Updating dataflow: OttRawToCurated

Author: marvinbuss

code/datafactory/dataflow/OttRawToCurated.json

@@ -39,6 +39,10 @@
 				{
 					"name": "FilterEmptyHappinessScore",
 					"description": "Filter Empty Happiness Score Rows"
+				},
+				{
+					"name": "HarmonizeColumn",
+					"description": "Harmonize columns: Connection, DeviceVendor"
 				}
 			],
 			"scriptLines": [
@@ -98,7 +102,9 @@
 				"     skipDuplicateMapInputs: true,",
 				"     skipDuplicateMapOutputs: true) ~> RenameColumns",
 				"RenameColumns filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore",
-				"FilterEmptyHappinessScore sink(allowSchemaDrift: true,",
+				"FilterEmptyHappinessScore derive(Connection = lower(Connection),",
+				"          DeviceVendor = lower(DeviceVendor)) ~> HarmonizeColumn",
+				"HarmonizeColumn sink(allowSchemaDrift: true,",
 				"     validateSchema: false,",
 				"     format: 'delta',",
 				"     fileSystem: ($sinkFileSystem),",