Merge pull request #23 from PerfectThymeTech/marvinbuss/filter_data

Filter Empty Happiness Scores

Author: marvinbuss

code/datafactory/dataflow/IptvRawToCurated.json

@@ -35,6 +35,10 @@
 				{
 					"name": "RenameColumns",
 					"description": "Rename Columns"
+				},
+				{
+					"name": "FilterEmptyHappinessScore",
+					"description": "Filter Empty Happiness Score Rows"
 				}
 			],
 			"scriptLines": [
@@ -97,7 +101,8 @@
 				"     ),",
 				"     skipDuplicateMapInputs: true,",
 				"     skipDuplicateMapOutputs: true) ~> RenameColumns",
-				"RenameColumns sink(allowSchemaDrift: true,",
+				"RenameColumns filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore",
+				"FilterEmptyHappinessScore sink(allowSchemaDrift: true,",
 				"     validateSchema: false,",
 				"     format: 'delta',",
 				"     fileSystem: ($sinkFileSystem),",

code/datafactory/dataflow/OttRawToCurated.json

@@ -35,6 +35,10 @@
 				{
 					"name": "RenameColumns",
 					"description": "Rename Columns"
+				},
+				{
+					"name": "FilterEmptyHappinessScore",
+					"description": "Filter Empty Happiness Score Rows"
 				}
 			],
 			"scriptLines": [
@@ -93,7 +97,8 @@
 				"     ),",
 				"     skipDuplicateMapInputs: true,",
 				"     skipDuplicateMapOutputs: true) ~> RenameColumns",
-				"RenameColumns sink(allowSchemaDrift: true,",
+				"RenameColumns filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore",
+				"FilterEmptyHappinessScore sink(allowSchemaDrift: true,",
 				"     validateSchema: false,",
 				"     format: 'delta',",
 				"     fileSystem: ($sinkFileSystem),",