Skip to content

Commit b9a71d6

Browse files
marvinbussmarvinbuss
committed
ARM template and parameters deployed on 6-17-2024 22:16:17, based on the collaboration branch's commit ID: b57f875 
1 parent 3ffdd78 commit b9a71d6

File tree

2 files changed

+96
-16
lines changed

2 files changed

+96
-16
lines changed

‎sc4-dev-df001/ARMTemplateForFactory.json‎

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1046,6 +1046,7 @@
10461046
},
10471047
"datasetParameters": {
10481048
"AzureDataLakeSource": {},
1049+
"KustoSinkReference": {},
10491050
"KustoSink": {}
10501051
},
10511052
"linkedServiceParameters": {
@@ -4137,6 +4138,14 @@
41374138
},
41384139
"name": "AzureDataLakeSource",
41394140
"description": "Source dataset in Azure Data Lake"
4141+
},
4142+
{
4143+
"linkedService": {
4144+
"referenceName": "AzureDataExplorer",
4145+
"type": "LinkedServiceReference"
4146+
},
4147+
"name": "KustoSinkReference",
4148+
"description": "Kusto Sink Reference Dataset"
41404149
}
41414150
],
41424151
"sinks": [
@@ -4153,7 +4162,11 @@
41534162
}
41544163
}
41554164
],
4156-
"transformations": [],
4165+
"transformations": [
4166+
{
4167+
"name": "ExistingHashes"
4168+
}
4169+
],
41574170
"scriptLines": [
41584171
"parameters{",
41594172
" sourceFileSystem as string,",
@@ -4176,15 +4189,41 @@
41764189
" DeviceVendor as string,",
41774190
" HappinessScore as float,",
41784191
" UserId as long,",
4179-
" EndTime as timestamp",
4192+
" EndTime as timestamp,",
4193+
" Hash as string",
41804194
" ),",
41814195
" allowSchemaDrift: true,",
41824196
" validateSchema: false,",
41834197
" ignoreNoFilesFound: false,",
41844198
" format: 'delta',",
41854199
" fileSystem: ($sourceFileSystem),",
41864200
" folderPath: ($sourceFolderPath)) ~> AzureDataLakeSource",
4187-
"AzureDataLakeSource sink(allowSchemaDrift: true,",
4201+
"source(output(",
4202+
" Country as string,",
4203+
" Isp as string,",
4204+
" CdnNodeHost as string,",
4205+
" Type as string,",
4206+
" Title as string,",
4207+
" SelectedQuality as string,",
4208+
" DeviceType as string,",
4209+
" Version as string,",
4210+
" Connection as string,",
4211+
" CommercilizationType as string,",
4212+
" DeviceVendor as string,",
4213+
" HappinessScore as double,",
4214+
" UserId as string,",
4215+
" EndTime as timestamp,",
4216+
" Hash as string",
4217+
" ),",
4218+
" allowSchemaDrift: true,",
4219+
" validateSchema: false,",
4220+
" format: 'table',",
4221+
" tableName: ($sinkTable),",
4222+
" store: 'azuredataexplorer') ~> KustoSinkReference",
4223+
"AzureDataLakeSource, KustoSinkReference exists(AzureDataLakeSource@Hash == KustoSinkReference@Hash,",
4224+
" negate:true,",
4225+
" broadcast: 'auto')~> ExistingHashes",
4226+
"ExistingHashes sink(allowSchemaDrift: true,",
41884227
" validateSchema: false,",
41894228
" format: 'table',",
41904229
" tableName: ($sinkTable),",
@@ -4295,9 +4334,9 @@
42954334
" escapeChar: '\\\\',",
42964335
" quoteChar: '\\\"',",
42974336
" columnNamesAsHeader: true) ~> AzureDataLakeSource",
4298-
"RenameColumns aggregate(groupBy(mycols = sha2(256,columns())),",
4337+
"AzureDataLakeSource aggregate(groupBy(hash = sha2(256,columns())),",
42994338
" each(match(true()), $$ = first($$))) ~> DistinctRows",
4300-
"AzureDataLakeSource select(mapColumn(",
4339+
"DistinctRows select(mapColumn(",
43014340
" Country,",
43024341
" Isp = ISP,",
43034342
" CdnNodeHost = {CDN Node Host},",
@@ -4311,11 +4350,12 @@
43114350
" DeviceVendor = {Device Vendor},",
43124351
" HappinessScore = {Happiness Score},",
43134352
" UserId = {User ID},",
4314-
" EndTime = {End Time}",
4353+
" EndTime = {End Time},",
4354+
" Hash = hash",
43154355
" ),",
43164356
" skipDuplicateMapInputs: true,",
43174357
" skipDuplicateMapOutputs: true) ~> RenameColumns",
4318-
"DistinctRows filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore",
4358+
"RenameColumns filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore",
43194359
"FilterEmptyHappinessScore derive(Connection = lower(Connection),",
43204360
" DeviceVendor = lower(DeviceVendor)) ~> HarmonizeColumn",
43214361
"HarmonizeColumn alterRow(upsertIf(true())) ~> AlterRow",
@@ -4332,7 +4372,7 @@
43324372
" insertable: false,",
43334373
" updateable: false,",
43344374
" upsertable: true,",
4335-
" keys:['Country','Isp','CdnNodeHost','Type','Title','SelectedQuality','DeviceType','Version','Connection','CommercilizationType','DeviceVendor','UserId','EndTime'],",
4375+
" keys:['Hash'],",
43364376
" umask: 0022,",
43374377
" preCommands: [],",
43384378
" postCommands: [],",

‎sc4-dev-df001/linkedTemplates/ArmTemplate_0.json‎

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -986,6 +986,14 @@
986986
},
987987
"name": "AzureDataLakeSource",
988988
"description": "Source dataset in Azure Data Lake"
989+
},
990+
{
991+
"linkedService": {
992+
"referenceName": "AzureDataExplorer",
993+
"type": "LinkedServiceReference"
994+
},
995+
"name": "KustoSinkReference",
996+
"description": "Kusto Sink Reference Dataset"
989997
}
990998
],
991999
"sinks": [
@@ -1002,7 +1010,11 @@
10021010
}
10031011
}
10041012
],
1005-
"transformations": [],
1013+
"transformations": [
1014+
{
1015+
"name": "ExistingHashes"
1016+
}
1017+
],
10061018
"scriptLines": [
10071019
"parameters{",
10081020
" sourceFileSystem as string,",
@@ -1025,15 +1037,41 @@
10251037
" DeviceVendor as string,",
10261038
" HappinessScore as float,",
10271039
" UserId as long,",
1028-
" EndTime as timestamp",
1040+
" EndTime as timestamp,",
1041+
" Hash as string",
10291042
" ),",
10301043
" allowSchemaDrift: true,",
10311044
" validateSchema: false,",
10321045
" ignoreNoFilesFound: false,",
10331046
" format: 'delta',",
10341047
" fileSystem: ($sourceFileSystem),",
10351048
" folderPath: ($sourceFolderPath)) ~> AzureDataLakeSource",
1036-
"AzureDataLakeSource sink(allowSchemaDrift: true,",
1049+
"source(output(",
1050+
" Country as string,",
1051+
" Isp as string,",
1052+
" CdnNodeHost as string,",
1053+
" Type as string,",
1054+
" Title as string,",
1055+
" SelectedQuality as string,",
1056+
" DeviceType as string,",
1057+
" Version as string,",
1058+
" Connection as string,",
1059+
" CommercilizationType as string,",
1060+
" DeviceVendor as string,",
1061+
" HappinessScore as double,",
1062+
" UserId as string,",
1063+
" EndTime as timestamp,",
1064+
" Hash as string",
1065+
" ),",
1066+
" allowSchemaDrift: true,",
1067+
" validateSchema: false,",
1068+
" format: 'table',",
1069+
" tableName: ($sinkTable),",
1070+
" store: 'azuredataexplorer') ~> KustoSinkReference",
1071+
"AzureDataLakeSource, KustoSinkReference exists(AzureDataLakeSource@Hash == KustoSinkReference@Hash,",
1072+
" negate:true,",
1073+
" broadcast: 'auto')~> ExistingHashes",
1074+
"ExistingHashes sink(allowSchemaDrift: true,",
10371075
" validateSchema: false,",
10381076
" format: 'table',",
10391077
" tableName: ($sinkTable),",
@@ -1144,9 +1182,9 @@
11441182
" escapeChar: '\\\\',",
11451183
" quoteChar: '\\\"',",
11461184
" columnNamesAsHeader: true) ~> AzureDataLakeSource",
1147-
"RenameColumns aggregate(groupBy(mycols = sha2(256,columns())),",
1185+
"AzureDataLakeSource aggregate(groupBy(hash = sha2(256,columns())),",
11481186
" each(match(true()), $$ = first($$))) ~> DistinctRows",
1149-
"AzureDataLakeSource select(mapColumn(",
1187+
"DistinctRows select(mapColumn(",
11501188
" Country,",
11511189
" Isp = ISP,",
11521190
" CdnNodeHost = {CDN Node Host},",
@@ -1160,11 +1198,12 @@
11601198
" DeviceVendor = {Device Vendor},",
11611199
" HappinessScore = {Happiness Score},",
11621200
" UserId = {User ID},",
1163-
" EndTime = {End Time}",
1201+
" EndTime = {End Time},",
1202+
" Hash = hash",
11641203
" ),",
11651204
" skipDuplicateMapInputs: true,",
11661205
" skipDuplicateMapOutputs: true) ~> RenameColumns",
1167-
"DistinctRows filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore",
1206+
"RenameColumns filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore",
11681207
"FilterEmptyHappinessScore derive(Connection = lower(Connection),",
11691208
" DeviceVendor = lower(DeviceVendor)) ~> HarmonizeColumn",
11701209
"HarmonizeColumn alterRow(upsertIf(true())) ~> AlterRow",
@@ -1181,7 +1220,7 @@
11811220
" insertable: false,",
11821221
" updateable: false,",
11831222
" upsertable: true,",
1184-
" keys:['Country','Isp','CdnNodeHost','Type','Title','SelectedQuality','DeviceType','Version','Connection','CommercilizationType','DeviceVendor','UserId','EndTime'],",
1223+
" keys:['Hash'],",
11851224
" umask: 0022,",
11861225
" preCommands: [],",
11871226
" postCommands: [],",
@@ -1819,6 +1858,7 @@
18191858
},
18201859
"datasetParameters": {
18211860
"AzureDataLakeSource": {},
1861+
"KustoSinkReference": {},
18221862
"KustoSink": {}
18231863
},
18241864
"linkedServiceParameters": {

0 commit comments

Comments
 (0)