Skip to content

Commit b43e88c

Browse files
marvinbussmarvinbuss
authored
Merge pull request #35 from PerfectThymeTech/adf_publish
Update ADF Templates
2 parents 35a4f18 + 334dd42 commit b43e88c

File tree

4 files changed

+531
-469
lines changed

4 files changed

+531
-469
lines changed

‎sc4-dev-df001/ARMTemplateForFactory.json‎

Lines changed: 123 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,11 @@
114114
"KustoSink": {}
115115
},
116116
"linkedServiceParameters": {
117+
"KustoSinkReference": {
118+
"linkedService": {
119+
"databaseName": "operationaldb"
120+
}
121+
},
117122
"KustoSink": {
118123
"linkedService": {
119124
"databaseName": {
@@ -1050,6 +1055,11 @@
10501055
"KustoSink": {}
10511056
},
10521057
"linkedServiceParameters": {
1058+
"KustoSinkReference": {
1059+
"linkedService": {
1060+
"databaseName": "operationaldb"
1061+
}
1062+
},
10531063
"KustoSink": {
10541064
"linkedService": {
10551065
"databaseName": {
@@ -2627,7 +2637,7 @@
26272637
},
26282638
"sourceFolderPath": {
26292639
"type": "string",
2630-
"defaultValue": "iptv/0156490b-efbe-4a27-af08-a237669f91cc"
2640+
"defaultValue": "iptv/4614736d-2afa-4739-80a7-d6b449958736"
26312641
},
26322642
"sourceFileName": {
26332643
"type": "string",
@@ -3270,7 +3280,7 @@
32703280
},
32713281
"sourceFolderPath": {
32723282
"type": "string",
3273-
"defaultValue": "ott/751cb63e-ce65-4f73-9894-6b1bdb8b7e1d"
3283+
"defaultValue": "ott/a44a15d8-7148-47e5-bf8b-e1d25c5d1935"
32743284
},
32753285
"sourceFileName": {
32763286
"type": "string",
@@ -3825,9 +3835,15 @@
38253835
"typeProperties": {
38263836
"endpoint": "[parameters('AzureDataExplorer_properties_typeProperties_endpoint')]",
38273837
"database": "[parameters('AzureDataExplorer_properties_typeProperties_database')]"
3838+
},
3839+
"connectVia": {
3840+
"referenceName": "IntegrationRuntime001",
3841+
"type": "IntegrationRuntimeReference"
38283842
}
38293843
},
3830-
"dependsOn": []
3844+
"dependsOn": [
3845+
"[concat(variables('factoryId'), '/integrationRuntimes/IntegrationRuntime001')]"
3846+
]
38313847
},
38323848
{
38333849
"name": "[concat(parameters('factoryName'), '/AzureDataLakeStorage')]",
@@ -3839,9 +3855,15 @@
38393855
"type": "AzureBlobFS",
38403856
"typeProperties": {
38413857
"url": "[parameters('AzureDataLakeStorage_properties_typeProperties_url')]"
3858+
},
3859+
"connectVia": {
3860+
"referenceName": "IntegrationRuntime001",
3861+
"type": "IntegrationRuntimeReference"
38423862
}
38433863
},
3844-
"dependsOn": []
3864+
"dependsOn": [
3865+
"[concat(variables('factoryId'), '/integrationRuntimes/IntegrationRuntime001')]"
3866+
]
38453867
},
38463868
{
38473869
"name": "[concat(parameters('factoryName'), '/AzureKeyVault')]",
@@ -4097,21 +4119,23 @@
40974119
" logsFolderPath as string",
40984120
"}",
40994121
"source(output(",
4100-
" AppVersion as string,",
41014122
" Country as string,",
4102-
" HappinessScore as float,",
4103-
" HwModel as string,",
41044123
" Service as string,",
41054124
" ServiceType as string,",
4106-
" StbModel as string,",
4107-
" StreamingProtocol as string,",
4108-
" Title as string,",
41094125
" Topology2 as string,",
41104126
" Topology3 as string,",
41114127
" Topology4 as string,",
41124128
" Topology5 as string,",
4113-
" TvModel as string,",
4114-
" UserId as long,",
4129+
" Title as string,",
4130+
" StbModel as string,",
4131+
" StbManufacturer as string,",
4132+
" StbSwVersion as string,",
4133+
" StreamingProtocol as string,",
4134+
" CdnHostname as string,",
4135+
" CdnPop as string,",
4136+
" ChannelName as string,",
4137+
" HappinessScore as float,",
4138+
" UserId as string,",
41154139
" EndTime as timestamp,",
41164140
" Hash as string",
41174141
" ),",
@@ -4122,23 +4146,26 @@
41224146
" fileSystem: ($sourceFileSystem),",
41234147
" folderPath: ($sourceFolderPath)) ~> AzureDataLakeSource",
41244148
"source(output(",
4125-
" AppVersion as string,",
41264149
" Country as string,",
4127-
" HappinessScore as double,",
4128-
" HwModel as string,",
41294150
" Service as string,",
41304151
" ServiceType as string,",
4131-
" StbModel as string,",
4132-
" StreamingProtocol as string,",
4133-
" Title as string,",
41344152
" Topology2 as string,",
41354153
" Topology3 as string,",
41364154
" Topology4 as string,",
41374155
" Topology5 as string,",
4138-
" TvModel as string,",
4156+
" Title as string,",
4157+
" StbModel as string,",
4158+
" StbManufacturer as string,",
4159+
" StbSwVersion as string,",
4160+
" StreamingProtocol as string,",
4161+
" CdnNodeHostname as string,",
4162+
" CdnPop as string,",
4163+
" ChannelName as string,",
4164+
" HappinessScore as double,",
41394165
" UserId as string,",
41404166
" EndTime as timestamp,",
4141-
" Hash as string",
4167+
" Hash as string,",
4168+
" CdnHostname as string",
41424169
" ),",
41434170
" allowSchemaDrift: true,",
41444171
" validateSchema: false,",
@@ -4229,22 +4256,24 @@
42294256
" logsFolderPath as string",
42304257
"}",
42314258
"source(output(",
4232-
" app_version as string,",
42334259
" country as string,",
4234-
" end_time as timestamp 'yyyy-MM-dd\\'T\\'HH:mm:ss.SSS\\'Z\\'',",
4235-
" happiness_score as float,",
4236-
" hw_model as string,",
42374260
" service as string,",
4261+
" topology_5 as string,",
4262+
" topology_4 as string,",
4263+
" topology_3 as string,",
4264+
" topology_2 as string,",
4265+
" title as string,",
42384266
" service_type as string,",
42394267
" stb_model as string,",
4268+
" stb_manufacturer as string,",
4269+
" stb_sw_version as string,",
42404270
" streaming_protocol as string,",
4241-
" title as string,",
4242-
" topology_2 as string,",
4243-
" topology_3 as string,",
4244-
" topology_4 as string,",
4245-
" topology_5 as string,",
4246-
" tv_model as string,",
4247-
" user_id as long",
4271+
" cdn_hostname as string,",
4272+
" cdn_pop as string,",
4273+
" channel_name as string,",
4274+
" happiness_score as float,",
4275+
" user_id as string,",
4276+
" end_time as timestamp 'yyyy-MM-dd\\'T\\'HH:mm:ss\\'Z\\''",
42484277
" ),",
42494278
" useSchema: false,",
42504279
" allowSchemaDrift: true,",
@@ -4261,20 +4290,22 @@
42614290
"AzureDataLakeSource aggregate(groupBy(hash = sha2(256,columns())),",
42624291
" each(match(true()), $$ = first($$))) ~> DistinctRows",
42634292
"DistinctRows select(mapColumn(",
4264-
" AppVersion = app_version,",
42654293
" Country = country,",
4266-
" HappinessScore = happiness_score,",
4267-
" HwModel = hw_model,",
42684294
" Service = service,",
42694295
" ServiceType = service_type,",
4270-
" StbModel = stb_model,",
4271-
" StreamingProtocol = streaming_protocol,",
4272-
" Title = title,",
42734296
" Topology2 = topology_2,",
42744297
" Topology3 = topology_3,",
42754298
" Topology4 = topology_4,",
42764299
" Topology5 = topology_5,",
4277-
" TvModel = tv_model,",
4300+
" Title = title,",
4301+
" StbModel = stb_model,",
4302+
" StbManufacturer = stb_manufacturer,",
4303+
" StbSwVersion = stb_sw_version,",
4304+
" StreamingProtocol = streaming_protocol,",
4305+
" CdnHostname = cdn_hostname,",
4306+
" CdnPop = cdn_pop,",
4307+
" ChannelName = channel_name,",
4308+
" HappinessScore = happiness_score,",
42784309
" UserId = user_id,",
42794310
" EndTime = end_time,",
42804311
" Hash = hash",
@@ -4373,18 +4404,19 @@
43734404
"}",
43744405
"source(output(",
43754406
" Country as string,",
4376-
" Isp as string,",
4377-
" CdnNodeHost as string,",
4378-
" Type as string,",
4407+
" Service as string,",
4408+
" ServiceType as string,",
4409+
" Device as string,",
43794410
" Title as string,",
4380-
" SelectedQuality as string,",
4381-
" DeviceType as string,",
4382-
" Version as string,",
4383-
" Connection as string,",
4384-
" CommercilizationType as string,",
4385-
" DeviceVendor as string,",
4411+
" Isp as string,",
4412+
" CdnHostname as string,",
4413+
" CdnPop as string,",
4414+
" AppReleaseVersion as string,",
4415+
" Browser as string,",
4416+
" Os as string,",
4417+
" Player as string,",
43864418
" HappinessScore as float,",
4387-
" UserId as long,",
4419+
" UserId as string,",
43884420
" EndTime as timestamp,",
43894421
" Hash as string",
43904422
" ),",
@@ -4396,20 +4428,22 @@
43964428
" folderPath: ($sourceFolderPath)) ~> AzureDataLakeSource",
43974429
"source(output(",
43984430
" Country as string,",
4399-
" Isp as string,",
4400-
" CdnNodeHost as string,",
4401-
" Type as string,",
4431+
" Service as string,",
4432+
" ServiceType as string,",
4433+
" Device as string,",
44024434
" Title as string,",
4403-
" SelectedQuality as string,",
4404-
" DeviceType as string,",
4405-
" Version as string,",
4406-
" Connection as string,",
4407-
" CommercilizationType as string,",
4408-
" DeviceVendor as string,",
4435+
" Isp as string,",
4436+
" CdnNodeHostname as string,",
4437+
" CdnPop as string,",
4438+
" AppReleaseVersion as string,",
4439+
" Browser as string,",
4440+
" Os as string,",
4441+
" Player as string,",
44094442
" HappinessScore as double,",
44104443
" UserId as string,",
44114444
" EndTime as timestamp,",
4412-
" Hash as string",
4445+
" Hash as string,",
4446+
" CdnHostname as string",
44134447
" ),",
44144448
" allowSchemaDrift: true,",
44154449
" validateSchema: false,",
@@ -4485,10 +4519,6 @@
44854519
"name": "FilterEmptyHappinessScore",
44864520
"description": "Filter Empty Happiness Score Rows"
44874521
},
4488-
{
4489-
"name": "HarmonizeColumn",
4490-
"description": "Harmonize columns: Connection, DeviceVendor"
4491-
},
44924522
{
44934523
"name": "AlterRow"
44944524
}
@@ -4504,20 +4534,21 @@
45044534
" logsFolderPath as string",
45054535
"}",
45064536
"source(output(",
4507-
" Country as string,",
4508-
" ISP as string,",
4509-
" {CDN Node Host} as string,",
4510-
" Type as string,",
4511-
" Title as string,",
4512-
" SELECTED_QUALITY as string,",
4513-
" DEVICE_TYPE as string,",
4514-
" VERSION as string,",
4515-
" CONNECTION as string,",
4516-
" {COMMERCIALIZATION TYPE} as string,",
4517-
" {Device Vendor} as string,",
4518-
" {Happiness Score} as float,",
4519-
" {User ID} as long,",
4520-
" {End Time} as timestamp",
4537+
" country as string,",
4538+
" service as string,",
4539+
" device as string,",
4540+
" title as string,",
4541+
" service_type as string,",
4542+
" isp as string,",
4543+
" cdn_hostname as string,",
4544+
" cdn_pop as string,",
4545+
" app_release_version as string,",
4546+
" browser as string,",
4547+
" os as string,",
4548+
" player as string,",
4549+
" happiness_score as float,",
4550+
" user_id as string,",
4551+
" end_time as timestamp 'yyyy-MM-dd\\'T\\'HH:mm:ss\\'Z\\''",
45214552
" ),",
45224553
" useSchema: false,",
45234554
" allowSchemaDrift: true,",
@@ -4534,28 +4565,27 @@
45344565
"AzureDataLakeSource aggregate(groupBy(hash = sha2(256,columns())),",
45354566
" each(match(true()), $$ = first($$))) ~> DistinctRows",
45364567
"DistinctRows select(mapColumn(",
4537-
" Country,",
4538-
" Isp = ISP,",
4539-
" CdnNodeHost = {CDN Node Host},",
4540-
" Type,",
4541-
" Title,",
4542-
" SelectedQuality = SELECTED_QUALITY,",
4543-
" DeviceType = DEVICE_TYPE,",
4544-
" Version = VERSION,",
4545-
" Connection = CONNECTION,",
4546-
" CommercilizationType = {COMMERCIALIZATION TYPE},",
4547-
" DeviceVendor = {Device Vendor},",
4548-
" HappinessScore = {Happiness Score},",
4549-
" UserId = {User ID},",
4550-
" EndTime = {End Time},",
4568+
" Country = country,",
4569+
" Service = service,",
4570+
" ServiceType = service_type,",
4571+
" Device = device,",
4572+
" Title = title,",
4573+
" Isp = isp,",
4574+
" CdnHostname = cdn_hostname,",
4575+
" CdnPop = cdn_pop,",
4576+
" AppReleaseVersion = app_release_version,",
4577+
" Browser = browser,",
4578+
" Os = os,",
4579+
" Player = player,",
4580+
" HappinessScore = happiness_score,",
4581+
" UserId = user_id,",
4582+
" EndTime = end_time,",
45514583
" Hash = hash",
45524584
" ),",
45534585
" skipDuplicateMapInputs: true,",
45544586
" skipDuplicateMapOutputs: true) ~> RenameColumns",
45554587
"RenameColumns filter(!isNull(HappinessScore)) ~> FilterEmptyHappinessScore",
4556-
"FilterEmptyHappinessScore derive(Connection = lower(Connection),",
4557-
" DeviceVendor = lower(DeviceVendor)) ~> HarmonizeColumn",
4558-
"HarmonizeColumn alterRow(upsertIf(true())) ~> AlterRow",
4588+
"FilterEmptyHappinessScore alterRow(upsertIf(true())) ~> AlterRow",
45594589
"AlterRow sink(allowSchemaDrift: true,",
45604590
" validateSchema: false,",
45614591
" format: 'delta',",

0 commit comments

Comments
 (0)