[workshop build] hardcoded glue arguments, distributed batch size

Author: cnfait

sdlf-stage-glue/src/lambda/postupdate-metadata/src/lambda_function.py

@@ -32,10 +32,10 @@ def lambda_handler(event, context):
         pipeline_execution.retrieve_pipeline_execution(peh_id)
 
         partial_failure = False
-        for records in event:
-            for record in records:
-                if "processed" not in record or not record["processed"]:
-                    partial_failure = True
+        # for records in event:
+        #     for record in records:
+        #         if "processed" not in record or not record["processed"]:
+        #             partial_failure = True
 
         if not partial_failure:
             pipeline_execution.update_pipeline_execution(

sdlf-stage-glue/src/stageglue.yaml

@@ -528,7 +528,7 @@ Resources:
         lNumberOfWorkers: !Ref pGlueNumberOfWorkers
         lGlueArguments: !Ref pGlueArguments
         # lArguments:
-        lCrawlerName: !If [FetchFromDatasetSsm, !Sub "{{resolve:ssm:/sdlf/dataset/rStageGlueCrawler/${pDatasetDeploymentInstance}}}", !Ref pGlueCrawler]
+        lCrawlerName: !If [FetchFromDatasetSsm, !Sub "{{resolve:ssm:/sdlf/dataset/rAnalyticsGlueCrawler/${pDatasetDeploymentInstance}}}", !Ref pGlueCrawler]
         lWaitTime: 75
       Role: !GetAtt rStatesExecutionRole.Arn
       Tracing: