latest tests

Author: Emerson Antony

cloudformation/sam-imagebuilder.yaml

@@ -133,7 +133,7 @@ Resources:
                 - echo Downloading requires source files from git
                 - git clone https://github.com/aws-samples/spark-on-aws-lambda.git
                 - cd spark-on-aws-lambda
-                - git checkout fixes
+                - git checkout dockerfilefixes
                 - echo Build started on `date`
                 - echo Building the Docker image...          
                 - docker build --build-arg FRAMEWORK=$FRAMEWORK -t $IMAGE_REPO_NAME:latest .

sparkLambdaHandler.py

@@ -38,10 +38,12 @@ def spark_submit(s3_bucket_script: str,input_script: str, event: dict)-> None:
     # Source input and output if available in event
     input_path = event.get('INPUT_PATH','')
     output_path = event.get('OUTPUT_PATH', '')
+    for key,value in event:
+        os.environ[key] = value
     # Run the spark-submit command on the local copy of teh script
     try:
         logger.info(f'Spark-Submitting the Spark script {input_script} from {s3_bucket_script}')
-        subprocess.run(["spark-submit", "/tmp/spark_script.py", "--event", json.dumps(event)], check=True, env=event )
+        subprocess.run(["spark-submit", "/tmp/spark_script.py", "--event", json.dumps(event)], check=True, env=os.environ)
     except Exception as e :
         logger.error(f'Error Spark-Submit with exception: {e}')
         raise e