rework sdlf-dataset deployspec

cnfait · cnfait · commit 565f3ea395d8 · 2024-10-22T16:44:53.000+02:00
to be more in line with the existing seedfarmer community

the model where we build packages will be using a standard buildspec instead, in a future PR
diff --git a/sdlf-dataset/deployspec.yaml b/sdlf-dataset/deployspec.yaml
@@ -1,142 +1,32 @@
-publishGenericEnvVariables: true
+publishGenericEnvVariables: True
 deploy:
   phases:
     install:
       commands:
+        - npm install -g aws-cdk
         - |-
-            # force Python3.12
-            rm -Rf ~/.venv/
-            pyenv global 3.12
-            python3 -m venv ~/.venv
-            . ~/.venv/bin/activate
-            pip install --upgrade pip setuptools
-            pip install aws-codeseeder~=1.1.0 seed-farmer==5.0.0
-        - |-
-            CFN_ENDPOINT="https://cloudformation.$AWS_REGION.amazonaws.com"
-            ARTIFACTS_BUCKET=$(aws cloudformation --endpoint-url "$CFN_ENDPOINT" describe-stacks --query "Stacks[?StackName=='aws-codeseeder-$SEEDFARMER_PROJECT_NAME'][].Outputs[?OutputKey=='Bucket'].OutputValue" --output text)
-
-            deps=(
-              "https://raw.githubusercontent.com/aws/serverless-application-model/develop/bin/sam-translate.py"
-              "https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip"
-              "https://raw.githubusercontent.com/awslabs/aws-serverless-data-lake-framework/main/sdlf-cicd/template-generic-cfn-module.yaml"
-            )
-            for u in "${deps[@]}"; do
-              aws s3api get-object --bucket "$ARTIFACTS_BUCKET" --key "${u##*/}" "${u##*/}" || {
-                curl -L -O "$u"
-                aws s3api put-object --bucket "$ARTIFACTS_BUCKET" --key "${u##*/}" --body "${u##*/}"
-              }
-            done
-        - |-
-            pip uninstall -y aws-sam-cli && unzip -q aws-sam-cli-linux-x86_64.zip -d sam-installation
-            ./sam-installation/install && sam --version
-            pip install "cfn-lint<1" cloudformation-cli poetry
-            npm install -g aws-cdk@2.159.1
-            poetry config virtualenvs.create false --local
+          pip install --upgrade pip setuptools
+          pip install poetry
+          poetry config virtualenvs.create false --local
+          poetry install -v
     build:
       commands:
         - |-
-          # deployment-type possible values: cfn-module, cdk-construct
-          # cfn-module creates a CloudFormation Registry module out of template.yaml
-          # cdk-construct publishes a pip library out of template.py on CodeArtifact
-          echo "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE"
-        - |-
-          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cfn-module" ]; then
-            # SEEDFARMER_PARAMETER_LIBRARY_MODULE can be used to pass the module name. If not, the module name is inferred from SEEDFARMER_MODULE_NAME
-            # by removing everything up to the first hyphen, then anything that isn't a letter/number, and lower-casing everything.
-            sf_module_name_without_prefix="${SEEDFARMER_MODULE_NAME#*-}"
-            sf_module_name_alnum="${sf_module_name_without_prefix//[^[:alnum:]]/}"
-            MODULE="${sf_module_name_alnum,,}"
-
-            : "${SEEDFARMER_PARAMETER_LIBRARY_ORG:=awslabs}" "${SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK:=sdlf}" "${SEEDFARMER_PARAMETER_LIBRARY_MODULE:=$MODULE}"
-            cd src || exit
-            sam package --template-file ./template.yaml --s3-bucket "$ARTIFACTS_BUCKET" --s3-prefix sdlf --output-template-file template.yaml
-            python3 ../sam-translate.py --template-file=template.yaml --output-template=translated-template.json
-
-            SSM_ENDPOINT="https://ssm.$AWS_REGION.amazonaws.com"
-            TEMPLATE_BASE_FILE_PATH="modules/$SEEDFARMER_PARAMETER_LIBRARY_ORG/$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK/$SEEDFARMER_PARAMETER_LIBRARY_MODULE"
-            aws s3api put-object --bucket "$ARTIFACTS_BUCKET" --key "$TEMPLATE_BASE_FILE_PATH/translated-template.json" --body translated-template.json
-            TEMPLATE_URL="https://$ARTIFACTS_BUCKET.s3.$AWS_REGION.amazonaws.com/$TEMPLATE_BASE_FILE_PATH/translated-template.json"
-            aws cloudformation --endpoint-url "$CFN_ENDPOINT" validate-template --template-url "$TEMPLATE_URL"
-
-            mkdir module
-            cd module || exit
-            cfn init --artifact-type MODULE --type-name "$SEEDFARMER_PARAMETER_LIBRARY_ORG::$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK::$SEEDFARMER_PARAMETER_LIBRARY_MODULE::MODULE" && rm fragments/sample.json
-            cp -i -a ../translated-template.json fragments/
-            cfn generate
-            zip -q -r "../$SEEDFARMER_PARAMETER_LIBRARY_MODULE.zip" .rpdk-config fragments/ schema.json
-
-            NEW_MODULE="$(sha256sum "../$SEEDFARMER_PARAMETER_LIBRARY_MODULE.zip" | cut -c1-12)"
-            aws s3api put-object --bucket "$ARTIFACTS_BUCKET" --key "$TEMPLATE_BASE_FILE_PATH-$NEW_MODULE.zip" --body "../$SEEDFARMER_PARAMETER_LIBRARY_MODULE.zip"
-
-            if CURRENT_MODULE=$(aws ssm --endpoint-url "$SSM_ENDPOINT" get-parameter --name "/SDLF/CFN/$SEEDFARMER_PARAMETER_LIBRARY_ORG-$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK-$SEEDFARMER_PARAMETER_LIBRARY_MODULE-MODULE" --query "Parameter.Value" --output text); then
-              echo "Current module hash: $CURRENT_MODULE / New module hash: $NEW_MODULE"
-              if [ "$NEW_MODULE" == "$CURRENT_MODULE" ]; then
-                echo "No change since last build, exiting module creation."
-                exit 0
-              fi
-            fi
-
-            STACK_NAME="sdlf-cfn-module-$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK-$SEEDFARMER_PARAMETER_LIBRARY_MODULE"
-            aws cloudformation --endpoint-url "$CFN_ENDPOINT" deploy \
-              --stack-name "$STACK_NAME" \
-              --template-file ../../template-generic-cfn-module.yaml \
-              --parameter-overrides \
-                  pArtifactsBucket="$ARTIFACTS_BUCKET" \
-                  pLibraryOrg="$SEEDFARMER_PARAMETER_LIBRARY_ORG" \
-                  pLibraryFramework="$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK" \
-                  pLibraryModule="$SEEDFARMER_PARAMETER_LIBRARY_MODULE" \
-                  pModuleGitRef="$NEW_MODULE" \
-              --tags Framework=sdlf || exit 1
-            echo "done"
-            cd .. && rm -Rf module
-          fi
-        - |-
-          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cdk-construct" ]; then
-            CA_ENDPOINT="https://codeartifact.$AWS_REGION.amazonaws.com"
-            CA_REPOSITORY_ENDPOINT=$(aws codeartifact --endpoint-url "$CA_ENDPOINT" get-repository-endpoint --domain "$SEEDFARMER_PARAMETER_CODEARTIFACT_DOMAIN" --domain-owner "$AWS_ACCOUNT_ID" --repository "$SEEDFARMER_PARAMETER_CODEARTIFACT_REPOSITORY" --format pypi --query repositoryEndpoint --output text)
-            CA_TOKEN=$(aws codeartifact --endpoint-url "$CA_ENDPOINT" get-authorization-token --domain "$SEEDFARMER_PARAMETER_CODEARTIFACT_DOMAIN" --domain-owner "$AWS_ACCOUNT_ID" --query authorizationToken --output text)
-            poetry source add private "https://aws:${CA_TOKEN}@${CA_REPOSITORY_ENDPOINT#https://}simple/"
-            poetry install -v
-            poetry config repositories.private "$CA_REPOSITORY_ENDPOINT"
-            poetry config http-basic.private aws "$CA_TOKEN"
-            poetry publish --skip-existing --build -r private || exit 1
-          fi
-    post_build:
-      commands:
-      - echo "Deploy successful"
+          cdk deploy --all --require-approval never --progress events --app "python src/app.py" --outputs-file ./cdk-exports.json \
+            --no-previous-parameters \
+            --parameters DatasetStack:pDatasetName="$SEEDFARMER_PARAMETER_DATASET" \
+            --parameters DatasetStack:pS3Prefix="$SEEDFARMER_PARAMETER_S3_PREFIX"
+        - seedfarmer metadata convert -f cdk-exports.json || true
 destroy:
   phases:
     install:
       commands:
+        - npm install -g aws-cdk
         - |-
-            pip install --upgrade pip setuptools
-            pip install poetry
+          pip install --upgrade pip setuptools
+          pip install poetry
+          poetry config virtualenvs.create false --local
+          poetry install -v
     build:
       commands:
-      - |-
-          echo "DESTROY! $SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE"
-      - |-
-          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cfn-module" ]; then
-            CFN_ENDPOINT="https://cloudformation.$AWS_REGION.amazonaws.com"
-            sf_module_name_without_prefix="${SEEDFARMER_MODULE_NAME#*-}"
-            sf_module_name_alnum="${sf_module_name_without_prefix//[^[:alnum:]]/}"
-            MODULE="${sf_module_name_alnum,,}"
-            : "${SEEDFARMER_PARAMETER_LIBRARY_ORG:=awslabs}" "${SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK:=sdlf}" "${SEEDFARMER_PARAMETER_LIBRARY_MODULE:=$MODULE}"
-            STACK_NAME="sdlf-cfn-module-$SEEDFARMER_PARAMETER_LIBRARY_FRAMEWORK-$SEEDFARMER_PARAMETER_LIBRARY_MODULE"
-            aws cloudformation --endpoint-url "$CFN_ENDPOINT" delete-stack --stack-name "$STACK_NAME" || exit 1
-          fi
-      - |-
-          if [ "$SEEDFARMER_PARAMETER_DEPLOYMENT_TYPE" = "cdk-construct" ]; then
-            CA_ENDPOINT="https://codeartifact.$AWS_REGION.amazonaws.com"
-            package_version_poetry=$(poetry version --short)
-            package_prerelease_id=$(echo "$package_version_poetry" | cut -d'-' -f2 | cut -d'.' -f1)
-            pypi_prerelease_id=$(test "$package_prerelease_id" = "rc" && echo "rc" || echo "${package_prerelease_id:0:1}")
-            pypi_package_version="$(echo "$package_version_poetry" | cut -d'-' -f1)$pypi_prerelease_id$(echo "$package_version_poetry" | cut -d'-' -f2 | cut -d'.' -f2)"
-            aws codeartifact --endpoint-url "$CA_ENDPOINT" delete-package-versions \
-              --domain "$SEEDFARMER_PARAMETER_CODEARTIFACT_DOMAIN" --domain-owner "$AWS_ACCOUNT_ID" \
-              --repository "$SEEDFARMER_PARAMETER_CODEARTIFACT_REPOSITORY" --format pypi --package "$(poetry version | cut -d' ' -f1)" --versions "$pypi_package_version"
-          fi
-    post_build:
-      commands:
-      - echo "Destroy successful"
-build_type: BUILD_GENERAL1_SMALL
+        - cdk destroy --force --all --app "python src/app.py"
diff --git a/sdlf-dataset/src/app.py b/sdlf-dataset/src/app.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+import aws_cdk as cdk
+
+from dataset import Dataset
+
+app = cdk.App()
+
+lakeformation_cicd_stack = cdk.Stack(app, "LakeFormationDatasetStack")
+
+cdk.aws_lakeformation.CfnDataLakeSettings(
+    lakeformation_cicd_stack,
+    "rDataLakeSettings",
+    admins=[
+        cdk.aws_lakeformation.CfnDataLakeSettings.DataLakePrincipalProperty(
+            data_lake_principal_identifier=cdk.Fn.sub(lakeformation_cicd_stack.synthesizer.cloud_formation_execution_role_arn)
+        ),
+    ],
+    mutation_type="APPEND",
+)
+
+stack = cdk.Stack(app, "DatasetStack")
+stack.add_dependency(lakeformation_cicd_stack)
+Dataset(stack, "dataset")
+
+app.synth()
