feat(rs): python udf deprecation scaffolding

.github/workflows/redshift-ded.yml

@@ -6,8 +6,7 @@ on:
 
 env:
   NODE_VERSION: 14
-  PYTHON2_VERSION: 2.7.18
-  PYTHON3_VERSION: 3.8.18
+  PYTHON_VERSION: 3.10.13
   VIRTUALENV_VERSION: 20.15.1
 
 jobs:
@@ -18,14 +17,15 @@ jobs:
       (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'dedicated_redshift')) ||
       (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dedicated_redshift'))
     runs-on: ubuntu-24.04
-    container: python:2.7.18-buster
     timeout-minutes: 20
     env:
       RS_HOST: ${{ secrets.RS_HOST_CD }}
       RS_DATABASE: ${{ secrets.RS_DATABASE_CD }}
       RS_USER: ${{ secrets.RS_USER_CD }}
       RS_PASSWORD: ${{ secrets.RS_PASSWORD_CD }}
-      RS_BUCKET: ${{ secrets.RS_BUCKET_CD }}
+      RS_LAMBDA_INVOKE_ROLE: ${{ secrets.RS_LAMBDA_INVOKE_ROLE_CD }}
+      RS_LAMBDA_EXECUTION_ROLE: ${{ secrets.RS_LAMBDA_EXECUTION_ROLE_CD }}
+      AWS_REGION: ${{ secrets.RS_REGION_CD }}
       AWS_ACCESS_KEY_ID: ${{ secrets.RS_AWS_ACCESS_KEY_ID_CD }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY_CD }}
     steps:
@@ -46,21 +46,16 @@ jobs:
           !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&
           !(startsWith(github.event.pull_request.head.ref, 'release/'))
         run: echo "RS_PREFIX=dedicated_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: Setup node
         uses: actions/setup-node@v1
         with:
           node-version: ${{ env.NODE_VERSION }}
       - name: Setup yarn
         run: npm install -g yarn
-      - name: Install dependencies
-        run: |
-          apt-get update
-          apt-get -y install rsync zip unzip curl glibc-source groff less
-      - name: Install AWS CLI
-        run: |
-          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-          unzip awscliv2.zip
-          ./aws/install
       - name: Setup virtualenv
         run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
       - name: Configure AWS Credentials
@@ -69,26 +64,26 @@ jobs:
           aws-access-key-id: ${{ secrets.RS_AWS_ACCESS_KEY_ID_CD }}
           aws-secret-access-key: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY_CD }}
           aws-region: ${{ secrets.RS_REGION_CD }}
-      - name: Setup virtualenv
-        run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
+      - name: Set RS_LAMBDA_PREFIX
+        run: echo "RS_LAMBDA_PREFIX=ded-${{ github.event.pull_request.number }}-" >> $GITHUB_ENV
       - name: Run deploy
         id: deploy
         if: github.event.action == 'synchronize' || github.event.action == 'labeled'
         run: |
-          cd clouds/redshift
-          make deploy dropfirst=1
+          make deploy cloud=redshift
       - name: Run remove
         id: remove
         if: github.event.action == 'unlabeled' || github.event.action == 'closed'
         run: |
-          cd clouds/redshift
-          make remove
+          make remove cloud=redshift drop-schema=1 || echo "Removal (best effort)"
       - name: Comment deploy PR
         if: steps.deploy.outcome == 'success' && github.event.action == 'labeled'
         uses: thollander/actions-comment-pull-request@v1
         with:
           message: |
             Dedicated ${{ env.RS_PREFIX }}carto environment deployed in Redshift host ${{ env.RS_HOST }} and ${{ env.RS_DATABASE }} database
+            - Gateway functions (Lambda): ${{ env.RS_LAMBDA_PREFIX }}*
+            - Clouds functions (SQL UDFs): deployed
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Comment remove PR
         if: steps.remove.outcome == 'success' && (github.event.action == 'unlabeled' || github.event.action == 'closed')

.github/workflows/redshift.yml

@@ -5,57 +5,59 @@ on:
     branches:
       - main
     paths:
+      - "gateway/**"
       - "clouds/redshift/**"
       - ".github/workflows/redshift.yml"
   pull_request:
     paths:
+      - "gateway/**"
       - "clouds/redshift/**"
       - ".github/workflows/redshift.yml"
   workflow_dispatch:
   workflow_call:
 
 env:
-  NODE_VERSION: 14
-  PYTHON2_VERSION: 2.7.18
-  PYTHON3_VERSION: 3.8.18
+  NODE_VERSION: 20.19
+  PYTHON_VERSION: 3.10.13
   VIRTUALENV_VERSION: 20.15.1
   GCLOUD_VERSION: 500.0.0
 
 jobs:
 
   test:
     runs-on: ubuntu-24.04
-    container: python:2.7.18-buster
-    timeout-minutes: 30
+    timeout-minutes: 40
     env:
-      RS_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_${{ github.run_attempt }}_
       RS_HOST: ${{ secrets.RS_HOST_CI }}
       RS_DATABASE: ${{ secrets.RS_DATABASE_CI }}
       RS_USER: ${{ secrets.RS_USER_CI }}
       RS_PASSWORD: ${{ secrets.RS_PASSWORD_CI }}
-      RS_BUCKET: ${{ secrets.RS_BUCKET_CI }}
+      RS_LAMBDA_EXECUTION_ROLE: ${{ secrets.RS_LAMBDA_EXECUTION_ROLE_CI }}
+      RS_LAMBDA_INVOKE_ROLE: ${{ secrets.RS_LAMBDA_INVOKE_ROLE_CI }}
+      AWS_REGION: us-east-1
       AWS_ACCESS_KEY_ID: ${{ secrets.RS_AWS_ACCESS_KEY_ID_CI }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY_CI }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
       - name: Check diff
         uses: technote-space/get-diff-action@v4
+      - name: Generate deployment prefix
+        run: |
+          SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-4)
+          SHORT_RUN=$(echo ${{ github.run_id }} | tail -c 4)
+          echo "RS_PREFIX=ci_${SHORT_SHA}_${SHORT_RUN}_" >> $GITHUB_ENV
+          echo "RS_LAMBDA_PREFIX=ci${SHORT_SHA}${SHORT_RUN}-" >> $GITHUB_ENV
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: Setup node
         uses: actions/setup-node@v1
         with:
           node-version: ${{ env.NODE_VERSION }}
       - name: Setup yarn
         run: npm install -g yarn
-      - name: Install dependencies
-        run: |
-          apt-get update
-          apt-get -y install rsync zip unzip curl glibc-source groff less
-      - name: Install AWS CLI
-        run: |
-          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-          unzip awscliv2.zip
-          ./aws/install
       - name: Setup virtualenv
         run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
       - name: Configure AWS Credentials
@@ -64,19 +66,21 @@ jobs:
           aws-access-key-id: ${{ secrets.RS_AWS_ACCESS_KEY_ID_CI }}
           aws-secret-access-key: ${{ secrets.RS_AWS_SECRET_ACCESS_KEY_CI }}
           aws-region: ${{ secrets.RS_REGION_CI }}
-      - name: Run linter and tests
+      - name: Run linter
+        run: make lint cloud=redshift
+      - name: Run deploy
+        run: make deploy cloud=redshift diff="$GIT_DIFF"
+      - name: Run test
+        run: make test cloud=redshift diff="$GIT_DIFF"
+      - name: Run remove
+        if: always()
         run: |
-          cd clouds/redshift
-          make lint && \
-          make deploy diff="$GIT_DIFF" && \
-          make test diff="$GIT_DIFF" && \
-          make remove
+          make remove cloud=redshift drop-schema=1
 
   deploy-internal:
     if: github.ref_name == 'main'
     needs: test
     runs-on: ubuntu-24.04
-    container: python:2.7.18-buster
     timeout-minutes: 20
     strategy:
       matrix:
@@ -85,47 +89,44 @@ jobs:
             database: RS_DATABASE_CD
             user: RS_USER_CD
             password: RS_PASSWORD_CD
-            bucket: RS_BUCKET_CD
             region: RS_REGION_CD
+            roles: RS_ROLES_CD
             aws_access_key_id: RS_AWS_ACCESS_KEY_ID_CD
             aws_secret_access_key: RS_AWS_SECRET_ACCESS_KEY_CD
           - host: RS_HOST_CI
             database: RS_DATABASE_CI
             user: RS_USER_CI
             password: RS_PASSWORD_CI
-            bucket: RS_BUCKET_CI
             region: RS_REGION_CI
+            roles: RS_ROLES_CI
             aws_access_key_id: RS_AWS_ACCESS_KEY_ID_CI
             aws_secret_access_key: RS_AWS_SECRET_ACCESS_KEY_CI
     env:
       RS_HOST: ${{ secrets[matrix.host] }}
       RS_DATABASE: ${{ secrets[matrix.database] }}
       RS_USER: ${{ secrets[matrix.user] }}
       RS_PASSWORD: ${{ secrets[matrix.password] }}
-      RS_BUCKET: ${{ secrets[matrix.bucket] }}
       RS_REGION: ${{ secrets[matrix.region] }}
+      RS_ROLES: ${{ secrets[matrix.roles] }}
+      AWS_REGION: ${{ secrets[matrix.region] }}
       AWS_ACCESS_KEY_ID: ${{ secrets[matrix.aws_access_key_id] }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets[matrix.aws_secret_access_key] }}
+      RS_LAMBDA_PREFIX: carto-at-
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
       - name: Check diff
         uses: technote-space/get-diff-action@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: Setup node
         uses: actions/setup-node@v1
         with:
           node-version: ${{ env.NODE_VERSION }}
       - name: Setup yarn
         run: npm install -g yarn
-      - name: Install dependencies
-        run: |
-          apt-get update
-          apt-get -y install rsync zip unzip curl glibc-source groff less
-      - name: Install AWS CLI
-        run: |
-          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-          unzip awscliv2.zip
-          ./aws/install
       - name: Setup virtualenv
         run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
       - name: Configure AWS Credentials
@@ -136,21 +137,23 @@ jobs:
           aws-region: ${{ env.RS_REGION }}
       - name: Run deploy
         run: |
-          cd clouds/redshift
-          make deploy diff="$GIT_DIFF" production=1
+          make deploy cloud=redshift production=1
 
   publish:
     if: github.ref_name == 'stable'
     needs: test
     runs-on: ubuntu-24.04
-    container: python:2.7.18-buster
     timeout-minutes: 10
     env:
       PACKAGE_BUCKET: gs://carto-analytics-toolbox-core/redshift
       LIBS_URL: https://storage.googleapis.com/carto-analytics-toolbox-core/redshift/latest/libs
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: Setup node
         uses: actions/setup-node@v1
         with:
@@ -159,15 +162,6 @@ jobs:
         run: npm install -g yarn
       - name: Setup virtualenv
         run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
-      - name: Install dependencies
-        run: |
-          apt-get update
-          apt-get -y install rsync zip unzip curl glibc-source groff less
-      - name: Install AWS CLI
-        run: |
-          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-          unzip awscliv2.zip
-          ./aws/install
       - name: Auth google
         uses: google-github-actions/auth@v2
         with:
@@ -181,12 +175,8 @@ jobs:
             project_id: ${{ secrets.GCLOUD_PRODUCTION_PROJECT }}
       - name: Create package
         run: |
-          cd clouds/redshift
-          make create-package
+          make create-package cloud=redshift production=1
       - name: Publish package
         run: |
-          cd clouds/redshift
           gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-redshift-latest.zip
-          gsutil -h "Content-Type:application/zip" -m cp build/libs/*.zip ${{ env.PACKAGE_BUCKET }}/latest/libs/
-          gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/
+          gsutil -h "Content-Type:application/zip" -m cp dist/carto-at-redshift-*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-redshift-latest.zip