feat(docker-gradle): localstack debug

* allow overriding base compose file
* wire debug-aws with localstack

Author: david-leifker

docker/build.gradle

@@ -371,7 +371,10 @@ quickstart_configs.each { taskName, config ->
 
                 // Only restart containers that had their modules rebuilt
                 if (containersToRestart) {
-                    def cmd = ["docker compose -p datahub --profile ${config.profile}"] + ['-f', compose_base] + ['restart'] + containersToRestart
+                    def composeFiles = dockerCompose."${taskName}".useComposeFiles.get()
+                    def composeFileArgs = composeFiles.collectMany { ['-f', it] }
+
+                    def cmd = ["docker compose -p ${project_name} --profile ${config.profile}"] + composeFileArgs + ['restart'] + containersToRestart
                     println(cmd.join(" "))
                     commandLine 'bash', '-c', cmd.join(" ")
                 } else {
@@ -392,7 +395,10 @@ quickstart_configs.each { taskName, config ->
                         containersToRestart << "${containerName}-${config.profile}"
                     }
                 }
-                def cmd = ["docker compose -p datahub --profile ${config.profile}"] + ['-f', compose_base] + ['up', '-d', '--no-deps'] + containersToRestart
+                def composeFiles = dockerCompose."${taskName}".useComposeFiles.get()
+                def composeFileArgs = composeFiles.collectMany { ['-f', it] }
+
+                def cmd = ["docker compose -p ${project_name} --profile ${config.profile}"] + composeFileArgs + ['up', '-d', '--no-deps'] + containersToRestart
                 println(cmd.join(" "))
                 commandLine 'bash', '-c', cmd.join(" ")
             }

docker/profiles/docker-compose.frontend.yml

@@ -66,6 +66,13 @@ services:
     depends_on:
       system-update-debug:
         condition: service_completed_successfully
+  frontend-debug-aws:
+    <<: *datahub-frontend-service-dev
+    profiles:
+      - debug-aws
+    depends_on:
+      system-update-debug-aws:
+        condition: service_completed_successfully
   frontend-debug-min:
     <<: *datahub-frontend-service-dev
     profiles:

docker/profiles/docker-compose.gms.yml

@@ -49,6 +49,13 @@ x-datahub-dev-telemetry-env: &datahub-dev-telemetry-env
   DATAHUB_SERVER_TYPE: ${DATAHUB_SERVER_TYPE:-dev}
   DATAHUB_TELEMETRY_ENABLED: ${DATAHUB_TELEMETRY_ENABLED:-true}
 
+x-localstack-env: &localstack-env
+  AWS_ENDPOINT_URL: ${DATAHUB_AWS_ENDPOINT_URL:-http://localstack:4566}
+  AWS_ACCESS_KEY_ID: ${DATAHUB_AWS_ACCESS_KEY_ID:-test}
+  AWS_SECRET_ACCESS_KEY: ${DATAHUB_AWS_ACCESS_KEY_ID:-test}
+  AWS_REGION: ${DATAHUB_AWS_REGION:-us-east-1}
+  DATAHUB_EXECUTOR_ROLE_ARN: ${DATAHUB_EXECUTOR_ROLE_ARN:-arn:aws:iam::000000000000:role/test-role}
+
 #################################
 # System Update
 #################################
@@ -274,6 +281,18 @@ services:
         condition: service_completed_successfully
       kafka-setup-dev:
         condition: service_completed_successfully
+  system-update-debug-aws: &system-update-debug
+    <<: *datahub-system-update-service-dev
+    profiles:
+      - debug-backend-aws
+      - debug-aws
+    depends_on:
+      mysql-setup-dev:
+        condition: service_completed_successfully
+      opensearch-setup-dev:
+        condition: service_completed_successfully
+      kafka-setup-dev:
+        condition: service_completed_successfully
   system-update-debug-consumers: # for consistent naming based on profile
     <<: *system-update-debug
     profiles:
@@ -378,16 +397,13 @@ services:
         condition: service_completed_successfully
   datahub-gms-debug-aws:
     <<: *datahub-gms-service-dev
+    environment:
+      <<: [*localstack-env, *datahub-gms-dev-env]
     profiles:
       - debug-backend-aws
-    environment:
-      <<: *datahub-gms-dev-env
-      AWS_ENDPOINT_URL: ${DATAHUB_AWS_ENDPOINT_URL:-http://localstack:4566}
-      AWS_ACCESS_KEY_ID: ${DATAHUB_AWS_ACCESS_KEY_ID:-test}
-      AWS_SECRET_ACCESS_KEY: ${DATAHUB_AWS_ACCESS_KEY_ID:-test}
-      AWS_REGION: ${DATAHUB_AWS_REGION:-us-east-1}
+      - debug-aws
     depends_on:
-      system-update-debug:
+      system-update-debug-aws:
         condition: service_completed_successfully
   datahub-gms-debug-min:
     <<: *datahub-gms-service-dev

docker/profiles/docker-compose.prerequisites.yml

@@ -21,6 +21,7 @@ x-mysql-profiles-dev: &mysql-profiles-dev
   - debug-neo4j
   - debug-elasticsearch
   - debug-backend-aws
+  - debug-aws
 x-mysql-profiles: &mysql-profiles
   - quickstart
   - quickstart-backend
@@ -36,6 +37,7 @@ x-mysql-profiles: &mysql-profiles
   - debug-neo4j
   - debug-elasticsearch
   - debug-backend-aws
+  - debug-aws
 
 x-postgres-profiles-quickstart: &postgres-profiles-quickstart
   - quickstart-postgres
@@ -78,6 +80,7 @@ x-opensearch-profiles-dev: &opensearch-profiles-dev
   - debug-consumers
   - debug-neo4j
   - debug-backend-aws
+  - debug-aws
 x-opensearch-profiles: &opensearch-profiles
   - quickstart
   - quickstart-backend
@@ -97,10 +100,12 @@ x-opensearch-profiles: &opensearch-profiles
   - debug-consumers
   - debug-neo4j
   - debug-backend-aws
+  - debug-aws
 
 # AWS Infra profiles
 x-profiles-aws: &aws-profiles
   - debug-backend-aws
+  - debug-aws
 
 # Debug vs Quickstart Profiles
 x-profiles-quickstart: &profiles-quickstart
@@ -124,6 +129,7 @@ x-profiles-dev: &profiles-dev
   - debug-neo4j
   - debug-elasticsearch
   - debug-backend-aws
+  - debug-aws
 
 services:
   mysql:

docker/quickstart/docker-compose.quickstart-profile.yml

@@ -243,6 +243,7 @@ services:
     - debug-neo4j
     - debug-elasticsearch
     - debug-backend-aws
+    - debug-aws
     command:
     - --character-set-server=utf8mb4
     - --collation-server=utf8mb4_bin
@@ -319,6 +320,7 @@ services:
     - debug-consumers
     - debug-neo4j
     - debug-backend-aws
+    - debug-aws
     deploy:
       resources:
         limits: