Skip to content

Commit 17ded40

Browse files
author
Eric Cornwell
committed
First migration commit
1 parent be466b6 commit 17ded40

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+14830
-0
lines changed

‎.gitignore‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
deployment/cdk/cdk.out
2+
deployment/cdk/node_modules
3+
deployment/cdk/package-lock.json
4+
deployment/cdk/outputs.json
5+
deployment/cdk/stacks/__pycache__
6+
deployment/cdk/stacks/components/__pycache__/
7+
deployment/terraform/.terraform
8+
deployment/terraform/.terraform.lock.hcl
9+
deployment/terraform/.terraform.tfstate.lock.info
10+
deployment/terraform/terraform.tfstate
11+
deployment/terraform/terraform.tfstate.backup
12+
deployment/terraform/outputs.json
13+
source/lambda/workflow_trigger/workflow_trigger.zip
14+
source/lambda/workflow_complete/workflow_complete.zip
15+
source/workflow-submissions
16+
source/containers/build-deploy-containers.log
17+
deployment/terraform/workflow-submissions/
18+
deployment/cdk/workflow-submissions/
19+
source/Gradio/.DS_Store
20+
source/venv/
21+
source/Gradio/workflow-submissions

‎deployment/cdk/app.py‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: LicenseRef-.amazon.com.-AmznSL-1.0
4+
# Licensed under the Amazon Software License http://aws.amazon.com/asl/
5+
#
6+
# Main entry into CDK App to build infrastructure stack
7+
#
8+
9+
import os
10+
import json
11+
import aws_cdk as cdk
12+
from stacks.infra_stack import GSWorkflowBaseStack
13+
from stacks.post_deploy_stack import GSWorkflowPostDeployStack
14+
15+
app = cdk.App()
16+
17+
# Load the app configuration from the config.json file
18+
try:
19+
with open("config.json", "r", encoding="utf-8") as config_file:
20+
config_data = json.load(config_file)
21+
except Exception as e:
22+
print(f"Could not read the app configuration file. {e}")
23+
raise e
24+
25+
# Set CDK environment variables
26+
environment = cdk.Environment(
27+
account=config_data['accountId'],
28+
region=config_data['region']
29+
)
30+
31+
# Comply with SageMaker path definitions
32+
current_path = os.path.dirname(os.path.realpath(__file__))
33+
build_args = {'CODE_PATH':'/opt/ml/code','MODEL_PATH':'/opt/ml/model'}
34+
35+
# Handle deploying and destroying groups of stacks
36+
select_all = False
37+
bundling_stacks = app.node.try_get_context("aws:cdk:bundling-stacks")
38+
is_destroy = app.node.try_get_context("destroy")
39+
bootstrap = app.node.try_get_context("bootstrap")
40+
41+
# Check if bundling_stacks exists and contains "**"
42+
if bundling_stacks and "**" in bundling_stacks:
43+
select_all = True
44+
45+
# Create the Base Stack
46+
if is_destroy or select_all or bootstrap or bundling_stacks is None or (bundling_stacks and "GSWorkflowBaseStack" in bundling_stacks):
47+
print("Creating base stack...")
48+
base_stack = GSWorkflowBaseStack(
49+
scope=app,
50+
id="GSWorkflowBaseStack",
51+
config_data=config_data,
52+
env=environment
53+
)
54+
55+
# Always include post-deploy stack in the app definition, even during destroy
56+
# Handle cases where bundling_stacks is None, empty list, or contains the stack name
57+
if select_all or bundling_stacks is None or len(bundling_stacks) == 0 or (bundling_stacks and "GSWorkflowPostDeployStack" in bundling_stacks):
58+
print("Post-deploy stack condition is TRUE")
59+
try:
60+
print("Creating post-deploy stack...")
61+
outputs_path = os.path.join(current_path, "outputs.json")
62+
print(f"Looking for outputs file at: {outputs_path}")
63+
print(f"File exists: {os.path.exists(outputs_path)}")
64+
65+
# Try to read existing outputs
66+
with open(outputs_path, "r") as f:
67+
output_data = json.load(f)
68+
print(f"Successfully loaded outputs data: {list(output_data.keys()) if output_data else 'empty'}")
69+
70+
post_deploy_stack = GSWorkflowPostDeployStack(
71+
scope=app,
72+
id="GSWorkflowPostDeployStack",
73+
config_data=config_data,
74+
output_json_path=outputs_path,
75+
build_args=build_args,
76+
dockerfile_path=os.path.join(current_path, "../../source/container"),
77+
env=environment
78+
)
79+
print("Post-deploy stack created successfully")
80+
81+
if 'base_stack' in locals():
82+
post_deploy_stack.add_dependency(base_stack)
83+
print("Added dependency on base stack")
84+
except (FileNotFoundError, KeyError) as e:
85+
print(f"Warning: Could not create post-deploy stack due to missing outputs: {e}")
86+
except Exception as e:
87+
print(f"Error creating post-deploy stack: {str(e)}")
88+
else:
89+
print("Post-deploy stack condition is FALSE")
90+
91+
app.synth()

‎deployment/cdk/cdk.json‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
{
2+
"app": "python app.py",
3+
"watch": {
4+
"include": [
5+
"**"
6+
],
7+
"exclude": [
8+
"README.md",
9+
"cdk*.json",
10+
"requirements*.txt",
11+
"source.bat",
12+
"**/__init__.py",
13+
"python/__pycache__",
14+
"tests"
15+
]
16+
},
17+
"context": {
18+
"@aws-cdk/aws-lambda:recognizeLayerVersion": true,
19+
"@aws-cdk/core:checkSecretUsage": true,
20+
"@aws-cdk/core:target-partitions": [
21+
"aws",
22+
"aws-cn"
23+
],
24+
"@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
25+
"@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
26+
"@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
27+
"@aws-cdk/aws-iam:minimizePolicies": true,
28+
"@aws-cdk/core:validateSnapshotRemovalPolicy": true,
29+
"@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
30+
"@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
31+
"@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
32+
"@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
33+
"@aws-cdk/core:enablePartitionLiterals": true,
34+
"@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
35+
"@aws-cdk/aws-iam:standardizedServicePrincipals": true,
36+
"@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
37+
"@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
38+
"@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
39+
"@aws-cdk/aws-route53-patters:useCertificate": true,
40+
"@aws-cdk/customresources:installLatestAwsSdkDefault": false,
41+
"@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
42+
"@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
43+
"@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
44+
"@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
45+
"@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
46+
"@aws-cdk/aws-redshift:columnId": true,
47+
"@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
48+
"@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
49+
"@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
50+
"@aws-cdk/aws-kms:aliasNameRef": true,
51+
"@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
52+
"@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
53+
"@aws-cdk/aws-efs:denyAnonymousAccess": true,
54+
"@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
55+
"@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true,
56+
"@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true,
57+
"@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true,
58+
"@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true,
59+
"@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true,
60+
"@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true,
61+
"@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true,
62+
"@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true,
63+
"@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true,
64+
"@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true,
65+
"@aws-cdk/aws-eks:nodegroupNameAttribute": true,
66+
"@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true,
67+
"@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true
68+
}
69+
}

‎deployment/cdk/config.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"comment": "NOTE: BEFORE DEPLOYING STACK, UPDATE THE ACCOUNT ID AND REGION BELOW",
3+
"accountId": "",
4+
"region": "us-east-1",
5+
"constructNamePrefix": "3dgs",
6+
"s3TriggerKey": "workflow-input",
7+
"adminEmail": "",
8+
"maintainS3ObjectsOnStackDeletion": "true"
9+
}

‎deployment/cdk/requirements.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
aws-cdk-lib==2.144.0
2+
constructs>=10.0.0,<11.0.0
3+
cdk-ecr-deployment==3.0.127
4+
boto3
5+
gradio

‎deployment/cdk/stacks/components/container_deployment.py‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
# SPDX-License-Identifier: LicenseRef-.amazon.com.-AmznSL-1.0
3+
# Licensed under the Amazon Software License http://aws.amazon.com/asl/
4+
5+
"""Main construct to build and push the container resources to ECR"""
6+
7+
from aws_cdk import (
8+
Environment,
9+
RemovalPolicy,
10+
aws_ecr as ecr,
11+
aws_iam as iam,
12+
aws_ecr_assets as ecr_assets,
13+
CfnOutput
14+
)
15+
from aws_cdk.aws_ecr_assets import DockerImageAsset
16+
import cdk_ecr_deployment
17+
from constructs import Construct
18+
import json
19+
import os
20+
21+
class ContainerDeployment(Construct):
22+
"""Class for Container Deployment Construct"""
23+
def __init__(
24+
self,
25+
scope: Construct,
26+
id: str,
27+
config_data: dict,
28+
output_data: dict,
29+
build_args: dict,
30+
dockerfile_path: str,
31+
env: Environment,
32+
**kwargs) -> None:
33+
super().__init__(scope, id, **kwargs)
34+
35+
try:
36+
# Validate input parameters
37+
if not dockerfile_path or not os.path.exists(dockerfile_path):
38+
raise ValueError(f"Invalid dockerfile path: {dockerfile_path}")
39+
if not os.path.exists(os.path.join(dockerfile_path, "Dockerfile")):
40+
raise ValueError(f"Dockerfile not found in {dockerfile_path}")
41+
42+
# Get ECR repository name
43+
print(output_data)
44+
# Check if output_data already has the GSWorkflowBaseStack key or is already the stack outputs
45+
if 'GSWorkflowBaseStack' in output_data:
46+
ecr_repo_name = output_data['GSWorkflowBaseStack']['ECRRepoName']
47+
else:
48+
ecr_repo_name = output_data['ECRRepoName']
49+
50+
if not ecr_repo_name:
51+
raise ValueError("ECR repository name not found in output data")
52+
53+
# Create deployment role with required permissions
54+
deployment_role = iam.Role(
55+
self,
56+
"ECRDeploymentRole",
57+
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
58+
managed_policies=[
59+
iam.ManagedPolicy.from_aws_managed_policy_name(
60+
"service-role/AWSLambdaBasicExecutionRole"
61+
)
62+
]
63+
)
64+
65+
# Add ECR permissions
66+
deployment_role.add_to_policy(
67+
iam.PolicyStatement(
68+
actions=[
69+
"ecr:GetAuthorizationToken",
70+
"ecr:BatchCheckLayerAvailability",
71+
"ecr:GetDownloadUrlForLayer",
72+
"ecr:GetRepositoryPolicy",
73+
"ecr:DescribeRepositories",
74+
"ecr:ListImages",
75+
"ecr:DescribeImages",
76+
"ecr:BatchGetImage",
77+
"ecr:InitiateLayerUpload",
78+
"ecr:UploadLayerPart",
79+
"ecr:CompleteLayerUpload",
80+
"ecr:PutImage"
81+
],
82+
resources=[f"arn:aws:ecr:{env.region}:{env.account}:repository/{ecr_repo_name}"]
83+
)
84+
)
85+
86+
# Build and assign the docker image for ECR
87+
self.asset = ecr_assets.DockerImageAsset(
88+
self,
89+
"DockerImage",
90+
asset_name=ecr_repo_name,
91+
directory=dockerfile_path,
92+
build_args=build_args,
93+
platform=ecr_assets.Platform.LINUX_AMD64,
94+
#build_options={
95+
# "platform": "linux/amd64"
96+
#}
97+
)
98+
99+
# Copy image from cdk docker image asset to ECR
100+
self.deployment = cdk_ecr_deployment.ECRDeployment(
101+
self,
102+
"DeployDockerImage",
103+
src=cdk_ecr_deployment.DockerImageName(self.asset.image_uri),
104+
dest=cdk_ecr_deployment.DockerImageName(
105+
f"{env.account}.dkr.ecr.{env.region}.amazonaws.com/{ecr_repo_name}:latest"
106+
),
107+
role=deployment_role,
108+
memory_limit=512,
109+
)
110+
111+
# Add dependencies
112+
if hasattr(self.asset, "node"):
113+
self.deployment.node.add_dependency(self.asset)
114+
115+
# Add outputs
116+
CfnOutput(
117+
self,
118+
"DockerImageUri",
119+
value=self.asset.image_uri,
120+
description="URI of the built Docker image"
121+
)
122+
123+
CfnOutput(
124+
self,
125+
"ECRRepositoryUri",
126+
value=f"{env.account}.dkr.ecr.{env.region}.amazonaws.com/{ecr_repo_name}",
127+
description="URI of the ECR repository"
128+
)
129+
130+
except Exception as e:
131+
print(f"Error in ContainerDeployment construct: {e}")
132+
raise e
133+
134+
@property
135+
def image_uri(self) -> str:
136+
"""Return the URI of the deployed Docker image"""
137+
return self.asset.image_uri if hasattr(self, 'asset') else None
138+
139+
@property
140+
def deployment_role(self) -> iam.Role:
141+
"""Return the deployment role"""
142+
return self.deployment.role if hasattr(self, 'deployment') else None
143+
144+
def add_to_role_policy(self, statement: iam.PolicyStatement):
145+
"""Add additional permissions to the deployment role"""
146+
if self.deployment_role:
147+
self.deployment_role.add_to_policy(statement)

0 commit comments

Comments
 (0)