Resiliency changes (#1113)

Co-authored-by: cyturney <cyturney@amazon.com>
Co-authored-by: Sai Vennam <svennam92@gmail.com>

Author: 3 people

hack/run-tests.sh

@@ -76,6 +76,10 @@ RESOURCES_PRECREATED=${RESOURCES_PRECREATED:-""}
 
 echo "Running test suite..."
 
+# get current IDs
+USER_ID=$(id -u)
+GROUP_ID=$(id -g)
+
 exit_code=0
 
 $CONTAINER_CLI run $background_args $dns_args \

hack/validate-terraform.sh

@@ -62,3 +62,13 @@ Resources:
       ManagedPolicyName: ${Env}-ide-labs2
       PolicyDocument:
         file: ./iam/policies/labs2.yaml
+  EksWorkshopLabsPolicy3:
+    Type: AWS::IAM::ManagedPolicy
+    DependsOn:
+      - EksWorkshopIdeRole
+    Properties:
+      Roles:
+        - !Ref EksWorkshopIdeRole
+      ManagedPolicyName: ${Env}-ide-labs3
+      PolicyDocument:
+        file: ./iam/policies/labs3.yaml

lab/iam/iam-role-cfn.yaml

@@ -9,6 +9,7 @@ Statement:
     Resource: ["*"]
   - Effect: Allow
     Action:
+      - ec2:StopInstances
       - ec2:TerminateInstances
     Resource: ["*"]
     Condition:

lab/iam/policies/ec2.yaml

@@ -23,12 +23,19 @@ Statement:
       - iam:CreatePolicy
       - iam:DeletePolicy
       - iam:GetPolicyVersion
+      - iam:DeletePolicyVersion
       - iam:ListPolicyVersions
       - iam:TagPolicy
       - iam:GetPolicy
     Resource:
       - !Sub arn:aws:iam::${AWS::AccountId}:policy/${Env}*
       - !Sub arn:aws:iam::${AWS::AccountId}:policy/eksctl-${Env}*
+  - Effect: Allow
+    Action:
+      - s3:ListAllMyBuckets
+      - iam:ListPolicies
+      - iam:ListRoles
+    Resource: ["*"]
   - Effect: Allow
     Action:
       - iam:CreateInstanceProfile

lab/iam/policies/iam.yaml

@@ -0,0 +1,70 @@
+Version: "2012-10-17"
+Statement:
+  - Effect: Allow
+    Action:
+      - s3:CreateBucket
+      - s3:DeleteBucket
+      - s3:PutObject
+      - s3:List*
+      - s3:Get*
+      - s3:GetObjectVersion
+      - s3:PutBucketPublicAccessBlock
+      - s3:PutBucketTagging
+      - s3:DeleteObject
+      - s3:DeleteObjectVersion
+    Resource:
+      - arn:aws:s3:::eks-workshop-canary-artifacts*
+      - arn:aws:s3:::aws-synthetics-library*
+  - Effect: Allow
+    Action:
+      - fis:CreateExperimentTemplate
+      - fis:CreateExperimentTemplate
+      - fis:GetExperimentTemplate
+      - fis:ListExperimentTemplates
+      - fis:DeleteExperimentTemplate
+      - fis:UpdateExperimentTemplate
+      - fis:TagResource
+      - fis:UntagResource
+      - fis:StartExperiment
+      - fis:GetExperiment
+      - fis:ListExperiments
+    Resource:
+      - !Sub arn:aws:fis:${AWS::Region}:${AWS::AccountId}:action/aws:eks:*
+      - !Sub arn:aws:fis:${AWS::Region}:${AWS::AccountId}:action/aws:ssm:*
+      - !Sub arn:aws:fis:${AWS::Region}:${AWS::AccountId}:experiment-template/*
+      - !Sub arn:aws:fis:${AWS::Region}:${AWS::AccountId}:experiment/*
+
+  - Effect: Allow
+    Action:
+      - synthetics:CreateCanary
+      - synthetics:DeleteCanary
+      - synthetics:DescribeCanaries
+      - synthetics:StartCanary
+      - synthetics:StopCanary
+      - synthetics:UpdateCanary
+    Resource:
+      - !Sub arn:aws:synthetics:${AWS::Region}:${AWS::AccountId}:canary:${Env}*
+  - Effect: Allow
+    Action:
+      - cloudwatch:PutMetricAlarm
+      - cloudwatch:PutMetricData
+      - cloudwatch:GetMetricStatistics
+      - cloudwatch:ListMetrics
+    Resource:
+      - !Sub arn:aws:cloudwatch:${AWS::Region}:${AWS::AccountId}:alarm:${Env}*
+  - Effect: Allow
+    Action:
+      - lambda:CreateFunction
+      - lambda:UpdateFunctionCode
+      - lambda:GetFunctionConfiguration
+      - lambda:UpdateFunctionConfiguration
+      - lambda:GetFunction
+      - lambda:DeleteFunction
+      - lambda:InvokeFunction
+      - lambda:AddPermission
+      - lambda:RemovePermission
+      - lambda:PublishLayerVersion
+      - lambda:PublishVersion
+    Resource:
+      - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:*${Env}*
+      - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:layer:*${Env}*

lab/iam/policies/labs3.yaml

@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "5.66.0"
+      version = "5.72.0"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

manifests/.workshop/terraform/base.tf

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+set -e
+
+echo "Starting cleanup process..."
+
+# Function to safely delete a resource
+safe_delete() {
+    local cmd=$1
+    local resource=$2
+    echo "Attempting to delete $resource..."
+    if $cmd 2>/dev/null; then
+        echo "$resource deleted successfully."
+    else
+        echo "Failed to delete $resource or it doesn't exist. Continuing..."
+    fi
+}
+
+# Delete Kubernetes resources
+echo "Cleaning up Kubernetes resources..."
+kubectl delete ingress,deployment,service -n ui --all --ignore-not-found
+kubectl delete role,rolebinding -n ui --all --ignore-not-found
+kubectl delete namespace chaos-mesh --ignore-not-found
+
+# Uninstall Helm charts
+echo "Uninstalling Helm charts..."
+helm uninstall aws-load-balancer-controller -n kube-system || true
+helm uninstall chaos-mesh -n chaos-mesh || true
+
+# Delete ALBs
+echo "Cleaning up ALBs..."
+for alb_arn in $(aws elbv2 describe-load-balancers --query "LoadBalancers[?starts_with(LoadBalancerName, 'k8s-ui-ui-') || starts_with(LoadBalancerName, 'k8s-default-ui-')].LoadBalancerArn" --output text); do
+    safe_delete "aws elbv2 delete-load-balancer --load-balancer-arn $alb_arn" "ALB $alb_arn"
+done
+
+# Delete IAM Roles and Policies
+echo "Cleaning up IAM roles and policies..."
+for role_prefix in "fis-execution-role-eks-workshop" "canary-execution-role-eks-workshop"; do
+    for role in $(aws iam list-roles --query "Roles[?starts_with(RoleName, '${role_prefix}')].RoleName" --output text); do
+        echo "Processing role: $role"
+        for policy in $(aws iam list-attached-role-policies --role-name $role --query "AttachedPolicies[*].PolicyArn" --output text); do
+            safe_delete "aws iam detach-role-policy --role-name $role --policy-arn $policy" "attached policy $policy from role $role"
+        done
+        for policy in $(aws iam list-role-policies --role-name $role --query "PolicyNames" --output text); do
+            safe_delete "aws iam delete-role-policy --role-name $role --policy-name $policy" "inline policy $policy from role $role"
+        done
+        safe_delete "aws iam delete-role --role-name $role" "IAM role $role"
+    done
+done
+
+for policy_prefix in "eks-resiliency-fis-policy" "eks-resiliency-canary-policy"; do
+    for policy_arn in $(aws iam list-policies --scope Local --query "Policies[?starts_with(PolicyName, '${policy_prefix}')].Arn" --output text); do
+        safe_delete "aws iam delete-policy --policy-arn $policy_arn" "IAM policy $policy_arn"
+    done
+done
+
+# Delete S3 buckets
+echo "Cleaning up S3 buckets..."
+for bucket in $(aws s3api list-buckets --query "Buckets[?starts_with(Name, 'eks-workshop-canary-artifacts-')].Name" --output text); do
+    aws s3 rm s3://$bucket --recursive
+    safe_delete "aws s3api delete-bucket --bucket $bucket" "S3 bucket $bucket"
+done
+
+# Delete CloudWatch Synthetics canary and alarm
+CANARY_NAME="eks-workshop-canary"
+ALARM_NAME="eks-workshop-canary-alarm"
+
+echo "Cleaning up CloudWatch Synthetics canary and alarm..."
+if aws synthetics get-canary --name $CANARY_NAME &>/dev/null; then
+    aws synthetics stop-canary --name $CANARY_NAME || true
+    sleep 30
+    safe_delete "aws synthetics delete-canary --name $CANARY_NAME" "CloudWatch Synthetics canary $CANARY_NAME"
+fi
+
+safe_delete "aws cloudwatch delete-alarms --alarm-names $ALARM_NAME" "CloudWatch alarm $ALARM_NAME"
+
+echo "Cleanup process completed. Please check for any remaining resources manually."