From f1ab9551239c8060e101181b571ffb839e0f6d9a Mon Sep 17 00:00:00 2001 From: Matt Smithson Date: Wed, 7 May 2025 11:27:23 +0100 Subject: [PATCH 1/5] fix: updated terraform and script and execution readme --- README-EXECUTE.md | 2 +- scripts/execute.sh | 11 ++++++++--- terraform/backend.hcl | 4 ++++ terraform/backend.hcl.temp | 4 ++++ terraform/data.tf | 1 + terraform/main.tf | 2 +- terraform/terraform.tf | 5 +++++ terraform/variables.tf | 6 +----- 8 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 terraform/backend.hcl create mode 100644 terraform/backend.hcl.temp create mode 100644 terraform/data.tf create mode 100644 terraform/terraform.tf diff --git a/README-EXECUTE.md b/README-EXECUTE.md index c8d32913..4d050b00 100644 --- a/README-EXECUTE.md +++ b/README-EXECUTE.md @@ -6,7 +6,7 @@ Independently of how you've deployed the state machine, you can execute it in a You'll find a few sample scripts in the `scripts` folder. -Feel free to customize the `scripts/sample-execution-input.json`, and then run `scripts/execute.sh`. +Feel free to customize the `scripts/sample-execution-input.json` or add a new json file, and then run `scripts/execute.sh `. The script will start a state machine execution, wait for the execution to complete (polling), and then show the execution results. diff --git a/scripts/execute.sh b/scripts/execute.sh index e5ae0aef..b6ed94b9 100644 --- a/scripts/execute.sh +++ b/scripts/execute.sh @@ -1,10 +1,15 @@ #!/bin/bash # config STACK_NAME=lambda-power-tuning -INPUT=$(cat scripts/sample-execution-input.json) # or use a static string +INPUT_FILE="${2}" +INPUT=$(cat "${INPUT_FILE}") # or use a static string # retrieve state machine ARN -STATE_MACHINE_ARN=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`StateMachineARN`].OutputValue' --output text) + +# we don't use this as CF isn't used +#STATE_MACHINE_ARN=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`StateMachineARN`].OutputValue' --output text) +STATE_MACHINE_ARN="${1}" + # start execution EXECUTION_ARN=$(aws stepfunctions start-execution --state-machine-arn $STATE_MACHINE_ARN --input "$INPUT" --query 'executionArn' --output text) @@ -30,7 +35,7 @@ do echo $STATUS echo "Execution output: " # retrieve output - aws stepfunctions describe-execution --execution-arn $EXECUTION_ARN --query 'output' --output text + aws stepfunctions describe-execution --execution-arn $EXECUTION_ARN --query 'output' --output text | cat break fi done diff --git a/terraform/backend.hcl b/terraform/backend.hcl new file mode 100644 index 00000000..b37ded3c --- /dev/null +++ b/terraform/backend.hcl @@ -0,0 +1,4 @@ +bucket = "sg-ssd-{stage}-{workload}-local-tfstate" +region = "eu-west-2" +key = "environments-{workload}/{environment}/{component}.tfstate" +dynamodb_table = "sg-ssd-{stage}-{workload}-local-tfstatelock" diff --git a/terraform/backend.hcl.temp b/terraform/backend.hcl.temp new file mode 100644 index 00000000..14e3c292 --- /dev/null +++ b/terraform/backend.hcl.temp @@ -0,0 +1,4 @@ +bucket = "sg-ssd-dev-ds-local-tfstate" +region = "eu-west-2" +key = "environments-ds/shared/aws-lambda-power-tuning.tfstate" +dynamodb_table = "sg-ssd-dev-ds-local-tfstatelock" diff --git a/terraform/data.tf b/terraform/data.tf new file mode 100644 index 00000000..d78fce49 --- /dev/null +++ b/terraform/data.tf @@ -0,0 +1 @@ +data "aws_caller_identity" "current" {} \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf index 98bdc0df..f72967ca 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,4 +1,4 @@ module "power_tuning" { source = "./module" - account_id = var.account_id + account_id = data.aws_caller_identity.current.account_id } \ No newline at end of file diff --git a/terraform/terraform.tf b/terraform/terraform.tf new file mode 100644 index 00000000..ed9e4aea --- /dev/null +++ b/terraform/terraform.tf @@ -0,0 +1,5 @@ +terraform { + backend "s3" { + encrypt = true + } +} \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf index ecf2de1d..844bbaa1 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,7 +1,3 @@ variable "aws_region" { - default = "eu-west-1" -} - -variable "account_id" { - default = "123456789101" + default = "eu-west-2" } \ No newline at end of file From af3678e1de22ca6c85be2d64ec893715ec9bd5d0 Mon Sep 17 00:00:00 2001 From: Matt Smithson Date: Wed, 7 May 2025 12:05:05 +0100 Subject: [PATCH 2/5] fix: added backend.hcl at root level --- backend.hcl | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 backend.hcl diff --git a/backend.hcl b/backend.hcl new file mode 100644 index 00000000..b37ded3c --- /dev/null +++ b/backend.hcl @@ -0,0 +1,4 @@ +bucket = "sg-ssd-{stage}-{workload}-local-tfstate" +region = "eu-west-2" +key = "environments-{workload}/{environment}/{component}.tfstate" +dynamodb_table = "sg-ssd-{stage}-{workload}-local-tfstatelock" From 3639c998ed2000209e49f52d556995567fc86bbd Mon Sep 17 00:00:00 2001 From: Matt Smithson Date: Mon, 12 May 2025 09:28:35 +0100 Subject: [PATCH 3/5] feat: added executor timeout to terraform --- terraform/module/json_files/state_machine.json | 2 +- terraform/module/lambda.tf | 2 +- terraform/module/locals.tf | 1 + terraform/module/variables.tf | 6 ++++++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/terraform/module/json_files/state_machine.json b/terraform/module/json_files/state_machine.json index bcb6bd8e..3d91928e 100644 --- a/terraform/module/json_files/state_machine.json +++ b/terraform/module/json_files/state_machine.json @@ -61,7 +61,7 @@ "Type": "Task", "Resource": "${executorArn}", "End": true, - "TimeoutSeconds": 30, + "TimeoutSeconds": ${totalExecutionTimeout}, "Retry": [{ "ErrorEquals": ["States.ALL"], "IntervalSeconds": 3, diff --git a/terraform/module/lambda.tf b/terraform/module/lambda.tf index 7db7f542..77414fc5 100644 --- a/terraform/module/lambda.tf +++ b/terraform/module/lambda.tf @@ -80,7 +80,7 @@ resource "aws_lambda_function" "executor" { handler = "executor.handler" layers = [aws_lambda_layer_version.lambda_layer.arn] memory_size = 128 - timeout = 30 + timeout = var.total_execution_timeout # The filebase64sha256() function is available in Terraform 0.11.12 and later # For Terraform 0.11.11 and earlier, use the base64sha256() function and the file() function: diff --git a/terraform/module/locals.tf b/terraform/module/locals.tf index 0d592f18..72240c09 100644 --- a/terraform/module/locals.tf +++ b/terraform/module/locals.tf @@ -13,6 +13,7 @@ locals { initializerArn = aws_lambda_function.initializer.arn, publisherArn = aws_lambda_function.publisher.arn, executorArn = aws_lambda_function.executor.arn, + totalExecutionTimeout = var.total_execution_timeout cleanerArn = aws_lambda_function.cleaner.arn, analyzerArn = aws_lambda_function.analyzer.arn, optimizerArn = aws_lambda_function.optimizer.arn diff --git a/terraform/module/variables.tf b/terraform/module/variables.tf index b4b6e952..6eb2da33 100644 --- a/terraform/module/variables.tf +++ b/terraform/module/variables.tf @@ -7,6 +7,12 @@ variable "lambda_function_prefix" { description = "Prefix used for the names of Lambda functions, Step Functions state machines, IAM roles, and IAM policies." } +variable "total_execution_timeout" { + default = 300 + type = number + description = "Maximum invocation timeout (in seconds) for the Executor step, after which you get a States.Timeout error" +} + variable "role_path_override" { default = "" type = string From 1bc96feede18f6123c43cf1a5212dd3dd2bfa165 Mon Sep 17 00:00:00 2001 From: Matt Smithson Date: Mon, 12 May 2025 09:39:04 +0100 Subject: [PATCH 4/5] fix: remove changed files --- README-EXECUTE.md | 2 +- backend.hcl | 4 ---- scripts/execute.sh | 9 ++------- terraform/backend.hcl | 4 ---- terraform/backend.hcl.temp | 4 ---- terraform/data.tf | 1 - terraform/main.tf | 2 +- terraform/module/variables.tf | 6 ------ terraform/terraform.tf | 5 ----- terraform/variables.tf | 6 +++++- 10 files changed, 9 insertions(+), 34 deletions(-) delete mode 100644 backend.hcl delete mode 100644 terraform/backend.hcl delete mode 100644 terraform/backend.hcl.temp delete mode 100644 terraform/data.tf delete mode 100644 terraform/terraform.tf diff --git a/README-EXECUTE.md b/README-EXECUTE.md index 4d050b00..c8d32913 100644 --- a/README-EXECUTE.md +++ b/README-EXECUTE.md @@ -6,7 +6,7 @@ Independently of how you've deployed the state machine, you can execute it in a You'll find a few sample scripts in the `scripts` folder. -Feel free to customize the `scripts/sample-execution-input.json` or add a new json file, and then run `scripts/execute.sh `. +Feel free to customize the `scripts/sample-execution-input.json`, and then run `scripts/execute.sh`. The script will start a state machine execution, wait for the execution to complete (polling), and then show the execution results. diff --git a/backend.hcl b/backend.hcl deleted file mode 100644 index b37ded3c..00000000 --- a/backend.hcl +++ /dev/null @@ -1,4 +0,0 @@ -bucket = "sg-ssd-{stage}-{workload}-local-tfstate" -region = "eu-west-2" -key = "environments-{workload}/{environment}/{component}.tfstate" -dynamodb_table = "sg-ssd-{stage}-{workload}-local-tfstatelock" diff --git a/scripts/execute.sh b/scripts/execute.sh index b6ed94b9..34446c36 100644 --- a/scripts/execute.sh +++ b/scripts/execute.sh @@ -1,15 +1,10 @@ #!/bin/bash # config STACK_NAME=lambda-power-tuning -INPUT_FILE="${2}" -INPUT=$(cat "${INPUT_FILE}") # or use a static string +INPUT=$(cat scripts/sample-execution-input.json) # or use a static string # retrieve state machine ARN - -# we don't use this as CF isn't used -#STATE_MACHINE_ARN=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`StateMachineARN`].OutputValue' --output text) -STATE_MACHINE_ARN="${1}" - +STATE_MACHINE_ARN=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`StateMachineARN`].OutputValue' --output text) # start execution EXECUTION_ARN=$(aws stepfunctions start-execution --state-machine-arn $STATE_MACHINE_ARN --input "$INPUT" --query 'executionArn' --output text) diff --git a/terraform/backend.hcl b/terraform/backend.hcl deleted file mode 100644 index b37ded3c..00000000 --- a/terraform/backend.hcl +++ /dev/null @@ -1,4 +0,0 @@ -bucket = "sg-ssd-{stage}-{workload}-local-tfstate" -region = "eu-west-2" -key = "environments-{workload}/{environment}/{component}.tfstate" -dynamodb_table = "sg-ssd-{stage}-{workload}-local-tfstatelock" diff --git a/terraform/backend.hcl.temp b/terraform/backend.hcl.temp deleted file mode 100644 index 14e3c292..00000000 --- a/terraform/backend.hcl.temp +++ /dev/null @@ -1,4 +0,0 @@ -bucket = "sg-ssd-dev-ds-local-tfstate" -region = "eu-west-2" -key = "environments-ds/shared/aws-lambda-power-tuning.tfstate" -dynamodb_table = "sg-ssd-dev-ds-local-tfstatelock" diff --git a/terraform/data.tf b/terraform/data.tf deleted file mode 100644 index d78fce49..00000000 --- a/terraform/data.tf +++ /dev/null @@ -1 +0,0 @@ -data "aws_caller_identity" "current" {} \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf index f72967ca..98bdc0df 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,4 +1,4 @@ module "power_tuning" { source = "./module" - account_id = data.aws_caller_identity.current.account_id + account_id = var.account_id } \ No newline at end of file diff --git a/terraform/module/variables.tf b/terraform/module/variables.tf index 6eb2da33..b4b6e952 100644 --- a/terraform/module/variables.tf +++ b/terraform/module/variables.tf @@ -7,12 +7,6 @@ variable "lambda_function_prefix" { description = "Prefix used for the names of Lambda functions, Step Functions state machines, IAM roles, and IAM policies." } -variable "total_execution_timeout" { - default = 300 - type = number - description = "Maximum invocation timeout (in seconds) for the Executor step, after which you get a States.Timeout error" -} - variable "role_path_override" { default = "" type = string diff --git a/terraform/terraform.tf b/terraform/terraform.tf deleted file mode 100644 index ed9e4aea..00000000 --- a/terraform/terraform.tf +++ /dev/null @@ -1,5 +0,0 @@ -terraform { - backend "s3" { - encrypt = true - } -} \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf index 844bbaa1..ecf2de1d 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,3 +1,7 @@ variable "aws_region" { - default = "eu-west-2" + default = "eu-west-1" +} + +variable "account_id" { + default = "123456789101" } \ No newline at end of file From 38bb5189eeeb1359c0e635a32b2588639607690d Mon Sep 17 00:00:00 2001 From: Matt Smithson Date: Mon, 12 May 2025 09:40:26 +0100 Subject: [PATCH 5/5] fix: remove changed files --- scripts/execute.sh | 2 +- terraform/module/variables.tf | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/execute.sh b/scripts/execute.sh index 34446c36..e5ae0aef 100644 --- a/scripts/execute.sh +++ b/scripts/execute.sh @@ -30,7 +30,7 @@ do echo $STATUS echo "Execution output: " # retrieve output - aws stepfunctions describe-execution --execution-arn $EXECUTION_ARN --query 'output' --output text | cat + aws stepfunctions describe-execution --execution-arn $EXECUTION_ARN --query 'output' --output text break fi done diff --git a/terraform/module/variables.tf b/terraform/module/variables.tf index b4b6e952..68e02608 100644 --- a/terraform/module/variables.tf +++ b/terraform/module/variables.tf @@ -7,6 +7,13 @@ variable "lambda_function_prefix" { description = "Prefix used for the names of Lambda functions, Step Functions state machines, IAM roles, and IAM policies." } +variable "executor_function_timeout" { + default = 30 + type = number + description = "Timeout for the executor function" +} + + variable "role_path_override" { default = "" type = string