From be7f52a89fe9518081c334e7184c9322b3558808 Mon Sep 17 00:00:00 2001 From: Robert - Localstack Date: Wed, 19 Jun 2024 21:22:48 +0300 Subject: [PATCH 1/2] Add docs on how to share s3 resources to another account --- .../README.md | 217 ++++++++++++++++++ .../bucket/main.go | 11 + .../bucket/main_test.go | 10 + .../source_bucket_policy.json | 21 ++ 4 files changed, 259 insertions(+) create mode 100644 multi-account-multi-region-s3-access/README.md create mode 100644 multi-account-multi-region-s3-access/bucket/main.go create mode 100644 multi-account-multi-region-s3-access/bucket/main_test.go create mode 100644 multi-account-multi-region-s3-access/source_bucket_policy.json diff --git a/multi-account-multi-region-s3-access/README.md b/multi-account-multi-region-s3-access/README.md new file mode 100644 index 0000000..c223b8b --- /dev/null +++ b/multi-account-multi-region-s3-access/README.md @@ -0,0 +1,217 @@ +# Localstack Demo: Access S3 resources from different account and different region + +## Profiles + +The following profiles will have to be created: + +* Profile `ls-a-admin`: admin user of account A. + +* Profile `ls-b-admin`: admin user of account B. + +* Profile `ls-a`: account A user that creates the S3 bucket and afferent resources inside the bucket. + +* Profile `ls-b`: account B user that copies the resources from account A user's S3 bucket into a bucket it owns. + +## Create Users + +Create the following profiles in `~/.aws/config`: + +``` +[profile ls-a-admin] +region=us-east-1 +output=json +endpoint_url=https://localhost.localstack.cloud:4566 + +[profile ls-b-admin] +region=eu-central-1 +output=json +endpoint_url=https://localhost.localstack.cloud:4566 +``` + +The `~/.aws/credentials` would initially look as follows: + +``` +[ls-a-admin] +aws_access_key_id=000000000001 +aws_secret_access_key=test + +[ls-b-admin] +aws_access_key_id=000000000002 +aws_secret_access_key=test +``` + +Then, create the actual `ls-a` and `ls-b` users: + +```shell +aws iam create-user --user-name ls-a --profile ls-a-admin +aws iam create-user --user-name ls-b --profile ls-b-admin +``` + +The following outputs would be returned: + +```shell +localstack@macintosh serverless-data-processing-pipeline % aws iam create-user --user-name ls-a --profile ls-a-admin +{ + "User": { + "Path": "/", + "UserName": "ls-a", + "UserId": "m963bjpmc9sh0khhd9co", + "Arn": "arn:aws:iam::000000000001:user/ls-a", + "CreateDate": "2024-06-19T16:57:16.763000Z" + } +} +localstack@macintosh serverless-data-processing-pipeline % aws iam create-user --user-name ls-b --profile ls-b-admin +{ + "User": { + "Path": "/", + "UserName": "ls-b", + "UserId": "juo7catmp7dj7y66kk3g", + "Arn": "arn:aws:iam::000000000002:user/ls-b", + "CreateDate": "2024-06-19T17:04:34.649000Z" + } +} +``` + +## Attach Policy to Users + +Create policies `pa` and `pb` for user A and respectively user B: + +```shell +aws iam create-policy --policy-name pa --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' --profile ls-a-admin +aws iam create-policy --policy-name pb --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' --profile ls-b-admin +``` + +```shell +localstack@macintosh serverless-data-processing-pipeline % aws iam create-policy --policy-name pa --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' --profile ls-a-admin +{ + "Policy": { + "PolicyName": "pa", + "PolicyId": "A9LSZ6ZQ1IZPK4CI9S5AJ", + "Arn": "arn:aws:iam::000000000001:policy/pa", + "Path": "/", + "DefaultVersionId": "v1", + "AttachmentCount": 0, + "CreateDate": "2024-06-19T17:09:44.758000Z", + "UpdateDate": "2024-06-19T17:09:44.758000Z", + "Tags": [] + } +} +localstack@macintosh serverless-data-processing-pipeline % aws iam create-policy --policy-name pb --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' --profile ls-b-admin +{ + "Policy": { + "PolicyName": "pb", + "PolicyId": "AMAKM9JGV5WB9NWZP218Q", + "Arn": "arn:aws:iam::000000000002:policy/pb", + "Path": "/", + "DefaultVersionId": "v1", + "AttachmentCount": 0, + "CreateDate": "2024-06-19T17:09:54.443000Z", + "UpdateDate": "2024-06-19T17:09:54.443000Z", + "Tags": [] + } +} +``` + +And then attach said policies to each user: + +```shell +aws iam attach-user-policy --user-name ls-a --policy-arn arn:aws:iam::000000000001:policy/pa --profile ls-a-admin +aws iam attach-user-policy --user-name ls-b --policy-arn arn:aws:iam::000000000002:policy/pb --profile ls-b-admin +``` + +## Create User Profiles + +Generate the AWS keys for each respective user: + +```shell +aws iam create-access-key --user-name ls-a --profile ls-a-admin +aws iam create-access-key --user-name ls-b --profile ls-b-admin +``` + +```shell +localstack@macintosh serverless-data-processing-pipeline % aws iam create-access-key --user-name ls-a --profile ls-a-admin +{ + "AccessKey": { + "UserName": "ls-a", + "AccessKeyId": "LKIAQAAAAAAA4ZFEPLMV", + "Status": "Active", + "SecretAccessKey": "NV/Uft4BtS1ZgnXOOFvYc+xKZj921lCbDRducl9N", + "CreateDate": "2024-06-19T17:16:59Z" + } +} +localstack@macintosh serverless-data-processing-pipeline % aws iam create-access-key --user-name ls-b --profile ls-b-admin +{ + "AccessKey": { + "UserName": "ls-b", + "AccessKeyId": "LKIAQAAAAAABCYRWOBQD", + "Status": "Active", + "SecretAccessKey": "/asqAjMJvpFt/dHeGW7ILG8ZZQGQLabx2KOtILRp", + "CreateDate": "2024-06-19T17:17:05Z" + } +} +``` + +And then create the respective `ls-a` and `ls-b` profiles in `~/.aws/config` and `~/.aws/credentials` based on each user's access key id and secret access key: + +```text +[profile ls-a] +region=us-east-1 +output=json +endpoint_url=https://localhost.localstack.cloud:4566 + +[profile ls-b] +region=eu-central-1 +output=json +endpoint_url=https://localhost.localstack.cloud:4566 +``` + +Finally, verify that listing S3 buckets works: + +```shell +aws s3 ls --profile ls-a +aws s3 ls --profile ls-b +``` + +## Create S3 Resource in Account A + +Create the S3 bucket and add a few objects to the bucket: + +```shell +aws s3 mb s3://source --profile ls-a +aws s3 sync ./bucket s3://source --profile ls-a +``` + +## Attach Bucket Policy + +Let's attach a bucket policy to the bucket in account A that allows user from account B to access the objects on said bucket: + +```shell +aws s3api put-bucket-policy --bucket source --policy file://source_bucket_policy.json --profile ls-a +``` + +Where `source_bucket_policy.json` would look like this: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:Get*" + ], + "Principal": { "AWS": "arn:aws:iam::000000000002:user/ls-b" }, + "Resource": "arn:aws:s3:::source/*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:List*" + ], + "Principal": { "AWS": "arn:aws:iam::000000000002:user/ls-b" }, + "Resource": "arn:aws:s3:::source" + } + ] +} + +``` diff --git a/multi-account-multi-region-s3-access/bucket/main.go b/multi-account-multi-region-s3-access/bucket/main.go new file mode 100644 index 0000000..d78aad9 --- /dev/null +++ b/multi-account-multi-region-s3-access/bucket/main.go @@ -0,0 +1,11 @@ +package main + +import "fmt" + +func helloworld() string { + return "Hello World!!" +} + +func main() { + fmt.Println(helloworld()) +} diff --git a/multi-account-multi-region-s3-access/bucket/main_test.go b/multi-account-multi-region-s3-access/bucket/main_test.go new file mode 100644 index 0000000..c9952bb --- /dev/null +++ b/multi-account-multi-region-s3-access/bucket/main_test.go @@ -0,0 +1,10 @@ +package main + +import "testing" + +func TestHelloWorld(t *testing.T) { + if helloworld() != "Hello World!!" { + t.Fatal("Test fail") + } +} + diff --git a/multi-account-multi-region-s3-access/source_bucket_policy.json b/multi-account-multi-region-s3-access/source_bucket_policy.json new file mode 100644 index 0000000..de3efad --- /dev/null +++ b/multi-account-multi-region-s3-access/source_bucket_policy.json @@ -0,0 +1,21 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:Get*" + ], + "Principal": { "AWS": "arn:aws:iam::000000000002:user/ls-b" }, + "Resource": "arn:aws:s3:::source/*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:List*" + ], + "Principal": { "AWS": "arn:aws:iam::000000000002:user/ls-b" }, + "Resource": "arn:aws:s3:::source" + } + ] +} From a908e48b5e7e8882e0a4c46376f98cdd9a669f79 Mon Sep 17 00:00:00 2001 From: Robert - Localstack Date: Tue, 25 Jun 2024 22:42:42 +0300 Subject: [PATCH 2/2] Add test script, README and makefile targets --- multi-account-multi-region-s3-access/Makefile | 35 +++ .../README.md | 213 ++---------------- multi-account-multi-region-s3-access/run.sh | 47 ++++ 3 files changed, 104 insertions(+), 191 deletions(-) create mode 100644 multi-account-multi-region-s3-access/Makefile create mode 100755 multi-account-multi-region-s3-access/run.sh diff --git a/multi-account-multi-region-s3-access/Makefile b/multi-account-multi-region-s3-access/Makefile new file mode 100644 index 0000000..d63acd8 --- /dev/null +++ b/multi-account-multi-region-s3-access/Makefile @@ -0,0 +1,35 @@ +# Enforce IAM to test cross-account cross-region access +export ENFORCE_IAM=1 +export DEBUG=1 + +SHELL := /bin/bash + +usage: ## Show this help + @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//' + +install: ## Install dependencies + @which localstack || pip install localstack + @which awslocal || pip install awscli-local + +run: ## Run the cross-account cross-region experiment of copying data from one S3 bucket to another + ./run.sh + +start: ## Start LocalStack + localstack start -d + +stop: ## Stop LocalStack + @echo + localstack stop + +ready: ## Wait for LocalStack to be ready + @echo Waiting on the LocalStack container... + @localstack wait -t 30 && echo Localstack is ready to use! || (echo Gave up waiting on LocalStack, exiting. && exit 1) + +logs: ## Retrieve logs from LocalStack + @localstack logs > logs.txt + +test-ci: ## Run CI test + make start install ready run; return_code=`echo $$?`;\ + make logs; make stop; exit $$return_code; + +.PHONY: usage install start run stop ready logs test-ci diff --git a/multi-account-multi-region-s3-access/README.md b/multi-account-multi-region-s3-access/README.md index c223b8b..d5ff161 100644 --- a/multi-account-multi-region-s3-access/README.md +++ b/multi-account-multi-region-s3-access/README.md @@ -1,217 +1,48 @@ # Localstack Demo: Access S3 resources from different account and different region -## Profiles +Simple demo script to showcase the accessing of S3 resources from a different AWS account using bucket policies and IAM users with specific IAM policies attached to their identities. +The script uses a couple of AWS profiles to achieve that: -The following profiles will have to be created: +* Admin user of account A with account ID `000000000001`. -* Profile `ls-a-admin`: admin user of account A. +* Admin user of account B with account ID `000000000002`. -* Profile `ls-b-admin`: admin user of account B. +* Account A user that creates the S3 bucket and subsequent resources inside the bucket. -* Profile `ls-a`: account A user that creates the S3 bucket and afferent resources inside the bucket. +* Account B user that copies the resources from account A user's S3 bucket `source` into a bucket `target` it owns. -* Profile `ls-b`: account B user that copies the resources from account A user's S3 bucket into a bucket it owns. +## Prerequisites -## Create Users +* LocalStack +* Docker +* Python 3.6+ / Python Pip +* `make` -Create the following profiles in `~/.aws/config`: +## Installing -``` -[profile ls-a-admin] -region=us-east-1 -output=json -endpoint_url=https://localhost.localstack.cloud:4566 - -[profile ls-b-admin] -region=eu-central-1 -output=json -endpoint_url=https://localhost.localstack.cloud:4566 -``` - -The `~/.aws/credentials` would initially look as follows: - -``` -[ls-a-admin] -aws_access_key_id=000000000001 -aws_secret_access_key=test - -[ls-b-admin] -aws_access_key_id=000000000002 -aws_secret_access_key=test -``` - -Then, create the actual `ls-a` and `ls-b` users: - -```shell -aws iam create-user --user-name ls-a --profile ls-a-admin -aws iam create-user --user-name ls-b --profile ls-b-admin -``` - -The following outputs would be returned: - -```shell -localstack@macintosh serverless-data-processing-pipeline % aws iam create-user --user-name ls-a --profile ls-a-admin -{ - "User": { - "Path": "/", - "UserName": "ls-a", - "UserId": "m963bjpmc9sh0khhd9co", - "Arn": "arn:aws:iam::000000000001:user/ls-a", - "CreateDate": "2024-06-19T16:57:16.763000Z" - } -} -localstack@macintosh serverless-data-processing-pipeline % aws iam create-user --user-name ls-b --profile ls-b-admin -{ - "User": { - "Path": "/", - "UserName": "ls-b", - "UserId": "juo7catmp7dj7y66kk3g", - "Arn": "arn:aws:iam::000000000002:user/ls-b", - "CreateDate": "2024-06-19T17:04:34.649000Z" - } -} -``` - -## Attach Policy to Users - -Create policies `pa` and `pb` for user A and respectively user B: - -```shell -aws iam create-policy --policy-name pa --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' --profile ls-a-admin -aws iam create-policy --policy-name pb --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' --profile ls-b-admin -``` +To install the dependencies: ```shell -localstack@macintosh serverless-data-processing-pipeline % aws iam create-policy --policy-name pa --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' --profile ls-a-admin -{ - "Policy": { - "PolicyName": "pa", - "PolicyId": "A9LSZ6ZQ1IZPK4CI9S5AJ", - "Arn": "arn:aws:iam::000000000001:policy/pa", - "Path": "/", - "DefaultVersionId": "v1", - "AttachmentCount": 0, - "CreateDate": "2024-06-19T17:09:44.758000Z", - "UpdateDate": "2024-06-19T17:09:44.758000Z", - "Tags": [] - } -} -localstack@macintosh serverless-data-processing-pipeline % aws iam create-policy --policy-name pb --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' --profile ls-b-admin -{ - "Policy": { - "PolicyName": "pb", - "PolicyId": "AMAKM9JGV5WB9NWZP218Q", - "Arn": "arn:aws:iam::000000000002:policy/pb", - "Path": "/", - "DefaultVersionId": "v1", - "AttachmentCount": 0, - "CreateDate": "2024-06-19T17:09:54.443000Z", - "UpdateDate": "2024-06-19T17:09:54.443000Z", - "Tags": [] - } -} +make install ``` -And then attach said policies to each user: +## Starting LocalStack -```shell -aws iam attach-user-policy --user-name ls-a --policy-arn arn:aws:iam::000000000001:policy/pa --profile ls-a-admin -aws iam attach-user-policy --user-name ls-b --policy-arn arn:aws:iam::000000000002:policy/pb --profile ls-b-admin -``` - -## Create User Profiles - -Generate the AWS keys for each respective user: - -```shell -aws iam create-access-key --user-name ls-a --profile ls-a-admin -aws iam create-access-key --user-name ls-b --profile ls-b-admin -``` - -```shell -localstack@macintosh serverless-data-processing-pipeline % aws iam create-access-key --user-name ls-a --profile ls-a-admin -{ - "AccessKey": { - "UserName": "ls-a", - "AccessKeyId": "LKIAQAAAAAAA4ZFEPLMV", - "Status": "Active", - "SecretAccessKey": "NV/Uft4BtS1ZgnXOOFvYc+xKZj921lCbDRducl9N", - "CreateDate": "2024-06-19T17:16:59Z" - } -} -localstack@macintosh serverless-data-processing-pipeline % aws iam create-access-key --user-name ls-b --profile ls-b-admin -{ - "AccessKey": { - "UserName": "ls-b", - "AccessKeyId": "LKIAQAAAAAABCYRWOBQD", - "Status": "Active", - "SecretAccessKey": "/asqAjMJvpFt/dHeGW7ILG8ZZQGQLabx2KOtILRp", - "CreateDate": "2024-06-19T17:17:05Z" - } -} -``` - -And then create the respective `ls-a` and `ls-b` profiles in `~/.aws/config` and `~/.aws/credentials` based on each user's access key id and secret access key: - -```text -[profile ls-a] -region=us-east-1 -output=json -endpoint_url=https://localhost.localstack.cloud:4566 - -[profile ls-b] -region=eu-central-1 -output=json -endpoint_url=https://localhost.localstack.cloud:4566 -``` - -Finally, verify that listing S3 buckets works: +Make sure that LocalStack is started: ```shell -aws s3 ls --profile ls-a -aws s3 ls --profile ls-b +LOCALSTACK_AUTH_TOKEN=... make start ``` -## Create S3 Resource in Account A +## Running -Create the S3 bucket and add a few objects to the bucket: +Run the sample demo script: ```shell -aws s3 mb s3://source --profile ls-a -aws s3 sync ./bucket s3://source --profile ls-a +make run ``` -## Attach Bucket Policy +## License -Let's attach a bucket policy to the bucket in account A that allows user from account B to access the objects on said bucket: +This code is available under the Apache 2.0 license. -```shell -aws s3api put-bucket-policy --bucket source --policy file://source_bucket_policy.json --profile ls-a -``` - -Where `source_bucket_policy.json` would look like this: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:Get*" - ], - "Principal": { "AWS": "arn:aws:iam::000000000002:user/ls-b" }, - "Resource": "arn:aws:s3:::source/*" - }, - { - "Effect": "Allow", - "Action": [ - "s3:List*" - ], - "Principal": { "AWS": "arn:aws:iam::000000000002:user/ls-b" }, - "Resource": "arn:aws:s3:::source" - } - ] -} - -``` diff --git a/multi-account-multi-region-s3-access/run.sh b/multi-account-multi-region-s3-access/run.sh new file mode 100755 index 0000000..7a38ebf --- /dev/null +++ b/multi-account-multi-region-s3-access/run.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -euxo pipefail +export AWS_SECRET_ACCESS_KEY=test + +# Create `ls-a` and `ls-b` IAM users using the root accounts of each user +AWS_ACCESS_KEY_ID=000000000001 awslocal iam create-user --user-name ls-a +AWS_ACCESS_KEY_ID=000000000002 awslocal iam create-user --user-name ls-b + +# Create IAM policies for each of the IAM users using the root accounts of each user +AWS_ACCESS_KEY_ID=000000000001 awslocal iam create-policy --policy-name pa --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' +AWS_ACCESS_KEY_ID=000000000002 awslocal iam create-policy --policy-name pb --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' + +# Attach the IAM policies to each IAM user using the root accounts of each user +AWS_ACCESS_KEY_ID=000000000001 awslocal iam attach-user-policy --user-name ls-a --policy-arn arn:aws:iam::000000000001:policy/pa +AWS_ACCESS_KEY_ID=000000000002 awslocal iam attach-user-policy --user-name ls-b --policy-arn arn:aws:iam::000000000002:policy/pb + +# Create access keys for each user using the root accounts of each user +CREDENTIALS_A=$(AWS_ACCESS_KEY_ID=000000000001 awslocal iam create-access-key --user-name ls-a) +CREDENTIALS_B=$(AWS_ACCESS_KEY_ID=000000000002 awslocal iam create-access-key --user-name ls-b) + +# Retrieve the access key id of each user +# In LocalStack, the secret access key is not strictly enforced +# But the access key id is +USER_ACCESS_KEY_ID_A=`jq -r .AccessKey.AccessKeyId <<< $CREDENTIALS_A` +USER_ACCESS_KEY_ID_B=`jq -r .AccessKey.AccessKeyId <<< $CREDENTIALS_B` + +# Create `source` bucket in `ls-a` user's account +AWS_ACCESS_KEY_ID=$USER_ACCESS_KEY_ID_A awslocal s3 mb s3://source +AWS_ACCESS_KEY_ID=$USER_ACCESS_KEY_ID_A awslocal s3 sync ./bucket s3://source + +# Attach a bucket policy so that user `ls-b` can access it +AWS_ACCESS_KEY_ID=$USER_ACCESS_KEY_ID_A awslocal s3api put-bucket-policy --bucket source --policy file://source_bucket_policy.json + +# Attempt to access bucket `source` using `ls-a` user +AWS_ACCESS_KEY_ID=$USER_ACCESS_KEY_ID_B awslocal s3 ls s3://source +AWS_ACCESS_KEY_ID=$USER_ACCESS_KEY_ID_B awslocal s3api list-object-versions --bucket source --prefix main.go + +# Sync buckets `source` and `target` using `ls-b` user +AWS_ACCESS_KEY_ID=$USER_ACCESS_KEY_ID_B awslocal s3 mb s3://target +AWS_ACCESS_KEY_ID=$USER_ACCESS_KEY_ID_B awslocal s3 sync s3://source s3://target +AWS_ACCESS_KEY_ID=$USER_ACCESS_KEY_ID_B awslocal s3api list-object-versions --bucket target --prefix main.go + +# Fail the script if somehow user A can access the resources of bucket `target` +echo "Check if s3api list-object-versions commnands fails as expected" +if AWS_ACCESS_KEY_ID=$USER_ACCESS_KEY_ID_A awslocal s3api list-object-versions --bucket target --prefix main.go; then + exit 1 +fi