From 12438feb48a2950b50dafc944c8d7933d291fda8 Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Wed, 4 Apr 2018 01:25:33 -0700 Subject: [PATCH 01/11] Change instance's volume size to a variable --- bin/create-ec2-machine-database.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/create-ec2-machine-database.sh b/bin/create-ec2-machine-database.sh index 4ed330e..d6e5792 100755 --- a/bin/create-ec2-machine-database.sh +++ b/bin/create-ec2-machine-database.sh @@ -42,7 +42,7 @@ aws ec2 run-instances \ --security-group-ids $SECURITYGROUPIDS \ --subnet-id $SUBNETID\ --region $REGION \ - --block-device-mappings "[{\"DeviceName\":\"/dev/sdb\",\"Ebs\":{\"VolumeSize\":8,\"VolumeType\":\"gp2\",\"DeleteOnTermination\":true}}]" \ + --block-device-mappings "[{\"DeviceName\":\"/dev/sdb\",\"Ebs\":{\"VolumeSize\":$VOLUMESIZE,\"VolumeType\":\"gp2\",\"DeleteOnTermination\":true}}]" \ --tag-specifications $TAG_SPECS \ --query 'Instances[0].InstanceId' \ > $INSTANCE_ID_FILE From 58f334cce8284dbe481d0c6adc34e1285b283af3 Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Wed, 4 Apr 2018 01:28:38 -0700 Subject: [PATCH 02/11] Update: Make instance's volume type a variable loaded from file --- bin/create-ec2-machine-database.sh | 3 ++- bin/ec2-profile-database-development.sh | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/create-ec2-machine-database.sh b/bin/create-ec2-machine-database.sh index d6e5792..473875a 100755 --- a/bin/create-ec2-machine-database.sh +++ b/bin/create-ec2-machine-database.sh @@ -24,6 +24,7 @@ fi # SECURITYGROUPIDS='sg-28154957' # SUBNETID='subnet-8794fddf' # VOLUMESIZE='8' +# VOLUMETYPE='gp2' EC2PROFILE=$2 INSTANCE_ID= @@ -42,7 +43,7 @@ aws ec2 run-instances \ --security-group-ids $SECURITYGROUPIDS \ --subnet-id $SUBNETID\ --region $REGION \ - --block-device-mappings "[{\"DeviceName\":\"/dev/sdb\",\"Ebs\":{\"VolumeSize\":$VOLUMESIZE,\"VolumeType\":\"gp2\",\"DeleteOnTermination\":true}}]" \ + --block-device-mappings "[{\"DeviceName\":\"/dev/sdb\",\"Ebs\":{\"VolumeSize\":$VOLUMESIZE,\"VolumeType\":\"$VOLUMETYPE\",\"DeleteOnTermination\":true}}]" \ --tag-specifications $TAG_SPECS \ --query 'Instances[0].InstanceId' \ > $INSTANCE_ID_FILE diff --git a/bin/ec2-profile-database-development.sh b/bin/ec2-profile-database-development.sh index 3e6f2cc..98edf6f 100755 --- a/bin/ec2-profile-database-development.sh +++ b/bin/ec2-profile-database-development.sh @@ -8,3 +8,5 @@ REGION='us-west-2' SECURITYGROUPIDS='sg-28154957' SUBNETID='subnet-8794fddf' VOLUMESIZE='8' +VOLUMETYPE='gp2' + From 6e8ea8e5021f9d658ab439a6ae542e77f64ba86b Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Wed, 4 Apr 2018 01:31:12 -0700 Subject: [PATCH 03/11] Update: DeleteOnTermination becomes a variable loaded from file --- bin/create-ec2-machine-database.sh | 3 ++- bin/ec2-profile-database-development.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/create-ec2-machine-database.sh b/bin/create-ec2-machine-database.sh index 473875a..d319999 100755 --- a/bin/create-ec2-machine-database.sh +++ b/bin/create-ec2-machine-database.sh @@ -17,6 +17,7 @@ fi # Source EC2 specs from a separate file # The ./ec2-profile.sh should contain the following variables # DEVICENAME='/dev/sdb' +# DELETEONTERM='true' # IMAGEID='ami-7f43f307' # INSTANCETYPE='t2.micro' # KEYNAME='hackoregon-2018-database-dev-env' @@ -43,7 +44,7 @@ aws ec2 run-instances \ --security-group-ids $SECURITYGROUPIDS \ --subnet-id $SUBNETID\ --region $REGION \ - --block-device-mappings "[{\"DeviceName\":\"/dev/sdb\",\"Ebs\":{\"VolumeSize\":$VOLUMESIZE,\"VolumeType\":\"$VOLUMETYPE\",\"DeleteOnTermination\":true}}]" \ + --block-device-mappings "[{\"DeviceName\":\"/dev/sdb\",\"Ebs\":{\"VolumeSize\":$VOLUMESIZE,\"VolumeType\":\"$VOLUMETYPE\",\"DeleteOnTermination\":$DELETEONTERM}}]" \ --tag-specifications $TAG_SPECS \ --query 'Instances[0].InstanceId' \ > $INSTANCE_ID_FILE diff --git a/bin/ec2-profile-database-development.sh b/bin/ec2-profile-database-development.sh index 98edf6f..289771c 100755 --- a/bin/ec2-profile-database-development.sh +++ b/bin/ec2-profile-database-development.sh @@ -1,6 +1,7 @@ #!/bin/bash -e DEVICENAME='/dev/sdb' +DELETEONTERM='true' IMAGEID='ami-7f43f307' INSTANCETYPE='t2.micro' KEYNAME='hackoregon-2018-database-dev-env' From ab1c7c66ed9b5c7f38faa7932ac6f7672e6c698c Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Wed, 4 Apr 2018 01:33:38 -0700 Subject: [PATCH 04/11] Improve comment around $EC2PROFILE loading --- bin/create-ec2-machine-database.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/create-ec2-machine-database.sh b/bin/create-ec2-machine-database.sh index d319999..5e04512 100755 --- a/bin/create-ec2-machine-database.sh +++ b/bin/create-ec2-machine-database.sh @@ -14,8 +14,9 @@ if [ "$#" -ne 2 ]; then exit 1 fi -# Source EC2 specs from a separate file -# The ./ec2-profile.sh should contain the following variables +EC2PROFILE=$2 # Source EC2 specs from a separate file +# The $EC2PROFILE must contain the following variables (values are merely just example) +# # DEVICENAME='/dev/sdb' # DELETEONTERM='true' # IMAGEID='ami-7f43f307' @@ -27,7 +28,6 @@ fi # VOLUMESIZE='8' # VOLUMETYPE='gp2' -EC2PROFILE=$2 INSTANCE_ID= INSTANCE_ID_FILE='./tmp_instance_id' INSTANCE_NAME=$1 From 776f89990e976821215a3d212252057947966523 Mon Sep 17 00:00:00 2001 From: Mike Lonergan Date: Sat, 7 Apr 2018 18:19:28 -0700 Subject: [PATCH 05/11] Definitely do *not* want to lose the databases The /data volume needs to be preserved if we happen to terminate the databases VM, at least until we have a better backup strategy --- bin/ec2-profile-database-development.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ec2-profile-database-development.sh b/bin/ec2-profile-database-development.sh index 289771c..1f0e954 100755 --- a/bin/ec2-profile-database-development.sh +++ b/bin/ec2-profile-database-development.sh @@ -1,7 +1,7 @@ #!/bin/bash -e DEVICENAME='/dev/sdb' -DELETEONTERM='true' +DELETEONTERM='false' IMAGEID='ami-7f43f307' INSTANCETYPE='t2.micro' KEYNAME='hackoregon-2018-database-dev-env' From 75b42aad9a05ab87af3c296fbc6d64830ddc809f Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Wed, 11 Apr 2018 23:54:29 -0700 Subject: [PATCH 06/11] Add CloudFormation template --- infrastructure/ec2-db.yaml | 108 +++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 infrastructure/ec2-db.yaml diff --git a/infrastructure/ec2-db.yaml b/infrastructure/ec2-db.yaml new file mode 100644 index 0000000..b24c05e --- /dev/null +++ b/infrastructure/ec2-db.yaml @@ -0,0 +1,108 @@ +# Itention: +# Create a ec2 instance that has read permission to the existing s3 instance(s) + +# USAGE: +# Run: +# aws cloudformation create-stack --stack-name --template-body file:///absolute/path/to/this-file.yaml --capabilities CAPABILITY_NAMED_IAM + +# TODO: +# - separate parameters into a file +# - write wrapper for the aws cli command + + +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: 'AWS CloudFormation to create a ec2 instance that has read permission to the existing s3 instance(s)' + +Parameters: + + InstanceType: + Description: Instance type used to build the machine(s) + Type: String + Default: t2.micro + + ImageId: + Description: AMI ID used to build the machine(s) + Type: String + Default: ami-7f43f307 + + AvailabilityZone: + Description: Avalaibility Zone to deploy within (different than region) + Type: String + Default: us-west-2a + + SubnetId: + Description: Subnet's ID to be located at + Type: String + Default: subnet-8794fddf + + SecurityGroupId: + Description: The Security Groups to use for the EC2 hosts + Type: String + Default: sg-28154957 + +Resources: + + DBInstance: + Type: AWS::EC2::Instance + Properties: + InstanceType: t2.micro # !Ref InstanceType + ImageId: ami-7f43f307 # !Ref ImageId + SecurityGroupIds: + - + sg-28154957 # !Ref SecurityGroupId + AvailabilityZone: us-west-2a # !Ref AvailabilityZone + SubnetId: subnet-8794fddf # !Ref SubnetId + IamInstanceProfile: + !Ref InstanceProfile + BlockDeviceMappings: + - + DeviceName: /dev/sdb # !Ref DeviceName + Ebs: + VolumeType: gp2 # !Ref VolumeType + VolumeSize: 8 # !Ref VolumeSize + DeleteOnTermination: True # False # !Ref DeleteOnTermination + KeyName: hackoregon-2018-database-dev-env # !Ref KeyName + Tags: + - + Key: Name + Value: DB # !Ref InstanceName + + Role: + Type: AWS::IAM::Role + Properties: + RoleName: db-role + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + #Policies: + # - !Ref RolePolicies + + RolePolicies: + Type: AWS::IAM::Policy + Properties: + PolicyName: ec2-read-s3-policy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - "s3:GetObject" + - "s3:ListBucket" + Resource: "*" + Roles: + - !Ref Role + + InstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: "/" + Roles: + - !Ref Role From 1da6dc8c49f05913c411b3fd4e192b4172e8841a Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Thu, 12 Apr 2018 00:03:10 -0700 Subject: [PATCH 07/11] Roll back 2 intentionally file changes Since the improvement/fix script has been merged yet, and I just accidentally branch out from a non-master branch, again :( --- bin/create-ec2-machine-database.sh | 12 +++++------- bin/ec2-profile-database-development.sh | 5 +---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/bin/create-ec2-machine-database.sh b/bin/create-ec2-machine-database.sh index 5e04512..fca5480 100755 --- a/bin/create-ec2-machine-database.sh +++ b/bin/create-ec2-machine-database.sh @@ -14,11 +14,9 @@ if [ "$#" -ne 2 ]; then exit 1 fi -EC2PROFILE=$2 # Source EC2 specs from a separate file -# The $EC2PROFILE must contain the following variables (values are merely just example) -# +# Source EC2 specs from a separate file +# The ./ec2-profile.sh should contain the following variables # DEVICENAME='/dev/sdb' -# DELETEONTERM='true' # IMAGEID='ami-7f43f307' # INSTANCETYPE='t2.micro' # KEYNAME='hackoregon-2018-database-dev-env' @@ -26,8 +24,8 @@ EC2PROFILE=$2 # Source EC2 specs from a separate file # SECURITYGROUPIDS='sg-28154957' # SUBNETID='subnet-8794fddf' # VOLUMESIZE='8' -# VOLUMETYPE='gp2' +EC2PROFILE=$2 INSTANCE_ID= INSTANCE_ID_FILE='./tmp_instance_id' INSTANCE_NAME=$1 @@ -44,7 +42,7 @@ aws ec2 run-instances \ --security-group-ids $SECURITYGROUPIDS \ --subnet-id $SUBNETID\ --region $REGION \ - --block-device-mappings "[{\"DeviceName\":\"/dev/sdb\",\"Ebs\":{\"VolumeSize\":$VOLUMESIZE,\"VolumeType\":\"$VOLUMETYPE\",\"DeleteOnTermination\":$DELETEONTERM}}]" \ + --block-device-mappings "[{\"DeviceName\":\"/dev/sdb\",\"Ebs\":{\"VolumeSize\":8,\"VolumeType\":\"gp2\",\"DeleteOnTermination\":true}}]" \ --tag-specifications $TAG_SPECS \ --query 'Instances[0].InstanceId' \ > $INSTANCE_ID_FILE @@ -60,4 +58,4 @@ aws ec2 describe-instances \ --instance-ids $INSTANCE_ID \ --query 'Reservations[0].Instances[0].PublicIpAddress' -rm -f $INSTANCE_ID_FILE +rm -f $INSTANCE_ID_FILE \ No newline at end of file diff --git a/bin/ec2-profile-database-development.sh b/bin/ec2-profile-database-development.sh index 1f0e954..4e03266 100755 --- a/bin/ec2-profile-database-development.sh +++ b/bin/ec2-profile-database-development.sh @@ -1,13 +1,10 @@ #!/bin/bash -e DEVICENAME='/dev/sdb' -DELETEONTERM='false' IMAGEID='ami-7f43f307' INSTANCETYPE='t2.micro' KEYNAME='hackoregon-2018-database-dev-env' REGION='us-west-2' SECURITYGROUPIDS='sg-28154957' SUBNETID='subnet-8794fddf' -VOLUMESIZE='8' -VOLUMETYPE='gp2' - +VOLUMESIZE='8' \ No newline at end of file From 83920cdc3b2562d190a50aa42dbd98fdca27f0db Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Thu, 12 Apr 2018 21:17:31 -0700 Subject: [PATCH 08/11] Update policy that limit access only to a certain s3 bucket --- infrastructure/ec2-db.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/ec2-db.yaml b/infrastructure/ec2-db.yaml index b24c05e..64442b6 100644 --- a/infrastructure/ec2-db.yaml +++ b/infrastructure/ec2-db.yaml @@ -96,7 +96,7 @@ Resources: Action: - "s3:GetObject" - "s3:ListBucket" - Resource: "*" + Resource: "arn:aws:s3:::hacko-data-archive/*" Roles: - !Ref Role From a3b76e2b587725fe5161bc720f2f1236033942ee Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Thu, 12 Apr 2018 21:21:51 -0700 Subject: [PATCH 09/11] Update: Persist Block Device after instance Termination --- infrastructure/ec2-db.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/ec2-db.yaml b/infrastructure/ec2-db.yaml index 64442b6..ff0a2e2 100644 --- a/infrastructure/ec2-db.yaml +++ b/infrastructure/ec2-db.yaml @@ -61,7 +61,7 @@ Resources: Ebs: VolumeType: gp2 # !Ref VolumeType VolumeSize: 8 # !Ref VolumeSize - DeleteOnTermination: True # False # !Ref DeleteOnTermination + DeleteOnTermination: False # True # !Ref DeleteOnTermination KeyName: hackoregon-2018-database-dev-env # !Ref KeyName Tags: - From d80952c0ab71e16184df6f96c41b44cdc7079fb5 Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Thu, 19 Apr 2018 17:51:32 -0700 Subject: [PATCH 10/11] Separate IAM Role away from EC2 instance launching --- infrastructure/ec2-db.yaml | 69 ++++++++++++++++---------------- infrastructure/ec2-iam-role.yaml | 44 ++++++++++++++++++++ 2 files changed, 78 insertions(+), 35 deletions(-) create mode 100644 infrastructure/ec2-iam-role.yaml diff --git a/infrastructure/ec2-db.yaml b/infrastructure/ec2-db.yaml index ff0a2e2..c1da06a 100644 --- a/infrastructure/ec2-db.yaml +++ b/infrastructure/ec2-db.yaml @@ -5,10 +5,8 @@ # Run: # aws cloudformation create-stack --stack-name --template-body file:///absolute/path/to/this-file.yaml --capabilities CAPABILITY_NAMED_IAM -# TODO: -# - separate parameters into a file -# - write wrapper for the aws cli command - +# PREREQUISITES: +# - The IAM role for this instance must already exist --- AWSTemplateFormatVersion: '2010-09-09' @@ -68,41 +66,42 @@ Resources: Key: Name Value: DB # !Ref InstanceName - Role: - Type: AWS::IAM::Role - Properties: - RoleName: db-role - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - ec2.amazonaws.com - Action: - - sts:AssumeRole - Path: "/" - #Policies: - # - !Ref RolePolicies + # Role: + # Type: AWS::IAM::Role + # Properties: + # RoleName: db-role + # AssumeRolePolicyDocument: + # Version: '2012-10-17' + # Statement: + # - Effect: Allow + # Principal: + # Service: + # - ec2.amazonaws.com + # Action: + # - sts:AssumeRole + # Path: "/" + # #Policies: + # # - !Ref RolePolicies - RolePolicies: - Type: AWS::IAM::Policy - Properties: - PolicyName: ec2-read-s3-policy - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - "s3:GetObject" - - "s3:ListBucket" - Resource: "arn:aws:s3:::hacko-data-archive/*" - Roles: - - !Ref Role + # RolePolicies: + # Type: AWS::IAM::Policy + # Properties: + # PolicyName: ec2-read-s3-policy + # PolicyDocument: + # Version: '2012-10-17' + # Statement: + # - Effect: Allow + # Action: + # - "s3:GetObject" + # - "s3:ListBucket" + # Resource: "arn:aws:s3:::hacko-data-archive/*" + # Roles: + # - !Ref Role InstanceProfile: Type: AWS::IAM::InstanceProfile Properties: Path: "/" Roles: - - !Ref Role + # - !Ref Role + - db-role diff --git a/infrastructure/ec2-iam-role.yaml b/infrastructure/ec2-iam-role.yaml new file mode 100644 index 0000000..a8034f3 --- /dev/null +++ b/infrastructure/ec2-iam-role.yaml @@ -0,0 +1,44 @@ +# Itention: +# Create an IAM role for EC2 instance + +# USAGE: +# Run: +# aws cloudformation create-stack --stack-name --template-body file:///absolute/path/to/this-file.yaml --capabilities CAPABILITY_NAMED_IAM + +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: 'AWS CloudFormation Template to create EC2 instances' + +Resources: + + Role: + Type: AWS::IAM::Role + Properties: + RoleName: db-role + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + #Policies: + # - !Ref RolePolicies + + RolePolicies: + Type: AWS::IAM::Policy + Properties: + PolicyName: ec2-read-s3-policy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - "s3:GetObject" + - "s3:ListBucket" + Resource: "arn:aws:s3:::hacko-data-archive/*" + Roles: + - !Ref Role From caf9dc8ffa3a7e4c6984eb534618017468044f12 Mon Sep 17 00:00:00 2001 From: Khuong Luu <6059347+khashf@users.noreply.github.com> Date: Thu, 19 Apr 2018 17:54:00 -0700 Subject: [PATCH 11/11] Rename directory --- {infrastructure => cloudformation}/ec2-db.yaml | 0 {infrastructure => cloudformation}/ec2-iam-role.yaml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {infrastructure => cloudformation}/ec2-db.yaml (100%) rename {infrastructure => cloudformation}/ec2-iam-role.yaml (100%) diff --git a/infrastructure/ec2-db.yaml b/cloudformation/ec2-db.yaml similarity index 100% rename from infrastructure/ec2-db.yaml rename to cloudformation/ec2-db.yaml diff --git a/infrastructure/ec2-iam-role.yaml b/cloudformation/ec2-iam-role.yaml similarity index 100% rename from infrastructure/ec2-iam-role.yaml rename to cloudformation/ec2-iam-role.yaml