Skip to content

Commit 8e0fa94

Browse files
dlpzxdlpzx
authored
[sdlf-cicd] extract create_repository into new file, add API-support gitlab (#543)
* [sdlf-cicd] extract create_repository into new file, add API-support gitlab * [sdlf-cicd] custom resource with GitLab API call for initial repositories
1 parent d12f75f commit 8e0fa94

File tree

6 files changed

+382
-257
lines changed

6 files changed

+382
-257
lines changed

‎docs/constructs/cicd.md‎

Lines changed: 2 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -91,50 +91,15 @@ Rnabled by setting `pEnableLambdaLayerBuilder` to `true` when deploying `templat
9191

9292
### GitLab
9393

94-
- Create a dedicated user on GitLab. Currently the user must be named: `sdlf`.
95-
- Create an access token with the `sdlf` user. The token name must be named `aws`. Permissions must be `api` and `write_repository`.
96-
- Create [CodeConnections](https://docs.aws.amazon.com/codepipeline/latest/userguide/connections-gitlab-managed.html) for the self-managed GitLab instance
94+
The creation of GitLab repositories will be performed through the GitLab API.
9795

9896
Populate:
9997

10098
- `/SDLF/GitLab/Url` :: secure-string :: GitLab URL **with** trailing `/`
10199
- `/SDLF/GitLab/AccessToken` :: secure-string :: User access token
100+
- `/SDLF/GitLab/NamespaceId` :: secure-string :: User/Enterprise namespace ID
102101
- `/SDLF/GitLab/CodeConnection` :: string :: CodeConnections ARN
103102

104-
Create CloudFormation role:
105-
106-
```
107-
{
108-
"Version": "2012-10-17",
109-
"Statement": [
110-
{
111-
"Effect": "Allow",
112-
"Principal": {
113-
"Service": "resources.cloudformation.amazonaws.com"
114-
},
115-
"Action": "sts:AssumeRole",
116-
"Condition": {
117-
"StringEquals": {
118-
"aws:SourceAccount": "111111111111"
119-
}
120-
}
121-
]
122-
}
123-
```
124-
125-
Enable `GitLab::Projects::Project` third-party resource type in CloudFormation Registry.
126-
127-
Add configuration (use of ssm-secure is mandatory):
128-
129-
```
130-
{
131-
"GitLabAccess": {
132-
"AccessToken": "{{resolve:ssm-secure:/SDLF/GitLab/AccessToken:1}}",
133-
"Url": "{{resolve:ssm-secure:/SDLF/GitLab/Url:1}}"
134-
}
135-
}
136-
```
137-
138103
## Interface
139104

140105
There is no external interface.

‎sdlf-cicd/lambda/domain-cicd/src/lambda_function.py‎

Lines changed: 9 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import boto3
99
from botocore.client import Config
1010
from botocore.exceptions import ClientError
11+
from repository_manager import create_repositories
1112

1213
logger = logging.getLogger()
1314
logger.setLevel(logging.INFO)
@@ -117,52 +118,6 @@ def delete_domain_team_role_stack(cloudformation, team):
117118
return (stack_name, "stack_delete_complete")
118119

119120

120-
def create_team_repository_cicd_stack(domain, team_name, template_body_url, cloudformation_role):
121-
response = {}
122-
cloudformation_waiter_type = None
123-
stack_name = f"sdlf-cicd-teams-{domain}-{team_name}-repository"
124-
stack_parameters = [
125-
{
126-
"ParameterKey": "pDomain",
127-
"ParameterValue": domain,
128-
"UsePreviousValue": False,
129-
},
130-
{
131-
"ParameterKey": "pTeamName",
132-
"ParameterValue": team_name,
133-
"UsePreviousValue": False,
134-
},
135-
]
136-
stack_arguments = dict(
137-
StackName=stack_name,
138-
TemplateURL=template_body_url,
139-
Parameters=stack_parameters,
140-
Capabilities=[
141-
"CAPABILITY_AUTO_EXPAND",
142-
],
143-
RoleARN=cloudformation_role,
144-
Tags=[
145-
{"Key": "Framework", "Value": "sdlf"},
146-
],
147-
)
148-
149-
try:
150-
response = cloudformation.create_stack(**stack_arguments)
151-
cloudformation_waiter_type = "stack_create_complete"
152-
except cloudformation.exceptions.AlreadyExistsException:
153-
try:
154-
response = cloudformation.update_stack(**stack_arguments)
155-
cloudformation_waiter_type = "stack_update_complete"
156-
except ClientError as err:
157-
if "No updates are to be performed" in err.response["Error"]["Message"]:
158-
pass
159-
else:
160-
raise err
161-
162-
logger.info("RESPONSE: %s", response)
163-
return (stack_name, cloudformation_waiter_type)
164-
165-
166121
def create_team_pipeline_cicd_stack(
167122
domain,
168123
environment,
@@ -467,51 +422,14 @@ def lambda_handler(event, context):
467422
###### CREATE STACKS FOR TEAMS ######
468423
for domain, domain_details in domains.items():
469424
# create team repository if it hasn't been created already
470-
cloudformation_waiters = {
471-
"stack_create_complete": [],
472-
"stack_update_complete": [],
473-
}
474-
for team in domain_details["teams"]:
475-
stack_details = create_team_repository_cicd_stack(
476-
domain,
477-
team,
478-
template_cicd_team_repository_url,
479-
cloudformation_role,
480-
)
481-
if stack_details[1]:
482-
cloudformation_waiters[stack_details[1]].append(stack_details[0])
483-
cloudformation_create_waiter = cloudformation.get_waiter("stack_create_complete")
484-
cloudformation_update_waiter = cloudformation.get_waiter("stack_update_complete")
485-
for stack in cloudformation_waiters["stack_create_complete"]:
486-
cloudformation_create_waiter.wait(StackName=stack, WaiterConfig={"Delay": 30, "MaxAttempts": 10})
487-
for stack in cloudformation_waiters["stack_update_complete"]:
488-
cloudformation_update_waiter.wait(StackName=stack, WaiterConfig={"Delay": 30, "MaxAttempts": 10})
489-
490-
if git_platform == "CodeCommit":
491-
for team in domain_details["teams"]:
492-
repository_name = f"{main_repository_prefix}{domain}-{team}"
493-
env_branches = ["dev", "test"]
494-
for env_branch in env_branches:
495-
try:
496-
codecommit.create_branch(
497-
repositoryName=repository_name,
498-
branchName=env_branch,
499-
commitId=codecommit.get_branch(
500-
repositoryName=repository_name,
501-
branchName="main",
502-
)["branch"]["commitId"],
503-
)
504-
logger.info(
505-
"Branch %s created in repository %s",
506-
env_branch,
507-
repository_name,
508-
)
509-
except codecommit.exceptions.BranchNameExistsException:
510-
logger.info(
511-
"Branch %s already created in repository %s",
512-
env_branch,
513-
repository_name,
514-
)
425+
create_repositories(
426+
git_platform,
427+
domain_details,
428+
domain,
429+
template_cicd_team_repository_url,
430+
cloudformation_role,
431+
main_repository_prefix,
432+
)
515433

516434
# and create a CICD stack per team that will be used to deploy team resources in the child account
517435
cloudformation_waiters = {

‎sdlf-cicd/lambda/domain-cicd/src/repository_manager.py‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
import json
2+
import logging
3+
import os
4+
import ssl
5+
from urllib.request import HTTPError, Request, URLError, urlopen
6+
7+
import boto3
8+
from botocore.exceptions import ClientError
9+
10+
logger = logging.getLogger()
11+
12+
ssm_endpoint_url = "https://ssm." + os.getenv("AWS_REGION") + ".amazonaws.com"
13+
ssm = boto3.client("ssm", endpoint_url=ssm_endpoint_url)
14+
codecommit_endpoint_url = "https://codecommit." + os.getenv("AWS_REGION") + ".amazonaws.com"
15+
codecommit = boto3.client("codecommit", endpoint_url=codecommit_endpoint_url)
16+
cloudformation_endpoint_url = "https://cloudformation." + os.getenv("AWS_REGION") + ".amazonaws.com"
17+
cloudformation = boto3.client("cloudformation", endpoint_url=cloudformation_endpoint_url)
18+
19+
20+
def _create_team_repository_cicd_stack(domain, team_name, template_body_url, cloudformation_role):
21+
response = {}
22+
cloudformation_waiter_type = None
23+
stack_name = f"sdlf-cicd-teams-{domain}-{team_name}-repository"
24+
stack_parameters = [
25+
{
26+
"ParameterKey": "pDomain",
27+
"ParameterValue": domain,
28+
"UsePreviousValue": False,
29+
},
30+
{
31+
"ParameterKey": "pTeamName",
32+
"ParameterValue": team_name,
33+
"UsePreviousValue": False,
34+
},
35+
]
36+
stack_arguments = dict(
37+
StackName=stack_name,
38+
TemplateURL=template_body_url,
39+
Parameters=stack_parameters,
40+
Capabilities=[
41+
"CAPABILITY_AUTO_EXPAND",
42+
],
43+
RoleARN=cloudformation_role,
44+
Tags=[
45+
{"Key": "Framework", "Value": "sdlf"},
46+
],
47+
)
48+
49+
try:
50+
response = cloudformation.create_stack(**stack_arguments)
51+
cloudformation_waiter_type = "stack_create_complete"
52+
except cloudformation.exceptions.AlreadyExistsException:
53+
try:
54+
response = cloudformation.update_stack(**stack_arguments)
55+
cloudformation_waiter_type = "stack_update_complete"
56+
except ClientError as err:
57+
if "No updates are to be performed" in err.response["Error"]["Message"]:
58+
pass
59+
else:
60+
raise err
61+
62+
logger.info("RESPONSE: %s", response)
63+
return (stack_name, cloudformation_waiter_type)
64+
65+
66+
def _create_codecommit_repositories(
67+
domain_details, domain, template_cicd_team_repository_url, cloudformation_role, main_repository_prefix
68+
):
69+
"""Create CodeCommit repositories and branches for teams"""
70+
cloudformation_waiters = {
71+
"stack_create_complete": [],
72+
"stack_update_complete": [],
73+
}
74+
for team in domain_details["teams"]:
75+
stack_details = _create_team_repository_cicd_stack(
76+
domain,
77+
team,
78+
template_cicd_team_repository_url,
79+
cloudformation_role,
80+
)
81+
if stack_details[1]:
82+
cloudformation_waiters[stack_details[1]].append(stack_details[0])
83+
84+
cloudformation_create_waiter = cloudformation.get_waiter("stack_create_complete")
85+
cloudformation_update_waiter = cloudformation.get_waiter("stack_update_complete")
86+
for stack in cloudformation_waiters["stack_create_complete"]:
87+
cloudformation_create_waiter.wait(StackName=stack, WaiterConfig={"Delay": 30, "MaxAttempts": 10})
88+
for stack in cloudformation_waiters["stack_update_complete"]:
89+
cloudformation_update_waiter.wait(StackName=stack, WaiterConfig={"Delay": 30, "MaxAttempts": 10})
90+
91+
# Create branches for each team repository
92+
for team in domain_details["teams"]:
93+
repository_name = f"{main_repository_prefix}{domain}-{team}"
94+
env_branches = ["dev", "test"]
95+
for env_branch in env_branches:
96+
try:
97+
codecommit.create_branch(
98+
repositoryName=repository_name,
99+
branchName=env_branch,
100+
commitId=codecommit.get_branch(
101+
repositoryName=repository_name,
102+
branchName="main",
103+
)["branch"]["commitId"],
104+
)
105+
logger.info(
106+
"Branch %s created in repository %s",
107+
env_branch,
108+
repository_name,
109+
)
110+
except codecommit.exceptions.BranchNameExistsException:
111+
logger.info(
112+
"Branch %s already created in repository %s",
113+
env_branch,
114+
repository_name,
115+
)
116+
117+
118+
def _create_gitlab_repositories(domain_details, domain, template_cicd_team_repository_url, cloudformation_role):
119+
"""Create GitLab repositories for teams"""
120+
for team in domain_details["teams"]:
121+
# Create GitLab repository via API
122+
# !Sub ${pMainRepositoriesPrefix}${pDomain}-${pTeamName}
123+
repository = f"sdlf-main-{domain}-{team}"
124+
gitlab_url = ssm.get_parameter(Name="/SDLF/GitLab/Url", WithDecryption=True)["Parameter"]["Value"]
125+
gitlab_accesstoken = ssm.get_parameter(Name="/SDLF/GitLab/AccessToken", WithDecryption=True)["Parameter"][
126+
"Value"
127+
]
128+
namespace_id = ssm.get_parameter(Name="/SDLF/GitLab/NamespaceId", WithDecryption=True)["Parameter"]["Value"]
129+
130+
url = f"{gitlab_url}api/v4/projects/"
131+
headers = {"Content-Type": "application/json", "PRIVATE-TOKEN": gitlab_accesstoken}
132+
data = {
133+
"name": repository,
134+
"description": repository,
135+
"path": repository,
136+
"namespace_id": namespace_id,
137+
"initialize_with_readme": "false",
138+
}
139+
json_data = json.dumps(data).encode("utf-8")
140+
req = Request(url, data=json_data, headers=headers, method="POST")
141+
unverified_context = ssl._create_unverified_context()
142+
try:
143+
with urlopen(req, context=unverified_context) as response:
144+
response_body = response.read().decode("utf-8")
145+
logger.info(response_body)
146+
except HTTPError as e:
147+
logger.warning(
148+
f"HTTP error occurred: {e.code} {e.reason}. Most likely the repository {repository} already exists"
149+
)
150+
except URLError as e:
151+
logger.error(f"URL error occurred: {e.reason}")
152+
153+
# Create CloudFormation stacks for GitLab repositories
154+
cloudformation_waiters = {
155+
"stack_create_complete": [],
156+
"stack_update_complete": [],
157+
}
158+
for team in domain_details["teams"]:
159+
stack_details = _create_team_repository_cicd_stack(
160+
domain,
161+
team,
162+
template_cicd_team_repository_url,
163+
cloudformation_role,
164+
)
165+
if stack_details[1]:
166+
cloudformation_waiters[stack_details[1]].append(stack_details[0])
167+
168+
cloudformation_create_waiter = cloudformation.get_waiter("stack_create_complete")
169+
cloudformation_update_waiter = cloudformation.get_waiter("stack_update_complete")
170+
for stack in cloudformation_waiters["stack_create_complete"]:
171+
cloudformation_create_waiter.wait(StackName=stack, WaiterConfig={"Delay": 30, "MaxAttempts": 10})
172+
for stack in cloudformation_waiters["stack_update_complete"]:
173+
cloudformation_update_waiter.wait(StackName=stack, WaiterConfig={"Delay": 30, "MaxAttempts": 10})
174+
175+
176+
def _create_github_repositories(domain_details, domain, template_cicd_team_repository_url, cloudformation_role):
177+
"""Create GitHub repositories for teams"""
178+
# GitHub repositories are created via CloudFormation template
179+
cloudformation_waiters = {
180+
"stack_create_complete": [],
181+
"stack_update_complete": [],
182+
}
183+
for team in domain_details["teams"]:
184+
stack_details = _create_team_repository_cicd_stack(
185+
domain,
186+
team,
187+
template_cicd_team_repository_url,
188+
cloudformation_role,
189+
)
190+
if stack_details[1]:
191+
cloudformation_waiters[stack_details[1]].append(stack_details[0])
192+
193+
cloudformation_create_waiter = cloudformation.get_waiter("stack_create_complete")
194+
cloudformation_update_waiter = cloudformation.get_waiter("stack_update_complete")
195+
for stack in cloudformation_waiters["stack_create_complete"]:
196+
cloudformation_create_waiter.wait(StackName=stack, WaiterConfig={"Delay": 30, "MaxAttempts": 10})
197+
for stack in cloudformation_waiters["stack_update_complete"]:
198+
cloudformation_update_waiter.wait(StackName=stack, WaiterConfig={"Delay": 30, "MaxAttempts": 10})
199+
200+
201+
def create_repositories(
202+
git_platform,
203+
domain_details,
204+
domain,
205+
template_cicd_team_repository_url,
206+
cloudformation_role,
207+
main_repository_prefix=None,
208+
):
209+
"""Create team repositories based on git platform"""
210+
if git_platform == "CodeCommit":
211+
_create_codecommit_repositories(
212+
domain_details, domain, template_cicd_team_repository_url, cloudformation_role, main_repository_prefix
213+
)
214+
elif git_platform == "GitHub":
215+
_create_github_repositories(domain_details, domain, template_cicd_team_repository_url, cloudformation_role)
216+
elif git_platform == "GitLab":
217+
_create_gitlab_repositories(domain_details, domain, template_cicd_team_repository_url, cloudformation_role)
218+
else:
219+
raise logging.exception("Git provider {} is not supported".format(git_platform))

0 commit comments

Comments
 (0)