Skip to content

Commit fa0048f

Browse files
authored
Actually use manylinux2014 docker image. (#194)
Actually use the manylinux2014 docker images, instead of just running on an EC2 instance that's similar enough that we happen to pass the wheel audit. Also, setup.py improvements: - accept `cmake3` as well as `cmake`. - slight cleanup of how AWS_LIBCRYPTO_INSTALL is used.
1 parent 1504ce3 commit fa0048f

9 files changed

+133
-53
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ lib64/
8383
parts/
8484
sdist/
8585
var/
86+
wheelhouse/
8687
wheels/
8788
*.egg-info/
8889
.installed.cfg

codebuild/cd/manylinux-x64-build.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ phases:
1919
- auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp36-cp36m-linux_x86_64.whl
2020
- /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
2121
- auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp37-cp37m-linux_x86_64.whl
22+
- /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
23+
- auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp38-cp38-linux_x86_64.whl
24+
- /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
25+
- auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp39-cp39-linux_x86_64.whl
2226
- cp -r wheelhouse ../dist
2327
- cp dist/*.tar.gz ../dist/
2428
post_build:

codebuild/cd/manylinux-x86-build.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ phases:
1919
- auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp36-cp36m-linux_i686.whl
2020
- /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
2121
- auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp37-cp37m-linux_i686.whl
22+
- /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
23+
- auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp38-cp38-linux_i686.whl
24+
- /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
25+
- auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp39-cp39-linux_i686.whl
2226
- cp -r wheelhouse ../dist
2327
post_build:
2428
commands:
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
#run build-wheels script in manylinux2014 docker image
3+
set -ex
4+
5+
DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt/manylinux2014-aarch64:latest
6+
7+
$(aws --region us-east-1 ecr get-login --no-include-email)
8+
9+
docker pull $DOCKER_IMAGE
10+
11+
docker run \
12+
--mount type=bind,source=`pwd`,target=/aws-crt-python \
13+
--workdir /aws-crt-python \
14+
--entrypoint /aws-crt-python/continuous-delivery/build-wheels-manylinux2014-aarch64.sh \
15+
$DOCKER_IMAGE
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
#assumes image based on manylinux2014 + extras (cmake3, libcrypto, etc)
3+
set -ex
4+
5+
/opt/python/cp38-cp38/bin/python ./continuous-delivery/update-version.py
6+
7+
export AWS_LIBCRYPTO_INSTALL=/opt/openssl
8+
9+
/opt/python/cp35-cp35m/bin/python setup.py sdist bdist_wheel
10+
auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp35*.whl
11+
12+
/opt/python/cp36-cp36m/bin/python setup.py sdist bdist_wheel
13+
auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp36*.whl
14+
15+
/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
16+
auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp37*.whl
17+
18+
/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
19+
auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp38*.whl
20+
21+
/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
22+
auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp39*.whl
23+
24+
rm dist/*.whl
25+
cp -rv wheelhouse/* dist/
26+
27+
#now you just need to run twine (that's in a different script)

continuous-delivery/build-wheels-manylinux2014-arm.sh

Lines changed: 0 additions & 25 deletions
This file was deleted.
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
#run build-wheels script in manylinux2014 docker image
3+
set -ex
4+
5+
DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt/manylinux2014-x64:latest
6+
7+
$(aws --region us-east-1 ecr get-login --no-include-email)
8+
9+
docker pull $DOCKER_IMAGE
10+
11+
docker run \
12+
--mount type=bind,source=`pwd`,target=/aws-crt-python \
13+
--workdir /aws-crt-python \
14+
--entrypoint /aws-crt-python/continuous-delivery/build-wheels-manylinux2014-x86_64.sh \
15+
$DOCKER_IMAGE

continuous-delivery/build-wheels-manylinux2014-x86_64.sh

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,26 @@
11
#!/bin/bash
2-
#before running this, you'll need cmake3 and a compiler. These python versions are just
3-
#using the default python installers from python.org. Each version needs updated pip, wheel, and setuptools
2+
#assumes image based on manylinux2014 + extras (cmake3, libcrypto, etc)
43
set -ex
54

6-
/usr/local/bin/python3.8 ./continuous-delivery/update-version.py
5+
/opt/python/cp38-cp38/bin/python ./continuous-delivery/update-version.py
76

87
export AWS_LIBCRYPTO_INSTALL=/opt/openssl
98

10-
/usr/local/bin/python3.5m setup.py sdist bdist_wheel
9+
/opt/python/cp35-cp35m/bin/python setup.py sdist bdist_wheel
1110
auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp35*.whl
1211

13-
/usr/local/bin/python3.6m setup.py sdist bdist_wheel
12+
/opt/python/cp36-cp36m/bin/python setup.py sdist bdist_wheel
1413
auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp36*.whl
1514

16-
/usr/local/bin/python3.7m setup.py sdist bdist_wheel
15+
/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
1716
auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp37*.whl
1817

19-
/usr/local/bin/python3.8 setup.py sdist bdist_wheel
18+
/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
2019
auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp38*.whl
2120

21+
/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
22+
auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp39*.whl
23+
2224
rm dist/*.whl
2325
cp -rv wheelhouse/* dist/
2426

setup.py

Lines changed: 58 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import platform
1010
import setuptools
1111
import setuptools.command.build_ext
12+
import shutil
1213
import subprocess
1314
import sys
1415

@@ -73,17 +74,54 @@ def determine_generator_args():
7374
return []
7475

7576

76-
cmake_found = False
77+
cmake_found = None
7778

7879

79-
def check_cmake_installed():
80+
def get_cmake_path():
8081
global cmake_found
81-
if not cmake_found:
82-
try:
83-
subprocess.check_call(['cmake', '--version'])
84-
cmake_found = True
85-
except Exception:
86-
raise Exception("'cmake' not found. cmake must be installed to build from source.")
82+
if cmake_found:
83+
return cmake_found
84+
85+
for cmake_alias in ['cmake3', 'cmake']:
86+
cmake_found = shutil.which(cmake_alias)
87+
if cmake_found:
88+
return cmake_found
89+
90+
raise Exception("CMake must be installed to build from source.")
91+
92+
93+
def get_libcrypto_static_library(libcrypto_dir):
94+
lib_path = os.path.join(libcrypto_dir, 'lib64', 'libcrypt')
95+
if is_64bit() and os.path.exists(lib_path):
96+
return lib_path
97+
98+
lib_path = os.path.join(libcrypto_dir, 'lib32', 'libcrypto.a')
99+
if is_32bit() and os.path.exists(lib_path):
100+
return lib_path
101+
102+
lib_path = os.path.join(libcrypto_dir, 'lib', 'libcrypto.a')
103+
if os.path.exists(lib_path):
104+
return lib_path
105+
106+
raise Exception('Bad AWS_LIBCRYPTO_INSTALL, file not found: ' + lib_path)
107+
108+
109+
def get_libcrypto_paths():
110+
# return None if not using libcrypto
111+
if sys.platform == 'darwin' or sys.platform == 'win32':
112+
return None
113+
libcrypto_dir = os.environ.get('AWS_LIBCRYPTO_INSTALL')
114+
if not libcrypto_dir:
115+
return None
116+
117+
# find include dir
118+
include_dir = os.path.join(libcrypto_dir, 'include')
119+
expected_file = os.path.join(include_dir, 'openssl', 'crypto.h')
120+
if not os.path.exists(expected_file):
121+
raise Exception('Bad AWS_LIBCRYPTO_INSTALL, file not found: ' + expected_file)
122+
123+
static_library = get_libcrypto_static_library(libcrypto_dir)
124+
return {'include_dir': include_dir, 'static_library': static_library}
87125

88126

89127
class AwsLib:
@@ -110,14 +148,10 @@ def __init__(self, name, extra_cmake_args=[]):
110148
DEP_BUILD_DIR = os.path.join(PROJECT_DIR, 'build', 'deps')
111149
DEP_INSTALL_PATH = os.environ.get('AWS_C_INSTALL', os.path.join(DEP_BUILD_DIR, 'install'))
112150

113-
AWS_LIBCRYPTO_INSTALL = None
114-
if sys.platform != 'darwin' and sys.platform != 'win32':
115-
AWS_LIBCRYPTO_INSTALL = os.environ.get('AWS_LIBCRYPTO_INSTALL', os.path.join(DEP_BUILD_DIR, 'libcrypto'))
116-
117151

118152
class awscrt_build_ext(setuptools.command.build_ext.build_ext):
119-
def _build_dependency(self, aws_lib):
120-
check_cmake_installed()
153+
def _build_dependency(self, aws_lib, libcrypto_paths):
154+
cmake = get_cmake_path()
121155

122156
prev_cwd = os.getcwd() # restore cwd at end of function
123157
lib_source_dir = os.path.join(PROJECT_DIR, 'crt', aws_lib.name)
@@ -137,7 +171,7 @@ def _build_dependency(self, aws_lib):
137171
os.chdir(lib_build_dir)
138172

139173
# cmake configure
140-
cmake_args = ['cmake']
174+
cmake_args = [cmake]
141175
cmake_args.extend(determine_generator_args())
142176
cmake_args.extend(determine_cross_compile_args())
143177
cmake_args.extend([
@@ -151,10 +185,9 @@ def _build_dependency(self, aws_lib):
151185
cmake_args.append('-DCMAKE_INCLUDE_PATH="{}"'.format(';'.join(self.include_dirs)))
152186
if self.library_dirs:
153187
cmake_args.append('-DCMAKE_LIBRARY_PATH="{}"'.format(';'.join(self.library_dirs)))
154-
if AWS_LIBCRYPTO_INSTALL:
155-
cmake_args.append('-DLibCrypto_INCLUDE_DIR={}/include'.format(AWS_LIBCRYPTO_INSTALL))
156-
cmake_args.append('-DLibCrypto_STATIC_LIBRARY={}/lib/libcrypto.a'.format(AWS_LIBCRYPTO_INSTALL))
157-
self.library_dirs.append('{}/lib'.format(AWS_LIBCRYPTO_INSTALL))
188+
if libcrypto_paths:
189+
cmake_args.append('-DLibCrypto_INCLUDE_DIR={}'.format(libcrypto_paths['include_dir']))
190+
cmake_args.append('-DLibCrypto_STATIC_LIBRARY={}'.format(libcrypto_paths['static_library']))
158191
cmake_args.extend(aws_lib.extra_cmake_args)
159192
cmake_args.append(lib_source_dir)
160193

@@ -163,7 +196,7 @@ def _build_dependency(self, aws_lib):
163196

164197
# cmake build/install
165198
build_cmd = [
166-
'cmake',
199+
cmake,
167200
'--build', './',
168201
'--config', build_type,
169202
'--target', 'install',
@@ -174,9 +207,13 @@ def _build_dependency(self, aws_lib):
174207
os.chdir(prev_cwd)
175208

176209
def run(self):
210+
libcrypto_paths = get_libcrypto_paths()
211+
if libcrypto_paths:
212+
self.library_dirs.append(os.path.dirname(libcrypto_paths['static_library']))
213+
177214
# build dependencies
178215
for lib in AWS_LIBS:
179-
self._build_dependency(lib)
216+
self._build_dependency(lib, libcrypto_paths)
180217

181218
# update paths so awscrt_ext can access dependencies
182219
self.include_dirs.append(os.path.join(DEP_INSTALL_PATH, 'include'))

0 commit comments

Comments
 (0)