Skip to content

Commit 6d9cef7

Browse files
RishabhJain2018RishabhJain2018
authored
Worker: Add worker container with python version 3.12 (#4636)
* Update python version in worker to 3.13 * Minor Fixes * Update to Python 3.12 version * Upgrade the python version to 3.12 * Sorted dependencies in worker py3_12 * Minor Fixes in the production setup * Fix tests and dev build
1 parent 0327ff0 commit 6d9cef7

File tree

19 files changed

+229
-106
lines changed

19 files changed

+229
-106
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ jobs:
3939
script:
4040
# Build steps
4141
- if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || travis_terminate 1; fi
42-
- docker-compose --profile worker --profile statsd build || travis_terminate 1;
42+
- docker-compose --profile worker_py3_7 --profile worker_py3_8 --profile worker_py3_12 --profile statsd build || travis_terminate 1;
4343

4444
# Frontend Tests
4545
- docker-compose run nodejs bash -c "gulp dev && karma start --single-run && gulp staging" || travis_terminate 1;

‎apps/challenges/aws_utils.py‎

Lines changed: 81 additions & 75 deletions
Large diffs are not rendered by default.

‎docker-compose-production.yml‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,19 +37,27 @@ services:
3737
awslogs-group: celery_production
3838
awslogs-create-group: "true"
3939

40-
worker:
41-
image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-worker:${COMMIT_ID}
40+
worker_py3_7:
41+
image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-worker-py3.7:${COMMIT_ID}
4242
build:
4343
context: ./
44-
dockerfile: docker/prod/worker/Dockerfile
44+
dockerfile: docker/prod/worker_py3_7/Dockerfile
4545
env_file:
4646
- docker/prod/docker_production.env
4747

48-
worker_py3.8:
48+
worker_py3_8:
4949
image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-worker-py3.8:${COMMIT_ID}
5050
build:
5151
context: ./
52-
dockerfile: docker/prod/worker_py3.8/Dockerfile
52+
dockerfile: docker/prod/worker_py3_8/Dockerfile
53+
env_file:
54+
- docker/prod/docker_production.env
55+
56+
worker_py3_12:
57+
image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-worker-py3.12:${COMMIT_ID}
58+
build:
59+
context: ./
60+
dockerfile: docker/prod/worker_py3_12/Dockerfile
5361
env_file:
5462
- docker/prod/docker_production.env
5563

@@ -78,7 +86,7 @@ services:
7886
build:
7987
context: ./
8088
dockerfile: docker/prod/worker/Dockerfile
81-
command: ["python", "-m", "scripts.workers.remote_submission_worker"]
89+
command: [ "python", "-m", "scripts.workers.remote_submission_worker" ]
8290
env_file:
8391
- docker/prod/docker_remote_worker.env
8492

@@ -166,4 +174,3 @@ services:
166174
- '--web.external-url=http://localhost:9093/alert_manager'
167175
ports:
168176
- '9093:9093'
169-

‎docker-compose-staging.yml‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,19 +37,27 @@ services:
3737
awslogs-group: celery_staging
3838
awslogs-create-group: 'true'
3939

40-
worker:
41-
image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-staging-worker:${COMMIT_ID}
40+
worker_py3_7:
41+
image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-staging-worker-py3.7:${COMMIT_ID}
4242
build:
4343
context: ./
44-
dockerfile: docker/prod/worker/Dockerfile
44+
dockerfile: docker/prod/worker_py3_7/Dockerfile
4545
env_file:
4646
- docker/prod/docker_staging.env
4747

48-
worker_py3.8:
48+
worker_py3_8:
4949
image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-staging-worker-py3.8:${COMMIT_ID}
5050
build:
5151
context: ./
52-
dockerfile: docker/prod/worker_py3.8/Dockerfile
52+
dockerfile: docker/prod/worker_py3_8/Dockerfile
53+
env_file:
54+
- docker/prod/docker_staging.env
55+
56+
worker_py3_12:
57+
image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-staging-worker-py3.12:${COMMIT_ID}
58+
build:
59+
context: ./
60+
dockerfile: docker/prod/worker_py3_12/Dockerfile
5361
env_file:
5462
- docker/prod/docker_staging.env
5563

@@ -78,7 +86,7 @@ services:
7886
build:
7987
context: ./
8088
dockerfile: docker/prod/worker/Dockerfile
81-
command: ['python', '-m', 'scripts.workers.remote_submission_worker']
89+
command: [ 'python', '-m', 'scripts.workers.remote_submission_worker' ]
8290
env_file:
8391
- docker/prod/docker_remote_worker.env
8492

@@ -105,7 +113,7 @@ services:
105113
- '--web.external-url=http://localhost:9090/prometheus'
106114
ports:
107115
- '9090:9090'
108-
116+
109117
grafana:
110118
hostname: grafana
111119
image: grafana/grafana:latest
@@ -129,29 +137,29 @@ services:
129137
- '9125:9125/udp'
130138
- '9125:9125/tcp'
131139
- '9102:9102'
132-
140+
133141
node_exporter:
134142
hostname: node_exporter
135143
image: prom/node-exporter
136144
ports:
137145
- '9100:9100'
138-
146+
139147
nginx-ingress:
140148
image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-staging-nginx-ingress:${COMMIT_ID}
141149
build:
142150
context: ./
143151
dockerfile: docker/prod/nginx-ingress/Dockerfile
144152
args:
145153
MONITORING_ENV: staging
146-
depends_on:
154+
depends_on:
147155
- prometheus
148156
- grafana
149157
- statsd-exporter
150158
- alert-manager
151159
ports:
152160
- '80:80'
153161
- '443:443'
154-
162+
155163
alert-manager:
156164
hostname: alert_manager
157165
image: prom/alertmanager:v0.24.0

‎docker-compose.yml‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,31 +27,44 @@ services:
2727
volumes:
2828
- .:/code
2929

30-
worker:
30+
worker_py3_7:
3131
env_file:
3232
- docker/dev/docker.env
3333
build:
3434
context: ./
35-
dockerfile: docker/dev/worker/Dockerfile
35+
dockerfile: docker/dev/worker_py3_7/Dockerfile
3636
depends_on:
3737
- django
3838
volumes:
3939
- .:/code
4040
profiles:
41-
- worker # Only built when this profile is specified
41+
- worker_py3_7 # Only built when this profile is specified
4242

4343
worker_py3_8:
4444
env_file:
4545
- docker/dev/docker.env
4646
build:
4747
context: ./
48-
dockerfile: docker/dev/worker_py3.8/Dockerfile
48+
dockerfile: docker/dev/worker_py3_8/Dockerfile
4949
depends_on:
5050
- django
5151
volumes:
5252
- .:/code
5353
profiles:
54-
- worker # Only built when this profile is specified
54+
- worker_py3_8 # Only built when this profile is specified
55+
56+
worker_py3_12:
57+
env_file:
58+
- docker/dev/docker.env
59+
build:
60+
context: ./
61+
dockerfile: docker/dev/worker_py3_12/Dockerfile
62+
depends_on:
63+
- django
64+
volumes:
65+
- .:/code
66+
profiles:
67+
- worker_py3_12 # Only built when this profile is specified
5568

5669
nodejs:
5770
hostname: nodejs

‎docker/dev/django/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ RUN mkdir /code
88
WORKDIR /code
99

1010
ADD requirements/* /code/
11+
RUN pip install --no-cache-dir --no-compile --progress-bar off -r django.txt
1112
RUN pip install --no-cache-dir --no-compile --progress-bar off -r dev.txt
1213

1314
ADD . /code/

‎docker/dev/worker_py3_12/Dockerfile‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
FROM python:3.12.3
2+
3+
ENV PYTHONUNBUFFERED 1
4+
5+
RUN apt-get update && \
6+
apt-get install -q -y \
7+
default-jre \
8+
default-jdk \
9+
libpq-dev \
10+
libcurl4-openssl-dev \
11+
build-essential \
12+
python3-dev \
13+
curl && \
14+
rm -rf /var/lib/apt/lists/*
15+
16+
17+
RUN mkdir /code
18+
WORKDIR /code
19+
20+
ADD requirements/* /code/
21+
22+
RUN pip install -r dev.txt
23+
RUN pip install -r worker_py3_12.txt
24+
25+
ADD . /code
26+
27+
CMD ["./docker/wait-for-it.sh", "django:8000", "--", "python", "-m", "scripts.workers.submission_worker"]

‎docker/dev/worker/Dockerfile‎ renamed to ‎docker/dev/worker_py3_7/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,9 @@ WORKDIR /code
1212
ADD requirements/* /code/
1313

1414
RUN pip install -U cffi service_identity cython==0.29 setuptools==57.5.0
15+
RUN pip install -r django.txt
1516
RUN pip install -r dev.txt
16-
RUN pip install -r worker.txt
17+
RUN pip install -r worker_py3_7.txt
1718

1819
ADD . /code
1920

‎docker/dev/worker_py3.8/Dockerfile‎ renamed to ‎docker/dev/worker_py3_8/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ RUN mkdir /code
66
WORKDIR /code
77

88
ADD requirements/* /code/
9+
RUN pip install -r django.txt
910
RUN pip install -r dev.txt
1011

1112
RUN apt-get update && \
@@ -15,7 +16,7 @@ RUN apt-get update && \
1516
rm -rf /var/lib/apt/lists/*
1617

1718
RUN pip install -U cffi service_identity cython==0.29 setuptools==57.5.0
18-
RUN pip install -r worker.txt
19+
RUN pip install -r worker_py3_8.txt
1920

2021
ADD . /code
2122

‎docker/prod/django/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ RUN mkdir /code
66
WORKDIR /code
77
ADD requirements/* /code/
88
RUN pip install -r prod.txt
9+
RUN pip install -r django.txt
910

1011
ADD . /code
1112

0 commit comments

Comments
 (0)