From 9338d69b3afefac0b589685a7bd3cb45059f16d5 Mon Sep 17 00:00:00 2001 From: Akhil Date: Thu, 15 Oct 2020 11:07:41 +0100 Subject: [PATCH 1/5] Add all changes --- .env.sample | 15 +++++++ .gitignore | 6 +++ README.md | 81 +++++++++++++++++++++++++++++++++++ dockerfiles/mlflow/dockerfile | 0 entrypoint | 0 local.yml | 60 ++++++++++++++++++++++++++ production.yml | 30 +++++++++++++ start | 13 ++++++ 8 files changed, 205 insertions(+) create mode 100644 .env.sample create mode 100644 dockerfiles/mlflow/dockerfile create mode 100644 entrypoint create mode 100644 local.yml create mode 100644 production.yml create mode 100644 start diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..cef00c1 --- /dev/null +++ b/.env.sample @@ -0,0 +1,15 @@ +# connection settings +DOMAIN_NAME=localhost +SERVICE_NAME=mlflow +SERVICE_PORT=5000 +# tell the user where they are logging into +REALM=testsite +# required for letsencrypt certificate email updates +EMAIL_ADDRESS= + +# database settings +POSTGRES_USER= +POSTGRES_PASSWORD= +POSTGRES_HOST= +POSTGRES_PORT= +POSTGRES_DB= diff --git a/.gitignore b/.gitignore index b6e4761..ae6bc32 100644 --- a/.gitignore +++ b/.gitignore @@ -125,5 +125,11 @@ venv.bak/ .dmypy.json dmypy.json +# Project specific +.htpasswd +config/traefik.yml + # Pyre type checker .pyre/ +.history/ +.devcontainer/ diff --git a/README.md b/README.md index b3df2ce..a412e8f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,83 @@ # traefik-basic-auth + Traefik in Docker with HTTP basic auth configured. +MLflow Production Docker + +A production ready1 docker-compose deployment of MLflow using Traefik with HTTP Basic Auth. + +1 This project is provided as is, and without a warranty of any kind, see [LICENSE](../blob/master/LICENSE) for full details. + +## Setup + +Install requirements: + +- Apache Utils (provides `htpasswd` command) +- GNU gettext (provides `envsubst` command) + +```sh +sudo apt-get install -y apache2-utils gettext +``` + +Create a new `.htpasswd` file with a single user (using Bcrypt hashing): + +```sh +htpasswd -cB config/.htpasswd +``` + +Add additional users with: + +```sh +htpasswd -B config/.htpasswd +``` + +Create a `.env` in the root directory with the following variables completed: + +```sh +# connection settings +DOMAIN_NAME=localhost +SERVICE_NAME=mlflow +SERVICE_PORT=5000 +# tell the user where they are logging into +REALM=testsite +# required for letsencrypt certificate email updates +EMAIL_ADDRESS=example@example.com +# Database settings +POSTGRES_USER= +POSTGRES_PASSWORD= +POSTGRES_HOST= +POSTGRES_PORT= +POSTGRES_DB= +# Mlflow settings +MLFLOW_ARTIFACT_URI=s3://bucket-name/ +# If using bentoml for deployment +``` + +Finally write a `traefik.yml` config file using the `.env` variables: + +```sh +./write_template.sh +``` + +## Running + +Start all containers: + +```sh +docker-compose up -d --build +``` + +MLflow is accessible on port 443 over HTTPS (port 80 redirects to 443). Use the username and password you created above to log in. + +## Database + +The metadata database for mlflow is configured to use an RDS database in a private subnet (no internet access). In situations where we need to directly access the database, we need to set up ssh tunneling. In order to do this run the following: + +```sh +ssh -i "private.pem" -f -N -L 5433::5432 @ -v +``` + +This create ssh tunneling allowing you to connect to tracking database with the host `localhost` port `5433` and the expected other credentials (1password). + +NOTE: The "private.pem" key here is the one used to ssh into the server. + +For further information, look [here](https://aws.amazon.com/premiumsupport/knowledge-center/rds-connect-using-bastion-host-linux/`). diff --git a/dockerfiles/mlflow/dockerfile b/dockerfiles/mlflow/dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/entrypoint b/entrypoint new file mode 100644 index 0000000..e69de29 diff --git a/local.yml b/local.yml new file mode 100644 index 0000000..e874370 --- /dev/null +++ b/local.yml @@ -0,0 +1,60 @@ + +--- +version: "3" + +volumes: + traefik_acme_data: {} + local_postgres_data: {} + +services: + traefik: + build: + context: . + dockerfile: ./dockerfiles/traefik/Dockerfile + image: mlflow_production_traefik + restart: always + volumes: + - traefik_acme_data:/etc/traefik/acme/ + - ./config/.htpasswd:/etc/traefik/.htpasswd:ro + - ./config/traefik.yml:/etc/traefik/traefik.yml:ro + ports: + - "0.0.0.0:80:8080" + - "0.0.0.0:443:8443" + + + mlflow: + build: + context: . + dockerfile: ./dockerfiles/mlflow/Dockerfile + image: mlflow_production_server + restart: always + command: /start + ports: + - "5000:5000" + depends_on: + - postgres + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_PORT=${POSTGRES_PORT} + - POSTGRES_DB=${POSTGRES_DB} + - MLFLOW_ARTIFACT_URI=${MLFLOW_ARTIFACT_URI} + # - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + # - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + # - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} + + postgres: + image: postgres:11.8 + restart: always + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_PORT=${POSTGRES_PORT} + - POSTGRES_DB=${POSTGRES_DB} + - MLFLOW_ARTIFACT_URI=${MLFLOW_ARTIFACT_URI} + ports: + - "5432:5432" + volumes: + - local_postgres_data:/var/lib/postgresql/data \ No newline at end of file diff --git a/production.yml b/production.yml new file mode 100644 index 0000000..3fa4612 --- /dev/null +++ b/production.yml @@ -0,0 +1,30 @@ +--- +version: "3" + +volumes: + traefik_acme_data: {} + +services: + traefik: + build: + context: . + dockerfile: ./dockerfiles/traefik/Dockerfile + image: mlflow_production_traefik + restart: always + volumes: + - traefik_acme_data:/etc/traefik/acme/ + - ./config/.htpasswd:/etc/traefik/.htpasswd:ro + - ./config/traefik.yml:/etc/traefik/traefik.yml:ro + ports: + - "0.0.0.0:80:8080" + - "0.0.0.0:443:8443" + + mlflow: + build: + context: . + dockerfile: ./dockerfiles/mlflow/Dockerfile + image: mlflow_production_server + restart: always + command: /start + env_file: + - .env \ No newline at end of file diff --git a/start b/start new file mode 100644 index 0000000..7fc0b11 --- /dev/null +++ b/start @@ -0,0 +1,13 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +DB_URI=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} +mlflow db upgrade $DB_URI + +mlflow server \ + --backend-store-uri ${DB_URI} \ + --default-artifact-root ${MLFLOW_ARTIFACT_URI} \ + --host 0.0.0.0 \ No newline at end of file From fa5149b74e3def332a581e7ab09c3bc1a88032a4 Mon Sep 17 00:00:00 2001 From: Akhil Date: Thu, 15 Oct 2020 22:42:20 +0100 Subject: [PATCH 2/5] Remove duplicates from gitignore --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index e62ef28..2e439c4 100644 --- a/.gitignore +++ b/.gitignore @@ -129,10 +129,6 @@ venv.bak/ .dmypy.json dmypy.json -# Project specific -.htpasswd -config/traefik.yml - # Pyre type checker .pyre/ .history/ From 4fe02b2ddc996cfceeb1df903aefbca00be58997 Mon Sep 17 00:00:00 2001 From: Akhil Date: Thu, 15 Oct 2020 22:50:12 +0100 Subject: [PATCH 3/5] Update entrypoint --- entrypoint | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/entrypoint b/entrypoint index 903b188..1d6cf63 100644 --- a/entrypoint +++ b/entrypoint @@ -4,28 +4,11 @@ set -euo pipefail export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" -postgres_ready() { -python << END -import sys -import psycopg2 -import os -try: - psycopg2.connect( - dbname="${POSTGRES_DB}", - user="${POSTGRES_USER}", - password="${POSTGRES_PASSWORD}", - host="${POSTGRES_HOST}", - port="${POSTGRES_PORT}", - ) -except psycopg2.OperationalError: - sys.exit(-1) -sys.exit(0) -END -} -until postgres_ready; do - >&2 echo 'Waiting for PostgreSQL to become available...' - sleep 1 +until nc -z postgres:5432 &>/dev/null; do + >&2 echo 'Sleeping till postgres is ready...' + sleep 1 done >&2 echo 'PostgreSQL is available' -exec "$@" \ No newline at end of file + +exec "$@" \ No newline at end of file From b7c19b7d2cb66bec6e4d6791b5b86c0ce81da0cb Mon Sep 17 00:00:00 2001 From: Akhil Date: Thu, 15 Oct 2020 22:55:41 +0100 Subject: [PATCH 4/5] use env file --- local.yml | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/local.yml b/local.yml index e874370..22ea78f 100644 --- a/local.yml +++ b/local.yml @@ -33,27 +33,14 @@ services: - "5000:5000" depends_on: - postgres - environment: - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - POSTGRES_HOST=${POSTGRES_HOST} - - POSTGRES_PORT=${POSTGRES_PORT} - - POSTGRES_DB=${POSTGRES_DB} - - MLFLOW_ARTIFACT_URI=${MLFLOW_ARTIFACT_URI} - # - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - # - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - # - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} + env_file: + - .env postgres: image: postgres:11.8 restart: always - environment: - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - POSTGRES_HOST=${POSTGRES_HOST} - - POSTGRES_PORT=${POSTGRES_PORT} - - POSTGRES_DB=${POSTGRES_DB} - - MLFLOW_ARTIFACT_URI=${MLFLOW_ARTIFACT_URI} + env_file: + - .env ports: - "5432:5432" volumes: From 60ec930c2acff6925e86b6fb6cf69379ac874d06 Mon Sep 17 00:00:00 2001 From: Akhil Date: Fri, 16 Oct 2020 00:39:07 +0100 Subject: [PATCH 5/5] Add requirements.txt --- entrypoint | 3 ++- requirements.txt | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 requirements.txt diff --git a/entrypoint b/entrypoint index 1d6cf63..3bb34d2 100644 --- a/entrypoint +++ b/entrypoint @@ -4,7 +4,8 @@ set -euo pipefail export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" -until nc -z postgres:5432 &>/dev/null; do + +until nc -z postgres:5432 > /dev/null 2>&1; do >&2 echo 'Sleeping till postgres is ready...' sleep 1 done diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cf68ff6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,52 @@ +alembic==1.4.1 +azure-core==1.8.1 +azure-storage-blob==12.5.0 +boto3==1.15.3 +botocore==1.18.3 +certifi==2020.6.20 +cffi==1.14.3 +chardet==3.0.4 +click==7.1.2 +cloudpickle==1.6.0 +cryptography==3.1.1 +databricks-cli==0.11.0 +docker==4.3.1 +entrypoints==0.3 +Flask==1.1.2 +gitdb==4.0.5 +GitPython==3.1.8 +gorilla==0.3.0 +gunicorn==20.0.4 +idna==2.10 +isodate==0.6.0 +itsdangerous==1.1.0 +Jinja2==2.11.2 +jmespath==0.10.0 +Mako==1.1.3 +MarkupSafe==1.1.1 +mlflow==1.11.0 +msrest==0.6.19 +numpy==1.19.2 +oauthlib==3.1.0 +pandas==1.1.2 +prometheus-client==0.8.0 +prometheus-flask-exporter==0.18.0 +protobuf==3.13.0 +psycopg2-binary==2.8.6 +pycparser==2.20 +python-dateutil==2.8.1 +python-editor==1.0.4 +pytz==2020.1 +PyYAML==5.3.1 +querystring-parser==1.2.4 +requests==2.24.0 +requests-oauthlib==1.3.0 +s3transfer==0.3.3 +six==1.15.0 +smmap==3.0.4 +SQLAlchemy==1.3.13 +sqlparse==0.3.1 +tabulate==0.8.7 +urllib3==1.25.10 +websocket-client==0.57.0 +Werkzeug==1.0.1 \ No newline at end of file