Support environment overrides in entrypoint.sh

Author: brendan-kellam

Dockerfile

@@ -173,7 +173,6 @@ ENV DATA_DIR=/data
 ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
 ENV DATABASE_DATA_DIR=$DATA_CACHE_DIR/db
 ENV REDIS_DATA_DIR=$DATA_CACHE_DIR/redis
-ENV REDIS_URL="redis://localhost:6379"
 ENV SRC_TENANT_ENFORCEMENT_MODE=strict
 ENV SOURCEBOT_PUBLIC_KEY_PATH=/app/public.pem
 

entrypoint.sh

@@ -1,28 +1,57 @@
 #!/bin/sh
-set -e
 
-# Check if DATABASE_URL is not set
-if [ -z "$DATABASE_URL" ]; then
-    # Check if the individual database variables are set and construct the URL
-    if [ -n "$DATABASE_HOST" ] && [ -n "$DATABASE_USERNAME" ] && [ -n "$DATABASE_PASSWORD" ]  && [ -n "$DATABASE_NAME" ]; then
-        DATABASE_URL="postgresql://${DATABASE_USERNAME}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}"
+# Exit immediately if a command fails
+set -e
+# Disable auto-exporting of variables
+set +a
+
+# If a CONFIG_PATH is set, resolve the environment overrides from the config file.
+# The overrides will be written into variables scopped to the current shell. This is
+# required in case one of the variables used in this entrypoint is overriden (e.g.,
+# DATABASE_URL, REDIS_URL, etc.)
+if [ -n "$CONFIG_PATH" ]; then
+    echo -e "\e[34m[Info] Resolving environment overrides from $CONFIG_PATH...\e[0m"
+
+    set +e # Disable exist on error so we can capture EXIT_CODE
+    OVERRIDES_OUTPUT=$(SKIP_ENV_VALIDATION=1 yarn tool:resolve-env-overrides 2>&1)
+    EXIT_CODE=$?
+    set -e # Re-enable exit on error
+
+    if [ $EXIT_CODE -eq 0 ]; then
+        eval "$OVERRIDES_OUTPUT"
+    else
+        echo -e "\e[31m[Error] Failed to resolve environment overrides.\e[0m"
+        echo "$OVERRIDES_OUTPUT"
+        exit 1
+    fi
+fi
 
-        if [ -n "$DATABASE_ARGS" ]; then
-            DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS"
-        fi
+# Descontruct the database URL from the individual variables if DATABASE_URL is not set
+if [ -z "$DATABASE_URL" ] && [ -n "$DATABASE_HOST" ] && [ -n "$DATABASE_USERNAME" ] && [ -n "$DATABASE_PASSWORD" ]  && [ -n "$DATABASE_NAME" ]; then
+    DATABASE_URL="postgresql://${DATABASE_USERNAME}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}"
 
-        export DATABASE_URL
-    else
-        # Otherwise, fallback to a default value
-        DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot"
-        export DATABASE_URL
+    if [ -n "$DATABASE_ARGS" ]; then
+        DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS"
     fi
 fi
 
-if [ "$DATABASE_URL" = "postgresql://postgres@localhost:5432/sourcebot" ]; then
-    DATABASE_EMBEDDED="true"
+if [ -z "$DATABASE_URL" ]; then
+    echo -e "\e[34m[Info] DATABASE_URL is not set. Using embeded database.\e[0m"
+    export DATABASE_EMBEDDED="true"
+    export DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot"
+else
+    export DATABASE_EMBEDDED="false"
+fi
+
+if [ -z "$REDIS_URL" ]; then
+    echo -e "\e[34m[Info] REDIS_URL is not set. Using embeded redis.\e[0m"
+    export REDIS_EMBEDDED="true"
+    export REDIS_URL="redis://localhost:6379"
+else
+    export REDIS_EMBEDDED="false"
 fi
 
+
 echo -e "\e[34m[Info] Sourcebot version: $NEXT_PUBLIC_SOURCEBOT_VERSION\e[0m"
 
 # If we don't have a PostHog key, then we need to disable telemetry.
@@ -59,7 +88,7 @@ if [ "$DATABASE_EMBEDDED" = "true" ] && [ ! -d "$DATABASE_DATA_DIR" ]; then
 fi
 
 # Create the redis data directory if it doesn't exist
-if [ ! -d "$REDIS_DATA_DIR" ]; then
+if [ "$REDIS_EMBEDDED" = "true" ] && [ ! -d "$REDIS_DATA_DIR" ]; then
     mkdir -p $REDIS_DATA_DIR
 fi
 
@@ -149,7 +178,6 @@ fi
 
 echo "{\"version\": \"$NEXT_PUBLIC_SOURCEBOT_VERSION\", \"install_id\": \"$SOURCEBOT_INSTALL_ID\"}" > "$FIRST_RUN_FILE"
 
-
 # Start the database and wait for it to be ready before starting any other service
 if [ "$DATABASE_EMBEDDED" = "true" ]; then
     su postgres -c "postgres -D $DATABASE_DATA_DIR" &
@@ -171,7 +199,7 @@ fi
 
 # Run a Database migration
 echo -e "\e[34m[Info] Running database migration...\e[0m"
-yarn workspace @sourcebot/db prisma:migrate:prod
+DATABASE_URL="$DATABASE_URL" yarn workspace @sourcebot/db prisma:migrate:prod
 
 # Create the log directory
 mkdir -p /var/log/sourcebot

packages/backend/src/index.ts

@@ -2,7 +2,7 @@ import "./instrument.js";
 
 import { PrismaClient } from "@sourcebot/db";
 import { createLogger } from "@sourcebot/shared";
-import { env, getConfigSettings, hasEntitlement } from '@sourcebot/shared';
+import { env, getConfigSettings, hasEntitlement, getDBConnectionString } from '@sourcebot/shared';
 import { existsSync } from 'fs';
 import { mkdir } from 'fs/promises';
 import { Redis } from 'ioredis';
@@ -31,7 +31,7 @@ if (!existsSync(indexPath)) {
 const prisma = new PrismaClient({
     datasources: {
         db: {
-            url: env.DATABASE_URL,
+            url: getDBConnectionString(),
         },
     },
 });

packages/shared/package.json

@@ -6,7 +6,8 @@
     "scripts": {
         "build": "tsc",
         "build:watch": "tsc-watch --preserveWatchOutput",
-        "postinstall": "yarn build"
+        "postinstall": "yarn build",
+        "tool:resolve-env-overrides": "tsx tools/resolveEnvOverrides.ts"
     },
     "dependencies": {
         "@google-cloud/secret-manager": "^6.1.1",
@@ -26,6 +27,7 @@
         "@types/micromatch": "^4.0.9",
         "@types/node": "^22.7.5",
         "tsc-watch": "6.2.1",
+        "tsx": "^4.19.1",
         "typescript": "^5.7.3"
     },
     "exports": {

packages/shared/src/db.ts

@@ -0,0 +1,15 @@
+import { env } from "./env.server.js";
+
+export const getDBConnectionString = (): string | undefined => {
+    if (env.DATABASE_URL) {
+        return env.DATABASE_URL;
+    }
+    else if (env.DATABASE_HOST && env.DATABASE_USERNAME && env.DATABASE_PASSWORD && env.DATABASE_NAME) {
+        let databaseUrl = `postgresql://${env.DATABASE_USERNAME}:${env.DATABASE_PASSWORD}@${env.DATABASE_HOST}/${env.DATABASE_NAME}`;
+        if (env.DATABASE_ARGS) {
+            databaseUrl += `?${env.DATABASE_ARGS}`;
+        }
+
+        return databaseUrl;
+    }
+}

packages/shared/src/env.server.ts

@@ -1,9 +1,9 @@
 import { createEnv } from "@t3-oss/env-core";
 import { z } from "zod";
-import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
-import { getTokenFromConfig } from "./crypto.js";
 import { loadConfig } from "./utils.js";
 import { tenancyModeSchema } from "./types.js";
+import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
+import { getTokenFromConfig } from "./crypto.js";
 
 // Booleans are specified as 'true' or 'false' strings.
 const booleanSchema = z.enum(["true", "false"]);
@@ -13,16 +13,14 @@ const booleanSchema = z.enum(["true", "false"]);
 // @see: https://zod.dev/?id=coercion-for-primitives
 const numberSchema = z.coerce.number();
 
-
-const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotConfig): Promise<Record<string, string>> => {
+export const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotConfig): Promise<Record<string, string>> => {
     if (!config.environmentOverrides) {
         return {};
     }
 
     const resolved: Record<string, string> = {};
 
     const start = performance.now();
-    console.debug('resolving environment variable overrides');
 
     for (const [key, override] of Object.entries(config.environmentOverrides)) {
         switch (override.type) {
@@ -122,7 +120,16 @@ export const env = createEnv({
         CONFIG_MAX_REPOS_NO_TOKEN: numberSchema.default(Number.MAX_SAFE_INTEGER),
         NODE_ENV: z.enum(["development", "test", "production"]),
         SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default('false'),
-        DATABASE_URL: z.string().url(),
+
+        // Database variables
+        // Either DATABASE_URL or DATABASE_HOST, DATABASE_USERNAME, DATABASE_PASSWORD, and DATABASE_NAME must be set.
+        // @see: shared/src/db.ts for more details.
+        DATABASE_URL: z.string().url().optional(),
+        DATABASE_HOST: z.string().optional(),
+        DATABASE_USERNAME: z.string().optional(),
+        DATABASE_PASSWORD: z.string().optional(),
+        DATABASE_NAME: z.string().optional(),
+        DATABASE_ARGS: z.string().optional(),
 
         SOURCEBOT_TENANCY_MODE: tenancyModeSchema.default("single"),
         CONFIG_PATH: z.string(),

packages/shared/src/index.server.ts

@@ -27,7 +27,8 @@ export {
 } from "./utils.js";
 export * from "./constants.js";
 export {
-    env
+    env,
+    resolveEnvironmentVariableOverridesFromConfig,
 } from "./env.server.js";
 export {
     createLogger,
@@ -42,4 +43,7 @@ export {
     hashSecret,
     generateApiKey,
     verifySignature,
-} from "./crypto.js";
+} from "./crypto.js";
+export {
+    getDBConnectionString,
+} from "./db.js";

packages/shared/tools/resolveEnvOverrides.ts

@@ -0,0 +1,30 @@
+// The following script loads the config.json file and resolves any environment variable overrides.
+// It then writes then to stdout in the format of `KEY="VALUE"`.
+// This is used by entrypoint.sh to set them as variables.
+(async () => {
+    if (!process.env.CONFIG_PATH) {
+        console.error('CONFIG_PATH is not set');
+        process.exit(1);
+    }
+
+    // Silence all console logs so we don't pollute stdout.
+    const originalConsoleLog = console.log;
+    console.log = () => {};
+    console.debug = () => {};
+    console.info = () => {};
+    console.warn = () => {};
+    // console.error = () => {}; // Keep errors
+
+    const { loadConfig } = await import("../src/utils.js");
+    const { resolveEnvironmentVariableOverridesFromConfig } = await import("../src/env.server.js");
+
+    const config = await loadConfig(process.env.CONFIG_PATH);
+    const overrides = await resolveEnvironmentVariableOverridesFromConfig(config);
+
+    for (const [key, value] of Object.entries(overrides)) {
+        const escapedValue = value.replace(/"/g, '\\"');
+        originalConsoleLog(`${key}="${escapedValue}"`);
+    }
+
+    process.exit(0);
+})();

packages/web/src/prisma.ts

@@ -1,11 +1,13 @@
 import 'server-only';
-import { env } from "@sourcebot/shared";
+import { env, getDBConnectionString } from "@sourcebot/shared";
 import { Prisma, PrismaClient } from "@sourcebot/db";
 import { hasEntitlement } from "@sourcebot/shared";
 
 // @see: https://authjs.dev/getting-started/adapters/prisma
 const globalForPrisma = globalThis as unknown as { prisma: PrismaClient }
 
+const dbConnectionString = getDBConnectionString();
+
 // @NOTE: In almost all cases, the userScopedPrismaClientExtension should be used
 // (since actions & queries are scoped to a particular user). There are some exceptions
 // (e.g., in initialize.ts).
@@ -14,15 +16,15 @@ const globalForPrisma = globalThis as unknown as { prisma: PrismaClient }
 // all of the actions & queries to use the userScopedPrismaClientExtension to avoid
 // accidental misuse.
 export const prisma = globalForPrisma.prisma || new PrismaClient({
-    // @note: even though DATABASE_URL is of type string, we need to check if it's defined
-    // because this code will be executed at build time, and env.DATABASE_URL will be undefined.
-    ...(env.DATABASE_URL ? {
+    // @note: this code is evaluated at build time, and will throw exceptions if these env vars are not set.
+    // Here we explicitly check if the DATABASE_URL or the individual database variables are set, and only
+    ...(dbConnectionString !== undefined ? {
         datasources: {
             db: {
-                url: env.DATABASE_URL,
+                url: dbConnectionString,
             },
         }
-    } : {})
+    }: {}),
 })
 if (env.NODE_ENV !== "production") globalForPrisma.prisma = prisma
 

supervisord.conf

@@ -36,7 +36,7 @@ redirect_stderr=true
 [program:redis]
 command=redis-server --dir %(ENV_REDIS_DATA_DIR)s
 priority=10
-autostart=true
+autostart=%(ENV_REDIS_EMBEDDED)s
 autorestart=true
 startretries=3
 stdout_logfile=/dev/fd/1