move web to use shared env package

Author: brendan-kellam

.env.development

@@ -84,7 +84,7 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
 # NEXT_PUBLIC_SOURCEBOT_VERSION=
 
 # CONFIG_MAX_REPOS_NO_TOKEN=
-# NODE_ENV=
+NODE_ENV=development
 # SOURCEBOT_TENANCY_MODE=single
 
 # NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT=

packages/backend/package.json

@@ -32,7 +32,6 @@
         "@sourcebot/db": "workspace:*",
         "@sourcebot/schemas": "workspace:*",
         "@sourcebot/shared": "workspace:*",
-        "@t3-oss/env-core": "^0.12.0",
         "@types/express": "^5.0.0",
         "argparse": "^2.0.1",
         "azure-devops-node-api": "^15.1.1",
@@ -52,6 +51,6 @@
         "posthog-node": "^4.2.1",
         "prom-client": "^15.1.3",
         "simple-git": "^3.27.0",
-        "zod": "^3.24.3"
+        "zod": "^3.25.74"
     }
 }

packages/backend/src/instrument.ts

@@ -1,6 +1,6 @@
 import * as Sentry from "@sentry/node";
 import { createLogger } from "@sourcebot/shared";
-import { env } from "@sourcebot/shared";
+import { env } from "@sourcebot/shared/client";
 
 const logger = createLogger('instrument');
 

packages/backend/src/posthog.ts

@@ -1,12 +1,13 @@
+import { env as clientEnv } from "@sourcebot/shared/client";
 import { env } from "@sourcebot/shared";
 import { PostHog } from 'posthog-node';
 import { PosthogEvent, PosthogEventMap } from './posthogEvents.js';
 
 let posthog: PostHog | undefined = undefined;
 
-if (env.NEXT_PUBLIC_POSTHOG_PAPIK) {
+if (clientEnv.NEXT_PUBLIC_POSTHOG_PAPIK) {
     posthog = new PostHog(
-        env.NEXT_PUBLIC_POSTHOG_PAPIK,
+        clientEnv.NEXT_PUBLIC_POSTHOG_PAPIK,
         {
             host: "https://us.i.posthog.com",
         }
@@ -23,7 +24,7 @@ export function captureEvent<E extends PosthogEvent>(event: E, properties: Posth
         event: event,
         properties: {
             ...properties,
-            sourcebot_version: env.NEXT_PUBLIC_SOURCEBOT_VERSION,
+            sourcebot_version: clientEnv.NEXT_PUBLIC_SOURCEBOT_VERSION,
         },
     });
 }

packages/shared/package.json

@@ -20,7 +20,7 @@
         "strip-json-comments": "^5.0.1",
         "triple-beam": "^1.4.1",
         "winston": "^3.15.0",
-        "zod": "^3.24.3"
+        "zod": "^3.25.74"
     },
     "devDependencies": {
         "@types/micromatch": "^4.0.9",

packages/shared/src/crypto.ts

@@ -1,6 +1,6 @@
 import crypto from 'crypto';
 import fs from 'fs';
-import { env } from './env.js';
+import { env } from './env.server.js';
 import { Token } from '@sourcebot/schemas/v3/shared.type';
 import { SecretManagerServiceClient } from "@google-cloud/secret-manager";
 

packages/shared/src/entitlements.ts

@@ -1,7 +1,8 @@
 import { base64Decode } from "./utils.js";
 import { z } from "zod";
 import { createLogger } from "./logger.js";
-import { env } from "./env.js";
+import { env } from "./env.server.js";
+import { env as clientEnv } from "./env.client.js";
 import { SOURCEBOT_SUPPORT_EMAIL, SOURCEBOT_UNLIMITED_SEATS } from "./constants.js";
 import { verifySignature } from "./crypto.js";
 
@@ -89,8 +90,8 @@ export const getLicenseKey = (): LicenseKeyPayload | null => {
 }
 
 export const getPlan = (): Plan => {
-    if (env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT) {
-        if (env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo") {
+    if (clientEnv.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT) {
+        if (clientEnv.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo") {
             return "cloud:demo";
         }
 

packages/shared/src/env.client.ts

@@ -0,0 +1,27 @@
+import { createEnv } from "@t3-oss/env-core";
+import { z } from "zod";
+import { SOURCEBOT_CLOUD_ENVIRONMENT } from "./constants.js";
+
+export const env = createEnv({
+    clientPrefix: "NEXT_PUBLIC_",
+    client: {
+        NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(),
+        NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"),
+        NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(),
+        NEXT_PUBLIC_SENTRY_BACKEND_DSN: z.string().optional(),
+        NEXT_PUBLIC_SENTRY_ENVIRONMENT: z.string().optional(),
+        NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: z.string().optional(),
+        NEXT_PUBLIC_LANGFUSE_BASE_URL: z.string().optional()
+    },
+    runtimeEnvStrict: {
+        NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT,
+        NEXT_PUBLIC_SOURCEBOT_VERSION: process.env.NEXT_PUBLIC_SOURCEBOT_VERSION,
+        NEXT_PUBLIC_POSTHOG_PAPIK: process.env.NEXT_PUBLIC_POSTHOG_PAPIK,
+        NEXT_PUBLIC_SENTRY_BACKEND_DSN: process.env.NEXT_PUBLIC_SENTRY_BACKEND_DSN,
+        NEXT_PUBLIC_SENTRY_ENVIRONMENT: process.env.NEXT_PUBLIC_SENTRY_ENVIRONMENT,
+        NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY,
+        NEXT_PUBLIC_LANGFUSE_BASE_URL: process.env.NEXT_PUBLIC_LANGFUSE_BASE_URL,
+    },
+    emptyStringAsUndefined: true,
+    skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
+});

packages/web/src/env.mjs packages/shared/src/env.server.tspackages/web/src/env.mjs renamed to packages/shared/src/env.server.ts

@@ -1,24 +1,69 @@
-import { createEnv } from "@t3-oss/env-nextjs";
+import { createEnv } from "@t3-oss/env-core";
 import { z } from "zod";
-import { SOURCEBOT_CLOUD_ENVIRONMENT } from "@sourcebot/shared/client";
+import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
+import { getTokenFromConfig } from "./crypto.js";
+import { loadConfig } from "./utils.js";
+import { tenancyModeSchema } from "./types.js";
 
 // Booleans are specified as 'true' or 'false' strings.
 const booleanSchema = z.enum(["true", "false"]);
-export const tenancyModeSchema = z.enum(["multi", "single"]);
 
 // Numbers are treated as strings in .env files.
 // coerce helps us convert them to numbers.
 // @see: https://zod.dev/?id=coercion-for-primitives
 const numberSchema = z.coerce.number();
 
+
+const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotConfig): Promise<Record<string, string>> => {
+    if (!config.environmentOverrides) {
+        return {};
+    }
+
+    const resolved: Record<string, string> = {};
+    console.debug('resolving environment variable overrides');
+
+    for (const [key, override] of Object.entries(config.environmentOverrides)) {
+        switch (override.type) {
+            case 'token':
+                resolved[key] = await getTokenFromConfig(override.value);
+                break;
+            case 'boolean':
+                resolved[key] = override.value ? 'true' : 'false';
+                break;
+            case 'number':
+                resolved[key] = override.value.toString();
+                break;
+            case 'string':
+                resolved[key] = override.value;
+                break;
+        }
+    }
+
+    return resolved;
+}
+
+// Merge process.env with environment variables resolved from config.json
+const runtimeEnv = await (async () => {
+    const configPath = process.env.CONFIG_PATH;
+    if (!configPath) {
+        return process.env;
+    }
+
+    const config = await loadConfig(configPath);
+    const overrides = await resolveEnvironmentVariableOverridesFromConfig(config);
+    return {
+        ...process.env,
+        ...overrides,
+    }
+})();
+
 export const env = createEnv({
     server: {
         // Zoekt
         ZOEKT_WEBSERVER_URL: z.string().url().default("http://localhost:6070"),
 
         // Auth
         FORCE_ENABLE_ANONYMOUS_ACCESS: booleanSchema.default('false'),
-
         AUTH_SECRET: z.string(),
         AUTH_URL: z.string().url(),
         AUTH_CREDENTIALS_LOGIN_ENABLED: booleanSchema.default('true'),
@@ -75,7 +120,7 @@ export const env = createEnv({
         DATABASE_URL: z.string().url(),
 
         SOURCEBOT_TENANCY_MODE: tenancyModeSchema.default("single"),
-        CONFIG_PATH: z.string().optional(),
+        CONFIG_PATH: z.string(),
 
         // Misc UI flags
         SECURITY_CARD_ENABLED: booleanSchema.default('false'),
@@ -137,31 +182,30 @@ export const env = createEnv({
         // @NOTE: Take care to update actions.ts when changing the name of this.
         EXPERIMENT_SELF_SERVE_REPO_INDEXING_GITHUB_TOKEN: z.string().optional(),
         EXPERIMENT_EE_PERMISSION_SYNC_ENABLED: booleanSchema.default('false'),
-    },
-    // @NOTE: Please make sure of the following:
-    // - Make sure you destructure all client variables in
-    //   the `experimental__runtimeEnv` block below.
-    // - Update the Dockerfile to pass these variables as build-args.
-    client: {
-        // PostHog
-        NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(),
 
-        // Misc
-        NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default('unknown'),
+        SOURCEBOT_ENCRYPTION_KEY: z.string(),
+        SOURCEBOT_INSTALL_ID: z.string().default("unknown"),
 
-        NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(),
+        FALLBACK_GITHUB_CLOUD_TOKEN: z.string().optional(),
+        FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(),
+        FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(),
 
-        NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: z.string().optional(),
-        NEXT_PUBLIC_LANGFUSE_BASE_URL: z.string().optional()
-    },
-    // For Next.js >= 13.4.4, you only need to destructure client variables:
-    experimental__runtimeEnv: {
-        NEXT_PUBLIC_POSTHOG_PAPIK: process.env.NEXT_PUBLIC_POSTHOG_PAPIK,
-        NEXT_PUBLIC_SOURCEBOT_VERSION: process.env.NEXT_PUBLIC_SOURCEBOT_VERSION,
-        NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT,
-        NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY,
-        NEXT_PUBLIC_LANGFUSE_BASE_URL: process.env.NEXT_PUBLIC_LANGFUSE_BASE_URL,
+        REDIS_URL: z.string().url().default("redis://localhost:6379"),
+        REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0),
+        REDIS_REMOVE_ON_FAIL: numberSchema.default(100),
+
+        DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'),
+
+        CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000),
+        REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60),
+
+        GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10),
+
+        SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"),
+        SOURCEBOT_STRUCTURED_LOGGING_ENABLED: booleanSchema.default("false"),
+        SOURCEBOT_STRUCTURED_LOGGING_FILE: z.string().optional(),
     },
-    skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
+    runtimeEnv,
     emptyStringAsUndefined: true,
+    skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
 });