Merge branch 'main' into docs/product-image-generator

D-K-P · web-flow · commit fc4ede45010e · 2025-09-16T10:14:36.000+01:00
diff --git a/.changeset/angry-files-yawn.md b/.changeset/angry-files-yawn.md
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Improves our schema to JSON Schema conversion, fixes zod 4 and a few other schema libraries, also correctly sets the dependencies
diff --git a/.changeset/seven-taxis-glow.md b/.changeset/seven-taxis-glow.md
@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Attach to existing deployment for deployments triggered in the build server. If `TRIGGER_EXISTING_DEPLOYMENT_ID` env var is set, the `deploy` command now skips the deployment initialization.
diff --git a/CHANGESETS.md b/CHANGESETS.md
@@ -30,14 +30,16 @@ Please follow the best-practice of adding changesets in the same commit as the c
 
 ## Snapshot instructions
 
-1. Delete the `.changeset/pre.json` file (if it exists)
+1. Update the `.changeset/config.json` file to set the `"changelog"` field to this:
 
-2. Do a temporary commit (do NOT push this, you should undo it after)
+```json
+"changelog": "@changesets/cli/changelog",
+```
 
-3. Copy the `GITHUB_TOKEN` line from the .env file
+2. Do a temporary commit (do NOT push this, you should undo it after)
 
-4. Run `GITHUB_TOKEN=github_pat_12345 ./scripts/publish-prerelease.sh re2`
+3. Run `./scripts/publish-prerelease.sh prerelease`
 
-Make sure to replace the token with yours. `re2` is the tag that will be used for the pre-release.
+You can choose a different tag if you want, but usually `prerelease` is fine.
 
-5. Undo the commit where you deleted the pre.json file.
+5. Undo the commit where you updated the config.json file.
diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts
@@ -89,6 +89,7 @@ const Env = z.object({
   KUBERNETES_CPU_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(0.75), // Ratio of CPU limit, so 0.75 = 75% of CPU limit
   KUBERNETES_MEMORY_REQUEST_MIN_GB: z.coerce.number().min(0).default(0),
   KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit
+  KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB
 
   // Placement tags settings
   PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),
diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts
@@ -25,6 +25,7 @@ export class KubernetesWorkloadManager implements WorkloadManager {
   private readonly cpuRequestRatio = env.KUBERNETES_CPU_REQUEST_RATIO;
   private readonly memoryRequestMinGb = env.KUBERNETES_MEMORY_REQUEST_MIN_GB;
   private readonly memoryRequestRatio = env.KUBERNETES_MEMORY_REQUEST_RATIO;
+  private readonly memoryOverheadGb = env.KUBERNETES_MEMORY_OVERHEAD_GB;
 
   constructor(private opts: WorkloadManagerOptions) {
     this.k8s = createK8sApi();
@@ -319,9 +320,13 @@ export class KubernetesWorkloadManager implements WorkloadManager {
   }
 
   #getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities {
+    const memoryLimit = this.memoryOverheadGb
+      ? preset.memory + this.memoryOverheadGb
+      : preset.memory;
+
     return {
       cpu: `${preset.cpu}`,
-      memory: `${preset.memory}G`,
+      memory: `${memoryLimit}G`,
     };
   }
 
diff --git a/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts b/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts
@@ -144,7 +144,7 @@ export class DeploymentListPresenter {
   wd."git"
 FROM
   ${sqlDatabaseSchema}."WorkerDeployment" as wd
-INNER JOIN
+LEFT JOIN
   ${sqlDatabaseSchema}."User" as u ON wd."triggeredById" = u."id"
 WHERE
   wd."projectId" = ${project.id}
diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts
@@ -1,4 +1,5 @@
-import { LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { type GetDeploymentResponseBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
@@ -52,7 +53,10 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
     shortCode: deployment.shortCode,
     version: deployment.version,
     imageReference: deployment.imageReference,
-    errorData: deployment.errorData,
+    imagePlatform: deployment.imagePlatform,
+    externalBuildData:
+      deployment.externalBuildData as GetDeploymentResponseBody["externalBuildData"],
+    errorData: deployment.errorData as GetDeploymentResponseBody["errorData"],
     worker: deployment.worker
       ? {
           id: deployment.worker.friendlyId,
@@ -65,5 +69,5 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
           })),
         }
       : undefined,
-  });
+  } satisfies GetDeploymentResponseBody);
 }
diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts
@@ -108,6 +108,8 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
   const disableVersionSelection = environment.type === "DEVELOPMENT";
   const allowArbitraryQueues = backgroundWorkers.at(0)?.engine === "V1";
 
+  const payload = await prettyPrintPacket(run.payload, run.payloadType);
+
   return typedjson({
     concurrencyKey: run.concurrencyKey,
     maxAttempts: run.maxAttempts,
@@ -116,7 +118,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
     ttlSeconds: run.ttl ? parseDuration(run.ttl, "s") ?? undefined : undefined,
     idempotencyKey: run.idempotencyKey,
     runTags: run.runTags,
-    payload: await prettyPrintPacket(run.payload, run.payloadType),
+    payload,
     payloadType: run.payloadType,
     queue: run.queue,
     metadata: run.seedMetadata
diff --git a/apps/webapp/test/fairDequeuingStrategy.test.ts b/apps/webapp/test/fairDequeuingStrategy.test.ts
@@ -270,8 +270,8 @@ describe("FairDequeuingStrategy", () => {
 
       console.log("Second distribution took", distribute2Duration, "ms");
 
-      // Make sure the second call is more than 2 times faster than the first
-      expect(distribute2Duration).toBeLessThan(withTolerance(distribute1Duration / 2));
+      // Make sure the second call is faster than the first
+      expect(distribute2Duration).toBeLessThan(distribute1Duration);
 
       const startDistribute3 = performance.now();
 
@@ -284,8 +284,8 @@ describe("FairDequeuingStrategy", () => {
 
       console.log("Third distribution took", distribute3Duration, "ms");
 
-      // Make sure the third call is more than 4 times the second
-      expect(withTolerance(distribute3Duration)).toBeGreaterThan(distribute2Duration * 4);
+      // Make sure the third call is faster than the second
+      expect(withTolerance(distribute3Duration)).toBeGreaterThan(distribute2Duration);
     }
   );
 
diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx
@@ -183,7 +183,7 @@ This can happen in different situations, for example when using plain strings as
 
 ### Task run stalled executing
 
-If you see a `TASK_RUN_STALLED_EXECUTING` error it means that we didn't receive a heartbeat from your task before the stall timeout. We automatically heartbeat runs every 30 seconds, and the heartbeat timeout is 10 minutes.
+If you see a `TASK_RUN_STALLED_EXECUTING` error it means that we didn't receive a heartbeat from your task before the stall timeout. We automatically heartbeat runs every 30 seconds, and the heartbeat timeout is 5 minutes.
 
 <Note>
 
diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts
@@ -1239,6 +1239,7 @@ export class RunEngine {
             orgId: latestSnapshot.organizationId,
             projectId: latestSnapshot.projectId,
             checkpointId: latestSnapshot.checkpointId ?? undefined,
+            completedWaitpoints: latestSnapshot.completedWaitpoints,
             error: {
               type: "INTERNAL_ERROR",
               code: "TASK_RUN_DEQUEUED_MAX_RETRIES",
diff --git a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts
@@ -1139,6 +1139,7 @@ export class RunAttemptSystem {
     workerId,
     runnerId,
     checkpointId,
+    completedWaitpoints,
     tx,
   }: {
     run: TaskRun;
@@ -1154,6 +1155,10 @@ export class RunAttemptSystem {
     runnerId?: string;
     checkpointId?: string;
     tx?: PrismaClientOrTransaction;
+    completedWaitpoints?: {
+      id: string;
+      index?: number;
+    }[];
   }): Promise<{ wasRequeued: boolean } & ExecutionResult> {
     const prisma = tx ?? this.$.prisma;
 
@@ -1201,6 +1206,7 @@ export class RunAttemptSystem {
         workerId,
         runnerId,
         checkpointId,
+        completedWaitpoints,
       });
 
       return {
diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts
@@ -1,6 +1,9 @@
 import { intro, log, outro } from "@clack/prompts";
 import { getBranch, prepareDeploymentError, tryCatch } from "@trigger.dev/core/v3";
-import { InitializeDeploymentResponseBody } from "@trigger.dev/core/v3/schemas";
+import {
+  InitializeDeploymentRequestBody,
+  InitializeDeploymentResponseBody,
+} from "@trigger.dev/core/v3/schemas";
 import { Command, Option as CommandOption } from "commander";
 import { resolve } from "node:path";
 import { isCI } from "std-env";
@@ -306,19 +309,17 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
     return;
   }
 
-  const deploymentResponse = await projectClient.client.initializeDeployment({
-    contentHash: buildManifest.contentHash,
-    userId: authorization.auth.tokenType === "personal" ? authorization.userId : undefined,
-    gitMeta,
-    type: features.run_engine_v2 ? "MANAGED" : "V1",
-    runtime: buildManifest.runtime,
-  });
-
-  if (!deploymentResponse.success) {
-    throw new Error(`Failed to start deployment: ${deploymentResponse.error}`);
-  }
-
-  const deployment = deploymentResponse.data;
+  const deployment = await initializeOrAttachDeployment(
+    projectClient.client,
+    {
+      contentHash: buildManifest.contentHash,
+      userId: authorization.auth.tokenType === "personal" ? authorization.userId : undefined,
+      gitMeta,
+      type: features.run_engine_v2 ? "MANAGED" : "V1",
+      runtime: buildManifest.runtime,
+    },
+    envVars.TRIGGER_EXISTING_DEPLOYMENT_ID
+  );
   const isLocalBuild = !deployment.externalBuildData;
 
   // Fail fast if we know local builds will fail
@@ -619,7 +620,7 @@ export async function syncEnvVarsWithServer(
 
 async function failDeploy(
   client: CliApiClient,
-  deployment: Deployment,
+  deployment: Pick<Deployment, "id" | "shortCode">,
   error: { name: string; message: string },
   logs: string,
   $spinner: ReturnType<typeof spinner>,
@@ -735,6 +736,60 @@ async function failDeploy(
   }
 }
 
+async function initializeOrAttachDeployment(
+  apiClient: CliApiClient,
+  data: InitializeDeploymentRequestBody,
+  existingDeploymentId?: string
+): Promise<InitializeDeploymentResponseBody> {
+  if (existingDeploymentId) {
+    // In the build server we initialize the deployment before installing the project dependencies,
+    // so that the status is correctly reflected in the dashboard. In this case, we need to attach
+    // to the existing deployment and continue with the remote build process.
+    // This is a workaround to avoid major changes in the deploy command and workflow. In the future,
+    // we'll likely make the build server the entry point of the flow for building and deploying and also
+    // adapt the related deployment API endpoints.
+
+    const existingDeploymentOrError = await apiClient.getDeployment(existingDeploymentId);
+
+    if (!existingDeploymentOrError.success) {
+      throw new Error(
+        `Failed to attach to existing deployment: ${existingDeploymentOrError.error}`
+      );
+    }
+
+    const { imageReference, status } = existingDeploymentOrError.data;
+    if (!imageReference) {
+      // this is just an artifact of our current DB schema
+      // `imageReference` is stored as nullable, but it should always exist
+      throw new Error("Existing deployment does not have an image reference");
+    }
+
+    if (
+      status === "CANCELED" ||
+      status === "FAILED" ||
+      status === "TIMED_OUT" ||
+      status === "DEPLOYED"
+    ) {
+      throw new Error(`Existing deployment is in an unexpected state: ${status}`);
+    }
+
+    return {
+      ...existingDeploymentOrError.data,
+      imageTag: imageReference,
+    };
+  }
+
+  const newDeploymentOrError = await apiClient.initializeDeployment({
+    ...data,
+  });
+
+  if (!newDeploymentOrError.success) {
+    throw new Error(`Failed to start deployment: ${newDeploymentOrError.error}`);
+  }
+
+  return newDeploymentOrError.data;
+}
+
 export function verifyDirectory(dir: string, projectPath: string) {
   if (dir !== "." && !isDirectory(projectPath)) {
     if (dir === "staging" || dir === "prod" || dir === "preview") {
diff --git a/packages/cli-v3/src/entryPoints/dev-index-worker.ts b/packages/cli-v3/src/entryPoints/dev-index-worker.ts
@@ -18,7 +18,7 @@ import { registerResources } from "../indexing/registerResources.js";
 import { env } from "std-env";
 import { normalizeImportPath } from "../utilities/normalizeImportPath.js";
 import { detectRuntimeVersion } from "@trigger.dev/core/v3/build";
-import { schemaToJsonSchema, initializeSchemaConverters } from "@trigger.dev/schema-to-json";
+import { schemaToJsonSchema } from "@trigger.dev/schema-to-json";
 
 sourceMapSupport.install({
   handleUncaughtExceptions: false,
@@ -193,8 +193,6 @@ await new Promise<void>((resolve) => {
 });
 
 async function convertSchemasToJsonSchemas(tasks: TaskManifest[]): Promise<TaskManifest[]> {
-  await initializeSchemaConverters();
-
   const convertedTasks = tasks.map((task) => {
     const schema = resourceCatalog.getTaskSchema(task.id);
 
diff --git a/packages/cli-v3/src/entryPoints/managed-index-worker.ts b/packages/cli-v3/src/entryPoints/managed-index-worker.ts
@@ -18,7 +18,7 @@ import { registerResources } from "../indexing/registerResources.js";
 import { env } from "std-env";
 import { normalizeImportPath } from "../utilities/normalizeImportPath.js";
 import { detectRuntimeVersion } from "@trigger.dev/core/v3/build";
-import { schemaToJsonSchema, initializeSchemaConverters } from "@trigger.dev/schema-to-json";
+import { schemaToJsonSchema } from "@trigger.dev/schema-to-json";
 
 sourceMapSupport.install({
   handleUncaughtExceptions: false,
@@ -201,8 +201,6 @@ await new Promise<void>((resolve) => {
 });
 
 async function convertSchemasToJsonSchemas(tasks: TaskManifest[]): Promise<TaskManifest[]> {
-  await initializeSchemaConverters();
-
   const convertedTasks = tasks.map((task) => {
     const schema = resourceCatalog.getTaskSchema(task.id);
 
diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts
@@ -467,6 +467,8 @@ export const GetDeploymentResponseBody = z.object({
   shortCode: z.string(),
   version: z.string(),
   imageReference: z.string().nullish(),
+  imagePlatform: z.string(),
+  externalBuildData: ExternalBuildData.optional().nullable(),
   errorData: DeploymentErrorData.nullish(),
   worker: z
     .object({
diff --git a/packages/core/src/v3/utils/ioSerialization.ts b/packages/core/src/v3/utils/ioSerialization.ts
diff --git a/packages/core/test/ioSerialization.test.ts b/packages/core/test/ioSerialization.test.ts
diff --git a/packages/schema-to-json/package.json b/packages/schema-to-json/package.json
diff --git a/packages/schema-to-json/src/index.ts b/packages/schema-to-json/src/index.ts
diff --git a/packages/schema-to-json/tests/index.test.ts b/packages/schema-to-json/tests/index.test.ts
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/references/hello-world/src/trigger/circularPayload.ts b/references/hello-world/src/trigger/circularPayload.ts
diff --git a/references/hello-world/src/trigger/jsonSchema.ts b/references/hello-world/src/trigger/jsonSchema.ts

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +"trigger.dev": patch
 +---
++
 +Improves our schema to JSON Schema conversion, fixes zod 4 and a few other schema libraries, also correctly sets the dependencies