feat(core): Add experimental debug ID based source map upload to Rollup and Vite plugins (#192)

Author: Luca Forstner

packages/bundler-plugin-core/package.json

@@ -36,10 +36,11 @@
     "fix": "eslint ./src ./test --format stylish --fix"
   },
   "dependencies": {
-    "@sentry/cli": "^2.10.0",
+    "@sentry/cli": "^2.17.0",
     "@sentry/node": "^7.19.0",
     "@sentry/tracing": "^7.19.0",
     "find-up": "5.0.0",
+    "glob": "9.3.2",
     "magic-string": "0.27.0",
     "unplugin": "1.0.1"
   },

packages/bundler-plugin-core/src/debug-id.ts

@@ -0,0 +1,162 @@
+import * as fs from "fs";
+import MagicString from "magic-string";
+import * as path from "path";
+import * as util from "util";
+import { Logger } from "./sentry/logger";
+import { stringToUUID } from "./utils";
+
+// TODO: Find a more elaborate process to generate this. (Maybe with type checking and built-in minification)
+const DEBUG_ID_INJECTOR_SNIPPET =
+  ';!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{},n=(new Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="__SENTRY_DEBUG_ID__",e._sentryDebugIdIdentifier="sentry-dbid-__SENTRY_DEBUG_ID__")}catch(e){}}();';
+
+export function injectDebugIdSnippetIntoChunk(code: string) {
+  const debugId = stringToUUID(code); // generate a deterministic debug ID
+  const ms = new MagicString(code);
+
+  const codeToInject = DEBUG_ID_INJECTOR_SNIPPET.replace(/__SENTRY_DEBUG_ID__/g, debugId);
+
+  // We need to be careful not to inject the snippet before any `"use strict";`s.
+  // As an additional complication `"use strict";`s may come after any number of comments.
+  const commentUseStrictRegex =
+    /^(?:\s*|\/\*(.|\r|\n)*?\*\/|\/\/.*?[\n\r])*(?:"use strict";|'use strict';)?/;
+
+  if (code.match(commentUseStrictRegex)?.[0]) {
+    // Add injected code after any comments or "use strict" at the beginning of the bundle.
+    ms.replace(commentUseStrictRegex, (match) => `${match}${codeToInject}`);
+  } else {
+    // ms.replace() doesn't work when there is an empty string match (which happens if
+    // there is neither, a comment, nor a "use strict" at the top of the chunk) so we
+    // need this special case here.
+    ms.prepend(codeToInject);
+  }
+
+  return {
+    code: ms.toString(),
+    map: ms.generateMap(),
+  };
+}
+
+export async function prepareBundleForDebugIdUpload(
+  bundleFilePath: string,
+  uploadFolder: string,
+  uniqueUploadName: string,
+  logger: Logger
+) {
+  let bundleContent;
+  try {
+    bundleContent = await util.promisify(fs.readFile)(bundleFilePath, "utf8");
+  } catch (e) {
+    logger.warn(`Could not read bundle to determine debug ID and source map: ${bundleFilePath}`);
+    return;
+  }
+
+  const debugId = determineDebugIdFromBundleSource(bundleContent);
+  if (debugId === undefined) {
+    logger.warn(`Could not determine debug ID from bundle: ${bundleFilePath}`);
+    return;
+  }
+
+  bundleContent += `\n//# debugId=${debugId}`;
+  const writeSourceFilePromise = util.promisify(fs.writeFile)(
+    path.join(uploadFolder, `${uniqueUploadName}.js`),
+    bundleContent,
+    "utf-8"
+  );
+
+  const writeSourceMapFilePromise = determineSourceMapPathFromBundle(
+    bundleFilePath,
+    bundleContent,
+    logger
+  ).then(async (sourceMapPath): Promise<void> => {
+    if (sourceMapPath) {
+      return await prepareSourceMapForDebugIdUpload(
+        sourceMapPath,
+        path.join(uploadFolder, `${uniqueUploadName}.js.map`),
+        debugId,
+        logger
+      );
+    }
+  });
+
+  return Promise.all([writeSourceFilePromise, writeSourceMapFilePromise]);
+}
+
+/**
+ * Looks for a particular string pattern (`sdbid-[debug ID]`) in the bundle
+ * source and extracts the bundle's debug ID from it.
+ *
+ * The string pattern is injected via the debug ID injection snipped.
+ */
+function determineDebugIdFromBundleSource(code: string): string | undefined {
+  const match = code.match(
+    /sentry-dbid-([0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})/
+  );
+
+  if (match) {
+    return match[1];
+  } else {
+    return undefined;
+  }
+}
+
+/**
+ * Applies a set of heuristics to find the source map for a particular bundle.
+ *
+ * @returns the path to the bundle's source map or `undefined` if none could be found.
+ */
+async function determineSourceMapPathFromBundle(
+  bundlePath: string,
+  bundleSource: string,
+  logger: Logger
+): Promise<string | undefined> {
+  // 1. try to find source map at `sourceMappingURL` location
+  const sourceMappingUrlMatch = bundleSource.match(/^\/\/# sourceMappingURL=(.*)$/);
+  if (sourceMappingUrlMatch) {
+    const sourceMappingUrl = path.normalize(sourceMappingUrlMatch[1] as string);
+    if (path.isAbsolute(sourceMappingUrl)) {
+      return sourceMappingUrl;
+    } else {
+      return path.join(path.dirname(bundlePath), sourceMappingUrl);
+    }
+  }
+
+  // 2. try to find source map at path adjacent to chunk source, but with `.map` appended
+  try {
+    const adjacentSourceMapFilePath = bundlePath + ".map";
+    await util.promisify(fs.access)(adjacentSourceMapFilePath);
+    return adjacentSourceMapFilePath;
+  } catch (e) {
+    // noop
+  }
+
+  logger.warn(`Could not determine source map path for bundle: ${bundlePath}`);
+  return undefined;
+}
+
+/**
+ * Reads a source map, injects debug ID fields, and writes the source map to the target path.
+ */
+async function prepareSourceMapForDebugIdUpload(
+  sourceMapPath: string,
+  targetPath: string,
+  debugId: string,
+  logger: Logger
+): Promise<void> {
+  try {
+    const sourceMapFileContent = await util.promisify(fs.readFile)(sourceMapPath, {
+      encoding: "utf8",
+    });
+
+    const map = JSON.parse(sourceMapFileContent) as Record<string, string>;
+
+    // For now we write both fields until we know what will become the standard - if ever.
+    map["debug_id"] = debugId;
+    map["debugId"] = debugId;
+
+    await util.promisify(fs.writeFile)(targetPath, JSON.stringify(map), {
+      encoding: "utf8",
+    });
+  } catch (e) {
+    logger.warn(`Failed to prepare source map for debug ID upload: ${sourceMapPath}`);
+  }
+}

packages/bundler-plugin-core/src/index.ts

@@ -8,6 +8,7 @@ import {
   finalizeRelease,
   setCommits,
   uploadSourceMaps,
+  uploadDebugIdSourcemaps,
 } from "./sentry/releasePipeline";
 import "@sentry/tracing";
 import SentryCli from "@sentry/cli";
@@ -22,15 +23,20 @@ import { createLogger, Logger } from "./sentry/logger";
 import { InternalOptions, normalizeUserOptions, validateOptions } from "./options-mapping";
 import { getSentryCli } from "./sentry/cli";
 import { makeMain } from "@sentry/node";
+import os from "os";
 import path from "path";
 import fs from "fs";
+import util from "util";
 import { getDependencies, getPackageJson, parseMajorVersion } from "./utils";
+import { glob } from "glob";
+import { injectDebugIdSnippetIntoChunk, prepareBundleForDebugIdUpload } from "./debug-id";
 
 const ALLOWED_TRANSFORMATION_FILE_ENDINGS = [".js", ".ts", ".jsx", ".tsx", ".mjs"];
 
 const releaseInjectionFilePath = require.resolve(
   "@sentry/bundler-plugin-core/sentry-release-injection-file"
 );
+
 /**
  * The sentry bundler plugin concerns itself with two things:
  * - Release injection
@@ -286,7 +292,37 @@ const unplugin = createUnplugin<Options>((options, unpluginMetaContext) => {
 
       const releaseName = await releaseNamePromise;
 
+      let tmpUploadFolder: string | undefined;
+
       try {
+        if (internalOptions._experiments.debugIdUpload) {
+          const debugIdChunkFilePaths = (
+            await glob(internalOptions._experiments.debugIdUpload.include, {
+              absolute: true,
+              nodir: true,
+              ignore: internalOptions._experiments.debugIdUpload.ignore,
+            })
+          ).filter((p) => p.endsWith(".js") || p.endsWith(".mjs"));
+
+          const sourceFileUploadFolderPromise = util.promisify(fs.mkdtemp)(
+            path.join(os.tmpdir(), "sentry-bundler-plugin-upload-")
+          );
+
+          await Promise.all(
+            debugIdChunkFilePaths.map(async (chunkFilePath, chunkIndex): Promise<void> => {
+              await prepareBundleForDebugIdUpload(
+                chunkFilePath,
+                await sourceFileUploadFolderPromise,
+                String(chunkIndex),
+                logger
+              );
+            })
+          );
+
+          tmpUploadFolder = await sourceFileUploadFolderPromise;
+          await uploadDebugIdSourcemaps(internalOptions, ctx, tmpUploadFolder, releaseName);
+        }
+
         await createNewRelease(internalOptions, ctx, releaseName);
         await cleanArtifacts(internalOptions, ctx, releaseName);
         await uploadSourceMaps(internalOptions, ctx, releaseName);
@@ -302,6 +338,11 @@ const unplugin = createUnplugin<Options>((options, unpluginMetaContext) => {
         });
         handleError(e, logger, internalOptions.errorHandler);
       } finally {
+        if (tmpUploadFolder) {
+          fs.rm(tmpUploadFolder, { recursive: true, force: true }, () => {
+            // We don't care if this errors
+          });
+        }
         releasePipelineSpan?.finish();
         transaction?.finish();
         await sentryClient.flush().then(null, () => {
@@ -314,6 +355,30 @@ const unplugin = createUnplugin<Options>((options, unpluginMetaContext) => {
         level: "info",
       });
     },
+    rollup: {
+      renderChunk(code, chunk) {
+        if (
+          options._experiments?.debugIdUpload &&
+          [".js", ".mjs"].some((ending) => chunk.fileName.endsWith(ending)) // chunks could be any file (html, md, ...)
+        ) {
+          return injectDebugIdSnippetIntoChunk(code);
+        } else {
+          return null; // returning null means not modifying the chunk at all
+        }
+      },
+    },
+    vite: {
+      renderChunk(code, chunk) {
+        if (
+          options._experiments?.debugIdUpload &&
+          [".js", ".mjs"].some((ending) => chunk.fileName.endsWith(ending)) // chunks could be any file (html, md, ...)
+        ) {
+          return injectDebugIdSnippetIntoChunk(code);
+        } else {
+          return null; // returning null means not modifying the chunk at all
+        }
+      },
+    },
   };
 });
 

packages/bundler-plugin-core/src/sentry/releasePipeline.ts

@@ -87,6 +87,39 @@ export async function uploadSourceMaps(
   ctx.logger.info("Successfully uploaded source maps.");
 }
 
+export async function uploadDebugIdSourcemaps(
+  options: InternalOptions,
+  ctx: BuildContext,
+  folderPathToUpload: string,
+  releaseName: string
+): Promise<void> {
+  const span = addSpanToTransaction(ctx, "function.plugin.upload_debug_id_sourcemaps");
+  ctx.logger.info("Uploading debug ID Sourcemaps.");
+
+  // Since our internal include entries contain all top-level sourcemaps options,
+  // we only need to pass the include option here.
+  try {
+    await ctx.cli.releases.uploadSourceMaps(releaseName, {
+      include: [
+        {
+          paths: [folderPathToUpload],
+          rewrite: false,
+          dist: options.dist,
+        },
+      ],
+      useArtifactBundle: true,
+    });
+  } catch (e) {
+    ctx.hub.captureException(new Error("CLI Error: Uploading debug ID source maps failed"));
+    throw e;
+  } finally {
+    span?.finish();
+  }
+
+  ctx.hub.addBreadcrumb({ level: "info", message: "Successfully uploaded debug ID source maps." });
+  ctx.logger.info("Successfully uploaded debug ID source maps.");
+}
+
 export async function setCommits(
   options: InternalOptions,
   ctx: BuildContext,

packages/bundler-plugin-core/src/sentry/telemetry.ts

@@ -96,6 +96,7 @@ export function addPluginOptionInformationToHub(
     errorHandler,
     deploy,
     include,
+    _experiments,
   } = options;
 
   hub.setTag("include", include.length > 1 ? "multiple-entries" : "single-entry");
@@ -124,6 +125,9 @@ export function addPluginOptionInformationToHub(
   if (errorHandler) {
     hub.setTag("error-handler", "custom");
   }
+  if (_experiments.debugIdUpload) {
+    hub.setTag("debug-id-upload", true);
+  }
 
   hub.setTag("node", process.version);
 

packages/bundler-plugin-core/src/tsconfig.json

@@ -6,6 +6,7 @@
     "esModuleInterop": true,
     "resolveJsonModule": true, // needed to import package.json
     "types": ["node"],
+    "target": "ES6", // needed for some iterator features
     "lib": ["ES2020", "DOM"] // es2020 needed for "new Set()", DOM needed for various bundler types
   }
 }

packages/bundler-plugin-core/src/types.ts

@@ -214,17 +214,34 @@ export type Options = Omit<IncludeEntry, "paths"> & {
   uploadSourceMaps?: boolean;
 
   /**
-   * These options are considered experimental and subject to change.
-   *
-   * _experiments.injectBuildInformation:
-   * If set to true, the plugin will inject an additional `SENTRY_BUILD_INFO` variable.
-   * This contains information about the build, e.g. dependencies, node version and other useful data.
-   *
-   * Defaults to `false`.
-   * @hidden
+   * Options that are considered experimental and subject to change.
    */
   _experiments?: {
+    /**
+     * If set to true, the plugin will inject an additional `SENTRY_BUILD_INFO` variable.
+     * This contains information about the build, e.g. dependencies, node version and other useful data.
+     *
+     * Defaults to `false`.
+     */
     injectBuildInformation?: boolean;
+
+    /**
+     * Configuration for debug ID upload.
+     *
+     * Note: Currently only functional for Vite and Rollup.
+     */
+    debugIdUpload?: {
+      /**
+       * Glob paths to files that should get be injected with a debug ID and uploaded.
+       */
+      include: string | string[];
+      /**
+       * Glob paths to files that should be ignored for debug ID injection and upload.
+       *
+       * Default: `[]`
+       */
+      ignore?: string | string[];
+    };
   };
 };