sdk-core: reduce breadcrumbs size (#228)

* sdk-core: add limitObjectDepth

* sdk-core: add breadcrumb limits options

* sdk-core: limit breadcrumb attributes object depth

* sdk-core: limit breadcrumb message length

* sdk-core: add jsonSize

* sdk-core: add breadcrumb size check and move limiting after intercept and level/type checks

* sdk-core: add breadcrumb factories

* sdk-core: add breadcrumb file size limiting in InMemoryBreadcrumbsStorage

* sdk-core: update docs for breadcrumb limits

* sdk-core: allow for unlimited breacrumbs in InMemoryBreadcrumbsStorage

* sdk-core: PR suggestion changes

* sdk-core: fix jsonSize returning invalid sizes for objects with toJSON

* sdk-core: refactored OverwritingArray, added pop and shift

* sdk-core: InMemoryBreadcrumbsStorage: shift breadcrumbs if larger than limit

* sdk-core: clear items removed by pop and shift in OverwritingArray

* sdk-core: reduce breadcrumb size PR changes

* node, react-native: update defaults to compile

* sdk-core: allow returning undefined from BreadcrumbsStorage.add

---------

Co-authored-by: Sebastian Alex <sebastian.alex@saucelabs.com>

Author: perf2711

packages/node/src/BacktraceClient.ts

@@ -48,7 +48,7 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
                 FileBreadcrumbsStorage.create(
                     this.sessionFiles,
                     fileSystem,
-                    clientSetup.options.breadcrumbs?.maximumBreadcrumbs ?? 100,
+                    (clientSetup.options.breadcrumbs?.maximumBreadcrumbs ?? 100) || 100,
                 ),
             );
         }

packages/react-native/src/BacktraceClient.ts

@@ -59,7 +59,7 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
                 FileBreadcrumbsStorage.create(
                     fileSystem,
                     this.sessionFiles,
-                    clientSetup.options.breadcrumbs?.maximumBreadcrumbs ?? 100,
+                    (clientSetup.options.breadcrumbs?.maximumBreadcrumbs ?? 100) || 100,
                 ),
             );
         }

packages/sdk-core/src/builder/BacktraceCoreClientBuilder.ts

@@ -2,7 +2,11 @@ import { BacktraceReportSubmission } from '../model/http/BacktraceReportSubmissi
 import { BacktraceRequestHandler } from '../model/http/BacktraceRequestHandler.js';
 import { BacktraceModule } from '../modules/BacktraceModule.js';
 import { BacktraceAttributeProvider } from '../modules/attribute/BacktraceAttributeProvider.js';
-import { BreadcrumbsEventSubscriber, BreadcrumbsStorage } from '../modules/breadcrumbs/index.js';
+import {
+    BreadcrumbsEventSubscriber,
+    BreadcrumbsStorage,
+    BreadcrumbsStorageFactory,
+} from '../modules/breadcrumbs/index.js';
 import { BacktraceStackTraceConverter } from '../modules/converter/index.js';
 import { BacktraceSessionProvider } from '../modules/metrics/BacktraceSessionProvider.js';
 import { MetricsQueue } from '../modules/metrics/MetricsQueue.js';
@@ -39,12 +43,22 @@ export abstract class BacktraceCoreClientBuilder<S extends Partial<CoreClientSet
         return this;
     }
 
-    public useBreadcrumbsStorage(storage: BreadcrumbsStorage): this {
+    public useBreadcrumbsStorage(storageFactory: BreadcrumbsStorageFactory): this;
+    /**
+     * @deprecated Use `useBreadcrumbsStorage` with `BreadcrumbsStorageFactory`.
+     */
+    public useBreadcrumbsStorage(storage: BreadcrumbsStorage): this;
+    public useBreadcrumbsStorage(storage: BreadcrumbsStorage | BreadcrumbsStorageFactory): this {
         if (!this.clientSetup.breadcrumbsSetup) {
             this.clientSetup.breadcrumbsSetup = {};
         }
 
-        this.clientSetup.breadcrumbsSetup.storage = storage;
+        if (typeof storage === 'function') {
+            this.clientSetup.breadcrumbsSetup.storage = storage;
+        } else {
+            this.clientSetup.breadcrumbsSetup.storage = () => storage;
+        }
+
         return this;
     }
 

packages/sdk-core/src/common/jsonEscaper.ts

@@ -5,12 +5,10 @@ export function jsonEscaper() {
     // in TypeScript add "this: any" param to avoid compliation errors - as follows
     //    return function (this: any, field: any, value: any) {
     return function (this: unknown, key: string, value: unknown) {
-        if (!key) {
-            return value;
-        }
         if (value === null) {
             return value;
         }
+
         const valueType = typeof value;
 
         if (valueType === 'bigint') {
@@ -28,7 +26,7 @@ export function jsonEscaper() {
             keys.pop();
         }
         if (ancestors.includes(value)) {
-            return `[Circular].${keys.join('.')}.${key}`;
+            return `[Circular].${keys.filter((k) => !!k).join('.')}.${key}`;
         }
         keys.push(key);
         ancestors.push(value);

packages/sdk-core/src/common/jsonSize.ts

@@ -0,0 +1,137 @@
+type JsonReplacer = (this: unknown, key: string, value: unknown) => unknown;
+
+function stringifiedSize<T>(value: T): number {
+    return JSON.stringify(value).length;
+}
+
+function toStringSize<T extends { toString(): string }>(value: T): number {
+    return value.toString().length;
+}
+
+const stringSize = (value: string) => stringifiedSize(value);
+const numberSize = toStringSize<number>;
+const bigintSize = toStringSize<bigint>;
+const symbolSize = 0;
+const functionSize = 0;
+const booleanSize = (value: boolean) => (value ? 4 : 5);
+const undefinedSize = 0;
+const nullSize = 'null'.length;
+
+function arraySize(array: unknown[], replacer?: JsonReplacer): number {
+    const bracketLength = 2;
+    const commaLength = array.length - 1;
+    let elementsLength = 0;
+    for (let i = 0; i < array.length; i++) {
+        const element = array[i];
+        switch (typeof element) {
+            case 'function':
+            case 'symbol':
+            case 'undefined':
+                elementsLength += nullSize;
+                break;
+            default:
+                elementsLength += _jsonSize(array, i.toString(), element, replacer);
+        }
+    }
+
+    return bracketLength + commaLength + elementsLength;
+}
+
+const objectSize = (obj: object, replacer?: JsonReplacer): number => {
+    const entries = Object.entries(obj);
+    const bracketLength = 2;
+
+    let entryCount = 0;
+    let entriesLength = 0;
+
+    for (const [k, v] of entries) {
+        const valueSize = _jsonSize(obj, k, v, replacer);
+        if (valueSize === 0) {
+            continue;
+        }
+
+        entryCount++;
+
+        // +1 adds the comma size
+        entriesLength += keySize(k) + valueSize + 1;
+    }
+
+    // -1 removes previously added last comma size (there is no trailing comma)
+    const commaLength = Math.max(0, entryCount - 1);
+
+    return bracketLength + commaLength + entriesLength;
+};
+
+function keySize(key: unknown): number {
+    const QUOTE_SIZE = 2;
+
+    if (key === null) {
+        return nullSize + QUOTE_SIZE;
+    } else if (key === undefined) {
+        return '"undefined"'.length;
+    }
+
+    switch (typeof key) {
+        case 'string':
+            return stringSize(key);
+        case 'number':
+            return numberSize(key) + QUOTE_SIZE;
+        case 'boolean':
+            return booleanSize(key) + QUOTE_SIZE;
+        case 'symbol':
+            return symbolSize; // key not used in JSON
+        default:
+            return stringSize(key.toString());
+    }
+}
+
+function _jsonSize(parent: unknown, key: string, value: unknown, replacer?: JsonReplacer): number {
+    if (value && typeof value === 'object' && 'toJSON' in value && typeof value.toJSON === 'function') {
+        value = value.toJSON() as object;
+    }
+
+    value = replacer ? replacer.call(parent, key, value) : value;
+    if (value === null) {
+        return nullSize;
+    } else if (value === undefined) {
+        return undefinedSize;
+    }
+
+    if (Array.isArray(value)) {
+        return arraySize(value, replacer);
+    }
+
+    switch (typeof value) {
+        case 'bigint':
+            return bigintSize(value);
+        case 'boolean':
+            return booleanSize(value);
+        case 'function':
+            return functionSize;
+        case 'number':
+            return numberSize(value);
+        case 'object':
+            return objectSize(value, replacer);
+        case 'string':
+            return stringSize(value);
+        case 'symbol':
+            return symbolSize;
+        case 'undefined':
+            return undefinedSize;
+    }
+
+    return 0;
+}
+
+/**
+ * Calculates size of the object as it would be serialized into JSON.
+ *
+ * _Should_ return the same value as `JSON.stringify(value, replacer).length`.
+ * This may not be 100% accurate, but should work for our requirements.
+ * @param value Value to compute length for.
+ * @param replacer A function that transforms the results as in `JSON.stringify`.
+ * @returns Final string length.
+ */
+export function jsonSize(value: unknown, replacer?: JsonReplacer): number {
+    return _jsonSize(undefined, '', value, replacer);
+}

packages/sdk-core/src/common/limitObjectDepth.ts

@@ -0,0 +1,30 @@
+type DeepPartial<T extends object> = Partial<{ [K in keyof T]: T[K] extends object ? DeepPartial<T[K]> : T[K] }>;
+
+const REMOVED_PLACEHOLDER = '<removed>';
+
+export type Limited<T extends object> = DeepPartial<T> | typeof REMOVED_PLACEHOLDER;
+
+export function limitObjectDepth<T extends object>(obj: T, depth: number): Limited<T> {
+    if (!(depth < Infinity)) {
+        return obj;
+    }
+
+    if (depth < 0) {
+        return REMOVED_PLACEHOLDER;
+    }
+
+    const limitIfObject = (value: unknown) =>
+        typeof value === 'object' && value ? limitObjectDepth(value, depth - 1) : value;
+
+    const result: DeepPartial<T> = {};
+    for (const key in obj) {
+        const value = obj[key];
+        if (Array.isArray(value)) {
+            result[key] = value.map(limitIfObject) as never;
+        } else {
+            result[key] = limitIfObject(value) as never;
+        }
+    }
+
+    return result;
+}

packages/sdk-core/src/dataStructures/OverwritingArray.ts

@@ -1,47 +1,120 @@
-import { OverwritingArrayIterator } from './OverwritingArrayIterator.js';
+import { ConstrainedNumber, clamped, wrapped } from './numbers.js';
 
 export class OverwritingArray<T> {
     private _array: T[];
-    private _index = 0;
-    private _size = 0;
-    private _startIndex = 0;
-    constructor(public readonly capacity: number) {
-        this._array = this.createArray();
+
+    private readonly _headConstraint: ConstrainedNumber;
+    private readonly _lengthConstraint: ConstrainedNumber;
+
+    private _head = 0;
+    private _length = 0;
+
+    private get head() {
+        return this._head;
+    }
+
+    private set head(value: number) {
+        this._head = this._headConstraint(value);
     }
-    public add(value: T): this {
-        this._array[this._index] = value;
-        this._index = this.incrementIndex(this._index);
-        this._startIndex = this.incrementStartingIndex();
-        this._size = this.incrementSize();
-        return this;
+
+    public get length() {
+        return this._length;
     }
 
-    public clear(): void {
-        this._array = this.createArray();
+    public set length(value: number) {
+        this._length = this._lengthConstraint(value);
     }
 
-    public values(): IterableIterator<T> {
-        return new OverwritingArrayIterator<T>(this._array, this._startIndex, this._size);
+    private get start() {
+        return this._headConstraint(this.head - this.length);
     }
 
-    [Symbol.iterator](): IterableIterator<T> {
-        return new OverwritingArrayIterator<T>(this._array, this._startIndex, this._size);
+    constructor(
+        public readonly capacity: number,
+        items?: T[],
+    ) {
+        this._array = new Array(capacity);
+
+        // Head must be always between 0 and capacity.
+        // If lower than 0, it needs to go from the end
+        // If larger than capacity, it needs to go from the start
+        // Wrapping solves this
+        this._headConstraint = wrapped(0, capacity);
+
+        // Length must be always no less than 0 and no larger than capacity
+        this._lengthConstraint = clamped(0, capacity);
+
+        if (items) {
+            this.push(...items);
+        }
     }
 
-    private incrementIndex(index: number) {
-        return (index + 1) % this.capacity;
+    public add(item: T) {
+        return this.pushOne(item);
     }
 
-    private incrementStartingIndex() {
-        if (this._size !== this.capacity) {
-            return this._startIndex;
+    public push(...items: T[]): number {
+        for (const item of items) {
+            this.pushOne(item);
         }
-        return this.incrementIndex(this._startIndex);
+        return this.length;
+    }
+
+    public pop(): T | undefined {
+        this.head--;
+        const element = this._array[this.head];
+        this._array[this.head] = undefined as never;
+        this.length--;
+        return element;
     }
-    private incrementSize() {
-        return Math.min(this.capacity, this._size + 1);
+
+    public shift(): T | undefined {
+        const element = this._array[this.start];
+        this._array[this.start] = undefined as never;
+        this.length--;
+        return element;
+    }
+
+    public at(index: number): T | undefined {
+        return this._array[this.index(index)];
     }
-    private createArray() {
-        return new Array(this.capacity);
+
+    public *values(): IterableIterator<T> {
+        for (let i = 0; i < this.length; i++) {
+            const index = this.index(i);
+            yield this._array[index];
+        }
+    }
+
+    public *keys(): IterableIterator<number> {
+        for (let i = 0; i < this.length; i++) {
+            yield i;
+        }
+    }
+
+    public *entries(): IterableIterator<[number, T]> {
+        for (let i = 0; i < this.length; i++) {
+            const index = this.index(i);
+            yield [i, this._array[index]];
+        }
+    }
+
+    public [Symbol.iterator]() {
+        return this.values();
+    }
+
+    private pushOne(item: T) {
+        this._array[this.head] = item;
+        this.head++;
+        this.length++;
+    }
+
+    private index(value: number) {
+        if (!this.length) {
+            return this._headConstraint(value);
+        }
+
+        const index = (value % this.length) + this.start;
+        return this._headConstraint(index);
     }
 }