Implement Deno benchmark extract

action.yml

@@ -11,7 +11,7 @@ inputs:
     required: true
     default: 'Benchmark'
   tool:
-    description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest", "googlecpp", "catch2", "julia", "benchmarkdotnet", "customBiggerIsBetter", "customSmallerIsBetter"'
+    description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest", "googlecpp", "catch2", "julia", "benchmarkdotnet", "deno", "customBiggerIsBetter", "customSmallerIsBetter"'
     required: true
   output-file-path:
     description: 'A path to file which contains the benchmark output'

src/config.ts

@@ -38,6 +38,7 @@ export const VALID_TOOLS = [
     'julia',
     'jmh',
     'benchmarkdotnet',
+    'deno',
     'customBiggerIsBetter',
     'customSmallerIsBetter',
 ] as const;

src/extract.ts

@@ -29,7 +29,7 @@
 }
 
 interface PullRequest {
     [key: string]: any;
     number: number;
     html_url?: string;
     body?: string;
@@ -222,6 +222,41 @@
     Benchmarks: BenchmarkDotnetBenchmark[];
 }
 
+export interface DenoBenchmarkResult {
+    ok?: {
+        n: number;
+        min: number;
+        max: number;
+        avg: number;
+        p75: number;
+        p95: number;
+        p99: number;
+        p995: number;
+        p999: number;
+        highPrecision: boolean;
+        usedExplicitTimers: boolean;
+    };
+    failed?: {
+        exceptionMessage: string;
+    };
+}
+
+export interface DenoBenchmark {
+    origin: string;
+    group?: string;
+    name: string;
+    baseline: boolean;
+    result?: DenoBenchmarkResult;
+    results?: DenoBenchmarkResult[];
+}
+
+export interface DenoBenchmarkJson {
+    version: number;
+    runtime: string;
+    cpu: string;
+    benches: DenoBenchmark[];
+}
+
 function getHumanReadableUnitValue(seconds: number): [number, string] {
     if (seconds < 1.0e-6) {
         return [seconds * 1e9, 'nsec'];
@@ -436,7 +471,7 @@
             const extra = `mean: ${mean} ${meanUnit}\nrounds: ${stats.rounds}`;
             return { name, value, unit, range, extra };
         });
     } catch (err: any) {
         throw new Error(
             `Output file for 'pytest' must be JSON file generated by --benchmark-json option: ${err.message}`,
         );
@@ -447,7 +482,7 @@
     let json: GoogleCppBenchmarkJson;
     try {
         json = JSON.parse(output);
     } catch (err: any) {
         throw new Error(
             `Output file for 'googlecpp' must be JSON file generated by --benchmark_format=json option: ${err.message}`,
         );
@@ -575,7 +610,7 @@
     return ret;
 }
 
 function extractJuliaBenchmarkHelper([_, bench]: JuliaBenchmarkGroup, labels: string[] = []): BenchmarkResult[] {
     const res: BenchmarkResult[] = [];
     for (const key in bench.data) {
         const value = bench.data[key];
@@ -606,7 +641,7 @@
     let json: JuliaBenchmarkJson;
     try {
         json = JSON.parse(output);
     } catch (err: any) {
         throw new Error(
             `Output file for 'julia' must be JSON file generated by BenchmarkTools.save("output.json", suit::BenchmarkGroup) :  ${err.message}`,
         );
@@ -624,7 +659,7 @@
     let json: JmhBenchmarkJson[];
     try {
         json = JSON.parse(output);
     } catch (err: any) {
         throw new Error(`Output file for 'jmh' must be JSON file generated by -rf json option: ${err.message}`);
     }
     return json.map((b) => {
@@ -641,7 +676,7 @@
     let json: BenchmarkDotNetBenchmarkJson;
     try {
         json = JSON.parse(output);
     } catch (err: any) {
         throw new Error(
             `Output file for 'benchmarkdotnet' must be JSON file generated by '--exporters json' option or by adding the JsonExporter to your run config: ${err.message}`,
         );
@@ -662,7 +697,7 @@
         return json.map(({ name, value, unit, range, extra }) => {
             return { name, value, unit, range, extra };
         });
     } catch (err: any) {
         throw new Error(
             `Output file for 'custom-(bigger|smaller)-is-better' must be JSON file containing an array of entries in BenchmarkResult format: ${err.message}`,
         );
@@ -690,6 +725,44 @@
     return results;
 }
 
+function extractDenoBenchmarkResult(output: string): BenchmarkResult[] {
+    let json: DenoBenchmarkJson;
+    try {
+        json = JSON.parse(output);
+    } catch (err: any) {
+        throw new Error(
+            `Output file for 'benchmarkdotnet' must be JSON file generated by '--exporters json' option or by adding the JsonExporter to your run config: ${err.message}`,
+        );
+    }
+
+    if (json.version !== 1) {
+        throw new Error(`Unsupported version of deno benchmark output: ${json.version}`);
+    }
+
+    const results: BenchmarkResult[] = [];
+
+    for (const bench of json.benches) {
+        let result: DenoBenchmarkResult;
+        if (bench.result) {
+            result = bench.result;
+        } else if (bench.results && bench.results.length > 0) {
+            result = bench.results[0];
+        } else {
+            continue;
+        }
+        if (result.ok) {
+            results.push({
+                name: `${bench.origin}/${bench.group ?? '-'}/${bench.name}`,
+                value: result.ok.avg,
+                unit: 'ns',
+                extra: `n=${result.ok.n}`,
+            });
+        }
+    }
+
+    return results;
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool, githubToken, ref } = config;
@@ -732,6 +805,9 @@
         case 'benchmarkluau':
             benches = extractLuauBenchmarkResult(output);
             break;
+        case 'deno':
+            benches = extractDenoBenchmarkResult(output);
+            break;
         default:
             throw new Error(`FATAL: Unexpected tool: '${tool}'`);
     }

src/write.ts

@@ -81,6 +81,8 @@ function biggerIsBetter(tool: ToolType): boolean {
             return false;
         case 'benchmarkdotnet':
             return false;
+        case 'deno':
+            return false;
         case 'customBiggerIsBetter':
             return true;
         case 'customSmallerIsBetter':

test/__snapshots__/extract.spec.ts.snap

@@ -487,6 +487,64 @@ exports[`extractResult() extracts benchmark output from customSmallerIsBetter -
 }
 `;
 
+exports[`extractResult() extracts benchmark output from deno - deno_output1.json 1`] = `
+{
+  "benches": [
+    {
+      "extra": "n=49",
+      "name": "file:///dev/bench_me.js/-/Deno.UnsafePointerView#getUint32",
+      "unit": "ns",
+      "value": 1308.7523755102038,
+    },
+  ],
+  "commit": {
+    "author": null,
+    "committer": null,
+    "id": "123456789abcdef",
+    "message": "this is dummy",
+    "timestamp": "dummy timestamp",
+    "url": "https://github.com/dummy/repo",
+  },
+  "date": 1712131503296,
+  "tool": "deno",
+}
+`;
+
+exports[`extractResult() extracts benchmark output from deno - deno_output2.json 1`] = `
+{
+  "benches": [
+    {
+      "extra": "n=5321",
+      "name": "file:///Users/garciat/Desktop/wgpu-memory/benchmarks/scalar.bench.ts/Float32 writeAt/wgpu-memory",
+      "unit": "ns",
+      "value": 73232,
+    },
+    {
+      "extra": "n=6689",
+      "name": "file:///Users/garciat/Desktop/wgpu-memory/benchmarks/scalar.bench.ts/Float32 writeAt/DataView",
+      "unit": "ns",
+      "value": 56036,
+    },
+    {
+      "extra": "n=8493",
+      "name": "file:///Users/garciat/Desktop/wgpu-memory/benchmarks/scalar.bench.ts/Float32 writeAt/TypedArray",
+      "unit": "ns",
+      "value": 39802,
+    },
+  ],
+  "commit": {
+    "author": null,
+    "committer": null,
+    "id": "123456789abcdef",
+    "message": "this is dummy",
+    "timestamp": "dummy timestamp",
+    "url": "https://github.com/dummy/repo",
+  },
+  "date": 1712131503296,
+  "tool": "deno",
+}
+`;
+
 exports[`extractResult() extracts benchmark output from go - go_fiber_output.txt 1`] = `
 {
   "benches": [

test/data/extract/deno_output1.json

@@ -0,0 +1,25 @@
+{
+    "version": 1,
+    "runtime": "Deno/1.31.0 x86_64-apple-darwin",
+    "cpu": "Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz",
+    "benches": [
+      {
+        "origin": "file:///dev/bench_me.js",
+        "group": null,
+        "name": "Deno.UnsafePointerView#getUint32",
+        "baseline": false,
+        "result": {
+            "ok": {
+            "n": 49,
+            "min": 1251.9348,
+            "max": 1441.2696,
+            "avg": 1308.7523755102038,
+            "p75": 1324.1055,
+            "p99": 1441.2696,
+            "p995": 1441.2696,
+            "p999": 1441.2696
+            }
+        }
+      }
+    ]
+  }

test/data/extract/deno_output2.json

@@ -0,0 +1,73 @@
+{
+    "version": 1,
+    "runtime": "Deno/2.1.4 aarch64-apple-darwin",
+    "cpu": "Apple M1 Max",
+    "benches": [
+      {
+        "origin": "file:///Users/garciat/Desktop/wgpu-memory/benchmarks/scalar.bench.ts",
+        "group": "Float32 writeAt",
+        "name": "wgpu-memory",
+        "baseline": false,
+        "results": [
+          {
+            "ok": {
+              "n": 5321,
+              "min": 61917.0,
+              "max": 1205959.0,
+              "avg": 73232.0,
+              "p75": 64750.0,
+              "p99": 118500.0,
+              "p995": 122000.0,
+              "p999": 206583.0,
+              "highPrecision": true,
+              "usedExplicitTimers": true
+            }
+          }
+        ]
+      },
+      {
+        "origin": "file:///Users/garciat/Desktop/wgpu-memory/benchmarks/scalar.bench.ts",
+        "group": "Float32 writeAt",
+        "name": "DataView",
+        "baseline": false,
+        "results": [
+          {
+            "ok": {
+              "n": 6689,
+              "min": 54166.0,
+              "max": 602791.0,
+              "avg": 56036.0,
+              "p75": 54542.0,
+              "p99": 94000.0,
+              "p995": 94125.0,
+              "p999": 102250.0,
+              "highPrecision": true,
+              "usedExplicitTimers": true
+            }
+          }
+        ]
+      },
+      {
+        "origin": "file:///Users/garciat/Desktop/wgpu-memory/benchmarks/scalar.bench.ts",
+        "group": "Float32 writeAt",
+        "name": "TypedArray",
+        "baseline": true,
+        "results": [
+          {
+            "ok": {
+              "n": 8493,
+              "min": 38667.0,
+              "max": 225917.0,
+              "avg": 39802.0,
+              "p75": 38958.0,
+              "p99": 59708.0,
+              "p995": 59958.0,
+              "p999": 133042.0,
+              "highPrecision": true,
+              "usedExplicitTimers": true
+            }
+          }
+        ]
+      }
+    ]
+  }

test/extract.spec.ts

@@ -135,6 +135,14 @@ describe('extractResult()', function () {
             tool: 'benchmarkdotnet',
             file: 'benchmarkdotnet.json',
         },
+        {
+            tool: 'deno',
+            file: 'deno_output1.json',
+        },
+        {
+            tool: 'deno',
+            file: 'deno_output2.json',
+        },
         {
             tool: 'customBiggerIsBetter',
             file: 'customBiggerIsBetter_output.json',