feat: add MacOS support

.github/workflows/ci.yml

@@ -87,30 +87,3 @@ jobs:
         env:
           CODSPEED_SKIP_UPLOAD: true
           CODSPEED_DEBUG: true
-
-  walltime-macos-test:
-    runs-on: macos-latest
-    steps:
-      - uses: "actions/checkout@v4"
-        with:
-          fetch-depth: 0
-          submodules: true
-      - uses: pnpm/action-setup@v2
-      - uses: actions/setup-node@v3
-        with:
-          cache: pnpm
-          node-version-file: .nvmrc
-      - name: Restore turbo cache
-        uses: ./.github/actions/turbo-cache
-      - run: pnpm install --frozen-lockfile --prefer-offline
-      - run: pnpm turbo run build
-
-      - name: Run benchmarks
-        uses: CodSpeedHQ/action@main
-        env:
-          CODSPEED_SKIP_UPLOAD: "true"
-          # Samply fails to profile pnpm targets for now
-          CODSPEED_PROFILER_ENABLED: "false"
-        with:
-          run: pnpm turbo run bench --filter=@codspeed/vitest-plugin
-          mode: walltime

.github/workflows/codspeed.yml

@@ -91,6 +91,34 @@ jobs:
             pnpm --workspace-concurrency 1 -r bench-tinybench
             pnpm --workspace-concurrency 1 -r bench-vitest
 
+  codspeed-walltime-macos:
+    runs-on: macos-latest
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: pnpm/action-setup@v2
+      - uses: actions/setup-node@v3
+        with:
+          cache: pnpm
+          node-version-file: .nvmrc
+      - name: Restore turbo cache
+        uses: ./.github/actions/turbo-cache
+      - run: pnpm install --frozen-lockfile --prefer-offline
+      - run: pnpm turbo run build
+
+      - name: Run macOS-only benchmarks
+        uses: CodSpeedHQ/action@main
+        with:
+          working-directory: packages/vitest-plugin
+          # Only run the macOS-only bench file: the rest of the suite already
+          # runs on the linux walltime job, and uploading the same benchmark
+          # twice for one commit is not supported.
+          run: pnpm turbo run bench --env-mode=loose --filter=@codspeed/vitest-plugin -- macos
+          mode: walltime
+          runner-version: branch:sip-resign-exec-redirect
+
   electron-e2e:
     name: Run electron inbox e2e
     runs-on: codspeed-macro

packages/tinybench-plugin/src/shared.ts

@@ -1,4 +1,11 @@
-import { InstrumentHooks, setupCore, teardownCore } from "@codspeed/core";
+import {
+  getInstrumentMode,
+  InstrumentHooks,
+  MARKER_TYPE_BENCHMARK_END,
+  MARKER_TYPE_BENCHMARK_START,
+  setupCore,
+  teardownCore,
+} from "@codspeed/core";
 import { Bench, Fn, Task } from "tinybench";
 import { getTaskUri } from "./uri";
 
@@ -63,25 +70,43 @@ export abstract class BaseBenchRunner {
   protected abstract finalizeAsyncRun(): Task[];
   protected abstract finalizeSyncRun(): Task[];
 
+  private sendRunMarkers(runStart: bigint, runEnd: bigint): void {
+    if (getInstrumentMode() !== "walltime") {
+      return;
+    }
+    InstrumentHooks.addMarker(
+      process.pid,
+      MARKER_TYPE_BENCHMARK_START,
+      runStart,
+    );
+    InstrumentHooks.addMarker(process.pid, MARKER_TYPE_BENCHMARK_END, runEnd);
+  }
+
   public setupBenchMethods(): void {
     this.bench.run = async () => {
       this.setupBenchRun();
 
+      const runStart = InstrumentHooks.currentTimestamp();
       for (const task of this.bench.tasks) {
         const uri = this.getTaskUri(task);
         await this.runTaskAsync(task, uri);
       }
+      const runEnd = InstrumentHooks.currentTimestamp();
+      this.sendRunMarkers(runStart, runEnd);
 
       return this.finalizeAsyncRun();
     };
 
     this.bench.runSync = () => {
       this.setupBenchRun();
 
+      const runStart = InstrumentHooks.currentTimestamp();
       for (const task of this.bench.tasks) {
         const uri = this.getTaskUri(task);
         this.runTaskSync(task, uri);
       }
+      const runEnd = InstrumentHooks.currentTimestamp();
+      this.sendRunMarkers(runStart, runEnd);
 
       return this.finalizeSyncRun();
     };

packages/tinybench-plugin/tests/index.integ.test.ts

@@ -16,6 +16,8 @@ const mockCore = vi.hoisted(() => {
       startBenchmark: vi.fn(),
       stopBenchmark: vi.fn(),
       setExecutedBenchmark: vi.fn(),
+      currentTimestamp: vi.fn().mockReturnValue(0n),
+      addMarker: vi.fn(),
     },
     optimizeFunction: vi
       .fn()

packages/vitest-plugin/benches/macos.bench.ts

@@ -0,0 +1,16 @@
+import { bench, describe } from "vitest";
+
+const isMacOS = process.platform === "darwin";
+
+function fibo(n: number): number {
+  if (n < 2) return 1;
+  return fibo(n - 1) + fibo(n - 2);
+}
+
+// macOS-only benchmark: skipped on every other platform, so it only runs on
+// the `walltime-macos-test` CI job (see .github/workflows/ci.yml).
+describe.skipIf(!isMacOS)("macos only", () => {
+  bench("fibo darwin", () => {
+    fibo(30);
+  });
+});

packages/vitest-plugin/src/walltime/index.ts

@@ -1,9 +1,10 @@
 import {
   InstrumentHooks,
+  MARKER_TYPE_BENCHMARK_END,
+  MARKER_TYPE_BENCHMARK_START,
   setupCore,
   writeWalltimeResults,
 } from "@codspeed/core";
-import { Fn } from "tinybench";
 import {
   RunnerTaskEventPack,
   RunnerTaskResultPack,
@@ -66,6 +67,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
     this.isTinybenchHookedWithCodspeed = true;
 
     const originalRun = tinybench.Task.prototype.run;
+    const pid = process.pid;
 
     const getSuiteUri = (): string => {
       if (this.currentSuiteId === null) {
@@ -75,21 +77,21 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
     };
 
     tinybench.Task.prototype.run = async function () {
-      const { fn } = this as { fn: Fn };
       const suiteUri = getSuiteUri();
 
-      function __codspeed_root_frame__() {
-        return fn();
-      }
-      (this as { fn: Fn }).fn = __codspeed_root_frame__;
-
       InstrumentHooks.startBenchmark();
+      const runStart = InstrumentHooks.currentTimestamp();
       await originalRun.call(this);
+      const runEnd = InstrumentHooks.currentTimestamp();
       InstrumentHooks.stopBenchmark();
 
+      // Emit a single marker pair covering the whole measurement run
+      InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, runStart);
+      InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd);
+
       // Look up the URI by task name
       const uri = `${suiteUri}::${this.name}`;
-      InstrumentHooks.setExecutedBenchmark(process.pid, uri);
+      InstrumentHooks.setExecutedBenchmark(pid, uri);
 
       return this;
     };