feat: support running TypeScript benchmarks with tsx

.github/actions/test-native-abi/action.yml

@@ -1,15 +1,16 @@
 name: Test Native ABI
 description: >
-  Set up the given Node.js major and run the core package's integ tests
-  in-band, without rebuilding, to catch native addon ABI mismatches and
-  V8-version-specific natives-syntax behavior. The flags mirror what the
+  Set up the given Node.js version and run the core package's integ tests
+  in-band, without rebuilding, to catch native addon ABI mismatches,
+  V8-version-specific natives-syntax behavior and Node-version-specific
+  module hooks in the tsx loader. The flags mirror what the
   runner passes in a real benchmark process; they cannot go through
   NODE_OPTIONS (--allow-natives-syntax is rejected there), so jest runs
   in-band under a flagged node instead of forking workers.
 
 inputs:
   node-version:
-    description: Node.js major version to test under
+    description: Node.js version to test under
     required: true
 
 runs:
@@ -25,3 +26,4 @@ runs:
         "$(node -p 'require.resolve("jest/bin/jest")')"
         -c jest.config.integ.js --runInBand
         tests/index.integ.test.ts tests/optimization.integ.test.ts
+        tests/tsx.integ.test.ts

.github/workflows/ci.yml

@@ -60,6 +60,14 @@ jobs:
       - uses: ./.github/actions/test-native-abi
         with:
           node-version: "26"
+      # tsx registers its module hooks off-thread before Node 22.22.3 and
+      # 24.11.1, and Node only has `module.registerHooks` from 22.15
+      - uses: ./.github/actions/test-native-abi
+        with:
+          node-version: "22.14.0"
+      - uses: ./.github/actions/test-native-abi
+        with:
+          node-version: "24.10.0"
 
   list-examples:
     runs-on: "ubuntu-latest"

CLAUDE.md

@@ -115,4 +115,9 @@ so exercise all three of these when developing or reviewing a plugin change
 Running these locally outside the CodSpeed runner is expected to log
 `instrument-hooks: failed to write environment.json` and skip actual measurement
 writes — the point is to verify the plugin's control flow and output per mode,
-not to produce real measurements.
+not to produce real measurements.
+
+For real measurements and profiles, run under the runner without uploading:
+`CODSPEED_SKIP_UPLOAD=true codspeed run --mode <mode> --profile-folder <dir> -- <command>`.
+In simulation, `--simulation-track-subprocess` splits each benchmark's callgrind
+costs per thread, which exposes work from other threads counted in a benchmark.

examples/with-typescript-cjs/package.json

@@ -2,16 +2,16 @@
   "name": "with-typescript-cjs",
   "private": true,
   "scripts": {
-    "bench-benchmark-js": "node -r esbuild-register bench/benchmark.js/index.bench.ts",
-    "bench-tinybench": "node -r esbuild-register bench/tinybench/index.bench.ts"
+    "bench-benchmark-js": "node --import tsx bench/benchmark.js/index.bench.ts",
+    "bench-tinybench": "node --import tsx bench/tinybench/index.bench.ts"
   },
   "devDependencies": {
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
-    "esbuild-register": "^3.4.2",
     "tinybench": "^4.0.1",
+    "tsx": "^4.23.12",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-simple-cjs/package.json

@@ -2,16 +2,16 @@
   "name": "with-typescript-simple-cjs",
   "private": true,
   "scripts": {
-    "bench-benchmark-js": "node -r esbuild-register benchmark-js.ts",
-    "bench-tinybench": "node -r esbuild-register tinybench.ts"
+    "bench-benchmark-js": "tsx benchmark-js.ts",
+    "bench-tinybench": "tsx tinybench.ts"
   },
   "devDependencies": {
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
-    "esbuild-register": "^3.4.2",
     "tinybench": "^4.0.1",
+    "tsx": "^4.23.12",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-simple-esm/package.json

@@ -3,8 +3,8 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "bench-benchmark-js": "node --import tsx benchmark-js.ts",
-    "bench-tinybench": "node --import tsx tinybench.ts"
+    "bench-benchmark-js": "tsx benchmark-js.ts",
+    "bench-tinybench": "tsx tinybench.ts"
   },
   "devDependencies": {
     "@codspeed/benchmark.js-plugin": "workspace:*",

examples/with-typescript-simple-esm/tinybench.ts

@@ -1,6 +1,13 @@
 import { withCodSpeed } from "@codspeed/tinybench-plugin";
 import { Bench } from "tinybench";
 
+function fibonacci(n: number): number {
+  if (n < 2) {
+    return n;
+  }
+  return fibonacci(n - 1) + fibonacci(n - 2);
+}
+
 const bench = withCodSpeed(new Bench({ time: 100 }));
 
 bench
@@ -17,8 +24,11 @@ bench
     a = b + a;
     b = a - b;
     a = b - a;
+  })
+  .add("async fibonacci 15", async () => {
+    await Promise.resolve();
+    fibonacci(15);
   });
 
-bench.run().then(() => {
-  console.table(bench.table());
-});
+await bench.run();
+console.table(bench.table());

package.json

@@ -23,7 +23,6 @@
     "@typescript-eslint/eslint-plugin": "^8.57.0",
     "@typescript-eslint/parser": "^8.57.0",
     "esbuild": "^0.17.16",
-    "esbuild-register": "^3.4.2",
     "eslint": "^8.57.1",
     "eslint-import-resolver-typescript": "^3.6.3",
     "eslint-plugin-import": "^2.32.0",

packages/benchmark.js-plugin/package.json

@@ -22,7 +22,7 @@
     "typecheck": "tsc --noEmit --pretty",
     "format": "prettier --config ../../.prettierrc.json --ignore-path ../../.prettierignore --check .",
     "fix-format": "prettier --config ../../.prettierrc.json --ignore-path ../../.prettierignore --write .",
-    "bench": "node -r esbuild-register benches/sample.ts",
+    "bench": "node --import tsx benches/sample.ts",
     "clean": "rm -rf dist"
   },
   "author": "Arthur Pastel <arthur@codspeed.io>",
@@ -34,7 +34,8 @@
     "@types/benchmark": "^2.1.2",
     "@types/lodash": "^4.14.195",
     "benchmark": "^2.1.4",
-    "jest-mock-extended": "^3.0.4"
+    "jest-mock-extended": "^3.0.4",
+    "tsx": "^4.23.12"
   },
   "dependencies": {
     "@codspeed/core": "workspace:^6.1.0",

packages/core/package.json

@@ -36,7 +36,8 @@
     "node-addon-api": "^5.1.0",
     "node-gyp": "^12.2.0",
     "openapi-typescript-codegen": "^0.23.0",
-    "prebuildify": "^6.0.1"
+    "prebuildify": "^6.0.1",
+    "tsx": "^4.23.12"
   },
   "dependencies": {
     "axios": "^1.4.0",

packages/core/rollup.config.mjs

@@ -36,4 +36,9 @@ export default defineConfig([
       warn(warning);
     },
   },
+  {
+    input: "src/tsxLoader.ts",
+    output: { file: "dist/tsx-loader.cjs", format: "cjs", sourcemap: true },
+    plugins: jsPlugins(pkg.version),
+  },
 ]);

packages/core/src/introspection.ts

@@ -1,7 +1,11 @@
+import { spawnSync } from "child_process";
 import { writeFileSync } from "fs";
+import os from "os";
 import path from "path";
+import { isMainThread } from "worker_threads";
 
-import { getInstrumentMode } from "./runnerMode";
+import { getCodspeedRunnerMode, getInstrumentMode } from "./runnerMode";
+import { findTsxPreload } from "./tsx";
 
 const CUSTOM_INTROSPECTION_EXIT_CODE = 0;
 
@@ -13,6 +17,11 @@ export const getV8Flags = () => {
 
   const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"];
 
+  if (findTsxPreload(process.execArgv) !== undefined) {
+    // `--require` preloads run before `--import` ones, such as tsx's
+    flags.push(`--require=${path.join(__dirname, "tsx-loader.cjs")}`);
+  }
+
   switch (instrumentMode) {
     case "analysis": {
       flags.push(
@@ -70,17 +79,59 @@ export const getV8Flags = () => {
   return flags;
 };
 
+const RERUN_ENV = "__CODSPEED_NODE_CORE_RERUN__";
+const WHITESPACE = /\s/;
+
+const rerunWithFlags = (flags: string[]): never => {
+  const args = [...flags, ...process.execArgv, ...process.argv.slice(1)];
+  // `execve` keeps the pid, `spawnSync` makes it the rerun's parent pid
+  const env = { ...process.env, [RERUN_ENV]: String(process.pid) };
+  if (process.execve) {
+    process.execve(process.execPath, [process.execPath, ...args], env);
+  }
+  const { status, signal } = spawnSync(process.execPath, args, {
+    stdio: "inherit",
+    env,
+  });
+  process.exit(status ?? 128 + os.constants.signals[signal!]);
+};
+
 export const tryIntrospect = () => {
   if (process.env.__CODSPEED_NODE_CORE_INTROSPECTION_PATH__ !== undefined) {
     const introspectionMetadata = {
-      flags: getV8Flags(),
+      // The runner splits the flags on whitespace. Under tsx, the rerun below
+      // adds back the ones left out.
+      flags: getV8Flags().filter((flag) => !WHITESPACE.test(flag)),
     };
     writeFileSync(
       process.env.__CODSPEED_NODE_CORE_INTROSPECTION_PATH__,
       JSON.stringify(introspectionMetadata),
     );
     process.exit(CUSTOM_INTROSPECTION_EXIT_CODE);
   }
+
+  const isRerun = [process.pid, process.ppid]
+    .map(String)
+    .includes(process.env[RERUN_ENV] ?? "");
+  delete process.env[RERUN_ENV];
+  // The `tsx` CLI runs the script in a child node process, without the flags
+  // given to the CLI's own process. Rerun at most once, so a flag that never
+  // shows up in `execArgv` cannot loop, and only a process that can be
+  // replaced: not a worker thread, nor one with an IPC channel to its parent.
+  const canRerunUnderTsx =
+    !isRerun &&
+    isMainThread &&
+    process.channel === undefined &&
+    getCodspeedRunnerMode() !== "disabled" &&
+    findTsxPreload(process.execArgv) !== undefined &&
+    process.argv[1] !== undefined;
+  if (!canRerunUnderTsx) {
+    return;
+  }
+  const flags = getV8Flags();
+  if (flags.some((flag) => !process.execArgv.includes(flag))) {
+    rerunWithFlags(flags);
+  }
 };
 
 export const checkV8Flags = () => {

packages/core/src/tsx.ts

@@ -0,0 +1,48 @@
+import { createRequire } from "module";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const PRELOAD_FLAGS = ["--import", "--require", "-r", "--loader"];
+const TSX_PACKAGE_PATH = /[\\/]node_modules[\\/]tsx[\\/]/;
+
+const isBareTsxSpecifier = (specifier: string) =>
+  specifier === "tsx" || specifier.startsWith("tsx/");
+
+const isTsxSpecifier = (specifier: string) =>
+  isBareTsxSpecifier(specifier) || TSX_PACKAGE_PATH.test(specifier);
+
+/**
+ * Returns the tsx module preloaded by these node arguments: the `tsx`
+ * specifier of `node --import tsx`, or the loader path the `tsx` CLI passes to
+ * the node process running the script.
+ */
+export const findTsxPreload = (execArgv: string[]): string | undefined => {
+  for (const [i, arg] of execArgv.entries()) {
+    const separator = arg.indexOf("=");
+    const flag = separator === -1 ? arg : arg.slice(0, separator);
+    const value = separator === -1 ? execArgv[i + 1] : arg.slice(separator + 1);
+    if (
+      PRELOAD_FLAGS.includes(flag) &&
+      value !== undefined &&
+      isTsxSpecifier(value)
+    ) {
+      return value;
+    }
+  }
+  return undefined;
+};
+
+export const getTsxPackageDir = (tsxPreload: string): string => {
+  if (isBareTsxSpecifier(tsxPreload)) {
+    // Bare preload specifiers resolve from the working directory
+    const tsxPackageJson = createRequire(
+      path.join(process.cwd(), "index.js"),
+    ).resolve("tsx/package.json");
+    return path.dirname(tsxPackageJson);
+  }
+  const tsxPath = path.resolve(
+    tsxPreload.startsWith("file:") ? fileURLToPath(tsxPreload) : tsxPreload,
+  );
+  const match = TSX_PACKAGE_PATH.exec(tsxPath)!;
+  return tsxPath.slice(0, match.index + match[0].length - 1);
+};