repomix-mirror/.github/scripts/perf-benchmark/bench-run.mjs

import { execFileSync } from 'node:child_process';
import { writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';

const [prDir, mainDir] = process.argv.slice(2);
const output = join(tmpdir(), 'repomix-bench-output.txt');
const runs = Number(process.env.BENCH_RUNS) || 20;

const prBin = join(prDir, 'bin', 'repomix.cjs');
const mainBin = join(mainDir, 'bin', 'repomix.cjs');

function run(bin, dir) {
  const start = Date.now();
  execFileSync(process.execPath, [bin, dir, '--output', output], { stdio: 'ignore' });
  return Date.now() - start;
}

// Warmup both branches to stabilize OS page cache and JIT
console.error('Warming up...');
for (let i = 0; i < 2; i++) {
  try {
    run(prBin, prDir);
  } catch {}
  try {
    run(mainBin, mainDir);
  } catch {}
}

// Interleaved measurement: alternate PR and main each iteration
// so both branches experience similar runner load conditions.
// Even/odd alternation neutralizes ordering bias from CPU/filesystem cache warming.
const prTimes = [];
const mainTimes = [];
for (let i = 0; i < runs; i++) {
  console.error(`Run ${i + 1}/${runs}`);
  if (i % 2 === 0) {
    try {
      prTimes.push(run(prBin, prDir));
    } catch (e) {
      console.error(`PR run ${i + 1} failed: ${e.message}`);
    }
    try {
      mainTimes.push(run(mainBin, mainDir));
    } catch (e) {
      console.error(`main run ${i + 1} failed: ${e.message}`);
    }
  } else {
    try {
      mainTimes.push(run(mainBin, mainDir));
    } catch (e) {
      console.error(`main run ${i + 1} failed: ${e.message}`);
    }
    try {
      prTimes.push(run(prBin, prDir));
    } catch (e) {
      console.error(`PR run ${i + 1} failed: ${e.message}`);
    }
  }
}

if (prTimes.length === 0 || mainTimes.length === 0) {
  console.error('All benchmark runs failed');
  process.exit(1);
}

function stats(times) {
  times.sort((a, b) => a - b);
  const median = times[Math.floor(times.length / 2)];
  const q1 = times[Math.floor(times.length * 0.25)];
  const q3 = times[Math.floor(times.length * 0.75)];
  return { median, iqr: q3 - q1 };
}

const pr = stats(prTimes);
const main = stats(mainTimes);

console.error(`PR median: ${pr.median}ms (±${pr.iqr}ms)`);
console.error(`main median: ${main.median}ms (±${main.iqr}ms)`);

const result = { pr: pr.median, prIqr: pr.iqr, main: main.median, mainIqr: main.iqr };
writeFileSync(join(process.env.RUNNER_TEMP, 'bench-result.json'), JSON.stringify(result));