scripts/repro/collect-stats.mjs

GitHub
#!/usr/bin/env node

// Runs every reproduction matrix with --json and writes a summary to
// scripts/repro/stats.json.  Re-run after rule changes to update the
// checked-in baseline so other sessions can read it without re-running
// all matrices (~2 min).
//
// Usage:
//   node scripts/repro/collect-stats.mjs            # update stats.json
//   node scripts/repro/collect-stats.mjs --check     # exit non-zero if stats.json is stale

import { execSync, spawnSync } from "node:child_process";
import { readFileSync, writeFileSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";

const scriptDir = dirname(fileURLToPath(import.meta.url));
const repoRoot = join(scriptDir, "../..");
const statsPath = join(scriptDir, "stats.json");

const matrices = [
  "array-spread-rest",
  "async-await",
  "conditional-switch",
  "enum",
  "for-of-iteration",
  "object-rest-spread",
  "optional-nullish",
  "parameters",
  "swc-minifier",
  "template-literal",
];

function gitCommit() {
  try {
    return execSync("git rev-parse --short HEAD", { cwd: repoRoot, encoding: "utf8" }).trim();
  } catch {
    return "unknown";
  }
}

function runMatrix(name) {
  const script = join(scriptDir, `${name}-matrix/matrix.mjs`);
  const result = spawnSync("node", [script, "--json"], {
    cwd: repoRoot,
    encoding: "utf8",
    maxBuffer: 1024 * 1024 * 50,
    timeout: 5 * 60 * 1000,
    stdio: ["pipe", "pipe", "pipe"],
  });
  if (result.status !== 0) {
    console.error(`  ${name}: failed (exit ${result.status})`);
    return null;
  }
  try {
    const data = JSON.parse(result.stdout);
    const matrix = {
      name,
      yes: data.summary.yes,
      no: data.summary.no,
      error: data.summary.error ?? 0,
      total: data.summary.yes + data.summary.no,
      pct: data.summary.pct,
    };
    const errorSamples = (data.rows ?? [])
      .filter((row) => row.status !== "yes" && row.status !== "no")
      .slice(0, 3)
      .map((row) => ({
        snippet: row.snippet,
        tools: row.tools,
        status: row.status,
        notes: row.notes,
      }));
    Object.defineProperty(matrix, "errorSamples", { value: errorSamples });
    return matrix;
  } catch {
    console.error(`  ${name}: invalid JSON output`);
    return null;
  }
}

function comparableStats(stats) {
  return { aggregate: stats.aggregate, matrices: stats.matrices };
}

function formatMatrix(matrix) {
  if (!matrix) return "<missing>";
  return `${matrix.yes}/${matrix.total} (${matrix.pct}%), no=${matrix.no}, error=${matrix.error ?? 0}`;
}

function formatValue(value) {
  return value === undefined ? "<missing>" : JSON.stringify(value);
}

function formatErrorSample(sample) {
  const tools = Array.isArray(sample.tools) ? sample.tools.join(", ") : "<unknown tool>";
  const note = String(sample.notes ?? "").replace(/\s+/g, " ").trim();
  return `${sample.snippet ?? "<unknown snippet>"} / ${tools} / ${sample.status}: ${note}`;
}

function printStatsDiff(recorded, measured) {
  console.error("  matrix diffs:");

  let printed = false;
  const recordedMatrices = new Map((recorded.matrices ?? []).map((matrix) => [matrix.name, matrix]));
  const measuredMatrices = new Map((measured.matrices ?? []).map((matrix) => [matrix.name, matrix]));
  const names = [
    ...matrices,
    ...[...recordedMatrices.keys()].filter((name) => !matrices.includes(name)),
    ...[...measuredMatrices.keys()].filter((name) => !matrices.includes(name) && !recordedMatrices.has(name)),
  ];

  const aggregateFields = ["yes", "total", "pct"];
  const aggregateDiffs = aggregateFields
    .filter((field) => recorded.aggregate?.[field] !== measured.aggregate?.[field])
    .map((field) => `${field}: ${formatValue(recorded.aggregate?.[field])} -> ${formatValue(measured.aggregate?.[field])}`);
  if (aggregateDiffs.length > 0) {
    printed = true;
    console.error(`    aggregate: ${aggregateDiffs.join(", ")}`);
  }

  for (const name of names) {
    const before = recordedMatrices.get(name);
    const after = measuredMatrices.get(name);
    if (!before || !after) {
      printed = true;
      console.error(`    ${name}: recorded ${formatMatrix(before)}; measured ${formatMatrix(after)}`);
      continue;
    }

    const diffs = ["yes", "no", "error", "total", "pct"]
      .filter((field) => (before[field] ?? 0) !== (after[field] ?? 0))
      .map((field) => `${field}: ${formatValue(before[field] ?? 0)} -> ${formatValue(after[field] ?? 0)}`);
    if (diffs.length > 0) {
      printed = true;
      console.error(`    ${name}: recorded ${formatMatrix(before)}; measured ${formatMatrix(after)}; ${diffs.join(", ")}`);
      for (const sample of after.errorSamples ?? []) {
        console.error(`      measured error sample: ${formatErrorSample(sample)}`);
      }
    }
  }

  if (!printed) {
    console.error("    no scalar field differences found; compare stats.json ordering or extra fields");
  }
}

const checkMode = process.argv.includes("--check");

console.log("Running reproduction matrices...");
const results = [];
for (const name of matrices) {
  process.stdout.write(`  ${name}...`);
  const result = runMatrix(name);
  if (result) {
    const errorSuffix = result.error ? ` / ${result.error} error` : "";
    console.log(` ${result.yes}/${result.total}${errorSuffix} (${result.pct}%)`);
    results.push(result);
  }
}

const yes = results.reduce((s, r) => s + r.yes, 0);
const total = results.reduce((s, r) => s + r.total, 0);
const pct = total > 0 ? +((yes / total) * 100).toFixed(1) : 0;

const stats = {
  commit: gitCommit(),
  date: new Date().toISOString().slice(0, 10),
  aggregate: { yes, total, pct },
  matrices: results,
};

const json = JSON.stringify(stats, null, 2) + "\n";

if (checkMode) {
  let existing;
  try {
    existing = JSON.parse(readFileSync(statsPath, "utf8"));
  } catch {
    console.error("\nstats.json is missing or invalid. Run without --check to create it.");
    process.exit(1);
  }
  // Compare only the measured numbers. The commit/date fields are
  // provenance: they change on every commit and every day, so including
  // them would make --check permanently stale anywhere but the machine
  // that just regenerated the file (e.g. always-failing in CI).
  const measured = (s) => JSON.stringify(comparableStats(s), null, 2);
  if (measured(existing) !== measured(stats)) {
    console.error("\nstats.json is stale. Run `node scripts/repro/collect-stats.mjs` to update.");
    console.error(`  recorded:  ${existing.aggregate?.yes}/${existing.aggregate?.total} (${existing.aggregate?.pct}%)`);
    console.error(`  measured:  ${yes}/${total} (${pct}%)`);
    printStatsDiff(comparableStats(existing), comparableStats(stats));
    process.exit(1);
  }
  console.log(`\nstats.json is up to date: ${yes}/${total} (${pct}%)`);
} else {
  writeFileSync(statsPath, json);
  console.log(`\nWrote ${statsPath}: ${yes}/${total} (${pct}%)`);
}