scripts/correctness/test262-collect-stats.mjs
GitHub#!/usr/bin/env node
// Reads committed test262 baseline summaries from docs/test262-baselines/ and
// writes a compact stats file to scripts/correctness/test262-stats.json.
// Does NOT re-run tests — it parses the markdown summaries that are already
// checked in. Re-run baselines first if you want fresh numbers.
//
// Usage:
// node scripts/correctness/test262-collect-stats.mjs # update all
// node scripts/correctness/test262-collect-stats.mjs --producer swc-minify # update one producer
// node scripts/correctness/test262-collect-stats.mjs --slice classes --slice scope # update specific slices
// node scripts/correctness/test262-collect-stats.mjs --check # exit non-zero if stale
//
// When --producer or --slice is given, only matching entries are re-collected;
// the rest are kept from the existing stats file.
import { execSync } from "node:child_process";
import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { join, dirname, basename } from "node:path";
import { fileURLToPath } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const repoRoot = join(scriptDir, "../..");
const baselinesDir = join(repoRoot, "docs", "test262-baselines");
const statsPath = join(scriptDir, "test262-stats.json");
const allProducers = ["terser-light", "swc-minify", "esbuild-minify"];
function parseArgs(argv) {
const options = { producers: [], slices: [], check: false };
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (arg === "--check") {
options.check = true;
} else if (arg === "--producer") {
const value = argv[++i];
if (!value || value.startsWith("-")) throw new Error("--producer requires a value");
if (!allProducers.includes(value)) throw new Error(`unknown producer: ${value}`);
options.producers.push(value);
} else if (arg === "--slice") {
const value = argv[++i];
if (!value || value.startsWith("-")) throw new Error("--slice requires a value");
options.slices.push(value);
} else {
throw new Error(`unknown option: ${arg}`);
}
}
return options;
}
function gitCommit() {
try {
return execSync("git rev-parse --short HEAD", { cwd: repoRoot, encoding: "utf8" }).trim();
} catch {
return "unknown";
}
}
function parseSummaryTotals(content) {
const match = content.match(
/\|\s*(\d+)\s*\|\s*(\d+)\s*\|\s*(\d+)\s*\|\s*(\d+)\s*\|\s*(\d+)\s*\|\s*(\d+)\s*\|\s*(\d+)\s*\|/,
);
if (!match) return null;
return {
discovered: Number(match[1]),
runnable: Number(match[2]),
skipped: Number(match[3]),
unsupported: Number(match[4]),
rejected: Number(match[5]),
passed: Number(match[6]),
failed: Number(match[7]),
};
}
function isComplete(content) {
return /^- complete: true$/m.test(content);
}
function collectSlices(producerDir) {
if (!existsSync(producerDir)) return [];
return readdirSync(producerDir)
.filter((f) => f.endsWith(".md"))
.map((f) => basename(f, ".md"))
.sort();
}
function loadExistingStats() {
try {
return JSON.parse(readFileSync(statsPath, "utf8"));
} catch {
return null;
}
}
function entryKey(producer, slice) {
return `${producer}\0${slice}`;
}
function recomputeAggregate(entries) {
let runnable = 0;
let passed = 0;
let failed = 0;
for (const entry of entries) {
runnable += entry.runnable;
passed += entry.passed;
failed += entry.failed;
}
const decidable = passed + failed;
const pct = decidable > 0 ? +((passed / decidable) * 100).toFixed(1) : 0;
return {
producers: allProducers.length,
entries: entries.length,
runnable,
passed,
failed,
pct,
};
}
const options = parseArgs(process.argv.slice(2));
const filterProducers = options.producers.length > 0 ? options.producers : allProducers;
const filterSlices = new Set(options.slices);
const isFiltered = options.producers.length > 0 || options.slices.length > 0;
console.log(
isFiltered
? `Collecting test262 baseline stats (${filterProducers.join(", ")}${filterSlices.size > 0 ? ` / ${[...filterSlices].join(", ")}` : ""})...`
: "Collecting test262 baseline stats...",
);
const freshEntries = new Map();
for (const producer of filterProducers) {
const dir = join(baselinesDir, producer);
let slices = collectSlices(dir);
if (filterSlices.size > 0) {
slices = slices.filter((s) => filterSlices.has(s));
}
for (const slice of slices) {
const summaryPath = join(dir, `${slice}.md`);
const content = readFileSync(summaryPath, "utf8");
const complete = isComplete(content);
const totals = parseSummaryTotals(content);
if (!totals) {
console.error(` ${producer}/${slice}: could not parse totals`);
continue;
}
if (!complete) {
console.error(` ${producer}/${slice}: incomplete`);
}
const entry = { producer, slice, complete, ...totals };
freshEntries.set(entryKey(producer, slice), entry);
process.stdout.write(` ${producer}/${slice}: ${totals.passed}/${totals.runnable}`);
console.log(totals.failed > 0 ? ` (${totals.failed} FAILED)` : "");
}
}
// Merge: start from existing entries, overwrite with freshly collected ones.
const existing = loadExistingStats();
const mergedMap = new Map();
if (isFiltered && existing) {
for (const entry of existing.entries) {
mergedMap.set(entryKey(entry.producer, entry.slice), entry);
}
}
for (const [key, entry] of freshEntries) {
mergedMap.set(key, entry);
}
const mergedEntries = [...mergedMap.values()].sort(
(a, b) => a.producer.localeCompare(b.producer) || a.slice.localeCompare(b.slice),
);
const aggregate = recomputeAggregate(mergedEntries);
const stats = {
commit: gitCommit(),
date: new Date().toISOString().slice(0, 10),
aggregate,
entries: mergedEntries,
};
const json = JSON.stringify(stats, null, 2) + "\n";
if (options.check) {
if (!existing) {
console.error("\ntest262-stats.json does not exist. Run without --check to create it.");
process.exit(1);
}
const existingNormalized = { ...existing, commit: stats.commit, date: stats.date };
if (JSON.stringify(existingNormalized) !== JSON.stringify(stats)) {
console.error("\ntest262-stats.json is stale. Run `node scripts/correctness/test262-collect-stats.mjs` to update.");
process.exit(1);
}
console.log(`\ntest262-stats.json is up to date: ${aggregate.passed}/${aggregate.passed + aggregate.failed} (${aggregate.pct}%)`);
} else {
writeFileSync(statsPath, json);
console.log(`\nWrote ${statsPath}: ${aggregate.passed}/${aggregate.passed + aggregate.failed} (${aggregate.pct}%)`);
}