scripts/correctness/test262-baseline-matrix.mjs
GitHub#!/usr/bin/env node
import { spawn, spawnSync } from "node:child_process";
import { existsSync, readFileSync } from "node:fs";
import { fileURLToPath, pathToFileURL } from "node:url";
import { join, relative, resolve } from "node:path";
import {
acceptTest262BaselineCandidate,
test262BaselineCandidatePath,
validateTest262BaselineCandidate,
} from "./test262-baseline.mjs";
const repoRoot = resolve(fileURLToPath(new URL("../..", import.meta.url)));
const roundTripScript = join(repoRoot, "scripts", "correctness", "test262-roundtrip.mjs");
export const normalBaselineProducers = ["terser-light", "swc-minify", "esbuild-minify"];
export const moduleGraphBaselineProducers = [
"none",
"babel-env-terser",
];
export const baselineProducers = unique([
...normalBaselineProducers,
...moduleGraphBaselineProducers,
]);
export const baselineSlices = [
"default",
"classes",
"destructuring",
"async-generators",
"scope",
"control-flow",
"calls",
"operators",
"templates",
"literals",
"block-scope-syntax",
"variables",
"assignment-target-type",
"arguments-object",
"identifiers",
"function-code",
"asi",
"keywords",
"reserved-words",
"modules",
];
export function parseMatrixArgs(argv) {
const options = {
dryRun: false,
missingOnly: false,
skipBuild: false,
producers: [],
slices: [],
limit: "all",
test262Root: null,
level: null,
knownBlockers: null,
caseTimeoutMs: "15000",
toolRoot: null,
details: false,
keepTemp: false,
updateBaselines: false,
acceptCandidates: false,
};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === "--dry-run") {
options.dryRun = true;
} else if (arg === "--missing") {
options.missingOnly = true;
} else if (arg === "--skip-build") {
options.skipBuild = true;
} else if (arg === "--producer") {
options.producers.push(readRequiredValue(argv, ++i, arg));
} else if (arg === "--slice") {
options.slices.push(readRequiredValue(argv, ++i, arg));
} else if (arg === "--limit") {
options.limit = readRequiredValue(argv, ++i, arg);
} else if (arg === "--test262") {
options.test262Root = readRequiredValue(argv, ++i, arg);
} else if (arg === "--level") {
options.level = readRequiredValue(argv, ++i, arg);
} else if (arg === "--known-blockers") {
options.knownBlockers = readRequiredValue(argv, ++i, arg);
} else if (arg === "--case-timeout-ms") {
options.caseTimeoutMs = readRequiredValue(argv, ++i, arg);
} else if (arg === "--tool-root") {
options.toolRoot = readRequiredValue(argv, ++i, arg);
} else if (arg === "--details") {
options.details = true;
} else if (arg === "--keep-temp") {
options.keepTemp = true;
} else if (arg === "--update") {
options.updateBaselines = true;
} else if (arg === "--accept") {
options.acceptCandidates = true;
} else if (arg === "--help" || arg === "-h") {
options.help = true;
} else {
throw new Error(`unknown option: ${arg}`);
}
}
validateRequestedValues("--producer", options.producers, baselineProducers);
validateRequestedValues("--slice", options.slices, [...baselineSlices, "module-graph"]);
if (options.acceptCandidates && options.updateBaselines) {
throw new Error("--accept cannot be combined with --update");
}
if (options.acceptCandidates && options.missingOnly) {
throw new Error("--accept cannot be combined with --missing");
}
return options;
}
export function buildBaselineMatrixJobs(options = {}) {
const producers = unique(options.producers?.length ? options.producers : baselineProducers);
const requestedSlices = unique(options.slices ?? []);
const slices = requestedSlices.length > 0
? requestedSlices.filter((slice) => slice !== "module-graph")
: baselineSlices;
const limit = options.limit ?? "all";
const jobs = producers.flatMap((producer) => {
const producerJobs = normalBaselineProducers.includes(producer)
? slices.map((slice) => createMatrixJob({ producer, slice, preset: slice, limit, options }))
: [];
if (
moduleGraphBaselineProducers.includes(producer) &&
(requestedSlices.length === 0 || requestedSlices.includes("module-graph"))
) {
producerJobs.push(
createMatrixJob({
producer,
slice: "module-graph",
preset: "modules",
limit,
options,
}),
);
}
return producerJobs;
});
if (options.missingOnly) {
return jobs.filter((job) => !isCompleteSummary(job.summary));
}
return jobs;
}
function createMatrixJob({ producer, slice, preset, limit, options }) {
const outputDir = slice === "module-graph"
? join(repoRoot, "docs", "test262-baselines", "module-graph")
: join(repoRoot, "docs", "test262-baselines", producer);
const outputName = slice === "module-graph" ? producer : slice;
const summary = join(outputDir, `${outputName}.md`);
const baseline = join(outputDir, `${outputName}.json`);
const candidate = test262BaselineCandidatePath(baseline);
const args = [
roundTripScript,
"--preset",
preset,
"--pipeline",
producer,
"--limit",
String(limit),
"--summary",
summary,
"--baseline",
baseline,
];
if (options.updateBaselines) args.push("--update-baseline");
pushOptionalPair(args, "--test262", options.test262Root);
pushOptionalPair(args, "--level", options.level);
pushOptionalPair(args, "--known-blockers", options.knownBlockers);
pushOptionalPair(args, "--case-timeout-ms", options.caseTimeoutMs);
pushOptionalPair(args, "--tool-root", options.toolRoot);
if (options.details) args.push("--details");
if (options.keepTemp) args.push("--keep-temp");
return {
producer,
slice,
summary,
baseline,
candidate,
command: process.execPath,
args,
};
}
export function formatCommand(job) {
return [job.command, ...job.args].map(shellQuote).join(" ");
}
function defaultConcurrency() {
return 1;
}
async function runPool(items, worker, concurrency = defaultConcurrency()) {
let cursor = 0;
const run = async () => {
while (cursor < items.length) {
const index = cursor++;
await worker(items[index], index);
}
};
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run));
}
function spawnJobAsync(job) {
return new Promise((resolvePromise) => {
const child = spawn(job.command, job.args, {
cwd: repoRoot,
stdio: ["ignore", "pipe", "pipe"],
});
let stderr = "";
child.stdout.resume();
child.stderr.on("data", (chunk) => (stderr += chunk));
child.on("error", (err) => resolvePromise({ code: 1, stderr: err.message }));
child.on("close", (code) => resolvePromise({ code: code ?? 1, stderr }));
});
}
export async function runBaselineMatrix(options) {
const jobs = buildBaselineMatrixJobs(options);
if (jobs.length === 0) {
console.error("No Test262 baseline jobs match the selected producers and slices.");
return 1;
}
if (options.acceptCandidates) {
if (options.dryRun) {
for (const job of jobs) {
console.log(`ACCEPT ${relative(repoRoot, job.candidate)} -> ${relative(repoRoot, job.baseline)}`);
}
return 0;
}
const accepted = acceptBaselineMatrixCandidates(jobs);
for (const item of accepted) {
console.log(`ACCEPTED ${item.producer} / ${item.slice} -> ${relative(repoRoot, item.baseline)}`);
}
if (accepted.length === 0) {
console.error("No Test262 baseline candidates found for the selected jobs.");
return 1;
}
return 0;
}
if (options.dryRun) {
for (const job of jobs) {
console.log(formatCommand(job));
}
return 0;
}
if (!options.skipBuild && !process.env.WAKARU) {
console.log("Building wakaru debug binary...");
const result = spawnSync("cargo", ["build", "-p", "wakaru-cli"], {
cwd: repoRoot,
stdio: "inherit",
});
if (result.status !== 0) {
return result.status ?? 1;
}
}
const concurrency = defaultConcurrency();
let failCount = 0;
console.log(`Running ${jobs.length} jobs with concurrency ${concurrency}...`);
await runPool(jobs, async (job) => {
const displaySummary = relative(repoRoot, job.summary);
const { code, stderr } = await spawnJobAsync(job);
if (code !== 0) {
console.log(`FAIL ${job.producer} / ${job.slice} -> ${displaySummary} (exit ${code})`);
if (stderr.trim()) console.log(stderr.trim());
failCount++;
} else {
console.log(`DONE ${job.producer} / ${job.slice} -> ${displaySummary}`);
}
}, concurrency);
return failCount > 0 ? 1 : 0;
}
export function acceptBaselineMatrixCandidates(jobs) {
const accepted = jobs.filter((job) => existsSync(job.candidate));
for (const job of accepted) {
validateTest262BaselineCandidate(job.baseline);
}
for (const job of accepted) {
acceptTest262BaselineCandidate(job.baseline);
}
return accepted;
}
export function usage() {
return `Usage:
node scripts/correctness/test262-baseline-matrix.mjs [options]
Options:
--producer <name> Producer to run. Repeatable. Default: ${baselineProducers.join(", ")}
--slice <name> Test262 slice to run. Repeatable; also accepts module-graph
--limit <n|all> Runnable test limit passed through. Default: all
--test262 <dir> Test262 checkout passed through
--level <level> Wakaru rewrite level passed through
--known-blockers <file> Known blocker manifest passed through
--case-timeout-ms <n> Per-test timeout. Default: 15000
--tool-root <dir> Tool package directory passed through
--details Print detailed round-trip output
--keep-temp Keep temporary round-trip files
--missing Run only missing or incomplete summaries
--update Explicitly rewrite selected JSON baselines and summaries
--accept Accept selected .json.new candidates without rerunning tests
--skip-build Do not build wakaru before running jobs
--dry-run Print commands without running them
`;
}
function readRequiredValue(argv, index, flag) {
const value = argv[index];
if (value == null || value.startsWith("-")) {
throw new Error(`${flag} requires a value`);
}
return value;
}
function validateRequestedValues(flag, requested, supported) {
for (const value of requested) {
if (!supported.includes(value)) {
throw new Error(`unsupported ${flag} ${value}`);
}
}
}
function pushOptionalPair(args, flag, value) {
if (value != null) {
args.push(flag, String(value));
}
}
function unique(values) {
return [...new Set(values)];
}
function isCompleteSummary(path) {
return existsSync(path) && /^- complete: true$/m.test(readFileSync(path, "utf8"));
}
function shellQuote(value) {
if (/^[A-Za-z0-9_./:\\-]+$/.test(value)) {
return value;
}
return `"${value.replaceAll('"', '\\"')}"`;
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
try {
const options = parseMatrixArgs(process.argv.slice(2));
if (options.help) {
console.log(usage());
process.exit(0);
}
process.exitCode = await runBaselineMatrix(options);
} catch (error) {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
}
}