scripts/correctness/test262-baseline.mjs

GitHub
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname } from "node:path";

export const test262BaselineSchemaVersion = 3;
const runtimeTempRoot = tmpdir().replaceAll("\\", "/").replace(/\/$/, "");
const runtimeTempPathPattern = new RegExp(
  `${escapeRegExp(runtimeTempRoot)}(?:/[^\\s:]+)?`,
  "g",
);

export function createTest262Baseline(report) {
  if (!report.complete) {
    throw new Error("cannot create a Test262 baseline from an incomplete report");
  }
  const outcomes = report.results
    .filter((result) => result.status !== "passed")
    .map(normalizeOutcome)
    .sort(compareOutcome);
  return {
    schemaVersion: test262BaselineSchemaVersion,
    ...createTest262BaselineIdentity(report.options),
    totals: baselineTotals(report.totals),
    outcomes,
  };
}

export function preflightTest262Baseline(path, options) {
  if (!existsSync(path)) {
    return false;
  }
  const expected = loadTest262Baseline(path);
  assertSameIdentity(expected, createTest262BaselineIdentity(options));
  return true;
}

export function loadTest262Baseline(path) {
  if (!existsSync(path)) {
    throw new Error(`missing Test262 baseline ${path}; rerun with --update-baseline`);
  }
  const baseline = JSON.parse(readFileSync(path, "utf8"));
  if (baseline.schemaVersion !== test262BaselineSchemaVersion) {
    throw new Error(
      `unsupported Test262 baseline schema ${baseline.schemaVersion} in ${path}`,
    );
  }
  return baseline;
}

export function saveTest262Baseline(path, baseline) {
  mkdirSync(dirname(path), { recursive: true });
  writeFileSync(path, `${JSON.stringify(baseline, null, 2)}\n`);
}

export function test262BaselineCandidatePath(path) {
  return `${path}.new`;
}

export function validateTest262BaselineCandidate(path) {
  const candidatePath = test262BaselineCandidatePath(path);
  const candidate = loadTest262Baseline(candidatePath);
  const reviewedBaselineSha256 = candidate._candidate?.reviewedBaselineSha256;
  if (
    !(reviewedBaselineSha256 === null ||
      (typeof reviewedBaselineSha256 === "string" &&
        /^[0-9a-f]{64}$/.test(reviewedBaselineSha256)))
  ) {
    throw new Error(
      `Test262 baseline candidate ${candidatePath} has no valid freshness metadata; rerun the comparison`,
    );
  }
  const currentBaselineSha256 = baselineFileSha256(path);
  if (reviewedBaselineSha256 !== currentBaselineSha256) {
    throw new Error(
      `stale Test262 baseline candidate ${candidatePath}: reviewed baseline changed after the comparison; rerun it`,
    );
  }
  const baseline = { ...candidate };
  delete baseline._candidate;
  return { path, candidatePath, baseline };
}

export function acceptTest262BaselineCandidate(path) {
  const validated = validateTest262BaselineCandidate(path);
  saveTest262Baseline(path, validated.baseline);
  const { candidatePath } = validated;
  rmSync(candidatePath);
  return validated;
}

export function compareTest262Baseline(expected, actual) {
  assertSameIdentity(expected, actual);
  const expectedByKey = new Map(expected.outcomes.map((outcome) => [outcomeKey(outcome), outcome]));
  const actualByKey = new Map(actual.outcomes.map((outcome) => [outcomeKey(outcome), outcome]));
  const newOutcomes = actual.outcomes.filter((outcome) => !expectedByKey.has(outcomeKey(outcome)));
  const unexpectedPasses = expected.outcomes.filter(
    (outcome) => !actualByKey.has(outcomeKey(outcome)),
  );
  const totalsChanged = canonicalJson(expected.totals) !== canonicalJson(actual.totals);
  return {
    clean: newOutcomes.length === 0 && unexpectedPasses.length === 0 && !totalsChanged,
    newOutcomes,
    unexpectedPasses,
    totalsChanged,
    expectedTotals: expected.totals,
    actualTotals: actual.totals,
  };
}

export function applyTest262Baseline(report, { path, update }) {
  const actual = createTest262Baseline(report);
  const candidatePath = test262BaselineCandidatePath(path);
  if (update) {
    saveTest262Baseline(path, actual);
    rmSync(candidatePath, { force: true });
    return {
      path,
      candidatePath: null,
      updated: true,
      clean: true,
      newOutcomes: [],
      unexpectedPasses: [],
      totalsChanged: false,
    };
  }
  const reviewedBaselineSha256 = baselineFileSha256(path);
  let comparison;
  try {
    const expected = loadTest262Baseline(path);
    comparison = compareTest262Baseline(expected, actual);
  } catch (error) {
    saveTest262BaselineCandidate(candidatePath, actual, reviewedBaselineSha256);
    const message = error instanceof Error ? error.message : String(error);
    throw new Error(`${message}\nCandidate baseline written to ${candidatePath}`, {
      cause: error,
    });
  }
  if (comparison.clean) {
    rmSync(candidatePath, { force: true });
  } else {
    saveTest262BaselineCandidate(candidatePath, actual, reviewedBaselineSha256);
  }
  return {
    path,
    candidatePath: comparison.clean ? null : candidatePath,
    updated: false,
    ...comparison,
  };
}

function saveTest262BaselineCandidate(path, baseline, reviewedBaselineSha256) {
  saveTest262Baseline(path, {
    ...baseline,
    _candidate: { reviewedBaselineSha256 },
  });
}

function baselineFileSha256(path) {
  return existsSync(path) ? sha256(readFileSync(path)) : null;
}

export function validateTest262BaselineOptions(options) {
  if (options.updateBaseline && !options.baseline) {
    throw new Error("--update-baseline requires --baseline <file>");
  }
  if (!options.baseline) {
    return;
  }
  if (Number.isFinite(options.limit)) {
    throw new Error("baseline comparison requires --limit all");
  }
  if (options.rerunFrom) {
    throw new Error("filtered --rerun-from runs cannot compare or update a complete baseline");
  }
  if (!options.presets || options.presets.length !== 1) {
    throw new Error("baseline comparison requires exactly one --preset");
  }
  if (!options.updateBaseline && !existsSync(options.baseline)) {
    throw new Error(`missing Test262 baseline ${options.baseline}; rerun with --update-baseline`);
  }
}

export function fingerprintTest262Outcome(result) {
  const evidence = {
    status: result.status,
    phase: result.phase ?? null,
    reason: result.reason ?? null,
    diagnostic: stableDiagnostic(result.error),
    transformed: hashEvidence(result.transformed),
    decompiled: hashEvidence(result.decompiled),
  };
  return sha256(canonicalJson(evidence));
}

function normalizeOutcome(result) {
  return {
    path: result.path,
    variant: result.variant ?? result.variants?.join(",") ?? "case",
    status: result.status,
    kind: result.reason ?? result.phase ?? result.status,
    fingerprint: fingerprintTest262Outcome(result),
    summary: stableDiagnostic(result.error) || result.reason || result.phase || result.status,
  };
}

function stableDiagnostic(value) {
  if (!value) {
    return "";
  }
  const lines = String(value)
    .replace(/\u001b\[[0-9;]*m/g, "")
    .split(/\r?\n/)
    .filter((line) => !/^\s+at\s/.test(line))
    .map((line) => line.replaceAll("\\", "/"))
    .map((line) => line.replace(/\/(?:private\/)?var\/folders\/[^\s:]+/g, "<tmp>"))
    .map((line) => line.replace(/\/tmp\/[^\s:]+/g, "<tmp>"))
    .map((line) =>
      line.replace(
        /[A-Za-z]:\/Users\/[^/\s:]+\/AppData\/Local\/Temp(?:\/[^\s:]+)?/gi,
        "<tmp>",
      ),
    )
    .map((line) => line.replace(runtimeTempPathPattern, "<tmp>"))
    .map((line) => line.replace(/file:\/{2,3}<tmp>/g, "file://<tmp>"));
  const typedError = lines.find((line) => /^[A-Za-z_$][\w$]*Error(?::|$)/.test(line));
  if (typedError) {
    return typedError;
  }
  return lines
    .filter((line) => line.trim().length > 0 && !/^\s*\^+\s*$/.test(line))
    .slice(0, 3)
    .join("\n");
}

function escapeRegExp(value) {
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

function hashEvidence(value) {
  if (value == null) {
    return null;
  }
  return sha256(canonicalJson(value));
}

function baselineTotals(totals) {
  return {
    discovered: totals.discovered,
    runnable: totals.runnable,
    skipped: totals.skipped,
    unsupported: totals.unsupported,
    rejected: totals.rejected,
    passed: totals.passed,
    failed: totals.failed,
  };
}

function assertSameIdentity(expected, actual) {
  for (const [label, left, right] of [
    ["Test262 revision", expected.test262, actual.test262],
    ["harness version", expected.harness, actual.harness],
    ["runtime environment", expected.environment, actual.environment],
    ["producer", expected.producer, actual.producer],
    ["Wakaru options", expected.wakaru, actual.wakaru],
    ["selection", expected.selection, actual.selection],
  ]) {
    if (canonicalJson(left) !== canonicalJson(right)) {
      throw new Error(
        `Test262 baseline ${label} mismatch: expected ${canonicalJson(left)}, actual ${canonicalJson(right)}`,
      );
    }
  }
}

function createTest262BaselineIdentity(options) {
  return {
    test262: {
      revision: options.test262Revision,
    },
    harness: {
      version: options.harnessVersion,
    },
    environment: {
      nodeMajor: options.nodeMajor,
    },
    producer: options.producer,
    wakaru: {
      level: options.level,
      caseTimeoutMs: options.caseTimeoutMs,
    },
    selection: {
      presets: options.presets ?? [],
      paths: options.paths,
    },
  };
}

function outcomeKey(outcome) {
  return [
    outcome.path,
    outcome.variant,
    outcome.status,
    outcome.kind,
    outcome.fingerprint,
  ].join("\0");
}

function compareOutcome(left, right) {
  return compareCodeUnits(outcomeKey(left), outcomeKey(right));
}

function canonicalJson(value) {
  return JSON.stringify(canonicalize(value));
}

function canonicalize(value) {
  if (Array.isArray(value)) {
    return value.map(canonicalize);
  }
  if (value && typeof value === "object") {
    return Object.fromEntries(
      Object.entries(value)
        .sort(([left], [right]) => compareCodeUnits(left, right))
        .map(([key, child]) => [key, canonicalize(child)]),
    );
  }
  return value;
}

function compareCodeUnits(left, right) {
  return left < right ? -1 : left > right ? 1 : 0;
}

function sha256(value) {
  return createHash("sha256").update(value).digest("hex");
}