sandbox-analyze.ts
GitHub#!/usr/bin/env bun
/**
* Sandbox benchmark analyzer: reads artifacts from sandbox-runner output,
* extracts debug logs and claim dirs, and produces a coverage report.
*
* Usage:
* bun run .claude/skills/benchmark-sandbox/sandbox-analyze.ts [options]
*
* --run-dir <path> Path to a specific run directory (default: latest in sandbox-results/)
* --json Output JSON report instead of human-readable
* --help Print usage and exit
*/
import { readFile, readdir, stat } from "node:fs/promises";
import { join, resolve } from "node:path";
import { parseArgs } from "node:util";
import { homedir } from "node:os";
import type { RunSummary, ScenarioSummary } from "./types.js";
// ---------------------------------------------------------------------------
// CLI
// ---------------------------------------------------------------------------
const { values: flags } = parseArgs({
options: {
"run-dir": { type: "string" },
json: { type: "boolean", default: false },
help: { type: "boolean", default: false },
},
strict: true,
});
if (flags.help) {
console.log(`Usage: bun run .claude/skills/benchmark-sandbox/sandbox-analyze.ts [options]
--run-dir <path> Path to a specific run directory
--json Output JSON report
--help Print usage`);
process.exit(0);
}
const DEFAULT_RESULTS = join(homedir(), "dev", "vercel-plugin-testing", "sandbox-results");
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
interface HookCoverage {
sessionStart: boolean;
preToolUse: boolean;
postToolUse: boolean;
userPromptSubmit: boolean;
}
interface ScenarioAnalysis {
slug: string;
sandboxId: string;
success: boolean;
timedOut: boolean;
durationSec: number;
expectedSkills: string[];
claimedSkills: string[];
missingSkills: string[];
unexpectedSkills: string[];
coveragePercent: number;
hookCoverage: HookCoverage;
debugLogLines: number;
stderrTraceLines: number;
error?: string;
}
interface CoverageReport {
runId: string;
timestamp: string;
snapshotId: string;
totalDurationSec: number;
scenarioCount: number;
passCount: number;
failCount: number;
timeoutCount: number;
hookFiredCount: number;
scenarios: ScenarioAnalysis[];
skillInjectionMatrix: Record<string, string[]>; // skill → slugs where injected
allExpectedSkills: string[];
allClaimedSkills: string[];
coveragePercent: number; // expected skills that were actually claimed
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
async function findLatestRunDir(): Promise<string> {
const entries = await readdir(DEFAULT_RESULTS);
const runDirs = entries.filter((e) => e.startsWith("run-")).sort().reverse();
if (runDirs.length === 0) {
console.error(`No run directories found in ${DEFAULT_RESULTS}`);
process.exit(1);
}
return join(DEFAULT_RESULTS, runDirs[0]);
}
async function readJsonFile<T>(path: string): Promise<T> {
const content = await readFile(path, "utf-8");
return JSON.parse(content) as T;
}
async function countLines(path: string): Promise<number> {
try {
const content = await readFile(path, "utf-8");
return content.split("\n").length;
} catch {
return 0;
}
}
async function dirExists(path: string): Promise<boolean> {
try {
const s = await stat(path);
return s.isDirectory();
} catch {
return false;
}
}
function parseHookCoverageFromDebugLogs(logContent: string): HookCoverage {
return {
sessionStart:
logContent.includes("SessionStart") || logContent.includes("session-start"),
preToolUse:
logContent.includes("PreToolUse") ||
logContent.includes("pretooluse") ||
logContent.includes("skill-inject"),
postToolUse:
logContent.includes("PostToolUse") ||
logContent.includes("posttooluse") ||
logContent.includes("validate"),
userPromptSubmit:
logContent.includes("UserPromptSubmit") ||
logContent.includes("user-prompt-submit") ||
logContent.includes("prompt-signal"),
};
}
// ---------------------------------------------------------------------------
// Analysis
// ---------------------------------------------------------------------------
async function analyzeRun(runDir: string): Promise<CoverageReport> {
const manifest = await readJsonFile<RunSummary>(join(runDir, "run-manifest.json"));
const scenarios: ScenarioAnalysis[] = [];
const skillInjectionMatrix: Record<string, string[]> = {};
for (const s of manifest.scenarios) {
const slugDir = join(runDir, s.slug);
// Read debug logs if available
let debugLogContent = "";
const debugLogsDir = join(slugDir, "debug-logs");
if (await dirExists(debugLogsDir)) {
const logFiles = await readdir(debugLogsDir).catch(() => []);
for (const f of logFiles) {
debugLogContent += await readFile(join(debugLogsDir, f), "utf-8").catch(() => "");
}
}
// Read stderr trace
const stderrLines = await countLines(join(slugDir, "stderr-trace.txt"));
// Parse hook coverage from debug logs + stderr
let stderrContent = "";
try {
stderrContent = await readFile(join(slugDir, "stderr-trace.txt"), "utf-8");
} catch { /* no stderr file */ }
const combinedLogs = debugLogContent + "\n" + stderrContent;
const hookCoverage = parseHookCoverageFromDebugLogs(combinedLogs);
// If manifest already has hook evidence, merge it in
if (s.hook_evidence.pre_tool_use_in_stderr) hookCoverage.preToolUse = true;
if (s.hook_evidence.user_prompt_in_stderr) hookCoverage.userPromptSubmit = true;
// Derive boolean flags from status
const success = s.status === "pass";
const timedOut = s.status === "timeout";
// Compute skill coverage per scenario
const expectedSet = new Set(s.expected_skills);
const claimedSet = new Set(s.claimed_skills);
const missingSkills = s.expected_skills.filter((sk) => !claimedSet.has(sk));
const unexpectedSkills = s.claimed_skills.filter((sk) => !expectedSet.has(sk));
const coveredCount = s.expected_skills.filter((sk) => claimedSet.has(sk)).length;
const scenarioCoverage =
s.expected_skills.length > 0 ? Math.round((coveredCount / s.expected_skills.length) * 100) : 100;
// Track skill injection matrix
for (const skill of s.claimed_skills) {
if (!skillInjectionMatrix[skill]) skillInjectionMatrix[skill] = [];
skillInjectionMatrix[skill].push(s.slug);
}
scenarios.push({
slug: s.slug,
sandboxId: s.sandbox_id,
success,
timedOut,
durationSec: Math.round(s.duration_ms / 1000),
expectedSkills: s.expected_skills,
claimedSkills: s.claimed_skills,
missingSkills,
unexpectedSkills,
coveragePercent: scenarioCoverage,
hookCoverage,
debugLogLines: debugLogContent.split("\n").length,
stderrTraceLines: stderrLines,
error: s.error,
});
}
// Aggregate stats — average per-scenario coverage, not global unique skills
const allExpected = [...new Set(manifest.scenarios.flatMap((s) => s.expected_skills))].sort();
const allClaimed = [...new Set(manifest.scenarios.flatMap((s) => s.claimed_skills))].sort();
const scenariosWithExpectations = scenarios.filter((s) => s.expectedSkills.length > 0);
const coveragePercent =
scenariosWithExpectations.length > 0
? Math.round(
scenariosWithExpectations.reduce((sum, s) => sum + s.coveragePercent, 0) /
scenariosWithExpectations.length,
)
: 0;
return {
runId: manifest.run_id,
timestamp: manifest.timestamp,
snapshotId: manifest.snapshot.snapshot_id,
totalDurationSec: Math.round(manifest.timing.total_duration_ms / 1000),
scenarioCount: scenarios.length,
passCount: scenarios.filter((s) => s.success).length,
failCount: scenarios.filter((s) => !s.success && !s.timedOut).length,
timeoutCount: scenarios.filter((s) => s.timedOut).length,
hookFiredCount: scenarios.filter(
(s) => s.hookCoverage.preToolUse || s.hookCoverage.userPromptSubmit,
).length,
scenarios,
skillInjectionMatrix,
allExpectedSkills: allExpected,
allClaimedSkills: allClaimed,
coveragePercent,
};
}
// ---------------------------------------------------------------------------
// Output
// ---------------------------------------------------------------------------
function printReport(report: CoverageReport): void {
console.log("=".repeat(70));
console.log("SANDBOX BENCHMARK COVERAGE REPORT");
console.log("=".repeat(70));
console.log(`Run ID: ${report.runId}`);
console.log(`Timestamp: ${report.timestamp}`);
console.log(`Snapshot: ${report.snapshotId}`);
console.log(`Duration: ${report.totalDurationSec}s`);
console.log();
// Summary
console.log("--- Summary ---");
console.log(`Scenarios: ${report.scenarioCount}`);
console.log(`Passed: ${report.passCount}`);
console.log(`Failed: ${report.failCount}`);
console.log(`Timed out: ${report.timeoutCount}`);
console.log(`Hooks fired: ${report.hookFiredCount}/${report.scenarioCount}`);
console.log(`Skill coverage: ${report.coveragePercent}% (${report.allClaimedSkills.length}/${report.allExpectedSkills.length} expected skills seen)`);
console.log();
// Scenario table
console.log("--- Scenarios ---");
console.log(
`${"Slug".padEnd(28)} ${"Status".padEnd(10)} ${"Cov%".padEnd(6)} ${"Hooks".padEnd(8)} ${"Expected".padEnd(20)} ${"Claimed".padEnd(20)} Missing`,
);
console.log("-".repeat(116));
for (const s of report.scenarios) {
const status = s.timedOut ? "TIMEOUT" : s.success ? "PASS" : "FAIL";
const covPct = `${s.coveragePercent}%`;
const hooks = [
s.hookCoverage.preToolUse ? "P" : "-",
s.hookCoverage.postToolUse ? "V" : "-",
s.hookCoverage.userPromptSubmit ? "U" : "-",
s.hookCoverage.sessionStart ? "S" : "-",
].join("");
const expected = s.expectedSkills.join(",");
const claimed = s.claimedSkills.join(",") || "(none)";
const missing = s.missingSkills.join(",") || "-";
console.log(
`${s.slug.padEnd(28)} ${status.padEnd(10)} ${covPct.padEnd(6)} ${hooks.padEnd(8)} ${expected.padEnd(20)} ${claimed.padEnd(20)} ${missing}`,
);
if (s.error) {
console.log(` ERROR: ${s.error.slice(0, 100)}`);
}
}
console.log();
console.log("Hook legend: P=PreToolUse V=PostToolUse(validate) U=UserPromptSubmit S=SessionStart");
console.log();
// Skill injection matrix
console.log("--- Skill Injection Matrix ---");
const sortedSkills = Object.entries(report.skillInjectionMatrix).sort(
(a, b) => b[1].length - a[1].length,
);
for (const [skill, slugs] of sortedSkills) {
console.log(` ${skill.padEnd(25)} → ${slugs.join(", ")}`);
}
if (sortedSkills.length === 0) {
console.log(" (no skills claimed)");
}
console.log();
// Missing coverage
const uncoveredSkills = report.allExpectedSkills.filter(
(sk) => !report.allClaimedSkills.includes(sk),
);
if (uncoveredSkills.length > 0) {
console.log("--- Uncovered Expected Skills ---");
for (const sk of uncoveredSkills) {
const scenarios = report.scenarios
.filter((s) => s.expectedSkills.includes(sk))
.map((s) => s.slug);
console.log(` ${sk.padEnd(25)} expected in: ${scenarios.join(", ")}`);
}
console.log();
}
// Failures
const failures = report.scenarios.filter((s) => !s.success);
if (failures.length > 0) {
console.log("--- Failures ---");
for (const f of failures) {
console.log(` ${f.slug}: ${f.timedOut ? "TIMEOUT" : f.error ?? "unknown error"}`);
console.log(` debug logs: ${f.debugLogLines} lines, stderr: ${f.stderrTraceLines} lines`);
}
console.log();
}
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
const runDir = flags["run-dir"] ? resolve(flags["run-dir"]) : await findLatestRunDir();
console.log(`Analyzing: ${runDir}\n`);
const report = await analyzeRun(runDir);
if (flags.json) {
console.log(JSON.stringify(report, null, 2));
} else {
printReport(report);
}
const MIN_COVERAGE_THRESHOLD = 50;
const allPassed = report.passCount === report.scenarioCount;
const coverageMet = report.coveragePercent >= MIN_COVERAGE_THRESHOLD;
if (!coverageMet) {
console.log(
`\nFAILED: Coverage ${report.coveragePercent}% is below minimum threshold of ${MIN_COVERAGE_THRESHOLD}%`,
);
}
process.exit(allPassed && coverageMet ? 0 : 1);
}
main().catch((err) => {
console.error("Fatal:", err);
process.exit(2);
});