run-eval.ts
GitHub#!/usr/bin/env bun
/**
* Sandbox eval runner with 3-phase pipeline: Build → Verify → Deploy.
*
* Phase 1 (BUILD): Claude Code builds the app in a fresh sandbox.
* Phase 2 (VERIFY): A follow-up Claude Code session uses agent-browser to
* walk through user stories, fixing issues until all pass.
* Phase 3 (DEPLOY): A third Claude Code session links to vercel-labs, runs
* `vercel deploy`, and fixes build errors (up to 3 retries).
* Deployed apps have deployment protection enabled by default.
*
* Skills are tracked across all 3 phases — each phase may trigger additional
* skill injections as new files/patterns are created.
*
* Usage:
* bun run .claude/skills/benchmark-sandbox/run-eval.ts [options]
* --concurrency N Max parallel sandboxes (default 5, max 10)
* --timeout MS Per-phase timeout in ms (default 1800000 = 30 min)
* --keep-alive Keep sandboxes running after eval
* --keep-hours N Hours to keep alive (default 8)
* --skip-verify Skip the agent-browser verification phase
* --skip-deploy Skip the Vercel deploy phase
* --scenarios a,b,c Only run specific scenarios by slug
* --scenarios-file f Load scenarios from a JSON file
*/
import { Sandbox } from "@vercel/sandbox";
import { readdir, readFile, stat, mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { homedir } from "node:os";
import { execSync } from "node:child_process";
// ---------------------------------------------------------------------------
// Config
// ---------------------------------------------------------------------------
const SANDBOX_HOME = "/home/vercel-sandbox";
const SANDBOX_PLUGIN_DIR = `${SANDBOX_HOME}/vercel-plugin`;
const LOCAL_PLUGIN_DIR = join(homedir(), "dev", "vercel-plugin");
const UPLOAD_DIRS = ["hooks", "skills", "generated"];
const RESULTS_DIR = join(homedir(), "dev", "vercel-plugin-testing", "sandbox-results");
const args = process.argv.slice(2);
const getArg = (name: string, fallback: number) =>
args.includes(`--${name}`) ? parseInt(args[args.indexOf(`--${name}`) + 1], 10) : fallback;
const CONCURRENCY = Math.min(Math.max(getArg("concurrency", 5), 1), 10);
const TIMEOUT_MS = getArg("timeout", 1_800_000);
let runId = "";
const KEEP_ALIVE = args.includes("--keep-alive");
const KEEP_ALIVE_HOURS = getArg("keep-hours", 8);
const SKIP_VERIFY = args.includes("--skip-verify");
const SKIP_DEPLOY = args.includes("--skip-deploy");
const BUILD_STATUS_INTERVAL_MS = 10_000;
const BUILD_WAIT_POLL_MS = 10_000;
const BUILD_STALE_THRESHOLD_MS = 120_000; // If no new debug log line for 2 min, consider build done
const SCENARIO_FILTER = args.includes("--scenarios")
? args[args.indexOf("--scenarios") + 1]?.split(",").map(s => s.trim()) ?? []
: [];
const SCENARIOS_FILE = args.includes("--scenarios-file")
? args[args.indexOf("--scenarios-file") + 1]
: undefined;
// ---------------------------------------------------------------------------
// Scenarios — loaded from --scenarios-file if provided, otherwise defaults
// ---------------------------------------------------------------------------
interface Scenario {
slug: string;
prompt: string;
expectedSkills: string[];
userStories: [string, string, string];
}
const SCENARIOS: Scenario[] = [
{
slug: "ai-writing-assistant",
prompt: `Build a Next.js AI writing assistant app. Requirements:
- Use AI SDK (\`ai\` package) with \`streamText\` and the anthropic provider (\`@ai-sdk/anthropic\`) for real AI responses
- Create /api/chat route handler using AI SDK's \`streamText\` that streams a writing assistant response
- The assistant should help rewrite, expand, or summarize text the user provides
- Use shadcn/ui (Textarea for input, Button, Card for output, Tabs for mode: rewrite/expand/summarize)
- Use SWR (\`swr\`) for client-side state management
- Use Geist font via next/font
- Add middleware.ts that logs request paths with timestamps
- Link the project to my vercel-labs team so we can deploy it later
After building all files, start the dev server on port 3000 with \`npx next dev --port 3000\`.`,
expectedSkills: ["ai-sdk", "swr", "shadcn", "routing-middleware", "geist", "nextjs"],
userStories: [
"As a user, I can see a text area where I can paste or type content to be processed by AI",
"As a user, I can select a mode (rewrite, expand, or summarize) and click a button to get an AI response",
"As a user, I can see the AI-generated response appear with streaming text output",
],
},
{
slug: "ai-code-reviewer",
prompt: `Build a Next.js AI code review tool. Requirements:
- Use AI SDK (\`ai\` package) with \`streamText\` and the anthropic provider (\`@ai-sdk/anthropic\`) for real AI code analysis
- Create /api/review route that accepts code and returns AI review comments with streaming
- Use shadcn/ui (Textarea with monospace font for code input, Card for review results, Badge for severity, Button)
- Use Vercel Flags (\`@vercel/flags/next\`) with a flag to toggle between "quick review" and "deep review" modes
- Create a flags.ts with the review mode flag using \`flag()\`
- Add /api/cron/stats route for tracking review statistics (mock implementation)
- Add structured observability logging in all API routes (JSON with timestamp, level, message, duration)
- Link the project to my vercel-labs team
After building all files, start the dev server on port 3000 with \`npx next dev --port 3000\`.`,
expectedSkills: ["ai-sdk", "vercel-flags", "shadcn", "cron-jobs", "observability", "nextjs", "vercel-functions"],
userStories: [
"As a user, I can see a code input area where I can paste code for review",
"As a user, I can click a Review button and see AI-generated code review comments appear",
"As a user, I can see severity indicators (like badges) on the review feedback",
],
},
{
slug: "ai-flashcard-trainer",
prompt: `Build a Next.js AI flashcard study app. Requirements:
- Use AI SDK (\`ai\` package) with \`generateText\` and the anthropic provider (\`@ai-sdk/anthropic\`) to generate flashcard content from a topic
- Create /api/generate route that takes a topic and returns 5 flashcards (question + answer) as JSON using AI
- Create /api/quiz route that uses AI to evaluate user answers and provide feedback
- Store flashcard decks in-memory via a /api/decks CRUD route (GET returns all decks, POST creates new)
- Use shadcn/ui (Card for flashcards with flip animation via CSS, Button, Input, Progress bar for score)
- Use SWR for fetching decks on the client
- Use Vercel KV / runtime cache pattern (mock with in-memory Map) for caching generated decks
- Use Geist font
- Link the project to my vercel-labs team
After building all files, start the dev server on port 3000 with \`npx next dev --port 3000\`.`,
expectedSkills: ["ai-sdk", "swr", "shadcn", "runtime-cache", "geist", "nextjs", "vercel-functions"],
userStories: [
"As a user, I can enter a topic and click Generate to have AI create flashcards",
"As a user, I can see flashcards displayed and flip them to reveal the answer",
"As a user, I can see a score or progress indicator showing how many cards I got right",
],
},
{
slug: "ai-meeting-summarizer",
prompt: `Build a Next.js AI meeting notes summarizer. Requirements:
- Use AI SDK (\`ai\` package) with \`streamText\` and the anthropic provider (\`@ai-sdk/anthropic\`) for streaming summaries
- Create /api/summarize route that takes meeting notes text and streams an AI summary with action items
- Create /api/meetings CRUD routes (GET, POST) storing meetings in-memory
- Use shadcn/ui (Textarea for notes input, Card for summary output, Table for action items, Button, Dialog)
- Use Satori (\`satori\`) in an /api/og/[id] route to generate OG image cards showing meeting title and date
- Add middleware.ts with request timing and path logging
- Use edge runtime for the /api/og route
- Use Vercel Functions for all other API routes
- Link the project to my vercel-labs team
After building all files, start the dev server on port 3000 with \`npx next dev --port 3000\`.`,
expectedSkills: ["ai-sdk", "satori", "shadcn", "routing-middleware", "nextjs", "vercel-functions"],
userStories: [
"As a user, I can paste meeting notes into a text area and click Summarize",
"As a user, I can see an AI-generated summary with key points streamed to the page",
"As a user, I can see extracted action items displayed in a list or table",
],
},
{
slug: "ai-deploy-analyzer",
prompt: `Build a Next.js deployment health analyzer with AI insights. Requirements:
- Create /api/deployments route returning mock deployment data (10 deployments with status, url, timestamp, duration)
- Use AI SDK (\`ai\` package) with \`generateText\` and the anthropic provider (\`@ai-sdk/anthropic\`) in /api/analyze route that takes deployment data and returns AI health analysis
- Use Vercel Flags (\`@vercel/flags/next\`) to toggle "show AI insights" with a \`flag()\` definition
- Create flags.ts with the feature flag
- Use shadcn/ui (Table for deployments, Badge for status, Card for AI insights, Tabs, Alert)
- Add /api/cron/health-check route that returns a health status JSON
- Add vercel.json with crons config for the health check
- Add structured observability logging (JSON with timestamp, level, message) in every API route
- Link the project to my vercel-labs team
After building all files, start the dev server on port 3000 with \`npx next dev --port 3000\`.`,
expectedSkills: ["ai-sdk", "vercel-flags", "shadcn", "cron-jobs", "observability", "nextjs", "vercel-functions"],
userStories: [
"As a user, I can see a table of deployments with status badges showing health",
"As a user, I can click an Analyze button and see AI-generated health insights appear",
"As a user, I can see alert or warning cards highlighting any deployment issues the AI found",
],
},
];
// Load scenarios from file if --scenarios-file is provided
let ACTIVE_SCENARIOS = SCENARIOS;
if (SCENARIOS_FILE) {
try {
const raw = require("fs").readFileSync(SCENARIOS_FILE, "utf-8");
ACTIVE_SCENARIOS = JSON.parse(raw) as Scenario[];
console.log(`Loaded ${ACTIVE_SCENARIOS.length} scenarios from ${SCENARIOS_FILE}`);
} catch (e: any) {
console.error(`Failed to load scenarios from ${SCENARIOS_FILE}: ${e.message}`);
process.exit(1);
}
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function elapsed(start: number): string {
return `${((performance.now() - start) / 1000).toFixed(0)}s`;
}
function resolveApiKey(): string {
if (process.env.ANTHROPIC_API_KEY) return process.env.ANTHROPIC_API_KEY;
try {
return execSync('security find-generic-password -a "$USER" -s "ANTHROPIC_AUTH_TOKEN" -w', {
encoding: "utf-8", timeout: 5000,
}).trim();
} catch {}
console.error("Missing ANTHROPIC_API_KEY"); process.exit(1);
}
function resolveVercelToken(): string | undefined {
try {
return JSON.parse(require("fs").readFileSync(join(homedir(), ".local/share/com.vercel.cli/auth.json"), "utf-8")).token;
} catch { return undefined; }
}
async function collectPluginFiles(): Promise<Array<{ path: string; content: Buffer }>> {
const files: Array<{ path: string; content: Buffer }> = [];
async function walk(dir: string): Promise<void> {
const entries = await readdir(join(LOCAL_PLUGIN_DIR, dir), { withFileTypes: true });
for (const entry of entries) {
const relPath = join(dir, entry.name);
const fullPath = join(LOCAL_PLUGIN_DIR, relPath);
if (entry.isDirectory()) {
if (["node_modules", ".git", "src", ".claude", "tests", "scripts", ".playground"].includes(entry.name)) continue;
await walk(relPath);
} else if (entry.isFile()) {
if (entry.name.endsWith(".mts") || entry.name.endsWith(".test.ts")) continue;
const s = await stat(fullPath);
if (s.size > 200_000) continue;
files.push({ path: join(SANDBOX_PLUGIN_DIR, relPath), content: await readFile(fullPath) });
}
}
}
for (const dir of UPLOAD_DIRS) await walk(dir);
for (const f of ["hooks/hooks.json", "package.json"]) {
try { files.push({ path: join(SANDBOX_PLUGIN_DIR, f), content: await readFile(join(LOCAL_PLUGIN_DIR, f)) }); } catch {}
}
return files;
}
async function sh(sandbox: any, cmd: string): Promise<string> {
try { const r = await sandbox.runCommand("sh", ["-c", cmd]); return (await r.stdout()).trim(); }
catch { return "(cmd failed)"; }
}
function normalizeShellOutput(output: string): string {
return output === "(cmd failed)" ? "" : output.trim();
}
export const SNAPSHOT_PREP_EXTENSION_MS = 1_800_000;
export function calculateSandboxLifetimeMs(phaseTimeoutMs: number): number {
return phaseTimeoutMs + phaseTimeoutMs + 300_000;
}
export function calculateKeepAliveExtensionMs(keepAlive: boolean, keepAliveHours: number): number | null {
return keepAlive ? keepAliveHours * 3600_000 : null;
}
export function isExpectedRunCommandTimeout(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error ?? "");
return /timed out|deadline exceeded|operation timed out|request timeout/i.test(message);
}
export function parseBuildExitCode(output: string): number | null {
const normalized = normalizeShellOutput(output);
if (!normalized) return null;
if (/^\d+$/.test(normalized)) return parseInt(normalized, 10);
const matches = [...normalized.matchAll(/EXIT:(\d+)/g)];
if (matches.length === 0) return null;
return parseInt(matches[matches.length - 1][1], 10);
}
type BuildWaitInput = {
claudeProbe: string;
debugLine: string;
elapsedMs: number;
timeoutMs: number;
lastDebugLineChanged?: boolean; // true if debug line differs from previous poll
msSinceLastDebugChange?: number; // ms since debug log last had a new line
};
export type BuildWaitState = {
claudeRunning: boolean;
sessionEnded: boolean;
timedOut: boolean;
stale: boolean; // debug log hasn't changed in BUILD_STALE_THRESHOLD_MS
shouldKeepPolling: boolean;
};
export function evaluateBuildWaitState(input: BuildWaitInput): BuildWaitState {
const probe = normalizeShellOutput(input.claudeProbe);
// Exit file contains a number when done, "RUNNING" when still going, empty on error
const claudeRunning = probe === "RUNNING" || probe === "";
const sessionEnded = /\bSessionEnd\b/.test(normalizeShellOutput(input.debugLine));
const timedOut = input.elapsedMs >= input.timeoutMs;
const stale = (input.msSinceLastDebugChange ?? 0) >= BUILD_STALE_THRESHOLD_MS;
return {
claudeRunning,
sessionEnded,
timedOut,
stale,
shouldKeepPolling: !timedOut && !stale && claudeRunning && !sessionEnded,
};
}
function buildVerificationPrompt(userStories: string[]): string {
const stories = userStories.map((s, i) => `${i + 1}. ${s}`).join("\n");
return `BEFORE anything else, fix the AI authentication to use OIDC (not API keys):
1. Check if the project has a .env.local with VERCEL_OIDC_TOKEN. If yes, OIDC is available.
2. Find ALL files that create an Anthropic provider (search for \`createAnthropic\` or \`@ai-sdk/anthropic\`).
3. Update each one to use OIDC auth with the AI Gateway instead of ANTHROPIC_API_KEY:
\`\`\`ts
import { createAnthropic } from "@ai-sdk/anthropic";
const anthropic = createAnthropic({
baseURL: process.env.ANTHROPIC_BASE_URL || "https://ai-gateway.vercel.sh/v1",
headers: {
"Authorization": \`Bearer \${process.env.VERCEL_OIDC_TOKEN}\`,
},
apiKey: "unused", // required by SDK but OIDC auth takes precedence via headers
});
\`\`\`
4. Make sure EVERY route that calls the AI uses this pattern. Do NOT use process.env.ANTHROPIC_API_KEY.
5. Log whether OIDC auth was configured: \`console.log("AI auth: OIDC=" + !!process.env.VERCEL_OIDC_TOKEN)\`
Then make sure the dev server is running. Check if http://localhost:3000 responds. If not, run \`npx next dev --port 3000\` in the background and wait for it to be ready.
Then use agent-browser to verify these user stories:
${stories}
For EACH story, follow this exact workflow:
1. agent-browser open http://localhost:3000
2. agent-browser wait --load networkidle
3. agent-browser screenshot --annotate
4. agent-browser snapshot -i
5. Interact with the UI (click buttons, fill inputs, etc.) to test the story
6. agent-browser screenshot --annotate (capture the result)
7. Determine if the story PASSED or FAILED
If a story FAILS:
- Fix the code to make it pass
- Restart the dev server if needed: kill the old one and run \`npx next dev --port 3000\` again
- Re-verify the story
After testing all stories, output a summary in this exact format:
VERIFICATION_RESULTS:
STORY_1: PASS or FAIL
STORY_2: PASS or FAIL
STORY_3: PASS or FAIL`;
}
// ---------------------------------------------------------------------------
// Structured scoring via haiku (runs inside sandbox, no hooks)
// ---------------------------------------------------------------------------
interface HaikuScore {
[key: string]: unknown;
}
async function scoreWithHaiku(
sandbox: any,
claudeBin: string,
prompt: string,
schema: Record<string, unknown>,
label: string,
slug: string,
t0: number,
): Promise<HaikuScore | null> {
const schemaJson = JSON.stringify(schema).replace(/'/g, "'\\''"); // escape single quotes for shell
const promptPath = `/tmp/score-${label}.txt`;
await sandbox.writeFiles([{ path: promptPath, content: Buffer.from(prompt) }]);
const cmd = `${claudeBin} -p "$(cat ${promptPath})" --json-schema '${schemaJson}' --output-format json --model haiku --setting-sources ""`;
try {
const sr = await sandbox.runCommand("sh", ["-c", cmd], { signal: AbortSignal.timeout(120_000) }); // 2 min max
const out = (await sr.stdout()).trim();
const parsed = JSON.parse(out);
// claude -p --output-format json wraps the result — structured_output has our schema data
return parsed.structured_output ?? parsed;
} catch (e: any) {
console.log(` [${slug}] ${label} score failed: ${e.message?.slice(0, 80)}`);
return null;
}
}
const STORIES_SCHEMA = {
type: "object",
properties: {
stories: {
type: "array",
items: {
type: "object",
properties: {
index: { type: "number" },
status: { type: "string", enum: ["pass", "fail"] },
reason: { type: "string" },
},
required: ["index", "status", "reason"],
},
},
},
required: ["stories"],
};
const BUILD_SCORE_SCHEMA = {
type: "object",
properties: {
completeness: { type: "string", enum: ["complete", "partial", "minimal", "empty"] },
hasApiRoutes: { type: "boolean" },
hasUIComponents: { type: "boolean" },
hasAIFeature: { type: "boolean" },
devServerRunning: { type: "boolean" },
missingFeatures: { type: "array", items: { type: "string" } },
summary: { type: "string" },
},
required: ["completeness", "hasApiRoutes", "hasUIComponents", "hasAIFeature", "devServerRunning", "missingFeatures", "summary"],
};
const DEPLOY_SCORE_SCHEMA = {
type: "object",
properties: {
deployed: { type: "boolean" },
url: { type: "string" },
buildSucceeded: { type: "boolean" },
errors: { type: "array", items: { type: "string" } },
summary: { type: "string" },
},
required: ["deployed", "buildSucceeded", "summary"],
};
// ---------------------------------------------------------------------------
// Per-scenario runner
// ---------------------------------------------------------------------------
interface VerificationResult {
ran: boolean;
exitCode: number;
stories: Array<{ index: number; status: "pass" | "fail" | "unknown" }>;
output: string;
}
interface ScenarioTiming {
sandboxCreateMs: number | null;
installMs: number | null;
pluginInstallMs: number | null;
buildStartMs: number | null;
buildEndMs: number | null;
verifyStartMs: number | null;
verifyEndMs: number | null;
deployStartMs: number | null;
deployEndMs: number | null;
extractMs: number | null;
}
interface ArtifactManifestEntry {
fileName: string;
sizeBytes: number;
description: string;
}
interface ScenarioResult {
slug: string;
sandboxId: string;
snapshotId?: string;
success: boolean;
durationMs: number;
claimedSkills: string[];
expectedSkills: string[];
projectFiles: string[];
appUrl?: string;
deployUrl?: string;
sourcePath?: string;
error?: string;
pollHistory: Array<{ elapsed: string; skills: string[]; files: number }>;
verification?: VerificationResult;
buildScore?: HaikuScore | null;
deployScore?: HaikuScore | null;
timing: ScenarioTiming;
}
function captureElapsedMs(start: number, end = performance.now()): number {
return Math.max(0, Math.round(end - start));
}
function captureRelativeMs(start: number, end = performance.now()): number {
return Math.max(0, Math.round(end - start));
}
export function calculatePhaseDuration(startMs: number | null | undefined, endMs: number | null | undefined): number | null {
if (typeof startMs !== "number" || typeof endMs !== "number") return null;
return Math.max(0, endMs - startMs);
}
function formatTimingDuration(durationMs: number | null | undefined): string {
return typeof durationMs === "number" ? `${(durationMs / 1000).toFixed(1)}s` : "skip";
}
export function renderTimingMarkdownTable(results: Array<{ slug: string; timing?: ScenarioTiming }>): string {
let md = `## Timing\n\n`;
md += `| Scenario | Sandbox | Install | Plugin | Build | Verify | Deploy | Extract |\n`;
md += `|----------|---------|---------|--------|-------|--------|--------|---------|\n`;
for (const result of results) {
const timing = result.timing;
md += `| ${result.slug} | ${formatTimingDuration(timing?.sandboxCreateMs)} | ${formatTimingDuration(timing?.installMs)} | ${formatTimingDuration(timing?.pluginInstallMs)} | ${formatTimingDuration(calculatePhaseDuration(timing?.buildStartMs, timing?.buildEndMs))} | ${formatTimingDuration(calculatePhaseDuration(timing?.verifyStartMs, timing?.verifyEndMs))} | ${formatTimingDuration(calculatePhaseDuration(timing?.deployStartMs, timing?.deployEndMs))} | ${formatTimingDuration(timing?.extractMs)} |\n`;
}
return md;
}
export function buildArtifactManifest(entries: ArtifactManifestEntry[]) {
return {
generatedAt: new Date().toISOString(),
artifactCount: entries.length,
artifacts: [...entries].sort((a, b) => a.fileName.localeCompare(b.fileName)),
};
}
export function summarizeToolCalls(grepOutput: string): string {
const normalized = normalizeShellOutput(grepOutput);
if (!normalized) return "No executePreToolHooks entries found.";
return normalized
.split("\n")
.filter(Boolean)
.map((line) => {
const timestampMatch = line.match(/"timestamp":"([^"]+)"/) ?? line.match(/\d{4}-\d{2}-\d{2}T[^\s"]+/);
const timestamp = timestampMatch
? timestampMatch[1] ?? timestampMatch[0]
: "unknown-timestamp";
const toolMatch =
line.match(/"tool(?:Name)?":"([^"]+)"/i)
?? line.match(/\btool(?:Name)?[=: ]+["']?([A-Za-z0-9._-]+)["']?/i)
?? line.match(/"tool_name":"([^"]+)"/i);
const toolName = toolMatch
? toolMatch[1]
: "unknown-tool";
return `${timestamp} | ${toolName} | ${line}`;
})
.join("\n");
}
async function readSandboxFileBuffer(sandbox: any, path: string): Promise<Buffer | null> {
const stream = await sandbox.readFile({ path });
if (!stream) return null;
const chunks: Buffer[] = [];
for await (const chunk of stream) chunks.push(Buffer.from(chunk));
return Buffer.concat(chunks);
}
async function writeArchiveArtifact(
archiveDir: string,
fileName: string,
description: string,
content: Buffer | string,
): Promise<ArtifactManifestEntry> {
const artifactPath = join(archiveDir, fileName);
await writeFile(artifactPath, content);
const artifactStats = await stat(artifactPath);
return {
fileName,
sizeBytes: artifactStats.size,
description,
};
}
async function runScenario(
scenario: Scenario,
apiKey: string,
baseUrl: string,
vercelToken: string | undefined,
pluginFiles: Array<{ path: string; content: Buffer }>,
): Promise<ScenarioResult> {
const t0 = performance.now();
const projectDir = `${SANDBOX_HOME}/${scenario.slug}`;
const pollHistory: ScenarioResult["pollHistory"] = [];
const timing: ScenarioTiming = {
sandboxCreateMs: null,
installMs: null,
pluginInstallMs: null,
buildStartMs: null,
buildEndMs: null,
verifyStartMs: null,
verifyEndMs: null,
deployStartMs: null,
deployEndMs: null,
extractMs: null,
};
let sandbox: InstanceType<typeof Sandbox> | undefined;
try {
// 1. Create sandbox with port 3000
console.log(` [${scenario.slug}] Creating sandbox...`);
const sandboxCreateStartedAt = performance.now();
sandbox = await Sandbox.create({
runtime: "node24",
ports: [3000],
env: {
ANTHROPIC_API_KEY: apiKey,
ANTHROPIC_BASE_URL: baseUrl,
VERCEL_PLUGIN_LOG_LEVEL: "trace",
...(vercelToken ? { VERCEL_TOKEN: vercelToken } : {}),
},
timeout: calculateSandboxLifetimeMs(TIMEOUT_MS),
} as any);
timing.sandboxCreateMs = captureElapsedMs(sandboxCreateStartedAt);
let appUrl: string | undefined;
try { appUrl = sandbox.domain(3000); } catch {}
console.log(` [${scenario.slug}] Sandbox ${sandbox.sandboxId}${appUrl ? ` | ${appUrl}` : ""} (${elapsed(t0)})`);
// 2. Install Claude Code + Vercel CLI + agent-browser
const installStartedAt = performance.now();
await sandbox.runCommand("sh", ["-c", "npm install -g @anthropic-ai/claude-code vercel agent-browser"]);
timing.installMs = captureElapsedMs(installStartedAt);
const claudeBin = await sh(sandbox, "which claude");
const abBin = await sh(sandbox, "which agent-browser");
console.log(` [${scenario.slug}] claude=${claudeBin} agent-browser=${abBin} (${elapsed(t0)})`);
// 3. Vercel CLI auth
if (vercelToken) {
await sandbox.writeFiles([{
path: `${SANDBOX_HOME}/.local/share/com.vercel.cli/auth.json`,
content: Buffer.from(JSON.stringify({ token: vercelToken })),
}]);
}
// 4. Project setup + plugin
const pluginInstallStartedAt = performance.now();
await sandbox.runCommand("sh", ["-c", `mkdir -p ${projectDir} && cd ${projectDir} && npm init -y`]);
await sandbox.writeFiles(pluginFiles);
await sh(sandbox, `cd ${projectDir} && npx -y add-plugin ${SANDBOX_PLUGIN_DIR} -s project -y --target claude-code 2>&1 | tail -1`);
timing.pluginInstallMs = captureElapsedMs(pluginInstallStartedAt);
console.log(` [${scenario.slug}] Plugin installed (${elapsed(t0)})`);
// 5. Phase 1: Build the app
await sandbox.writeFiles([{ path: "/tmp/prompt.txt", content: Buffer.from(scenario.prompt) }]);
const settingsPath = `${projectDir}/.claude/settings.json`;
const buildCmd = `cd ${projectDir} && ${claudeBin} --dangerously-skip-permissions --debug --settings ${settingsPath} "$(cat /tmp/prompt.txt)"`;
const buildStartedAt = Date.now();
timing.buildStartMs = captureRelativeMs(t0);
const logFile = "/tmp/claude-build-round-1.log";
const errFile = "/tmp/claude-build-round-1.err";
const exitFile = "/tmp/claude-build-round-1.exit";
let lastDebugLine = "";
let lastDebugChangeAt = Date.now();
async function pollBuildProgress(includeProcessState = false): Promise<BuildWaitState | null> {
try {
const [skillsRaw, fileCountRaw, port3000Raw, claudeProbeRaw, debugLineRaw] = await Promise.all([
sh(sandbox!, "ls /tmp/vercel-plugin-*-seen-skills.d/ 2>/dev/null"),
sh(sandbox!, `find ${projectDir} -maxdepth 3 -not -path '*/node_modules/*' -not -path '*/.git/*' -not -path '*/.next/*' -not -path '*/.claude/*' -newer /tmp/prompt.txt -type f 2>/dev/null | wc -l`),
sh(sandbox!, "curl -s -o /dev/null -w '%{http_code}' http://localhost:3000 2>/dev/null || echo 'down'"),
includeProcessState
? sh(sandbox!, `cat ${exitFile} 2>/dev/null || echo "RUNNING"`)
: Promise.resolve(""),
includeProcessState
? sh(sandbox!, "ls -t ~/.claude/debug/*.txt 2>/dev/null | head -1 | xargs tail -1 2>/dev/null || true")
: Promise.resolve(""),
]);
const skills = normalizeShellOutput(skillsRaw).split("\n").filter(Boolean);
const fileCount = parseInt(normalizeShellOutput(fileCountRaw), 10) || 0;
const port3000 = normalizeShellOutput(port3000Raw) || "down";
if (!appUrl && port3000 !== "000down" && port3000 !== "down") {
try { appUrl = sandbox!.domain(3000); } catch {}
}
pollHistory.push({ elapsed: elapsed(t0), skills, files: fileCount });
// Track debug log staleness
const debugLine = normalizeShellOutput(debugLineRaw);
if (debugLine && debugLine !== lastDebugLine) {
lastDebugLine = debugLine;
lastDebugChangeAt = Date.now();
}
const msSinceLastDebugChange = Date.now() - lastDebugChangeAt;
const waitState = includeProcessState
? evaluateBuildWaitState({
claudeProbe: claudeProbeRaw,
debugLine: debugLineRaw,
elapsedMs: Date.now() - buildStartedAt,
timeoutMs: TIMEOUT_MS,
lastDebugLineChanged: debugLine !== lastDebugLine,
msSinceLastDebugChange,
})
: null;
const staleSuffix = waitState?.stale ? ` | STALE(${Math.round(msSinceLastDebugChange / 1000)}s)` : "";
const claudeStatus = waitState
? (waitState.claudeRunning ? "running" : `exited(${normalizeShellOutput(claudeProbeRaw)})`)
: "";
const waitSuffix = waitState
? ` | claude=${claudeStatus} | sessionEnd=${waitState.sessionEnded ? "yes" : "no"}${waitState.timedOut ? " | wait=timeout" : ""}${staleSuffix}${debugLine ? ` | debug=${debugLine.slice(-120)}` : ""}`
: "";
console.log(` [${scenario.slug}] ${elapsed(t0)} | skills: ${skills.join(", ") || "(none)"} | files: ${fileCount} | :3000=${port3000}${waitSuffix}`);
return waitState;
} catch {
return null;
}
}
async function readClaudeExitCode(exitPath: string, logPath: string): Promise<number | null> {
for (let attempt = 1; attempt <= 3; attempt++) {
const exitRaw = await sh(sandbox!, `cat ${exitPath} 2>/dev/null || grep -o 'EXIT:[0-9]\\+' ${logPath} 2>/dev/null | tail -1`);
const parsed = parseBuildExitCode(exitRaw);
if (parsed !== null) return parsed;
await new Promise((resolve) => setTimeout(resolve, 1_000));
}
return null;
}
console.log(` [${scenario.slug}] Phase 1: BUILD started (single launch + fire-and-forget poll after API timeout) (${elapsed(t0)})`);
const pollInterval = setInterval(() => {
void pollBuildProgress(false);
}, BUILD_STATUS_INTERVAL_MS);
let buildExit = -1;
let buildLaunchTimedOut = false;
// Redirect stdout/stderr to files so we can capture them even if runCommand times out.
const wrappedCmd = `${buildCmd} > ${logFile} 2> ${errFile}; status=$?; echo "EXIT:$status" >> ${logFile}; printf '%s' "$status" > ${exitFile}`;
try {
const r = await sandbox.runCommand("sh", ["-c", wrappedCmd]);
buildExit = (r as any).exitCode ?? 0;
} catch (e: any) {
if (isExpectedRunCommandTimeout(e)) {
buildLaunchTimedOut = true;
buildExit = 124;
console.log(` [${scenario.slug}] Build launch hit the expected 300s API timeout; waiting on sandbox process state (${elapsed(t0)})`);
} else {
console.log(` [${scenario.slug}] Build launch failed | attempt=runCommand | state=launching | error=${e.message?.slice(0, 200)} (${elapsed(t0)})`);
buildExit = 1;
}
} finally {
clearInterval(pollInterval);
}
if (buildLaunchTimedOut) {
// After the HTTP timeout the shell keeps running inside the sandbox, so poll
// process/debug state until Claude exits or the phase deadline is exhausted.
while (true) {
const waitState = await pollBuildProgress(true);
if (!waitState) {
if (Date.now() - buildStartedAt >= TIMEOUT_MS) {
buildExit = 124;
console.log(` [${scenario.slug}] Build wait probe stalled past phase timeout (${TIMEOUT_MS / 1000}s) (${elapsed(t0)})`);
break;
}
console.log(` [${scenario.slug}] Build wait probe failed; retrying in ${BUILD_WAIT_POLL_MS / 1000}s (${elapsed(t0)})`);
await new Promise((resolve) => setTimeout(resolve, BUILD_WAIT_POLL_MS));
continue;
}
if (!waitState.shouldKeepPolling) {
if (waitState.timedOut) {
buildExit = 124;
console.log(` [${scenario.slug}] Build wait reached phase timeout (${TIMEOUT_MS / 1000}s) (${elapsed(t0)})`);
} else if (waitState.stale) {
const exitCode = await readClaudeExitCode(exitFile, logFile);
if (exitCode !== null) buildExit = exitCode;
console.log(` [${scenario.slug}] Build stale — no new debug log activity for ${BUILD_STALE_THRESHOLD_MS / 1000}s, treating as done | exit=${exitCode ?? "unknown"} (${elapsed(t0)})`);
} else {
const exitCode = await readClaudeExitCode(exitFile, logFile);
if (exitCode !== null) buildExit = exitCode;
console.log(` [${scenario.slug}] Build process finished | claude=${waitState.claudeRunning ? "running" : "exited"} | sessionEnd=${waitState.sessionEnded ? "yes" : "no"} | exit=${exitCode ?? "unknown"} (${elapsed(t0)})`);
}
break;
}
await new Promise((resolve) => setTimeout(resolve, BUILD_WAIT_POLL_MS));
}
}
// Grab tail of stdout/stderr no matter what happened.
try {
const tail = await sh(sandbox, `tail -30 ${logFile} 2>/dev/null`);
const errTail = await sh(sandbox, `tail -10 ${errFile} 2>/dev/null`);
if (tail) console.log(` [${scenario.slug}] Build stdout (last 30 lines):\n${tail}`);
if (errTail) console.log(` [${scenario.slug}] Build stderr (last 10 lines):\n${errTail}`);
} catch {}
timing.buildEndMs = captureRelativeMs(t0);
if (buildExit === 0) {
console.log(` [${scenario.slug}] Build completed (${elapsed(t0)})`);
}
// Extract artifacts after build
const projectFilesList = (await sh(sandbox, `find ${projectDir} -maxdepth 3 -not -path '*/node_modules/*' -not -path '*/.git/*' -not -path '*/.next/*' -not -path '*/.claude/*' -type f 2>/dev/null | head -40`)).split("\n").filter(Boolean);
// 5b. Snapshot the sandbox after build (safety net — stops source sandbox, creates restore point)
let snapshotId: string | undefined;
if (projectFilesList.length > 1) {
try {
console.log(` [${scenario.slug}] Snapshotting after build... (${elapsed(t0)})`);
try { await sandbox.extendTimeout(SNAPSHOT_PREP_EXTENSION_MS); } catch {}
const snap = await sandbox.snapshot({ expiration: 4 * 3600_000 }); // 4 hour expiry
snapshotId = snap.snapshotId;
console.log(` [${scenario.slug}] Snapshot created: ${snapshotId} (${elapsed(t0)})`);
// Snapshotting stopped the old sandbox — create a new one from the snapshot
sandbox = await Sandbox.create({
runtime: "node24",
ports: [3000],
env: {
ANTHROPIC_API_KEY: apiKey,
ANTHROPIC_BASE_URL: baseUrl,
VERCEL_PLUGIN_LOG_LEVEL: "trace",
...(vercelToken ? { VERCEL_TOKEN: vercelToken } : {}),
},
timeout: calculateSandboxLifetimeMs(TIMEOUT_MS),
source: { type: "snapshot", snapshotId },
} as any);
try { appUrl = sandbox.domain(3000); } catch {}
console.log(` [${scenario.slug}] Restored from snapshot: ${sandbox.sandboxId}${appUrl ? ` | ${appUrl}` : ""} (${elapsed(t0)})`);
} catch (e: any) {
console.log(` [${scenario.slug}] Snapshot failed (continuing with original sandbox): ${e.message?.slice(0, 100)} (${elapsed(t0)})`);
}
}
const claimedSkills = (await sh(sandbox, "ls /tmp/vercel-plugin-*-seen-skills.d/ 2>/dev/null")).split("\n").filter(Boolean);
console.log(` [${scenario.slug}] Build done (exit=${buildExit}) | skills=${claimedSkills.length} | files=${projectFilesList.length} (${elapsed(t0)})`);
// Score build completeness with haiku
let buildScore: HaikuScore | null = null;
if (projectFilesList.length > 0) {
const fileList = projectFilesList.map(f => f.replace(`${SANDBOX_HOME}/${scenario.slug}/`, "")).join("\n");
buildScore = await scoreWithHaiku(sandbox, claudeBin,
`A Next.js app was built with this prompt:\n"${scenario.prompt.slice(0, 500)}"\n\nThe following files were created:\n${fileList}\n\nPort 3000 status: ${await sh(sandbox, "curl -s -o /dev/null -w '%{http_code}' http://localhost:3000 2>/dev/null || echo down")}\n\nScore the build completeness.`,
BUILD_SCORE_SCHEMA, "build", scenario.slug, t0,
);
if (buildScore) {
console.log(` [${scenario.slug}] Build score: ${(buildScore as any).completeness} | API=${(buildScore as any).hasApiRoutes} UI=${(buildScore as any).hasUIComponents} AI=${(buildScore as any).hasAIFeature} (${elapsed(t0)})`);
}
}
// 6. Start dev server (if not already running from the build prompt)
let port3000Up = false;
const portCheck = await sh(sandbox, "curl -s -o /dev/null -w '%{http_code}' http://localhost:3000 2>/dev/null");
if (portCheck === "200" || portCheck === "307") {
port3000Up = true;
console.log(` [${scenario.slug}] Dev server already running (${elapsed(t0)})`);
} else {
const hasNext = await sh(sandbox, `test -f ${projectDir}/node_modules/.bin/next && echo YES || echo NO`);
if (hasNext === "YES") {
console.log(` [${scenario.slug}] Starting dev server... (${elapsed(t0)})`);
await sh(sandbox, `cd ${projectDir} && nohup npx next dev --port 3000 --turbopack > /tmp/next-dev.log 2>&1 & echo started`);
for (let i = 0; i < 30; i++) {
await new Promise(r => setTimeout(r, 3000));
const status = await sh(sandbox, "curl -s -o /dev/null -w '%{http_code}' http://localhost:3000 2>/dev/null");
if (status === "200" || status === "307") {
port3000Up = true;
try { appUrl = sandbox.domain(3000); } catch {}
console.log(` [${scenario.slug}] Dev server UP: ${appUrl} (${elapsed(t0)})`);
break;
}
}
}
}
// 7. Extend timeout only when the sandbox should stay alive after the eval
const keepAliveExtensionMs = calculateKeepAliveExtensionMs(KEEP_ALIVE, KEEP_ALIVE_HOURS);
if (keepAliveExtensionMs) {
try {
await sandbox.extendTimeout(keepAliveExtensionMs);
console.log(` [${scenario.slug}] Keep-alive timeout extended (${elapsed(t0)})`);
} catch (e: any) {
console.log(` [${scenario.slug}] extendTimeout: ${e.message?.slice(0, 60)}`);
}
}
// 7b. Link project to Vercel + pull env (OIDC credentials for AI Gateway during verify)
const vercelProjectName = `${scenario.slug}-${new Date().toISOString().replace(/[-:T]/g, "").slice(0, 12)}`.toLowerCase();
if (vercelToken) {
try {
const linkOut = await sh(sandbox, `cd ${projectDir} && unset VERCEL_TOKEN && vercel link --yes --scope vercel-labs --project ${vercelProjectName} 2>&1 | tail -3`);
console.log(` [${scenario.slug}] Linked to vercel-labs/${vercelProjectName}: ${linkOut.split("\n").pop()} (${elapsed(t0)})`);
const envOut = await sh(sandbox, `cd ${projectDir} && unset VERCEL_TOKEN && vercel env pull .env.local --yes 2>&1 | tail -3`);
console.log(` [${scenario.slug}] Pulled env: ${envOut.split("\n").pop()} (${elapsed(t0)})`);
} catch (e: any) {
console.log(` [${scenario.slug}] Vercel link/env pull failed: ${e.message?.slice(0, 80)} (${elapsed(t0)})`);
}
}
// Ensure ANTHROPIC_BASE_URL has /v1 suffix for the app's AI SDK (@ai-sdk/anthropic appends /messages)
// Don't write the vck_* API key — the app should use the OIDC token from vercel env pull
try {
const existing = await sh(sandbox, `cat ${projectDir}/.env.local 2>/dev/null`);
const appBaseUrl = baseUrl.endsWith("/v1") ? baseUrl : `${baseUrl}/v1`;
if (!existing.includes("ANTHROPIC_BASE_URL") || !existing.includes("/v1")) {
const lines = existing.trim() ? existing.trim() + "\n" : "";
const envWithBaseUrl = lines + `ANTHROPIC_BASE_URL=${appBaseUrl}\n`;
await sandbox.writeFiles([{ path: `${projectDir}/.env.local`, content: Buffer.from(envWithBaseUrl) }]);
console.log(` [${scenario.slug}] Ensured ANTHROPIC_BASE_URL=${appBaseUrl} in .env.local (${elapsed(t0)})`);
}
} catch {}
// 8. Phase 2: Verification with agent-browser
let verification: VerificationResult | undefined;
if (!SKIP_VERIFY && projectFilesList.length > 1) {
console.log(` [${scenario.slug}] Phase 2: VERIFY with agent-browser (${elapsed(t0)})`);
timing.verifyStartMs = captureRelativeMs(t0);
const verifyPrompt = buildVerificationPrompt(scenario.userStories);
await sandbox.writeFiles([{ path: "/tmp/verify.txt", content: Buffer.from(verifyPrompt) }]);
const verifyLogFile = "/tmp/claude-verify.log";
const verifyErrFile = "/tmp/claude-verify.err";
const verifyExitFile = "/tmp/claude-verify.exit";
const verifyCmd = `cd ${projectDir} && ${claudeBin} --dangerously-skip-permissions --debug --settings ${settingsPath} "$(cat /tmp/verify.txt)"`;
const wrappedVerifyCmd = `${verifyCmd} > ${verifyLogFile} 2> ${verifyErrFile}; status=$?; echo "EXIT:$status" >> ${verifyLogFile}; printf '%s' "$status" > ${verifyExitFile}`;
let verifyExit = -1;
let verifyOut = "";
// Fire-and-forget: launch verify, catch the 300s API timeout, then poll for exit
try {
const vr = await sandbox.runCommand("sh", ["-c", wrappedVerifyCmd]);
verifyExit = (vr as any).exitCode ?? 0;
} catch (e: any) {
if (isExpectedRunCommandTimeout(e)) {
console.log(` [${scenario.slug}] Verify hit 300s API timeout; polling for completion (${elapsed(t0)})`);
// Poll for verify exit
const verifyStartedAt = Date.now();
while (Date.now() - verifyStartedAt < 1_200_000) { // 20 min max
await new Promise(r => setTimeout(r, BUILD_WAIT_POLL_MS));
const exitRaw = await sh(sandbox, `cat ${verifyExitFile} 2>/dev/null || echo RUNNING`);
const probe = normalizeShellOutput(exitRaw);
if (probe !== "RUNNING" && probe !== "") {
verifyExit = parseInt(probe, 10) || 0;
console.log(` [${scenario.slug}] Verify exited(${verifyExit}) (${elapsed(t0)})`);
break;
}
// Log progress
const tail = await sh(sandbox, `tail -1 ${verifyLogFile} 2>/dev/null`);
console.log(` [${scenario.slug}] Verify polling... | ${elapsed(t0)} | ${normalizeShellOutput(tail).slice(-100)}`);
}
if (verifyExit === -1) {
verifyExit = 124;
console.log(` [${scenario.slug}] Verify timed out after 20min (${elapsed(t0)})`);
}
} else {
verifyExit = 1;
console.log(` [${scenario.slug}] Verify launch failed: ${e.message?.slice(0, 100)} (${elapsed(t0)})`);
}
} finally {
timing.verifyEndMs = captureRelativeMs(t0);
}
// Always capture verify stdout — this is the critical data we've been missing
try {
verifyOut = await sh(sandbox, `cat ${verifyLogFile} 2>/dev/null`);
const verifyErr = await sh(sandbox, `cat ${verifyErrFile} 2>/dev/null`);
const outLines = normalizeShellOutput(verifyOut).split("\n");
console.log(` [${scenario.slug}] Verify stdout: ${outLines.length} lines, last: ${outLines.slice(-3).join(" | ").slice(-200)}`);
if (normalizeShellOutput(verifyErr)) {
console.log(` [${scenario.slug}] Verify stderr (last 3 lines): ${normalizeShellOutput(verifyErr).split("\n").slice(-3).join(" | ").slice(-200)}`);
}
} catch {}
// Re-extract skills after verify phase (agent-browser + fixes trigger more)
const postVerifySkills = (await sh(sandbox, "ls /tmp/vercel-plugin-*-seen-skills.d/ 2>/dev/null")).split("\n").filter(Boolean);
if (postVerifySkills.length > claimedSkills.length) {
const newSkills = postVerifySkills.filter(s => !claimedSkills.includes(s));
if (newSkills.length > 0) {
console.log(` [${scenario.slug}] +${newSkills.length} skills from verify: ${newSkills.join(", ")}`);
claimedSkills.push(...newSkills);
}
}
// Score verification results with haiku structured output
const storyList = scenario.userStories.map((s, i) => `${i + 1}. ${s}`).join("\n");
const verifyScore = await scoreWithHaiku(sandbox, claudeBin,
`You are scoring whether user stories were verified in a web app test session.\n\nThe user stories were:\n${storyList}\n\nThe verification session output (last 2000 chars):\n${verifyOut.slice(-2000)}\n\nFor each story, determine if it PASSED or FAILED based on evidence in the output. If the output mentions screenshots, successful interactions, or confirmations — mark as pass. If there are errors, missing elements, or the story was never tested — mark as fail. Give a brief reason for each.`,
STORIES_SCHEMA, "verify", scenario.slug, t0,
);
let stories: VerificationResult["stories"] = scenario.userStories.map((_, i) => ({
index: i + 1, status: "unknown" as const,
}));
if (verifyScore && Array.isArray((verifyScore as any).stories)) {
stories = (verifyScore as any).stories.map((s: any) => ({
index: s.index,
status: s.status === "pass" ? "pass" as const : "fail" as const,
}));
}
verification = { ran: true, exitCode: verifyExit, stories, output: verifyOut.slice(-500) };
const passCount = stories.filter(s => s.status === "pass").length;
console.log(` [${scenario.slug}] Verify: ${passCount}/${stories.length} passed (exit=${verifyExit}) (${elapsed(t0)})`)
} else if (SKIP_VERIFY) {
console.log(` [${scenario.slug}] Verification skipped (--skip-verify)`);
} else {
console.log(` [${scenario.slug}] Verification skipped (only ${projectFilesList.length} files built)`);
}
// 9. Phase 3: Deploy to Vercel for permanent URL
// Uses Claude Code for skill tracking during deploy + build error fixes.
// Deployment protection is on by default for vercel-labs team.
let deployUrl: string | undefined;
let deployScore: HaikuScore | null = null;
if (!SKIP_DEPLOY && vercelToken && projectFilesList.length > 3) {
console.log(` [${scenario.slug}] Phase 3: DEPLOY (${elapsed(t0)})`);
const deployStartedAt = Date.now();
timing.deployStartMs = captureRelativeMs(t0);
const deployPrompt = `Deploy this app to Vercel. Follow these steps:
1. Run: vercel link --yes --scope vercel-labs --project ${vercelProjectName}
2. Run: vercel deploy --yes
3. If the deploy fails with a build error, fix the code and try again (up to 3 attempts).
Important:
- Do NOT set or use VERCEL_TOKEN env var — the CLI auth is already configured
- If you see tsconfig or type errors, fix them before retrying
- Deployment protection is enabled by default, which is what we want`;
await sandbox.writeFiles([{ path: "/tmp/deploy.txt", content: Buffer.from(deployPrompt) }]);
const deployCmd = `cd ${projectDir} && unset VERCEL_TOKEN && ${claudeBin} --dangerously-skip-permissions --debug --settings ${settingsPath} "$(cat /tmp/deploy.txt)"`;
const deployLogFile = "/tmp/claude-deploy.log";
const deployErrFile = "/tmp/claude-deploy.err";
const deployExitFile = "/tmp/claude-deploy.exit";
const wrappedDeployCmd = `${deployCmd} > ${deployLogFile} 2> ${deployErrFile}; status=$?; echo "EXIT:$status" >> ${deployLogFile}; printf '%s' "$status" > ${deployExitFile}`;
let lastDeployDebugLine = "";
let lastDeployDebugChangeAt = Date.now();
async function pollDeployProgress(): Promise<BuildWaitState | null> {
try {
const [claudeProbeRaw, debugLineRaw] = await Promise.all([
sh(sandbox!, `cat ${deployExitFile} 2>/dev/null || echo "RUNNING"`),
sh(sandbox!, "ls -t ~/.claude/debug/*.txt 2>/dev/null | head -1 | xargs tail -1 2>/dev/null || true"),
]);
const debugLine = normalizeShellOutput(debugLineRaw);
if (debugLine && debugLine !== lastDeployDebugLine) {
lastDeployDebugLine = debugLine;
lastDeployDebugChangeAt = Date.now();
}
const msSinceLastDebugChange = Date.now() - lastDeployDebugChangeAt;
const waitState = evaluateBuildWaitState({
claudeProbe: claudeProbeRaw,
debugLine: debugLineRaw,
elapsedMs: Date.now() - deployStartedAt,
timeoutMs: TIMEOUT_MS,
lastDebugLineChanged: debugLine !== lastDeployDebugLine,
msSinceLastDebugChange,
});
const staleSuffix = waitState.stale ? ` | STALE(${Math.round(msSinceLastDebugChange / 1000)}s)` : "";
const claudeStatus = waitState.claudeRunning ? "running" : `exited(${normalizeShellOutput(claudeProbeRaw)})`;
console.log(` [${scenario.slug}] Deploy wait | claude=${claudeStatus} | sessionEnd=${waitState.sessionEnded ? "yes" : "no"}${waitState.timedOut ? " | wait=timeout" : ""}${staleSuffix}${debugLine ? ` | debug=${debugLine.slice(-120)}` : ""} (${elapsed(t0)})`);
return waitState;
} catch {
return null;
}
}
let deployExit = -1;
let deployOut = "";
let deployLaunchTimedOut = false;
try {
const dr = await sandbox.runCommand("sh", ["-c", wrappedDeployCmd]);
deployExit = (dr as any).exitCode ?? 0;
} catch (e: any) {
if (isExpectedRunCommandTimeout(e)) {
deployLaunchTimedOut = true;
deployExit = 124;
console.log(` [${scenario.slug}] Deploy launch hit the expected 300s API timeout; waiting on sandbox process state (${elapsed(t0)})`);
} else {
deployExit = 1;
console.log(` [${scenario.slug}] Deploy launch failed | attempt=runCommand | state=launching | error=${e.message?.slice(0, 200)} (${elapsed(t0)})`);
}
}
if (deployLaunchTimedOut) {
while (true) {
const waitState = await pollDeployProgress();
if (!waitState) {
if (Date.now() - deployStartedAt >= TIMEOUT_MS) {
deployExit = 124;
console.log(` [${scenario.slug}] Deploy wait probe stalled past phase timeout (${TIMEOUT_MS / 1000}s) (${elapsed(t0)})`);
break;
}
console.log(` [${scenario.slug}] Deploy wait probe failed; retrying in ${BUILD_WAIT_POLL_MS / 1000}s (${elapsed(t0)})`);
await new Promise((resolve) => setTimeout(resolve, BUILD_WAIT_POLL_MS));
continue;
}
if (!waitState.shouldKeepPolling) {
if (waitState.timedOut) {
deployExit = 124;
console.log(` [${scenario.slug}] Deploy wait reached phase timeout (${TIMEOUT_MS / 1000}s) (${elapsed(t0)})`);
} else if (waitState.stale) {
const exitCode = await readClaudeExitCode(deployExitFile, deployLogFile);
if (exitCode !== null) deployExit = exitCode;
console.log(` [${scenario.slug}] Deploy stale — no new debug log activity for ${BUILD_STALE_THRESHOLD_MS / 1000}s, treating as done | exit=${exitCode ?? "unknown"} (${elapsed(t0)})`);
} else {
const exitCode = await readClaudeExitCode(deployExitFile, deployLogFile);
if (exitCode !== null) deployExit = exitCode;
console.log(` [${scenario.slug}] Deploy process finished | claude=${waitState.claudeRunning ? "running" : "exited"} | sessionEnd=${waitState.sessionEnded ? "yes" : "no"} | exit=${exitCode ?? "unknown"} (${elapsed(t0)})`);
}
break;
}
await new Promise((resolve) => setTimeout(resolve, BUILD_WAIT_POLL_MS));
}
}
deployOut = await sh(sandbox, `cat ${deployLogFile} 2>/dev/null`);
timing.deployEndMs = captureRelativeMs(t0);
// Extract deploy URL from output (scan for any vercel.app URL)
const urlMatch = deployOut.match(/(https:\/\/[^\s]+\.vercel\.app)/);
if (urlMatch) {
deployUrl = urlMatch[1];
console.log(` [${scenario.slug}] Deployed: ${deployUrl} (${elapsed(t0)})`);
}
// Re-extract skills after deploy phase
const postDeploySkills = (await sh(sandbox, "ls /tmp/vercel-plugin-*-seen-skills.d/ 2>/dev/null")).split("\n").filter(Boolean);
if (postDeploySkills.length > claimedSkills.length) {
const newSkills = postDeploySkills.filter(s => !claimedSkills.includes(s));
if (newSkills.length > 0) {
console.log(` [${scenario.slug}] +${newSkills.length} skills from deploy: ${newSkills.join(", ")}`);
claimedSkills.push(...newSkills);
}
}
// Score deploy results with haiku
deployScore = await scoreWithHaiku(sandbox, claudeBin,
`A Vercel deploy was attempted. Here's the output (last 1500 chars):\n\n${deployOut.slice(-1500)}\n\nDid the deploy succeed? Extract the URL if present. List any errors.`,
DEPLOY_SCORE_SCHEMA, "deploy", scenario.slug, t0,
);
if (deployScore) {
const ds = deployScore as any;
console.log(` [${scenario.slug}] Deploy score: deployed=${ds.deployed} build=${ds.buildSucceeded}${ds.url ? ` url=${ds.url}` : ""} (${elapsed(t0)})`);
// If haiku found a URL we missed, use it
if (!deployUrl && ds.url && ds.url.includes("vercel.app")) {
deployUrl = ds.url;
console.log(` [${scenario.slug}] Deploy URL from haiku: ${deployUrl}`);
}
}
if (!deployUrl) {
console.log(` [${scenario.slug}] Deploy failed (exit=${deployExit}) (${elapsed(t0)})`);
}
}
// 10. Extract source code + debug logs to local filesystem
let sourcePath: string | undefined;
const archiveDir = join(RESULTS_DIR, runId, scenario.slug);
await mkdir(archiveDir, { recursive: true });
const extractStartedAt = performance.now();
const extractedArtifacts: ArtifactManifestEntry[] = [];
const trackArtifact = async (
fileName: string,
description: string,
content: Buffer | string,
): Promise<ArtifactManifestEntry> => {
const artifact = await writeArchiveArtifact(archiveDir, fileName, description, content);
extractedArtifacts.push(artifact);
console.log(` [${scenario.slug}] Artifact saved | file=${fileName} | sizeKB=${(artifact.sizeBytes / 1024).toFixed(0)} (${elapsed(t0)})`);
return artifact;
};
// Always extract debug logs (even if build produced few files)
try {
await sh(sandbox, `tar czf /tmp/debug-logs.tar.gz -C ${SANDBOX_HOME} .claude/debug/ 2>/dev/null`);
const debugBuffer = await readSandboxFileBuffer(sandbox, "/tmp/debug-logs.tar.gz");
if (!debugBuffer) throw new Error("sandbox file /tmp/debug-logs.tar.gz missing");
await trackArtifact("debug-logs.tar.gz", "Full Claude debug logs captured from ~/.claude/debug.", debugBuffer);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=debug-logs.tar.gz | attempted=tar-debug-logs | error=${e.message?.slice(0, 160)}`);
}
// Extract Claude Code stdout/stderr from build + verify + deploy phases
try {
await sh(sandbox, "tar czf /tmp/claude-output.tar.gz /tmp/claude-build-round-*.log /tmp/claude-build-round-*.err /tmp/claude-verify.log /tmp/claude-verify.err /tmp/claude-deploy.log /tmp/claude-deploy.err 2>/dev/null");
const claudeOutputBuffer = await readSandboxFileBuffer(sandbox, "/tmp/claude-output.tar.gz");
if (!claudeOutputBuffer) throw new Error("sandbox file /tmp/claude-output.tar.gz missing");
await trackArtifact("claude-output.tar.gz", "Claude build/verify/deploy stdout/stderr logs.", claudeOutputBuffer);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=claude-output.tar.gz | attempted=tar-claude-output | error=${e.message?.slice(0, 160)}`);
}
try {
await sh(sandbox, `cd ${SANDBOX_HOME} && tar czf /tmp/source.tar.gz --exclude='node_modules' --exclude='.next' --exclude='.git' ${scenario.slug}/ 2>/dev/null`);
const sourceBuffer = await readSandboxFileBuffer(sandbox, "/tmp/source.tar.gz");
if (!sourceBuffer) throw new Error("sandbox file /tmp/source.tar.gz missing");
const artifact = await trackArtifact("source.tar.gz", "Generated project source tree excluding node_modules, .next, and .git.", sourceBuffer);
sourcePath = join(archiveDir, artifact.fileName);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=source.tar.gz | attempted=tar-project-source | error=${e.message?.slice(0, 160)}`);
}
try {
const hookTrace = await sh(
sandbox,
"grep -RniE 'vercel-plugin|VERCEL_PLUGIN|PreToolUse|PostToolUse|SessionStart|SessionEnd|UserPromptSubmit' ~/.claude/debug 2>/dev/null || true",
);
const hookTraceOutput = normalizeShellOutput(hookTrace) || "No hook/session trace lines matched.";
await trackArtifact("hook-trace.txt", "Filtered hook and session trace lines from Claude debug logs.", `${hookTraceOutput}\n`);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=hook-trace.txt | attempted=grep-hook-trace | error=${e.message?.slice(0, 160)}`);
}
try {
const fileTree = await sh(
sandbox,
`find ${projectDir} -not -path '*/node_modules/*' -not -path '*/.next/*' -not -path '*/.git/*' -type f -exec ls -la {} + 2>/dev/null || true`,
);
const fileTreeOutput = normalizeShellOutput(fileTree) || "No project files found.";
await trackArtifact("file-tree.txt", "Readable file tree for the generated project.", `${fileTreeOutput}\n`);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=file-tree.txt | attempted=find-project-files | error=${e.message?.slice(0, 160)}`);
}
try {
const pluginEnv = await sh(sandbox, "env | grep -i vercel_plugin || true");
const pluginEnvOutput = normalizeShellOutput(pluginEnv) || "No VERCEL_PLUGIN environment variables found.";
await trackArtifact("plugin-env.txt", "Sandbox environment variables matching VERCEL_PLUGIN.", `${pluginEnvOutput}\n`);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=plugin-env.txt | attempted=grep-plugin-env | error=${e.message?.slice(0, 160)}`);
}
try {
const npmGlobals = await sh(sandbox, "npm ls -g --depth=0 2>/dev/null || true");
const npmGlobalsOutput = normalizeShellOutput(npmGlobals) || "npm global package listing returned no output.";
await trackArtifact("npm-globals.txt", "Globally installed npm packages inside the sandbox.", `${npmGlobalsOutput}\n`);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=npm-globals.txt | attempted=npm-ls-global | error=${e.message?.slice(0, 160)}`);
}
try {
const settingsExists = await sh(sandbox, `test -f ${settingsPath} && echo present || true`);
if (normalizeShellOutput(settingsExists) === "present") {
const settingsBuffer = await readSandboxFileBuffer(sandbox, settingsPath);
if (!settingsBuffer) throw new Error(`sandbox file ${settingsPath} missing`);
await trackArtifact("settings.json", "Claude settings generated for the scenario project.", settingsBuffer);
}
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=settings.json | attempted=copy-settings | error=${e.message?.slice(0, 160)}`);
}
try {
const toolCallsRaw = await sh(sandbox, "grep -Rni 'executePreToolHooks called' ~/.claude/debug 2>/dev/null || true");
await trackArtifact("tool-calls.txt", "Pre-tool hook calls with parsed timestamp and tool name.", `${summarizeToolCalls(toolCallsRaw)}\n`);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=tool-calls.txt | attempted=grep-pretool-hooks | error=${e.message?.slice(0, 160)}`);
}
try {
const claimDirs = await sh(sandbox, "ls -la /tmp/vercel-plugin-*-seen-skills.d/ 2>/dev/null || true");
const claimFiles = await sh(sandbox, "cat /tmp/vercel-plugin-*-seen-skills.txt 2>/dev/null || true");
const claimsOutput = [
"=== claim directories ===",
normalizeShellOutput(claimDirs) || "(none)",
"",
"=== claim files ===",
normalizeShellOutput(claimFiles) || "(none)",
].join("\n");
await trackArtifact("skill-claims.txt", "Seen-skills claim directories and files emitted by the plugin.", `${claimsOutput}\n`);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=skill-claims.txt | attempted=collect-skill-claims | error=${e.message?.slice(0, 160)}`);
}
try {
const manifestPath = join(archiveDir, "manifest.json");
const manifestEntry: ArtifactManifestEntry = {
fileName: "manifest.json",
sizeBytes: 0,
description: "Manifest of archived observability artifacts.",
};
const manifestEntries = [...extractedArtifacts, manifestEntry];
for (let attempt = 0; attempt < 3; attempt++) {
await writeFile(manifestPath, JSON.stringify(buildArtifactManifest(manifestEntries), null, 2));
const manifestStats = await stat(manifestPath);
if (manifestEntry.sizeBytes === manifestStats.size) break;
manifestEntry.sizeBytes = manifestStats.size;
}
if (!extractedArtifacts.some((artifact) => artifact.fileName === "manifest.json")) {
extractedArtifacts.push(manifestEntry);
}
console.log(` [${scenario.slug}] Artifact saved | file=manifest.json | sizeKB=${(manifestEntry.sizeBytes / 1024).toFixed(0)} (${elapsed(t0)})`);
} catch (e: any) {
console.log(` [${scenario.slug}] Artifact extract failed | artifact=manifest.json | attempted=write-manifest | error=${e.message?.slice(0, 160)}`);
}
timing.extractMs = captureElapsedMs(extractStartedAt);
console.log(` [${scenario.slug}] DONE (${elapsed(t0)}) | skills=${claimedSkills.length} | files=${projectFilesList.length}${deployUrl ? ` | ${deployUrl}` : appUrl ? ` | ${appUrl}` : ""}`);
// Clean up snapshot (it has a 4h expiry but be tidy)
if (snapshotId) {
try {
const { Snapshot } = await import("@vercel/sandbox");
const snap = await Snapshot.get({ snapshotId });
await snap.delete();
console.log(` [${scenario.slug}] Snapshot ${snapshotId} cleaned up (${elapsed(t0)})`);
} catch {}
}
return {
slug: scenario.slug,
sandboxId: sandbox.sandboxId,
snapshotId,
success: buildExit === 0 || buildExit === 124,
durationMs: performance.now() - t0,
claimedSkills,
expectedSkills: scenario.expectedSkills,
projectFiles: projectFilesList,
appUrl,
deployUrl,
sourcePath,
pollHistory,
verification,
buildScore,
deployScore,
timing,
};
} catch (err: any) {
console.error(` [${scenario.slug}] ERROR: ${err.message?.slice(0, 200)}`);
return {
slug: scenario.slug,
sandboxId: sandbox?.sandboxId ?? "unknown",
success: false,
durationMs: performance.now() - t0,
claimedSkills: [],
expectedSkills: scenario.expectedSkills,
projectFiles: [],
error: err.message?.slice(0, 400),
pollHistory,
timing,
};
} finally {
if (sandbox && !KEEP_ALIVE) {
try { await sandbox.stop(); } catch {}
}
}
}
// ---------------------------------------------------------------------------
// Report generation
// ---------------------------------------------------------------------------
async function generateReport(
runId: string,
results: ScenarioResult[],
totalMs: number,
resultsPath: string,
): Promise<string> {
const ts = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
const reportsDir = join(LOCAL_PLUGIN_DIR, ".reports");
await mkdir(reportsDir, { recursive: true });
const reportPath = join(reportsDir, `${ts}.md`);
const scenarioMap = Object.fromEntries(ACTIVE_SCENARIOS.map(s => [s.slug, s]));
const totalSkills = new Set(results.flatMap(r => r.claimedSkills));
const verified = results.filter(r => r.verification?.ran);
const totalStories = verified.reduce((a, r) => a + r.verification!.stories.length, 0);
const passedStories = verified.reduce((a, r) => a + r.verification!.stories.filter(s => s.status === "pass").length, 0);
let md = `# Sandbox Eval Report — ${ts}\n\n`;
md += `| Metric | Value |\n|--------|-------|\n`;
md += `| Run ID | \`${runId}\` |\n`;
md += `| Date | ${new Date().toISOString()} |\n`;
md += `| Duration | ${(totalMs / 1000).toFixed(0)}s |\n`;
md += `| Scenarios | ${results.length} |\n`;
md += `| Builds succeeded | ${results.filter(r => r.success).length}/${results.length} |\n`;
md += `| Unique skills injected | ${totalSkills.size} |\n`;
md += `| User stories verified | ${passedStories}/${totalStories} |\n`;
md += `| Results JSON | \`${resultsPath}/results.json\` |\n\n`;
// Summary table
md += `## Summary\n\n`;
md += `| Scenario | Build | Skills | Files | Verify | Duration | Deploy URL |\n`;
md += `|----------|-------|--------|-------|--------|----------|------------|\n`;
for (const r of results) {
const build = r.success ? "OK" : "FAIL";
const verify = r.verification
? `${r.verification.stories.filter(s => s.status === "pass").length}/${r.verification.stories.length}`
: "skip";
const url = r.deployUrl ? `[${r.slug}](${r.deployUrl})` : r.appUrl ? `[sandbox](${r.appUrl})` : "—";
md += `| ${r.slug} | ${build} | ${r.claimedSkills.length} | ${r.projectFiles.length} | ${verify} | ${(r.durationMs / 1000).toFixed(0)}s | ${url} |\n`;
}
md += `\n${renderTimingMarkdownTable(results)}\n`;
// Deployed + Live URLs
const deployed = results.filter(r => r.deployUrl);
const liveApps = results.filter(r => r.appUrl);
if (deployed.length > 0) {
md += `\n## Deployed URLs (permanent)\n\n`;
for (const r of deployed) md += `- **${r.slug}**: ${r.deployUrl}\n`;
}
if (liveApps.length > 0) {
md += `\n## Sandbox URLs (temporary)\n\n`;
for (const r of liveApps) md += `- **${r.slug}**: ${r.appUrl}\n`;
}
// Per-scenario details
md += `\n## Scenario Details\n`;
for (const r of results) {
const scenario = scenarioMap[r.slug];
md += `\n### ${r.slug}\n\n`;
md += `**Sandbox ID**: \`${r.sandboxId}\`\n`;
if (r.deployUrl) md += `**Deploy URL**: ${r.deployUrl}\n`;
if (r.appUrl) md += `**Sandbox URL**: ${r.appUrl}\n`;
md += `**Duration**: ${(r.durationMs / 1000).toFixed(0)}s\n`;
md += `**Build**: ${r.success ? "OK" : "FAIL"}`;
if (r.error) md += ` — \`${r.error.slice(0, 100)}\``;
md += `\n`;
if (r.sourcePath) {
md += `**Source archive**: \`${r.sourcePath}\`\n`;
md += `\nExtract source locally:\n`;
md += `\`\`\`bash\nmkdir -p /tmp/${r.slug} && tar xzf "${r.sourcePath}" -C /tmp/${r.slug}\ncd /tmp/${r.slug}/${r.slug} && npm install && npx next dev\n\`\`\`\n`;
}
md += `\n`;
// Prompt
if (scenario) {
md += `<details><summary>Build Prompt</summary>\n\n${scenario.prompt}\n\n</details>\n\n`;
}
// Skills
md += `**Skills injected (${r.claimedSkills.length})**:`;
if (r.claimedSkills.length > 0) {
md += ` ${r.claimedSkills.join(", ")}\n`;
} else {
md += ` (none)\n`;
}
// Expected vs actual
if (scenario) {
const expected = new Set(scenario.expectedSkills);
const actual = new Set(r.claimedSkills);
const hit = [...expected].filter(s => actual.has(s));
const miss = [...expected].filter(s => !actual.has(s));
const extra = [...actual].filter(s => !expected.has(s));
md += `**Expected**: ${scenario.expectedSkills.join(", ")}\n`;
md += `**Match**: ${hit.length}/${expected.size}`;
if (miss.length) md += ` | Missing: ${miss.join(", ")}`;
if (extra.length) md += ` | Bonus: ${extra.join(", ")}`;
md += `\n`;
}
// Project files
if (r.projectFiles.length > 0) {
md += `\n**Project files (${r.projectFiles.length})**:\n`;
md += `\`\`\`\n${r.projectFiles.map(f => f.split("/").slice(-2).join("/")).join("\n")}\n\`\`\`\n`;
}
// Skill injection timeline (from polls)
if (r.pollHistory.length > 0) {
md += `\n**Skill injection timeline**:\n`;
let prevSkills = new Set<string>();
for (const p of r.pollHistory) {
const curr = new Set(p.skills);
const newSkills = [...curr].filter(s => !prevSkills.has(s));
if (newSkills.length > 0) {
md += `- ${p.elapsed}: +${newSkills.join(", ")} (total: ${curr.size}, files: ${p.files})\n`;
}
prevSkills = curr;
}
}
// Build score (from haiku)
if (r.buildScore) {
const bs = r.buildScore as any;
md += `\n**Build Score** (haiku): ${bs.completeness} | API=${bs.hasApiRoutes} UI=${bs.hasUIComponents} AI=${bs.hasAIFeature} Server=${bs.devServerRunning}\n`;
if (bs.missingFeatures?.length) md += `Missing: ${bs.missingFeatures.join(", ")}\n`;
if (bs.summary) md += `> ${bs.summary}\n`;
}
// Deploy score (from haiku)
if (r.deployScore) {
const ds = r.deployScore as any;
md += `\n**Deploy Score** (haiku): deployed=${ds.deployed} build=${ds.buildSucceeded}${ds.url ? ` url=${ds.url}` : ""}\n`;
if (ds.errors?.length) md += `Errors: ${ds.errors.join(", ")}\n`;
if (ds.summary) md += `> ${ds.summary}\n`;
}
// Verification
if (r.verification?.ran) {
md += `\n**Verification** (exit=${r.verification.exitCode}):\n`;
for (let i = 0; i < r.verification.stories.length; i++) {
const s = r.verification.stories[i];
const story = scenario?.userStories[i] ?? `Story ${s.index}`;
const icon = s.status === "pass" ? "PASS" : s.status === "fail" ? "FAIL" : "???";
md += `- **${icon}**: ${story}\n`;
}
if (r.verification.output) {
md += `\n<details><summary>Verification output (last 500 chars)</summary>\n\n\`\`\`\n${r.verification.output}\n\`\`\`\n\n</details>\n`;
}
}
}
// Aggregate skill coverage
md += `\n## Aggregate Skill Coverage\n\n`;
md += `**${totalSkills.size} unique skills** injected across ${results.length} scenarios:\n`;
md += [...totalSkills].sort().map(s => `\`${s}\``).join(", ") + "\n";
await writeFile(reportPath, md);
// Also write to results dir for easy access alongside results.json
await writeFile(join(resultsPath, "report.md"), md);
console.log(`\nReport: ${reportPath}`);
console.log(`Report copy: ${join(resultsPath, "report.md")}`);
return reportPath;
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
const t0 = performance.now();
runId = `eval-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}`;
const resultsPath = join(RESULTS_DIR, runId);
await mkdir(resultsPath, { recursive: true });
const filtered = SCENARIO_FILTER.length > 0
? ACTIVE_SCENARIOS.filter(s => SCENARIO_FILTER.includes(s.slug))
: ACTIVE_SCENARIOS;
console.log("=== Sandbox Eval Runner: Build → Verify → Deploy ===");
console.log(`Scenarios: ${filtered.length}${SCENARIO_FILTER.length ? ` (filtered: ${SCENARIO_FILTER.join(", ")})` : ""}`);
console.log(`Concurrency: ${CONCURRENCY}`);
console.log(`Timeout: ${TIMEOUT_MS / 1000}s per phase`);
console.log(`Phases: Build${SKIP_VERIFY ? "" : " → Verify"}${SKIP_DEPLOY ? "" : " → Deploy"}`);
console.log(`Keep-alive: ${KEEP_ALIVE ? `${KEEP_ALIVE_HOURS}h` : "OFF"}`);
console.log(`Results: ${resultsPath}\n`);
const apiKey = resolveApiKey();
const baseUrl = "https://ai-gateway.vercel.sh";
const vercelToken = resolveVercelToken();
console.log("Collecting plugin files...");
const pluginFiles = await collectPluginFiles();
console.log(` ${pluginFiles.length} files (${(pluginFiles.reduce((a, f) => a + f.content.length, 0) / 1024).toFixed(0)}KB)\n`);
const queue = [...filtered];
const results: ScenarioResult[] = [];
async function worker(): Promise<void> {
while (queue.length > 0) {
const scenario = queue.shift()!;
console.log(`\n--- ${scenario.slug} ---`);
const result = await runScenario(scenario, apiKey, baseUrl, vercelToken, pluginFiles);
results.push(result);
// Write individual result immediately so it survives crashes
try {
const scenarioDir = join(resultsPath, result.slug);
await mkdir(scenarioDir, { recursive: true });
await writeFile(join(scenarioDir, "result.json"), JSON.stringify(result, null, 2));
// Also update the aggregate results.json with everything so far
await writeFile(join(resultsPath, "results.json"), JSON.stringify({ runId, results, totalMs: performance.now() - t0, complete: false }, null, 2));
} catch {}
}
}
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, filtered.length) }, () => worker()));
// Save final results (marks complete: true)
const totalMs = performance.now() - t0;
await writeFile(join(resultsPath, "results.json"), JSON.stringify({ runId, results, totalMs, complete: true }, null, 2));
// Generate markdown report
await generateReport(runId, results, totalMs, resultsPath);
// Print summary
console.log("\n\n=== SUMMARY ===");
console.log(`${"Slug".padEnd(22)} ${"Build".padEnd(6)} ${"Skills".padEnd(6)} ${"Files".padEnd(6)} ${"Verify".padEnd(10)} Duration`);
console.log("-".repeat(80));
for (const r of results) {
const build = r.success ? "OK" : "FAIL";
const verify = r.verification
? `${r.verification.stories.filter(s => s.status === "pass").length}/${r.verification.stories.length}`
: "skip";
console.log(`${r.slug.padEnd(22)} ${build.padEnd(6)} ${String(r.claimedSkills.length).padEnd(6)} ${String(r.projectFiles.length).padEnd(6)} ${verify.padEnd(10)} ${(r.durationMs / 1000).toFixed(0)}s`);
}
// Verification details
const verified = results.filter(r => r.verification?.ran);
if (verified.length > 0) {
console.log("\n=== VERIFICATION DETAILS ===");
for (const r of verified) {
console.log(`\n ${r.slug}:`);
for (const s of r.verification!.stories) {
const icon = s.status === "pass" ? "✓" : s.status === "fail" ? "✗" : "?";
console.log(` ${icon} Story ${s.index}: ${s.status.toUpperCase()}`);
}
}
const totalStories = verified.reduce((a, r) => a + r.verification!.stories.length, 0);
const passedStories = verified.reduce((a, r) => a + r.verification!.stories.filter(s => s.status === "pass").length, 0);
console.log(`\n Total: ${passedStories}/${totalStories} stories passed`);
}
// App URLs
const appsWithUrls = results.filter(r => r.appUrl);
if (appsWithUrls.length > 0) {
console.log("\n=== APP URLs ===");
for (const r of appsWithUrls) console.log(` ${r.slug}: ${r.appUrl}`);
}
// Skill coverage
console.log("\n=== SKILL COVERAGE ===");
for (const r of results) {
const expected = new Set(r.expectedSkills);
const actual = new Set(r.claimedSkills);
const hit = [...expected].filter(s => actual.has(s));
const miss = [...expected].filter(s => !actual.has(s));
const extra = [...actual].filter(s => !expected.has(s));
console.log(` ${r.slug}: ${hit.length}/${expected.size} expected | +${extra.length} bonus | -${miss.length} missing`);
if (miss.length) console.log(` missing: ${miss.join(", ")}`);
}
if (!KEEP_ALIVE) {
const allPassed = results.every(r => r.success);
process.exit(allPassed ? 0 : 1);
}
// Keep-alive mode
if (appsWithUrls.length > 0) {
console.log(`\n=== SANDBOXES KEPT ALIVE (${KEEP_ALIVE_HOURS}h) ===`);
for (const r of appsWithUrls) console.log(` ${r.slug}: ${r.appUrl}`);
await writeFile(join(resultsPath, "live-urls.json"), JSON.stringify(
Object.fromEntries(appsWithUrls.map(r => [r.slug, { url: r.appUrl, sandboxId: r.sandboxId }])),
null, 2,
));
console.log(`\nPress Ctrl+C to stop all sandboxes.\n`);
await new Promise(() => {});
}
}
if (import.meta.main) {
main().catch(e => { console.error("Fatal:", e); process.exit(2); });
}