types.ts

GitHub
/**
 * Stable contracts for the sandbox benchmark runner.
 *
 * schema_version tracks breaking changes. Consumers should check this field
 * and reject unknown versions rather than silently misinterpreting data.
 */

// ---------------------------------------------------------------------------
// RunnerEvent — structured NDJSON log entries (stderr when --log-format json)
// ---------------------------------------------------------------------------

export const RUNNER_EVENT_NAMES = [
  "runner.start",
  "runner.completed",
  "runner.failed",
  "snapshot.cache.hit",
  "snapshot.cache.miss",
  "snapshot.create.started",
  "snapshot.create.succeeded",
  "snapshot.verify.started",
  "snapshot.verify.succeeded",
  "snapshot.verify.failed",
  "plugin.upload.started",
  "plugin.upload.chunk",
  "plugin.upload.completed",
  "scenario.start",
  "scenario.completed",
  "scenario.command.failed",
  "scenario.timeout",
  "artifact.extract.succeeded",
  "artifact.extract.failed",
] as const;

export type RunnerEventName = (typeof RUNNER_EVENT_NAMES)[number];

export interface RunnerEvent {
  schema_version: 1;
  run_id: string;
  event: RunnerEventName;
  timestamp: string; // ISO 8601
  payload: Record<string, unknown>;
}

// ---------------------------------------------------------------------------
// RunSummary — final machine-readable output (stdout when --json)
// ---------------------------------------------------------------------------

export type RunStatus = "pass" | "fail" | "timeout" | "error";

export interface ScenarioSummary {
  slug: string;
  sandbox_id: string;
  status: RunStatus;
  duration_ms: number;
  session_method: string;
  expected_skills: string[];
  claimed_skills: string[];
  hook_evidence: {
    claim_dirs: boolean;
    seen_file: boolean;
    debug_log_count: number;
    pre_tool_use_in_stderr: boolean;
    user_prompt_in_stderr: boolean;
  };
  error?: string;
}

export interface SnapshotMeta {
  snapshot_id: string;
  cached: boolean;
  age_hours?: number;
  creation_duration_ms?: number;
}

export interface RunSummary {
  schema_version: 1;
  run_id: string;
  status: RunStatus;
  timestamp: string; // ISO 8601
  snapshot: SnapshotMeta;
  scenarios: ScenarioSummary[];
  timing: {
    total_duration_ms: number;
    scenario_durations_ms: Record<string, number>;
  };
  config: {
    concurrency: number;
    timeout_ms: number;
    results_dir: string;
    mode: "full" | "quick" | "single";
  };
}

// ---------------------------------------------------------------------------
// RunnerError — typed error model with actionable hints
// ---------------------------------------------------------------------------

export const RUNNER_ERROR_CODES = [
  "SNAPSHOT_CACHE_STALE",
  "SNAPSHOT_VERIFY_FAILED",
  "PLUGIN_UPLOAD_CHUNK_FAILED",
  "SANDBOX_PATH_UNRESOLVED",
  "SCENARIO_LOAD_FAILED",
] as const;

export type RunnerErrorCode = (typeof RUNNER_ERROR_CODES)[number];

const ERROR_HINTS: Record<RunnerErrorCode, string> = {
  SNAPSHOT_CACHE_STALE:
    "Run with --force-snapshot to recreate the base snapshot.",
  SNAPSHOT_VERIFY_FAILED:
    "The cached snapshot may be corrupted. Delete the cache file and retry.",
  PLUGIN_UPLOAD_CHUNK_FAILED:
    "Sandbox writeFiles() failed for a chunk. Check sandbox API limits or retry.",
  SANDBOX_PATH_UNRESOLVED:
    "A sandbox path could not be resolved. Verify SANDBOX_PLUGIN_DIR and project directory.",
  SCENARIO_LOAD_FAILED:
    "Scenarios could not be loaded. Ensure scripts/benchmark-scenarios.js exists or benchmark-runner.ts is parseable.",
};

const ERROR_RETRYABLE: Record<RunnerErrorCode, boolean> = {
  SNAPSHOT_CACHE_STALE: true,
  SNAPSHOT_VERIFY_FAILED: true,
  PLUGIN_UPLOAD_CHUNK_FAILED: true,
  SANDBOX_PATH_UNRESOLVED: false,
  SCENARIO_LOAD_FAILED: false,
};

export class RunnerError extends Error {
  readonly code: RunnerErrorCode;
  readonly hint: string;
  readonly retryable: boolean;
  readonly cause?: Error;

  constructor(code: RunnerErrorCode, message: string, cause?: Error) {
    super(message);
    this.name = "RunnerError";
    this.code = code;
    this.hint = ERROR_HINTS[code];
    this.retryable = ERROR_RETRYABLE[code];
    this.cause = cause;
  }

  toJSON() {
    return {
      code: this.code,
      message: this.message,
      hint: this.hint,
      retryable: this.retryable,
      cause: this.cause?.message,
    };
  }
}

// ---------------------------------------------------------------------------
// Exit codes
// ---------------------------------------------------------------------------

/** Exit 0: all scenarios passed */
export const EXIT_ALL_PASS = 0;
/** Exit 1: one or more scenarios failed/timed out */
export const EXIT_SOME_FAIL = 1;
/** Exit 2: fatal runner error (snapshot failure, scenario load failure, etc.) */
export const EXIT_FATAL = 2;

// ---------------------------------------------------------------------------
// Helpers to convert internal ScenarioResult → ScenarioSummary
// ---------------------------------------------------------------------------

export function scenarioStatus(result: {
  success: boolean;
  timedOut: boolean;
  error?: string;
}): RunStatus {
  if (result.timedOut) return "timeout";
  if (result.error) return "error";
  if (result.success) return "pass";
  return "fail";
}

export function overallStatus(scenarios: ScenarioSummary[]): RunStatus {
  if (scenarios.some((s) => s.status === "error")) return "error";
  if (scenarios.every((s) => s.status === "pass")) return "pass";
  if (scenarios.some((s) => s.status === "timeout")) return "timeout";
  return "fail";
}