Curio/tests/golden/run-misconception-eval.ts

/**
 * Misconception hit-rate eval.
 *
 * Measures: given a response exhibiting a known misconception, does the
 * grader return the correct tag when the library is provided?
 *
 * Two conditions run per case:
 *   WITH library  — grader receives the misconception library (expected: hit the tag)
 *   WITHOUT library — grader receives empty library (baseline: "novel" or "none")
 *
 * CI gate: WITH-library hit rate must exceed HIT_RATE_THRESHOLD.
 */

import { generateObject } from 'ai';
import { GradeOutputSchema } from '../../src/schemas/grading';
import type { GradeOutput } from '../../src/schemas/grading';
import { prompts } from '../../src/lib/llm/prompts';
import { llmClient } from '../../src/lib/llm/client';
import {
  MISCONCEPTION_CASES,
  HIT_RATE_THRESHOLD,
  BASELINE_HIT_RATE,
} from './misconception-cases';
import type { MisconceptionCase } from './misconception-cases';

// ── Prompt builder ────────────────────────────────────────────────────────────

function buildPrompt(
  c: MisconceptionCase,
  library: Array<{ tag: string; signature: string }>,
): string {
  const rubricLines = c.rubric
    .map((r) => `  - [${r.id}] ${r.criterion} (weight: ${r.weight.toFixed(2)})`)
    .join('\n');

  const miscLines =
    library.length > 0
      ? library.map((m) => `  - tag: "${m.tag}"\n    signature: ${m.signature}`).join('\n')
      : '  (none)';

  return `Checkpoint prompt: ${c.checkpointPrompt}

Reference answer: ${c.referenceAnswer}

Rubric criteria:
${rubricLines}

Known misconceptions for this concept:
${miscLines}

Learner's response:
"""
${c.learnerResponse}
"""`;
}

// ── Single grading call ───────────────────────────────────────────────────────

async function grade(
  c: MisconceptionCase,
  library: Array<{ tag: string; signature: string }>,
): Promise<{ grade: GradeOutput | null; error?: string; latencyMs: number }> {
  const start = Date.now();
  try {
    const { object } = await generateObject({
      model: llmClient.grader,
      schema: GradeOutputSchema,
      system: prompts.GRADE_RESPONSE.template,
      prompt: buildPrompt(c, library),
    });
    return { grade: object, latencyMs: Date.now() - start };
  } catch (err) {
    return {
      grade: null,
      error: err instanceof Error ? err.message : String(err),
      latencyMs: Date.now() - start,
    };
  }
}

// ── Runner ────────────────────────────────────────────────────────────────────

async function main(): Promise<void> {
  console.log(`\nCurio — misconception hit-rate eval (${MISCONCEPTION_CASES.length} cases × 2 conditions)\n`);
  console.log(`  Grader: ${process.env.LLM_GRADER_PROVIDER ?? 'openai'} / ${process.env.LLM_GRADER_MODEL ?? 'gpt-4o-mini'}`);
  console.log(`  Hit-rate threshold: ${(HIT_RATE_THRESHOLD * 100).toFixed(0)}%\n`);

  let withHits = 0;
  let baselineHits = 0;
  let errors = 0;

  for (const c of MISCONCEPTION_CASES) {
    process.stdout.write(`  [${c.id}] "${c.label}"…\r`);

    const [withResult, baselineResult] = await Promise.all([
      grade(c, c.library),
      grade(c, []),
    ]);

    const withTag = withResult.grade?.misconception_tag;
    const baselineTag = baselineResult.grade?.misconception_tag;

    const withHit = withTag === c.targetTag;
    const baselineHit = baselineTag === c.targetTag;

    if (withResult.error || baselineResult.error) errors++;
    if (withHit) withHits++;
    if (baselineHit) baselineHits++;

    const withStatus = withResult.error ? '  ERR ' : withHit ? '✓ hit ' : '✗ miss';
    const bStatus = baselineResult.error ? '  ERR ' : baselineHit ? '✓ hit ' : '✗ miss';

    console.log(
      `  [${c.id}]  WITH=${withStatus} (tag="${withTag ?? 'null'}")  BASE=${bStatus} (tag="${baselineTag ?? 'null'}")`,
    );
  }

  const n = MISCONCEPTION_CASES.length;
  const withRate = n > 0 ? withHits / n : 0;
  const baseRate = n > 0 ? baselineHits / n : 0;
  const lift = withRate - baseRate;

  console.log('\n── Results ─────────────────────────────────────────────────────────────');
  console.log(`  Cases:            ${n} (${errors} errors)`);
  console.log(`  WITH library:     ${withHits}/${n} = ${(withRate * 100).toFixed(1)}%  (threshold: ${(HIT_RATE_THRESHOLD * 100).toFixed(0)}%)`);
  console.log(`  Baseline (no lib): ${baselineHits}/${n} = ${(baseRate * 100).toFixed(1)}%  (expected: ${(BASELINE_HIT_RATE * 100).toFixed(0)}%)`);
  console.log(`  Lift:             +${(lift * 100).toFixed(1)}pp`);

  const failed = withRate < HIT_RATE_THRESHOLD;

  if (failed) {
    console.log(
      `\n✗ FAIL: hit rate ${(withRate * 100).toFixed(1)}% below threshold ${(HIT_RATE_THRESHOLD * 100).toFixed(0)}%`,
    );
  } else {
    console.log('\n✓ PASS: misconception hit rate within threshold\n');
  }

  process.exit(failed ? 1 : 0);
}

main().catch((err) => {
  console.error('Eval runner crashed:', err);
  process.exit(1);
});