Curio/tests/golden/content-cases.ts

/**
 * Frozen golden eval cases for content verification (T1 + T2 cascade).
 *
 * Each case provides a synthetic segment + checkpoint + source chunks.
 * The verifier should catch bad content (unsupported claims, wrong reference answers)
 * and pass good content (well-grounded, correct reference answer).
 *
 * expectedT1Verdict: what T1 (cheap model) should return.
 * isBadContent: true → verifier MUST return fail (false-pass = catastrophic).
 * isGoodContent: true → verifier should return pass or uncertain (false-fail = trust erosion).
 */

export interface ContentCase {
  id: string;
  label: string;
  segmentText: string;
  sourceChunks: Array<{ docRef: string; text: string }>;
  checkpointPrompt: string;
  /** Only provided to T2, not T1. */
  referenceAnswer: string;
  rubric: Array<{ id: string; criterion: string; weight: number }>;
  expectedT1Verdict: 'pass' | 'fail' | 'uncertain';
  isBadContent: boolean;
  isGoodContent: boolean;
}

const CLOSURE_CHUNK = {
  docRef: 'js-closures/definition',
  text: `A closure is a function that retains access to its lexical scope even after the outer function has returned. When a function is created, it captures a reference to its surrounding scope. This means variables defined in the outer function remain accessible to the inner function — they are not garbage collected as long as the inner function exists.

Example:
\`\`\`js
function makeGreeter(name) {
  return function() {
    return 'Hello, ' + name;
  };
}
const greetAlice = makeGreeter('Alice');
greetAlice(); // 'Hello, Alice'
\`\`\`

In this example, the inner function closes over the \`name\` variable. Even after \`makeGreeter\` returns, the inner function can still access \`name\`.`,
};

const LOOP_CHUNK = {
  docRef: 'js-closures/loop-bug',
  text: `A common pitfall: closures in loops with \`var\`.

\`\`\`js
for (var i = 0; i < 3; i++) {
  setTimeout(() => console.log(i), 0);
}
// logs: 3, 3, 3
\`\`\`

Because \`var\` is function-scoped (not block-scoped), all three arrow functions share the same \`i\` variable. When the callbacks run, the loop has finished and \`i\` is 3.

Fix: use \`let\`, which creates a new binding per iteration:
\`\`\`js
for (let i = 0; i < 3; i++) {
  setTimeout(() => console.log(i), 0);
}
// logs: 0, 1, 2
\`\`\``,
};

export const CONTENT_CASES: ContentCase[] = [
  // ── GOOD CONTENT ────────────────────────────────────────────────────────────

  {
    id: 'content-good-001',
    label: 'well-grounded segment — should pass T1',
    segmentText: `A closure is a function that keeps a reference to the variables in its outer scope, even after the outer function has finished running. This is possible because JavaScript functions retain a reference to the scope in which they were created — their "lexical environment."

When you call \`makeGreeter('Alice')\`, the returned function closes over the \`name\` variable. The outer function's stack frame is gone, but \`name\` lives on because the inner function holds a reference to it.

\`\`\`js
function makeGreeter(name) {
  return function() {
    return 'Hello, ' + name;
  };
}
const greetAlice = makeGreeter('Alice');
greetAlice(); // 'Hello, Alice'
\`\`\`

This pattern is the foundation of data privacy in JavaScript: variables declared in the outer function are inaccessible from outside, reachable only through the returned function.`,
    sourceChunks: [CLOSURE_CHUNK],
    checkpointPrompt: 'In your own words, explain what happens to the `name` variable after `makeGreeter` returns.',
    referenceAnswer: 'The name variable is retained in memory because the returned inner function closes over it. Even though makeGreeter has returned, name is not garbage-collected — the inner function holds a reference to the lexical scope where name was defined.',
    rubric: [
      { id: 'r1', criterion: 'Explains that name persists because the inner function holds a reference', weight: 0.6 },
      { id: 'r2', criterion: 'Notes that makeGreeter has returned but name survives', weight: 0.4 },
    ],
    expectedT1Verdict: 'pass',
    isBadContent: false,
    isGoodContent: true,
  },

  {
    id: 'content-good-002',
    label: 'loop bug explanation — well-grounded, should pass T1',
    segmentText: `The classic closure-in-loop bug catches many developers by surprise. When you use \`var\` in a for loop and create functions inside the loop body, all those functions share a single \`i\` variable — because \`var\` is function-scoped, not block-scoped.

\`\`\`js
for (var i = 0; i < 3; i++) {
  setTimeout(() => console.log(i), 0);
}
// Logs: 3, 3, 3
\`\`\`

By the time the callbacks fire, the loop has run to completion and \`i\` equals 3. The fix is straightforward: replace \`var\` with \`let\`. Because \`let\` is block-scoped, each iteration gets its own separate \`i\` binding.`,
    sourceChunks: [LOOP_CHUNK],
    checkpointPrompt: 'Why does the loop log "3" three times, and how do you fix it?',
    referenceAnswer: 'var is function-scoped so all callbacks share the same i variable. By the time they run, i is 3. Fix: replace var with let to get a new i binding per iteration.',
    rubric: [
      { id: 'r1', criterion: 'Identifies var scoping as the root cause', weight: 0.5 },
      { id: 'r2', criterion: 'Explains that all callbacks see the final value of i', weight: 0.3 },
      { id: 'r3', criterion: 'Proposes let as the fix', weight: 0.2 },
    ],
    expectedT1Verdict: 'pass',
    isBadContent: false,
    isGoodContent: true,
  },

  // ── BAD CONTENT ─────────────────────────────────────────────────────────────

  {
    id: 'content-bad-001',
    label: 'unsupported claim — closures copy values (not references)',
    segmentText: `A closure captures a copy of the variables in its outer scope at the moment the inner function is created. This means the inner function has a snapshot of the values, not a live reference to them.

\`\`\`js
function makeCounter() {
  let count = 0;
  return {
    increment: () => ++count,
    getValue: () => count,
  };
}
\`\`\`

When the returned methods run, they use the snapshot copy of \`count\` that was captured at creation time.`,
    sourceChunks: [CLOSURE_CHUNK],
    checkpointPrompt: 'Does a closure capture a copy of variables or a live reference?',
    referenceAnswer: 'A closure captures a live reference to the variable, not a copy. Changes to the variable are reflected when the inner function reads it.',
    rubric: [
      { id: 'r1', criterion: 'States it is a live reference, not a copy', weight: 1.0 },
    ],
    expectedT1Verdict: 'fail',
    isBadContent: true,
    isGoodContent: false,
  },

  {
    id: 'content-bad-002',
    label: 'wrong reference answer — claims var is block-scoped',
    segmentText: `The loop bug occurs because JavaScript's \`var\` is block-scoped, so each iteration of the loop has its own \`i\` variable. However, the asynchronous nature of \`setTimeout\` means the callbacks run after the loop completes, reading the final value of \`i\`.`,
    sourceChunks: [LOOP_CHUNK],
    checkpointPrompt: 'What is the scoping behavior of var in a for loop?',
    referenceAnswer: 'var is block-scoped in for loops, so each iteration creates its own i binding.',
    rubric: [
      { id: 'r1', criterion: 'Correctly identifies var scoping (block vs function)', weight: 1.0 },
    ],
    expectedT1Verdict: 'fail',
    isBadContent: true,
    isGoodContent: false,
  },
];

export const CONTENT_VERIFY_THRESHOLDS = {
  /** Max fraction of bad-content cases that T1 passes (false-pass rate). */
  maxFalsePassRate: 0.15,
  /** Max fraction of good-content cases that T1 fails (false-fail rate). */
  maxFalseFailRate: 0.15,
};