Curio/tests/golden/grading-cases.ts

/**
 * Frozen golden eval cases for response grading.
 *
 * ADD cases here when a new failure mode is found in prod.
 * NEVER edit existing cases (they are the historical baseline).
 * Append new cases only.
 *
 * isCorrect=true  → learner is right; grading as 'misconception' is a false-fail
 * isCorrect=false → learner is wrong; grading as 'mastered' is a false-pass
 *
 * Key invariant (CLAUDE.md §6): false-fail rate must stay ≤ FALSE_FAIL_THRESHOLD.
 */

export interface GoldenCase {
  id: string;
  /** Short human label for reports. */
  label: string;
  checkpointPrompt: string;
  referenceAnswer: string;
  rubric: Array<{ id: string; criterion: string; weight: number }>;
  misconceptions: Array<{ tag: string; signature: string }>;
  learnerResponse: string;
  expectedVerdict: 'mastered' | 'partial' | 'misconception' | 'uncertain';
  /** If true, grading as 'misconception' counts as a false-fail. */
  isCorrect: boolean;
}

export const CASES: GoldenCase[] = [
  // ── Closure definition ──────────────────────────────────────────────────────

  {
    id: 'closure-def-001',
    label: 'closure definition — full correct answer',
    checkpointPrompt:
      'In your own words, explain what a closure is and why it matters in JavaScript.',
    referenceAnswer:
      'A closure is a function that retains access to its outer (lexical) scope even after the outer function has finished executing. This matters because it lets inner functions use variables from their enclosing scope, enabling patterns like data privacy and factory functions.',
    rubric: [
      { id: 'r1', criterion: 'Mentions that the function retains access to its outer/enclosing scope', weight: 0.5 },
      { id: 'r2', criterion: 'Notes that this works even after the outer function has returned/finished', weight: 0.3 },
      { id: 'r3', criterion: 'Gives at least one reason why this matters (data privacy, factory, etc.)', weight: 0.2 },
    ],
    misconceptions: [
      { tag: 'closure-is-object', signature: 'Learner describes closure as an object or class instance' },
      { tag: 'closure-requires-return', signature: 'Learner thinks a closure only exists when the inner function is returned' },
    ],
    learnerResponse:
      'A closure is when a function keeps access to variables from its outer function even after that outer function has already returned. It is useful because you can hide data inside a function and only expose what you want.',
    expectedVerdict: 'mastered',
    isCorrect: true,
  },

  {
    id: 'closure-def-002',
    label: 'closure definition — correct but terse',
    checkpointPrompt:
      'In your own words, explain what a closure is and why it matters in JavaScript.',
    referenceAnswer:
      'A closure is a function that retains access to its outer (lexical) scope even after the outer function has finished executing.',
    rubric: [
      { id: 'r1', criterion: 'Mentions outer/enclosing scope retention', weight: 0.6 },
      { id: 'r2', criterion: 'Acknowledges the function survives beyond the outer scope', weight: 0.4 },
    ],
    misconceptions: [
      { tag: 'closure-is-object', signature: 'Learner describes closure as an object' },
    ],
    learnerResponse:
      'A closure captures variables from its surrounding scope so the function can still use them later.',
    expectedVerdict: 'partial',
    isCorrect: true,
  },

  {
    id: 'closure-def-003',
    label: 'closure definition — "inner function" only, incomplete',
    checkpointPrompt:
      'In your own words, explain what a closure is and why it matters in JavaScript.',
    referenceAnswer:
      'A closure is a function that retains access to its outer (lexical) scope even after the outer function has finished executing.',
    rubric: [
      { id: 'r1', criterion: 'Mentions outer/enclosing scope retention', weight: 0.6 },
      { id: 'r2', criterion: 'Notes persistence beyond outer function lifetime', weight: 0.4 },
    ],
    misconceptions: [
      { tag: 'closure-is-object', signature: 'Learner describes closure as an object' },
    ],
    learnerResponse:
      'A closure is just a function defined inside another function.',
    expectedVerdict: 'partial',
    isCorrect: true,
  },

  {
    id: 'closure-def-004',
    label: 'closure definition — object misconception',
    checkpointPrompt:
      'In your own words, explain what a closure is and why it matters in JavaScript.',
    referenceAnswer:
      'A closure is a function that retains access to its outer (lexical) scope even after the outer function has finished executing.',
    rubric: [
      { id: 'r1', criterion: 'Mentions outer/enclosing scope retention', weight: 0.6 },
      { id: 'r2', criterion: 'Notes persistence beyond outer function lifetime', weight: 0.4 },
    ],
    misconceptions: [
      { tag: 'closure-is-object', signature: 'Learner describes closure as an object or instance' },
    ],
    learnerResponse:
      'A closure is like a JavaScript object that bundles together a function and its state. It is similar to a class instance.',
    expectedVerdict: 'misconception',
    isCorrect: false,
  },

  // ── Loop variable bug ────────────────────────────────────────────────────────

  {
    id: 'loop-bug-001',
    label: 'loop bug — correct let/scope explanation',
    checkpointPrompt:
      'The following loop logs "3" three times instead of 0, 1, 2. Explain why, and fix it.\n\n```js\nfor (var i = 0; i < 3; i++) {\n  setTimeout(() => console.log(i), 0);\n}\n```',
    referenceAnswer:
      'var is function-scoped, so all three callbacks share the same i variable. By the time the callbacks run, i is 3. Fix: replace var with let, which creates a new binding per iteration.',
    rubric: [
      { id: 'r1', criterion: 'Identifies that var is function/global-scoped (not block-scoped)', weight: 0.4 },
      { id: 'r2', criterion: 'Explains that all callbacks share the same i and see its final value', weight: 0.4 },
      { id: 'r3', criterion: 'Proposes using let (or IIFE) as the fix', weight: 0.2 },
    ],
    misconceptions: [
      { tag: 'async-timing', signature: 'Learner attributes the bug to setTimeout running after the loop because it is asynchronous/slow, not to scoping' },
    ],
    learnerResponse:
      'Because var is function-scoped, all three arrow functions close over the same i variable. By the time setTimeout fires, the loop has finished and i equals 3. Switching to let creates a fresh binding for each iteration, so each callback captures its own copy.',
    expectedVerdict: 'mastered',
    isCorrect: true,
  },

  {
    id: 'loop-bug-002',
    label: 'loop bug — correct but omits fix',
    checkpointPrompt:
      'The following loop logs "3" three times instead of 0, 1, 2. Explain why, and fix it.\n\n```js\nfor (var i = 0; i < 3; i++) {\n  setTimeout(() => console.log(i), 0);\n}\n```',
    referenceAnswer:
      'var is function-scoped, so all callbacks share the same i. By the time they run, i is 3. Fix with let.',
    rubric: [
      { id: 'r1', criterion: 'Identifies var scoping as the cause', weight: 0.5 },
      { id: 'r2', criterion: 'Notes all callbacks see the final value of i', weight: 0.3 },
      { id: 'r3', criterion: 'Mentions let or IIFE as fix', weight: 0.2 },
    ],
    misconceptions: [
      { tag: 'async-timing', signature: 'Blames asynchronous timing rather than scoping' },
    ],
    learnerResponse:
      'var does not have block scope, so all the callbacks reference the same i. Once the loop finishes i is 3, so they all log 3.',
    expectedVerdict: 'partial',
    isCorrect: true,
  },

  {
    id: 'loop-bug-003',
    label: 'loop bug — timing misconception',
    checkpointPrompt:
      'The following loop logs "3" three times instead of 0, 1, 2. Explain why, and fix it.\n\n```js\nfor (var i = 0; i < 3; i++) {\n  setTimeout(() => console.log(i), 0);\n}\n```',
    referenceAnswer:
      'var is function-scoped; all callbacks share the same i. Fix with let.',
    rubric: [
      { id: 'r1', criterion: 'Identifies var scoping as root cause', weight: 0.6 },
      { id: 'r2', criterion: 'Notes final value of i is captured', weight: 0.4 },
    ],
    misconceptions: [
      { tag: 'async-timing', signature: 'Attributes the bug to setTimeout being asynchronous or slow rather than to var scoping' },
    ],
    learnerResponse:
      'setTimeout is asynchronous, so by the time the callbacks run the loop has already finished. That is why they all log 3. You can fix it by passing i as an argument to setTimeout.',
    expectedVerdict: 'misconception',
    isCorrect: false,
  },

  // ── Data privacy via closures ────────────────────────────────────────────────

  {
    id: 'data-privacy-001',
    label: 'data privacy — correct encapsulation explanation',
    checkpointPrompt:
      'Using closures, write a counter that exposes increment and getValue but keeps the count variable private. Explain how the closure enforces privacy.',
    referenceAnswer:
      'The count variable lives in the outer function\'s scope. The returned object\'s methods (increment, getValue) close over count and are the only way to access it. No external code can reach count directly because it is not in global scope.',
    rubric: [
      { id: 'r1', criterion: 'Correctly implements or describes an outer function holding count', weight: 0.3 },
      { id: 'r2', criterion: 'Explains that the inner functions close over count', weight: 0.4 },
      { id: 'r3', criterion: 'Notes that outside code cannot access count directly', weight: 0.3 },
    ],
    misconceptions: [
      { tag: 'privacy-by-naming', signature: 'Learner thinks naming a variable with underscore or using let makes it private without closures' },
    ],
    learnerResponse:
      'You define count inside a function and return an object with methods that reference count. Because count is local to the outer function, nothing outside can touch it directly — only the returned methods can read or change it through the closure.',
    expectedVerdict: 'mastered',
    isCorrect: true,
  },

  {
    id: 'data-privacy-002',
    label: 'data privacy — underscore-naming misconception',
    checkpointPrompt:
      'Using closures, write a counter that exposes increment and getValue but keeps the count variable private. Explain how the closure enforces privacy.',
    referenceAnswer:
      'count lives in the outer function scope; closures over it are the only access path.',
    rubric: [
      { id: 'r1', criterion: 'Identifies that count is in outer function scope', weight: 0.5 },
      { id: 'r2', criterion: 'States that only the inner methods can access count', weight: 0.5 },
    ],
    misconceptions: [
      { tag: 'privacy-by-naming', signature: 'Suggests naming convention (e.g. _count) alone enforces privacy without closure scope' },
    ],
    learnerResponse:
      'You make the variable private by naming it _count with an underscore. This signals to other developers that they should not access it directly.',
    expectedVerdict: 'misconception',
    isCorrect: false,
  },

  // ── Factory functions ────────────────────────────────────────────────────────

  {
    id: 'factory-001',
    label: 'factory — correct independent state',
    checkpointPrompt:
      'Explain why two calls to makeCounter() produce independent counters that do not share state.',
    referenceAnswer:
      'Each call to makeCounter() creates a new execution context with its own count variable. The returned functions close over their own count, so incrementing one counter does not affect the other.',
    rubric: [
      { id: 'r1', criterion: 'Identifies that each call creates a new scope/execution context', weight: 0.5 },
      { id: 'r2', criterion: 'Notes each returned function closes over its own separate count', weight: 0.5 },
    ],
    misconceptions: [
      { tag: 'shared-global', signature: 'Learner believes both counters share a single count variable because they come from the same function definition' },
    ],
    learnerResponse:
      'Every time makeCounter is called, JavaScript creates a new function scope with its own count. The functions returned by each call close over their respective count, so they are completely independent.',
    expectedVerdict: 'mastered',
    isCorrect: true,
  },

  {
    id: 'factory-002',
    label: 'factory — shared-global misconception',
    checkpointPrompt:
      'Explain why two calls to makeCounter() produce independent counters that do not share state.',
    referenceAnswer:
      'Each call creates a new scope with its own count. Returned functions close over their own count.',
    rubric: [
      { id: 'r1', criterion: 'Each call creates a new scope', weight: 0.5 },
      { id: 'r2', criterion: 'Each returned function closes over its own count', weight: 0.5 },
    ],
    misconceptions: [
      { tag: 'shared-global', signature: 'Learner thinks both counters share a count because they came from the same function' },
    ],
    learnerResponse:
      'They actually do share state if you use the same makeCounter function, because count is defined in the same place in the code.',
    expectedVerdict: 'misconception',
    isCorrect: false,
  },
];

/** Threshold: false-fail rate above this value fails the eval. */
export const FALSE_FAIL_THRESHOLD = 0.10; // 10%