feat: initial commit — Curio mastery tutor
Full Next.js App Router application with: - AI-graded lesson checkpoints (BKT/Elo mastery tracking) - Auth.js v5: credentials, Google, GitHub, generic OIDC - Anonymous-session-first with migrate-on-signin - Admin panel: users, blueprints, reports, site settings - Password reset + email verification (nodemailer/SMTP) - Site config: require_auth + signups_enabled flags - server-only guards on all DB/generation/verification modules - PostgreSQL 16 + pgvector, Redis cache, Drizzle ORM - 271 unit tests (Vitest), golden-eval harness, Playwright e2e stubs Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Frozen golden eval cases for content verification (T1 + T2 cascade).
|
||||
*
|
||||
* Each case provides a synthetic segment + checkpoint + source chunks.
|
||||
* The verifier should catch bad content (unsupported claims, wrong reference answers)
|
||||
* and pass good content (well-grounded, correct reference answer).
|
||||
*
|
||||
* expectedT1Verdict: what T1 (cheap model) should return.
|
||||
* isBadContent: true → verifier MUST return fail (false-pass = catastrophic).
|
||||
* isGoodContent: true → verifier should return pass or uncertain (false-fail = trust erosion).
|
||||
*/
|
||||
|
||||
export interface ContentCase {
|
||||
id: string;
|
||||
label: string;
|
||||
segmentText: string;
|
||||
sourceChunks: Array<{ docRef: string; text: string }>;
|
||||
checkpointPrompt: string;
|
||||
/** Only provided to T2, not T1. */
|
||||
referenceAnswer: string;
|
||||
rubric: Array<{ id: string; criterion: string; weight: number }>;
|
||||
expectedT1Verdict: 'pass' | 'fail' | 'uncertain';
|
||||
isBadContent: boolean;
|
||||
isGoodContent: boolean;
|
||||
}
|
||||
|
||||
const CLOSURE_CHUNK = {
|
||||
docRef: 'js-closures/definition',
|
||||
text: `A closure is a function that retains access to its lexical scope even after the outer function has returned. When a function is created, it captures a reference to its surrounding scope. This means variables defined in the outer function remain accessible to the inner function — they are not garbage collected as long as the inner function exists.
|
||||
|
||||
Example:
|
||||
\`\`\`js
|
||||
function makeGreeter(name) {
|
||||
return function() {
|
||||
return 'Hello, ' + name;
|
||||
};
|
||||
}
|
||||
const greetAlice = makeGreeter('Alice');
|
||||
greetAlice(); // 'Hello, Alice'
|
||||
\`\`\`
|
||||
|
||||
In this example, the inner function closes over the \`name\` variable. Even after \`makeGreeter\` returns, the inner function can still access \`name\`.`,
|
||||
};
|
||||
|
||||
const LOOP_CHUNK = {
|
||||
docRef: 'js-closures/loop-bug',
|
||||
text: `A common pitfall: closures in loops with \`var\`.
|
||||
|
||||
\`\`\`js
|
||||
for (var i = 0; i < 3; i++) {
|
||||
setTimeout(() => console.log(i), 0);
|
||||
}
|
||||
// logs: 3, 3, 3
|
||||
\`\`\`
|
||||
|
||||
Because \`var\` is function-scoped (not block-scoped), all three arrow functions share the same \`i\` variable. When the callbacks run, the loop has finished and \`i\` is 3.
|
||||
|
||||
Fix: use \`let\`, which creates a new binding per iteration:
|
||||
\`\`\`js
|
||||
for (let i = 0; i < 3; i++) {
|
||||
setTimeout(() => console.log(i), 0);
|
||||
}
|
||||
// logs: 0, 1, 2
|
||||
\`\`\``,
|
||||
};
|
||||
|
||||
export const CONTENT_CASES: ContentCase[] = [
|
||||
// ── GOOD CONTENT ────────────────────────────────────────────────────────────
|
||||
|
||||
{
|
||||
id: 'content-good-001',
|
||||
label: 'well-grounded segment — should pass T1',
|
||||
segmentText: `A closure is a function that keeps a reference to the variables in its outer scope, even after the outer function has finished running. This is possible because JavaScript functions retain a reference to the scope in which they were created — their "lexical environment."
|
||||
|
||||
When you call \`makeGreeter('Alice')\`, the returned function closes over the \`name\` variable. The outer function's stack frame is gone, but \`name\` lives on because the inner function holds a reference to it.
|
||||
|
||||
\`\`\`js
|
||||
function makeGreeter(name) {
|
||||
return function() {
|
||||
return 'Hello, ' + name;
|
||||
};
|
||||
}
|
||||
const greetAlice = makeGreeter('Alice');
|
||||
greetAlice(); // 'Hello, Alice'
|
||||
\`\`\`
|
||||
|
||||
This pattern is the foundation of data privacy in JavaScript: variables declared in the outer function are inaccessible from outside, reachable only through the returned function.`,
|
||||
sourceChunks: [CLOSURE_CHUNK],
|
||||
checkpointPrompt: 'In your own words, explain what happens to the `name` variable after `makeGreeter` returns.',
|
||||
referenceAnswer: 'The name variable is retained in memory because the returned inner function closes over it. Even though makeGreeter has returned, name is not garbage-collected — the inner function holds a reference to the lexical scope where name was defined.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Explains that name persists because the inner function holds a reference', weight: 0.6 },
|
||||
{ id: 'r2', criterion: 'Notes that makeGreeter has returned but name survives', weight: 0.4 },
|
||||
],
|
||||
expectedT1Verdict: 'pass',
|
||||
isBadContent: false,
|
||||
isGoodContent: true,
|
||||
},
|
||||
|
||||
{
|
||||
id: 'content-good-002',
|
||||
label: 'loop bug explanation — well-grounded, should pass T1',
|
||||
segmentText: `The classic closure-in-loop bug catches many developers by surprise. When you use \`var\` in a for loop and create functions inside the loop body, all those functions share a single \`i\` variable — because \`var\` is function-scoped, not block-scoped.
|
||||
|
||||
\`\`\`js
|
||||
for (var i = 0; i < 3; i++) {
|
||||
setTimeout(() => console.log(i), 0);
|
||||
}
|
||||
// Logs: 3, 3, 3
|
||||
\`\`\`
|
||||
|
||||
By the time the callbacks fire, the loop has run to completion and \`i\` equals 3. The fix is straightforward: replace \`var\` with \`let\`. Because \`let\` is block-scoped, each iteration gets its own separate \`i\` binding.`,
|
||||
sourceChunks: [LOOP_CHUNK],
|
||||
checkpointPrompt: 'Why does the loop log "3" three times, and how do you fix it?',
|
||||
referenceAnswer: 'var is function-scoped so all callbacks share the same i variable. By the time they run, i is 3. Fix: replace var with let to get a new i binding per iteration.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Identifies var scoping as the root cause', weight: 0.5 },
|
||||
{ id: 'r2', criterion: 'Explains that all callbacks see the final value of i', weight: 0.3 },
|
||||
{ id: 'r3', criterion: 'Proposes let as the fix', weight: 0.2 },
|
||||
],
|
||||
expectedT1Verdict: 'pass',
|
||||
isBadContent: false,
|
||||
isGoodContent: true,
|
||||
},
|
||||
|
||||
// ── BAD CONTENT ─────────────────────────────────────────────────────────────
|
||||
|
||||
{
|
||||
id: 'content-bad-001',
|
||||
label: 'unsupported claim — closures copy values (not references)',
|
||||
segmentText: `A closure captures a copy of the variables in its outer scope at the moment the inner function is created. This means the inner function has a snapshot of the values, not a live reference to them.
|
||||
|
||||
\`\`\`js
|
||||
function makeCounter() {
|
||||
let count = 0;
|
||||
return {
|
||||
increment: () => ++count,
|
||||
getValue: () => count,
|
||||
};
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
When the returned methods run, they use the snapshot copy of \`count\` that was captured at creation time.`,
|
||||
sourceChunks: [CLOSURE_CHUNK],
|
||||
checkpointPrompt: 'Does a closure capture a copy of variables or a live reference?',
|
||||
referenceAnswer: 'A closure captures a live reference to the variable, not a copy. Changes to the variable are reflected when the inner function reads it.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'States it is a live reference, not a copy', weight: 1.0 },
|
||||
],
|
||||
expectedT1Verdict: 'fail',
|
||||
isBadContent: true,
|
||||
isGoodContent: false,
|
||||
},
|
||||
|
||||
{
|
||||
id: 'content-bad-002',
|
||||
label: 'wrong reference answer — claims var is block-scoped',
|
||||
segmentText: `The loop bug occurs because JavaScript's \`var\` is block-scoped, so each iteration of the loop has its own \`i\` variable. However, the asynchronous nature of \`setTimeout\` means the callbacks run after the loop completes, reading the final value of \`i\`.`,
|
||||
sourceChunks: [LOOP_CHUNK],
|
||||
checkpointPrompt: 'What is the scoping behavior of var in a for loop?',
|
||||
referenceAnswer: 'var is block-scoped in for loops, so each iteration creates its own i binding.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Correctly identifies var scoping (block vs function)', weight: 1.0 },
|
||||
],
|
||||
expectedT1Verdict: 'fail',
|
||||
isBadContent: true,
|
||||
isGoodContent: false,
|
||||
},
|
||||
];
|
||||
|
||||
export const CONTENT_VERIFY_THRESHOLDS = {
|
||||
/** Max fraction of bad-content cases that T1 passes (false-pass rate). */
|
||||
maxFalsePassRate: 0.15,
|
||||
/** Max fraction of good-content cases that T1 fails (false-fail rate). */
|
||||
maxFalseFailRate: 0.15,
|
||||
};
|
||||
@@ -0,0 +1,264 @@
|
||||
/**
|
||||
* Frozen golden eval cases for response grading.
|
||||
*
|
||||
* ADD cases here when a new failure mode is found in prod.
|
||||
* NEVER edit existing cases (they are the historical baseline).
|
||||
* Append new cases only.
|
||||
*
|
||||
* isCorrect=true → learner is right; grading as 'misconception' is a false-fail
|
||||
* isCorrect=false → learner is wrong; grading as 'mastered' is a false-pass
|
||||
*
|
||||
* Key invariant (CLAUDE.md §6): false-fail rate must stay ≤ FALSE_FAIL_THRESHOLD.
|
||||
*/
|
||||
|
||||
export interface GoldenCase {
|
||||
id: string;
|
||||
/** Short human label for reports. */
|
||||
label: string;
|
||||
checkpointPrompt: string;
|
||||
referenceAnswer: string;
|
||||
rubric: Array<{ id: string; criterion: string; weight: number }>;
|
||||
misconceptions: Array<{ tag: string; signature: string }>;
|
||||
learnerResponse: string;
|
||||
expectedVerdict: 'mastered' | 'partial' | 'misconception' | 'uncertain';
|
||||
/** If true, grading as 'misconception' counts as a false-fail. */
|
||||
isCorrect: boolean;
|
||||
}
|
||||
|
||||
export const CASES: GoldenCase[] = [
|
||||
// ── Closure definition ──────────────────────────────────────────────────────
|
||||
|
||||
{
|
||||
id: 'closure-def-001',
|
||||
label: 'closure definition — full correct answer',
|
||||
checkpointPrompt:
|
||||
'In your own words, explain what a closure is and why it matters in JavaScript.',
|
||||
referenceAnswer:
|
||||
'A closure is a function that retains access to its outer (lexical) scope even after the outer function has finished executing. This matters because it lets inner functions use variables from their enclosing scope, enabling patterns like data privacy and factory functions.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Mentions that the function retains access to its outer/enclosing scope', weight: 0.5 },
|
||||
{ id: 'r2', criterion: 'Notes that this works even after the outer function has returned/finished', weight: 0.3 },
|
||||
{ id: 'r3', criterion: 'Gives at least one reason why this matters (data privacy, factory, etc.)', weight: 0.2 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'closure-is-object', signature: 'Learner describes closure as an object or class instance' },
|
||||
{ tag: 'closure-requires-return', signature: 'Learner thinks a closure only exists when the inner function is returned' },
|
||||
],
|
||||
learnerResponse:
|
||||
'A closure is when a function keeps access to variables from its outer function even after that outer function has already returned. It is useful because you can hide data inside a function and only expose what you want.',
|
||||
expectedVerdict: 'mastered',
|
||||
isCorrect: true,
|
||||
},
|
||||
|
||||
{
|
||||
id: 'closure-def-002',
|
||||
label: 'closure definition — correct but terse',
|
||||
checkpointPrompt:
|
||||
'In your own words, explain what a closure is and why it matters in JavaScript.',
|
||||
referenceAnswer:
|
||||
'A closure is a function that retains access to its outer (lexical) scope even after the outer function has finished executing.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Mentions outer/enclosing scope retention', weight: 0.6 },
|
||||
{ id: 'r2', criterion: 'Acknowledges the function survives beyond the outer scope', weight: 0.4 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'closure-is-object', signature: 'Learner describes closure as an object' },
|
||||
],
|
||||
learnerResponse:
|
||||
'A closure captures variables from its surrounding scope so the function can still use them later.',
|
||||
expectedVerdict: 'partial',
|
||||
isCorrect: true,
|
||||
},
|
||||
|
||||
{
|
||||
id: 'closure-def-003',
|
||||
label: 'closure definition — "inner function" only, incomplete',
|
||||
checkpointPrompt:
|
||||
'In your own words, explain what a closure is and why it matters in JavaScript.',
|
||||
referenceAnswer:
|
||||
'A closure is a function that retains access to its outer (lexical) scope even after the outer function has finished executing.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Mentions outer/enclosing scope retention', weight: 0.6 },
|
||||
{ id: 'r2', criterion: 'Notes persistence beyond outer function lifetime', weight: 0.4 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'closure-is-object', signature: 'Learner describes closure as an object' },
|
||||
],
|
||||
learnerResponse:
|
||||
'A closure is just a function defined inside another function.',
|
||||
expectedVerdict: 'partial',
|
||||
isCorrect: true,
|
||||
},
|
||||
|
||||
{
|
||||
id: 'closure-def-004',
|
||||
label: 'closure definition — object misconception',
|
||||
checkpointPrompt:
|
||||
'In your own words, explain what a closure is and why it matters in JavaScript.',
|
||||
referenceAnswer:
|
||||
'A closure is a function that retains access to its outer (lexical) scope even after the outer function has finished executing.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Mentions outer/enclosing scope retention', weight: 0.6 },
|
||||
{ id: 'r2', criterion: 'Notes persistence beyond outer function lifetime', weight: 0.4 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'closure-is-object', signature: 'Learner describes closure as an object or instance' },
|
||||
],
|
||||
learnerResponse:
|
||||
'A closure is like a JavaScript object that bundles together a function and its state. It is similar to a class instance.',
|
||||
expectedVerdict: 'misconception',
|
||||
isCorrect: false,
|
||||
},
|
||||
|
||||
// ── Loop variable bug ────────────────────────────────────────────────────────
|
||||
|
||||
{
|
||||
id: 'loop-bug-001',
|
||||
label: 'loop bug — correct let/scope explanation',
|
||||
checkpointPrompt:
|
||||
'The following loop logs "3" three times instead of 0, 1, 2. Explain why, and fix it.\n\n```js\nfor (var i = 0; i < 3; i++) {\n setTimeout(() => console.log(i), 0);\n}\n```',
|
||||
referenceAnswer:
|
||||
'var is function-scoped, so all three callbacks share the same i variable. By the time the callbacks run, i is 3. Fix: replace var with let, which creates a new binding per iteration.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Identifies that var is function/global-scoped (not block-scoped)', weight: 0.4 },
|
||||
{ id: 'r2', criterion: 'Explains that all callbacks share the same i and see its final value', weight: 0.4 },
|
||||
{ id: 'r3', criterion: 'Proposes using let (or IIFE) as the fix', weight: 0.2 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'async-timing', signature: 'Learner attributes the bug to setTimeout running after the loop because it is asynchronous/slow, not to scoping' },
|
||||
],
|
||||
learnerResponse:
|
||||
'Because var is function-scoped, all three arrow functions close over the same i variable. By the time setTimeout fires, the loop has finished and i equals 3. Switching to let creates a fresh binding for each iteration, so each callback captures its own copy.',
|
||||
expectedVerdict: 'mastered',
|
||||
isCorrect: true,
|
||||
},
|
||||
|
||||
{
|
||||
id: 'loop-bug-002',
|
||||
label: 'loop bug — correct but omits fix',
|
||||
checkpointPrompt:
|
||||
'The following loop logs "3" three times instead of 0, 1, 2. Explain why, and fix it.\n\n```js\nfor (var i = 0; i < 3; i++) {\n setTimeout(() => console.log(i), 0);\n}\n```',
|
||||
referenceAnswer:
|
||||
'var is function-scoped, so all callbacks share the same i. By the time they run, i is 3. Fix with let.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Identifies var scoping as the cause', weight: 0.5 },
|
||||
{ id: 'r2', criterion: 'Notes all callbacks see the final value of i', weight: 0.3 },
|
||||
{ id: 'r3', criterion: 'Mentions let or IIFE as fix', weight: 0.2 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'async-timing', signature: 'Blames asynchronous timing rather than scoping' },
|
||||
],
|
||||
learnerResponse:
|
||||
'var does not have block scope, so all the callbacks reference the same i. Once the loop finishes i is 3, so they all log 3.',
|
||||
expectedVerdict: 'partial',
|
||||
isCorrect: true,
|
||||
},
|
||||
|
||||
{
|
||||
id: 'loop-bug-003',
|
||||
label: 'loop bug — timing misconception',
|
||||
checkpointPrompt:
|
||||
'The following loop logs "3" three times instead of 0, 1, 2. Explain why, and fix it.\n\n```js\nfor (var i = 0; i < 3; i++) {\n setTimeout(() => console.log(i), 0);\n}\n```',
|
||||
referenceAnswer:
|
||||
'var is function-scoped; all callbacks share the same i. Fix with let.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Identifies var scoping as root cause', weight: 0.6 },
|
||||
{ id: 'r2', criterion: 'Notes final value of i is captured', weight: 0.4 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'async-timing', signature: 'Attributes the bug to setTimeout being asynchronous or slow rather than to var scoping' },
|
||||
],
|
||||
learnerResponse:
|
||||
'setTimeout is asynchronous, so by the time the callbacks run the loop has already finished. That is why they all log 3. You can fix it by passing i as an argument to setTimeout.',
|
||||
expectedVerdict: 'misconception',
|
||||
isCorrect: false,
|
||||
},
|
||||
|
||||
// ── Data privacy via closures ────────────────────────────────────────────────
|
||||
|
||||
{
|
||||
id: 'data-privacy-001',
|
||||
label: 'data privacy — correct encapsulation explanation',
|
||||
checkpointPrompt:
|
||||
'Using closures, write a counter that exposes increment and getValue but keeps the count variable private. Explain how the closure enforces privacy.',
|
||||
referenceAnswer:
|
||||
'The count variable lives in the outer function\'s scope. The returned object\'s methods (increment, getValue) close over count and are the only way to access it. No external code can reach count directly because it is not in global scope.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Correctly implements or describes an outer function holding count', weight: 0.3 },
|
||||
{ id: 'r2', criterion: 'Explains that the inner functions close over count', weight: 0.4 },
|
||||
{ id: 'r3', criterion: 'Notes that outside code cannot access count directly', weight: 0.3 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'privacy-by-naming', signature: 'Learner thinks naming a variable with underscore or using let makes it private without closures' },
|
||||
],
|
||||
learnerResponse:
|
||||
'You define count inside a function and return an object with methods that reference count. Because count is local to the outer function, nothing outside can touch it directly — only the returned methods can read or change it through the closure.',
|
||||
expectedVerdict: 'mastered',
|
||||
isCorrect: true,
|
||||
},
|
||||
|
||||
{
|
||||
id: 'data-privacy-002',
|
||||
label: 'data privacy — underscore-naming misconception',
|
||||
checkpointPrompt:
|
||||
'Using closures, write a counter that exposes increment and getValue but keeps the count variable private. Explain how the closure enforces privacy.',
|
||||
referenceAnswer:
|
||||
'count lives in the outer function scope; closures over it are the only access path.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Identifies that count is in outer function scope', weight: 0.5 },
|
||||
{ id: 'r2', criterion: 'States that only the inner methods can access count', weight: 0.5 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'privacy-by-naming', signature: 'Suggests naming convention (e.g. _count) alone enforces privacy without closure scope' },
|
||||
],
|
||||
learnerResponse:
|
||||
'You make the variable private by naming it _count with an underscore. This signals to other developers that they should not access it directly.',
|
||||
expectedVerdict: 'misconception',
|
||||
isCorrect: false,
|
||||
},
|
||||
|
||||
// ── Factory functions ────────────────────────────────────────────────────────
|
||||
|
||||
{
|
||||
id: 'factory-001',
|
||||
label: 'factory — correct independent state',
|
||||
checkpointPrompt:
|
||||
'Explain why two calls to makeCounter() produce independent counters that do not share state.',
|
||||
referenceAnswer:
|
||||
'Each call to makeCounter() creates a new execution context with its own count variable. The returned functions close over their own count, so incrementing one counter does not affect the other.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Identifies that each call creates a new scope/execution context', weight: 0.5 },
|
||||
{ id: 'r2', criterion: 'Notes each returned function closes over its own separate count', weight: 0.5 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'shared-global', signature: 'Learner believes both counters share a single count variable because they come from the same function definition' },
|
||||
],
|
||||
learnerResponse:
|
||||
'Every time makeCounter is called, JavaScript creates a new function scope with its own count. The functions returned by each call close over their respective count, so they are completely independent.',
|
||||
expectedVerdict: 'mastered',
|
||||
isCorrect: true,
|
||||
},
|
||||
|
||||
{
|
||||
id: 'factory-002',
|
||||
label: 'factory — shared-global misconception',
|
||||
checkpointPrompt:
|
||||
'Explain why two calls to makeCounter() produce independent counters that do not share state.',
|
||||
referenceAnswer:
|
||||
'Each call creates a new scope with its own count. Returned functions close over their own count.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Each call creates a new scope', weight: 0.5 },
|
||||
{ id: 'r2', criterion: 'Each returned function closes over its own count', weight: 0.5 },
|
||||
],
|
||||
misconceptions: [
|
||||
{ tag: 'shared-global', signature: 'Learner thinks both counters share a count because they came from the same function' },
|
||||
],
|
||||
learnerResponse:
|
||||
'They actually do share state if you use the same makeCounter function, because count is defined in the same place in the code.',
|
||||
expectedVerdict: 'misconception',
|
||||
isCorrect: false,
|
||||
},
|
||||
];
|
||||
|
||||
/** Threshold: false-fail rate above this value fails the eval. */
|
||||
export const FALSE_FAIL_THRESHOLD = 0.10; // 10%
|
||||
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Frozen cases for misconception-hit-rate eval.
|
||||
*
|
||||
* Each case: learner response exhibits a specific misconception tag.
|
||||
* The grader should return that tag (hit) vs "novel"/"none" (miss).
|
||||
*
|
||||
* Baseline comparison: grader WITHOUT misconception library → measures
|
||||
* how many it gets as "novel". With library: should hit the known tag.
|
||||
*/
|
||||
|
||||
export interface MisconceptionCase {
|
||||
id: string;
|
||||
label: string;
|
||||
checkpointPrompt: string;
|
||||
referenceAnswer: string;
|
||||
rubric: Array<{ id: string; criterion: string; weight: number }>;
|
||||
/** The target misconception — what the grader SHOULD return as misconception_tag. */
|
||||
targetTag: string;
|
||||
/** The full library entry — used to run WITH-library condition. */
|
||||
library: Array<{ tag: string; signature: string }>;
|
||||
learnerResponse: string;
|
||||
}
|
||||
|
||||
export const MISCONCEPTION_CASES: MisconceptionCase[] = [
|
||||
{
|
||||
id: 'mc-hit-001',
|
||||
label: 'closure-is-object — should tag correctly with library',
|
||||
checkpointPrompt: 'In your own words, explain what a closure is.',
|
||||
referenceAnswer: 'A closure is a function that retains access to its lexical scope after the outer function returns.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Mentions scope retention', weight: 0.6 },
|
||||
{ id: 'r2', criterion: 'Notes outer function lifetime', weight: 0.4 },
|
||||
],
|
||||
targetTag: 'closure-is-object',
|
||||
library: [
|
||||
{
|
||||
tag: 'closure-is-object',
|
||||
signature: 'Learner describes a closure as an object, class instance, or data container rather than as a function with scope access',
|
||||
},
|
||||
{
|
||||
tag: 'closure-requires-return',
|
||||
signature: 'Learner states that a closure only exists or "activates" when the inner function is explicitly returned from the outer function',
|
||||
},
|
||||
],
|
||||
learnerResponse:
|
||||
'A closure is basically a JavaScript object that wraps a function together with some private state. It is similar to creating an instance of a class.',
|
||||
},
|
||||
|
||||
{
|
||||
id: 'mc-hit-002',
|
||||
label: 'async-timing — should tag correctly with library',
|
||||
checkpointPrompt: 'Why does this loop log "3" three times? for (var i=0; i<3; i++) setTimeout(()=>console.log(i), 0)',
|
||||
referenceAnswer: 'var is function-scoped; all callbacks share one i. By the time they fire, i is 3. Fix with let.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Identifies var scoping as cause', weight: 0.7 },
|
||||
{ id: 'r2', criterion: 'Notes final value of i', weight: 0.3 },
|
||||
],
|
||||
targetTag: 'async-timing',
|
||||
library: [
|
||||
{
|
||||
tag: 'async-timing',
|
||||
signature: 'Learner attributes the bug to setTimeout being asynchronous or running after the loop, without mentioning that var scoping is the actual cause',
|
||||
},
|
||||
{
|
||||
tag: 'var-block-scoped',
|
||||
signature: 'Learner claims var is block-scoped like let, or confuses var and let scoping behavior',
|
||||
},
|
||||
],
|
||||
learnerResponse:
|
||||
'Because setTimeout runs asynchronously, the callback does not execute until after the loop has already completed. By that time i is already 3, so all three logs show 3.',
|
||||
},
|
||||
|
||||
{
|
||||
id: 'mc-hit-003',
|
||||
label: 'privacy-by-naming — should tag correctly with library',
|
||||
checkpointPrompt: 'How do closures enforce data privacy? Give an example.',
|
||||
referenceAnswer: 'The private variable lives in the outer function scope; only the returned methods can access it through the closure. No external code can reach it.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Variable in outer scope, not accessible externally', weight: 0.6 },
|
||||
{ id: 'r2', criterion: 'Returned functions are the only access path', weight: 0.4 },
|
||||
],
|
||||
targetTag: 'privacy-by-naming',
|
||||
library: [
|
||||
{
|
||||
tag: 'privacy-by-naming',
|
||||
signature: 'Learner claims data privacy is achieved by naming convention (prefixing with underscore, using ALL_CAPS) rather than by scoping variables inside a closure',
|
||||
},
|
||||
{
|
||||
tag: 'closure-is-object',
|
||||
signature: 'Learner describes a closure as an object with private properties',
|
||||
},
|
||||
],
|
||||
learnerResponse:
|
||||
'You can achieve data privacy in JavaScript by prefixing your variables with an underscore, like _count. This is a convention that tells other developers the variable is private and should not be touched.',
|
||||
},
|
||||
|
||||
{
|
||||
id: 'mc-hit-004',
|
||||
label: 'shared-global — should tag correctly with library',
|
||||
checkpointPrompt: 'Why do two calls to makeCounter() produce independent counters?',
|
||||
referenceAnswer: 'Each call creates a new execution context with its own count. Returned functions close over their own separate count.',
|
||||
rubric: [
|
||||
{ id: 'r1', criterion: 'Each call creates new scope', weight: 0.5 },
|
||||
{ id: 'r2', criterion: 'Each closure has own count', weight: 0.5 },
|
||||
],
|
||||
targetTag: 'shared-global',
|
||||
library: [
|
||||
{
|
||||
tag: 'shared-global',
|
||||
signature: 'Learner believes that because both counters come from the same function definition, they share a single count variable in memory',
|
||||
},
|
||||
],
|
||||
learnerResponse:
|
||||
'Actually if you call makeCounter twice, both counters will share the same count variable because they both reference the same function definition. You would need to store count somewhere unique to avoid this.',
|
||||
},
|
||||
];
|
||||
|
||||
/** Fraction of cases where grader returns correct tag. Must beat baseline. */
|
||||
export const HIT_RATE_THRESHOLD = 0.65;
|
||||
|
||||
/** Baseline: expected hit rate without any misconception library (grader returns "novel"). */
|
||||
export const BASELINE_HIT_RATE = 0.0;
|
||||
@@ -0,0 +1,142 @@
|
||||
/**
|
||||
* Content verification eval — T1 cascade accuracy.
|
||||
*
|
||||
* Tests the T1 verifier against cases where content is known to be good or bad.
|
||||
* Measures false-pass rate (bad content slips through) and false-fail rate
|
||||
* (good content incorrectly rejected).
|
||||
*
|
||||
* CI gate: false-pass rate ≤ 15%, false-fail rate ≤ 15%.
|
||||
*/
|
||||
|
||||
import { generateObject } from 'ai';
|
||||
import { VerifyOutputSchema } from '../../src/schemas/verification';
|
||||
import type { VerifyOutput } from '../../src/schemas/verification';
|
||||
import { prompts } from '../../src/lib/llm/prompts';
|
||||
import { llmClient } from '../../src/lib/llm/client';
|
||||
import { CONTENT_CASES, CONTENT_VERIFY_THRESHOLDS } from './content-cases';
|
||||
import type { ContentCase } from './content-cases';
|
||||
|
||||
// ── Prompt builder (mirrors verify-content.ts T1 logic) ──────────────────────
|
||||
|
||||
function buildT1Prompt(c: ContentCase): string {
|
||||
const sourcesText = c.sourceChunks.map((s) => `[${s.docRef}]\n${s.text}`).join('\n\n---\n\n');
|
||||
|
||||
return `Source chunks:
|
||||
${sourcesText}
|
||||
|
||||
Segment text to verify:
|
||||
"""
|
||||
${c.segmentText}
|
||||
"""
|
||||
|
||||
Checkpoint prompt (blind self-solve — answer from sources only, NOT from segment):
|
||||
${c.checkpointPrompt}
|
||||
|
||||
Fact-check the segment and independently solve the checkpoint.`;
|
||||
}
|
||||
|
||||
// ── Single verification call ──────────────────────────────────────────────────
|
||||
|
||||
async function verify(
|
||||
c: ContentCase,
|
||||
): Promise<{ output: VerifyOutput | null; error?: string; latencyMs: number }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const { object } = await generateObject({
|
||||
model: llmClient.grader,
|
||||
schema: VerifyOutputSchema,
|
||||
system: prompts.VERIFY_CONTENT_T1.template,
|
||||
prompt: buildT1Prompt(c),
|
||||
});
|
||||
return { output: object, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return {
|
||||
output: null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
latencyMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Runner ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log(`\nCurio — content verification eval (${CONTENT_CASES.length} cases)\n`);
|
||||
console.log(` Verifier: ${process.env.LLM_GRADER_PROVIDER ?? 'openai'} / ${process.env.LLM_GRADER_MODEL ?? 'gpt-4o-mini'} (T1)`);
|
||||
console.log(
|
||||
` Thresholds: false-pass ≤ ${(CONTENT_VERIFY_THRESHOLDS.maxFalsePassRate * 100).toFixed(0)}%,` +
|
||||
` false-fail ≤ ${(CONTENT_VERIFY_THRESHOLDS.maxFalseFailRate * 100).toFixed(0)}%\n`,
|
||||
);
|
||||
|
||||
const results: Array<{
|
||||
case: ContentCase;
|
||||
output: VerifyOutput | null;
|
||||
error?: string;
|
||||
isFalsePass: boolean;
|
||||
isFalseFail: boolean;
|
||||
latencyMs: number;
|
||||
}> = [];
|
||||
|
||||
for (const c of CONTENT_CASES) {
|
||||
process.stdout.write(` [${c.id}] "${c.label}"…\r`);
|
||||
const { output, error, latencyMs } = await verify(c);
|
||||
|
||||
const verdict = output?.verdict ?? null;
|
||||
const isFalsePass = c.isBadContent && verdict === 'pass';
|
||||
const isFalseFail = c.isGoodContent && verdict === 'fail';
|
||||
|
||||
results.push({ case: c, output, error, isFalsePass, isFalseFail, latencyMs });
|
||||
|
||||
const flag = error ? ' ERR ' : isFalsePass ? '✗ FP ' : isFalseFail ? '✗ FF ' : '✓ ';
|
||||
const ms = `${latencyMs}ms`.padStart(6);
|
||||
console.log(
|
||||
` [${c.id}] ${flag} verdict=${verdict ?? 'null'} expected=${c.expectedT1Verdict} ${ms} "${c.label}"`,
|
||||
);
|
||||
|
||||
if (error) console.log(` Error: ${error}`);
|
||||
if (output && output.claims.some((cl) => cl.status !== 'supported')) {
|
||||
const unsupported = output.claims.filter((cl) => cl.status !== 'supported');
|
||||
console.log(` Unsupported claims: ${unsupported.map((cl) => `"${cl.text.slice(0, 60)}…"`).join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Metrics ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const ran = results.filter((r) => !r.error);
|
||||
const errors = results.filter((r) => r.error);
|
||||
const badCases = ran.filter((r) => r.case.isBadContent);
|
||||
const goodCases = ran.filter((r) => r.case.isGoodContent);
|
||||
const falsePasses = ran.filter((r) => r.isFalsePass);
|
||||
const falseFails = ran.filter((r) => r.isFalseFail);
|
||||
|
||||
const falsePassRate = badCases.length > 0 ? falsePasses.length / badCases.length : 0;
|
||||
const falseFailRate = goodCases.length > 0 ? falseFails.length / goodCases.length : 0;
|
||||
|
||||
console.log('\n── Results ─────────────────────────────────────────────────────────────');
|
||||
console.log(` Cases run: ${ran.length} / ${CONTENT_CASES.length} (${errors.length} errors)`);
|
||||
console.log(` False-pass: ${falsePasses.length}/${badCases.length} = ${(falsePassRate * 100).toFixed(1)}% (bad content passing — max ${(CONTENT_VERIFY_THRESHOLDS.maxFalsePassRate * 100).toFixed(0)}%)`);
|
||||
console.log(` False-fail: ${falseFails.length}/${goodCases.length} = ${(falseFailRate * 100).toFixed(1)}% (good content rejected — max ${(CONTENT_VERIFY_THRESHOLDS.maxFalseFailRate * 100).toFixed(0)}%)`);
|
||||
|
||||
let failed = false;
|
||||
|
||||
if (falsePassRate > CONTENT_VERIFY_THRESHOLDS.maxFalsePassRate) {
|
||||
console.log(`\n✗ FAIL: false-pass rate ${(falsePassRate * 100).toFixed(1)}% exceeds threshold`);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (falseFailRate > CONTENT_VERIFY_THRESHOLDS.maxFalseFailRate) {
|
||||
console.log(`\n✗ FAIL: false-fail rate ${(falseFailRate * 100).toFixed(1)}% exceeds threshold`);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (!failed) {
|
||||
console.log('\n✓ PASS: content verification quality within thresholds\n');
|
||||
}
|
||||
|
||||
process.exit(failed ? 1 : 0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Eval runner crashed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,196 @@
|
||||
/**
|
||||
* Golden eval harness — response grading accuracy.
|
||||
*
|
||||
* Runs real-model grading calls against the frozen labeled cases in grading-cases.ts.
|
||||
* Measures false-fail rate and exits non-zero if it exceeds FALSE_FAIL_THRESHOLD.
|
||||
*
|
||||
* Usage:
|
||||
* pnpm test:golden
|
||||
*
|
||||
* Required env: OPENAI_API_KEY (grader is openai/gpt-4o-mini by default)
|
||||
* Optional env: ANTHROPIC_API_KEY (only if LLM_GRADER_PROVIDER=anthropic)
|
||||
*
|
||||
* CI gate: exits 1 if false-fail rate > 10% OR accuracy < 70%.
|
||||
*/
|
||||
|
||||
import { generateObject } from 'ai';
|
||||
import { CASES, FALSE_FAIL_THRESHOLD } from './grading-cases';
|
||||
import type { GoldenCase } from './grading-cases';
|
||||
import { GradeOutputSchema } from '../../src/schemas/grading';
|
||||
import type { GradeOutput } from '../../src/schemas/grading';
|
||||
import { prompts } from '../../src/lib/llm/prompts';
|
||||
import { llmClient } from '../../src/lib/llm/client';
|
||||
|
||||
// ── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface CaseResult {
|
||||
case: GoldenCase;
|
||||
actual: GradeOutput | null;
|
||||
error?: string;
|
||||
/** verdict matched expected */
|
||||
verdictMatch: boolean;
|
||||
/** isCorrect case graded as misconception */
|
||||
isFalseFail: boolean;
|
||||
/** !isCorrect case graded as mastered */
|
||||
isFalsePass: boolean;
|
||||
latencyMs: number;
|
||||
}
|
||||
|
||||
// ── Prompt builder (mirrors grader.ts logic) ─────────────────────────────────
|
||||
|
||||
function buildPrompt(c: GoldenCase): string {
|
||||
const rubricLines = c.rubric
|
||||
.map((r) => ` - [${r.id}] ${r.criterion} (weight: ${r.weight.toFixed(2)})`)
|
||||
.join('\n');
|
||||
|
||||
const miscLines =
|
||||
c.misconceptions.length > 0
|
||||
? c.misconceptions.map((m) => ` - tag: "${m.tag}"\n signature: ${m.signature}`).join('\n')
|
||||
: ' (none)';
|
||||
|
||||
return `Checkpoint prompt: ${c.checkpointPrompt}
|
||||
|
||||
Reference answer: ${c.referenceAnswer}
|
||||
|
||||
Rubric criteria:
|
||||
${rubricLines}
|
||||
|
||||
Known misconceptions for this concept:
|
||||
${miscLines}
|
||||
|
||||
Learner's response:
|
||||
"""
|
||||
${c.learnerResponse}
|
||||
"""`;
|
||||
}
|
||||
|
||||
// ── Runner ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function runCase(c: GoldenCase): Promise<CaseResult> {
|
||||
const start = Date.now();
|
||||
let actual: GradeOutput | null = null;
|
||||
let error: string | undefined;
|
||||
|
||||
try {
|
||||
const { object } = await generateObject({
|
||||
model: llmClient.grader,
|
||||
schema: GradeOutputSchema,
|
||||
system: prompts.GRADE_RESPONSE.template,
|
||||
prompt: buildPrompt(c),
|
||||
});
|
||||
actual = object;
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
const latencyMs = Date.now() - start;
|
||||
const verdictMatch = actual?.verdict === c.expectedVerdict;
|
||||
const isFalseFail = c.isCorrect && actual?.verdict === 'misconception';
|
||||
const isFalsePass = !c.isCorrect && actual?.verdict === 'mastered';
|
||||
|
||||
return { case: c, actual, error, verdictMatch, isFalseFail, isFalsePass, latencyMs };
|
||||
}
|
||||
|
||||
function printResult(r: CaseResult, idx: number, total: number): void {
|
||||
const status = r.error
|
||||
? ' ERROR '
|
||||
: r.isFalseFail
|
||||
? '✗ FF '
|
||||
: r.isFalsePass
|
||||
? '✗ FP '
|
||||
: r.verdictMatch
|
||||
? '✓ '
|
||||
: '✗ ';
|
||||
|
||||
const expected = r.case.expectedVerdict.padEnd(12);
|
||||
const actual = r.actual ? r.actual.verdict.padEnd(12) : 'null ';
|
||||
const ms = `${r.latencyMs}ms`.padStart(6);
|
||||
|
||||
console.log(
|
||||
` [${String(idx + 1).padStart(2)}/${total}] ${status} expected=${expected} actual=${actual} ${ms} "${r.case.label}"`,
|
||||
);
|
||||
|
||||
if (r.error) {
|
||||
console.log(` Error: ${r.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log(`\nCurio — golden grading eval (${CASES.length} cases)\n`);
|
||||
console.log(` Grader: ${process.env.LLM_GRADER_PROVIDER ?? 'openai'} / ${process.env.LLM_GRADER_MODEL ?? 'gpt-4o-mini'}`);
|
||||
console.log(` False-fail threshold: ${(FALSE_FAIL_THRESHOLD * 100).toFixed(0)}%\n`);
|
||||
|
||||
const results: CaseResult[] = [];
|
||||
|
||||
for (let i = 0; i < CASES.length; i++) {
|
||||
const c = CASES[i];
|
||||
process.stdout.write(` running [${i + 1}/${CASES.length}] "${c.label}"…\r`);
|
||||
const result = await runCase(c);
|
||||
results.push(result);
|
||||
printResult(result, i, CASES.length);
|
||||
}
|
||||
|
||||
// ── Metrics ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const ran = results.filter((r) => !r.error);
|
||||
const errors = results.filter((r) => r.error);
|
||||
const verdictMatches = ran.filter((r) => r.verdictMatch).length;
|
||||
const falseFails = ran.filter((r) => r.isFalseFail);
|
||||
const falsePasses = ran.filter((r) => r.isFalsePass);
|
||||
const correctCases = ran.filter((r) => r.case.isCorrect);
|
||||
|
||||
const accuracy = ran.length > 0 ? verdictMatches / ran.length : 0;
|
||||
const falseFailRate = correctCases.length > 0 ? falseFails.length / correctCases.length : 0;
|
||||
|
||||
console.log('\n── Results ─────────────────────────────────────────────────────────────');
|
||||
console.log(` Cases run: ${ran.length} / ${CASES.length} (${errors.length} errors)`);
|
||||
console.log(` Verdict accuracy: ${verdictMatches}/${ran.length} = ${(accuracy * 100).toFixed(1)}%`);
|
||||
console.log(` False-fail rate: ${falseFails.length}/${correctCases.length} = ${(falseFailRate * 100).toFixed(1)}% (threshold: ${(FALSE_FAIL_THRESHOLD * 100).toFixed(0)}%)`);
|
||||
console.log(` False-pass rate: ${falsePasses.length}/${ran.filter((r) => !r.case.isCorrect).length}`);
|
||||
|
||||
if (falseFails.length > 0) {
|
||||
console.log('\n── False-fails (correct learner marked wrong) ───────────────────────────');
|
||||
for (const ff of falseFails) {
|
||||
console.log(` [${ff.case.id}] "${ff.case.label}"`);
|
||||
console.log(` Response: "${ff.case.learnerResponse.slice(0, 80)}…"`);
|
||||
console.log(` Verdict: ${ff.actual?.verdict} Evidence: "${ff.actual?.evidence_span}"`);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.log('\n── Errors ───────────────────────────────────────────────────────────────');
|
||||
for (const r of errors) {
|
||||
console.log(` [${r.case.id}] ${r.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── CI gate ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const ACCURACY_FLOOR = 0.70;
|
||||
let failed = false;
|
||||
|
||||
if (falseFailRate > FALSE_FAIL_THRESHOLD) {
|
||||
console.log(
|
||||
`\n✗ FAIL: false-fail rate ${(falseFailRate * 100).toFixed(1)}% exceeds threshold ${(FALSE_FAIL_THRESHOLD * 100).toFixed(0)}%`,
|
||||
);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (accuracy < ACCURACY_FLOOR) {
|
||||
console.log(
|
||||
`\n✗ FAIL: verdict accuracy ${(accuracy * 100).toFixed(1)}% below floor ${(ACCURACY_FLOOR * 100).toFixed(0)}%`,
|
||||
);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (!failed) {
|
||||
console.log('\n✓ PASS: grading quality within thresholds\n');
|
||||
}
|
||||
|
||||
process.exit(failed ? 1 : 0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Eval runner crashed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,145 @@
|
||||
/**
|
||||
* Misconception hit-rate eval.
|
||||
*
|
||||
* Measures: given a response exhibiting a known misconception, does the
|
||||
* grader return the correct tag when the library is provided?
|
||||
*
|
||||
* Two conditions run per case:
|
||||
* WITH library — grader receives the misconception library (expected: hit the tag)
|
||||
* WITHOUT library — grader receives empty library (baseline: "novel" or "none")
|
||||
*
|
||||
* CI gate: WITH-library hit rate must exceed HIT_RATE_THRESHOLD.
|
||||
*/
|
||||
|
||||
import { generateObject } from 'ai';
|
||||
import { GradeOutputSchema } from '../../src/schemas/grading';
|
||||
import type { GradeOutput } from '../../src/schemas/grading';
|
||||
import { prompts } from '../../src/lib/llm/prompts';
|
||||
import { llmClient } from '../../src/lib/llm/client';
|
||||
import {
|
||||
MISCONCEPTION_CASES,
|
||||
HIT_RATE_THRESHOLD,
|
||||
BASELINE_HIT_RATE,
|
||||
} from './misconception-cases';
|
||||
import type { MisconceptionCase } from './misconception-cases';
|
||||
|
||||
// ── Prompt builder ────────────────────────────────────────────────────────────
|
||||
|
||||
function buildPrompt(
|
||||
c: MisconceptionCase,
|
||||
library: Array<{ tag: string; signature: string }>,
|
||||
): string {
|
||||
const rubricLines = c.rubric
|
||||
.map((r) => ` - [${r.id}] ${r.criterion} (weight: ${r.weight.toFixed(2)})`)
|
||||
.join('\n');
|
||||
|
||||
const miscLines =
|
||||
library.length > 0
|
||||
? library.map((m) => ` - tag: "${m.tag}"\n signature: ${m.signature}`).join('\n')
|
||||
: ' (none)';
|
||||
|
||||
return `Checkpoint prompt: ${c.checkpointPrompt}
|
||||
|
||||
Reference answer: ${c.referenceAnswer}
|
||||
|
||||
Rubric criteria:
|
||||
${rubricLines}
|
||||
|
||||
Known misconceptions for this concept:
|
||||
${miscLines}
|
||||
|
||||
Learner's response:
|
||||
"""
|
||||
${c.learnerResponse}
|
||||
"""`;
|
||||
}
|
||||
|
||||
// ── Single grading call ───────────────────────────────────────────────────────
|
||||
|
||||
async function grade(
|
||||
c: MisconceptionCase,
|
||||
library: Array<{ tag: string; signature: string }>,
|
||||
): Promise<{ grade: GradeOutput | null; error?: string; latencyMs: number }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const { object } = await generateObject({
|
||||
model: llmClient.grader,
|
||||
schema: GradeOutputSchema,
|
||||
system: prompts.GRADE_RESPONSE.template,
|
||||
prompt: buildPrompt(c, library),
|
||||
});
|
||||
return { grade: object, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return {
|
||||
grade: null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
latencyMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Runner ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log(`\nCurio — misconception hit-rate eval (${MISCONCEPTION_CASES.length} cases × 2 conditions)\n`);
|
||||
console.log(` Grader: ${process.env.LLM_GRADER_PROVIDER ?? 'openai'} / ${process.env.LLM_GRADER_MODEL ?? 'gpt-4o-mini'}`);
|
||||
console.log(` Hit-rate threshold: ${(HIT_RATE_THRESHOLD * 100).toFixed(0)}%\n`);
|
||||
|
||||
let withHits = 0;
|
||||
let baselineHits = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const c of MISCONCEPTION_CASES) {
|
||||
process.stdout.write(` [${c.id}] "${c.label}"…\r`);
|
||||
|
||||
const [withResult, baselineResult] = await Promise.all([
|
||||
grade(c, c.library),
|
||||
grade(c, []),
|
||||
]);
|
||||
|
||||
const withTag = withResult.grade?.misconception_tag;
|
||||
const baselineTag = baselineResult.grade?.misconception_tag;
|
||||
|
||||
const withHit = withTag === c.targetTag;
|
||||
const baselineHit = baselineTag === c.targetTag;
|
||||
|
||||
if (withResult.error || baselineResult.error) errors++;
|
||||
if (withHit) withHits++;
|
||||
if (baselineHit) baselineHits++;
|
||||
|
||||
const withStatus = withResult.error ? ' ERR ' : withHit ? '✓ hit ' : '✗ miss';
|
||||
const bStatus = baselineResult.error ? ' ERR ' : baselineHit ? '✓ hit ' : '✗ miss';
|
||||
|
||||
console.log(
|
||||
` [${c.id}] WITH=${withStatus} (tag="${withTag ?? 'null'}") BASE=${bStatus} (tag="${baselineTag ?? 'null'}")`,
|
||||
);
|
||||
}
|
||||
|
||||
const n = MISCONCEPTION_CASES.length;
|
||||
const withRate = n > 0 ? withHits / n : 0;
|
||||
const baseRate = n > 0 ? baselineHits / n : 0;
|
||||
const lift = withRate - baseRate;
|
||||
|
||||
console.log('\n── Results ─────────────────────────────────────────────────────────────');
|
||||
console.log(` Cases: ${n} (${errors} errors)`);
|
||||
console.log(` WITH library: ${withHits}/${n} = ${(withRate * 100).toFixed(1)}% (threshold: ${(HIT_RATE_THRESHOLD * 100).toFixed(0)}%)`);
|
||||
console.log(` Baseline (no lib): ${baselineHits}/${n} = ${(baseRate * 100).toFixed(1)}% (expected: ${(BASELINE_HIT_RATE * 100).toFixed(0)}%)`);
|
||||
console.log(` Lift: +${(lift * 100).toFixed(1)}pp`);
|
||||
|
||||
const failed = withRate < HIT_RATE_THRESHOLD;
|
||||
|
||||
if (failed) {
|
||||
console.log(
|
||||
`\n✗ FAIL: hit rate ${(withRate * 100).toFixed(1)}% below threshold ${(HIT_RATE_THRESHOLD * 100).toFixed(0)}%`,
|
||||
);
|
||||
} else {
|
||||
console.log('\n✓ PASS: misconception hit rate within threshold\n');
|
||||
}
|
||||
|
||||
process.exit(failed ? 1 : 0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Eval runner crashed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user