feat: full feature buildout — streaming, i18n, mastery map, admin, jobs

Progressive lesson streaming via onSegment callback (fixes SSE for non-English
users — locale was shadowed in lesson-reader useEffect). Adds: BullMQ workers,
Redis stream buffer, token budget enforcement, Langfuse tracing, golden-eval
runner, Playwright e2e scaffolding, lesson depth/locale/preferences schema,
mastery map UI, admin panel (blueprints/users/reports/quality/misconceptions),
image queries, source citations, view transitions, reading animations, i18n
(next-intl), PDF export, surprise endpoint, and 402 passing unit tests.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

jobs/generate-lesson-job.ts

@@ -6,13 +6,14 @@
  * On failure: increments attempt count; sets status to 'failed' after processing.
  */
 import { Worker } from 'bullmq';
-import { eq } from 'drizzle-orm';
+import { eq, and } from 'drizzle-orm';
 import { db } from '../src/lib/db';
 import { jobs, blueprints, concepts, lessons } from '../src/lib/db/schema';
 import { generateLesson } from '../src/lib/generation/generate-lesson';
 import { verifyLesson } from '../src/lib/verification/verify-content';
 import { getLessonResponse } from '../src/lib/db/queries';
 import { setCachedLesson } from '../src/lib/cache/lesson';
+import { appendStreamSegment, markStreamDone } from '../src/lib/cache/stream';
 import { enqueuePromoteBlueprint } from '../src/lib/jobs/queue';
 import type { GenerateLessonJobData } from '../src/lib/jobs/queue';
 
@@ -23,7 +24,7 @@ const connection = {
 const worker = new Worker<GenerateLessonJobData>(
   'generate-lesson',
   async (job) => {
-    const { intentKey, blueprintId, idempotencyKey } = job.data;
+    const { intentKey, blueprintId, idempotencyKey, locale = 'en', depth = 'standard', ageGroup = 'adult', difficultyLevel = 1, ord = 0 } = job.data;
 
     // 1. Idempotency check — find the job row by idempotency key
     const [jobRow] = await db
@@ -34,9 +35,9 @@ const worker = new Worker<GenerateLessonJobData>(
 
     if (jobRow?.status === 'done') {
       // Already completed — warm cache if needed and return
-      const cached = await getLessonResponse(intentKey);
+      const cached = await getLessonResponse(intentKey, undefined, locale, depth as import('../src/lib/generation/depth').LessonDepth, ageGroup as import('../src/schemas/preferences').AgeGroup);
       if (cached?.state === 'ready') {
-        await setCachedLesson(intentKey, cached);
+        await setCachedLesson(intentKey, locale, cached, depth as import('../src/lib/generation/depth').LessonDepth, ageGroup as import('../src/schemas/preferences').AgeGroup);
       }
       return;
     }
@@ -47,15 +48,31 @@ const worker = new Worker<GenerateLessonJobData>(
       .set({ status: 'running', attempts: (jobRow?.attempts ?? 0) + 1 })
       .where(eq(jobs.id, idempotencyKey));
 
-    // 2. Check lesson doesn't already exist
-    const existing = await getLessonResponse(intentKey);
-    if (existing?.state === 'ready') {
-      await setCachedLesson(intentKey, existing);
+    // 2. Check lesson at this ord doesn't already exist
+    const [existingLesson] = await db
+      .select({ id: lessons.id })
+      .from(lessons)
+      .where(
+        and(
+          eq(lessons.blueprintId, blueprintId),
+          eq(lessons.locale, locale),
+          eq(lessons.depth, depth),
+          eq(lessons.ageGroup, ageGroup),
+          eq(lessons.ord, ord),
+        ),
+      )
+      .limit(1);
+
+    if (existingLesson) {
+      const cached = await getLessonResponse(intentKey, undefined, locale, depth as import('../src/lib/generation/depth').LessonDepth, ageGroup as import('../src/schemas/preferences').AgeGroup);
+      if (cached?.state === 'ready') {
+        await setCachedLesson(intentKey, locale, cached, depth as import('../src/lib/generation/depth').LessonDepth, ageGroup as import('../src/schemas/preferences').AgeGroup);
+      }
       await db.update(jobs).set({ status: 'done' }).where(eq(jobs.id, idempotencyKey));
       return;
     }
 
-    // 3. Fetch blueprint + concepts
+    // 3. Fetch blueprint + concepts at this ord
     const [blueprint] = await db
       .select()
       .from(blueprints)
@@ -64,32 +81,57 @@ const worker = new Worker<GenerateLessonJobData>(
 
     if (!blueprint) throw new Error(`Blueprint not found: ${blueprintId}`);
 
+    // For ord=0 use all concepts (backward compat); for ord>0 use concepts at that ord.
     const conceptRows = await db
       .select({ id: concepts.id, name: concepts.name, ord: concepts.ord, retrievalCtxRef: concepts.retrievalCtxRef })
       .from(concepts)
-      .where(eq(concepts.blueprintId, blueprintId));
+      .where(
+        ord === 0
+          ? eq(concepts.blueprintId, blueprintId)
+          : and(eq(concepts.blueprintId, blueprintId), eq(concepts.ord, ord)),
+      );
 
-    if (conceptRows.length === 0) throw new Error(`No concepts for blueprint: ${blueprintId}`);
+    if (conceptRows.length === 0) throw new Error(`No concepts at ord=${ord} for blueprint: ${blueprintId}`);
 
-    // 4. Generate lesson
+    // 4. Generate lesson — onSegment pushes each segment to Redis as it's persisted,
+    //    so SSE clients receive progressive delivery without waiting for the full batch.
     const { lessonId } = await generateLesson({
       blueprintId,
-      lessonOrd: 0,
+      lessonOrd: ord,
       topicTitle: blueprint.title,
       conceptsToGenerate: conceptRows.map((c) => ({
         id: c.id,
         name: c.name,
         retrievalCtxRef: c.retrievalCtxRef,
       })),
+      locale,
+      depth: depth as import('../src/lib/generation/depth').LessonDepth,
+      ageGroup: ageGroup as import('../src/schemas/preferences').AgeGroup,
+      difficultyLevel: difficultyLevel as 1 | 2 | 3,
+      onSegment: (seg) =>
+        appendStreamSegment(
+          intentKey, locale, seg,
+          depth as import('../src/lib/generation/depth').LessonDepth,
+          ageGroup as import('../src/schemas/preferences').AgeGroup,
+          ord,
+        ),
     });
 
+    // Signal SSE clients that the stream is complete.
+    await markStreamDone(
+      intentKey, locale,
+      depth as import('../src/lib/generation/depth').LessonDepth,
+      ageGroup as import('../src/schemas/preferences').AgeGroup,
+      ord,
+    );
+
     // 5. T1 verification (non-blocking for serve — T2 + promotion handled by promote-blueprint job)
     await verifyLesson({ lessonId, runT2: false });
 
-    // 6. Write to Redis cache
-    const lesson = await getLessonResponse(intentKey);
+    // 6. Write to Redis cache (position-keyed)
+    const lesson = await getLessonResponse(intentKey, undefined, locale, depth as import('../src/lib/generation/depth').LessonDepth, ageGroup as import('../src/schemas/preferences').AgeGroup);
     if (lesson?.state === 'ready') {
-      await setCachedLesson(intentKey, lesson);
+      await setCachedLesson(intentKey, locale, lesson, depth as import('../src/lib/generation/depth').LessonDepth, ageGroup as import('../src/schemas/preferences').AgeGroup, ord);
     }
 
     // 7. Update job ledger