arnaudne/Epicure
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: split photo-import into vision recognition + text generation

Browse Source 
The photo-import flow used one vision-capable model to both read the
photo and structure the full recipe (quantities, steps, timing) in a
single call. Split it into two: a vision model recognizes what's in
the picture, then a text model reconstructs the recipe from that
description — same pattern the pantry photo-scan already uses for
recognition, and lets structuring use whichever model is actually
configured for text generation. Still counts as one AI-quota unit.

v0.37.0
This commit is contained in:
Arnaud
2026-07-17 16:12:39 +02:00
parent 5763bd3318
commit f0340859fa
12 changed files with 157 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/web/lib/ai/features/import-photo.ts
+20 -44
View File
@@ -1,54 +1,30 @@
import { generateObject } from "ai";
import { z } from "zod";
import { resolveModel, type AiConfig } from "../factory";
import { dietaryTagsSchema, ingredientSchema, stepSchema } from "./recipe-schema";
import type { AiConfig } from "../factory";
import { recognizePhoto } from "./recognize-photo";
import { generateRecipeFromRecognition, type GeneratedRecipeFromRecognition } from "./generate-recipe-from-recognition";
const ImportedRecipeSchema = z.object({
found: z.boolean().describe("false if the image does not contain a recognizable recipe (e.g. random photo, no visible ingredients/instructions)"),
recipeType: z.enum(["dish", "drink"]).describe("\"drink\" only for a beverage/cocktail (no cooking step) — \"dish\" for anything else, including if unsure."),
title: z.string(),
description: z.string().optional(),
baseServings: z.number().int().min(1).optional(),
prepMins: z.number().int().min(0).optional(),
cookMins: z.number().int().min(0).optional(),
difficulty: z.enum(["easy", "medium", "hard"]).optional(),
dietaryTags: dietaryTagsSchema.optional(),
ingredients: z.array(ingredientSchema(z.string())),
steps: z.array(stepSchema),
});
export type ImportedRecipe = z.infer<typeof ImportedRecipeSchema>;
const LANG: Record<string, string> = { en: "English", fr: "French" };
export type ImportedRecipe =
| { found: false; recipeType: "dish" | "drink"; title: string; ingredients: []; steps: [] }
| ({ found: true; recipeType: "dish" | "drink" } & GeneratedRecipeFromRecognition);
/** Orchestrates the two-step photo-import flow: a vision model recognizes
* what's in the photo (`recognizePhoto`), then a text model reconstructs the
* full structured recipe from that recognition (`generateRecipeFromRecognition`).
* Splitting the call lets each step use the model best suited to it — vision
* for the photo, text for structuring — instead of one model doing both. */
export async function importFromPhoto(
imageBase64: string,
mimeType: "image/jpeg" | "image/png" | "image/webp",
config?: AiConfig,
visionConfig: AiConfig | undefined,
textConfig: AiConfig | undefined,
locale?: string,
): Promise<ImportedRecipe> {
const model = resolveModel(config);
const lang = LANG[locale ?? "en"] ?? "English";
const langInstruction = lang !== "English" ? ` Write the entire recipe (title, description, ingredient names, step instructions) in ${lang}, regardless of the language used in the image.` : "";
const recognition = await recognizePhoto(imageBase64, mimeType, visionConfig);
const { object } = await generateObject({
model,
schema: ImportedRecipeSchema,
system:
`You are a recipe extraction specialist. Extract the complete recipe from the provided image. Be precise with ingredient quantities and cooking instructions. If the image does not contain a recognizable recipe (no visible ingredients or cooking instructions), set found to false and leave the other fields minimal/empty. Set recipeType to "drink" only for a beverage/cocktail with no cooking involved — never set cookMins for a drink.${langInstruction}`,
messages: [
{
role: "user",
content: [
{ type: "image", image: imageBase64, mediaType: mimeType },
{ type: "text", text: "Extract the complete recipe from this image." },
],
},
],
});
if (object.recipeType === "drink") {
return { ...object, cookMins: undefined };
if (!recognition.found) {
return { found: false, recipeType: recognition.recipeType, title: recognition.title, ingredients: [], steps: [] };
}
return object;
const generated = await generateRecipeFromRecognition(recognition, { ...textConfig, language: locale ?? "en" });
return { found: true, recipeType: recognition.recipeType, ...generated };
}