Torben Sorensen
498fbd9e0e
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 21m5s
932 lines
37 KiB
TypeScript
932 lines
37 KiB
TypeScript
// src/services/aiService.server.ts
|
|
/**
|
|
* @file This file contains all server-side functions that directly interact with the Google AI (Gemini) API.
|
|
* It is intended to be used only by the backend (e.g., server.ts) and should never be imported into client-side code.
|
|
* The `.server.ts` naming convention helps enforce this separation.
|
|
*/
|
|
import { GoogleGenAI, type GenerateContentResponse, type Content, type Tool } from '@google/genai';
|
|
import fsPromises from 'node:fs/promises';
|
|
import type { Logger } from 'pino';
|
|
import { z } from 'zod';
|
|
import { pRateLimit } from 'p-ratelimit';
|
|
import type {
|
|
FlyerItem,
|
|
MasterGroceryItem,
|
|
ExtractedFlyerItem,
|
|
UserProfile,
|
|
ExtractedCoreData,
|
|
FlyerInsert,
|
|
Flyer,
|
|
} from '../types';
|
|
import { FlyerProcessingError } from './processingErrors';
|
|
import * as db from './db/index.db';
|
|
import { flyerQueue } from './queueService.server';
|
|
import type { Job } from 'bullmq';
|
|
import { createFlyerAndItems } from './db/flyer.db';
|
|
import { generateFlyerIcon } from '../utils/imageProcessor';
|
|
import path from 'path';
|
|
import { ValidationError } from './db/errors.db'; // Keep this import for ValidationError
|
|
import {
|
|
AiFlyerDataSchema,
|
|
ExtractedFlyerItemSchema,
|
|
} from '../types/ai'; // Import consolidated schemas
|
|
|
|
interface FlyerProcessPayload extends Partial<ExtractedCoreData> {
|
|
checksum?: string;
|
|
originalFileName?: string;
|
|
extractedData?: Partial<ExtractedCoreData>;
|
|
data?: FlyerProcessPayload; // For nested data structures
|
|
}
|
|
|
|
// Helper to safely extract an error message from unknown `catch` values.
|
|
const errMsg = (e: unknown) => {
|
|
if (e instanceof Error) return e.message;
|
|
if (typeof e === 'object' && e !== null && 'message' in e)
|
|
return String((e as { message: unknown }).message);
|
|
return String(e || 'An unknown error occurred.');
|
|
};
|
|
|
|
/**
|
|
* Defines the contract for a file system utility. This interface allows for
|
|
* dependency injection, making the AIService testable without hitting the real file system.
|
|
*/
|
|
interface IFileSystem {
|
|
readFile(path: string): Promise<Buffer>;
|
|
}
|
|
|
|
/**
|
|
* Defines the contract for an AI model client. This allows for dependency injection,
|
|
* making the AIService testable without making real API calls to Google.
|
|
*/
|
|
interface IAiClient {
|
|
generateContent(request: {
|
|
contents: Content[];
|
|
tools?: Tool[];
|
|
useLiteModels?: boolean;
|
|
}): Promise<GenerateContentResponse>;
|
|
}
|
|
|
|
/**
|
|
* Defines the shape of a single flyer item as returned by the AI.
|
|
* This type is intentionally loose to accommodate potential null/undefined values
|
|
* from the AI before they are cleaned and normalized.
|
|
*/
|
|
export type RawFlyerItem = {
|
|
item: string | null;
|
|
price_display: string | null | undefined;
|
|
price_in_cents: number | null | undefined;
|
|
quantity: string | null | undefined;
|
|
category_name: string | null | undefined;
|
|
master_item_id?: number | null | undefined;
|
|
};
|
|
|
|
export class DuplicateFlyerError extends FlyerProcessingError {
|
|
constructor(message: string, public flyerId: number) {
|
|
super(message, 'DUPLICATE_FLYER', message);
|
|
}
|
|
}
|
|
|
|
export class AIService {
|
|
private aiClient: IAiClient;
|
|
private fs: IFileSystem;
|
|
private rateLimiter: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
private logger: Logger;
|
|
// The fallback list is ordered by preference (speed/cost vs. power).
|
|
// We try the fastest models first, then the more powerful 'pro' model as a high-quality fallback,
|
|
// and finally the 'lite' model as a last resort.
|
|
private readonly models = [ 'gemini-3-flash-preview','gemini-2.5-pro', 'gemini-2.5-flash', 'gemini-2.5-flash-lite','gemini-2.0-flash-001','gemini-2.0-flash','gemini-2.0-flash-exp','gemini-2.0-flash-lite-001','gemini-2.0-flash-lite', 'gemma-3-27b-it', 'gemma-3-12b-it'];
|
|
private readonly models_lite = ["gemma-3-4b-it", "gemma-3-2b-it", "gemma-3-1b-it"];
|
|
|
|
constructor(logger: Logger, aiClient?: IAiClient, fs?: IFileSystem) {
|
|
this.logger = logger;
|
|
this.logger.info('---------------- [AIService] Constructor Start ----------------');
|
|
if (aiClient) {
|
|
this.logger.info(
|
|
'[AIService Constructor] Using provided mock AI client. This indicates a TEST environment.',
|
|
);
|
|
this.aiClient = aiClient;
|
|
} else {
|
|
this.logger.info(
|
|
'[AIService Constructor] No mock client provided. Initializing Google GenAI client for PRODUCTION-LIKE environment.',
|
|
);
|
|
// Determine if we are in any kind of test environment.
|
|
// VITEST_POOL_ID is reliably set by Vitest during test runs.
|
|
const isTestEnvironment = process.env.NODE_ENV === 'test' || !!process.env.VITEST_POOL_ID;
|
|
this.logger.info(
|
|
{
|
|
isTestEnvironment,
|
|
nodeEnv: process.env.NODE_ENV,
|
|
vitestPoolId: process.env.VITEST_POOL_ID,
|
|
hasApiKey: !!process.env.GEMINI_API_KEY,
|
|
},
|
|
'[AIService Constructor] Environment check',
|
|
);
|
|
|
|
const apiKey = process.env.GEMINI_API_KEY;
|
|
if (!apiKey) {
|
|
this.logger.warn('[AIService] GEMINI_API_KEY is not set.');
|
|
// Allow initialization without key in test/build environments if strictly needed
|
|
if (!isTestEnvironment) {
|
|
this.logger.error('[AIService] GEMINI_API_KEY is required in non-test environments.');
|
|
throw new Error('GEMINI_API_KEY environment variable not set for server-side AI calls.');
|
|
} else {
|
|
this.logger.warn(
|
|
'[AIService Constructor] GEMINI_API_KEY is missing, but this is a test environment, so proceeding.',
|
|
);
|
|
}
|
|
}
|
|
// In test mode without injected client, we might not have a key.
|
|
// The stubs below protect against calling the undefined client.
|
|
// This is the correct modern SDK pattern. We instantiate the main client.
|
|
const genAI = apiKey ? new GoogleGenAI({ apiKey }) : null;
|
|
if (!genAI) {
|
|
this.logger.warn(
|
|
'[AIService] GoogleGenAI client could not be initialized (likely missing API key in test environment). Using mock placeholder.',
|
|
);
|
|
}
|
|
|
|
// We create a shim/adapter that matches the old structure but uses the new SDK call pattern.
|
|
// This preserves the dependency injection pattern used throughout the class.
|
|
this.aiClient = genAI
|
|
? {
|
|
generateContent: async (request) => {
|
|
if (!request.contents || request.contents.length === 0) {
|
|
this.logger.error(
|
|
{ request },
|
|
'[AIService Adapter] generateContent called with no content, which is invalid.',
|
|
);
|
|
throw new Error('AIService.generateContent requires at least one content element.');
|
|
}
|
|
|
|
const { useLiteModels, ...apiReq } = request;
|
|
const models = useLiteModels ? this.models_lite : this.models;
|
|
return this._generateWithFallback(genAI, apiReq, models);
|
|
},
|
|
}
|
|
: {
|
|
// This is the updated mock for testing, matching the new response shape.
|
|
generateContent: async () => {
|
|
this.logger.warn(
|
|
'[AIService] Mock generateContent called. This should only happen in tests when no API key is available.',
|
|
);
|
|
return { text: '[]' } as unknown as GenerateContentResponse;
|
|
},
|
|
};
|
|
}
|
|
|
|
this.fs = fs || fsPromises;
|
|
|
|
if (aiClient) {
|
|
this.logger.warn(
|
|
'[AIService Constructor] Mock client detected. Rate limiter is DISABLED for testing.',
|
|
);
|
|
this.rateLimiter = <T>(fn: () => Promise<T>) => fn(); // Pass-through function
|
|
} else {
|
|
const requestsPerMinute = parseInt(process.env.GEMINI_RPM || '5', 10);
|
|
this.logger.info(
|
|
`[AIService Constructor] Initializing production rate limiter to ${requestsPerMinute} RPM.`,
|
|
);
|
|
this.rateLimiter = pRateLimit({
|
|
interval: 60 * 1000,
|
|
rate: requestsPerMinute,
|
|
concurrency: requestsPerMinute,
|
|
});
|
|
}
|
|
this.logger.info('---------------- [AIService] Constructor End ----------------');
|
|
}
|
|
|
|
private async _generateWithFallback(
|
|
genAI: GoogleGenAI,
|
|
request: { contents: Content[]; tools?: Tool[] },
|
|
models: string[] = this.models,
|
|
): Promise<GenerateContentResponse> {
|
|
let lastError: Error | null = null;
|
|
|
|
for (const modelName of models) {
|
|
try {
|
|
this.logger.info(
|
|
`[AIService Adapter] Attempting to generate content with model: ${modelName}`,
|
|
);
|
|
const result = await genAI.models.generateContent({ model: modelName, ...request });
|
|
// If the call succeeds, return the result immediately.
|
|
return result;
|
|
} catch (error: unknown) {
|
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
const errorMessage = (lastError.message || '').toLowerCase(); // Make case-insensitive
|
|
|
|
// Check for specific error messages indicating quota issues or model unavailability.
|
|
if (
|
|
errorMessage.includes('quota') ||
|
|
errorMessage.includes('429') || // HTTP 429 Too Many Requests
|
|
errorMessage.includes('resource_exhausted') || // Make case-insensitive
|
|
errorMessage.includes('model is overloaded') ||
|
|
errorMessage.includes('not found') // Also retry if model is not found (e.g., regional availability or API version issue)
|
|
) {
|
|
this.logger.warn(
|
|
`[AIService Adapter] Model '${modelName}' failed due to quota/rate limit. Trying next model. Error: ${errorMessage}`,
|
|
);
|
|
continue; // Try the next model in the list.
|
|
} else {
|
|
// For other errors (e.g., invalid input, safety settings), fail immediately.
|
|
this.logger.error(
|
|
{ error: lastError },
|
|
`[AIService Adapter] Model '${modelName}' failed with a non-retriable error.`,
|
|
);
|
|
throw lastError;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If all models in the list have failed, throw the last error encountered.
|
|
this.logger.error(
|
|
{ lastError },
|
|
'[AIService Adapter] All AI models failed. Throwing last known error.',
|
|
);
|
|
throw lastError || new Error('All AI models failed to generate content.');
|
|
}
|
|
|
|
private async serverFileToGenerativePart(path: string, mimeType: string) {
|
|
const fileData = await this.fs.readFile(path);
|
|
return {
|
|
inlineData: {
|
|
data: fileData.toString('base64'),
|
|
mimeType,
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Constructs the detailed prompt for the AI to extract flyer data.
|
|
* @param masterItems A list of known grocery items to aid in matching.
|
|
* @param submitterIp The IP address of the user who submitted the flyer.
|
|
* @param userProfileAddress The profile address of the user.
|
|
* @returns A formatted string to be used as the AI prompt.
|
|
*/
|
|
private _buildFlyerExtractionPrompt(
|
|
masterItems: MasterGroceryItem[],
|
|
submitterIp?: string,
|
|
userProfileAddress?: string,
|
|
): string {
|
|
let locationHint = '';
|
|
if (userProfileAddress) {
|
|
locationHint = `The user who uploaded this flyer has a profile address of "${userProfileAddress}". Use this as a strong hint for the store's location.`;
|
|
} else if (submitterIp) {
|
|
locationHint = `The user uploaded this flyer from an IP address that suggests a location. Use this as a general hint for the store's region.`;
|
|
}
|
|
|
|
// Optimization: Instead of sending the whole masterItems object, send only the necessary fields.
|
|
// This significantly reduces the number of tokens used in the prompt.
|
|
const simplifiedMasterList = masterItems.map((item) => ({
|
|
id: item.master_grocery_item_id,
|
|
name: item.name,
|
|
}));
|
|
|
|
return `
|
|
# TASK
|
|
Analyze the provided flyer image(s) and extract key information into a single, valid JSON object.
|
|
|
|
# RULES
|
|
1. Extract the following top-level details for the flyer:
|
|
- "store_name": The name of the grocery store (e.g., "Walmart", "No Frills").
|
|
- "valid_from": The start date of the sale in YYYY-MM-DD format. Use null if not present.
|
|
- "valid_to": The end date of the sale in YYYY-MM-DD format. Use null if not present.
|
|
- "store_address": The physical address of the store. Use null if not present. ${locationHint}
|
|
|
|
2. Extract each individual sale item into an "items" array. For each item, provide:
|
|
- "item": The name of the product (e.g., "Coca-Cola Classic").
|
|
- "price_display": The exact sale price as a string (e.g., "$2.99", "2 for $5.00"). If no price is visible, use an empty string "".
|
|
- "price_in_cents": The primary numeric price in cents. For "$2.99", use 299. For "2 for $5.00", use 500. If no price is visible, you MUST use null.
|
|
- "quantity": A string describing the quantity or weight (e.g., "12x355mL", "500g", "each"). If no quantity is visible, use an empty string "".
|
|
- "master_item_id": Find the best matching item from the MASTER LIST provided below and return its "id". If no good match is found, you MUST use null.
|
|
- "category_name": The most appropriate category (e.g., "Beverages", "Meat & Seafood"). If unsure, use "Other/Miscellaneous".
|
|
|
|
3. Your entire output MUST be a single JSON object. Do not include any other text, explanations, or markdown formatting like \`\`\`json.
|
|
|
|
# EXAMPLES
|
|
- For an item "Red Seedless Grapes" on sale for "$1.99 /lb" that matches master item ID 45:
|
|
{ "item": "Red Seedless Grapes", "price_display": "$1.99 /lb", "price_in_cents": 199, "quantity": "/lb", "master_item_id": 45, "category_name": "Produce" }
|
|
- For an item "PC Cola 2L" on sale "3 for $5.00" that has no master item match:
|
|
{ "item": "PC Cola 2L", "price_display": "3 for $5.00", "price_in_cents": 500, "quantity": "2L", "master_item_id": null, "category_name": "Beverages" }
|
|
- For an item "Store-made Muffins" with no price listed:
|
|
{ "item": "Store-made Muffins", "price_display": "", "price_in_cents": null, "quantity": "6 pack", "master_item_id": 123, "category_name": "Bakery" }
|
|
|
|
# MASTER LIST
|
|
${JSON.stringify(simplifiedMasterList)}
|
|
|
|
# JSON OUTPUT
|
|
`;
|
|
}
|
|
|
|
/**
|
|
* Safely parses a JSON object from a string, typically from an AI response.
|
|
* @param responseText The raw text response from the AI.
|
|
* @returns The parsed JSON object, or null if parsing fails.
|
|
*/
|
|
private _parseJsonFromAiResponse<T>(responseText: string | undefined, logger: Logger): T | null {
|
|
// --- START HYPER-DIAGNOSTIC LOGGING ---
|
|
console.log('\n--- DIAGNOSING _parseJsonFromAiResponse ---');
|
|
console.log(
|
|
`1. Initial responseText (Type: ${typeof responseText}):`,
|
|
JSON.stringify(responseText),
|
|
);
|
|
// --- END HYPER-DIAGNOSTIC LOGGING ---
|
|
|
|
if (!responseText) {
|
|
logger.warn(
|
|
'[_parseJsonFromAiResponse] Response text is empty or undefined. Returning null.',
|
|
);
|
|
console.log('2. responseText is falsy. ABORTING.');
|
|
console.log('--- END DIAGNOSIS ---\n');
|
|
return null;
|
|
}
|
|
|
|
// Find the start of the JSON, which can be inside a markdown block
|
|
const markdownRegex = /```(json)?\s*([\s\S]*?)\s*```/;
|
|
const markdownMatch = responseText.match(markdownRegex);
|
|
console.log('2. Regex Result (markdownMatch):', markdownMatch);
|
|
|
|
let jsonString;
|
|
if (markdownMatch && markdownMatch[2] !== undefined) {
|
|
// Check for capture group
|
|
console.log('3. Regex matched. Processing Captured Group.');
|
|
console.log(
|
|
` - Captured content (Type: ${typeof markdownMatch[2]}, Length: ${markdownMatch[2].length}):`,
|
|
JSON.stringify(markdownMatch[2]),
|
|
);
|
|
logger.debug(
|
|
{ rawCapture: markdownMatch[2] },
|
|
'[_parseJsonFromAiResponse] Found JSON content within markdown code block.',
|
|
);
|
|
|
|
jsonString = markdownMatch[2].trim();
|
|
console.log(
|
|
`4. After trimming, jsonString is (Type: ${typeof jsonString}, Length: ${jsonString.length}):`,
|
|
JSON.stringify(jsonString),
|
|
);
|
|
logger.debug(
|
|
{ trimmedJsonString: jsonString },
|
|
'[_parseJsonFromAiResponse] Trimmed extracted JSON string.',
|
|
);
|
|
} else {
|
|
console.log(
|
|
'3. Regex did NOT match or capture group 2 is undefined. Will attempt to parse entire responseText.',
|
|
);
|
|
jsonString = responseText;
|
|
}
|
|
|
|
// Find the first '{' or '[' and the last '}' or ']' to isolate the JSON object.
|
|
const firstBrace = jsonString.indexOf('{');
|
|
const firstBracket = jsonString.indexOf('[');
|
|
console.log(
|
|
`5. Index search on jsonString: firstBrace=${firstBrace}, firstBracket=${firstBracket}`,
|
|
);
|
|
|
|
// Determine the starting point of the JSON content
|
|
const startIndex =
|
|
firstBrace === -1 || (firstBracket !== -1 && firstBracket < firstBrace)
|
|
? firstBracket
|
|
: firstBrace;
|
|
console.log('6. Calculated startIndex:', startIndex);
|
|
|
|
if (startIndex === -1) {
|
|
logger.error(
|
|
{ responseText },
|
|
"[_parseJsonFromAiResponse] Could not find starting '{' or '[' in response.",
|
|
);
|
|
console.log('7. startIndex is -1. ABORTING.');
|
|
console.log('--- END DIAGNOSIS ---\n');
|
|
return null;
|
|
}
|
|
|
|
const jsonSlice = jsonString.substring(startIndex);
|
|
console.log(
|
|
`8. Sliced string to be parsed (jsonSlice) (Length: ${jsonSlice.length}):`,
|
|
JSON.stringify(jsonSlice),
|
|
);
|
|
|
|
try {
|
|
console.log('9. Attempting JSON.parse on jsonSlice...');
|
|
const parsed = JSON.parse(jsonSlice) as T;
|
|
console.log('10. SUCCESS: JSON.parse succeeded.');
|
|
console.log('--- END DIAGNOSIS (SUCCESS) ---\n');
|
|
return parsed;
|
|
} catch (e) {
|
|
logger.error(
|
|
{ jsonSlice, error: e, errorMessage: (e as Error).message, stack: (e as Error).stack },
|
|
'[_parseJsonFromAiResponse] Failed to parse JSON slice.',
|
|
);
|
|
console.error('10. FAILURE: JSON.parse FAILED. Error:', e);
|
|
console.log('--- END DIAGNOSIS (FAILURE) ---\n');
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async extractItemsFromReceiptImage(
|
|
imagePath: string,
|
|
imageMimeType: string,
|
|
logger: Logger = this.logger,
|
|
): Promise<{ raw_item_description: string; price_paid_cents: number }[] | null> {
|
|
const prompt = `
|
|
Analyze the provided receipt image. Extract all purchased line items.
|
|
For each item, identify its description and total price.
|
|
Return the data as a valid JSON array of objects. Each object should have two keys:
|
|
1. "raw_item_description": a string containing the item's name as written on the receipt.
|
|
2. "price_paid_cents": an integer representing the total price for that line item in cents (do not include currency symbols).
|
|
|
|
Example format:
|
|
[
|
|
{ "raw_item_description": "ORGANIC BANANAS", "price_paid_cents": 129 },
|
|
{ "raw_item_description": "AVOCADO", "price_paid_cents": 299 }
|
|
]
|
|
|
|
Only output the JSON array. Do not include any other text, explanations, or markdown formatting.
|
|
`;
|
|
|
|
const imagePart = await this.serverFileToGenerativePart(imagePath, imageMimeType);
|
|
|
|
logger.info('[extractItemsFromReceiptImage] Entering method.');
|
|
try {
|
|
logger.debug('[extractItemsFromReceiptImage] PRE-RATE-LIMITER: Preparing to call AI.');
|
|
// Wrap the AI call with the rate limiter.
|
|
const result = await this.rateLimiter(() =>
|
|
this.aiClient.generateContent({
|
|
contents: [{ parts: [{ text: prompt }, imagePart] }],
|
|
}),
|
|
);
|
|
logger.debug(
|
|
'[extractItemsFromReceiptImage] POST-RATE-LIMITER: AI call successful, parsing response.',
|
|
);
|
|
|
|
// The response from the SDK is structured, we need to access the text part.
|
|
const text = result.text;
|
|
logger.debug(
|
|
{ rawText: text?.substring(0, 100) },
|
|
'[extractItemsFromReceiptImage] Raw text from AI.',
|
|
);
|
|
const parsedJson = this._parseJsonFromAiResponse<
|
|
{ raw_item_description: string; price_paid_cents: number }[]
|
|
>(text, logger);
|
|
|
|
if (!parsedJson) {
|
|
logger.error(
|
|
{ responseText: text },
|
|
'[extractItemsFromReceiptImage] Failed to parse valid JSON from response.',
|
|
);
|
|
throw new Error('AI response did not contain a valid JSON array.');
|
|
}
|
|
|
|
logger.info('[extractItemsFromReceiptImage] Successfully extracted items. Exiting method.');
|
|
return parsedJson;
|
|
} catch (apiError) {
|
|
logger.error(
|
|
{ err: apiError },
|
|
'[extractItemsFromReceiptImage] An error occurred during the process.',
|
|
);
|
|
throw apiError;
|
|
}
|
|
}
|
|
|
|
async extractCoreDataFromFlyerImage(
|
|
imagePaths: { path: string; mimetype: string }[],
|
|
masterItems: MasterGroceryItem[],
|
|
submitterIp?: string,
|
|
userProfileAddress?: string,
|
|
logger: Logger = this.logger,
|
|
): Promise<{
|
|
store_name: string | null;
|
|
valid_from: string | null;
|
|
valid_to: string | null;
|
|
store_address: string | null;
|
|
items: ExtractedFlyerItem[];
|
|
}> {
|
|
logger.info(
|
|
`[extractCoreDataFromFlyerImage] Entering method with ${imagePaths.length} image(s).`,
|
|
);
|
|
const prompt = this._buildFlyerExtractionPrompt(masterItems, submitterIp, userProfileAddress);
|
|
|
|
const imageParts = await Promise.all(
|
|
imagePaths.map((file) => this.serverFileToGenerativePart(file.path, file.mimetype)),
|
|
);
|
|
|
|
const totalImageSize = imageParts.reduce((acc, part) => acc + part.inlineData.data.length, 0);
|
|
logger.info(
|
|
`[aiService.server] Total base64 image data size for Gemini: ${(totalImageSize / (1024 * 1024)).toFixed(2)} MB`,
|
|
);
|
|
|
|
try {
|
|
logger.debug(
|
|
`[extractCoreDataFromFlyerImage] PRE-RATE-LIMITER: Preparing to call Gemini API.`,
|
|
);
|
|
const geminiCallStartTime = process.hrtime.bigint();
|
|
|
|
// Wrap the AI call with the rate limiter.
|
|
const result = await this.rateLimiter(() => {
|
|
logger.debug(
|
|
'[extractCoreDataFromFlyerImage] INSIDE-RATE-LIMITER: Executing generateContent call.',
|
|
);
|
|
return this.aiClient.generateContent({
|
|
contents: [{ parts: [{ text: prompt }, ...imageParts] }],
|
|
});
|
|
});
|
|
logger.debug('[extractCoreDataFromFlyerImage] POST-RATE-LIMITER: AI call completed.');
|
|
|
|
const geminiCallEndTime = process.hrtime.bigint();
|
|
const durationMs = Number(geminiCallEndTime - geminiCallStartTime) / 1_000_000;
|
|
logger.info(
|
|
`[aiService.server] Gemini API call for flyer processing completed in ${durationMs.toFixed(2)} ms.`,
|
|
);
|
|
|
|
const text = result.text;
|
|
logger.debug(
|
|
`[aiService.server] Raw Gemini response text (first 500 chars): ${text?.substring(0, 500)}`,
|
|
);
|
|
|
|
const extractedData = this._parseJsonFromAiResponse<z.infer<typeof AiFlyerDataSchema>>(
|
|
text,
|
|
logger,
|
|
);
|
|
|
|
if (!extractedData) {
|
|
logger.error(
|
|
{ responseText: text },
|
|
'[extractCoreDataFromFlyerImage] AI response did not contain a valid JSON object after parsing.',
|
|
);
|
|
throw new Error('AI response did not contain a valid JSON object.');
|
|
}
|
|
|
|
// Normalize the items to create a clean data structure.
|
|
logger.debug('[extractCoreDataFromFlyerImage] Normalizing extracted items.');
|
|
const normalizedItems = Array.isArray(extractedData.items)
|
|
? this._normalizeExtractedItems(extractedData.items)
|
|
: [];
|
|
|
|
logger.info(
|
|
`[extractCoreDataFromFlyerImage] Successfully processed flyer data for store: ${extractedData.store_name}. Exiting method.`,
|
|
);
|
|
return { ...extractedData, items: normalizedItems };
|
|
} catch (apiError) {
|
|
logger.error({ err: apiError }, '[extractCoreDataFromFlyerImage] The entire process failed.');
|
|
throw apiError;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Normalizes the raw items returned by the AI, ensuring fields are in the correct format.
|
|
* @param items An array of raw flyer items from the AI.
|
|
* @returns A normalized array of flyer items.
|
|
*/
|
|
private _normalizeExtractedItems(items: RawFlyerItem[]): ExtractedFlyerItem[] {
|
|
return items.map((item: RawFlyerItem) => ({
|
|
...item,
|
|
// Ensure 'item' is always a string, defaulting to 'Unknown Item' if null/undefined.
|
|
item:
|
|
item.item === null || item.item === undefined || String(item.item).trim() === ''
|
|
? 'Unknown Item'
|
|
: String(item.item),
|
|
price_display:
|
|
item.price_display === null || item.price_display === undefined
|
|
? ''
|
|
: String(item.price_display),
|
|
quantity: item.quantity === null || item.quantity === undefined ? '' : String(item.quantity),
|
|
category_name:
|
|
item.category_name === null || item.category_name === undefined
|
|
? 'Other/Miscellaneous'
|
|
: String(item.category_name),
|
|
// Ensure undefined is converted to null to match the Zod schema.
|
|
price_in_cents: item.price_in_cents ?? null,
|
|
master_item_id: item.master_item_id ?? undefined,
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* SERVER-SIDE FUNCTION
|
|
* Extracts a specific piece of text from a cropped area of an image.
|
|
* @param imagePath The path to the original image file on the server.
|
|
* @param cropArea The coordinates and dimensions { x, y, width, height } to crop.
|
|
* @param extractionType The type of data to extract, which determines the AI prompt.
|
|
* @returns A promise that resolves to the extracted text.
|
|
*/
|
|
async extractTextFromImageArea(
|
|
imagePath: string,
|
|
imageMimeType: string,
|
|
cropArea: { x: number; y: number; width: number; height: number },
|
|
extractionType: 'store_name' | 'dates' | 'item_details',
|
|
logger: Logger = this.logger,
|
|
): Promise<{ text: string | undefined }> {
|
|
logger.info(
|
|
`[extractTextFromImageArea] Entering method for extraction type: ${extractionType}.`,
|
|
);
|
|
// 1. Define prompts based on the extraction type
|
|
const prompts = {
|
|
store_name: 'What is the store name in this image? Respond with only the name.',
|
|
dates:
|
|
'What are the sale dates in this image? Respond with the date range as text (e.g., "Jan 1 - Jan 7").',
|
|
item_details:
|
|
'Extract the item name, price, and quantity from this image. Respond with the text as seen.',
|
|
};
|
|
|
|
const prompt = prompts[extractionType] || 'Extract the text from this image.';
|
|
|
|
// 2. Crop the image using sharp
|
|
logger.debug('[extractTextFromImageArea] Cropping image with sharp.');
|
|
const sharp = (await import('sharp')).default;
|
|
const croppedImageBuffer = await sharp(imagePath)
|
|
.extract({
|
|
left: Math.round(cropArea.x),
|
|
top: Math.round(cropArea.y),
|
|
width: Math.round(cropArea.width),
|
|
height: Math.round(cropArea.height),
|
|
})
|
|
.toBuffer();
|
|
|
|
// 3. Convert cropped buffer to GenerativePart
|
|
const imagePart = {
|
|
inlineData: {
|
|
data: croppedImageBuffer.toString('base64'),
|
|
mimeType: imageMimeType,
|
|
},
|
|
};
|
|
|
|
// 4. Call the AI model
|
|
try {
|
|
logger.debug(`[extractTextFromImageArea] PRE-RATE-LIMITER: Preparing to call AI.`);
|
|
// Wrap the AI call with the rate limiter.
|
|
const result = await this.rateLimiter(() => {
|
|
logger.debug(`[extractTextFromImageArea] INSIDE-RATE-LIMITER: Executing generateContent.`);
|
|
return this.aiClient.generateContent({
|
|
contents: [{ parts: [{ text: prompt }, imagePart] }],
|
|
});
|
|
});
|
|
logger.debug('[extractTextFromImageArea] POST-RATE-LIMITER: AI call completed.');
|
|
|
|
const text = result.text?.trim();
|
|
logger.info(
|
|
`[extractTextFromImageArea] Gemini rescan completed. Extracted text: "${text}". Exiting method.`,
|
|
);
|
|
return { text };
|
|
} catch (apiError) {
|
|
logger.error(
|
|
{ err: apiError },
|
|
`[extractTextFromImageArea] An error occurred for type ${extractionType}.`,
|
|
);
|
|
throw apiError;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generates a simple recipe suggestion based on a list of ingredients.
|
|
* Uses the 'lite' models for faster/cheaper generation.
|
|
* @param ingredients List of available ingredients.
|
|
* @param logger Logger instance.
|
|
* @returns The recipe suggestion text.
|
|
*/
|
|
async generateRecipeSuggestion(
|
|
ingredients: string[],
|
|
logger: Logger = this.logger,
|
|
): Promise<string | null> {
|
|
const prompt = `Suggest a simple recipe using these ingredients: ${ingredients.join(', ')}. Keep it brief.`;
|
|
|
|
try {
|
|
const result = await this.rateLimiter(() =>
|
|
this.aiClient.generateContent({
|
|
contents: [{ parts: [{ text: prompt }] }],
|
|
useLiteModels: true,
|
|
}),
|
|
);
|
|
return result.text || null;
|
|
} catch (error) {
|
|
logger.error({ err: error }, 'Failed to generate recipe suggestion');
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* SERVER-SIDE FUNCTION
|
|
* Uses Google Maps grounding to find nearby stores and plan a shopping trip.
|
|
* @param items The items from the flyer.
|
|
* @param store The store associated with the flyer.
|
|
* @param userLocation The user's current geographic coordinates.
|
|
* @returns A text response with trip planning advice and a list of map sources.
|
|
*/
|
|
async planTripWithMaps(
|
|
items: FlyerItem[],
|
|
store: { name: string } | undefined,
|
|
userLocation: GeolocationCoordinates,
|
|
logger: Logger = this.logger,
|
|
): Promise<{ text: string; sources: { uri: string; title: string }[] }> {
|
|
// Return a 501 Not Implemented error as this feature is disabled.
|
|
logger.warn('[AIService] planTripWithMaps called, but feature is disabled. Throwing error.');
|
|
throw new Error("The 'planTripWithMaps' feature is currently disabled due to API costs.");
|
|
|
|
/* const topItems = items.slice(0, 5).map(i => i.item).join(', ');
|
|
const storeName = store?.name || 'the grocery store';
|
|
|
|
try {
|
|
// Wrap the AI call with the rate limiter.
|
|
const result = await this.rateLimiter(() => this.aiClient.generateContent({
|
|
contents: [{ parts: [{ text: `My current location is latitude ${userLocation.latitude}, longitude ${userLocation.longitude}.
|
|
I have a shopping list with items like ${topItems}. Find the nearest ${storeName} to me and suggest the best route.
|
|
Also, are there any other specialty stores nearby (like a bakery or butcher) that might have good deals on related items?`}]}],
|
|
tools: [{ "googleSearch": {} }],
|
|
}));
|
|
|
|
// In a real implementation, you would render the map URLs from the sources.
|
|
// The new SDK provides the search queries used, not a direct list of web attributions.
|
|
// We will transform these queries into searchable links to fulfill the contract of the function.
|
|
const searchQueries = result.candidates?.[0]?.groundingMetadata?.webSearchQueries || [];
|
|
const sources = searchQueries.map((query: string) => ({
|
|
uri: `https://www.google.com/search?q=${encodeURIComponent(query)}`,
|
|
title: query
|
|
}));
|
|
|
|
return { text: result.text ?? '', sources };
|
|
} catch (apiError) {
|
|
logger.error({ err: apiError }, "Google GenAI API call failed in planTripWithMaps");
|
|
throw apiError;
|
|
}
|
|
*/
|
|
}
|
|
|
|
|
|
async enqueueFlyerProcessing(
|
|
file: Express.Multer.File,
|
|
checksum: string,
|
|
userProfile: UserProfile | undefined,
|
|
submitterIp: string,
|
|
logger: Logger,
|
|
): Promise<Job> {
|
|
// 1. Check for duplicate flyer
|
|
const existingFlyer = await db.flyerRepo.findFlyerByChecksum(checksum, logger);
|
|
if (existingFlyer) {
|
|
// Throw a specific error for the route to handle
|
|
throw new DuplicateFlyerError(
|
|
'This flyer has already been processed.',
|
|
existingFlyer.flyer_id,
|
|
);
|
|
}
|
|
|
|
// 2. Construct user address string
|
|
let userProfileAddress: string | undefined = undefined;
|
|
if (userProfile?.address) {
|
|
userProfileAddress = [
|
|
userProfile.address.address_line_1,
|
|
userProfile.address.address_line_2,
|
|
userProfile.address.city,
|
|
userProfile.address.province_state,
|
|
userProfile.address.postal_code,
|
|
userProfile.address.country,
|
|
]
|
|
.filter(Boolean)
|
|
.join(', ');
|
|
}
|
|
|
|
// 3. Add job to the queue
|
|
const job = await flyerQueue.add('process-flyer', {
|
|
filePath: file.path,
|
|
originalFileName: file.originalname,
|
|
checksum: checksum,
|
|
userId: userProfile?.user.user_id,
|
|
submitterIp: submitterIp,
|
|
userProfileAddress: userProfileAddress,
|
|
});
|
|
|
|
logger.info(
|
|
`Enqueued flyer for processing. File: ${file.originalname}, Job ID: ${job.id}`,
|
|
);
|
|
|
|
return job;
|
|
}
|
|
|
|
private _parseLegacyPayload(
|
|
body: any,
|
|
logger: Logger,
|
|
): { parsed: FlyerProcessPayload; extractedData: Partial<ExtractedCoreData> | null | undefined } {
|
|
let parsed: FlyerProcessPayload = {};
|
|
|
|
try {
|
|
parsed = typeof body === 'string' ? JSON.parse(body) : body || {};
|
|
} catch (e) {
|
|
logger.warn({ error: errMsg(e) }, '[AIService] Failed to parse top-level request body string.');
|
|
return { parsed: {}, extractedData: {} };
|
|
}
|
|
|
|
// If the real payload is nested inside a 'data' property (which could be a string),
|
|
// we parse it out but keep the original `parsed` object for top-level properties like checksum.
|
|
let potentialPayload: FlyerProcessPayload = parsed;
|
|
if (parsed.data) {
|
|
if (typeof parsed.data === 'string') {
|
|
try {
|
|
potentialPayload = JSON.parse(parsed.data);
|
|
} catch (e) {
|
|
logger.warn({ error: errMsg(e) }, '[AIService] Failed to parse nested "data" property string.');
|
|
}
|
|
} else if (typeof parsed.data === 'object') {
|
|
potentialPayload = parsed.data;
|
|
}
|
|
}
|
|
|
|
// The extracted data is either in an `extractedData` key or is the payload itself.
|
|
const extractedData = potentialPayload.extractedData ?? potentialPayload;
|
|
|
|
// Merge for checksum lookup: properties in the outer `parsed` object (like a top-level checksum)
|
|
// take precedence over any same-named properties inside `potentialPayload`.
|
|
const finalParsed = { ...potentialPayload, ...parsed };
|
|
|
|
return { parsed: finalParsed, extractedData };
|
|
}
|
|
|
|
async processLegacyFlyerUpload(
|
|
file: Express.Multer.File,
|
|
body: any,
|
|
userProfile: UserProfile | undefined,
|
|
logger: Logger,
|
|
): Promise<Flyer> {
|
|
const { parsed, extractedData: initialExtractedData } = this._parseLegacyPayload(body, logger);
|
|
let extractedData = initialExtractedData;
|
|
|
|
const checksum = parsed.checksum ?? parsed?.data?.checksum ?? '';
|
|
if (!checksum) {
|
|
throw new ValidationError([], 'Checksum is required.');
|
|
}
|
|
|
|
const existingFlyer = await db.flyerRepo.findFlyerByChecksum(checksum, logger);
|
|
if (existingFlyer) {
|
|
throw new DuplicateFlyerError('This flyer has already been processed.', existingFlyer.flyer_id);
|
|
}
|
|
|
|
const originalFileName = parsed.originalFileName ?? parsed?.data?.originalFileName ?? file.originalname;
|
|
|
|
if (!extractedData || typeof extractedData !== 'object') {
|
|
logger.warn({ bodyData: parsed }, 'Missing extractedData in legacy payload.');
|
|
extractedData = {};
|
|
}
|
|
|
|
const rawItems = extractedData.items ?? [];
|
|
const itemsArray = Array.isArray(rawItems) ? rawItems : typeof rawItems === 'string' ? JSON.parse(rawItems) : [];
|
|
const itemsForDb = itemsArray.map((item: Partial<ExtractedFlyerItem>) => ({
|
|
...item,
|
|
master_item_id: item.master_item_id === null ? undefined : item.master_item_id,
|
|
quantity: item.quantity ?? 1,
|
|
view_count: 0,
|
|
click_count: 0,
|
|
updated_at: new Date().toISOString(),
|
|
}));
|
|
|
|
const storeName = extractedData.store_name && String(extractedData.store_name).trim().length > 0 ? String(extractedData.store_name) : 'Unknown Store (auto)';
|
|
if (storeName.startsWith('Unknown')) {
|
|
logger.warn('extractedData.store_name missing; using fallback store name.');
|
|
}
|
|
|
|
const iconsDir = path.join(path.dirname(file.path), 'icons');
|
|
const iconFileName = await generateFlyerIcon(file.path, iconsDir, logger);
|
|
|
|
// Construct proper URLs including protocol and host to satisfy DB constraints.
|
|
let baseUrl = (process.env.FRONTEND_URL || process.env.BASE_URL || '').trim();
|
|
if (!baseUrl || !baseUrl.startsWith('http')) {
|
|
const port = process.env.PORT || 3000;
|
|
const fallbackUrl = `http://localhost:${port}`;
|
|
if (baseUrl) {
|
|
logger.warn(
|
|
`FRONTEND_URL/BASE_URL is invalid or incomplete ('${baseUrl}'). Falling back to default local URL: ${fallbackUrl}`,
|
|
);
|
|
}
|
|
baseUrl = fallbackUrl;
|
|
}
|
|
baseUrl = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl;
|
|
|
|
const iconUrl = `${baseUrl}/flyer-images/icons/${iconFileName}`;
|
|
const imageUrl = `${baseUrl}/flyer-images/${file.filename}`;
|
|
|
|
const flyerData: FlyerInsert = {
|
|
file_name: originalFileName,
|
|
image_url: imageUrl,
|
|
icon_url: iconUrl,
|
|
checksum: checksum,
|
|
store_name: storeName,
|
|
valid_from: extractedData.valid_from ?? null,
|
|
valid_to: extractedData.valid_to ?? null,
|
|
store_address: extractedData.store_address ?? null,
|
|
item_count: 0,
|
|
status: 'needs_review',
|
|
uploaded_by: userProfile?.user.user_id,
|
|
};
|
|
|
|
const { flyer: newFlyer, items: newItems } = await createFlyerAndItems(flyerData, itemsForDb, logger);
|
|
|
|
logger.info(`Successfully processed legacy flyer: ${newFlyer.file_name} (ID: ${newFlyer.flyer_id}) with ${newItems.length} items.`);
|
|
|
|
await db.adminRepo.logActivity({
|
|
userId: userProfile?.user.user_id,
|
|
action: 'flyer_processed',
|
|
displayText: `Processed a new flyer for ${flyerData.store_name}.`,
|
|
details: { flyerId: newFlyer.flyer_id, storeName: flyerData.store_name },
|
|
}, logger);
|
|
|
|
return newFlyer;
|
|
}
|
|
}
|
|
|
|
// Export a singleton instance of the service for use throughout the application.
|
|
import { logger } from './logger.server';
|
|
export const aiService = new AIService(logger);
|