torbo/flyer-crawler.projectium.com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

one lazy ai
All checks were successful
Deploy to Web Server flyer-crawler.projectium.com / deploy (push) Successful in 4m26s
Details

Browse Source
This commit is contained in:
Torben Sorensen
2025-12-02 21:00:24 -08:00
parent 62c30420fa
commit 67d1b9b077
5 changed files with 80 additions and 419 deletions
Download Patch File Download Diff File Expand all files Collapse all files

314
src/App.tsx
View File

@@ -17,13 +17,9 @@ import { PriceHistoryChart } from './features/charts/PriceHistoryChart';
import * as apiClient from './services/apiClient';
import { FlyerList } from './features/flyer/FlyerList';
import { recordProcessingTime, getAverageProcessingTime } from './utils/processingTimer';
import { ProcessingStatus } from './features/flyer/ProcessingStatus';
import { generateFileChecksum } from './utils/checksum';
import { convertPdfToImageFiles } from './utils/pdfConverter';
import { BulkImportSummary } from './features/flyer/BulkImportSummary';
import { withTimeout } from './utils/timeout';
import { ProfileManager } from './pages/admin/components/ProfileManager';
import { ShoppingListComponent } from './features/shopping/ShoppingList';
import { FlyerUploader } from './features/flyer/FlyerUploader';
import { VoiceAssistant } from './features/voice-assistant/VoiceAssistant';
import { AdminPage } from './pages/admin/AdminPage';
import { AdminRoute } from './components/AdminRoute';
@@ -60,30 +56,34 @@ function App() {
const [user, setUser] = useState<User | null>(null); // Moved user state to the top
// --- Data Fetching ---
const [flyers, setFlyers] = useState<Flyer[]>([]);
const [masterItems, setMasterItems] = useState<MasterGroceryItem[]>([]);
const [watchedItems, setWatchedItems] = useState<MasterGroceryItem[]>([]);
const [shoppingLists, setShoppingLists] = useState<ShoppingList[]>([]); // This was a duplicate, fixed.
const [flyers, setFlyers] = useState<Flyer[]>([]);
const [masterItems, setMasterItems] = useState<MasterGroceryItem[]>([]);
const [watchedItems, setWatchedItems] = useState<MasterGroceryItem[]>([]);
const [shoppingLists, setShoppingLists] = useState<ShoppingList[]>([]); // This was a duplicate, fixed.
useEffect(() => {
const fetchData = async () => {
try {
const [flyersRes, masterItemsRes, watchedItemsRes, shoppingListsRes] = await Promise.all([
apiClient.fetchFlyers(),
apiClient.fetchMasterItems(),
user ? apiClient.fetchWatchedItems() : Promise.resolve(new Response(JSON.stringify([]))),
user ? apiClient.fetchShoppingLists() : Promise.resolve(new Response(JSON.stringify([]))),
]);
setFlyers(await flyersRes.json());
setMasterItems(await masterItemsRes.json());
setWatchedItems(await watchedItemsRes.json());
setShoppingLists(await shoppingListsRes.json());
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
}
};
fetchData();
}, [user]);
const refetchFlyers = useCallback(async () => {
try {
const flyersRes = await apiClient.fetchFlyers();
setFlyers(await flyersRes.json());
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
}
}, []);
useEffect(() => {
const fetchData = async () => {
const [masterItemsRes, watchedItemsRes, shoppingListsRes] = await Promise.all([
apiClient.fetchMasterItems(),
user ? apiClient.fetchWatchedItems() : Promise.resolve(new Response(JSON.stringify([]))),
user ? apiClient.fetchShoppingLists() : Promise.resolve(new Response(JSON.stringify([]))),
]);
setMasterItems(await masterItemsRes.json());
setWatchedItems(await watchedItemsRes.json());
setShoppingLists(await shoppingListsRes.json());
};
refetchFlyers(); // Initial fetch
fetchData();
}, [user, refetchFlyers]);
const [selectedFlyer, setSelectedFlyer] = useState<Flyer | null>(null);
const [flyerItems, setFlyerItems] = useState<FlyerItem[]>([]);
@@ -94,14 +94,8 @@ function App() {
const [activeDeals, setActiveDeals] = useState<DealItem[]>([]);
const [activeDealsLoading, setActiveDealsLoading] = useState(false);
const [totalActiveItems, setTotalActiveItems] = useState(0);
const [isProcessing, setIsProcessing] = useState(false);
const [error, setError] = useState<string | null>(null);
const [processingProgress, setProcessingProgress] = useState(0);
const [currentFile, setCurrentFile] = useState<string | null>(null);
const [fileCount, setFileCount] = useState<{current: number, total: number} | null>(null);
const [importSummary, setImportSummary] = useState<{
processed: string[];
skipped: string[];
@@ -132,9 +126,7 @@ function App() {
setSelectedFlyer(updatedFlyer);
};
const [processingStages, setProcessingStages] = useState<ProcessingStage[]>([]);
const [estimatedTime, setEstimatedTime] = useState(0);
const [pageProgress, setPageProgress] = useState<{current: number, total: number} | null>(null);
const [activeListId, setActiveListId] = useState<number | null>(null);
@@ -285,13 +277,6 @@ function App() {
const resetState = useCallback(() => {
setSelectedFlyer(null);
setFlyerItems([]);
setError(null);
setProcessingProgress(0);
setProcessingStages([]);
setImportSummary(null);
setCurrentFile(null);
setPageProgress(null);
setFileCount(null);
}, []);
const handleFlyerSelect = useCallback(async (flyer: Flyer) => {
@@ -310,10 +295,10 @@ function App() {
}, []);
useEffect(() => {
if (!isProcessing && !selectedFlyer && flyers.length > 0) {
if (!selectedFlyer && flyers.length > 0) {
handleFlyerSelect(flyers[0]);
}
}, [flyers, selectedFlyer, handleFlyerSelect, isProcessing]);
}, [flyers, selectedFlyer, handleFlyerSelect]);
useEffect(() => {
const findActiveDeals = async () => {
@@ -420,225 +405,6 @@ function App() {
calculateTotalActiveItems();
}, [flyers]);
const processAndUploadFlyer = async (files: File[], checksum: string, originalFileName: string, updateStage?: (index: number, updates: Partial<ProcessingStage>) => void) => {
let stageIndex = 0;
// Stage: Validating Flyer
updateStage?.(stageIndex, { status: 'in-progress' });
const isFlyerResponse = await withTimeout(aiApiClient.isImageAFlyer(files[0]), 15000);
const { is_flyer: isFlyer } = await isFlyerResponse.json();
if (!isFlyer) {
throw new Error("The uploaded image does not appear to be a grocery flyer.");
}
updateStage?.(stageIndex++, { status: 'completed' }); // stageIndex is now 1
const pageCount = files.length;
const coreDataTimeout = 60000 * pageCount;
const nonCriticalTimeout = 30000;
// Granular stages for core data extraction
const storeInfoStageIndex = stageIndex; // Stage 1: Extracting Store Name & Sale Dates
const itemExtractionStageIndex = stageIndex + 1; // Stage 2: Extracting All Items from Flyer
// Mark both stages as in-progress for the single AI call
updateStage?.(storeInfoStageIndex, { status: 'in-progress' });
updateStage?.(itemExtractionStageIndex, { status: 'in-progress', detail: pageCount > 1 ? `(${pageCount} pages)` : undefined });
let progressInterval: number | undefined;
let extractedData;
try {
if (pageCount > 1) {
let currentPage = 0;
const intervalTime = 2500;
// Attach progress bar to the item extraction stage
progressInterval = window.setInterval(() => { // This was a duplicate test, fixed.
currentPage++;
if (currentPage <= pageCount) {
updateStage?.(itemExtractionStageIndex, { progress: { current: currentPage, total: pageCount } });
} else {
clearInterval(progressInterval); // This was a duplicate test, fixed.
}
}, intervalTime);
}
extractedData = await withTimeout(aiApiClient.extractCoreDataFromImage(files, masterItems), coreDataTimeout);
await extractedData.clone().json(); // Parse the Response object to ensure it's valid JSON, but we don't need the result here.
// Mark both stages as completed after the AI call finishes // This was a duplicate test, fixed.
updateStage?.(storeInfoStageIndex, { status: 'completed' });
updateStage?.(itemExtractionStageIndex, { status: 'completed', progress: null });
} finally {
if (progressInterval) {
clearInterval(progressInterval);
}
}
// If extractedData is null or undefined at this point, it means the AI call failed.
if (!extractedData) {
throw new Error("Core data extraction failed. The AI did not return valid data.");
}
const { store_name, valid_from, valid_to, items: extractedItems } = await extractedData.json();
stageIndex += 2; // Increment by 2 for the stages we just completed. stageIndex is now 3
// Stage: Extracting Store Address
let storeAddress: string | null = null;
try {
updateStage?.(stageIndex, { status: 'in-progress' });
const addressResponse = await withTimeout(aiApiClient.extractAddressFromImage(files[0]), nonCriticalTimeout);
storeAddress = (await addressResponse.json()).address;
updateStage?.(stageIndex++, { status: 'completed' }); // stageIndex is now 4
} catch (e) {
const errorMessage = e instanceof Error ? e.message : String(e);
logger.warn("Non-critical step failed: Address extraction.", { error: errorMessage });
updateStage?.(stageIndex++, { status: 'error', detail: '(Skipped)' }); // stageIndex is now 4
}
// Stage: Extracting Store Logo
let storeLogoBase64: string | null = null;
try {
updateStage?.(stageIndex, { status: 'in-progress' });
const logoData = await withTimeout(aiApiClient.extractLogoFromImage(files.slice(0, 1)), nonCriticalTimeout);
storeLogoBase64 = (await logoData.json()).store_logo_base_64;
updateStage?.(stageIndex++, { status: 'completed' }); // stageIndex is now 5
} catch (e) {
const errorMessage = e instanceof Error ? e.message : String(e);
logger.warn("Non-critical step failed: Logo extraction.", { error: errorMessage });
updateStage?.(stageIndex++, { status: 'error', detail: '(Skipped)' }); // stageIndex is now 5
}
// Stage: Uploading and Saving Data to Backend
updateStage?.(stageIndex, { status: 'in-progress' });
const backendRawResponse = await apiClient.processFlyerFile(files[0], checksum, originalFileName, { store_name, valid_from, valid_to, items: extractedItems, store_address: storeAddress });
const backendResponse = await backendRawResponse.json();
updateStage?.(stageIndex, { status: 'completed', detail: backendResponse.message });
// Fire-and-forget logo upload if a logo was extracted and the store doesn't already have one.
if (storeLogoBase64 && backendResponse.flyer.store_id && !backendResponse.flyer.store?.logo_url) {
const logoFile = await (await fetch(storeLogoBase64)).blob();
apiClient.uploadLogoAndUpdateStore(backendResponse.flyer.store_id, new File([logoFile], 'logo.png', { type: 'image/png' })) // This returns a Response
.catch(e => logger.warn("Non-critical error: Failed to upload store logo.", { error: e }));
}
return { newFlyer: backendResponse.flyer, items: [] }; // Items are now saved on backend, return empty array
};
const setupProcessingStages = (isPdf: boolean) => {
const pendingStatus: StageStatus = 'pending';
const baseStages: ProcessingStage[] = [
{ name: 'Validating Flyer', status: pendingStatus, critical: true },
{ name: 'Extracting Store Name & Sale Dates', status: pendingStatus, critical: true },
{ name: 'Extracting All Items from Flyer', status: pendingStatus, critical: true },
{ name: 'Extracting Store Address', status: pendingStatus, critical: false },
{ name: 'Extracting Store Logo', status: pendingStatus, critical: false },
{ name: 'Uploading and Saving to Database', status: pendingStatus, critical: true },
];
if (isPdf) {
return [
{ name: 'Analyzing PDF', status: pendingStatus, critical: true },
{ name: 'Converting PDF to Images', status: pendingStatus, critical: true },
...baseStages
];
}
return baseStages;
};
const handleProcessFiles = useCallback(async (files: FileList) => {
if (files.length === 0) return;
// If a user is not logged in, transition them to an 'ANONYMOUS' state
// upon their first major interaction (uploading a flyer). This allows
// the app to handle session-based data without requiring a full login.
// This state can be used to prompt the user to save their work by creating
// an account later.
if (authStatus === 'SIGNED_OUT') {
setAuthStatus('ANONYMOUS');
logger.info("User is not signed in. Transitioning to ANONYMOUS session for this upload.");
}
resetState();
setIsProcessing(true);
setProcessingProgress(0);
const summary = {
processed: [] as string[],
skipped: [] as string[],
errors: [] as { fileName: string; message: string }[],
};
const avgTime = getAverageProcessingTime();
setEstimatedTime(avgTime * files.length);
for (let i = 0; i < files.length; i++) {
const originalFile = files[i];
setCurrentFile(originalFile.name);
setFileCount({ current: i + 1, total: files.length });
setPageProgress(null);
const isPdf = originalFile.type === 'application/pdf';
setProcessingStages(setupProcessingStages(isPdf));
const updateStage = (index: number, updates: Partial<ProcessingStage>) => {
setProcessingStages(prev =>
prev.map((stage, j) => (j === index ? { ...stage, ...updates } : stage))
);
};
let currentStageIndex = 0;
const startTime = Date.now();
try {
let filesToProcess: File[];
let checksum = '';
if (isPdf) {
updateStage(currentStageIndex, { status: 'in-progress' });
const onPdfProgress = (currentPage: number, totalPages: number) => {
setPageProgress({ current: currentPage, total: totalPages });
};
const { imageFiles, pageCount } = await convertPdfToImageFiles(originalFile, onPdfProgress);
filesToProcess = imageFiles;
setPageProgress(null);
updateStage(currentStageIndex++, { status: 'completed', detail: `(${pageCount} pages)` });
updateStage(currentStageIndex++, { status: 'completed' });
} else {
filesToProcess = [originalFile];
}
checksum = await generateFileChecksum(originalFile);
const processAndUploadUpdateStage = (idx: number, updates: Partial<ProcessingStage>) => updateStage(idx + currentStageIndex, updates);
await processAndUploadFlyer(filesToProcess, checksum, originalFile.name, processAndUploadUpdateStage);
summary.processed.push(originalFile.name);
} catch (e) {
const errorMessage = e instanceof Error ? e.message : String(e);
logger.error(`Failed to process ${originalFile.name}:`, { error: errorMessage });
summary.errors.push({ fileName: originalFile.name, message: errorMessage });
setProcessingStages(prev => prev.map(stage => {
if (stage.status === 'in-progress' && (stage.critical ?? true)) {
return {...stage, status: 'error'};
}
return stage;
}));
await new Promise(resolve => setTimeout(resolve, 2000));
} finally {
const duration = (Date.now() - startTime) / 1000;
recordProcessingTime(duration);
}
setProcessingProgress(((i + 1) / files.length) * 100);
}
// Data will be re-fetched automatically by the useApiOnMount hooks if we re-trigger them,
// but for now, we'll just update the local state. A full page refresh would also work.
setImportSummary(summary);
setIsProcessing(false);
setCurrentFile(null); // This was a duplicate, fixed.
setPageProgress(null);
setFileCount(null);
}, [resetState, masterItems, authStatus]); // Removed fetchFlyers, fetchMasterItems
const handleAddWatchedItem = useCallback(async (itemName: string, category: string) => {
if (!user) return;
try {
@@ -872,27 +638,13 @@ function App() {
<div className="lg:col-span-1 flex flex-col space-y-6">
<FlyerList flyers={flyers} onFlyerSelect={handleFlyerSelect} selectedFlyerId={selectedFlyer?.flyer_id || null} />
{(
<BulkImporter
onProcess={handleProcessFiles}
isProcessing={isProcessing}
/>
)}
<FlyerUploader onProcessingComplete={refetchFlyers} />
</div>
<div className="lg:col-span-2 flex flex-col space-y-6">
{error && <ErrorDisplay message={error} />} {/* This was a duplicate, fixed. */}
{isProcessing ? (
<ProcessingStatus
stages={processingStages}
estimatedTime={estimatedTime}
currentFile={currentFile}
pageProgress={pageProgress}
bulkProgress={processingProgress}
bulkFileCount={fileCount}
/>
) : selectedFlyer ? (
{selectedFlyer ? (
<>
<FlyerDisplay
imageUrl={selectedFlyer.image_url}
@@ -919,8 +671,6 @@ function App() {
</>
)}
</>
) : importSummary ? (
<BulkImportSummary summary={importSummary} onDismiss={() => setImportSummary(null)} /> // This was a duplicate, fixed.
) : (
<div className="text-center p-8 bg-white dark:bg-gray-800 rounded-lg border border-gray-200 dark:border-gray-700 h-full flex flex-col justify-center min-h-[400px]">
<h2 className="text-xl font-semibold text-gray-700 dark:text-gray-200">Welcome to Flyer Crawler!</h2>