ci: Bump version to 0.9.101 [skip ci]

.gitea/workflows/deploy-to-prod.yml

@@ -171,7 +171,7 @@ jobs:
             else
               echo "Version mismatch (Running: $RUNNING_VERSION -> Deployed: $NEW_VERSION) or app not running. Reloading PM2..."
             fi
-            pm2 startOrReload ecosystem.config.cjs --env production --update-env && pm2 save
+            pm2 startOrReload ecosystem.config.cjs --update-env && pm2 save
             echo "Production backend server reloaded successfully."
           else
             echo "Version $NEW_VERSION is already running. Skipping PM2 reload."

.gitea/workflows/deploy-to-test.yml

@@ -476,10 +476,11 @@ jobs:
           echo "Cleaning up errored or stopped PM2 processes..."
           node -e "const exec = require('child_process').execSync; try { const list = JSON.parse(exec('pm2 jlist').toString()); list.forEach(p => { if (p.pm2_env.status === 'errored' || p.pm2_env.status === 'stopped') { console.log('Deleting ' + p.pm2_env.status + ' process: ' + p.name + ' (' + p.pm2_env.pm_id + ')'); try { exec('pm2 delete ' + p.pm2_env.pm_id); } catch(e) { console.error('Failed to delete ' + p.pm2_env.pm_id); } } }); } catch (e) { console.error('Error cleaning up processes:', e); }"
 
-          # Use `startOrReload` with the ecosystem file. This is the standard, idempotent way to deploy.
-          # It will START the process if it's not running, or RELOAD it if it is.
+          # Use `startOrReload` with the TEST ecosystem file. This starts test-specific processes
+          # (flyer-crawler-api-test, flyer-crawler-worker-test, flyer-crawler-analytics-worker-test)
+          # that run separately from production processes.
           # We also add `&& pm2 save` to persist the process list across server reboots.
-          pm2 startOrReload ecosystem.config.cjs --env test --update-env && pm2 save
+          pm2 startOrReload ecosystem.config.test.cjs --update-env && pm2 save
           echo "Test backend server reloaded successfully."
 
           # After a successful deployment, update the schema hash in the database.

ecosystem.config.cjs

@@ -2,18 +2,28 @@
 // This file is the standard way to configure applications for PM2.
 // It allows us to define all the settings for our application in one place.
 // The .cjs extension is required because the project's package.json has "type": "module".
+//
+// IMPORTANT: This file defines SEPARATE apps for production and test environments.
+// Production apps: flyer-crawler-api, flyer-crawler-worker, flyer-crawler-analytics-worker
+// Test apps: flyer-crawler-api-test, flyer-crawler-worker-test, flyer-crawler-analytics-worker-test
+//
+// Use ecosystem.config.test.cjs for test deployments (contains only test apps).
+// Use this file (ecosystem.config.cjs) for production deployments.
 
 // --- Environment Variable Validation ---
+// NOTE: We only WARN about missing secrets, not exit.
+// Calling process.exit(1) prevents PM2 from reading the apps array.
+// The actual application will fail to start if secrets are missing,
+// which PM2 will handle with its restart logic.
 const requiredSecrets = ['DB_HOST', 'JWT_SECRET', 'GEMINI_API_KEY'];
 const missingSecrets = requiredSecrets.filter(key => !process.env[key]);
 
 if (missingSecrets.length > 0) {
-  console.warn('\n[ecosystem.config.cjs] ⚠️  WARNING: The following environment variables are MISSING in the shell:');
+  console.warn('\n[ecosystem.config.cjs] WARNING: The following environment variables are MISSING:');
   missingSecrets.forEach(key => console.warn(`   - ${key}`));
-  console.warn('[ecosystem.config.cjs] The application may crash if these are required for startup.\n');
-  process.exit(1); // Fail fast so PM2 doesn't attempt to start a broken app
+  console.warn('[ecosystem.config.cjs] The application may fail to start if these are required.\n');
 } else {
-  console.log('[ecosystem.config.cjs] ✅ Critical environment variables are present.');
+  console.log('[ecosystem.config.cjs] Critical environment variables are present.');
 }
 
 // --- Shared Environment Variables ---
@@ -35,125 +45,67 @@ const sharedEnv = {
   SMTP_USER: process.env.SMTP_USER,
   SMTP_PASS: process.env.SMTP_PASS,
   SMTP_FROM_EMAIL: process.env.SMTP_FROM_EMAIL,
+  SENTRY_DSN: process.env.SENTRY_DSN,
+  SENTRY_ENVIRONMENT: process.env.SENTRY_ENVIRONMENT,
+  SENTRY_ENABLED: process.env.SENTRY_ENABLED,
 };
 
 module.exports = {
   apps: [
+    // =========================================================================
+    // PRODUCTION APPS
+    // =========================================================================
     {
-      // --- API Server ---
+      // --- Production API Server ---
       name: 'flyer-crawler-api',
-      // Note: The process names below are referenced in .gitea/workflows/ for status checks.
       script: './node_modules/.bin/tsx',
       args: 'server.ts',
+      cwd: '/var/www/flyer-crawler.projectium.com',
       max_memory_restart: '500M',
-      // Production Optimization: Run in cluster mode to utilize all CPU cores
       instances: 'max',
       exec_mode: 'cluster',
-      kill_timeout: 5000, // Allow 5s for graceful shutdown of API requests
+      kill_timeout: 5000,
       log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
-      
-      // Restart Logic
       max_restarts: 40,
       exp_backoff_restart_delay: 100,
       min_uptime: '10s',
-
-      // Production Environment Settings
-      env_production: {
+      env: {
         NODE_ENV: 'production',
-        name: 'flyer-crawler-api',
-        cwd: '/var/www/flyer-crawler.projectium.com',
-        WORKER_LOCK_DURATION: '120000',
-        ...sharedEnv,
-      },
-      // Test Environment Settings
-      env_test: {
-        NODE_ENV: 'test',
-        name: 'flyer-crawler-api-test',
-        cwd: '/var/www/flyer-crawler-test.projectium.com',
-        WORKER_LOCK_DURATION: '120000',
-        ...sharedEnv,
-      },
-      // Development Environment Settings
-      env_development: {
-        NODE_ENV: 'development',
-        name: 'flyer-crawler-api-dev',
-        watch: true,
-        ignore_watch: ['node_modules', 'logs', '*.log', 'flyer-images', '.git'],
         WORKER_LOCK_DURATION: '120000',
         ...sharedEnv,
       },
     },
     {
-      // --- General Worker ---
+      // --- Production General Worker ---
       name: 'flyer-crawler-worker',
       script: './node_modules/.bin/tsx',
       args: 'src/services/worker.ts',
+      cwd: '/var/www/flyer-crawler.projectium.com',
       max_memory_restart: '1G',
-      kill_timeout: 10000, // Workers may need more time to complete a job
+      kill_timeout: 10000,
       log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
-      
-      // Restart Logic
       max_restarts: 40,
       exp_backoff_restart_delay: 100,
       min_uptime: '10s',
-
-      // Production Environment Settings
-      env_production: {
+      env: {
         NODE_ENV: 'production',
-        name: 'flyer-crawler-worker',
-        cwd: '/var/www/flyer-crawler.projectium.com',
-        ...sharedEnv,
-      },
-      // Test Environment Settings
-      env_test: {
-        NODE_ENV: 'test',
-        name: 'flyer-crawler-worker-test',
-        cwd: '/var/www/flyer-crawler-test.projectium.com',
-        ...sharedEnv,
-      },
-      // Development Environment Settings
-      env_development: {
-        NODE_ENV: 'development',
-        name: 'flyer-crawler-worker-dev',
-        watch: true,
-        ignore_watch: ['node_modules', 'logs', '*.log', 'flyer-images', '.git'],
         ...sharedEnv,
       },
     },
     {
-      // --- Analytics Worker ---
+      // --- Production Analytics Worker ---
       name: 'flyer-crawler-analytics-worker',
       script: './node_modules/.bin/tsx',
       args: 'src/services/worker.ts',
+      cwd: '/var/www/flyer-crawler.projectium.com',
       max_memory_restart: '1G',
       kill_timeout: 10000,
       log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
-      
-      // Restart Logic
       max_restarts: 40,
       exp_backoff_restart_delay: 100,
       min_uptime: '10s',
-
-      // Production Environment Settings
-      env_production: {
+      env: {
         NODE_ENV: 'production',
-        name: 'flyer-crawler-analytics-worker',
-        cwd: '/var/www/flyer-crawler.projectium.com',
-        ...sharedEnv,
-      },
-      // Test Environment Settings
-      env_test: {
-        NODE_ENV: 'test',
-        name: 'flyer-crawler-analytics-worker-test',
-        cwd: '/var/www/flyer-crawler-test.projectium.com',
-        ...sharedEnv,
-      },
-      // Development Environment Settings
-      env_development: {
-        NODE_ENV: 'development',
-        name: 'flyer-crawler-analytics-worker-dev',
-        watch: true,
-        ignore_watch: ['node_modules', 'logs', '*.log', 'flyer-images', '.git'],
         ...sharedEnv,
       },
     },

ecosystem.config.test.cjs

@@ -0,0 +1,113 @@
+// ecosystem.config.test.cjs
+// PM2 configuration for the TEST environment only.
+// This file defines test-specific apps that run alongside production apps.
+//
+// Test apps: flyer-crawler-api-test, flyer-crawler-worker-test, flyer-crawler-analytics-worker-test
+//
+// These apps:
+// - Run from /var/www/flyer-crawler-test.projectium.com
+// - Use NODE_ENV='test' (enables file logging in logger.server.ts)
+// - Use Redis database 1 (isolated from production which uses database 0)
+// - Have distinct PM2 process names to avoid conflicts with production
+
+// --- Environment Variable Validation ---
+// NOTE: We only WARN about missing secrets, not exit.
+// Calling process.exit(1) prevents PM2 from reading the apps array.
+// The actual application will fail to start if secrets are missing,
+// which PM2 will handle with its restart logic.
+const requiredSecrets = ['DB_HOST', 'JWT_SECRET', 'GEMINI_API_KEY'];
+const missingSecrets = requiredSecrets.filter(key => !process.env[key]);
+
+if (missingSecrets.length > 0) {
+  console.warn('\n[ecosystem.config.test.cjs] WARNING: The following environment variables are MISSING:');
+  missingSecrets.forEach(key => console.warn(`   - ${key}`));
+  console.warn('[ecosystem.config.test.cjs] The application may fail to start if these are required.\n');
+} else {
+  console.log('[ecosystem.config.test.cjs] Critical environment variables are present.');
+}
+
+// --- Shared Environment Variables ---
+const sharedEnv = {
+  DB_HOST: process.env.DB_HOST,
+  DB_USER: process.env.DB_USER,
+  DB_PASSWORD: process.env.DB_PASSWORD,
+  DB_NAME: process.env.DB_NAME,
+  REDIS_URL: process.env.REDIS_URL,
+  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
+  FRONTEND_URL: process.env.FRONTEND_URL,
+  JWT_SECRET: process.env.JWT_SECRET,
+  GEMINI_API_KEY: process.env.GEMINI_API_KEY,
+  GOOGLE_MAPS_API_KEY: process.env.GOOGLE_MAPS_API_KEY,
+  SMTP_HOST: process.env.SMTP_HOST,
+  SMTP_PORT: process.env.SMTP_PORT,
+  SMTP_SECURE: process.env.SMTP_SECURE,
+  SMTP_USER: process.env.SMTP_USER,
+  SMTP_PASS: process.env.SMTP_PASS,
+  SMTP_FROM_EMAIL: process.env.SMTP_FROM_EMAIL,
+  SENTRY_DSN: process.env.SENTRY_DSN,
+  SENTRY_ENVIRONMENT: process.env.SENTRY_ENVIRONMENT,
+  SENTRY_ENABLED: process.env.SENTRY_ENABLED,
+};
+
+module.exports = {
+  apps: [
+    // =========================================================================
+    // TEST APPS
+    // =========================================================================
+    {
+      // --- Test API Server ---
+      name: 'flyer-crawler-api-test',
+      script: './node_modules/.bin/tsx',
+      args: 'server.ts',
+      cwd: '/var/www/flyer-crawler-test.projectium.com',
+      max_memory_restart: '500M',
+      // Test environment: single instance (no cluster) to conserve resources
+      instances: 1,
+      exec_mode: 'fork',
+      kill_timeout: 5000,
+      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
+      max_restarts: 40,
+      exp_backoff_restart_delay: 100,
+      min_uptime: '10s',
+      env: {
+        NODE_ENV: 'test',
+        WORKER_LOCK_DURATION: '120000',
+        ...sharedEnv,
+      },
+    },
+    {
+      // --- Test General Worker ---
+      name: 'flyer-crawler-worker-test',
+      script: './node_modules/.bin/tsx',
+      args: 'src/services/worker.ts',
+      cwd: '/var/www/flyer-crawler-test.projectium.com',
+      max_memory_restart: '1G',
+      kill_timeout: 10000,
+      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
+      max_restarts: 40,
+      exp_backoff_restart_delay: 100,
+      min_uptime: '10s',
+      env: {
+        NODE_ENV: 'test',
+        ...sharedEnv,
+      },
+    },
+    {
+      // --- Test Analytics Worker ---
+      name: 'flyer-crawler-analytics-worker-test',
+      script: './node_modules/.bin/tsx',
+      args: 'src/services/worker.ts',
+      cwd: '/var/www/flyer-crawler-test.projectium.com',
+      max_memory_restart: '1G',
+      kill_timeout: 10000,
+      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
+      max_restarts: 40,
+      exp_backoff_restart_delay: 100,
+      min_uptime: '10s',
+      env: {
+        NODE_ENV: 'test',
+        ...sharedEnv,
+      },
+    },
+  ],
+};

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "flyer-crawler",
-  "version": "0.9.99",
+  "version": "0.9.101",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "flyer-crawler",
-      "version": "0.9.99",
+      "version": "0.9.101",
       "dependencies": {
         "@bull-board/api": "^6.14.2",
         "@bull-board/express": "^6.14.2",

package.json

@@ -1,7 +1,7 @@
 {
   "name": "flyer-crawler",
   "private": true,
-  "version": "0.9.99",
+  "version": "0.9.101",
   "type": "module",
   "scripts": {
     "dev": "concurrently \"npm:start:dev\" \"vite\"",

sql/initial_schema.sql

@@ -943,13 +943,21 @@ CREATE TABLE IF NOT EXISTS public.receipts (
     status TEXT DEFAULT 'pending' NOT NULL CHECK (status IN ('pending', 'processing', 'completed', 'failed')),
     raw_text TEXT,
     created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
-    processed_at TIMESTAMPTZ,    
-    updated_at TIMESTAMPTZ DEFAULT now() NOT NULL
+    processed_at TIMESTAMPTZ,
+    updated_at TIMESTAMPTZ DEFAULT now() NOT NULL,
+    -- Columns from migration 003_receipt_scanning_enhancements.sql
+    store_confidence NUMERIC(5,4) CHECK (store_confidence IS NULL OR (store_confidence >= 0 AND store_confidence <= 1)),
+    ocr_provider TEXT,
+    error_details JSONB,
+    retry_count INTEGER DEFAULT 0 CHECK (retry_count >= 0),
+    ocr_confidence NUMERIC(5,4) CHECK (ocr_confidence IS NULL OR (ocr_confidence >= 0 AND ocr_confidence <= 1)),
+    currency TEXT DEFAULT 'CAD'
 );
     -- CONSTRAINT receipts_receipt_image_url_check CHECK (receipt_image_url ~* '^https://?.*')
 COMMENT ON TABLE public.receipts IS 'Stores uploaded user receipts for purchase tracking and analysis.';
 CREATE INDEX IF NOT EXISTS idx_receipts_user_id ON public.receipts(user_id);
 CREATE INDEX IF NOT EXISTS idx_receipts_store_id ON public.receipts(store_id);
+CREATE INDEX IF NOT EXISTS idx_receipts_status_retry ON public.receipts(status, retry_count) WHERE status IN ('pending', 'failed') AND retry_count < 3;
 
 -- 53. Store individual line items extracted from a user receipt.
 CREATE TABLE IF NOT EXISTS public.receipt_items (

sql/master_schema_rollup.sql

@@ -962,13 +962,21 @@ CREATE TABLE IF NOT EXISTS public.receipts (
     status TEXT DEFAULT 'pending' NOT NULL CHECK (status IN ('pending', 'processing', 'completed', 'failed')),
     raw_text TEXT,
     created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
-    processed_at TIMESTAMPTZ,    
-    updated_at TIMESTAMPTZ DEFAULT now() NOT NULL
+    processed_at TIMESTAMPTZ,
+    updated_at TIMESTAMPTZ DEFAULT now() NOT NULL,
+    -- Columns from migration 003_receipt_scanning_enhancements.sql
+    store_confidence NUMERIC(5,4) CHECK (store_confidence IS NULL OR (store_confidence >= 0 AND store_confidence <= 1)),
+    ocr_provider TEXT,
+    error_details JSONB,
+    retry_count INTEGER DEFAULT 0 CHECK (retry_count >= 0),
+    ocr_confidence NUMERIC(5,4) CHECK (ocr_confidence IS NULL OR (ocr_confidence >= 0 AND ocr_confidence <= 1)),
+    currency TEXT DEFAULT 'CAD'
 );
     -- CONSTRAINT receipts_receipt_image_url_check CHECK (receipt_image_url ~* '^https?://.*'),
 COMMENT ON TABLE public.receipts IS 'Stores uploaded user receipts for purchase tracking and analysis.';
 CREATE INDEX IF NOT EXISTS idx_receipts_user_id ON public.receipts(user_id);
 CREATE INDEX IF NOT EXISTS idx_receipts_store_id ON public.receipts(store_id);
+CREATE INDEX IF NOT EXISTS idx_receipts_status_retry ON public.receipts(status, retry_count) WHERE status IN ('pending', 'failed') AND retry_count < 3;
 
 -- 53. Store individual line items extracted from a user receipt.
 CREATE TABLE IF NOT EXISTS public.receipt_items (

src/services/db/receipt.db.ts

@@ -28,7 +28,8 @@ interface ReceiptRow {
   raw_text: string | null;
   store_confidence: number | null;
   ocr_provider: OcrProvider | null;
-  error_details: string | null;
+  // JSONB columns are automatically parsed by pg driver
+  error_details: Record<string, unknown> | null;
   retry_count: number;
   ocr_confidence: number | null;
   currency: string;
@@ -1036,7 +1037,7 @@ export class ReceiptRepository {
       raw_text: row.raw_text,
       store_confidence: row.store_confidence !== null ? Number(row.store_confidence) : null,
       ocr_provider: row.ocr_provider,
-      error_details: row.error_details ? JSON.parse(row.error_details) : null,
+      error_details: row.error_details ?? null,
       retry_count: row.retry_count,
       ocr_confidence: row.ocr_confidence !== null ? Number(row.ocr_confidence) : null,
       currency: row.currency,

src/tests/integration/inventory.integration.test.ts

@@ -416,7 +416,14 @@ describe('Inventory/Expiry Integration Tests (/api/inventory)', () => {
         .send({ expiry_date: futureDate });
 
       expect(response.status).toBe(200);
-      expect(response.body.data.expiry_date).toContain(futureDate);
+      // Compare date portions only - the response is in UTC, which may differ by timezone offset
+      // e.g., '2026-02-27' sent becomes '2026-02-26T19:00:00.000Z' in UTC (for UTC-5 timezone)
+      const responseDate = new Date(response.body.data.expiry_date);
+      const sentDate = new Date(futureDate + 'T00:00:00');
+      // Dates should be within 24 hours of each other (same logical day)
+      expect(Math.abs(responseDate.getTime() - sentDate.getTime())).toBeLessThan(
+        24 * 60 * 60 * 1000,
+      );
     });
 
     it('should reject empty update body', async () => {

src/tests/integration/receipt.integration.test.ts

@@ -14,6 +14,14 @@ import { getPool } from '../../services/db/connection.db';
  * @vitest-environment node
  */
 
+// Mock Bull Board to prevent BullMQAdapter from validating queue instances
+vi.mock('@bull-board/api', () => ({
+  createBullBoard: vi.fn(),
+}));
+vi.mock('@bull-board/api/bullMQAdapter', () => ({
+  BullMQAdapter: vi.fn(),
+}));
+
 // Mock the queues to prevent actual background processing
 // IMPORTANT: Must include all queue exports that are imported by workers.server.ts
 vi.mock('../../services/queues.server', () => ({
@@ -88,7 +96,7 @@ describe('Receipt Processing Integration Tests (/api/receipts)', () => {
         createdReceiptIds,
       ]);
       await pool.query(
-        'DELETE FROM public.receipt_processing_logs WHERE receipt_id = ANY($1::int[])',
+        'DELETE FROM public.receipt_processing_log WHERE receipt_id = ANY($1::int[])',
         [createdReceiptIds],
       );
       await pool.query('DELETE FROM public.receipts WHERE receipt_id = ANY($1::int[])', [
@@ -337,8 +345,8 @@ describe('Receipt Processing Integration Tests (/api/receipts)', () => {
     beforeAll(async () => {
       const pool = getPool();
       const result = await pool.query(
-        `INSERT INTO public.receipts (user_id, receipt_image_url, status, error_message)
-         VALUES ($1, '/uploads/receipts/failed-test.jpg', 'failed', 'OCR failed')
+        `INSERT INTO public.receipts (user_id, receipt_image_url, status, error_details)
+         VALUES ($1, '/uploads/receipts/failed-test.jpg', 'failed', '{"message": "OCR failed"}'::jsonb)
          RETURNING receipt_id`,
         [testUser.user.user_id],
       );
@@ -551,12 +559,14 @@ describe('Receipt Processing Integration Tests (/api/receipts)', () => {
       receiptWithLogsId = receiptResult.rows[0].receipt_id;
       createdReceiptIds.push(receiptWithLogsId);
 
-      // Add processing logs
+      // Add processing logs - using correct table name and column names
+      // processing_step must be one of: upload, ocr_extraction, text_parsing, store_detection,
+      // item_extraction, item_matching, price_parsing, finalization
       await pool.query(
-        `INSERT INTO public.receipt_processing_logs (receipt_id, step, status, message)
-         VALUES ($1, 'ocr', 'completed', 'OCR completed successfully'),
+        `INSERT INTO public.receipt_processing_log (receipt_id, processing_step, status, error_message)
+         VALUES ($1, 'ocr_extraction', 'completed', 'OCR completed successfully'),
                 ($1, 'item_extraction', 'completed', 'Extracted 5 items'),
-                ($1, 'matching', 'completed', 'Matched 3 items')`,
+                ($1, 'item_matching', 'completed', 'Matched 3 items')`,
         [receiptWithLogsId],
       );
     });