ci: Bump version to 0.9.87 [skip ci]

.gitea/workflows/deploy-to-test.yml

@@ -198,8 +198,8 @@ jobs:
             --reporter=verbose --includeTaskLocation --testTimeout=10000 --silent=passed-only || true
 
           echo "--- Running E2E Tests ---"
-          # Run E2E tests using the dedicated E2E config which inherits from integration config.
-          # We still pass --coverage to enable it, but directory and timeout are now in the config.
+          # Run E2E tests using the dedicated E2E config.
+          # E2E uses port 3098, integration uses 3099 to avoid conflicts.
           npx vitest run --config vitest.config.e2e.ts --coverage \
             --coverage.exclude='**/*.test.ts' \
             --coverage.exclude='**/tests/**' \

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "flyer-crawler",
-  "version": "0.9.86",
+  "version": "0.9.87",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "flyer-crawler",
-      "version": "0.9.86",
+      "version": "0.9.87",
       "dependencies": {
         "@bull-board/api": "^6.14.2",
         "@bull-board/express": "^6.14.2",

package.json

@@ -1,7 +1,7 @@
 {
   "name": "flyer-crawler",
   "private": true,
-  "version": "0.9.86",
+  "version": "0.9.87",
   "type": "module",
   "scripts": {
     "dev": "concurrently \"npm:start:dev\" \"vite\"",

run-integration-tests.ps1

@@ -1,88 +0,0 @@
-# PowerShell script to run integration tests with containerized infrastructure
-# Sets up environment variables and runs the integration test suite
-
-Write-Host "=== Flyer Crawler Integration Test Runner ===" -ForegroundColor Cyan
-Write-Host ""
-
-# Check if containers are running
-Write-Host "Checking container status..." -ForegroundColor Yellow
-$postgresRunning = podman ps --filter "name=flyer-crawler-postgres" --format "{{.Names}}" 2>$null
-$redisRunning = podman ps --filter "name=flyer-crawler-redis" --format "{{.Names}}" 2>$null
-
-if (-not $postgresRunning) {
-    Write-Host "ERROR: PostgreSQL container is not running!" -ForegroundColor Red
-    Write-Host "Start it with: podman start flyer-crawler-postgres" -ForegroundColor Yellow
-    exit 1
-}
-
-if (-not $redisRunning) {
-    Write-Host "ERROR: Redis container is not running!" -ForegroundColor Red
-    Write-Host "Start it with: podman start flyer-crawler-redis" -ForegroundColor Yellow
-    exit 1
-}
-
-Write-Host "✓ PostgreSQL container: $postgresRunning" -ForegroundColor Green
-Write-Host "✓ Redis container: $redisRunning" -ForegroundColor Green
-Write-Host ""
-
-# Set environment variables for integration tests
-Write-Host "Setting environment variables..." -ForegroundColor Yellow
-
-$env:NODE_ENV = "test"
-$env:DB_HOST = "localhost"
-$env:DB_USER = "postgres"
-$env:DB_PASSWORD = "postgres"
-$env:DB_NAME = "flyer_crawler_dev"
-$env:DB_PORT = "5432"
-$env:REDIS_URL = "redis://localhost:6379"
-$env:REDIS_PASSWORD = ""
-$env:FRONTEND_URL = "http://localhost:5173"
-$env:VITE_API_BASE_URL = "http://localhost:3001/api"
-$env:JWT_SECRET = "test-jwt-secret-for-integration-tests"
-$env:NODE_OPTIONS = "--max-old-space-size=8192"
-
-Write-Host "✓ Environment configured" -ForegroundColor Green
-Write-Host ""
-
-# Display configuration
-Write-Host "Test Configuration:" -ForegroundColor Cyan
-Write-Host "  NODE_ENV:     $env:NODE_ENV"
-Write-Host "  Database:     $env:DB_HOST`:$env:DB_PORT/$env:DB_NAME"
-Write-Host "  Redis:        $env:REDIS_URL"
-Write-Host "  Frontend URL: $env:FRONTEND_URL"
-Write-Host ""
-
-# Check database connectivity
-Write-Host "Verifying database connection..." -ForegroundColor Yellow
-$dbCheck = podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1;" 2>&1
-if ($LASTEXITCODE -ne 0) {
-    Write-Host "ERROR: Cannot connect to database!" -ForegroundColor Red
-    Write-Host $dbCheck
-    exit 1
-}
-Write-Host "✓ Database connection successful" -ForegroundColor Green
-Write-Host ""
-
-# Check URL constraints are enabled
-Write-Host "Verifying URL constraints..." -ForegroundColor Yellow
-$constraints = podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -t -A -c "SELECT COUNT(*) FROM pg_constraint WHERE conname LIKE '%url_check';"
-Write-Host "✓ Found $constraints URL constraint(s)" -ForegroundColor Green
-Write-Host ""
-
-# Run integration tests
-Write-Host "=== Running Integration Tests ===" -ForegroundColor Cyan
-Write-Host ""
-
-npm run test:integration
-
-$exitCode = $LASTEXITCODE
-
-Write-Host ""
-if ($exitCode -eq 0) {
-    Write-Host "=== Integration Tests PASSED ===" -ForegroundColor Green
-} else {
-    Write-Host "=== Integration Tests FAILED ===" -ForegroundColor Red
-    Write-Host "Exit code: $exitCode" -ForegroundColor Red
-}
-
-exit $exitCode

run-tests.cmd

@@ -1,80 +0,0 @@
-@echo off
-REM Simple batch script to run integration tests with container infrastructure
-
-echo === Flyer Crawler Integration Test Runner ===
-echo.
-
-REM Check containers
-echo Checking container status...
-podman ps --filter "name=flyer-crawler-postgres" --format "{{.Names}}" >nul 2>&1
-if errorlevel 1 (
-    echo ERROR: PostgreSQL container is not running!
-    echo Start it with: podman start flyer-crawler-postgres
-    exit /b 1
-)
-
-podman ps --filter "name=flyer-crawler-redis" --format "{{.Names}}" >nul 2>&1
-if errorlevel 1 (
-    echo ERROR: Redis container is not running!
-    echo Start it with: podman start flyer-crawler-redis
-    exit /b 1
-)
-
-echo [OK] Containers are running
-echo.
-
-REM Set environment variables
-echo Setting environment variables...
-set NODE_ENV=test
-set DB_HOST=localhost
-set DB_USER=postgres
-set DB_PASSWORD=postgres
-set DB_NAME=flyer_crawler_dev
-set DB_PORT=5432
-set REDIS_URL=redis://localhost:6379
-set REDIS_PASSWORD=
-set FRONTEND_URL=http://localhost:5173
-set VITE_API_BASE_URL=http://localhost:3001/api
-set JWT_SECRET=test-jwt-secret-for-integration-tests
-set NODE_OPTIONS=--max-old-space-size=8192
-
-echo [OK] Environment configured
-echo.
-
-echo Test Configuration:
-echo   NODE_ENV:     %NODE_ENV%
-echo   Database:     %DB_HOST%:%DB_PORT%/%DB_NAME%
-echo   Redis:        %REDIS_URL%
-echo   Frontend URL: %FRONTEND_URL%
-echo.
-
-REM Verify database
-echo Verifying database connection...
-podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1;" >nul 2>&1
-if errorlevel 1 (
-    echo ERROR: Cannot connect to database!
-    exit /b 1
-)
-echo [OK] Database connection successful
-echo.
-
-REM Check URL constraints
-echo Verifying URL constraints...
-podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -t -A -c "SELECT COUNT(*) FROM pg_constraint WHERE conname LIKE '%%url_check';"
-echo.
-
-REM Run tests
-echo === Running Integration Tests ===
-echo.
-
-npm run test:integration
-
-if errorlevel 1 (
-    echo.
-    echo === Integration Tests FAILED ===
-    exit /b 1
-) else (
-    echo.
-    echo === Integration Tests PASSED ===
-    exit /b 0
-)

src/tests/setup/e2e-global-setup.ts

@@ -0,0 +1,240 @@
+// src/tests/setup/e2e-global-setup.ts
+import { execSync } from 'child_process';
+import fs from 'node:fs/promises';
+import path from 'path';
+import os from 'os';
+import type { Server } from 'http';
+import { logger } from '../../services/logger.server';
+import { getPool } from '../../services/db/connection.db';
+
+// --- DEBUG: Log when this file is first loaded/parsed ---
+const SETUP_LOAD_TIME = new Date().toISOString();
+console.error(`\n[E2E-SETUP-DEBUG] Module loaded at ${SETUP_LOAD_TIME}`);
+console.error(`[E2E-SETUP-DEBUG] Current working directory: ${process.cwd()}`);
+console.error(`[E2E-SETUP-DEBUG] NODE_ENV: ${process.env.NODE_ENV}`);
+console.error(`[E2E-SETUP-DEBUG] __filename: ${import.meta.url}`);
+
+// --- Centralized State for E2E Test Lifecycle ---
+let server: Server;
+// This will hold the single database pool instance for the entire test run.
+let globalPool: ReturnType<typeof getPool> | null = null;
+// Temporary directory for test file storage (to avoid modifying committed fixtures)
+let tempStorageDir: string | null = null;
+
+/**
+ * Cleans all BullMQ queues to ensure no stale jobs from previous test runs.
+ * This is critical because old jobs with outdated error messages can pollute test results.
+ */
+async function cleanAllQueues() {
+  console.error(`[PID:${process.pid}] [E2E QUEUE CLEANUP] Starting BullMQ queue cleanup...`);
+
+  try {
+    const {
+      flyerQueue,
+      cleanupQueue,
+      emailQueue,
+      analyticsQueue,
+      weeklyAnalyticsQueue,
+      tokenCleanupQueue,
+    } = await import('../../services/queues.server');
+    console.error(`[E2E QUEUE CLEANUP] Successfully imported queue modules`);
+
+    const queues = [
+      flyerQueue,
+      cleanupQueue,
+      emailQueue,
+      analyticsQueue,
+      weeklyAnalyticsQueue,
+      tokenCleanupQueue,
+    ];
+
+    for (const queue of queues) {
+      try {
+        const jobCounts = await queue.getJobCounts();
+        console.error(
+          `[E2E QUEUE CLEANUP] Queue "${queue.name}" before cleanup: ${JSON.stringify(jobCounts)}`,
+        );
+
+        await queue.obliterate({ force: true });
+        console.error(`  [E2E QUEUE CLEANUP] Cleaned queue: ${queue.name}`);
+      } catch (error) {
+        console.error(
+          `  [E2E QUEUE CLEANUP] Could not clean queue ${queue.name}: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        );
+      }
+    }
+    console.error(`[PID:${process.pid}] [E2E QUEUE CLEANUP] All queues cleaned successfully.`);
+  } catch (error) {
+    console.error(
+      `[PID:${process.pid}] [E2E QUEUE CLEANUP] CRITICAL ERROR during queue cleanup:`,
+      error,
+    );
+    // Don't throw - we want the tests to continue even if cleanup fails
+  }
+}
+
+export async function setup() {
+  console.error(`\n[E2E-SETUP-DEBUG] ========================================`);
+  console.error(`[E2E-SETUP-DEBUG] setup() function STARTED at ${new Date().toISOString()}`);
+  console.error(`[E2E-SETUP-DEBUG] ========================================`);
+
+  // Ensure we are in the correct environment for these tests.
+  process.env.NODE_ENV = 'test';
+  process.env.FRONTEND_URL = 'https://example.com';
+
+  // CRITICAL: Create a temporary directory for test file storage.
+  // This prevents tests from modifying or deleting committed fixture files.
+  // The temp directory is cleaned up in teardown().
+  tempStorageDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flyer-crawler-e2e-'));
+  const tempFlyerImagesDir = path.join(tempStorageDir, 'flyer-images');
+  await fs.mkdir(path.join(tempFlyerImagesDir, 'icons'), { recursive: true });
+  console.error(`[E2E-SETUP] Created temporary storage directory: ${tempFlyerImagesDir}`);
+
+  // CRITICAL: Set STORAGE_PATH before importing the server.
+  process.env.STORAGE_PATH = tempFlyerImagesDir;
+  console.error(`[E2E-SETUP] Set STORAGE_PATH to temporary directory: ${process.env.STORAGE_PATH}`);
+
+  console.error(`\n--- [PID:${process.pid}] Running E2E Test GLOBAL Setup ---`);
+  console.error(`[E2E-SETUP] STORAGE_PATH: ${process.env.STORAGE_PATH}`);
+  console.error(`[E2E-SETUP] REDIS_URL: ${process.env.REDIS_URL}`);
+  console.error(`[E2E-SETUP] REDIS_PASSWORD is set: ${!!process.env.REDIS_PASSWORD}`);
+
+  // Clean all queues BEFORE running any tests
+  console.error(`[E2E-SETUP] About to call cleanAllQueues()...`);
+  await cleanAllQueues();
+  console.error(`[E2E-SETUP] cleanAllQueues() completed.`);
+
+  // Seed the database for E2E tests
+  try {
+    console.log(`\n[PID:${process.pid}] Running database seed script for E2E tests...`);
+    execSync('npx cross-env NODE_ENV=test npx tsx src/db/seed.ts', { stdio: 'inherit' });
+    console.log(`[PID:${process.pid}] Database seed script finished.`);
+  } catch (error) {
+    console.error('Failed to reset and seed the test database. Aborting E2E tests.', error);
+    process.exit(1);
+  }
+
+  // Initialize the global pool instance once.
+  console.log(`[PID:${process.pid}] Initializing global database pool...`);
+  globalPool = getPool();
+
+  // Dynamic import AFTER env vars are set
+  console.error(`[E2E-SETUP-DEBUG] About to import server module...`);
+  const appModule = await import('../../../server');
+  console.error(`[E2E-SETUP-DEBUG] Server module imported successfully`);
+  const app = appModule.default;
+  console.error(`[E2E-SETUP-DEBUG] App object type: ${typeof app}`);
+
+  // Use a dedicated E2E test port (3098) to avoid conflicts with integration tests (3099)
+  // and production servers (3001)
+  const port = process.env.TEST_PORT || 3098;
+  console.error(`[E2E-SETUP-DEBUG] Attempting to start E2E server on port ${port}...`);
+
+  await new Promise<void>((resolve, reject) => {
+    let settled = false;
+    try {
+      server = app.listen(port, () => {
+        if (settled) return;
+        settled = true;
+        console.log(`In-process E2E test server started on port ${port}`);
+        console.error(
+          `[E2E-SETUP-DEBUG] Server listen callback invoked at ${new Date().toISOString()}`,
+        );
+        resolve();
+      });
+
+      server.on('error', (err: NodeJS.ErrnoException) => {
+        if (settled) return;
+        settled = true;
+        console.error(`[E2E-SETUP-DEBUG] Server error event:`, err.message);
+        if (err.code === 'EADDRINUSE') {
+          console.error(
+            `[E2E-SETUP-DEBUG] Port ${port} is already in use! ` +
+              `Set TEST_PORT env var to use a different port.`,
+          );
+        }
+        reject(err);
+      });
+    } catch (err) {
+      if (settled) return;
+      settled = true;
+      console.error(`[E2E-SETUP-DEBUG] Error during app.listen:`, err);
+      reject(err);
+    }
+  });
+
+  /**
+   * Ping the E2E test server to verify it's ready.
+   */
+  const pingTestBackend = async (): Promise<boolean> => {
+    const pingUrl = `http://localhost:${port}/api/health/ping`;
+    console.error(`[E2E-SETUP-DEBUG] Pinging: ${pingUrl}`);
+    try {
+      const response = await fetch(pingUrl);
+      console.error(`[E2E-SETUP-DEBUG] Ping response status: ${response.status}`);
+      if (!response.ok) {
+        console.error(`[E2E-SETUP-DEBUG] Ping response not OK: ${response.statusText}`);
+        return false;
+      }
+      const json = await response.json();
+      console.error(`[E2E-SETUP-DEBUG] Ping response JSON:`, JSON.stringify(json));
+      return json?.data?.message === 'pong';
+    } catch (e) {
+      const errMsg = e instanceof Error ? e.message : String(e);
+      console.error(`[E2E-SETUP-DEBUG] Ping exception: ${errMsg}`);
+      logger.debug({ error: e }, 'Ping failed while waiting for E2E server, this is expected.');
+      return false;
+    }
+  };
+
+  console.error(
+    `[E2E-SETUP-DEBUG] Server started, beginning ping loop at ${new Date().toISOString()}`,
+  );
+  console.error(`[E2E-SETUP-DEBUG] Server address info:`, server.address());
+
+  const maxRetries = 15;
+  const retryDelay = 1000;
+  for (let i = 0; i < maxRetries; i++) {
+    console.error(`[E2E-SETUP-DEBUG] Ping attempt ${i + 1}/${maxRetries}`);
+    if (await pingTestBackend()) {
+      console.log('E2E backend server is running and responsive.');
+      console.error(
+        `[E2E-SETUP-DEBUG] setup() function COMPLETED SUCCESSFULLY at ${new Date().toISOString()}`,
+      );
+      return;
+    }
+    console.log(
+      `[PID:${process.pid}] Waiting for E2E backend server... (attempt ${i + 1}/${maxRetries})`,
+    );
+    await new Promise((resolve) => setTimeout(resolve, retryDelay));
+  }
+
+  console.error(`[E2E-SETUP-DEBUG] All ${maxRetries} ping attempts failed!`);
+  console.error(`[E2E-SETUP-DEBUG] Server listening status: ${server.listening}`);
+  console.error(`[E2E-SETUP-DEBUG] Server address: ${JSON.stringify(server.address())}`);
+
+  throw new Error('E2E backend server failed to start.');
+}
+
+export async function teardown() {
+  console.log(`\n--- [PID:${process.pid}] Running E2E Test GLOBAL Teardown ---`);
+  // 1. Stop the server to release any resources it's holding.
+  if (server) {
+    await new Promise<void>((resolve) => server.close(() => resolve()));
+    console.log('In-process E2E test server stopped.');
+  }
+  // 2. Close the single, shared database pool.
+  if (globalPool) {
+    await globalPool.end();
+    console.log('E2E global database pool teardown complete.');
+  }
+  // 3. Clean up the temporary storage directory.
+  if (tempStorageDir) {
+    try {
+      await fs.rm(tempStorageDir, { recursive: true, force: true });
+      console.log(`Cleaned up E2E temporary storage directory: ${tempStorageDir}`);
+    } catch (error) {
+      console.error(`Warning: Could not clean up E2E temp directory ${tempStorageDir}:`, error);
+    }
+  }
+}

src/tests/setup/integration-global-setup.ts

@@ -136,12 +136,16 @@ export async function setup() {
   console.error(`[GLOBAL-SETUP-DEBUG] App object type: ${typeof app}`);
 
   // Programmatically start the server within the same process.
-  const port = process.env.PORT || 3001;
+  // Use a dedicated test port to avoid conflicts with production servers.
+  const port = process.env.TEST_PORT || process.env.PORT || 3099;
   console.error(`[GLOBAL-SETUP-DEBUG] Attempting to start server on port ${port}...`);
 
   await new Promise<void>((resolve, reject) => {
+    let settled = false; // Prevent double-resolution race condition
     try {
       server = app.listen(port, () => {
+        if (settled) return;
+        settled = true;
         console.log(`✅ In-process test server started on port ${port}`);
         console.error(
           `[GLOBAL-SETUP-DEBUG] Server listen callback invoked at ${new Date().toISOString()}`,
@@ -150,25 +154,33 @@ export async function setup() {
       });
 
       server.on('error', (err: NodeJS.ErrnoException) => {
+        if (settled) return;
+        settled = true;
         console.error(`[GLOBAL-SETUP-DEBUG] Server error event:`, err.message);
         if (err.code === 'EADDRINUSE') {
-          console.error(`[GLOBAL-SETUP-DEBUG] Port ${port} is already in use!`);
+          console.error(
+            `[GLOBAL-SETUP-DEBUG] Port ${port} is already in use! ` +
+              `Set TEST_PORT env var to use a different port.`,
+          );
         }
         reject(err);
       });
     } catch (err) {
+      if (settled) return;
+      settled = true;
       console.error(`[GLOBAL-SETUP-DEBUG] Error during app.listen:`, err);
       reject(err);
     }
   });
 
   /**
-   * A local ping function that respects the VITE_API_BASE_URL from the test environment.
-   * This is necessary because the global apiClient's URL is configured for browser use.
+   * A local ping function that pings the test server we just started.
+   * Uses the same port that the server was started on to avoid hitting
+   * a different server that might be running on the default port.
    */
   const pingTestBackend = async (): Promise<boolean> => {
-    const apiUrl = process.env.VITE_API_BASE_URL || 'http://localhost:3001/api';
-    const pingUrl = `${apiUrl.replace('/api', '')}/api/health/ping`;
+    // Always ping the port we started on, not what's in env vars
+    const pingUrl = `http://localhost:${port}/api/health/ping`;
     console.error(`[GLOBAL-SETUP-DEBUG] Pinging: ${pingUrl}`);
     try {
       const response = await fetch(pingUrl);

vitest.config.e2e.ts

@@ -1,26 +1,55 @@
+// vitest.config.e2e.ts
 import { defineConfig, mergeConfig } from 'vitest/config';
-import integrationConfig from './vitest.config.integration';
+import type { UserConfig } from 'vite';
+import viteConfig from './vite.config';
 
+// Ensure NODE_ENV is set to 'test' for all Vitest runs.
+process.env.NODE_ENV = 'test';
+
+// Define a type that includes the 'test' property from Vitest's config.
+type ViteConfigWithTest = UserConfig & { test?: UserConfig['test'] };
+
+const { test: _unusedTest, ...baseViteConfig } = viteConfig as ViteConfigWithTest;
+
+/**
+ * E2E test configuration.
+ * Uses a DIFFERENT port (3098) than integration tests (3099) to allow
+ * both test suites to run sequentially without port conflicts.
+ */
 const e2eConfig = mergeConfig(
-  integrationConfig,
+  baseViteConfig,
   defineConfig({
     test: {
       name: 'e2e',
+      environment: 'node',
       // Point specifically to E2E tests
       include: ['src/tests/e2e/**/*.e2e.test.ts'],
+      exclude: [],
+      // E2E tests use a different port to avoid conflicts with integration tests
+      env: {
+        NODE_ENV: 'test',
+        BASE_URL: 'https://example.com',
+        FRONTEND_URL: 'https://example.com',
+        // Use port 3098 for E2E tests (integration uses 3099)
+        TEST_PORT: '3098',
+        VITE_API_BASE_URL: 'http://localhost:3098/api',
+      },
+      // E2E tests have their own dedicated global setup file
+      globalSetup: './src/tests/setup/e2e-global-setup.ts',
+      setupFiles: ['./src/tests/setup/global.ts'],
       // Increase timeout for E2E flows that involve AI or full API chains
       testTimeout: 120000,
+      hookTimeout: 60000,
+      fileParallelism: false,
       coverage: {
+        provider: 'v8',
+        reporter: ['html', 'json-summary', 'json'],
         reportsDirectory: '.coverage/e2e',
+        reportOnFailure: true,
+        clean: true,
       },
     },
   }),
 );
 
-// Explicitly override the include array to ensure we don't inherit integration tests
-// (mergeConfig might concatenate arrays by default)
-if (e2eConfig.test) {
-  e2eConfig.test.include = ['src/tests/e2e/**/*.e2e.test.ts'];
-}
-
-export default e2eConfig;
+export default e2eConfig;

vitest.config.integration.ts

@@ -65,7 +65,10 @@ const finalConfig = mergeConfig(
         NODE_ENV: 'test',
         BASE_URL: 'https://example.com', // Use a standard domain to pass strict URL validation
         FRONTEND_URL: 'https://example.com',
-        PORT: '3000',
+        // Use a dedicated test port (3099) to avoid conflicts with production servers
+        // that might be running on port 3000 or 3001
+        TEST_PORT: '3099',
+        VITE_API_BASE_URL: 'http://localhost:3099/api',
       },
       // This setup script starts the backend server before tests run.
       globalSetup: './src/tests/setup/integration-global-setup.ts',