flyer-crawler.projectium.com/src/tests/integration/flyer-processing.integration.test.ts

// src/tests/integration/flyer-processing.integration.test.ts
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import supertest from 'supertest';
import app from '../../../server';
import fs from 'node:fs/promises';
import path from 'path';
import * as db from '../../services/db/index.db';
import { getPool } from '../../services/db/connection.db';
import { generateFileChecksum } from '../../utils/checksum';
import { logger } from '../../services/logger.server';
import type { UserProfile, ExtractedFlyerItem } from '../../types';
import { createAndLoginUser } from '../utils/testHelpers';
import { cleanupDb } from '../utils/cleanup';
import { cleanupFiles } from '../utils/cleanupFiles';
import piexif from 'piexifjs';
import exifParser from 'exif-parser';
import sharp from 'sharp';


/**
 * @vitest-environment node
 */

const request = supertest(app);

// Import the mocked service to control its behavior in tests.
import { aiService } from '../../services/aiService.server';

describe('Flyer Processing Background Job Integration Test', () => {
  const createdUserIds: string[] = [];
  const createdFlyerIds: number[] = [];
  const createdFilePaths: string[] = [];

  beforeAll(async () => {
    // This setup is now simpler as the worker handles fetching master items.
    // Setup default mock response for AI service
    const mockItems: ExtractedFlyerItem[] = [
      {
        item: 'Mocked Integration Item',
        price_display: '$1.99',
        price_in_cents: 199,
        quantity: 'each',
        category_name: 'Mock Category',
      },
    ];
    vi.spyOn(aiService, 'extractCoreDataFromFlyerImage').mockResolvedValue({
      store_name: 'Mock Store',
      valid_from: null,
      valid_to: null,
      store_address: null,
      items: mockItems,
    });
  });

  afterAll(async () => {
    // Use the centralized cleanup utility.
    await cleanupDb({
      userIds: createdUserIds,
      flyerIds: createdFlyerIds,
    });

    // Use the centralized file cleanup utility.
    await cleanupFiles(createdFilePaths);
  });

  /**
   * This is the new end-to-end test for the background job processing flow.
   * It uploads a file, polls for completion, and verifies the result in the database.
   */
  const runBackgroundProcessingTest = async (user?: UserProfile, token?: string) => {
    // Arrange: Load a mock flyer PDF.
    const imagePath = path.resolve(__dirname, '../assets/test-flyer-image.jpg');
    const imageBuffer = await fs.readFile(imagePath);
    // Create a unique buffer and filename for each test run to ensure a unique checksum.
    // This prevents a 409 Conflict error when the second test runs.
    const uniqueContent = Buffer.concat([imageBuffer, Buffer.from(Date.now().toString())]);
    const uniqueFileName = `test-flyer-image-${Date.now()}.jpg`;
    const mockImageFile = new File([uniqueContent], uniqueFileName, { type: 'image/jpeg' });
    const checksum = await generateFileChecksum(mockImageFile);

    // Track created files for cleanup
    const uploadDir = path.resolve(__dirname, '../../../flyer-images');
    createdFilePaths.push(path.join(uploadDir, uniqueFileName));
    // The icon name is derived from the original filename.
    const iconFileName = `icon-${path.parse(uniqueFileName).name}.webp`;
    createdFilePaths.push(path.join(uploadDir, 'icons', iconFileName));

    // Act 1: Upload the file to start the background job.
    const uploadReq = request
      .post('/api/ai/upload-and-process')
      .field('checksum', checksum)
      .attach('flyerFile', uniqueContent, uniqueFileName);
    if (token) {
      uploadReq.set('Authorization', `Bearer ${token}`);
    }
    const uploadResponse = await uploadReq;
    const { jobId } = uploadResponse.body;

    // Assert 1: Check that a job ID was returned.
    expect(jobId).toBeTypeOf('string');

    // Act 2: Poll for the job status until it completes.
    let jobStatus;
    const maxRetries = 60; // Poll for up to 180 seconds (60 * 3s)
    for (let i = 0; i < maxRetries; i++) {
      console.log(`Polling attempt ${i + 1}...`);
      await new Promise((resolve) => setTimeout(resolve, 3000)); // Wait 3 seconds between polls
      const statusReq = request.get(`/api/ai/jobs/${jobId}/status`);
      if (token) {
        statusReq.set('Authorization', `Bearer ${token}`);
      }
      const statusResponse = await statusReq;
      jobStatus = statusResponse.body;
      console.log(`Job status: ${JSON.stringify(jobStatus)}`);
      if (jobStatus.state === 'completed' || jobStatus.state === 'failed') {
        break;
      }
    }

    // Assert 2: Check that the job completed successfully.
    if (jobStatus?.state === 'failed') {
      console.error('[DEBUG] Job failed with reason:', jobStatus.failedReason);
      console.error('[DEBUG] Job stack trace:', jobStatus.stacktrace);
      console.error('[DEBUG] Full Job Status:', JSON.stringify(jobStatus, null, 2));
    }
    expect(jobStatus?.state).toBe('completed');
    const flyerId = jobStatus?.returnValue?.flyerId;
    expect(flyerId).toBeTypeOf('number');
    createdFlyerIds.push(flyerId); // Track for cleanup

    // Assert 3: Verify the flyer and its items were actually saved in the database.
    const savedFlyer = await db.flyerRepo.findFlyerByChecksum(checksum, logger);
    expect(savedFlyer).toBeDefined();
    expect(savedFlyer?.flyer_id).toBe(flyerId);
    expect(savedFlyer?.file_name).toBe(uniqueFileName);
    // Also add the final processed image path to the cleanup list.
    // This is important because JPEGs are re-processed to strip EXIF data, creating a new file.
    const savedImagePath = path.join(uploadDir, path.basename(savedFlyer!.image_url));
    createdFilePaths.push(savedImagePath);

    const items = await db.flyerRepo.getFlyerItems(flyerId, logger);
    // The stubbed AI response returns items, so we expect them to be here.
    expect(items.length).toBeGreaterThan(0);
    expect(items[0].item).toBeTypeOf('string');

    // Assert 4: Verify user association is correct.
    if (token) {
      expect(savedFlyer?.uploaded_by).toBe(user?.user.user_id);
    } else {
      expect(savedFlyer?.uploaded_by).toBe(null);
    }
  };

  it('should successfully process a flyer for an AUTHENTICATED user via the background queue', async ({
    onTestFinished,
  }) => {
    // Arrange: Create a new user specifically for this test.
    const email = `auth-flyer-user-${Date.now()}@example.com`;
    const { user: authUser, token } = await createAndLoginUser({
      email,
      fullName: 'Flyer Uploader',
      request,
    });
    createdUserIds.push(authUser.user.user_id); // Track for cleanup

    // Use a cleanup function to delete the user even if the test fails.
    onTestFinished(async () => {
      await getPool().query('DELETE FROM public.users WHERE user_id = $1', [authUser.user.user_id]);
    });

    // Act & Assert
    await runBackgroundProcessingTest(authUser, token);
  }, 240000); // Increase timeout to 240 seconds for this long-running test

  it('should successfully process a flyer for an ANONYMOUS user via the background queue', async () => {
    // Act & Assert: Call the test helper without a user or token.
    await runBackgroundProcessingTest();
  }, 240000); // Increase timeout to 240 seconds for this long-running test

  it(
    'should strip EXIF data from uploaded JPEG images during processing',
    async () => {
      // Arrange: Create a user for this test
      const { user: authUser, token } = await createAndLoginUser({
        email: `exif-user-${Date.now()}@example.com`,
        fullName: 'EXIF Tester',
        request,
      });
      createdUserIds.push(authUser.user.user_id);

      // 1. Create an image buffer with EXIF data
      const imagePath = path.resolve(__dirname, '../assets/test-flyer-image.jpg');
      const imageBuffer = await fs.readFile(imagePath);
      const jpegDataAsString = imageBuffer.toString('binary');

      const exifObj = {
        '0th': { [piexif.ImageIFD.Software]: 'Gemini Code Assist Test' },
        Exif: { [piexif.ExifIFD.DateTimeOriginal]: '2025:12:25 10:00:00' },
      };
      const exifBytes = piexif.dump(exifObj);
      const jpegWithExif = piexif.insert(exifBytes, jpegDataAsString);
      const imageWithExifBuffer = Buffer.from(jpegWithExif, 'binary');

      const uniqueFileName = `test-flyer-with-exif-${Date.now()}.jpg`;
      const mockImageFile = new File([imageWithExifBuffer], uniqueFileName, { type: 'image/jpeg' });
      const checksum = await generateFileChecksum(mockImageFile);

      // Track original and derived files for cleanup
      const uploadDir = path.resolve(__dirname, '../../../flyer-images');
      createdFilePaths.push(path.join(uploadDir, uniqueFileName));
      const iconFileName = `icon-${path.parse(uniqueFileName).name}.webp`;
      createdFilePaths.push(path.join(uploadDir, 'icons', iconFileName));

      // 2. Act: Upload the file and wait for processing
      const uploadResponse = await request
        .post('/api/ai/upload-and-process')
        .set('Authorization', `Bearer ${token}`)
        .field('checksum', checksum)
        .attach('flyerFile', imageWithExifBuffer, uniqueFileName);

      const { jobId } = uploadResponse.body;
      expect(jobId).toBeTypeOf('string');

      // Poll for job completion
      let jobStatus;
      const maxRetries = 30; // Poll for up to 90 seconds
      for (let i = 0; i < maxRetries; i++) {
        await new Promise((resolve) => setTimeout(resolve, 3000));
        const statusResponse = await request
          .get(`/api/ai/jobs/${jobId}/status`)
          .set('Authorization', `Bearer ${token}`);
        jobStatus = statusResponse.body;
        if (jobStatus.state === 'completed' || jobStatus.state === 'failed') {
          break;
        }
      }

      // 3. Assert
      if (jobStatus?.state === 'failed') {
        console.error('[DEBUG] EXIF test job failed:', jobStatus.failedReason);
      }
      expect(jobStatus?.state).toBe('completed');
      const flyerId = jobStatus?.data?.flyerId;
      expect(flyerId).toBeTypeOf('number');
      createdFlyerIds.push(flyerId);

      // 4. Verify EXIF data is stripped from the saved file
      const savedFlyer = await db.flyerRepo.findFlyerByChecksum(checksum, logger);
      expect(savedFlyer).toBeDefined();

      const savedImagePath = path.join(uploadDir, path.basename(savedFlyer!.image_url));
      createdFilePaths.push(savedImagePath); // Add final path for cleanup

      const savedImageBuffer = await fs.readFile(savedImagePath);
      const parser = exifParser.create(savedImageBuffer);
      const exifResult = parser.parse();

      // The `tags` object will be empty if no EXIF data is found.
      expect(exifResult.tags).toEqual({});
      expect(exifResult.tags.Software).toBeUndefined();
    },
    240000,
  );

  it(
    'should strip metadata from uploaded PNG images during processing',
    async () => {
      // Arrange: Create a user for this test
      const { user: authUser, token } = await createAndLoginUser({
        email: `png-meta-user-${Date.now()}@example.com`,
        fullName: 'PNG Metadata Tester',
        request,
      });
      createdUserIds.push(authUser.user.user_id);

      // 1. Create a PNG image buffer with custom metadata using sharp
      const imagePath = path.resolve(__dirname, '../assets/test-flyer-image.jpg');

      const imageWithMetadataBuffer = await sharp(imagePath)
        .png() // Convert to PNG
        .withMetadata({
          exif: {
            IFD0: {
              Copyright: 'Gemini Code Assist PNG Test',
            },
          },
        })
        .toBuffer();

      const uniqueFileName = `test-flyer-with-metadata-${Date.now()}.png`;
      const mockImageFile = new File([Buffer.from(imageWithMetadataBuffer)], uniqueFileName, { type: 'image/png' });
      const checksum = await generateFileChecksum(mockImageFile);

      // Track files for cleanup
      const uploadDir = path.resolve(__dirname, '../../../flyer-images');
      createdFilePaths.push(path.join(uploadDir, uniqueFileName));
      const iconFileName = `icon-${path.parse(uniqueFileName).name}.webp`;
      createdFilePaths.push(path.join(uploadDir, 'icons', iconFileName));

      // 2. Act: Upload the file and wait for processing
      const uploadResponse = await request
        .post('/api/ai/upload-and-process')
        .set('Authorization', `Bearer ${token}`)
        .field('checksum', checksum)
        .attach('flyerFile', imageWithMetadataBuffer, uniqueFileName);

      const { jobId } = uploadResponse.body;
      expect(jobId).toBeTypeOf('string');

      // Poll for job completion
      let jobStatus;
      const maxRetries = 30;
      for (let i = 0; i < maxRetries; i++) {
        await new Promise((resolve) => setTimeout(resolve, 3000));
        const statusResponse = await request
          .get(`/api/ai/jobs/${jobId}/status`)
          .set('Authorization', `Bearer ${token}`);
        jobStatus = statusResponse.body;
        if (jobStatus.state === 'completed' || jobStatus.state === 'failed') {
          break;
        }
      }

      // 3. Assert job completion
      if (jobStatus?.state === 'failed') {
        console.error('[DEBUG] PNG metadata test job failed:', jobStatus.failedReason);
      }
      expect(jobStatus?.state).toBe('completed');
      const flyerId = jobStatus?.data?.flyerId;
      expect(flyerId).toBeTypeOf('number');
      createdFlyerIds.push(flyerId);

      // 4. Verify metadata is stripped from the saved file
      const savedFlyer = await db.flyerRepo.findFlyerByChecksum(checksum, logger);
      expect(savedFlyer).toBeDefined();

      const savedImagePath = path.join(uploadDir, path.basename(savedFlyer!.image_url));
      createdFilePaths.push(savedImagePath); // Add final path for cleanup

      const savedImageMetadata = await sharp(savedImagePath).metadata();

      // The test should fail here initially because PNGs are not processed.
      // The `exif` property should be undefined after the fix.
      expect(savedImageMetadata.exif).toBeUndefined();
    },
    240000,
  );
});