flyer-crawler.projectium.com/server.ts

// server.ts
/**
 * IMPORTANT: Sentry initialization MUST happen before any other imports
 * to ensure all errors are captured, including those in imported modules.
 * See ADR-015: Application Performance Monitoring and Error Tracking.
 */
import { initSentry, getSentryMiddleware } from './src/services/sentry.server';
initSentry();

import express, { Request, Response, NextFunction } from 'express';
import { randomUUID } from 'crypto';
import helmet from 'helmet';
import timeout from 'connect-timeout';
import cookieParser from 'cookie-parser';
import listEndpoints from 'express-list-endpoints';
import { getPool } from './src/services/db/connection.db';

import passport from './src/config/passport';
import { logger } from './src/services/logger.server';

// Import routers
import authRouter from './src/routes/auth.routes';
import userRouter from './src/routes/user.routes';
import adminRouter from './src/routes/admin.routes';
import aiRouter from './src/routes/ai.routes';
import budgetRouter from './src/routes/budget.routes';
import flyerRouter from './src/routes/flyer.routes';
import recipeRouter from './src/routes/recipe.routes';
import personalizationRouter from './src/routes/personalization.routes';
import priceRouter from './src/routes/price.routes';
import statsRouter from './src/routes/stats.routes';
import gamificationRouter from './src/routes/gamification.routes';
import systemRouter from './src/routes/system.routes';
import healthRouter from './src/routes/health.routes';
import upcRouter from './src/routes/upc.routes';
import inventoryRouter from './src/routes/inventory.routes';
import receiptRouter from './src/routes/receipt.routes';
import dealsRouter from './src/routes/deals.routes';
import reactionsRouter from './src/routes/reactions.routes';
import { errorHandler } from './src/middleware/errorHandler';
import { backgroundJobService, startBackgroundJobs } from './src/services/backgroundJobService';
import type { UserProfile } from './src/types';

// API Documentation (ADR-018)
import swaggerUi from 'swagger-ui-express';
import { swaggerSpec } from './src/config/swagger';
import {
  analyticsQueue,
  weeklyAnalyticsQueue,
  gracefulShutdown,
  tokenCleanupQueue,
} from './src/services/queueService.server';
import { monitoringService } from './src/services/monitoringService.server';

// --- START DEBUG LOGGING ---
// Log the database connection details as seen by the SERVER PROCESS.
// This will confirm if the `--env-file` flag is working as expected.
logger.info('--- [SERVER PROCESS LOG] DATABASE CONNECTION ---');
logger.info(`  NODE_ENV: ${process.env.NODE_ENV}`);
logger.info(`  Host: ${process.env.DB_HOST}`);
logger.info(`  Port: ${process.env.DB_PORT}`);
logger.info(`  User: ${process.env.DB_USER}`);
logger.info(`  Database: ${process.env.DB_NAME}`);

// Query the users table to see what the server process sees on startup.
// Corrected the query to be unambiguous by specifying the table alias for each column.
// `id` and `email` come from the `users` table (u), and `role` comes from the `profiles` table (p).
getPool()
  .query(
    'SELECT u.user_id, u.email, p.role FROM public.users u JOIN public.profiles p ON u.user_id = p.user_id',
  )
  .then((res) => {
    logger.debug('[SERVER PROCESS] Users found in DB on startup:');
    console.table(res.rows);
  })
  .catch((err) => {
    logger.error({ err }, '[SERVER PROCESS] Could not query users table on startup.');
  });

logger.info('-----------------------------------------------\n');

const app = express();

// --- Security Headers Middleware (ADR-016) ---
// Helmet sets various HTTP headers to help protect the app from common web vulnerabilities.
// Must be applied early in the middleware chain, before any routes.
app.use(
  helmet({
    // Content Security Policy - configured for API + SPA frontend
    contentSecurityPolicy: {
      directives: {
        defaultSrc: ["'self'"],
        scriptSrc: ["'self'", "'unsafe-inline'"], // Allow inline scripts for React
        styleSrc: ["'self'", "'unsafe-inline'"], // Allow inline styles for Tailwind
        imgSrc: ["'self'", 'data:', 'blob:', 'https:'], // Allow images from various sources
        fontSrc: ["'self'", 'https:', 'data:'],
        connectSrc: ["'self'", 'https:', 'wss:'], // Allow API and WebSocket connections
        frameSrc: ["'none'"], // Disallow iframes
        objectSrc: ["'none'"], // Disallow plugins
        upgradeInsecureRequests: process.env.NODE_ENV === 'production' ? [] : null,
      },
    },
    // Cross-Origin settings for API
    crossOriginEmbedderPolicy: false, // Disabled to allow loading external images
    crossOriginResourcePolicy: { policy: 'cross-origin' }, // Allow cross-origin resource loading
    // Additional security headers
    hsts: {
      maxAge: 31536000, // 1 year in seconds
      includeSubDomains: true,
      preload: true,
    },
    referrerPolicy: { policy: 'strict-origin-when-cross-origin' },
  }),
);

// --- Core Middleware ---
// Increase the limit for JSON and URL-encoded bodies. This is crucial for handling large file uploads
// that are part of multipart/form-data requests, as the overall request size is checked.
// Setting a 50MB limit to accommodate large flyer images.
app.use(express.json({ limit: '100mb' }));
app.use(express.urlencoded({ limit: '100mb', extended: true }));
app.use(cookieParser()); // Middleware to parse cookies
app.use(passport.initialize()); // Initialize Passport

// --- Sentry Request Handler (ADR-015) ---
// Must be the first middleware after body parsers to capture request data for errors.
const sentryMiddleware = getSentryMiddleware();
app.use(sentryMiddleware.requestHandler);

// --- MOCK AUTH FOR TESTING ---
// This MUST come after passport.initialize() and BEFORE any of the API routes.
import { mockAuth } from './src/config/passport';
app.use(mockAuth);

// Add a request timeout middleware. This will help prevent requests from hanging indefinitely.
// We set a generous 5-minute timeout to accommodate slow AI processing for large flyers.
app.use(timeout('5m'));

// --- Logging Middleware ---
const getDurationInMilliseconds = (start: [number, number]): number => {
  const NS_PER_SEC = 1e9;
  const NS_TO_MS = 1e6;
  const diff = process.hrtime(start);
  return (diff[0] * NS_PER_SEC + diff[1]) / NS_TO_MS;
};

/**
 * Defines the structure for the detailed log object created for each request.
 * This ensures type safety and consistency in our structured logs.
 */
interface RequestLogDetails {
  user_id?: string;
  method: string;
  originalUrl: string;
  statusCode: number;
  statusMessage: string;
  duration: string;
  // The 'req' property is added conditionally for client/server errors.
  req?: { headers: express.Request['headers']; body: express.Request['body'] };
}

const requestLogger = (req: Request, res: Response, next: NextFunction) => {
  const requestId = randomUUID();
  const user = req.user as UserProfile | undefined;
  const start = process.hrtime();
  const { method, originalUrl } = req;

  // Create a request-scoped logger instance as per ADR-004
  // This attaches contextual info to every log message generated for this request.
  req.log = logger.child({
    request_id: requestId,
    user_id: user?.user.user_id, // This will be undefined until the auth middleware runs, but the logger will hold the reference.
    ip_address: req.ip,
  });

  req.log.debug({ method, originalUrl }, `[Request Logger] INCOMING`);

  res.on('finish', () => {
    const durationInMilliseconds = getDurationInMilliseconds(start);
    const { statusCode, statusMessage } = res;
    const finalUser = req.user as UserProfile | undefined;

    // The base log object includes details relevant for all status codes.
    const logDetails: RequestLogDetails = {
      user_id: finalUser?.user.user_id,
      method,
      originalUrl,
      statusCode,
      statusMessage,
      duration: durationInMilliseconds.toFixed(2),
    };

    // For failed requests, add the full request details for better debugging.
    // Pino's `redact` config will automatically sanitize sensitive headers and body fields.
    if (statusCode >= 400) {
      logDetails.req = { headers: req.headers, body: req.body };
    }
    if (statusCode >= 500) req.log.error(logDetails, 'Request completed with server error');
    else if (statusCode >= 400) req.log.warn(logDetails, 'Request completed with client error');
    else req.log.info(logDetails, 'Request completed successfully');
  });

  next();
};

app.use(requestLogger); // Use the logging middleware for all requests

// --- Security Warning ---
if (!process.env.JWT_SECRET) {
  logger.error('CRITICAL: JWT_SECRET is not set. The application cannot start securely.');
  process.exit(1);
}

// --- API Documentation (ADR-018) ---
// Only serve Swagger UI in non-production environments to prevent information disclosure.
if (process.env.NODE_ENV !== 'production') {
  app.use(
    '/docs/api-docs',
    swaggerUi.serve,
    swaggerUi.setup(swaggerSpec, {
      customCss: '.swagger-ui .topbar { display: none }',
      customSiteTitle: 'Flyer Crawler API Documentation',
    }),
  );

  // Expose raw OpenAPI JSON spec for tooling (SDK generation, testing, etc.)
  app.get('/docs/api-docs.json', (_req, res) => {
    res.setHeader('Content-Type', 'application/json');
    res.send(swaggerSpec);
  });

  logger.info('API Documentation available at /docs/api-docs');
}

// --- API Routes ---

// ADR-053: Worker Health Checks
// Expose queue metrics for monitoring.
app.get('/api/health/queues', async (req, res) => {
  try {
    const statuses = await monitoringService.getQueueStatuses();
    res.json(statuses);
  } catch (error) {
    logger.error({ err: error }, 'Failed to fetch queue statuses');
    res.status(503).json({ error: 'Failed to fetch queue statuses' });
  }
});

// The order of route registration is critical.
// More specific routes should be registered before more general ones.
// 1. Authentication routes for login, registration, etc.
app.use('/api/auth', authRouter); // This was a duplicate, fixed.
// 2. System routes for health checks, etc.
app.use('/api/health', healthRouter);
// 3. System routes for pm2 status, etc.
app.use('/api/system', systemRouter);
// 3. General authenticated user routes.
app.use('/api/users', userRouter);
// 4. AI routes, some of which use optional authentication.
app.use('/api/ai', aiRouter);
// 5. Admin routes, which are all protected by admin-level checks.
app.use('/api/admin', adminRouter); // This seems to be missing from the original file list, but is required.
// 6. Budgeting and spending analysis routes.
app.use('/api/budgets', budgetRouter);
// 7. Gamification routes for achievements.
app.use('/api/achievements', gamificationRouter);
// 8. Public flyer routes.
app.use('/api/flyers', flyerRouter);
// 8. Public recipe routes.
app.use('/api/recipes', recipeRouter);
// 9. Public personalization data routes (master items, etc.).
app.use('/api/personalization', personalizationRouter);
// 9.5. Price history routes.
app.use('/api/price-history', priceRouter);
// 10. Public statistics routes.
app.use('/api/stats', statsRouter);
// 11. UPC barcode scanning routes.
app.use('/api/upc', upcRouter);
// 12. Inventory and expiry tracking routes.
app.use('/api/inventory', inventoryRouter);
// 13. Receipt scanning routes.
app.use('/api/receipts', receiptRouter);
// 14. Deals and best prices routes.
app.use('/api/deals', dealsRouter);
// 15. Reactions/social features routes.
app.use('/api/reactions', reactionsRouter);

// --- Error Handling and Server Startup ---

// Sentry Error Handler (ADR-015) - captures errors and sends to Bugsink.
// Must come BEFORE the custom error handler but AFTER all routes.
app.use(sentryMiddleware.errorHandler);

// Global error handling middleware. This must be the last `app.use()` call.
app.use(errorHandler);

// --- Server Startup ---
// Only start the server and background jobs if the file is run directly,
// not when it's imported by another module (like the integration test setup).
// This prevents the server from trying to listen on a port during tests.
if (process.env.NODE_ENV !== 'test') {
  const PORT = process.env.PORT || 3001;
  app.listen(PORT, () => {
    logger.info(`Authentication server started on port ${PORT}`);
    console.log('--- REGISTERED API ROUTES ---');
    console.table(listEndpoints(app));
    console.log('-----------------------------');
  });

  // Start the scheduled background jobs
  startBackgroundJobs(
    backgroundJobService,
    analyticsQueue,
    weeklyAnalyticsQueue,
    tokenCleanupQueue,
    logger,
  );

  // --- Graceful Shutdown Handling ---
  process.on('SIGINT', () => gracefulShutdown('SIGINT'));
  process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
}

// Export the app for integration testing
export default app;