diff --git a/src/pages/admin/components/SystemCheck.tsx b/src/pages/admin/components/SystemCheck.tsx index df272a4a..09e4a3d2 100644 --- a/src/pages/admin/components/SystemCheck.tsx +++ b/src/pages/admin/components/SystemCheck.tsx @@ -1,7 +1,7 @@ // src/pages/admin/components/SystemCheck.tsx import React, { useState, useEffect, useCallback } from 'react'; import toast from 'react-hot-toast'; -import { loginUser, pingBackend, checkDbSchema, checkStorage, checkDbPoolHealth, checkPm2Status, checkRedisHealth, triggerFailingJob } from '../../../services/apiClient'; +import { loginUser, pingBackend, checkDbSchema, checkStorage, checkDbPoolHealth, checkPm2Status, checkRedisHealth, triggerFailingJob, clearGeocodeCache } from '../../../services/apiClient'; import { ShieldCheckIcon } from '../../../components/icons/ShieldCheckIcon'; import { LoadingSpinner } from '../../../components/LoadingSpinner'; import { CheckCircleIcon } from '../../../components/icons/CheckCircleIcon'; @@ -41,6 +41,54 @@ const initialChecks: Check[] = [ { id: CheckID.GEMINI, name: 'Gemini API Key', description: 'Verifies the GEMINI_API_KEY is set for AI features.', status: 'idle', message: '' }, ]; +const GeocodeCacheManager: React.FC = () => { + const [isLoading, setIsLoading] = useState(false); + + const handleClearCache = async () => { + if (!window.confirm('Are you sure you want to clear the entire geocoding cache? This action cannot be undone.')) { + return; + } + + setIsLoading(true); + try { + const response = await clearGeocodeCache(); + const data = await response.json(); + + if (!response.ok) { + throw new Error(data.message || 'An unknown error occurred.'); + } + + toast.success(data.message); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Failed to clear cache.'; + toast.error(errorMessage); + } finally { + setIsLoading(false); + } + }; + + return ( +
+

Geocoding Service

+

+ The application uses a Redis cache to store geocoding results and reduce API calls. You can manually clear this cache if you suspect the data is stale. +

+ +
+ ); +}; + export const SystemCheck: React.FC = () => { const [checks, setChecks] = useState(initialChecks); const [isRunning, setIsRunning] = useState(false); @@ -319,6 +367,7 @@ export const SystemCheck: React.FC = () => { ) : 'Trigger Failing Job'} + ); diff --git a/src/routes/admin.ts b/src/routes/admin.ts index 49e050be..e5cac7db 100644 --- a/src/routes/admin.ts +++ b/src/routes/admin.ts @@ -7,6 +7,7 @@ import multer from 'multer'; import * as db from '../services/db'; import { logger } from '../services/logger.server'; import { UserProfile } from '../types'; +import { clearGeocodeCache } from '../services/geocodingService.server'; // --- Bull Board (Job Queue UI) Imports --- import { createBullBoard } from '@bull-board/api'; @@ -274,4 +275,21 @@ router.post('/trigger/failing-job', async (req: Request, res: Response, next: Ne } }); +/** + * POST /api/admin/system/clear-geocode-cache - Clears the Redis cache for geocoded addresses. + * Requires admin privileges. + */ +router.post('/system/clear-geocode-cache', async (req: Request, res: Response, next: NextFunction) => { + const adminUser = req.user as UserProfile; + logger.info(`[Admin] Manual trigger for geocode cache clear received from user: ${adminUser.user_id}`); + + try { + const keysDeleted = await clearGeocodeCache(); + res.status(200).json({ message: `Successfully cleared the geocode cache. ${keysDeleted} keys were removed.` }); + } catch (error) { + logger.error('[Admin] Failed to clear geocode cache.', { error }); + next(error); + } +}); + export default router; \ No newline at end of file diff --git a/src/routes/system.ts b/src/routes/system.ts index 5fb29606..57ef0482 100644 --- a/src/routes/system.ts +++ b/src/routes/system.ts @@ -50,7 +50,7 @@ router.post('/geocode', async (req: Request, res: Response) => { const coordinates = await geocodeAddress(address); - if (!coordinates) { + if (!coordinates) { // This check remains, but now it only fails if BOTH services fail. return res.status(404).json({ message: 'Could not geocode the provided address.' }); } diff --git a/src/services/apiClient.ts b/src/services/apiClient.ts index e648cd7e..2ad10671 100644 --- a/src/services/apiClient.ts +++ b/src/services/apiClient.ts @@ -690,6 +690,15 @@ export const triggerFailingJob = async (tokenOverride?: string): Promise => { + return apiFetch(`/admin/system/clear-geocode-cache`, { method: 'POST' }, tokenOverride); +}; + export async function registerUser( email: string, password: string, diff --git a/src/services/db/flyer.ts b/src/services/db/flyer.ts index 8128ba2f..62e2b208 100644 --- a/src/services/db/flyer.ts +++ b/src/services/db/flyer.ts @@ -183,15 +183,19 @@ export async function createFlyerAndItems( latitude: coords?.lat, longitude: coords?.lng, }; - const addressRes = await client.query<{ address_id: number }>( + // First, attempt to insert the address. If it already exists, do nothing. + await client.query( `INSERT INTO public.addresses (address_line_1, city, province_state, postal_code, country, latitude, longitude, location) VALUES ($1, $2, $3, $4, $5, $6, $7, ${coords ? `ST_SetSRID(ST_MakePoint(${coords.lng}, ${coords.lat}), 4326)` : 'NULL'}) - ON CONFLICT (address_line_1) DO UPDATE - SET latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude, location = EXCLUDED.location, updated_at = NOW() - WHERE public.addresses.latitude IS NULL AND EXCLUDED.latitude IS NOT NULL - RETURNING address_id`, + ON CONFLICT (address_line_1) DO NOTHING`, [addressData.address_line_1, addressData.city, addressData.province_state, addressData.postal_code, addressData.country, addressData.latitude, addressData.longitude] ); + + // Now, select the address_id, which is guaranteed to exist either from the insert or because it was already there. + const addressRes = await client.query<{ address_id: number }>( + 'SELECT address_id FROM public.addresses WHERE address_line_1 = $1', + [addressData.address_line_1] + ); const addressId = addressRes.rows[0].address_id; // 3. Upsert the `store_locations` link. diff --git a/src/services/geocodingService.server.ts b/src/services/geocodingService.server.ts index 1a2d1db4..2a0d59c1 100644 --- a/src/services/geocodingService.server.ts +++ b/src/services/geocodingService.server.ts @@ -1,39 +1,113 @@ // src/services/geocodingService.server.ts import { logger } from './logger.server'; +import { geocodeWithNominatim } from './nominatimGeocodingService.server'; +import { connection as redis } from './queueService.server'; // Import the configured Redis connection const apiKey = process.env.GOOGLE_MAPS_API_KEY; +const REDIS_CACHE_EXPIRATION_SECONDS = 60 * 60 * 24 * 30; // 30 days /** - * Geocodes a physical address into latitude and longitude coordinates using the Google Maps API. + * Geocodes a physical address into latitude and longitude coordinates. + * It first attempts to use the Google Maps API. If that is unavailable or fails, + * it falls back to the free OpenStreetMap Nominatim API. + * Results are cached in Redis to avoid redundant API calls. * @param address The address string to geocode. * @returns A promise that resolves to an object with latitude and longitude, or null if not found. */ export async function geocodeAddress(address: string): Promise<{ lat: number; lng: number } | null> { + const cacheKey = `geocode:${address}`; + + // 1. Check Redis cache first. + try { + const cachedResult = await redis.get(cacheKey); + if (cachedResult) { + logger.info(`[GeocodingService] Redis cache hit for address: "${address}"`); + return JSON.parse(cachedResult); + } + } catch (error) { + logger.error('[GeocodingService] Redis GET command failed. Proceeding without cache.', { error }); + } + + logger.info(`[GeocodingService] Redis cache miss for address: "${address}". Fetching from API.`); + + // Helper function to set cache and return result + const setCacheAndReturn = async (coordinates: { lat: number; lng: number } | null) => { + if (coordinates) { + try { + await redis.set(cacheKey, JSON.stringify(coordinates), 'EX', REDIS_CACHE_EXPIRATION_SECONDS); + logger.info(`[GeocodingService] Successfully cached result for address: "${address}"`); + } catch (error) { + logger.error('[GeocodingService] Redis SET command failed. Result will not be cached.', { error }); + } + } + return coordinates; + }; + if (!apiKey) { logger.warn('[GeocodingService] GOOGLE_MAPS_API_KEY is not set. Geocoding is disabled.'); - return null; + // If Google API key is not set, immediately fall back to Nominatim. + logger.info('[GeocodingService] Falling back to Nominatim due to missing Google API key.'); + const result = await geocodeWithNominatim(address); + return setCacheAndReturn(result); } const url = `https://maps.googleapis.com/maps/api/geocode/json?address=${encodeURIComponent(address)}&key=${apiKey}`; try { + logger.info(`[GeocodingService] Attempting geocoding with Google for address: "${address}"`); const response = await fetch(url); const data = await response.json(); if (data.status !== 'OK' || !data.results || data.results.length === 0) { - logger.warn('[GeocodingService] Geocoding failed or returned no results.', { status: data.status, address }); - return null; + logger.warn('[GeocodingService] Geocoding with Google failed or returned no results.', { status: data.status, address }); + // Fallback to Nominatim on Google API failure. + logger.info('[GeocodingService] Falling back to Nominatim due to Google API failure.'); + const result = await geocodeWithNominatim(address); + return setCacheAndReturn(result); } const location = data.results[0].geometry.location; logger.info(`[GeocodingService] Successfully geocoded address: "${address}"`, { location }); - return { + const coordinates = { lat: location.lat, lng: location.lng, }; + return setCacheAndReturn(coordinates); } catch (error) { logger.error('[GeocodingService] An error occurred while calling the Google Maps Geocoding API.', { error }); - // We return null instead of throwing to prevent a single geocoding failure from blocking a user profile save. - return null; + // Fallback to Nominatim on network or other unexpected errors. + logger.info('[GeocodingService] Falling back to Nominatim due to Google API error.'); + const result = await geocodeWithNominatim(address); + return setCacheAndReturn(result); } +} + +/** + * Clears all geocoding entries from the Redis cache. + * This function iterates through all keys matching the 'geocode:*' pattern and deletes them. + * It uses the SCAN command to avoid blocking the Redis server. + * @returns The number of keys that were deleted. + */ +export async function clearGeocodeCache(): Promise { + let cursor = '0'; + let keysDeleted = 0; + const pattern = 'geocode:*'; + + logger.info('[GeocodingService] Starting to clear geocode cache...'); + + do { + // Scan for keys matching the pattern in batches of 100. + const [newCursor, keys] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100); + + if (keys.length > 0) { + const count = await redis.del(keys); + keysDeleted += count; + logger.debug(`[GeocodingService] Deleted ${count} keys from cache.`); + } + + cursor = newCursor; + } while (cursor !== '0'); + + logger.info(`[GeocodingService] Finished clearing geocode cache. Total keys deleted: ${keysDeleted}`); + return keysDeleted; } \ No newline at end of file diff --git a/src/services/nominatimGeocodingService.server.ts b/src/services/nominatimGeocodingService.server.ts new file mode 100644 index 00000000..08312352 --- /dev/null +++ b/src/services/nominatimGeocodingService.server.ts @@ -0,0 +1,38 @@ +// src/services/nominatimGeocodingService.server.ts +import { logger } from './logger.server'; + +/** + * Geocodes a physical address using the public OpenStreetMap Nominatim API. + * This serves as a free fallback to the Google Maps Geocoding API. + * @param address The address string to geocode. + * @returns A promise that resolves to an object with latitude and longitude, or null if not found. + */ +export async function geocodeWithNominatim(address: string): Promise<{ lat: number; lng: number } | null> { + // Nominatim requires a specific User-Agent header. + const headers = new Headers({ + 'User-Agent': 'FlyerCrawler/1.0 (flyer-crawler.projectium.com)', + }); + + const url = `https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(address)}&format=json&limit=1`; + + try { + logger.info(`[NominatimService] Attempting geocoding for address: "${address}"`); + const response = await fetch(url, { headers }); + const data = await response.json(); + + if (!Array.isArray(data) || data.length === 0) { + logger.warn('[NominatimService] Geocoding failed or returned no results.', { address }); + return null; + } + + const location = data[0]; + logger.info(`[NominatimService] Successfully geocoded address: "${address}"`, { location }); + return { + lat: parseFloat(location.lat), + lng: parseFloat(location.lon), + }; + } catch (error) { + logger.error('[NominatimService] An error occurred while calling the Nominatim API.', { error }); + return null; + } +} \ No newline at end of file