add geocoding fallback nominatim

src/pages/admin/components/SystemCheck.tsx

@@ -1,7 +1,7 @@
 // src/pages/admin/components/SystemCheck.tsx
 import React, { useState, useEffect, useCallback } from 'react';
 import toast from 'react-hot-toast';
-import { loginUser, pingBackend, checkDbSchema, checkStorage, checkDbPoolHealth, checkPm2Status, checkRedisHealth, triggerFailingJob } from '../../../services/apiClient';
+import { loginUser, pingBackend, checkDbSchema, checkStorage, checkDbPoolHealth, checkPm2Status, checkRedisHealth, triggerFailingJob, clearGeocodeCache } from '../../../services/apiClient';
 import { ShieldCheckIcon } from '../../../components/icons/ShieldCheckIcon';
 import { LoadingSpinner } from '../../../components/LoadingSpinner';
 import { CheckCircleIcon } from '../../../components/icons/CheckCircleIcon';
@@ -41,6 +41,54 @@ const initialChecks: Check[] = [
     { id: CheckID.GEMINI, name: 'Gemini API Key', description: 'Verifies the GEMINI_API_KEY is set for AI features.', status: 'idle', message: '' },
 ];
 
+const GeocodeCacheManager: React.FC = () => {
+  const [isLoading, setIsLoading] = useState(false);
+
+  const handleClearCache = async () => {
+    if (!window.confirm('Are you sure you want to clear the entire geocoding cache? This action cannot be undone.')) {
+      return;
+    }
+
+    setIsLoading(true);
+    try {
+      const response = await clearGeocodeCache();
+      const data = await response.json();
+
+      if (!response.ok) {
+        throw new Error(data.message || 'An unknown error occurred.');
+      }
+
+      toast.success(data.message);
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Failed to clear cache.';
+      toast.error(errorMessage);
+    } finally {
+      setIsLoading(false);
+    }
+  };
+
+  return (
+    <div className="mt-4">
+      <h4 className="font-medium text-gray-800 dark:text-gray-200">Geocoding Service</h4>
+      <p className="text-sm text-gray-500 dark:text-gray-400 mt-1">
+        The application uses a Redis cache to store geocoding results and reduce API calls. You can manually clear this cache if you suspect the data is stale.
+      </p>
+      <button
+        onClick={handleClearCache}
+        disabled={isLoading}
+        className="mt-3 inline-flex items-center justify-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-red-600 hover:bg-red-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-red-500 disabled:bg-red-400"
+      >
+        {isLoading ? (
+          <>
+            <div className="w-5 h-5 mr-2"><LoadingSpinner /></div>
+            <span>Clearing...</span>
+          </>
+        ) : ( 'Clear Geocode Cache' )}
+      </button>
+    </div>
+  );
+};
+
 export const SystemCheck: React.FC = () => {
     const [checks, setChecks] = useState<Check[]>(initialChecks);
     const [isRunning, setIsRunning] = useState(false);
@@ -319,6 +367,7 @@ export const SystemCheck: React.FC = () => {
                         ) : 'Trigger Failing Job'}
                     </button>
                 </div>
+                <GeocodeCacheManager />
             </div>
         </div>
     );

src/routes/admin.ts

@@ -7,6 +7,7 @@ import multer from 'multer';
 import * as db from '../services/db';
 import { logger } from '../services/logger.server';
 import { UserProfile } from '../types';
+import { clearGeocodeCache } from '../services/geocodingService.server';
 
 // --- Bull Board (Job Queue UI) Imports ---
 import { createBullBoard } from '@bull-board/api';
@@ -274,4 +275,21 @@ router.post('/trigger/failing-job', async (req: Request, res: Response, next: Ne
     }
 });
 
+/**
+ * POST /api/admin/system/clear-geocode-cache - Clears the Redis cache for geocoded addresses.
+ * Requires admin privileges.
+ */
+router.post('/system/clear-geocode-cache', async (req: Request, res: Response, next: NextFunction) => {
+    const adminUser = req.user as UserProfile;
+    logger.info(`[Admin] Manual trigger for geocode cache clear received from user: ${adminUser.user_id}`);
+
+    try {
+        const keysDeleted = await clearGeocodeCache();
+        res.status(200).json({ message: `Successfully cleared the geocode cache. ${keysDeleted} keys were removed.` });
+    } catch (error) {
+        logger.error('[Admin] Failed to clear geocode cache.', { error });
+        next(error);
+    }
+});
+
 export default router;

src/routes/system.ts

@@ -50,7 +50,7 @@ router.post('/geocode', async (req: Request, res: Response) => {
 
     const coordinates = await geocodeAddress(address);
 
-    if (!coordinates) {
+    if (!coordinates) { // This check remains, but now it only fails if BOTH services fail.
         return res.status(404).json({ message: 'Could not geocode the provided address.' });
     }
 

src/services/apiClient.ts

@@ -690,6 +690,15 @@ export const triggerFailingJob = async (tokenOverride?: string): Promise<Respons
     return apiFetch(`/admin/trigger/failing-job`, { method: 'POST' }, tokenOverride);
 };
 
+/**
+ * Triggers the clearing of the geocoding cache on the server.
+ * Requires admin privileges.
+ * @param tokenOverride Optional token for testing.
+ */
+export const clearGeocodeCache = async (tokenOverride?: string): Promise<Response> => {
+    return apiFetch(`/admin/system/clear-geocode-cache`, { method: 'POST' }, tokenOverride);
+};
+
 export async function registerUser(
     email: string,
     password: string,

src/services/db/flyer.ts

@@ -183,15 +183,19 @@ export async function createFlyerAndItems(
         latitude: coords?.lat,
         longitude: coords?.lng,
       };
-      const addressRes = await client.query<{ address_id: number }>(
+      // First, attempt to insert the address. If it already exists, do nothing.
+      await client.query(
         `INSERT INTO public.addresses (address_line_1, city, province_state, postal_code, country, latitude, longitude, location)
          VALUES ($1, $2, $3, $4, $5, $6, $7, ${coords ? `ST_SetSRID(ST_MakePoint(${coords.lng}, ${coords.lat}), 4326)` : 'NULL'})
-         ON CONFLICT (address_line_1) DO UPDATE 
-           SET latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude, location = EXCLUDED.location, updated_at = NOW()
-           WHERE public.addresses.latitude IS NULL AND EXCLUDED.latitude IS NOT NULL
-         RETURNING address_id`,
+         ON CONFLICT (address_line_1) DO NOTHING`,
         [addressData.address_line_1, addressData.city, addressData.province_state, addressData.postal_code, addressData.country, addressData.latitude, addressData.longitude]
       );
+
+      // Now, select the address_id, which is guaranteed to exist either from the insert or because it was already there.
+      const addressRes = await client.query<{ address_id: number }>(
+        'SELECT address_id FROM public.addresses WHERE address_line_1 = $1',
+        [addressData.address_line_1]
+      );
       const addressId = addressRes.rows[0].address_id;
 
       // 3. Upsert the `store_locations` link.

src/services/geocodingService.server.ts

@@ -1,39 +1,113 @@
 // src/services/geocodingService.server.ts
 import { logger } from './logger.server';
+import { geocodeWithNominatim } from './nominatimGeocodingService.server';
+import { connection as redis } from './queueService.server'; // Import the configured Redis connection
 
 const apiKey = process.env.GOOGLE_MAPS_API_KEY;
+const REDIS_CACHE_EXPIRATION_SECONDS = 60 * 60 * 24 * 30; // 30 days
 
 /**
- * Geocodes a physical address into latitude and longitude coordinates using the Google Maps API.
+ * Geocodes a physical address into latitude and longitude coordinates.
+ * It first attempts to use the Google Maps API. If that is unavailable or fails,
+ * it falls back to the free OpenStreetMap Nominatim API.
+ * Results are cached in Redis to avoid redundant API calls.
  * @param address The address string to geocode.
  * @returns A promise that resolves to an object with latitude and longitude, or null if not found.
  */
 export async function geocodeAddress(address: string): Promise<{ lat: number; lng: number } | null> {
+  const cacheKey = `geocode:${address}`;
+
+  // 1. Check Redis cache first.
+  try {
+    const cachedResult = await redis.get(cacheKey);
+    if (cachedResult) {
+      logger.info(`[GeocodingService] Redis cache hit for address: "${address}"`);
+      return JSON.parse(cachedResult);
+    }
+  } catch (error) {
+    logger.error('[GeocodingService] Redis GET command failed. Proceeding without cache.', { error });
+  }
+
+  logger.info(`[GeocodingService] Redis cache miss for address: "${address}". Fetching from API.`);
+
+  // Helper function to set cache and return result
+  const setCacheAndReturn = async (coordinates: { lat: number; lng: number } | null) => {
+    if (coordinates) {
+      try {
+        await redis.set(cacheKey, JSON.stringify(coordinates), 'EX', REDIS_CACHE_EXPIRATION_SECONDS);
+        logger.info(`[GeocodingService] Successfully cached result for address: "${address}"`);
+      } catch (error) {
+        logger.error('[GeocodingService] Redis SET command failed. Result will not be cached.', { error });
+      }
+    }
+    return coordinates;
+  };
+
   if (!apiKey) {
     logger.warn('[GeocodingService] GOOGLE_MAPS_API_KEY is not set. Geocoding is disabled.');
-    return null;
+    // If Google API key is not set, immediately fall back to Nominatim.
+    logger.info('[GeocodingService] Falling back to Nominatim due to missing Google API key.');
+    const result = await geocodeWithNominatim(address);
+    return setCacheAndReturn(result);
   }
 
   const url = `https://maps.googleapis.com/maps/api/geocode/json?address=${encodeURIComponent(address)}&key=${apiKey}`;
 
   try {
+    logger.info(`[GeocodingService] Attempting geocoding with Google for address: "${address}"`);
     const response = await fetch(url);
     const data = await response.json();
 
     if (data.status !== 'OK' || !data.results || data.results.length === 0) {
-      logger.warn('[GeocodingService] Geocoding failed or returned no results.', { status: data.status, address });
-      return null;
+      logger.warn('[GeocodingService] Geocoding with Google failed or returned no results.', { status: data.status, address });
+      // Fallback to Nominatim on Google API failure.
+      logger.info('[GeocodingService] Falling back to Nominatim due to Google API failure.');
+      const result = await geocodeWithNominatim(address);
+      return setCacheAndReturn(result);
     }
 
     const location = data.results[0].geometry.location;
     logger.info(`[GeocodingService] Successfully geocoded address: "${address}"`, { location });
-    return {
+    const coordinates = {
       lat: location.lat,
       lng: location.lng,
     };
+    return setCacheAndReturn(coordinates);
   } catch (error) {
     logger.error('[GeocodingService] An error occurred while calling the Google Maps Geocoding API.', { error });
-    // We return null instead of throwing to prevent a single geocoding failure from blocking a user profile save.
-    return null;
+    // Fallback to Nominatim on network or other unexpected errors.
+    logger.info('[GeocodingService] Falling back to Nominatim due to Google API error.');
+    const result = await geocodeWithNominatim(address);
+    return setCacheAndReturn(result);
   }
+}
+
+/**
+ * Clears all geocoding entries from the Redis cache.
+ * This function iterates through all keys matching the 'geocode:*' pattern and deletes them.
+ * It uses the SCAN command to avoid blocking the Redis server.
+ * @returns The number of keys that were deleted.
+ */
+export async function clearGeocodeCache(): Promise<number> {
+  let cursor = '0';
+  let keysDeleted = 0;
+  const pattern = 'geocode:*';
+
+  logger.info('[GeocodingService] Starting to clear geocode cache...');
+
+  do {
+    // Scan for keys matching the pattern in batches of 100.
+    const [newCursor, keys] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100);
+    
+    if (keys.length > 0) {
+      const count = await redis.del(keys);
+      keysDeleted += count;
+      logger.debug(`[GeocodingService] Deleted ${count} keys from cache.`);
+    }
+    
+    cursor = newCursor;
+  } while (cursor !== '0');
+
+  logger.info(`[GeocodingService] Finished clearing geocode cache. Total keys deleted: ${keysDeleted}`);
+  return keysDeleted;
 }

src/services/nominatimGeocodingService.server.ts

@@ -0,0 +1,38 @@
+// src/services/nominatimGeocodingService.server.ts
+import { logger } from './logger.server';
+
+/**
+ * Geocodes a physical address using the public OpenStreetMap Nominatim API.
+ * This serves as a free fallback to the Google Maps Geocoding API.
+ * @param address The address string to geocode.
+ * @returns A promise that resolves to an object with latitude and longitude, or null if not found.
+ */
+export async function geocodeWithNominatim(address: string): Promise<{ lat: number; lng: number } | null> {
+  // Nominatim requires a specific User-Agent header.
+  const headers = new Headers({
+    'User-Agent': 'FlyerCrawler/1.0 (flyer-crawler.projectium.com)',
+  });
+
+  const url = `https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(address)}&format=json&limit=1`;
+
+  try {
+    logger.info(`[NominatimService] Attempting geocoding for address: "${address}"`);
+    const response = await fetch(url, { headers });
+    const data = await response.json();
+
+    if (!Array.isArray(data) || data.length === 0) {
+      logger.warn('[NominatimService] Geocoding failed or returned no results.', { address });
+      return null;
+    }
+
+    const location = data[0];
+    logger.info(`[NominatimService] Successfully geocoded address: "${address}"`, { location });
+    return {
+      lat: parseFloat(location.lat),
+      lng: parseFloat(location.lon),
+    };
+  } catch (error) {
+    logger.error('[NominatimService] An error occurred while calling the Nominatim API.', { error });
+    return null;
+  }
+}