Refactor database environment variable usage across workflows and application code

- Updated Gitea workflows to standardize on `DB_NAME` instead of `DB_DATABASE` for database name references. - Modified deployment, backup, reset, and restore workflows to ensure consistent environment variable usage. - Removed dotenv dependency and preload script, transitioning to environment variable management directly in scripts. - Adjusted application code to utilize `DB_NAME` for database connections and logging. - Enhanced README to reflect changes in environment variable configuration and usage. - Cleaned up package.json scripts to remove unnecessary preload references.
2025-12-04 18:02:38 -08:00
parent 80d2b1ffe6
commit 9d552b7456
22 changed files with 127 additions and 176 deletions
--- a/.gitea/workflows/deploy.yml
+++ b/.gitea/workflows/deploy.yml
@@ -85,8 +85,7 @@ jobs:
          DB_PORT: ${{ secrets.DB_PORT }}
          DB_USER: ${{ secrets.DB_USER }}
          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
-          DB_DATABASE: "flyer-crawler-test"
-          DB_NAME: "flyer-crawler-test"
+          DB_NAME: "flyer-crawler-test" # Explicitly set for tests

          # --- Redis credentials for the test suite ---
          REDIS_URL: "redis://localhost:6379"
@@ -103,7 +102,7 @@ jobs:

        run: |
          # Fail-fast check to ensure secrets are configured in Gitea for testing.
-          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ] || [ -z "$GEMINI_API_KEY" ] || [ -z "$REDIS_PASSWORD" ]; then
+          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ] || [ -z "$GEMINI_API_KEY" ] || [ -z "$REDIS_PASSWORD_TEST" ]; then
            echo "ERROR: One or more test secrets (DB_*, GEMINI_API_KEY, REDIS_PASSWORD_TEST) are not set in Gitea repository secrets."
            exit 1
          fi
@@ -207,11 +206,11 @@ jobs:
          DB_HOST: ${{ secrets.DB_HOST }}
          DB_PORT: ${{ secrets.DB_PORT }}
          DB_USER: ${{ secrets.DB_USER }}
-          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
-          DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }} # Assumes a secret for the production DB name.
+          DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # This is used by psql
+          DB_NAME: ${{ secrets.DB_DATABASE_PROD }} # This is used by the application
        run: |
          # Fail-fast check to ensure secrets are configured in Gitea.
-          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
+          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
            echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_DATABASE_PROD) are not set in Gitea repository settings."
            exit 1
          fi
@@ -226,7 +225,7 @@ jobs:
          # The `psql` command requires PGPASSWORD to be set.
          # `\t` sets tuples-only mode and `\A` unaligns output to get just the raw value.
          # The `|| echo "none"` ensures the command doesn't fail if the table or row doesn't exist yet.          
-          DEPLOYED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A || echo "none")
+          DEPLOYED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A || echo "none")
          echo "Deployed DB Schema Hash: $DEPLOYED_HASH"

          # Check if the hash is "none" (command failed) OR if it's an empty string (table exists but is empty).
@@ -272,26 +271,51 @@ jobs:
          mkdir -p "$APP_PATH/flyer-images/icons" "$APP_PATH/flyer-images/archive" # Ensure all required subdirectories exist

          # 1. Copy the backend source code and project files first.
-          # CRITICAL: We exclude '.env', 'node_modules', '.git', 'dist', and now 'flyer-images' to protect user content.
-          rsync -avz --delete --exclude '.env' --exclude '.env.test' --exclude 'node_modules' --exclude '.git' --exclude 'dist' --exclude 'flyer-images' ./ "$APP_PATH/"
+          # CRITICAL: We exclude 'node_modules', '.git', 'dist', and 'flyer-images' to protect user content and avoid overwriting build artifacts.
+          rsync -avz --delete --exclude 'node_modules' --exclude '.git' --exclude 'dist' --exclude 'flyer-images' ./ "$APP_PATH/"

          # 2. Copy the built frontend assets into the same directory.
          # This will correctly place index.html and the assets/ folder in the webroot.
-          rsync -avz --exclude '.env.local' dist/ "/var/www/flyer-crawler.projectium.com"
+          rsync -avz dist/ "/var/www/flyer-crawler.projectium.com"
          echo "Application deployment complete."

      - name: Install Backend Dependencies and Restart Server
        env:
-          # These credentials are required for the psql command at the end of this step.
+          # --- Production Secrets Injection ---
+          # These secrets are injected into the environment for the PM2 process.
+          # Your Node.js application will read these directly from `process.env`.
+
+          # Database Credentials
          DB_HOST: ${{ secrets.DB_HOST }}
          DB_PORT: ${{ secrets.DB_PORT }}
          DB_USER: ${{ secrets.DB_USER }}
-          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
-          DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }}
+          DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
+          DB_NAME: ${{ secrets.DB_DATABASE_PROD }} # Standardize on the existing prod secret name
+
+          # Redis Credentials
+          REDIS_URL: "redis://localhost:6379" # Assuming Redis is on the same server
+          REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD_PROD }}
+
+          # Application Secrets
+          FRONTEND_URL: "https://flyer-crawler.projectium.com"
+          JWT_SECRET: ${{ secrets.JWT_SECRET }}
+          GEMINI_API_KEY: ${{ secrets.VITE_GOOGLE_GENAI_API_KEY }} # Re-use the same secret for the server
+          GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
+          WORKER_CONCURRENCY: ${{ secrets.WORKER_CONCURRENCY }}
+
+          # SMTP (email)
+          SMTP_HOST: "localhost"
+          SMTP_PORT: "1025"
+          SMTP_SECURE: "false"
+          SMTP_USER: ""
+          SMTP_PASS: ""
+          SMTP_FROM_EMAIL: "noreply@flyer-crawler.projectium.com"
+
+
        run: |
          # Fail-fast check to ensure secrets are configured in Gitea.
-          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
-            echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_DATABASE_PROD) are not set in Gitea repository settings."
+          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
+            echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_NAME_PROD) are not set in Gitea repository settings."
            exit 1
          fi

@@ -307,13 +331,13 @@ jobs:
          # After a successful deployment, update the schema hash in the database.
          # This ensures the next deployment will compare against this new state.
          echo "Updating schema hash in production database..."
-          CURRENT_HASH=$(cat sql/master_schema_rollup.sql | dos2unix | sha256sum | awk '{ print $1 }')          
-          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c \
+          CURRENT_HASH=$(cat sql/master_schema_rollup.sql | dos2unix | sha256sum | awk '{ print $1 }')
+          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c \
          "INSERT INTO public.schema_info (id, schema_hash, deployed_at) VALUES (1, '$CURRENT_HASH', NOW())
           ON CONFLICT (id) DO UPDATE SET schema_hash = EXCLUDED.schema_hash, deployed_at = NOW();"
          
          # Verify the hash was updated
-          UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
+          UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
          if [ "$CURRENT_HASH" = "$UPDATED_HASH" ]; then
            echo "✅ Schema hash successfully updated in the database to: $UPDATED_HASH"
          else
@@ -323,8 +347,7 @@ jobs:
      - name: Show PM2 Environment for Production
        run: |
          echo "--- Displaying recent PM2 logs for flyer-crawler-api ---"
-          # After a reload, the server restarts. We'll show the last 20 lines of the log
-          # to see the startup messages, which include the environment variables loaded from the .env file.
+          # After a reload, the server restarts. We'll show the last 20 lines of the log to see the startup messages.
          sleep 5 # Wait a few seconds for the app to start and log its output.
          pm2 describe flyer-crawler-api || echo "Could not find pm2 process."
          pm2 logs flyer-crawler-api --lines 20 --nostream || echo "Could not find pm2 process."
--- a/.gitea/workflows/manual-db-backup.yml
+++ b/.gitea/workflows/manual-db-backup.yml
@@ -22,12 +22,12 @@ jobs:
      DB_PORT: ${{ secrets.DB_PORT }}
      DB_USER: ${{ secrets.DB_USER }}
      DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
-      DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }}
+      DB_NAME: ${{ secrets.DB_NAME_PROD }}

    steps:
      - name: Validate Secrets
        run: |
-          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
+          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
            echo "ERROR: One or more production database secrets are not set in Gitea repository settings."
            exit 1
          fi
@@ -51,7 +51,7 @@ jobs:

          # Use pg_dump to create a plain-text SQL dump, then pipe it to gzip for compression.
          # This is more efficient than creating a large uncompressed file first.
-          PGPASSWORD="$DB_PASSWORD" pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" --clean --if-exists | gzip > "$BACKUP_FILENAME"
+          PGPASSWORD="$DB_PASSWORD" pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" --clean --if-exists | gzip > "$BACKUP_FILENAME"

          echo "✅ Database backup created successfully."
          echo "backup_filename=$BACKUP_FILENAME" >> $GITEA_ENV
--- a/.gitea/workflows/manual-db-reset.yml
+++ b/.gitea/workflows/manual-db-reset.yml
@@ -25,8 +25,8 @@ jobs:
      DB_HOST: ${{ secrets.DB_HOST }}
      DB_PORT: ${{ secrets.DB_PORT }}
      DB_USER: ${{ secrets.DB_USER }}
-      DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
-      DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }} # Assumes a secret for the production DB name.
+      DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
+      DB_NAME: ${{ secrets.DB_DATABASE_PROD }} # Used by the application

    steps:
      - name: Checkout Code
@@ -35,8 +35,8 @@ jobs:
      - name: Validate Secrets
        run: |
          # Fail-fast check to ensure secrets are configured in Gitea.
-          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
-            echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_DATABASE_PROD) are not set in Gitea repository settings."
+          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
+            echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_NAME_PROD) are not set in Gitea repository settings."
            exit 1
          fi
          echo "✅ All required database secrets are present."
@@ -66,7 +66,7 @@ jobs:
          echo "Attempting to back up data for user: $USER_EMAIL"

          # Get the user_id for the specified email.
-          USER_ID=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT user_id FROM public.users WHERE email = '$USER_EMAIL';" -t -A)
+          USER_ID=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT user_id FROM public.users WHERE email = '$USER_EMAIL';" -t -A)

          if [ -z "$USER_ID" ]; then
            echo "WARNING: User with email '$USER_EMAIL' not found. Skipping backup."
@@ -76,7 +76,7 @@ jobs:
            # Use pg_dump to create a data-only dump for all tables that have a direct or indirect
            # relationship to the user_id. This is a robust way to capture all related data.
            # We use --data-only and --column-inserts for maximum compatibility.
-            PGPASSWORD="$DB_PASSWORD" pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" \
+            PGPASSWORD="$DB_PASSWORD" pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
              --data-only --column-inserts \
              --table="public.users" --table="public.profiles" --table="public.shopping_lists" \
              --table="public.shopping_list_items" --table="public.user_watched_items" \
@@ -99,20 +99,20 @@ jobs:
      - name: Step 2 - Drop All Tables from Production DB
        run: |
          echo "Executing drop_tables.sql against the PRODUCTION database..."
-          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f sql/drop_tables.sql
+          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f sql/drop_tables.sql
          echo "✅ All tables dropped successfully."

      - name: Step 3 - Rebuild Schema from Master Rollup
        run: |
          echo "Executing master_schema_rollup.sql against the PRODUCTION database..."
-          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f sql/master_schema_rollup.sql
+          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f sql/master_schema_rollup.sql
          echo "✅ Schema rebuilt successfully."

      - name: Step 4 - (Optional) Restore Specific User Data
        if: ${{ gitea.event.inputs.user_email != '' && env.NO_USER_BACKUP != 'true' }}
        run: |
          echo "Restoring user data from filtered_backup.sql..."          
-          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f filtered_backup.sql
+          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f filtered_backup.sql
          echo "✅ User data restored successfully."

      - name: Step 5 - Update Schema Info Table
@@ -123,12 +123,12 @@ jobs:
          echo "New Schema Hash: $CURRENT_HASH"

          # Insert the new hash into the freshly created schema_info table.
-          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c \
+          PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c \
          "INSERT INTO public.schema_info (id, schema_hash, deployed_at) VALUES (1, '$CURRENT_HASH', NOW())
           ON CONFLICT (id) DO UPDATE SET schema_hash = EXCLUDED.schema_hash, deployed_at = NOW();"

          # Verify the hash was updated
-          UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
+          UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
          if [ "$CURRENT_HASH" = "$UPDATED_HASH" ]; then
            echo "✅ Schema hash successfully set in the database to: $UPDATED_HASH"
          else
--- a/.gitea/workflows/manual-db-restore.yml
+++ b/.gitea/workflows/manual-db-restore.yml
@@ -25,13 +25,13 @@ jobs:
      DB_PORT: ${{ secrets.DB_PORT }}
      DB_USER: ${{ secrets.DB_USER }}
      DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
-      DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }}
+      DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
      BACKUP_DIR: "/var/www/backups" # Define a dedicated directory for backups

    steps:
      - name: Validate Secrets and Inputs
        run: |
-          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
+          if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
            echo "ERROR: One or more production database secrets are not set in Gitea repository settings."
            exit 1
          fi
@@ -66,10 +66,10 @@ jobs:
          echo "Dropping and recreating the production database..."
          # Connect as the superuser (postgres) to drop the database.
          # First, terminate all active connections to the database.
-          sudo -u postgres psql -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${DB_DATABASE}';"
+          sudo -u postgres psql -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${DB_NAME}';"
          # Now, drop and recreate it.
-          sudo -u postgres psql -c "DROP DATABASE IF EXISTS \"${DB_DATABASE}\";"
-          sudo -u postgres psql -c "CREATE DATABASE \"${DB_DATABASE}\" WITH OWNER = ${DB_USER};"
+          sudo -u postgres psql -c "DROP DATABASE IF EXISTS \"${DB_NAME}\";"
+          sudo -u postgres psql -c "CREATE DATABASE \"${DB_NAME}\" WITH OWNER = ${DB_USER};"
          echo "✅ Database dropped and recreated successfully."

      - name: Step 3 - Restore Database from Backup
@@ -84,7 +84,7 @@ jobs:

          # Uncompress the gzipped file and pipe the SQL commands directly into psql.
          # This is efficient as it doesn't require an intermediate uncompressed file.
-          gunzip < "$BACKUP_FILE_PATH" | PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE"
+          gunzip < "$BACKUP_FILE_PATH" | PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME"

          echo "✅ Database restore completed successfully."