torbo/flyer-crawler.projectium.com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor database environment variable usage across workflows and application code
All checks were successful
Deploy to Web Server flyer-crawler.projectium.com / deploy (push) Successful in 3m53s
Details

Browse Source 
- Updated Gitea workflows to standardize on `DB_NAME` instead of `DB_DATABASE` for database name references.
- Modified deployment, backup, reset, and restore workflows to ensure consistent environment variable usage.
- Removed dotenv dependency and preload script, transitioning to environment variable management directly in scripts.
- Adjusted application code to utilize `DB_NAME` for database connections and logging.
- Enhanced README to reflect changes in environment variable configuration and usage.
- Cleaned up package.json scripts to remove unnecessary preload references.
This commit is contained in:
Torben Sorensen
2025-12-04 18:02:38 -08:00
parent 80d2b1ffe6
commit 9d552b7456
22 changed files with 127 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
.gitea/workflows/deploy.yml
View File

@@ -85,8 +85,7 @@ jobs:
DB_PORT: ${{ secrets.DB_PORT }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_DATABASE: "flyer-crawler-test"
DB_NAME: "flyer-crawler-test"
DB_NAME: "flyer-crawler-test" # Explicitly set for tests
# --- Redis credentials for the test suite ---
REDIS_URL: "redis://localhost:6379"
@@ -103,7 +102,7 @@ jobs:
run: |
# Fail-fast check to ensure secrets are configured in Gitea for testing.
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ] || [ -z "$GEMINI_API_KEY" ] || [ -z "$REDIS_PASSWORD" ]; then
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ] || [ -z "$GEMINI_API_KEY" ] || [ -z "$REDIS_PASSWORD_TEST" ]; then
echo "ERROR: One or more test secrets (DB_*, GEMINI_API_KEY, REDIS_PASSWORD_TEST) are not set in Gitea repository secrets."
exit 1
fi
@@ -207,11 +206,11 @@ jobs:
DB_HOST: ${{ secrets.DB_HOST }}
DB_PORT: ${{ secrets.DB_PORT }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }} # Assumes a secret for the production DB name.
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # This is used by psql
DB_NAME: ${{ secrets.DB_DATABASE_PROD }} # This is used by the application
run: |
# Fail-fast check to ensure secrets are configured in Gitea.
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_DATABASE_PROD) are not set in Gitea repository settings."
exit 1
fi
@@ -226,7 +225,7 @@ jobs:
# The `psql` command requires PGPASSWORD to be set.
# `\t` sets tuples-only mode and `\A` unaligns output to get just the raw value.
# The `|| echo "none"` ensures the command doesn't fail if the table or row doesn't exist yet.
DEPLOYED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A || echo "none")
DEPLOYED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A || echo "none")
echo "Deployed DB Schema Hash: $DEPLOYED_HASH"
# Check if the hash is "none" (command failed) OR if it's an empty string (table exists but is empty).
@@ -272,26 +271,51 @@ jobs:
mkdir -p "$APP_PATH/flyer-images/icons" "$APP_PATH/flyer-images/archive" # Ensure all required subdirectories exist
# 1. Copy the backend source code and project files first.
# CRITICAL: We exclude '.env', 'node_modules', '.git', 'dist', and now 'flyer-images' to protect user content.
rsync -avz --delete --exclude '.env' --exclude '.env.test' --exclude 'node_modules' --exclude '.git' --exclude 'dist' --exclude 'flyer-images' ./ "$APP_PATH/"
# CRITICAL: We exclude 'node_modules', '.git', 'dist', and 'flyer-images' to protect user content and avoid overwriting build artifacts.
rsync -avz --delete --exclude 'node_modules' --exclude '.git' --exclude 'dist' --exclude 'flyer-images' ./ "$APP_PATH/"
# 2. Copy the built frontend assets into the same directory.
# This will correctly place index.html and the assets/ folder in the webroot.
rsync -avz --exclude '.env.local' dist/ "/var/www/flyer-crawler.projectium.com"
rsync -avz dist/ "/var/www/flyer-crawler.projectium.com"
echo "Application deployment complete."
- name: Install Backend Dependencies and Restart Server
env:
# These credentials are required for the psql command at the end of this step.
# --- Production Secrets Injection ---
# These secrets are injected into the environment for the PM2 process.
# Your Node.js application will read these directly from `process.env`.
# Database Credentials
DB_HOST: ${{ secrets.DB_HOST }}
DB_PORT: ${{ secrets.DB_PORT }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
DB_NAME: ${{ secrets.DB_DATABASE_PROD }} # Standardize on the existing prod secret name
# Redis Credentials
REDIS_URL: "redis://localhost:6379" # Assuming Redis is on the same server
REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD_PROD }}
# Application Secrets
FRONTEND_URL: "https://flyer-crawler.projectium.com"
JWT_SECRET: ${{ secrets.JWT_SECRET }}
GEMINI_API_KEY: ${{ secrets.VITE_GOOGLE_GENAI_API_KEY }} # Re-use the same secret for the server
GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
WORKER_CONCURRENCY: ${{ secrets.WORKER_CONCURRENCY }}
# SMTP (email)
SMTP_HOST: "localhost"
SMTP_PORT: "1025"
SMTP_SECURE: "false"
SMTP_USER: ""
SMTP_PASS: ""
SMTP_FROM_EMAIL: "noreply@flyer-crawler.projectium.com"
run: |
# Fail-fast check to ensure secrets are configured in Gitea.
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_DATABASE_PROD) are not set in Gitea repository settings."
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_NAME_PROD) are not set in Gitea repository settings."
exit 1
fi
@@ -308,12 +332,12 @@ jobs:
# This ensures the next deployment will compare against this new state.
echo "Updating schema hash in production database..."
CURRENT_HASH=$(cat sql/master_schema_rollup.sql | dos2unix | sha256sum | awk '{ print $1 }')
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c \
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c \
"INSERT INTO public.schema_info (id, schema_hash, deployed_at) VALUES (1, '$CURRENT_HASH', NOW())
ON CONFLICT (id) DO UPDATE SET schema_hash = EXCLUDED.schema_hash, deployed_at = NOW();"
# Verify the hash was updated
UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
if [ "$CURRENT_HASH" = "$UPDATED_HASH" ]; then
echo "✅ Schema hash successfully updated in the database to: $UPDATED_HASH"
else
@@ -323,8 +347,7 @@ jobs:
- name: Show PM2 Environment for Production
run: |
echo "--- Displaying recent PM2 logs for flyer-crawler-api ---"
# After a reload, the server restarts. We'll show the last 20 lines of the log
# to see the startup messages, which include the environment variables loaded from the .env file.
# After a reload, the server restarts. We'll show the last 20 lines of the log to see the startup messages.
sleep 5 # Wait a few seconds for the app to start and log its output.
pm2 describe flyer-crawler-api || echo "Could not find pm2 process."
pm2 logs flyer-crawler-api --lines 20 --nostream || echo "Could not find pm2 process."

6
.gitea/workflows/manual-db-backup.yml
View File

@@ -22,12 +22,12 @@ jobs:
DB_PORT: ${{ secrets.DB_PORT }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }}
DB_NAME: ${{ secrets.DB_NAME_PROD }}
steps:
- name: Validate Secrets
run: |
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
echo "ERROR: One or more production database secrets are not set in Gitea repository settings."
exit 1
fi
@@ -51,7 +51,7 @@ jobs:
# Use pg_dump to create a plain-text SQL dump, then pipe it to gzip for compression.
# This is more efficient than creating a large uncompressed file first.
PGPASSWORD="$DB_PASSWORD" pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" --clean --if-exists | gzip > "$BACKUP_FILENAME"
PGPASSWORD="$DB_PASSWORD" pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" --clean --if-exists | gzip > "$BACKUP_FILENAME"
echo "✅ Database backup created successfully."
echo "backup_filename=$BACKUP_FILENAME" >> $GITEA_ENV

22
.gitea/workflows/manual-db-reset.yml
View File

@@ -25,8 +25,8 @@ jobs:
DB_HOST: ${{ secrets.DB_HOST }}
DB_PORT: ${{ secrets.DB_PORT }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }} # Assumes a secret for the production DB name.
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
DB_NAME: ${{ secrets.DB_DATABASE_PROD }} # Used by the application
steps:
- name: Checkout Code
@@ -35,8 +35,8 @@ jobs:
- name: Validate Secrets
run: |
# Fail-fast check to ensure secrets are configured in Gitea.
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_DATABASE_PROD) are not set in Gitea repository settings."
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_NAME_PROD) are not set in Gitea repository settings."
exit 1
fi
echo "✅ All required database secrets are present."
@@ -66,7 +66,7 @@ jobs:
echo "Attempting to back up data for user: $USER_EMAIL"
# Get the user_id for the specified email.
USER_ID=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT user_id FROM public.users WHERE email = '$USER_EMAIL';" -t -A)
USER_ID=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT user_id FROM public.users WHERE email = '$USER_EMAIL';" -t -A)
if [ -z "$USER_ID" ]; then
echo "WARNING: User with email '$USER_EMAIL' not found. Skipping backup."
@@ -76,7 +76,7 @@ jobs:
# Use pg_dump to create a data-only dump for all tables that have a direct or indirect
# relationship to the user_id. This is a robust way to capture all related data.
# We use --data-only and --column-inserts for maximum compatibility.
PGPASSWORD="$DB_PASSWORD" pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" \
PGPASSWORD="$DB_PASSWORD" pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
--data-only --column-inserts \
--table="public.users" --table="public.profiles" --table="public.shopping_lists" \
--table="public.shopping_list_items" --table="public.user_watched_items" \
@@ -99,20 +99,20 @@ jobs:
- name: Step 2 - Drop All Tables from Production DB
run: |
echo "Executing drop_tables.sql against the PRODUCTION database..."
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f sql/drop_tables.sql
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f sql/drop_tables.sql
echo "✅ All tables dropped successfully."
- name: Step 3 - Rebuild Schema from Master Rollup
run: |
echo "Executing master_schema_rollup.sql against the PRODUCTION database..."
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f sql/master_schema_rollup.sql
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f sql/master_schema_rollup.sql
echo "✅ Schema rebuilt successfully."
- name: Step 4 - (Optional) Restore Specific User Data
if: ${{ gitea.event.inputs.user_email != '' && env.NO_USER_BACKUP != 'true' }}
run: |
echo "Restoring user data from filtered_backup.sql..."
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -f filtered_backup.sql
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f filtered_backup.sql
echo "✅ User data restored successfully."
- name: Step 5 - Update Schema Info Table
@@ -123,12 +123,12 @@ jobs:
echo "New Schema Hash: $CURRENT_HASH"
# Insert the new hash into the freshly created schema_info table.
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c \
PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c \
"INSERT INTO public.schema_info (id, schema_hash, deployed_at) VALUES (1, '$CURRENT_HASH', NOW())
ON CONFLICT (id) DO UPDATE SET schema_hash = EXCLUDED.schema_hash, deployed_at = NOW();"
# Verify the hash was updated
UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
UPDATED_HASH=$(PGPASSWORD="$DB_PASSWORD" psql -v ON_ERROR_STOP=1 -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "SELECT schema_hash FROM public.schema_info WHERE id = 1;" -t -A)
if [ "$CURRENT_HASH" = "$UPDATED_HASH" ]; then
echo "✅ Schema hash successfully set in the database to: $UPDATED_HASH"
else

12
.gitea/workflows/manual-db-restore.yml
View File

@@ -25,13 +25,13 @@ jobs:
DB_PORT: ${{ secrets.DB_PORT }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_DATABASE: ${{ secrets.DB_DATABASE_PROD }}
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
BACKUP_DIR: "/var/www/backups" # Define a dedicated directory for backups
steps:
- name: Validate Secrets and Inputs
run: |
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_DATABASE" ]; then
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
echo "ERROR: One or more production database secrets are not set in Gitea repository settings."
exit 1
fi
@@ -66,10 +66,10 @@ jobs:
echo "Dropping and recreating the production database..."
# Connect as the superuser (postgres) to drop the database.
# First, terminate all active connections to the database.
sudo -u postgres psql -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${DB_DATABASE}';"
sudo -u postgres psql -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${DB_NAME}';"
# Now, drop and recreate it.
sudo -u postgres psql -c "DROP DATABASE IF EXISTS \"${DB_DATABASE}\";"
sudo -u postgres psql -c "CREATE DATABASE \"${DB_DATABASE}\" WITH OWNER = ${DB_USER};"
sudo -u postgres psql -c "DROP DATABASE IF EXISTS \"${DB_NAME}\";"
sudo -u postgres psql -c "CREATE DATABASE \"${DB_NAME}\" WITH OWNER = ${DB_USER};"
echo "✅ Database dropped and recreated successfully."
- name: Step 3 - Restore Database from Backup
@@ -84,7 +84,7 @@ jobs:
# Uncompress the gzipped file and pipe the SQL commands directly into psql.
# This is efficient as it doesn't require an intermediate uncompressed file.
gunzip < "$BACKUP_FILE_PATH" | PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_DATABASE"
gunzip < "$BACKUP_FILE_PATH" | PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME"
echo "✅ Database restore completed successfully."

6
.gitignore vendored
View File

@@ -22,9 +22,3 @@ dist-ssr
*.njsproj
*.sln
*.sw?
# secrets
.env
.env.local
.env.test

41
README.md
View File

@@ -27,12 +27,18 @@ We are working on an app to help people save money, by finding good deals that a
---
## Required Environment Variables & Setup
## Required Secrets & Configuration
This project requires several secret keys to function. Create a `.env` file in the root of the project and see the `env.example` file for a complete template.
This project is configured to run in a CI/CD environment and does not use `.env` files. All configuration and secrets must be provided as environment variables. For deployments using the included Gitea workflows, these must be configured as **repository secrets** in your Gitea instance.
- **For the AI Service**: `GEMINI_API_KEY`. This is your public-facing Google Gemini API key.
- **For the Database**: You will need to provide connection details for your PostgreSQL database, such as `DB_USER`, `DB_HOST`, `DB_DATABASE`, `DB_PASSWORD`, and `DB_PORT`.
- **`DB_HOST`, `DB_PORT`, `DB_USER`, `DB_PASSWORD`**: Credentials for your PostgreSQL server.
- **`DB_DATABASE_PROD`**: The name of your production database.
- **`REDIS_PASSWORD_PROD`**: The password for your production Redis instance.
- **`REDIS_PASSWORD_TEST`**: The password for your test Redis instance.
- **`JWT_SECRET`**: A long, random, and secret string for signing authentication tokens.
- **`VITE_GOOGLE_GENAI_API_KEY`**: Your Google Gemini API key.
- **`GOOGLE_MAPS_API_KEY`**: Your Google Maps Geocoding API key.
- **`WORKER_CONCURRENCY`**: The number of concurrent jobs the flyer processing worker should handle (e.g., `5`).
## Setup and Installation
@@ -55,7 +61,7 @@ This project requires several secret keys to function. Create a `.env` file in t
2. **Run the Application**:
```bash
npm run dev
npm run start:prod
```
### Step 3: Seed Development Users (Optional)
@@ -106,12 +112,6 @@ Under Authorized redirect URIs, click ADD URI and enter the URL where Google wil
Click Create. You will be given a Client ID and a Client Secret.
Add these credentials to your .env file at the project root:
plaintext
GOOGLE_CLIENT_ID="your-client-id-from-google"
GOOGLE_CLIENT_SECRET="your-client-secret-from-google"
2. Get GitHub OAuth Credentials
You'll need to obtain a Client ID and Client Secret from GitHub:
@@ -130,12 +130,6 @@ Click Register application.
You will be given a Client ID and a Client Secret.
Add these credentials to your .env file at the project root:
plaintext
GITHUB_CLIENT_ID="your-github-client-id"
GITHUB_CLIENT_SECRET="your-github-client-secret"
## connect to postgres on projectium.com
psql -h localhost -U flyer_crawler_user -d "flyer-crawler-prod" -W
@@ -204,16 +198,15 @@ bash
psql -U flyer_crawler_user -d "flyer-crawler-prod" -f sql/master_schema_rollup.sql
This single command creates all tables, extensions (pg_trgm, postgis), functions, and triggers, and seeds essential data like categories and master items.
Step 4: Seed the Admin Account
Your application has a separate script to create the initial admin user.
Ensure your .env file on the server is configured with the correct production database credentials (DB_USER, DB_PASSWORD, DB_DATABASE="flyer-crawler-prod", etc.).
Run the admin seeding script using tsx.
Step 4: Seed the Admin Account (If Needed)
Your application has a separate script to create the initial admin user. To run it, you must first set the required environment variables in your shell session.
bash
# Set variables for the current session
export DB_USER=flyer_crawler_user DB_PASSWORD=your_password DB_NAME="flyer-crawler-prod" ...
# Run the seeding script
npx tsx src/db/seed_admin_account.ts
Your production database is now ready! Your application can connect to it using the flyer_crawler_user role and the credentials in your .env file.
Your production database is now ready!
Part 2: Test Database Setup (for CI/CD)
Your Gitea workflow (deploy.yml) already automates the creation and teardown of the test database during the pipeline run. The steps below are for understanding what the workflow does and for manual setup if you ever need to run tests outside the CI pipeline.

12
ecosystem.config.cjs
View File

@@ -7,10 +7,8 @@ module.exports = {
apps: [
{
name: 'flyer-crawler-api', // The name of our API application in PM2
// Use tsx as the interpreter and pass the preload script via node_args.
script: './node_modules/.bin/tsx',
args: 'server.ts',
node_args: '-r ./preload.ts',
args: 'server.ts', // tsx will execute this file
// Explicitly set the working directory. This is crucial for reliability.
cwd: '/var/www/flyer-crawler.projectium.com',
env_production: {
@@ -19,10 +17,8 @@ module.exports = {
},
{
name: 'flyer-crawler-worker', // The name of our worker process in PM2
// Use the same preload mechanism for the worker.
script: './node_modules/.bin/tsx',
args: 'src/services/queueService.server.ts',
node_args: '-r ./preload.ts',
args: 'src/services/queueService.server.ts', // tsx will execute this file
// Explicitly set the working directory.
cwd: '/var/www/flyer-crawler.projectium.com',
env_production: {
@@ -31,11 +27,9 @@ module.exports = {
},
{
name: 'flyer-crawler-analytics-worker', // A dedicated worker for analytics
// Use the same preload mechanism.
script: './node_modules/.bin/tsx',
// Point to the main worker service file. It will start all workers.
args: 'src/services/queueService.server.ts',
node_args: '-r ./preload.ts',
args: 'src/services/queueService.server.ts', // tsx will execute this file
cwd: '/var/www/flyer-crawler.projectium.com',
env_production: {
NODE_ENV: 'production',

13
package-lock.json generated
View File

@@ -17,7 +17,6 @@
"connect-timeout": "^1.9.1",
"cookie-parser": "^1.4.7",
"date-fns": "^4.1.0",
"dotenv": "^17.2.3",
"express": "^5.1.0",
"express-list-endpoints": "^7.1.1",
"express-rate-limit": "^8.2.1",
@@ -8265,18 +8264,6 @@
"license": "MIT",
"peer": true
},
"node_modules/dotenv": {
"version": "17.2.3",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz",
"integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",

17
package.json
View File

@@ -8,18 +8,16 @@
"start": "npm run start:prod",
"build": "vite build",
"preview": "vite preview",
"test": "NODE_ENV=test tsx -r ./preload.ts ./node_modules/vitest/vitest.mjs run",
"test": "NODE_ENV=test tsx ./node_modules/vitest/vitest.mjs run",
"test:coverage": "npm run clean && npm run test:unit -- --coverage && npm run test:integration -- --coverage",
"test:unit": "NODE_ENV=test tsx -r ./preload.ts ./node_modules/vitest/vitest.mjs run --project unit -c vite.config.ts",
"test:integration": "NODE_ENV=test tsx -r ./preload.ts ./node_modules/vitest/vitest.mjs run --project integration -c vitest.config.integration.ts",
"test:unit": "NODE_ENV=test tsx ./node_modules/vitest/vitest.mjs run --project unit -c vite.config.ts",
"test:integration": "NODE_ENV=test tsx ./node_modules/vitest/vitest.mjs run --project integration -c vitest.config.integration.ts",
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
"clean": "rimraf coverage .coverage",
"start:dev": "NODE_ENV=development tsx -r ./preload.ts server.ts",
"start:prod": "NODE_ENV=production tsx -r ./preload.ts server.ts",
"start:test": "NODE_ENV=test NODE_V8_COVERAGE=.coverage/tmp/integration-server tsx -r ./preload.ts server.ts",
"db:reset:test": "NODE_ENV=test tsx -r ./preload.ts src/db/seed.ts",
"worker:dev": "NODE_ENV=development tsx -r ./preload.ts src/services/queueService.server.ts",
"worker:prod": "NODE_ENV=production tsx -r ./preload.ts src/services/queueService.server.ts"
"start:prod": "NODE_ENV=production tsx server.ts",
"start:test": "NODE_ENV=test NODE_V8_COVERAGE=.coverage/tmp/integration-server tsx server.ts",
"db:reset:test": "NODE_ENV=test tsx src/db/seed.ts",
"worker:prod": "NODE_ENV=production tsx src/services/queueService.server.ts"
},
"dependencies": {
"@bull-board/api": "^6.14.2",
@@ -31,7 +29,6 @@
"connect-timeout": "^1.9.1",
"cookie-parser": "^1.4.7",
"date-fns": "^4.1.0",
"dotenv": "^17.2.3",
"express": "^5.1.0",
"express-list-endpoints": "^7.1.1",
"express-rate-limit": "^8.2.1",

19
preload.ts
View File

@@ -1,19 +0,0 @@
// preload.ts
/**
* This script is preloaded using Node's --require flag.
* Its purpose is to load the correct environment file (.env or .env.test)
* before any other application code runs. This ensures that `process.env`
* is populated correctly for the given environment (production vs. testing).
*/
import dotenv from 'dotenv';
import path from 'path';
// Determine which .env file to load based on the NODE_ENV variable.
const envFile = process.env.NODE_ENV === 'test' ? '.env.test' : '.env';
const envPath = path.resolve(process.cwd(), envFile);
const result = dotenv.config({ path: envPath });
if (result.error) {
console.error(`[preload.ts] Error loading ${envFile}:`, result.error);
}

10
server.ts
View File

@@ -21,9 +21,6 @@ import healthRouter from './src/routes/health';
import { errorHandler } from './src/middleware/errorHandler';
import { startBackgroundJobs } from './src/services/backgroundJobService';
// Environment variables are now loaded by the `tsx` command in package.json scripts.
// This ensures the correct .env file is used for development vs. testing.
// --- START DEBUG LOGGING ---
// Log the database connection details as seen by the SERVER PROCESS.
// This will confirm if the `--env-file` flag is working as expected.
@@ -32,7 +29,7 @@ logger.info(` NODE_ENV: ${process.env.NODE_ENV}`);
logger.info(` Host: ${process.env.DB_HOST}`);
logger.info(` Port: ${process.env.DB_PORT}`);
logger.info(` User: ${process.env.DB_USER}`);
logger.info(` Database: ${process.env.DB_DATABASE}`);
logger.info(` Database: ${process.env.DB_NAME}`);
// Query the users table to see what the server process sees on startup.
// Corrected the query to be unambiguous by specifying the table alias for each column.
@@ -101,8 +98,9 @@ const requestLogger = (req: Request, res: Response, next: NextFunction) => {
app.use(requestLogger); // Use the logging middleware for all requests
// --- Security Warning ---
if ((process.env.JWT_SECRET || 'your_super_secret_jwt_key_change_this') === 'your_super_secret_jwt_key_change_this') {
logger.warn('Security Warning: JWT_SECRET is using a default, insecure value. Please set a strong secret in your .env file.');
if (!process.env.JWT_SECRET) {
logger.error('CRITICAL: JWT_SECRET is not set. The application cannot start securely.');
process.exit(1);
}
// --- API Routes ---

10
src/db/backup_user.ts
View File

@@ -28,22 +28,18 @@ The manual-db-reset.yml workflow would be simplified to call these scripts. The
* @file This script creates a data-only backup for a single user.
* It traces relationships across tables to export all relevant data as SQL INSERT statements.
*
* Usage:
* tsx src/db/backup_user.ts --email user@example.com --output user_backup.sql
* Usage (from project root, after setting environment variables):
* export DB_USER=... DB_PASSWORD=... && npx tsx src/db/backup_user.ts --email user@example.com --output user_backup.sql
*/
import { Pool, PoolClient } from 'pg';
import fs from 'fs/promises';
import dotenv from 'dotenv';
import { logger } from '../services/logger.server';
// Load environment variables from the root .env file
dotenv.config({ path: '../../.env' });
const pool = new Pool({
user: process.env.DB_USER,
host: process.env.DB_HOST,
database: process.env.DB_DATABASE_PROD, // IMPORTANT: This script targets the production DB
database: process.env.DB_NAME_PROD, // IMPORTANT: This script targets the production DB
password: process.env.DB_PASSWORD,
port: parseInt(process.env.DB_PORT || '5432', 10),
});

13
src/db/seed.ts
View File

@@ -8,19 +8,14 @@
import { Pool } from 'pg';
import bcrypt from 'bcrypt';
import dotenv from 'dotenv';
import { logger } from '../services/logger';
import { CATEGORIES } from '../types';
// Load environment variables from .env file
dotenv.config({ path: '../../.env' });
const pool = new Pool({
user: process.env.DB_USER || 'postgres',
host: process.env.DB_HOST || 'localhost',
// Default to 'flyer-crawler' for local development.
database: process.env.DB_NAME || 'flyer-crawler-test',
password: process.env.DB_PASSWORD || 'fake_test_db_password', // do not replace this - use appropriate .env file
user: process.env.DB_USER,
host: process.env.DB_HOST,
database: process.env.DB_NAME,
password: process.env.DB_PASSWORD,
port: parseInt(process.env.DB_PORT || '5432', 10),
});

15
src/db/seed_admin_account.ts
View File

@@ -1,18 +1,13 @@
// src/db/seed_admin_account.ts
import { Pool } from 'pg';
import bcrypt from 'bcrypt';
import dotenv from 'dotenv';
// Load environment variables from .env file
dotenv.config();
const pool = new Pool({
user: process.env.DB_USER || 'postgres',
host: process.env.DB_HOST || 'localhost',
// Default to 'flyer-crawler' for local development.
database: process.env.DB_NAME || 'flyer-crawler-dev',
password: process.env.DB_PASSWORD || 'fake_test_db_password', // do not replace this - use appropriate .env file
port: parseInt(process.env.DB_PORT || '5432', 10),
user: process.env.DB_USER,
host: process.env.DB_HOST,
database: process.env.DB_NAME,
password: process.env.DB_PASSWORD,
port: parseInt(process.env.DB_PORT || '5432', 10), // Keep fallback for port
});
const ADMIN_EMAIL = 'admin@example.com';

4
src/routes/auth.ts
View File

@@ -14,7 +14,7 @@ import { sendPasswordResetEmail } from '../services/emailService.server';
const router = Router();
const JWT_SECRET = process.env.JWT_SECRET || 'your_super_secret_jwt_key_change_this';
const JWT_SECRET = process.env.JWT_SECRET!;
// Conditionally disable rate limiting for the test environment
const isTestEnv = process.env.NODE_ENV === 'test';
@@ -166,7 +166,7 @@ router.post('/forgot-password', forgotPasswordLimiter, async (req, res, next) =>
await db.createPasswordResetToken(user.user_id, tokenHash, expiresAt);
const resetLink = `${process.env.FRONTEND_URL || 'http://localhost:5173'}/reset-password/${token}`;
const resetLink = `${process.env.FRONTEND_URL}/reset-password/${token}`;
try {
await sendPasswordResetEmail(email, resetLink);

2
src/routes/passport.ts
View File

@@ -12,7 +12,7 @@ import { logger } from '../services/logger.server';
import { UserProfile } from '../types';
import { omit } from '../utils/objectUtils';
const JWT_SECRET = process.env.JWT_SECRET || 'your_super_secret_jwt_key_change_this';
const JWT_SECRET = process.env.JWT_SECRET!;
const MAX_FAILED_ATTEMPTS = 5;
const LOCKOUT_DURATION_MINUTES = 15;

8
src/routes/public.routes.test.ts
View File

@@ -190,9 +190,9 @@ describe('Public Routes (/api)', () => {
describe('GET /recipes/by-sale-percentage', () => {
it('should return recipes based on sale percentage', async () => {
const mockRecipes: Recipe[] = [
{ recipe_id: 1, name: 'Pasta', description: null, instructions: null, avg_rating: 0, rating_count: 0, fork_count: 0, status: 'public', created_at: new Date().toISOString() }
{ recipe_id: 1, name: 'Pasta', description: null, instructions: null, avg_rating: 0, rating_count: 0, fork_count: 0, status: 'public', created_at: new Date().toISOString() },
];
vi.mocked(db.getRecipesBySalePercentage).mockResolvedValue(mockRecipes as any);
vi.mocked(db.getRecipesBySalePercentage).mockResolvedValue(mockRecipes);
const response = await supertest(app).get('/api/recipes/by-sale-percentage?minPercentage=75');
@@ -240,9 +240,9 @@ describe('Public Routes (/api)', () => {
describe('GET /recipes/by-ingredient-and-tag', () => {
it('should return recipes for a given ingredient and tag', async () => {
const mockRecipes: Recipe[] = [
{ recipe_id: 2, name: 'Chicken Tacos', description: null, instructions: null, avg_rating: 0, rating_count: 0, fork_count: 0, status: 'public', created_at: new Date().toISOString() }
{ recipe_id: 2, name: 'Chicken Tacos', description: null, instructions: null, avg_rating: 0, rating_count: 0, fork_count: 0, status: 'public', created_at: new Date().toISOString() },
];
vi.mocked(db.findRecipesByIngredientAndTag).mockResolvedValue(mockRecipes as any);
vi.mocked(db.findRecipesByIngredientAndTag).mockResolvedValue(mockRecipes);
const response = await supertest(app).get('/api/recipes/by-ingredient-and-tag?ingredient=chicken&tag=quick');

14
src/services/db/connection.db.ts
View File

@@ -12,22 +12,20 @@ const createPool = (): Pool => {
const poolId = Math.random().toString(36).substring(2, 8);
// --- START DEBUG LOGGING ---
// Log the database connection details being used by the server process.
// This helps confirm it's using the correct .env file for the environment.
//logger.info('--- [DB POOL] Creating new PostgreSQL connection pool. ---');
//logger.info(` Host: ${process.env.DB_HOST}`);
//logger.info(` Port: ${process.env.DB_PORT}`);
//logger.info(` User: ${process.env.DB_USER}`);
//logger.info(` Database: ${process.env.DB_DATABASE}`);
//logger.info(` Database: ${process.env.DB_NAME}`);
//logger.info(` Pool Instance ID: ${poolId}`);
//logger.info('----------------------------------------------------');
const poolConfig: PoolConfig = {
user: process.env.DB_USER || 'postgres',
host: process.env.DB_HOST || 'localhost',
database: process.env.DB_NAME || 'flyer-crawler-test',
password: process.env.DB_PASSWORD || 'fake_test_db_password', // do not replace this - use appropriate .env file
port: parseInt(process.env.DB_PORT || '5432', 10),
user: process.env.DB_USER,
host: process.env.DB_HOST,
database: process.env.DB_NAME,
password: process.env.DB_PASSWORD,
port: parseInt(process.env.DB_PORT || '5432', 10), // Keep fallback for port as it's less sensitive
};
// --- ADD DEBUG LOGGING HERE ---

6
src/services/queueService.server.ts
View File

@@ -12,7 +12,7 @@ import * as emailService from './emailService.server';
import * as db from './db/index.db';
import { generateFlyerIcon } from '../utils/imageProcessor';
export const connection = new IORedis(process.env.REDIS_URL || 'redis://127.0.0.1:6379', {
export const connection = new IORedis(process.env.REDIS_URL!, {
maxRetriesPerRequest: null, // Important for BullMQ
password: process.env.REDIS_PASSWORD, // Add the password from environment variables
});
@@ -224,7 +224,7 @@ export const flyerWorker = new Worker<FlyerJobData>(
// Control the number of concurrent jobs. This directly limits parallel calls to the AI API.
// It's sourced from an environment variable for easy configuration without code changes.
// The Google AI free tier limit is 60 RPM, so a low concurrency is safe.
concurrency: parseInt(process.env.WORKER_CONCURRENCY || '5', 10),
concurrency: parseInt(process.env.WORKER_CONCURRENCY!, 10),
}
);
@@ -307,7 +307,7 @@ export const cleanupWorker = new Worker<CleanupJobData>(
}
// 2. Determine the base path for the flyer images.
const storagePath = process.env.STORAGE_PATH || '/var/www/flyer-crawler.projectium.com/flyer-images';
const storagePath = process.env.STORAGE_PATH!;
// 3. Delete the main flyer image.
const mainImagePath = path.join(storagePath, path.basename(flyer.image_url));

2
src/tests/integration/auth.integration.test.ts
View File

@@ -29,7 +29,7 @@ describe('Authentication API Integration', () => {
console.log(` Host: ${process.env.DB_HOST}`);
console.log(` Port: ${process.env.DB_PORT}`);
console.log(` User: ${process.env.DB_USER}`);
console.log(` Database: ${process.env.DB_DATABASE}`);
console.log(` Database: ${process.env.DB_NAME}`);
console.log('-----------------------------------------------------\n');
// --- END DEBUG LOGGING ---

6
src/tests/setup/global-setup.ts
View File

@@ -6,11 +6,11 @@ import { execSync } from 'child_process';
const getPool = () => {
// This pool connects using the test-specific environment variables
// passed from the Gitea workflow (e.g., DB_DATABASE="flyer-crawler-test").
// passed from the Gitea workflow (e.g., DB_NAMEyer-crawler-test").
return new Pool({
user: process.env.DB_USER,
host: process.env.DB_HOST,
database: process.env.DB_DATABASE,
database: process.env.DB_NAME,
password: process.env.DB_PASSWORD,
port: parseInt(process.env.DB_PORT || '5432', 10),
});
@@ -29,7 +29,7 @@ export async function setup() {
console.log(` Host: ${process.env.DB_HOST}`);
console.log(` Port: ${process.env.DB_PORT}`);
console.log(` User: ${process.env.DB_USER}`);
console.log(` Database: ${process.env.DB_DATABASE}`);
console.log(` Database: ${process.env.DB_NAME}`);
console.log('-------------------------------------------\n');
// --- END DEBUG LOGGING ---
console.log('\nResetting test database schema...');

2
src/tests/setup/tests-setup-db.ts
View File

@@ -6,7 +6,7 @@ import { Pool } from 'pg';
export const testPool = new Pool({
user: process.env.DB_USER,
host: process.env.DB_HOST,
database: process.env.DB_DATABASE,
database: process.env.DB_NAME,
password: process.env.DB_PASSWORD,
port: parseInt(process.env.DB_PORT || '5432', 10),
});