Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a14816c8ee | ||
|
|
08b220e29c | ||
|
|
d41a3f1887 | ||
| 1f6cdc62d7 | |||
|
|
978c63bacd | ||
| 544eb7ae3c | |||
|
|
f6839f6e14 | ||
| 3fac29436a |
@@ -63,8 +63,8 @@ jobs:
|
||||
- name: Check for Production Database Schema Changes
|
||||
env:
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
run: |
|
||||
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
|
||||
@@ -117,8 +117,8 @@ jobs:
|
||||
env:
|
||||
# --- Production Secrets Injection ---
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
# Explicitly use database 0 for production (test uses database 1)
|
||||
REDIS_URL: 'redis://localhost:6379/0'
|
||||
|
||||
@@ -121,10 +121,11 @@ jobs:
|
||||
env:
|
||||
# --- Database credentials for the test suite ---
|
||||
# These are injected from Gitea secrets into the runner's environment.
|
||||
# CRITICAL: Use TEST-specific credentials that have CREATE privileges on the public schema.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_NAME: 'flyer-crawler-test' # Explicitly set for tests
|
||||
DB_USER: ${{ secrets.DB_USER_TEST }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
|
||||
|
||||
# --- Redis credentials for the test suite ---
|
||||
# CRITICAL: Use Redis database 1 to isolate tests from production (which uses db 0).
|
||||
@@ -328,10 +329,11 @@ jobs:
|
||||
- name: Check for Test Database Schema Changes
|
||||
env:
|
||||
# Use test database credentials for this check.
|
||||
# CRITICAL: Use TEST-specific credentials that have CREATE privileges on the public schema.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # This is used by psql
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }} # This is used by the application
|
||||
DB_USER: ${{ secrets.DB_USER_TEST }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
|
||||
run: |
|
||||
# Fail-fast check to ensure secrets are configured in Gitea.
|
||||
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
|
||||
@@ -427,9 +429,10 @@ jobs:
|
||||
# Your Node.js application will read these directly from `process.env`.
|
||||
|
||||
# Database Credentials
|
||||
# CRITICAL: Use TEST-specific credentials that have CREATE privileges on the public schema.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_TEST }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
|
||||
|
||||
# Redis Credentials (use database 1 to isolate from production)
|
||||
|
||||
@@ -20,9 +20,9 @@ jobs:
|
||||
# Use production database credentials for this entire job.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_PORT: ${{ secrets.DB_PORT }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_NAME: ${{ secrets.DB_NAME_PROD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
|
||||
steps:
|
||||
- name: Validate Secrets
|
||||
|
||||
@@ -23,9 +23,9 @@ jobs:
|
||||
env:
|
||||
# Use production database credentials for this entire job.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }} # Used by the application
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
|
||||
@@ -23,9 +23,9 @@ jobs:
|
||||
env:
|
||||
# Use test database credentials for this entire job.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }} # Used by the application
|
||||
DB_USER: ${{ secrets.DB_USER_TEST }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
|
||||
@@ -22,8 +22,8 @@ jobs:
|
||||
env:
|
||||
# Use production database credentials for this entire job.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
BACKUP_DIR: '/var/www/backups' # Define a dedicated directory for backups
|
||||
|
||||
|
||||
@@ -62,8 +62,8 @@ jobs:
|
||||
- name: Check for Production Database Schema Changes
|
||||
env:
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
run: |
|
||||
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
|
||||
@@ -113,8 +113,8 @@ jobs:
|
||||
env:
|
||||
# --- Production Secrets Injection ---
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
# Explicitly use database 0 for production (test uses database 1)
|
||||
REDIS_URL: 'redis://localhost:6379/0'
|
||||
|
||||
57
CLAUDE.md
57
CLAUDE.md
@@ -293,7 +293,7 @@ To add a new secret (e.g., `SENTRY_DSN`):
|
||||
|
||||
**Shared (used by both environments):**
|
||||
|
||||
- `DB_HOST`, `DB_USER`, `DB_PASSWORD` - Database credentials
|
||||
- `DB_HOST` - Database host (shared PostgreSQL server)
|
||||
- `JWT_SECRET` - Authentication
|
||||
- `GOOGLE_MAPS_API_KEY` - Google Maps
|
||||
- `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` - Google OAuth
|
||||
@@ -301,14 +301,16 @@ To add a new secret (e.g., `SENTRY_DSN`):
|
||||
|
||||
**Production-specific:**
|
||||
|
||||
- `DB_DATABASE_PROD` - Production database name
|
||||
- `DB_USER_PROD`, `DB_PASSWORD_PROD` - Production database credentials (`flyer_crawler_prod`)
|
||||
- `DB_DATABASE_PROD` - Production database name (`flyer-crawler`)
|
||||
- `REDIS_PASSWORD_PROD` - Redis password (uses database 0)
|
||||
- `VITE_GOOGLE_GENAI_API_KEY` - Gemini API key for production
|
||||
- `SENTRY_DSN`, `VITE_SENTRY_DSN` - Bugsink error tracking DSNs (production projects)
|
||||
|
||||
**Test-specific:**
|
||||
|
||||
- `DB_DATABASE_TEST` - Test database name
|
||||
- `DB_USER_TEST`, `DB_PASSWORD_TEST` - Test database credentials (`flyer_crawler_test`)
|
||||
- `DB_DATABASE_TEST` - Test database name (`flyer-crawler-test`)
|
||||
- `REDIS_PASSWORD_TEST` - Redis password (uses database 1 for isolation)
|
||||
- `VITE_GOOGLE_GENAI_API_KEY_TEST` - Gemini API key for test
|
||||
- `SENTRY_DSN_TEST`, `VITE_SENTRY_DSN_TEST` - Bugsink error tracking DSNs (test projects)
|
||||
@@ -322,6 +324,55 @@ The test environment (`flyer-crawler-test.projectium.com`) uses **both** Gitea C
|
||||
- **Redis database 1**: Isolates test job queues from production (which uses database 0)
|
||||
- **PM2 process names**: Suffixed with `-test` (e.g., `flyer-crawler-api-test`)
|
||||
|
||||
### Database User Setup (Test Environment)
|
||||
|
||||
**CRITICAL**: The test database requires specific PostgreSQL permissions to be configured manually. Schema ownership alone is NOT sufficient - explicit privileges must be granted.
|
||||
|
||||
**Database Users:**
|
||||
|
||||
| User | Database | Purpose |
|
||||
| -------------------- | -------------------- | ---------- |
|
||||
| `flyer_crawler_prod` | `flyer-crawler` | Production |
|
||||
| `flyer_crawler_test` | `flyer-crawler-test` | Testing |
|
||||
|
||||
**Required Setup Commands** (run as `postgres` superuser):
|
||||
|
||||
```bash
|
||||
# Connect as postgres superuser
|
||||
sudo -u postgres psql
|
||||
|
||||
# Create the test database and user (if not exists)
|
||||
CREATE DATABASE "flyer-crawler-test";
|
||||
CREATE USER flyer_crawler_test WITH PASSWORD 'your-password-here';
|
||||
|
||||
# Grant ownership and privileges
|
||||
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
|
||||
\c "flyer-crawler-test"
|
||||
ALTER SCHEMA public OWNER TO flyer_crawler_test;
|
||||
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
|
||||
|
||||
# Create required extension (must be done by superuser)
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
```
|
||||
|
||||
**Why These Steps Are Necessary:**
|
||||
|
||||
1. **Schema ownership alone is insufficient** - PostgreSQL requires explicit `GRANT CREATE, USAGE` privileges even when the user owns the schema
|
||||
2. **uuid-ossp extension** - Required by the application for UUID generation; must be created by a superuser before the app can use it
|
||||
3. **Separate users for prod/test** - Prevents accidental cross-environment data access; each environment has its own credentials in Gitea secrets
|
||||
|
||||
**Verification:**
|
||||
|
||||
```bash
|
||||
# Check schema privileges (should show 'UC' for flyer_crawler_test)
|
||||
psql -d "flyer-crawler-test" -c "\dn+ public"
|
||||
|
||||
# Expected output:
|
||||
# Name | Owner | Access privileges
|
||||
# -------+--------------------+------------------------------------------
|
||||
# public | flyer_crawler_test | flyer_crawler_test=UC/flyer_crawler_test
|
||||
```
|
||||
|
||||
### Dev Container Environment
|
||||
|
||||
The dev container runs its own **local Bugsink instance** - it does NOT connect to the production Bugsink server:
|
||||
|
||||
@@ -149,7 +149,7 @@ SELECT PostGIS_Full_Version();
|
||||
|
||||
Example output:
|
||||
|
||||
```
|
||||
```text
|
||||
PostgreSQL 14.19 (Ubuntu 14.19-0ubuntu0.22.04.1)
|
||||
POSTGIS="3.2.0 c3e3cc0" GEOS="3.10.2-CAPI-1.16.0" PROJ="8.2.1"
|
||||
```
|
||||
|
||||
@@ -42,9 +42,9 @@ jobs:
|
||||
env:
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_PORT: ${{ secrets.DB_PORT }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_NAME: ${{ secrets.DB_NAME_PROD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
|
||||
steps:
|
||||
- name: Validate Secrets
|
||||
|
||||
4
package-lock.json
generated
4
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "flyer-crawler",
|
||||
"version": "0.9.112",
|
||||
"version": "0.11.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "flyer-crawler",
|
||||
"version": "0.9.112",
|
||||
"version": "0.11.0",
|
||||
"dependencies": {
|
||||
"@bull-board/api": "^6.14.2",
|
||||
"@bull-board/express": "^6.14.2",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "flyer-crawler",
|
||||
"private": true,
|
||||
"version": "0.9.112",
|
||||
"version": "0.11.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "concurrently \"npm:start:dev\" \"vite\"",
|
||||
|
||||
Reference in New Issue
Block a user