Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dd067183ed | ||
| 9f3a070612 | |||
| 8a38befb1c |
@@ -114,7 +114,15 @@
|
||||
"Bash(git rm:*)",
|
||||
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" log -1 --format=\"%H %ci %s\")",
|
||||
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" config --get remote.origin.url)",
|
||||
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" fetch --dry-run -v)"
|
||||
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" fetch --dry-run -v)",
|
||||
"mcp__localerrors__get_project",
|
||||
"mcp__localerrors__get_issue",
|
||||
"mcp__localerrors__get_event"
|
||||
]
|
||||
}
|
||||
},
|
||||
"enabledMcpjsonServers": [
|
||||
"localerrors",
|
||||
"devdb",
|
||||
"gitea-projectium"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -36,10 +36,10 @@ NODE_ENV=development
|
||||
FRONTEND_URL=http://localhost:3000
|
||||
|
||||
# Flyer Base URL - used for seed data and flyer image URLs
|
||||
# Dev container: http://127.0.0.1
|
||||
# Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
|
||||
# Test: https://flyer-crawler-test.projectium.com
|
||||
# Production: https://flyer-crawler.projectium.com
|
||||
FLYER_BASE_URL=http://127.0.0.1
|
||||
FLYER_BASE_URL=https://localhost
|
||||
|
||||
# ===================
|
||||
# Authentication
|
||||
|
||||
@@ -15,6 +15,14 @@
|
||||
"@modelcontextprotocol/server-postgres",
|
||||
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
|
||||
]
|
||||
},
|
||||
"gitea-projectium": {
|
||||
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
|
||||
"args": ["run", "-t", "stdio"],
|
||||
"env": {
|
||||
"GITEA_HOST": "https://gitea.projectium.com",
|
||||
"GITEA_ACCESS_TOKEN": "b111259253aa3cadcb6a37618de03bf388f6235a"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
103
CLAUDE.md
103
CLAUDE.md
@@ -40,6 +40,7 @@ Ask before assuming. Never assume:
|
||||
1. **Memory**: `mcp__memory__read_graph` - Recall project context, credentials, known issues
|
||||
2. **Git**: `git log --oneline -10` - Recent changes
|
||||
3. **Containers**: `mcp__podman__container_list` - Running state
|
||||
4. **PM2 Status**: `podman exec flyer-crawler-dev pm2 status` - Process health (API, Worker, Vite)
|
||||
|
||||
---
|
||||
|
||||
@@ -47,12 +48,15 @@ Ask before assuming. Never assume:
|
||||
|
||||
### Essential Commands
|
||||
|
||||
| Command | Description |
|
||||
| ------------------------------------------------------------ | ----------------- |
|
||||
| `podman exec -it flyer-crawler-dev npm test` | Run all tests |
|
||||
| `podman exec -it flyer-crawler-dev npm run test:unit` | Unit tests only |
|
||||
| `podman exec -it flyer-crawler-dev npm run type-check` | TypeScript check |
|
||||
| `podman exec -it flyer-crawler-dev npm run test:integration` | Integration tests |
|
||||
| Command | Description |
|
||||
| ------------------------------------------------------------ | --------------------- |
|
||||
| `podman exec -it flyer-crawler-dev npm test` | Run all tests |
|
||||
| `podman exec -it flyer-crawler-dev npm run test:unit` | Unit tests only |
|
||||
| `podman exec -it flyer-crawler-dev npm run type-check` | TypeScript check |
|
||||
| `podman exec -it flyer-crawler-dev npm run test:integration` | Integration tests |
|
||||
| `podman exec -it flyer-crawler-dev pm2 status` | PM2 process status |
|
||||
| `podman exec -it flyer-crawler-dev pm2 logs` | View all PM2 logs |
|
||||
| `podman exec -it flyer-crawler-dev pm2 restart all` | Restart all processes |
|
||||
|
||||
### Key Patterns (with file locations)
|
||||
|
||||
@@ -65,14 +69,16 @@ Ask before assuming. Never assume:
|
||||
|
||||
### Key Files Quick Access
|
||||
|
||||
| Purpose | File |
|
||||
| ------------ | -------------------------------- |
|
||||
| Express app | `server.ts` |
|
||||
| Environment | `src/config/env.ts` |
|
||||
| Routes | `src/routes/*.routes.ts` |
|
||||
| Repositories | `src/services/db/*.db.ts` |
|
||||
| Workers | `src/services/workers.server.ts` |
|
||||
| Queues | `src/services/queues.server.ts` |
|
||||
| Purpose | File |
|
||||
| ----------------- | -------------------------------- |
|
||||
| Express app | `server.ts` |
|
||||
| Environment | `src/config/env.ts` |
|
||||
| Routes | `src/routes/*.routes.ts` |
|
||||
| Repositories | `src/services/db/*.db.ts` |
|
||||
| Workers | `src/services/workers.server.ts` |
|
||||
| Queues | `src/services/queues.server.ts` |
|
||||
| PM2 Config (Dev) | `ecosystem.dev.config.cjs` |
|
||||
| PM2 Config (Prod) | `ecosystem.config.cjs` |
|
||||
|
||||
---
|
||||
|
||||
@@ -96,6 +102,49 @@ Routes → Services → Repositories → Database
|
||||
|
||||
---
|
||||
|
||||
## Dev Container Architecture (ADR-014)
|
||||
|
||||
The dev container now matches production by using PM2 for process management.
|
||||
|
||||
### Process Management
|
||||
|
||||
| Component | Production | Dev Container |
|
||||
| ---------- | ---------------------- | ------------------------- |
|
||||
| API Server | PM2 cluster mode | PM2 fork mode + tsx watch |
|
||||
| Worker | PM2 process | PM2 process + tsx watch |
|
||||
| Frontend | Static files via NGINX | PM2 + Vite dev server |
|
||||
| Logs | PM2 logs -> Logstash | PM2 logs -> Logstash |
|
||||
|
||||
**PM2 Processes in Dev Container**:
|
||||
|
||||
- `flyer-crawler-api-dev` - API server (port 3001)
|
||||
- `flyer-crawler-worker-dev` - Background job worker
|
||||
- `flyer-crawler-vite-dev` - Vite frontend dev server (port 5173)
|
||||
|
||||
### Log Aggregation (ADR-050)
|
||||
|
||||
All logs flow to Bugsink via Logstash:
|
||||
|
||||
| Source | Log Location | Status |
|
||||
| ----------------- | --------------------------------- | ------ |
|
||||
| Backend (Pino) | `/var/log/pm2/api-*.log` | Active |
|
||||
| Worker (Pino) | `/var/log/pm2/worker-*.log` | Active |
|
||||
| Vite | `/var/log/pm2/vite-*.log` | Active |
|
||||
| PostgreSQL | `/var/log/postgresql/*.log` | Active |
|
||||
| Redis | `/var/log/redis/redis-server.log` | Active |
|
||||
| NGINX | `/var/log/nginx/*.log` | Active |
|
||||
| Frontend (Sentry) | Browser -> Bugsink SDK | Active |
|
||||
|
||||
**Key Files**:
|
||||
|
||||
- `ecosystem.dev.config.cjs` - PM2 development configuration
|
||||
- `scripts/dev-entrypoint.sh` - Container startup script
|
||||
- `docker/logstash/bugsink.conf` - Logstash pipeline configuration
|
||||
|
||||
**Full Dev Container Guide**: See [docs/development/DEV-CONTAINER.md](docs/development/DEV-CONTAINER.md)
|
||||
|
||||
---
|
||||
|
||||
## Common Workflows
|
||||
|
||||
### Adding a New API Endpoint
|
||||
@@ -251,18 +300,20 @@ Log aggregation: PostgreSQL + PM2 + Redis + NGINX → Bugsink (ADR-050)
|
||||
|
||||
## Documentation Quick Links
|
||||
|
||||
| Topic | Document |
|
||||
| ------------------- | ----------------------------------------------------- |
|
||||
| **Getting Started** | [QUICKSTART.md](docs/getting-started/QUICKSTART.md) |
|
||||
| **Architecture** | [OVERVIEW.md](docs/architecture/OVERVIEW.md) |
|
||||
| **Code Patterns** | [CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md) |
|
||||
| **Testing** | [TESTING.md](docs/development/TESTING.md) |
|
||||
| **Debugging** | [DEBUGGING.md](docs/development/DEBUGGING.md) |
|
||||
| **Database** | [DATABASE.md](docs/architecture/DATABASE.md) |
|
||||
| **Deployment** | [DEPLOYMENT.md](docs/operations/DEPLOYMENT.md) |
|
||||
| **Monitoring** | [MONITORING.md](docs/operations/MONITORING.md) |
|
||||
| **ADRs** | [docs/adr/index.md](docs/adr/index.md) |
|
||||
| **All Docs** | [docs/README.md](docs/README.md) |
|
||||
| Topic | Document |
|
||||
| ------------------- | -------------------------------------------------------------- |
|
||||
| **Getting Started** | [QUICKSTART.md](docs/getting-started/QUICKSTART.md) |
|
||||
| **Dev Container** | [DEV-CONTAINER.md](docs/development/DEV-CONTAINER.md) |
|
||||
| **Architecture** | [OVERVIEW.md](docs/architecture/OVERVIEW.md) |
|
||||
| **Code Patterns** | [CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md) |
|
||||
| **Testing** | [TESTING.md](docs/development/TESTING.md) |
|
||||
| **Debugging** | [DEBUGGING.md](docs/development/DEBUGGING.md) |
|
||||
| **Database** | [DATABASE.md](docs/architecture/DATABASE.md) |
|
||||
| **Deployment** | [DEPLOYMENT.md](docs/operations/DEPLOYMENT.md) |
|
||||
| **Monitoring** | [MONITORING.md](docs/operations/MONITORING.md) |
|
||||
| **Logstash** | [LOGSTASH-QUICK-REF.md](docs/operations/LOGSTASH-QUICK-REF.md) |
|
||||
| **ADRs** | [docs/adr/index.md](docs/adr/index.md) |
|
||||
| **All Docs** | [docs/README.md](docs/README.md) |
|
||||
|
||||
---
|
||||
|
||||
|
||||
146
Dockerfile.dev
146
Dockerfile.dev
@@ -54,6 +54,13 @@ RUN apt-get update && apt-get install -y \
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
|
||||
&& apt-get install -y nodejs
|
||||
|
||||
# ============================================================================
|
||||
# Install PM2 Globally (ADR-014 Parity)
|
||||
# ============================================================================
|
||||
# Install PM2 to match production architecture. This allows dev container to
|
||||
# run the same process management as production (cluster mode, workers, etc.)
|
||||
RUN npm install -g pm2
|
||||
|
||||
# ============================================================================
|
||||
# Install mkcert and Generate Self-Signed Certificates
|
||||
# ============================================================================
|
||||
@@ -63,6 +70,27 @@ RUN wget -O /usr/local/bin/mkcert https://github.com/FiloSottile/mkcert/releases
|
||||
&& chmod +x /usr/local/bin/mkcert
|
||||
|
||||
# Create certificates directory and generate localhost certificates
|
||||
# ============================================================================
|
||||
# IMPORTANT: Certificate includes MULTIPLE hostnames (SANs)
|
||||
# ============================================================================
|
||||
# The certificate is generated for 'localhost', '127.0.0.1', AND '::1' because:
|
||||
#
|
||||
# 1. Users may access the site via https://localhost/ OR https://127.0.0.1/
|
||||
# 2. Database stores image URLs using one hostname (typically 127.0.0.1)
|
||||
# 3. The seed script uses https://127.0.0.1 for image URLs (database constraint)
|
||||
# 4. NGINX is configured to accept BOTH hostnames (see docker/nginx/dev.conf)
|
||||
#
|
||||
# Without all hostnames in the certificate's Subject Alternative Names (SANs),
|
||||
# browsers would show ERR_CERT_AUTHORITY_INVALID when loading images or other
|
||||
# resources that use a different hostname than the one in the address bar.
|
||||
#
|
||||
# The mkcert command below creates a certificate valid for all three:
|
||||
# - localhost (IPv4 hostname)
|
||||
# - 127.0.0.1 (IPv4 address)
|
||||
# - ::1 (IPv6 loopback)
|
||||
#
|
||||
# See also: docker/nginx/dev.conf, docs/FLYER-URL-CONFIGURATION.md
|
||||
# ============================================================================
|
||||
RUN mkdir -p /app/certs \
|
||||
&& cd /app/certs \
|
||||
&& mkcert -install \
|
||||
@@ -211,119 +239,25 @@ exec /opt/bugsink/bin/gunicorn \\\n\
|
||||
&& chmod +x /usr/local/bin/start-bugsink.sh
|
||||
|
||||
# ============================================================================
|
||||
# Create Logstash Pipeline Configuration
|
||||
# Copy Logstash Pipeline Configuration
|
||||
# ============================================================================
|
||||
# ADR-015: Pino and Redis logs → Bugsink
|
||||
# ADR-015 + ADR-050: Multi-source log aggregation to Bugsink
|
||||
# Configuration file includes:
|
||||
# - Pino application logs (Backend API errors)
|
||||
# - PostgreSQL logs (including fn_log() structured output)
|
||||
# - NGINX access and error logs
|
||||
# See docker/logstash/bugsink.conf for full configuration
|
||||
RUN mkdir -p /etc/logstash/conf.d /app/logs
|
||||
|
||||
RUN echo 'input {\n\
|
||||
# Pino application logs\n\
|
||||
file {\n\
|
||||
path => "/app/logs/*.log"\n\
|
||||
codec => json\n\
|
||||
type => "pino"\n\
|
||||
tags => ["app"]\n\
|
||||
start_position => "beginning"\n\
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pino"\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Redis logs\n\
|
||||
file {\n\
|
||||
path => "/var/log/redis/*.log"\n\
|
||||
type => "redis"\n\
|
||||
tags => ["redis"]\n\
|
||||
start_position => "beginning"\n\
|
||||
sincedb_path => "/var/lib/logstash/sincedb_redis"\n\
|
||||
}\n\
|
||||
\n\
|
||||
# PostgreSQL function logs (ADR-050)\n\
|
||||
file {\n\
|
||||
path => "/var/log/postgresql/*.log"\n\
|
||||
type => "postgres"\n\
|
||||
tags => ["postgres", "database"]\n\
|
||||
start_position => "beginning"\n\
|
||||
sincedb_path => "/var/lib/logstash/sincedb_postgres"\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
filter {\n\
|
||||
# Pino error detection (level 50 = error, 60 = fatal)\n\
|
||||
if [type] == "pino" and [level] >= 50 {\n\
|
||||
mutate { add_tag => ["error"] }\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Redis log parsing\n\
|
||||
if [type] == "redis" {\n\
|
||||
grok {\n\
|
||||
match => { "message" => "%%{POSINT:pid}:%%{WORD:role} %%{MONTHDAY} %%{MONTH} %%{TIME} %%{WORD:loglevel} %%{GREEDYDATA:redis_message}" }\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Tag errors (WARNING/ERROR) for Bugsink forwarding\n\
|
||||
if [loglevel] in ["WARNING", "ERROR"] {\n\
|
||||
mutate { add_tag => ["error"] }\n\
|
||||
}\n\
|
||||
# Tag INFO-level operational events (startup, config, persistence)\n\
|
||||
else if [loglevel] == "INFO" {\n\
|
||||
mutate { add_tag => ["redis_operational"] }\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
# PostgreSQL function log parsing (ADR-050)\n\
|
||||
if [type] == "postgres" {\n\
|
||||
# Extract timestamp and process ID from PostgreSQL log prefix\n\
|
||||
# Format: "2026-01-18 10:30:00 PST [12345] user@database "\n\
|
||||
grok {\n\
|
||||
match => { "message" => "%%{TIMESTAMP_ISO8601:pg_timestamp} \\\\[%%{POSINT:pg_pid}\\\\] %%{USERNAME:pg_user}@%%{WORD:pg_database} %%{GREEDYDATA:pg_message}" }\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Check if this is a structured JSON log from fn_log()\n\
|
||||
# fn_log() emits JSON like: {"timestamp":"...","level":"WARNING","source":"postgresql","function":"award_achievement",...}\n\
|
||||
if [pg_message] =~ /^\\{.*"source":"postgresql".*\\}$/ {\n\
|
||||
json {\n\
|
||||
source => "pg_message"\n\
|
||||
target => "fn_log"\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Mark as error if level is WARNING or ERROR\n\
|
||||
if [fn_log][level] in ["WARNING", "ERROR"] {\n\
|
||||
mutate { add_tag => ["error", "db_function"] }\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Also catch native PostgreSQL errors\n\
|
||||
if [pg_message] =~ /^ERROR:/ or [pg_message] =~ /^FATAL:/ {\n\
|
||||
mutate { add_tag => ["error", "postgres_native"] }\n\
|
||||
}\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
output {\n\
|
||||
# Forward errors to Bugsink\n\
|
||||
if "error" in [tags] {\n\
|
||||
http {\n\
|
||||
url => "http://localhost:8000/api/store/"\n\
|
||||
http_method => "post"\n\
|
||||
format => "json"\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Store Redis operational logs (INFO level) to file\n\
|
||||
if "redis_operational" in [tags] {\n\
|
||||
file {\n\
|
||||
path => "/var/log/logstash/redis-operational-%%{+YYYY-MM-dd}.log"\n\
|
||||
codec => json_lines\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Debug output (comment out in production)\n\
|
||||
stdout { codec => rubydebug }\n\
|
||||
}\n\
|
||||
' > /etc/logstash/conf.d/bugsink.conf
|
||||
COPY docker/logstash/bugsink.conf /etc/logstash/conf.d/bugsink.conf
|
||||
|
||||
# Create Logstash directories
|
||||
RUN mkdir -p /var/lib/logstash && chown -R logstash:logstash /var/lib/logstash
|
||||
RUN mkdir -p /var/log/logstash && chown -R logstash:logstash /var/log/logstash
|
||||
|
||||
# Create PM2 log directory (ADR-014 Parity)
|
||||
# Logs written here will be picked up by Logstash (ADR-050)
|
||||
RUN mkdir -p /var/log/pm2
|
||||
|
||||
# ============================================================================
|
||||
# Configure Nginx
|
||||
# ============================================================================
|
||||
|
||||
105
certs/README.md
Normal file
105
certs/README.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# Development SSL Certificates
|
||||
|
||||
This directory contains SSL certificates for the development container HTTPS setup.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose | Generated By |
|
||||
| --------------- | ---------------------------------------------------- | -------------------------- |
|
||||
| `localhost.crt` | SSL certificate for localhost and 127.0.0.1 | mkcert (in Dockerfile.dev) |
|
||||
| `localhost.key` | Private key for localhost.crt | mkcert (in Dockerfile.dev) |
|
||||
| `mkcert-ca.crt` | Root CA certificate for trusting mkcert certificates | mkcert |
|
||||
|
||||
## Certificate Details
|
||||
|
||||
The `localhost.crt` certificate includes the following Subject Alternative Names (SANs):
|
||||
|
||||
- `DNS:localhost`
|
||||
- `IP Address:127.0.0.1`
|
||||
- `IP Address:::1` (IPv6 localhost)
|
||||
|
||||
This allows the development server to be accessed via both `https://localhost/` and `https://127.0.0.1/` without SSL errors.
|
||||
|
||||
## Installing the CA Certificate (Recommended)
|
||||
|
||||
To avoid SSL certificate warnings in your browser, install the mkcert CA certificate on your system.
|
||||
|
||||
### Windows
|
||||
|
||||
1. Double-click `mkcert-ca.crt`
|
||||
2. Click **"Install Certificate..."**
|
||||
3. Select **"Local Machine"** > Next
|
||||
4. Select **"Place all certificates in the following store"**
|
||||
5. Click **Browse** > Select **"Trusted Root Certification Authorities"** > OK
|
||||
6. Click **Next** > **Finish**
|
||||
7. Restart your browser
|
||||
|
||||
### macOS
|
||||
|
||||
```bash
|
||||
sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain certs/mkcert-ca.crt
|
||||
```
|
||||
|
||||
### Linux
|
||||
|
||||
```bash
|
||||
# Ubuntu/Debian
|
||||
sudo cp certs/mkcert-ca.crt /usr/local/share/ca-certificates/mkcert-ca.crt
|
||||
sudo update-ca-certificates
|
||||
|
||||
# Fedora/RHEL
|
||||
sudo cp certs/mkcert-ca.crt /etc/pki/ca-trust/source/anchors/
|
||||
sudo update-ca-trust
|
||||
```
|
||||
|
||||
### Firefox (All Platforms)
|
||||
|
||||
Firefox uses its own certificate store:
|
||||
|
||||
1. Open Firefox Settings
|
||||
2. Search for "Certificates"
|
||||
3. Click **"View Certificates"**
|
||||
4. Go to **"Authorities"** tab
|
||||
5. Click **"Import..."**
|
||||
6. Select `certs/mkcert-ca.crt`
|
||||
7. Check **"Trust this CA to identify websites"**
|
||||
8. Click **OK**
|
||||
|
||||
## After Installation
|
||||
|
||||
Once the CA certificate is installed:
|
||||
|
||||
- Your browser will trust all mkcert certificates without warnings
|
||||
- Access `https://localhost/` with no security warnings
|
||||
- Images from `https://127.0.0.1/flyer-images/` will load without SSL errors
|
||||
|
||||
## Regenerating Certificates
|
||||
|
||||
If you need to regenerate the certificates (e.g., after rebuilding the container):
|
||||
|
||||
```bash
|
||||
# Inside the container
|
||||
cd /app/certs
|
||||
mkcert localhost 127.0.0.1 ::1
|
||||
mv localhost+2.pem localhost.crt
|
||||
mv localhost+2-key.pem localhost.key
|
||||
nginx -s reload
|
||||
|
||||
# Copy the new CA to the host
|
||||
podman cp flyer-crawler-dev:/app/certs/mkcert-ca.crt ./certs/mkcert-ca.crt
|
||||
```
|
||||
|
||||
Then reinstall the CA certificate as described above.
|
||||
|
||||
## Security Note
|
||||
|
||||
**DO NOT** commit the private key (`localhost.key`) to version control in production projects. For this development-only project, the certificates are checked in for convenience since they're only used locally with self-signed certificates.
|
||||
|
||||
The certificates in this directory are automatically generated by the Dockerfile.dev and should not be used in production.
|
||||
|
||||
## See Also
|
||||
|
||||
- [Dockerfile.dev](../Dockerfile.dev) - Certificate generation (line ~69)
|
||||
- [docker/nginx/dev.conf](../docker/nginx/dev.conf) - NGINX SSL configuration
|
||||
- [docs/FLYER-URL-CONFIGURATION.md](../docs/FLYER-URL-CONFIGURATION.md) - URL configuration details
|
||||
- [docs/development/DEBUGGING.md](../docs/development/DEBUGGING.md) - SSL troubleshooting
|
||||
@@ -1,19 +1,25 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDCTCCAfGgAwIBAgIUHhZUK1vmww2wCepWPuVcU6d27hMwDQYJKoZIhvcNAQEL
|
||||
BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDExODAyMzM0NFoXDTI3MDEx
|
||||
ODAyMzM0NFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
|
||||
AAOCAQ8AMIIBCgKCAQEAuUJGtSZzd+ZpLi+efjrkxJJNfVxVz2VLhknNM2WKeOYx
|
||||
JTK/VaTYq5hrczy6fEUnMhDAJCgEPUFlOK3vn1gFJKNMN8m7arkLVk6PYtrx8CTw
|
||||
w78Q06FLITr6hR0vlJNpN4MsmGxYwUoUpn1j5JdfZF7foxNAZRiwoopf7ZJxltDu
|
||||
PIuFjmVZqdzR8c6vmqIqdawx/V6sL9fizZr+CDH3oTsTUirn2qM+1ibBtPDiBvfX
|
||||
omUsr6MVOcTtvnMvAdy9NfV88qwF7MEWBGCjXkoT1bKCLD8hjn8l7GjRmPcmMFE2
|
||||
GqWEvfJiFkBK0CgSHYEUwzo0UtVNeQr0k0qkDRub6QIDAQABo1MwUTAdBgNVHQ4E
|
||||
FgQU5VeD67yFLV0QNYbHaJ6u9cM6UbkwHwYDVR0jBBgwFoAU5VeD67yFLV0QNYbH
|
||||
aJ6u9cM6UbkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEABueA
|
||||
8ujAD+yjeP5dTgqQH1G0hlriD5LmlJYnktaLarFU+y+EZlRFwjdORF/vLPwSG+y7
|
||||
CLty/xlmKKQop70QzQ5jtJcsWzUjww8w1sO3AevfZlIF3HNhJmt51ihfvtJ7DVCv
|
||||
CNyMeYO0pBqRKwOuhbG3EtJgyV7MF8J25UEtO4t+GzX3jcKKU4pWP+kyLBVfeDU3
|
||||
MQuigd2LBwBQQFxZdpYpcXVKnAJJlHZIt68ycO1oSBEJO9fIF0CiAlC6ITxjtYtz
|
||||
oCjd6cCLKMJiC6Zg7t1Q17vGl+FdGyQObSsiYsYO9N3CVaeDdpyGCH0Rfa0+oZzu
|
||||
a5U9/l1FHlvpX980bw==
|
||||
MIIEJDCCAoygAwIBAgIQfbdj1KSREvW82wxWZcDRsDANBgkqhkiG9w0BAQsFADBf
|
||||
MR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExGjAYBgNVBAsMEXJvb3RA
|
||||
OTIxZjA1YmRkNDk4MSEwHwYDVQQDDBhta2NlcnQgcm9vdEA5MjFmMDViZGQ0OTgw
|
||||
HhcNMjYwMTIyMjAwMzIwWhcNMjgwNDIyMjAwMzIwWjBFMScwJQYDVQQKEx5ta2Nl
|
||||
cnQgZGV2ZWxvcG1lbnQgY2VydGlmaWNhdGUxGjAYBgNVBAsMEXJvb3RANzk2OWU1
|
||||
ZjA2MGM4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2u+HpnTr+Ecj
|
||||
X6Ur4I8YHVKiEIgOFozJWWAeCPSZg0lr9/z3UDl7QPfRKaDk11OMN7DB0Y/ZAIHj
|
||||
kMFdYc+ODOUAKGv/MvlM41OeRUXhrt0Gr/TJoDe9RLzs9ffASpqTHUNNrmyj/fLP
|
||||
M5VZU+4Y2POFq8qji6Otkvqr6wp+2CTS/fkAtenFS2X+Z5u6BBq1/KBSWZhUFSuJ
|
||||
sAGN9u+l20Cj/Sxp2nD1npvMEvPehFKEK2tZGgFr+X426jJ4Znd19EyZoI/xetG6
|
||||
ybSzBdQk1KbhWFa3LPuOG814m9Qh21vaL0Hj2hpqC3KEQ2jiuCovjRS+RBGatltl
|
||||
5m47Cj0UrQIDAQABo3YwdDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYB
|
||||
BQUHAwEwHwYDVR0jBBgwFoAUjf0NOmUuoqSSD1Mbu/3G+wYxvjEwLAYDVR0RBCUw
|
||||
I4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3DQEB
|
||||
CwUAA4IBgQAorThUyu4FxHkOafMp/4CmvptfNvYeYP81DW0wpJxGL68Bz+idvXKv
|
||||
JopX/ZAr+dDhS/TLeSnzt83W7TaBBHY+usvsiBx9+pZOVZIpreXRamPu7utmuC46
|
||||
dictMNGlRNX9bwAApOJ24NCpOVSIIKtjHcjl4idwUHqLVGS+3wsmxIILYxigzkuT
|
||||
fcK5vs0ItZWeuunsBAGb/U/Iu9zZ71rtmBejxNPyEvd8+bZC0m2mtV8C0Lpn58jZ
|
||||
FiEf5OHiOdWG9O/uh3QeVWkuKLmaH6a8VdKRSIlOxEEZdkUYlwuVvzeWgFw4kjm8
|
||||
rNWz0aIPovmcgLXoUG1d1D8klveGd3plF7N2p3xWqKmS6R6FJHx7MIxH6XmBATii
|
||||
x/193Sgzqe8mwXQr14ulg2M/B3ZWleNdD6SeieADSgvRKjnlO7xwUmcFrffnEKEG
|
||||
WcSPoGIuZ8V2pkYLh7ipPN+tFUbhmrWnsao5kQ0sqLlfscyO9hYQeQRtTjtC3P8r
|
||||
FJYOdBMOCzE=
|
||||
-----END CERTIFICATE-----
|
||||
|
||||
@@ -1,28 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC5Qka1JnN35mku
|
||||
L55+OuTEkk19XFXPZUuGSc0zZYp45jElMr9VpNirmGtzPLp8RScyEMAkKAQ9QWU4
|
||||
re+fWAUko0w3ybtquQtWTo9i2vHwJPDDvxDToUshOvqFHS+Uk2k3gyyYbFjBShSm
|
||||
fWPkl19kXt+jE0BlGLCiil/tknGW0O48i4WOZVmp3NHxzq+aoip1rDH9Xqwv1+LN
|
||||
mv4IMfehOxNSKufaoz7WJsG08OIG99eiZSyvoxU5xO2+cy8B3L019XzyrAXswRYE
|
||||
YKNeShPVsoIsPyGOfyXsaNGY9yYwUTYapYS98mIWQErQKBIdgRTDOjRS1U15CvST
|
||||
SqQNG5vpAgMBAAECggEAAnv0Dw1Mv+rRy4ZyxtObEVPXPRzoxnDDXzHP4E16BTye
|
||||
Fc/4pSBUIAUn2bPvLz0/X8bMOa4dlDcIv7Eu9Pvns8AY70vMaUReA80fmtHVD2xX
|
||||
1PCT0X3InnxRAYKstSIUIGs+aHvV5Z+iJ8F82soOStN1MU56h+JLWElL5deCPHq3
|
||||
tLZT8wM9aOZlNG72kJ71+DlcViahynQj8+VrionOLNjTJ2Jv/ByjM3GMIuSdBrgd
|
||||
Sl4YAcdn6ontjJGoTgI+e+qkBAPwMZxHarNGQgbS0yNVIJe7Lq4zIKHErU/ZSmpD
|
||||
GzhdVNzhrjADNIDzS7G+pxtz+aUxGtmRvOyopy8GAQKBgQDEPp2mRM+uZVVT4e1j
|
||||
pkKO1c3O8j24I5mGKwFqhhNs3qGy051RXZa0+cQNx63GokXQan9DIXzc/Il7Y72E
|
||||
z9bCFbcSWnlP8dBIpWiJm+UmqLXRyY4N8ecNnzL5x+Tuxm5Ij+ixJwXgdz/TLNeO
|
||||
MBzu+Qy738/l/cAYxwcF7mR7AQKBgQDxq1F95HzCxBahRU9OGUO4s3naXqc8xKCC
|
||||
m3vbbI8V0Exse2cuiwtlPPQWzTPabLCJVvCGXNru98sdeOu9FO9yicwZX0knOABK
|
||||
QfPyDeITsh2u0C63+T9DNn6ixI/T68bTs7DHawEYbpS7bR50BnbHbQrrOAo6FSXF
|
||||
yC7+Te+o6QKBgQCXEWSmo/4D0Dn5Usg9l7VQ40GFd3EPmUgLwntal0/I1TFAyiom
|
||||
gpcLReIogXhCmpSHthO1h8fpDfZ/p+4ymRRHYBQH6uHMKugdpEdu9zVVpzYgArp5
|
||||
/afSEqVZJwoSzWoELdQA23toqiPV2oUtDdiYFdw5nDccY1RHPp8nb7amAQKBgQDj
|
||||
f4DhYDxKJMmg21xCiuoDb4DgHoaUYA0xpii8cL9pq4KmBK0nVWFO1kh5Robvsa2m
|
||||
PB+EfNjkaIPepLxWbOTUEAAASoDU2JT9UoTQcl1GaUAkFnpEWfBB14TyuNMkjinH
|
||||
lLpvn72SQFbm8VvfoU4jgfTrZP/LmajLPR1v6/IWMQKBgBh9qvOTax/GugBAWNj3
|
||||
ZvF99rHOx0rfotEdaPcRN66OOiSWILR9yfMsTvwt1V0VEj7OqO9juMRFuIyB57gd
|
||||
Hs/zgbkuggqjr1dW9r22P/UpzpodAEEN2d52RSX8nkMOkH61JXlH2MyRX65kdExA
|
||||
VkTDq6KwomuhrU3z0+r/MSOn
|
||||
MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDa74emdOv4RyNf
|
||||
pSvgjxgdUqIQiA4WjMlZYB4I9JmDSWv3/PdQOXtA99EpoOTXU4w3sMHRj9kAgeOQ
|
||||
wV1hz44M5QAoa/8y+UzjU55FReGu3Qav9MmgN71EvOz198BKmpMdQ02ubKP98s8z
|
||||
lVlT7hjY84WryqOLo62S+qvrCn7YJNL9+QC16cVLZf5nm7oEGrX8oFJZmFQVK4mw
|
||||
AY3276XbQKP9LGnacPWem8wS896EUoQra1kaAWv5fjbqMnhmd3X0TJmgj/F60brJ
|
||||
tLMF1CTUpuFYVrcs+44bzXib1CHbW9ovQePaGmoLcoRDaOK4Ki+NFL5EEZq2W2Xm
|
||||
bjsKPRStAgMBAAECggEBALFhpHwe+xh7OpPBlR0pkpYfXyMZuKBYjMIW9/61frM6
|
||||
B3oywIWFLPFkV1js/Lvg+xgb48zQSTb6BdBAelJHAYY8+7XEWk2IYt1D4FWr2r/8
|
||||
X/Cr2bgvsO9CSpK2mltXhZ4N66BIcU3NLkdS178Ch6svErwvP/ZhNL6Czktug3rG
|
||||
S2fxpKqoVQhqWiEBV0vBWpw2oskvcpP2Btev2eaJeDmP1IkCaKIU9jJzmSg8UKj3
|
||||
AxJJ8lJvlDi2Z8mfB0BVIFZSI1s44LqbuPdITsWvG49srdAhyLkifcbY2r7eH8+s
|
||||
rFNswCaqqNwmzZXHMFedfvgVJHtCTGX/U8G55dcG/YECgYEA6PvcX+axT4aSaG+/
|
||||
fQpyW8TmNmIwS/8HD8rA5dSBB2AE+RnqbxxB1SWWk4w47R/riYOJCy6C88XUZWLn
|
||||
05cYotR9lA6tZOYqIPUIxfCHl4vDDBnOGAB0ga3z5vF3X7HV6uspPFcrGVJ9k42S
|
||||
BK1gk7Kj8RxVQMqXSqkR/pXJ4l0CgYEA8JBkZ4MRi1D74SAU+TY1rnUCTs2EnP1U
|
||||
TKAI9qHyvNJOFZaObWPzegkZryZTm+yTblwvYkryMjvsiuCdv6ro/VKc667F3OBs
|
||||
dJ/8+ylO0lArP9a23QHViNSvQmyi3bsw2UpIGQRGq2C4LxDceo5cYoVOWIlLl8u3
|
||||
LUuL0IfxdpECgYEAyMd0DPljyGLyfSoAXaPJFajDtA4+DOAEl+lk/yt43oAzCPD6
|
||||
hTJW0XcJIrJuxHsDoohGa+pzU90iwxTPMBtAUeLJLfTQHOn1WF2SZ/J3B3ScbCs4
|
||||
3ppVzQO580YYV9GLxl1ONf/w1muuaKBSO9GmLuJ+QeTm22U7qE23gixXxMkCgYEA
|
||||
3R7cK4l+huBZpgUnQitiDInhJS4jx2nUItq3YnxZ8tYckBtjr4lAM9xJj4VbNOew
|
||||
XLC/nUnmdeY+9yif153xq2hUdQ6hMPXYuxqUHwlJOmgWWQez7lHRRYS51ASnb8iw
|
||||
jgqJWvVjQAQXSKvm/X/9y1FdQmRw54aJSUk3quZKPQECgYA5DqFXn/jYOUCGfCUD
|
||||
RENWe+3fNZSJCWmr4yvjEy9NUT5VcA+/hZHbbLUnVHfAHxJmdGhTzFe+ZGWFAGwi
|
||||
Wo62BDu1fqHHegCWaFFluDnz8iZTxXtHMEGSBZnXC4f5wylcxRtCPUbrLMRTPo4O
|
||||
t85qeBu1dMq1XQtU6ab3w3W0nw==
|
||||
-----END PRIVATE KEY-----
|
||||
|
||||
27
certs/mkcert-ca.crt
Normal file
27
certs/mkcert-ca.crt
Normal file
@@ -0,0 +1,27 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIEjjCCAvagAwIBAgIRAJtq2Z0+W981Ct2dMVPb3bQwDQYJKoZIhvcNAQELBQAw
|
||||
XzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMRowGAYDVQQLDBFyb290
|
||||
QDkyMWYwNWJkZDQ5ODEhMB8GA1UEAwwYbWtjZXJ0IHJvb3RAOTIxZjA1YmRkNDk4
|
||||
MB4XDTI2MDEyMjA5NDcxNVoXDTM2MDEyMjA5NDcxNVowXzEeMBwGA1UEChMVbWtj
|
||||
ZXJ0IGRldmVsb3BtZW50IENBMRowGAYDVQQLDBFyb290QDkyMWYwNWJkZDQ5ODEh
|
||||
MB8GA1UEAwwYbWtjZXJ0IHJvb3RAOTIxZjA1YmRkNDk4MIIBojANBgkqhkiG9w0B
|
||||
AQEFAAOCAY8AMIIBigKCAYEAxXR5gXDwv5cfQSud1eEhwDuaSaf5kf8NtPnucZXY
|
||||
AN+/QW1+OEKJFuawj/YrSbL/yIB8sUSJToEYNJ4LAgzIZ4+TPYVvOIPqQnimfj98
|
||||
2AKCt73U/AMvoEpts7U0s37f5wF8o+BHXfChxyN//z96+wsQ+2+Q9QBGjirvScF+
|
||||
8FRnupcygDeGZ8x3JQVaEfEV6iYyXFl/4tEDVr9QX4avyUlf0vp1Y90TG3L42JYQ
|
||||
xDU37Ct9dqsxPCPOPjmkQi9HV5TeqLTs/4NdrEYOSk7bOVMzL8EHs2prRL7sWzYJ
|
||||
gRT+VXFPpoSCkZs1gS3FNXukTGx5LNsstyJZRa99cGgDcqvNseig06KUzZrRnCig
|
||||
kARLF/n8VTpHETEuTdxdnXJO3i2N/99mG/2/lej9HNDMaqg45ur5EhaFhHarXMtc
|
||||
wy7nfGoThzscZvpFbVHorhgRxzsUqRHTMHa9mUOYtShMA0IwFccHcczY3CDxXLg9
|
||||
hC+24pdiCxtRmi23JB10Th+nAgMBAAGjRTBDMA4GA1UdDwEB/wQEAwICBDASBgNV
|
||||
HRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBSN/Q06ZS6ipJIPUxu7/cb7BjG+MTAN
|
||||
BgkqhkiG9w0BAQsFAAOCAYEAghQL80XpPs70EbCmETYri5ryjJPbsxK3Z3QgBYoy
|
||||
/P/021eK5woLx4EDeEUyrRayOtLlDWiIMdV7FmlY4GpEdonVbqFJY2gghkIuQpkw
|
||||
lWqbk08F+iXe8PSGD1qz4y6enioCRAx+RaZ6jnVJtaC8AR257FVhGIzDBiOA+LDM
|
||||
L1yq6Bxxij17q9s5HL9KuzxWRMuXACHmaGBXHpl/1n4dIxi2lXRgp+1xCR/VPNPt
|
||||
/ZRy29kncd8Fxx+VEtc0muoJvRo4ttVWhBvAVJrkAeukjYKpcDDfRU6Y22o54jD/
|
||||
mseDb+0UPwSxaKbnGJlCcRbbh6i14bA4KfPq93bZX+Tlq9VCp8LvQSl3oU+23RVc
|
||||
KjBB9EJnoBBNGIY7VmRI+QowrnlP2wtg2fTNaPqULtjqA9frbMTP0xTlumLzGB+6
|
||||
9Da7/+AE2in3Aa8Xyry4BiXbk2L6c1xz/Cd1ZpFrSXAOEi1Xt7so/Ck0yM3c2KWK
|
||||
5aSfCjjOzONMPJyY1oxodJ1p
|
||||
-----END CERTIFICATE-----
|
||||
@@ -46,6 +46,10 @@ services:
|
||||
- node_modules_data:/app/node_modules
|
||||
# Mount PostgreSQL logs for Logstash access (ADR-050)
|
||||
- postgres_logs:/var/log/postgresql:ro
|
||||
# Mount Redis logs for Logstash access (ADR-050)
|
||||
- redis_logs:/var/log/redis:ro
|
||||
# Mount PM2 logs for Logstash access (ADR-050, ADR-014)
|
||||
- pm2_logs:/var/log/pm2
|
||||
ports:
|
||||
- '80:80' # HTTP redirect to HTTPS (matches production)
|
||||
- '443:443' # Frontend HTTPS (nginx proxies Vite 5173 → 443)
|
||||
@@ -166,10 +170,17 @@ services:
|
||||
redis:
|
||||
image: docker.io/library/redis:alpine
|
||||
container_name: flyer-crawler-redis
|
||||
# Run as root to allow entrypoint to set up log directory permissions
|
||||
# Redis will drop privileges after setup via gosu in entrypoint
|
||||
user: root
|
||||
ports:
|
||||
- '6379:6379'
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
# Create log volume for Logstash access (ADR-050)
|
||||
- redis_logs:/var/log/redis
|
||||
# Mount custom entrypoint for log directory setup (ADR-050)
|
||||
- ./docker/redis/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh:ro
|
||||
# Healthcheck ensures redis is ready before app starts
|
||||
healthcheck:
|
||||
test: ['CMD', 'redis-cli', 'ping']
|
||||
@@ -177,8 +188,17 @@ services:
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
# Enable persistence for development data
|
||||
command: redis-server --appendonly yes
|
||||
# Enable persistence and file logging for Logstash (ADR-050)
|
||||
# Redis log levels: debug, verbose, notice (default), warning
|
||||
# Custom entrypoint sets up log directory with correct permissions
|
||||
entrypoint: ['/usr/local/bin/docker-entrypoint.sh']
|
||||
command:
|
||||
- --appendonly
|
||||
- 'yes'
|
||||
- --logfile
|
||||
- /var/log/redis/redis-server.log
|
||||
- --loglevel
|
||||
- notice
|
||||
|
||||
# ===================
|
||||
# Named Volumes
|
||||
@@ -190,6 +210,10 @@ volumes:
|
||||
name: flyer-crawler-postgres-logs
|
||||
redis_data:
|
||||
name: flyer-crawler-redis-data
|
||||
redis_logs:
|
||||
name: flyer-crawler-redis-logs
|
||||
pm2_logs:
|
||||
name: flyer-crawler-pm2-logs
|
||||
node_modules_data:
|
||||
name: flyer-crawler-node-modules
|
||||
|
||||
|
||||
461
docker/logstash/bugsink.conf
Normal file
461
docker/logstash/bugsink.conf
Normal file
@@ -0,0 +1,461 @@
|
||||
# docker/logstash/bugsink.conf
|
||||
# ============================================================================
|
||||
# Logstash Pipeline Configuration for Bugsink Error Tracking
|
||||
# ============================================================================
|
||||
# This configuration aggregates logs from multiple sources and forwards errors
|
||||
# to Bugsink (Sentry-compatible error tracking) in the dev container.
|
||||
#
|
||||
# Sources:
|
||||
# - PM2 managed logs (/var/log/pm2/*.log) - API, Worker, Vite (ADR-014)
|
||||
# - Pino application logs (/app/logs/*.log) - JSON format (fallback)
|
||||
# - PostgreSQL logs (/var/log/postgresql/*.log) - Including fn_log() output
|
||||
# - NGINX logs (/var/log/nginx/*.log) - Access and error logs
|
||||
# - Redis logs (/var/log/redis/*.log) - Via shared volume (ADR-050)
|
||||
#
|
||||
# Bugsink Projects:
|
||||
# - Project 1: Backend API (Dev) - Pino errors, PostgreSQL errors
|
||||
# - Project 2: Frontend (Dev) - Configured via Sentry SDK in browser
|
||||
#
|
||||
# Related Documentation:
|
||||
# - docs/adr/0050-postgresql-function-observability.md
|
||||
# - docs/adr/0015-application-performance-monitoring-and-error-tracking.md
|
||||
# - docs/operations/LOGSTASH-QUICK-REF.md
|
||||
# ============================================================================
|
||||
|
||||
input {
|
||||
# ============================================================================
|
||||
# PM2 Managed Process Logs (ADR-014, ADR-050)
|
||||
# ============================================================================
|
||||
# PM2 manages all Node.js processes in the dev container, matching production.
|
||||
# Logs are written to /var/log/pm2 for Logstash integration.
|
||||
#
|
||||
# Process logs:
|
||||
# - api-out.log / api-error.log: API server (Pino JSON format)
|
||||
# - worker-out.log / worker-error.log: Background worker (Pino JSON format)
|
||||
# - vite-out.log / vite-error.log: Vite dev server (plain text)
|
||||
# ============================================================================
|
||||
|
||||
# PM2 API Server Logs (Pino JSON format)
|
||||
file {
|
||||
path => "/var/log/pm2/api-out.log"
|
||||
codec => json_lines
|
||||
type => "pm2_api"
|
||||
tags => ["app", "backend", "pm2", "api"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_api_out"
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/pm2/api-error.log"
|
||||
codec => json_lines
|
||||
type => "pm2_api"
|
||||
tags => ["app", "backend", "pm2", "api", "stderr"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_api_error"
|
||||
}
|
||||
|
||||
# PM2 Worker Logs (Pino JSON format)
|
||||
file {
|
||||
path => "/var/log/pm2/worker-out.log"
|
||||
codec => json_lines
|
||||
type => "pm2_worker"
|
||||
tags => ["app", "worker", "pm2"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_worker_out"
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/pm2/worker-error.log"
|
||||
codec => json_lines
|
||||
type => "pm2_worker"
|
||||
tags => ["app", "worker", "pm2", "stderr"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_worker_error"
|
||||
}
|
||||
|
||||
# PM2 Vite Logs (plain text format)
|
||||
file {
|
||||
path => "/var/log/pm2/vite-out.log"
|
||||
codec => plain
|
||||
type => "pm2_vite"
|
||||
tags => ["app", "frontend", "pm2", "vite"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_vite_out"
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/pm2/vite-error.log"
|
||||
codec => plain
|
||||
type => "pm2_vite"
|
||||
tags => ["app", "frontend", "pm2", "vite", "stderr"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_vite_error"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Pino Application Logs (Fallback Path)
|
||||
# ============================================================================
|
||||
# JSON-formatted logs from the Node.js application using Pino logger.
|
||||
# Note: Primary logs now go to /var/log/pm2. This is a fallback.
|
||||
# Log levels: 10=trace, 20=debug, 30=info, 40=warn, 50=error, 60=fatal
|
||||
file {
|
||||
path => "/app/logs/*.log"
|
||||
codec => json
|
||||
type => "pino"
|
||||
tags => ["app", "backend"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pino"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PostgreSQL Function Logs (ADR-050)
|
||||
# ============================================================================
|
||||
# Captures PostgreSQL log output including fn_log() structured JSON messages.
|
||||
# PostgreSQL is configured to write logs to /var/log/postgresql/ (shared volume).
|
||||
# Log format: "2026-01-22 00:00:00 UTC [5724] postgres@flyer_crawler_dev LOG: message"
|
||||
file {
|
||||
path => "/var/log/postgresql/*.log"
|
||||
type => "postgres"
|
||||
tags => ["postgres", "database"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_postgres"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# NGINX Logs
|
||||
# ============================================================================
|
||||
# Access logs for request monitoring and error logs for proxy/SSL issues.
|
||||
file {
|
||||
path => "/var/log/nginx/access.log"
|
||||
type => "nginx_access"
|
||||
tags => ["nginx", "access"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_nginx_access"
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/nginx/error.log"
|
||||
type => "nginx_error"
|
||||
tags => ["nginx", "error"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_nginx_error"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Redis Logs (ADR-050)
|
||||
# ============================================================================
|
||||
# Redis logs from the shared volume. Redis uses its own log format:
|
||||
# "<pid>:<role> <day> <month> <time> <loglevel> <message>"
|
||||
# Example: "1:M 22 Jan 2026 14:30:00.123 * Ready to accept connections"
|
||||
# Log levels: . (debug), - (verbose), * (notice), # (warning)
|
||||
file {
|
||||
path => "/var/log/redis/redis-server.log"
|
||||
type => "redis"
|
||||
tags => ["redis", "infra"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_redis"
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
# ============================================================================
|
||||
# PM2 API/Worker Log Processing (ADR-014)
|
||||
# ============================================================================
|
||||
# PM2 API and Worker logs are in Pino JSON format.
|
||||
# Process them the same way as direct Pino logs.
|
||||
if [type] in ["pm2_api", "pm2_worker"] {
|
||||
# Tag errors (level 50 = error, 60 = fatal)
|
||||
if [level] >= 50 {
|
||||
mutate { add_tag => ["error", "pm2_pino_error"] }
|
||||
|
||||
# Map Pino level to Sentry level and set error_message field
|
||||
if [level] == 60 {
|
||||
mutate { add_field => { "sentry_level" => "fatal" } }
|
||||
} else {
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
}
|
||||
|
||||
# Copy msg to error_message for consistent access in output
|
||||
mutate { add_field => { "error_message" => "%{msg}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PM2 Vite Log Processing (ADR-014)
|
||||
# ============================================================================
|
||||
# Vite logs are plain text. Look for error patterns.
|
||||
if [type] == "pm2_vite" {
|
||||
# Detect Vite build/compilation errors
|
||||
if [message] =~ /(?i)(error|failed|exception|cannot|enoent|eperm|EACCES|ECONNREFUSED)/ {
|
||||
mutate { add_tag => ["error", "vite_error"] }
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
mutate { add_field => { "error_message" => "Vite: %{message}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Pino Log Processing (Fallback)
|
||||
# ============================================================================
|
||||
if [type] == "pino" {
|
||||
# Tag errors (level 50 = error, 60 = fatal)
|
||||
if [level] >= 50 {
|
||||
mutate { add_tag => ["error", "pino_error"] }
|
||||
|
||||
# Map Pino level to Sentry level and set error_message field
|
||||
if [level] == 60 {
|
||||
mutate { add_field => { "sentry_level" => "fatal" } }
|
||||
} else {
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
}
|
||||
|
||||
# Copy msg to error_message for consistent access in output
|
||||
mutate { add_field => { "error_message" => "%{msg}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PostgreSQL Log Processing (ADR-050)
|
||||
# ============================================================================
|
||||
# PostgreSQL log format in dev container:
|
||||
# "2026-01-22 00:00:00 UTC [5724] postgres@flyer_crawler_dev LOG: message"
|
||||
# "2026-01-22 07:06:03 UTC [19851] postgres@flyer_crawler_dev ERROR: column "id" does not exist"
|
||||
if [type] == "postgres" {
|
||||
# Parse PostgreSQL log prefix with UTC timezone
|
||||
grok {
|
||||
match => { "message" => "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME} %{WORD:pg_timezone} \[%{POSINT:pg_pid}\] %{DATA:pg_user}@%{DATA:pg_database} %{WORD:pg_level}: ?%{GREEDYDATA:pg_message}" }
|
||||
tag_on_failure => ["_postgres_grok_failure"]
|
||||
}
|
||||
|
||||
# Check if this is a structured JSON log from fn_log()
|
||||
if [pg_message] =~ /^\{.*"source":"postgresql".*\}$/ {
|
||||
json {
|
||||
source => "pg_message"
|
||||
target => "fn_log"
|
||||
}
|
||||
|
||||
# Mark as error if level is WARNING or ERROR
|
||||
if [fn_log][level] in ["WARNING", "ERROR"] {
|
||||
mutate { add_tag => ["error", "db_function"] }
|
||||
mutate { add_field => { "sentry_level" => "warning" } }
|
||||
if [fn_log][level] == "ERROR" {
|
||||
mutate { replace => { "sentry_level" => "error" } }
|
||||
}
|
||||
# Use fn_log message for error_message
|
||||
mutate { add_field => { "error_message" => "%{[fn_log][message]}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# Catch native PostgreSQL errors (ERROR: or FATAL: prefix in pg_level)
|
||||
if [pg_level] == "ERROR" or [pg_level] == "FATAL" {
|
||||
mutate { add_tag => ["error", "postgres_native"] }
|
||||
|
||||
if [pg_level] == "FATAL" {
|
||||
mutate { add_field => { "sentry_level" => "fatal" } }
|
||||
} else {
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
}
|
||||
|
||||
# Use the full pg_message for error_message
|
||||
mutate { add_field => { "error_message" => "PostgreSQL %{pg_level}: %{pg_message}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# NGINX Access Log Processing
|
||||
# ============================================================================
|
||||
if [type] == "nginx_access" {
|
||||
grok {
|
||||
match => { "message" => '%{IPORHOST:client_ip} - %{DATA:user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:user_agent}"' }
|
||||
tag_on_failure => ["_nginx_access_grok_failure"]
|
||||
}
|
||||
|
||||
# Tag 5xx errors for Bugsink
|
||||
if [status] =~ /^5/ {
|
||||
mutate { add_tag => ["error", "nginx_5xx"] }
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
mutate { add_field => { "error_message" => "HTTP %{status} %{method} %{request}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# NGINX Error Log Processing
|
||||
# ============================================================================
|
||||
# NGINX error log format: "2026/01/22 17:55:01 [error] 16#16: *3 message..."
|
||||
if [type] == "nginx_error" {
|
||||
grok {
|
||||
match => { "message" => "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME} \[%{LOGLEVEL:nginx_level}\] %{POSINT:pid}#%{POSINT}: (\*%{POSINT:connection} )?%{GREEDYDATA:nginx_message}" }
|
||||
tag_on_failure => ["_nginx_error_grok_failure"]
|
||||
}
|
||||
|
||||
# Only process actual errors, not notices (like "signal process started")
|
||||
if [nginx_level] in ["error", "crit", "alert", "emerg"] {
|
||||
mutate { add_tag => ["error", "nginx_error"] }
|
||||
|
||||
# Map NGINX log level to Sentry level
|
||||
if [nginx_level] in ["crit", "alert", "emerg"] {
|
||||
mutate { add_field => { "sentry_level" => "fatal" } }
|
||||
} else {
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
}
|
||||
|
||||
mutate { add_field => { "error_message" => "NGINX [%{nginx_level}]: %{nginx_message}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Redis Log Processing (ADR-050)
|
||||
# ============================================================================
|
||||
# Redis log format: "<pid>:<role> <day> <month> <time>.<ms> <level> <message>"
|
||||
# Example: "1:M 22 Jan 14:30:00.123 * Ready to accept connections"
|
||||
# Roles: M=master, S=slave, C=sentinel, X=cluster
|
||||
# Levels: . (debug), - (verbose), * (notice), # (warning)
|
||||
if [type] == "redis" {
|
||||
# Parse Redis log format
|
||||
grok {
|
||||
match => { "message" => "%{POSINT:redis_pid}:%{WORD:redis_role} %{MONTHDAY} %{MONTH} %{YEAR}? ?%{TIME} %{DATA:redis_level_char} %{GREEDYDATA:redis_message}" }
|
||||
tag_on_failure => ["_redis_grok_failure"]
|
||||
}
|
||||
|
||||
# Map Redis level characters to human-readable levels
|
||||
# . = debug, - = verbose, * = notice, # = warning
|
||||
if [redis_level_char] == "#" {
|
||||
mutate {
|
||||
add_field => { "redis_level" => "warning" }
|
||||
add_tag => ["error", "redis_warning"]
|
||||
}
|
||||
mutate { add_field => { "sentry_level" => "warning" } }
|
||||
mutate { add_field => { "error_message" => "Redis WARNING: %{redis_message}" } }
|
||||
} else if [redis_level_char] == "*" {
|
||||
mutate { add_field => { "redis_level" => "notice" } }
|
||||
} else if [redis_level_char] == "-" {
|
||||
mutate { add_field => { "redis_level" => "verbose" } }
|
||||
} else if [redis_level_char] == "." {
|
||||
mutate { add_field => { "redis_level" => "debug" } }
|
||||
}
|
||||
|
||||
# Also detect error keywords in message content (e.g., ECONNREFUSED, OOM, etc.)
|
||||
if [redis_message] =~ /(?i)(error|failed|refused|denied|timeout|oom|crash|fatal|exception)/ {
|
||||
if "error" not in [tags] {
|
||||
mutate { add_tag => ["error", "redis_error"] }
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
mutate { add_field => { "error_message" => "Redis ERROR: %{redis_message}" } }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Generate Sentry Event ID for all errors
|
||||
# ============================================================================
|
||||
if "error" in [tags] {
|
||||
ruby {
|
||||
code => '
|
||||
require "securerandom"
|
||||
event.set("sentry_event_id", SecureRandom.hex(16))
|
||||
'
|
||||
}
|
||||
|
||||
# Ensure error_message has a fallback value
|
||||
if ![error_message] {
|
||||
mutate { add_field => { "error_message" => "%{message}" } }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
# ============================================================================
|
||||
# Forward Errors to Bugsink (Backend API Project)
|
||||
# ============================================================================
|
||||
# Bugsink uses Sentry-compatible API. Events must include:
|
||||
# - event_id: 32 hex characters (UUID without dashes)
|
||||
# - message: Human-readable error description
|
||||
# - level: fatal, error, warning, info, debug
|
||||
# - timestamp: Unix epoch in seconds
|
||||
# - platform: "node" for backend, "javascript" for frontend
|
||||
#
|
||||
# Authentication via X-Sentry-Auth header with project's public key.
|
||||
# Dev container DSN: http://cea01396c56246adb5878fa5ee6b1d22@localhost:8000/1
|
||||
# ============================================================================
|
||||
if "error" in [tags] {
|
||||
http {
|
||||
url => "http://localhost:8000/api/1/store/"
|
||||
http_method => "post"
|
||||
format => "json"
|
||||
headers => {
|
||||
"X-Sentry-Auth" => "Sentry sentry_key=cea01396c56246adb5878fa5ee6b1d22, sentry_version=7"
|
||||
"Content-Type" => "application/json"
|
||||
}
|
||||
# Transform event to Sentry format using regular fields (not @metadata)
|
||||
mapping => {
|
||||
"event_id" => "%{sentry_event_id}"
|
||||
"timestamp" => "%{@timestamp}"
|
||||
"level" => "%{sentry_level}"
|
||||
"platform" => "node"
|
||||
"logger" => "%{type}"
|
||||
"message" => "%{error_message}"
|
||||
"extra" => {
|
||||
"hostname" => "%{[host][name]}"
|
||||
"source_type" => "%{type}"
|
||||
"tags" => "%{tags}"
|
||||
"original_message" => "%{message}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Store Operational Logs to Files (for debugging/audit)
|
||||
# ============================================================================
|
||||
# NGINX access logs (all requests, not just errors)
|
||||
if [type] == "nginx_access" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/nginx-access-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# PostgreSQL operational logs (non-error)
|
||||
if [type] == "postgres" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/postgres-operational-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# Redis operational logs (non-error)
|
||||
if [type] == "redis" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/redis-operational-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# PM2 API operational logs (non-error) - ADR-014
|
||||
if [type] == "pm2_api" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/pm2-api-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# PM2 Worker operational logs (non-error) - ADR-014
|
||||
if [type] == "pm2_worker" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/pm2-worker-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# PM2 Vite operational logs (non-error) - ADR-014
|
||||
if [type] == "pm2_vite" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/pm2-vite-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Debug Output (for development only)
|
||||
# ============================================================================
|
||||
# Uncomment to see all processed events in Logstash stdout
|
||||
# stdout { codec => rubydebug }
|
||||
}
|
||||
@@ -9,13 +9,39 @@
|
||||
# - Frontend accessible on https://localhost (port 443)
|
||||
# - Backend API on http://localhost:3001
|
||||
# - Port 80 redirects to HTTPS
|
||||
#
|
||||
# IMPORTANT: Dual Hostname Configuration (localhost AND 127.0.0.1)
|
||||
# ============================================================================
|
||||
# The server_name directive includes BOTH 'localhost' and '127.0.0.1' to
|
||||
# prevent SSL certificate errors when resources use different hostnames.
|
||||
#
|
||||
# Problem Scenario:
|
||||
# 1. User accesses site via https://localhost/
|
||||
# 2. Database stores image URLs as https://127.0.0.1/flyer-images/...
|
||||
# 3. Browser treats these as different origins, showing ERR_CERT_AUTHORITY_INVALID
|
||||
#
|
||||
# Solution:
|
||||
# - mkcert generates certificates valid for: localhost, 127.0.0.1, ::1
|
||||
# - NGINX accepts requests to BOTH hostnames using the same certificate
|
||||
# - Users can access via either hostname without SSL warnings
|
||||
#
|
||||
# The self-signed certificate is generated in Dockerfile.dev with:
|
||||
# mkcert localhost 127.0.0.1 ::1
|
||||
#
|
||||
# This creates a certificate with Subject Alternative Names (SANs) for all
|
||||
# three hostnames, allowing NGINX to serve valid HTTPS for each.
|
||||
#
|
||||
# See also:
|
||||
# - Dockerfile.dev (certificate generation, ~line 69)
|
||||
# - docs/FLYER-URL-CONFIGURATION.md (URL configuration details)
|
||||
# - docs/development/DEBUGGING.md (SSL troubleshooting)
|
||||
# ============================================================================
|
||||
|
||||
# HTTPS Server (main)
|
||||
server {
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
server_name localhost;
|
||||
server_name localhost 127.0.0.1;
|
||||
|
||||
# SSL Configuration (self-signed certificates from mkcert)
|
||||
ssl_certificate /app/certs/localhost.crt;
|
||||
@@ -80,7 +106,7 @@ server {
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name localhost;
|
||||
server_name localhost 127.0.0.1;
|
||||
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
|
||||
24
docker/redis/docker-entrypoint.sh
Normal file
24
docker/redis/docker-entrypoint.sh
Normal file
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
# docker/redis/docker-entrypoint.sh
|
||||
# ============================================================================
|
||||
# Redis Entrypoint Script for Dev Container
|
||||
# ============================================================================
|
||||
# This script ensures the Redis log directory exists and is writable before
|
||||
# starting Redis. This is needed because the redis_logs volume is mounted
|
||||
# at /var/log/redis but Redis Alpine runs as a non-root user.
|
||||
#
|
||||
# Related: ADR-050 (Log aggregation via Logstash)
|
||||
# ============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
# Create log directory if it doesn't exist
|
||||
mkdir -p /var/log/redis
|
||||
|
||||
# Ensure redis user can write to the log directory
|
||||
# Note: In Alpine Redis image, redis runs as uid 999
|
||||
chown -R redis:redis /var/log/redis
|
||||
chmod 755 /var/log/redis
|
||||
|
||||
# Start Redis with the provided arguments
|
||||
exec redis-server "$@"
|
||||
@@ -12,6 +12,87 @@ Flyer image and icon URLs are environment-specific to ensure they point to the c
|
||||
| Test | `https://flyer-crawler-test.projectium.com` | `https://flyer-crawler-test.projectium.com/flyer-images/safeway-flyer.jpg` |
|
||||
| Production | `https://flyer-crawler.projectium.com` | `https://flyer-crawler.projectium.com/flyer-images/safeway-flyer.jpg` |
|
||||
|
||||
**Note:** The dev container accepts connections to **both** `https://localhost/` and `https://127.0.0.1/` thanks to the SSL certificate and NGINX configuration. See [SSL Certificate Configuration](#ssl-certificate-configuration-dev-container) below.
|
||||
|
||||
## SSL Certificate Configuration (Dev Container)
|
||||
|
||||
The dev container uses self-signed certificates generated by [mkcert](https://github.com/FiloSottile/mkcert) to enable HTTPS locally. This configuration solves a common mixed-origin SSL issue.
|
||||
|
||||
### The Problem
|
||||
|
||||
When users access the site via `https://localhost/` but image URLs in the database use `https://127.0.0.1/...`, browsers treat these as different origins. This causes `ERR_CERT_AUTHORITY_INVALID` errors when loading images, even though both hostnames point to the same server.
|
||||
|
||||
### The Solution
|
||||
|
||||
1. **Certificate Generation** (`Dockerfile.dev`):
|
||||
|
||||
```bash
|
||||
mkcert localhost 127.0.0.1 ::1
|
||||
```
|
||||
|
||||
This creates a certificate with Subject Alternative Names (SANs) for all three hostnames.
|
||||
|
||||
2. **NGINX Configuration** (`docker/nginx/dev.conf`):
|
||||
|
||||
```nginx
|
||||
server_name localhost 127.0.0.1;
|
||||
```
|
||||
|
||||
NGINX accepts requests to both hostnames using the same SSL certificate.
|
||||
|
||||
### How It Works
|
||||
|
||||
| Component | Configuration |
|
||||
| -------------------- | ---------------------------------------------------- |
|
||||
| SSL Certificate SANs | `localhost`, `127.0.0.1`, `::1` |
|
||||
| NGINX `server_name` | `localhost 127.0.0.1` |
|
||||
| Seed Script URLs | Uses `https://127.0.0.1` (works with DB constraints) |
|
||||
| User Access | Either `https://localhost/` or `https://127.0.0.1/` |
|
||||
|
||||
### Why This Matters
|
||||
|
||||
- **Database Constraints**: The `flyers` table has CHECK constraints requiring URLs to start with `http://` or `https://`. Relative URLs are not allowed.
|
||||
- **Consistent Behavior**: Users can access the site using either hostname without SSL warnings.
|
||||
- **Same Certificate**: Both hostnames use the same self-signed certificate, eliminating mixed-content errors.
|
||||
|
||||
### Verifying the Configuration
|
||||
|
||||
```bash
|
||||
# Check certificate SANs
|
||||
podman exec flyer-crawler-dev openssl x509 -in /app/certs/localhost.crt -text -noout | grep -A1 "Subject Alternative Name"
|
||||
|
||||
# Expected output:
|
||||
# X509v3 Subject Alternative Name:
|
||||
# DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1
|
||||
|
||||
# Test both hostnames respond
|
||||
curl -k https://localhost/health
|
||||
curl -k https://127.0.0.1/health
|
||||
```
|
||||
|
||||
### Troubleshooting SSL Issues
|
||||
|
||||
If you encounter `ERR_CERT_AUTHORITY_INVALID`:
|
||||
|
||||
1. **Check NGINX is running**: `podman exec flyer-crawler-dev nginx -t`
|
||||
2. **Verify certificate exists**: `podman exec flyer-crawler-dev ls -la /app/certs/`
|
||||
3. **Ensure both hostnames are in server_name**: Check `/etc/nginx/sites-available/default`
|
||||
4. **Rebuild container if needed**: The certificate is generated at build time
|
||||
|
||||
### Permanent Fix: Install CA Certificate (Recommended)
|
||||
|
||||
To permanently eliminate SSL certificate warnings, install the mkcert CA certificate on your system. This is optional but provides a better development experience.
|
||||
|
||||
The CA certificate is located at `certs/mkcert-ca.crt` in the project root. See [`certs/README.md`](../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
|
||||
|
||||
After installation:
|
||||
|
||||
- Your browser will trust all mkcert certificates without warnings
|
||||
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
|
||||
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
|
||||
|
||||
See also: [Debugging Guide - SSL Issues](development/DEBUGGING.md#ssl-certificate-issues)
|
||||
|
||||
## NGINX Static File Serving
|
||||
|
||||
All environments serve flyer images as static files with browser caching:
|
||||
@@ -41,7 +122,7 @@ Set `FLYER_BASE_URL` in your environment configuration:
|
||||
|
||||
```bash
|
||||
# Dev container (.env)
|
||||
FLYER_BASE_URL=https://127.0.0.1
|
||||
FLYER_BASE_URL=https://localhost
|
||||
|
||||
# Test environment
|
||||
FLYER_BASE_URL=https://flyer-crawler-test.projectium.com
|
||||
@@ -58,7 +139,7 @@ The seed script ([src/db/seed.ts](../src/db/seed.ts)) automatically uses the cor
|
||||
2. `NODE_ENV` value:
|
||||
- `production` → `https://flyer-crawler.projectium.com`
|
||||
- `test` → `https://flyer-crawler-test.projectium.com`
|
||||
- Default → `https://127.0.0.1`
|
||||
- Default → `https://localhost`
|
||||
|
||||
The seed script also copies test images from `src/tests/assets/` to `public/flyer-images/`:
|
||||
|
||||
@@ -78,8 +159,8 @@ podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev
|
||||
# Run the update (dev container uses HTTPS with self-signed certs)
|
||||
UPDATE flyers
|
||||
SET
|
||||
image_url = REPLACE(image_url, 'example.com', '127.0.0.1'),
|
||||
icon_url = REPLACE(icon_url, 'example.com', '127.0.0.1')
|
||||
image_url = REPLACE(image_url, 'example.com', 'localhost'),
|
||||
icon_url = REPLACE(icon_url, 'example.com', 'localhost')
|
||||
WHERE
|
||||
image_url LIKE '%example.com%'
|
||||
OR icon_url LIKE '%example.com%';
|
||||
@@ -131,8 +212,10 @@ export const getFlyerBaseUrl = (): string => {
|
||||
}
|
||||
|
||||
// Check if we're in dev container (DB_HOST=postgres is typical indicator)
|
||||
// Use 'localhost' instead of '127.0.0.1' to match the hostname users access
|
||||
// This avoids SSL certificate mixed-origin issues in browsers
|
||||
if (process.env.DB_HOST === 'postgres' || process.env.DB_HOST === '127.0.0.1') {
|
||||
return 'https://127.0.0.1';
|
||||
return 'https://localhost';
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
@@ -169,13 +252,13 @@ This approach ensures tests work correctly in all environments (dev container, C
|
||||
| `docker/nginx/dev.conf` | Added `/flyer-images/` location block for static file serving |
|
||||
| `.env.example` | Added `FLYER_BASE_URL` variable |
|
||||
| `sql/update_flyer_urls.sql` | SQL script for updating existing data |
|
||||
| Test files | Updated mock data to use `https://127.0.0.1` |
|
||||
| Test files | Updated mock data to use `https://localhost` |
|
||||
|
||||
## Summary
|
||||
|
||||
- Seed script now uses environment-specific HTTPS URLs
|
||||
- Seed script copies test images from `src/tests/assets/` to `public/flyer-images/`
|
||||
- NGINX serves `/flyer-images/` as static files with 7-day cache
|
||||
- Test files updated with `https://127.0.0.1`
|
||||
- Test files updated with `https://localhost` (not `127.0.0.1` to avoid SSL mixed-origin issues)
|
||||
- SQL script provided for updating existing data
|
||||
- Documentation updated for each environment
|
||||
|
||||
@@ -293,7 +293,23 @@ Google Gemini powers the AI extraction capabilities.
|
||||
|
||||
### Background Workers (BullMQ/PM2)
|
||||
|
||||
BullMQ workers handle asynchronous processing tasks.
|
||||
BullMQ workers handle asynchronous processing tasks. PM2 manages both the API server and worker processes in production and development environments (ADR-014).
|
||||
|
||||
**Dev Container PM2 Processes**:
|
||||
|
||||
| Process | Script | Purpose |
|
||||
| -------------------------- | ---------------------------------- | -------------------------- |
|
||||
| `flyer-crawler-api-dev` | `tsx watch server.ts` | API server with hot reload |
|
||||
| `flyer-crawler-worker-dev` | `tsx watch src/services/worker.ts` | Background job processing |
|
||||
| `flyer-crawler-vite-dev` | `vite --host` | Frontend dev server |
|
||||
|
||||
**Production PM2 Processes**:
|
||||
|
||||
| Process | Script | Purpose |
|
||||
| -------------------------------- | ------------------ | --------------------------- |
|
||||
| `flyer-crawler-api` | `tsx server.ts` | API server (cluster mode) |
|
||||
| `flyer-crawler-worker` | `tsx worker.ts` | Background job processing |
|
||||
| `flyer-crawler-analytics-worker` | `tsx analytics.ts` | Analytics report generation |
|
||||
|
||||
**Job Queues**:
|
||||
|
||||
@@ -651,9 +667,11 @@ Attempt 4: 4x delay (e.g., 20 seconds)
|
||||
| | Windows Host Machine | | Linux Dev Container | |
|
||||
| | - VS Code | | (flyer-crawler-dev) | |
|
||||
| | - Podman Desktop +---->+ - Node.js 22 | |
|
||||
| | - Git | | - PostgreSQL 16 | |
|
||||
| +-----------------------------------+ | - Redis 7 | |
|
||||
| | - Git | | - PM2 (process manager) | |
|
||||
| +-----------------------------------+ | - PostgreSQL 16 | |
|
||||
| | - Redis 7 | |
|
||||
| | - Bugsink (local) | |
|
||||
| | - Logstash (log aggregation) | |
|
||||
| +-----------------------------------+ |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ Common debugging strategies and troubleshooting patterns for Flyer Crawler.
|
||||
- [API Errors](#api-errors)
|
||||
- [Authentication Problems](#authentication-problems)
|
||||
- [Background Job Issues](#background-job-issues)
|
||||
- [SSL Certificate Issues](#ssl-certificate-issues)
|
||||
- [Frontend Issues](#frontend-issues)
|
||||
- [Performance Problems](#performance-problems)
|
||||
- [Debugging Tools](#debugging-tools)
|
||||
@@ -43,14 +44,24 @@ When something breaks, check these first:
|
||||
4. **Are there recent errors in logs?**
|
||||
|
||||
```bash
|
||||
# Application logs
|
||||
# PM2 logs (dev container)
|
||||
podman exec -it flyer-crawler-dev pm2 logs
|
||||
|
||||
# Container logs
|
||||
podman logs -f flyer-crawler-dev
|
||||
|
||||
# PM2 logs (production)
|
||||
pm2 logs flyer-crawler-api
|
||||
```
|
||||
|
||||
5. **Is Redis accessible?**
|
||||
5. **Are PM2 processes running?**
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
```
|
||||
|
||||
6. **Is Redis accessible?**
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-redis redis-cli ping
|
||||
```
|
||||
@@ -426,6 +437,92 @@ console.log('Expired:', decoded.exp < Date.now() / 1000);
|
||||
|
||||
---
|
||||
|
||||
## PM2 Process Issues (Dev Container)
|
||||
|
||||
### PM2 Process Not Starting
|
||||
|
||||
**Symptom**: `pm2 status` shows process as "errored" or "stopped"
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check process status
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
|
||||
# View process logs
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev --lines 50
|
||||
|
||||
# Show detailed process info
|
||||
podman exec -it flyer-crawler-dev pm2 show flyer-crawler-api-dev
|
||||
|
||||
# Check if port is in use
|
||||
podman exec -it flyer-crawler-dev netstat -tlnp | grep 3001
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Port already in use by another process
|
||||
- Missing environment variables
|
||||
- Syntax error in code
|
||||
- Database connection failure
|
||||
|
||||
**Solutions**:
|
||||
|
||||
```bash
|
||||
# Restart specific process
|
||||
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
|
||||
|
||||
# Restart all processes
|
||||
podman exec -it flyer-crawler-dev pm2 restart all
|
||||
|
||||
# Delete and recreate processes
|
||||
podman exec -it flyer-crawler-dev pm2 delete all
|
||||
podman exec -it flyer-crawler-dev pm2 start /app/ecosystem.dev.config.cjs
|
||||
```
|
||||
|
||||
### PM2 Log Access
|
||||
|
||||
**View logs in real-time**:
|
||||
|
||||
```bash
|
||||
# All processes
|
||||
podman exec -it flyer-crawler-dev pm2 logs
|
||||
|
||||
# Specific process
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
|
||||
|
||||
# Last 100 lines
|
||||
podman exec -it flyer-crawler-dev pm2 logs --lines 100
|
||||
```
|
||||
|
||||
**View log files directly**:
|
||||
|
||||
```bash
|
||||
# API logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-out.log
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-error.log
|
||||
|
||||
# Worker logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/worker-out.log
|
||||
|
||||
# Vite logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/vite-out.log
|
||||
```
|
||||
|
||||
### Redis Log Access
|
||||
|
||||
**View Redis logs**:
|
||||
|
||||
```bash
|
||||
# View Redis log file
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
|
||||
|
||||
# View from dev container (shared volume)
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/redis/redis-server.log
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Background Job Issues
|
||||
|
||||
### Jobs Not Processing
|
||||
@@ -433,10 +530,16 @@ console.log('Expired:', decoded.exp < Date.now() / 1000);
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check if worker is running
|
||||
# Check if worker is running (dev container)
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
|
||||
# Check worker logs (dev container)
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-worker-dev
|
||||
|
||||
# Check if worker is running (production)
|
||||
pm2 list
|
||||
|
||||
# Check worker logs
|
||||
# Check worker logs (production)
|
||||
pm2 logs flyer-crawler-worker
|
||||
|
||||
# Check Redis connection
|
||||
@@ -494,6 +597,77 @@ pm2 logs flyer-crawler-worker --lines 100
|
||||
|
||||
---
|
||||
|
||||
## SSL Certificate Issues
|
||||
|
||||
### Images Not Loading (ERR_CERT_AUTHORITY_INVALID)
|
||||
|
||||
**Symptom**: Flyer images fail to load with `ERR_CERT_AUTHORITY_INVALID` in browser console
|
||||
|
||||
**Cause**: Mixed hostname origins - user accesses via `localhost` but images use `127.0.0.1` (or vice versa)
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check which hostname images are using
|
||||
podman exec flyer-crawler-dev psql -U postgres -d flyer_crawler_dev \
|
||||
-c "SELECT image_url FROM flyers LIMIT 1;"
|
||||
|
||||
# Verify certificate includes both hostnames
|
||||
podman exec flyer-crawler-dev openssl x509 -in /app/certs/localhost.crt -text -noout | grep -A1 "Subject Alternative Name"
|
||||
|
||||
# Check NGINX accepts both hostnames
|
||||
podman exec flyer-crawler-dev grep "server_name" /etc/nginx/sites-available/default
|
||||
```
|
||||
|
||||
**Solution**: The dev container is configured to handle both hostnames:
|
||||
|
||||
1. Certificate includes SANs for `localhost`, `127.0.0.1`, and `::1`
|
||||
2. NGINX `server_name` directive includes both `localhost` and `127.0.0.1`
|
||||
|
||||
If you still see errors:
|
||||
|
||||
```bash
|
||||
# Rebuild container to regenerate certificate
|
||||
podman-compose down
|
||||
podman-compose build --no-cache flyer-crawler-dev
|
||||
podman-compose up -d
|
||||
```
|
||||
|
||||
See [FLYER-URL-CONFIGURATION.md](../FLYER-URL-CONFIGURATION.md#ssl-certificate-configuration-dev-container) for full details.
|
||||
|
||||
### Self-Signed Certificate Not Trusted
|
||||
|
||||
**Symptom**: Browser shows security warning for `https://localhost`
|
||||
|
||||
**Temporary Workaround**: Accept the warning by clicking "Advanced" > "Proceed to localhost"
|
||||
|
||||
**Permanent Fix (Recommended)**: Install the mkcert CA certificate to eliminate all SSL warnings.
|
||||
|
||||
The CA certificate is located at `certs/mkcert-ca.crt` in the project root. See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
|
||||
|
||||
After installation:
|
||||
|
||||
- Your browser will trust all mkcert certificates without warnings
|
||||
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
|
||||
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
|
||||
|
||||
### NGINX SSL Configuration Test
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Test NGINX configuration
|
||||
podman exec flyer-crawler-dev nginx -t
|
||||
|
||||
# Check if NGINX is listening on 443
|
||||
podman exec flyer-crawler-dev netstat -tlnp | grep 443
|
||||
|
||||
# Verify certificate files exist
|
||||
podman exec flyer-crawler-dev ls -la /app/certs/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Frontend Issues
|
||||
|
||||
### Hot Reload Not Working
|
||||
@@ -661,8 +835,10 @@ podman exec flyer-crawler-redis redis-cli info stats
|
||||
|
||||
## See Also
|
||||
|
||||
- [DEV-CONTAINER.md](DEV-CONTAINER.md) - Dev container guide (PM2, Logstash)
|
||||
- [TESTING.md](TESTING.md) - Testing strategies
|
||||
- [CODE-PATTERNS.md](CODE-PATTERNS.md) - Common patterns
|
||||
- [MONITORING.md](../operations/MONITORING.md) - Production monitoring
|
||||
- [LOGSTASH-QUICK-REF.md](../operations/LOGSTASH-QUICK-REF.md) - Log aggregation
|
||||
- [Bugsink Setup](../tools/BUGSINK-SETUP.md) - Error tracking
|
||||
- [DevOps Guide](../subagents/DEVOPS-GUIDE.md) - Container debugging
|
||||
|
||||
377
docs/development/DEV-CONTAINER.md
Normal file
377
docs/development/DEV-CONTAINER.md
Normal file
@@ -0,0 +1,377 @@
|
||||
# Dev Container Guide
|
||||
|
||||
Comprehensive documentation for the Flyer Crawler development container.
|
||||
|
||||
**Last Updated**: 2026-01-22
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Overview](#overview)
|
||||
2. [Architecture](#architecture)
|
||||
3. [PM2 Process Management](#pm2-process-management)
|
||||
4. [Log Aggregation](#log-aggregation)
|
||||
5. [Quick Reference](#quick-reference)
|
||||
6. [Troubleshooting](#troubleshooting)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
The dev container provides a production-like environment for local development. Key features:
|
||||
|
||||
- **PM2 Process Management** - Matches production architecture (ADR-014)
|
||||
- **Logstash Integration** - All logs forwarded to Bugsink (ADR-050)
|
||||
- **HTTPS by Default** - Self-signed certificates via mkcert
|
||||
- **Hot Reloading** - tsx watch mode for API and worker processes
|
||||
|
||||
### Container Services
|
||||
|
||||
| Service | Container Name | Purpose |
|
||||
| ----------- | ------------------------ | ------------------------------ |
|
||||
| Application | `flyer-crawler-dev` | Node.js app, Bugsink, Logstash |
|
||||
| PostgreSQL | `flyer-crawler-postgres` | Primary database with PostGIS |
|
||||
| Redis | `flyer-crawler-redis` | Cache and job queue backing |
|
||||
|
||||
### Access Points
|
||||
|
||||
| Service | URL | Notes |
|
||||
| ----------- | ------------------------ | ------------------------- |
|
||||
| Frontend | `https://localhost` | NGINX proxies Vite (5173) |
|
||||
| Backend API | `http://localhost:3001` | Express server |
|
||||
| Bugsink | `https://localhost:8443` | Error tracking UI |
|
||||
| PostgreSQL | `localhost:5432` | Direct database access |
|
||||
| Redis | `localhost:6379` | Direct Redis access |
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
### Production vs Development Parity
|
||||
|
||||
The dev container was updated to match production architecture (ADR-014):
|
||||
|
||||
| Component | Production | Dev Container (OLD) | Dev Container (NEW) |
|
||||
| ------------ | ---------------------- | ---------------------- | ----------------------- |
|
||||
| API Server | PM2 cluster mode | `npm run dev` (inline) | PM2 fork + tsx watch |
|
||||
| Worker | PM2 process | Inline with API | PM2 process + tsx watch |
|
||||
| Frontend | Static files via NGINX | Vite standalone | PM2 + Vite dev server |
|
||||
| Logs | PM2 logs -> Logstash | Console only | PM2 logs -> Logstash |
|
||||
| Process Mgmt | PM2 | None | PM2 |
|
||||
|
||||
### Container Startup Flow
|
||||
|
||||
When the container starts (`scripts/dev-entrypoint.sh`):
|
||||
|
||||
1. **NGINX** starts (HTTPS proxy for Vite and Bugsink)
|
||||
2. **Bugsink** starts (error tracking on port 8000)
|
||||
3. **Logstash** starts (log aggregation)
|
||||
4. **PM2** starts with `ecosystem.dev.config.cjs`:
|
||||
- `flyer-crawler-api-dev` - API server
|
||||
- `flyer-crawler-worker-dev` - Background worker
|
||||
- `flyer-crawler-vite-dev` - Vite dev server
|
||||
5. Container tails PM2 logs to stay alive
|
||||
|
||||
### Key Configuration Files
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------ | ---------------------------------- |
|
||||
| `ecosystem.dev.config.cjs` | PM2 development configuration |
|
||||
| `ecosystem.config.cjs` | PM2 production configuration |
|
||||
| `scripts/dev-entrypoint.sh` | Container startup script |
|
||||
| `docker/logstash/bugsink.conf` | Logstash pipeline configuration |
|
||||
| `docker/nginx/dev.conf` | NGINX development configuration |
|
||||
| `compose.dev.yml` | Docker Compose service definitions |
|
||||
| `Dockerfile.dev` | Container image definition |
|
||||
|
||||
---
|
||||
|
||||
## PM2 Process Management
|
||||
|
||||
### Process Overview
|
||||
|
||||
PM2 manages three processes in the dev container:
|
||||
|
||||
```
|
||||
+--------------------+ +------------------------+ +--------------------+
|
||||
| flyer-crawler- | | flyer-crawler- | | flyer-crawler- |
|
||||
| api-dev | | worker-dev | | vite-dev |
|
||||
+--------------------+ +------------------------+ +--------------------+
|
||||
| tsx watch | | tsx watch | | vite --host |
|
||||
| server.ts | | src/services/worker.ts | | |
|
||||
| Port: 3001 | | No port | | Port: 5173 |
|
||||
+--------------------+ +------------------------+ +--------------------+
|
||||
| | |
|
||||
v v v
|
||||
+------------------------------------------------------------------------+
|
||||
| /var/log/pm2/*.log |
|
||||
| (Logstash picks up for Bugsink) |
|
||||
+------------------------------------------------------------------------+
|
||||
```
|
||||
|
||||
### PM2 Commands
|
||||
|
||||
All commands should be run inside the container:
|
||||
|
||||
```bash
|
||||
# View process status
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
|
||||
# View all logs (tail -f style)
|
||||
podman exec -it flyer-crawler-dev pm2 logs
|
||||
|
||||
# View specific process logs
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
|
||||
|
||||
# Restart all processes
|
||||
podman exec -it flyer-crawler-dev pm2 restart all
|
||||
|
||||
# Restart specific process
|
||||
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
|
||||
|
||||
# Stop all processes
|
||||
podman exec -it flyer-crawler-dev pm2 delete all
|
||||
|
||||
# Show detailed process info
|
||||
podman exec -it flyer-crawler-dev pm2 show flyer-crawler-api-dev
|
||||
|
||||
# Monitor processes in real-time
|
||||
podman exec -it flyer-crawler-dev pm2 monit
|
||||
```
|
||||
|
||||
### PM2 Log Locations
|
||||
|
||||
| Process | stdout Log | stderr Log |
|
||||
| -------------------------- | ----------------------------- | ------------------------------- |
|
||||
| `flyer-crawler-api-dev` | `/var/log/pm2/api-out.log` | `/var/log/pm2/api-error.log` |
|
||||
| `flyer-crawler-worker-dev` | `/var/log/pm2/worker-out.log` | `/var/log/pm2/worker-error.log` |
|
||||
| `flyer-crawler-vite-dev` | `/var/log/pm2/vite-out.log` | `/var/log/pm2/vite-error.log` |
|
||||
|
||||
### NPM Scripts for PM2
|
||||
|
||||
The following npm scripts are available for PM2 management:
|
||||
|
||||
```bash
|
||||
# Start PM2 with dev config (inside container)
|
||||
npm run dev:pm2
|
||||
|
||||
# Restart all PM2 processes
|
||||
npm run dev:pm2:restart
|
||||
|
||||
# Stop all PM2 processes
|
||||
npm run dev:pm2:stop
|
||||
|
||||
# View PM2 status
|
||||
npm run dev:pm2:status
|
||||
|
||||
# View PM2 logs
|
||||
npm run dev:pm2:logs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Log Aggregation
|
||||
|
||||
### Log Flow Architecture (ADR-050)
|
||||
|
||||
All application logs flow through Logstash to Bugsink:
|
||||
|
||||
```
|
||||
+------------------+ +------------------+ +------------------+
|
||||
| PM2 Logs | | PostgreSQL | | Redis Logs |
|
||||
| /var/log/pm2/ | | /var/log/ | | /var/log/redis/ |
|
||||
+--------+---------+ | postgresql/ | +--------+---------+
|
||||
| +--------+---------+ |
|
||||
| | |
|
||||
v v v
|
||||
+------------------------------------------------------------------------+
|
||||
| LOGSTASH |
|
||||
| /etc/logstash/conf.d/bugsink.conf |
|
||||
+------------------------------------------------------------------------+
|
||||
| | |
|
||||
| +---------+---------+ |
|
||||
| | | |
|
||||
v v v v
|
||||
+------------------+ +------------------+ +------------------+
|
||||
| Errors -> | | Operational -> | | NGINX Logs -> |
|
||||
| Bugsink API | | /var/log/ | | /var/log/ |
|
||||
| (Project 1) | | logstash/*.log | | logstash/*.log |
|
||||
+------------------+ +------------------+ +------------------+
|
||||
```
|
||||
|
||||
### Log Sources
|
||||
|
||||
| Source | Log Path | Format | Errors To Bugsink |
|
||||
| ------------ | --------------------------------- | ---------- | ----------------- |
|
||||
| API Server | `/var/log/pm2/api-*.log` | Pino JSON | Yes |
|
||||
| Worker | `/var/log/pm2/worker-*.log` | Pino JSON | Yes |
|
||||
| Vite | `/var/log/pm2/vite-*.log` | Plain text | Yes (if error) |
|
||||
| PostgreSQL | `/var/log/postgresql/*.log` | PostgreSQL | Yes (ERROR/FATAL) |
|
||||
| Redis | `/var/log/redis/redis-server.log` | Redis | Yes (warnings) |
|
||||
| NGINX Access | `/var/log/nginx/access.log` | Combined | Yes (5xx only) |
|
||||
| NGINX Error | `/var/log/nginx/error.log` | NGINX | Yes |
|
||||
|
||||
### Viewing Logs
|
||||
|
||||
```bash
|
||||
# View Logstash processed logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-api-$(date +%Y-%m-%d).log
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
|
||||
|
||||
# View raw Redis logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
|
||||
|
||||
# Check Logstash status
|
||||
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
|
||||
```
|
||||
|
||||
### Bugsink Access
|
||||
|
||||
- **URL**: `https://localhost:8443`
|
||||
- **Login**: `admin@localhost` / `admin`
|
||||
- **Projects**:
|
||||
- Project 1: Backend API (errors from Pino, PostgreSQL, Redis)
|
||||
- Project 2: Frontend (errors from Sentry SDK in browser)
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference
|
||||
|
||||
### Starting the Dev Container
|
||||
|
||||
```bash
|
||||
# Start all services
|
||||
podman-compose -f compose.dev.yml up -d
|
||||
|
||||
# View container logs
|
||||
podman-compose -f compose.dev.yml logs -f
|
||||
|
||||
# Stop all services
|
||||
podman-compose -f compose.dev.yml down
|
||||
```
|
||||
|
||||
### Common Tasks
|
||||
|
||||
| Task | Command |
|
||||
| -------------------- | ------------------------------------------------------------------------------ |
|
||||
| Run tests | `podman exec -it flyer-crawler-dev npm test` |
|
||||
| Run type check | `podman exec -it flyer-crawler-dev npm run type-check` |
|
||||
| View PM2 status | `podman exec -it flyer-crawler-dev pm2 status` |
|
||||
| View PM2 logs | `podman exec -it flyer-crawler-dev pm2 logs` |
|
||||
| Restart API | `podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev` |
|
||||
| Access PostgreSQL | `podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev` |
|
||||
| Access Redis CLI | `podman exec -it flyer-crawler-redis redis-cli` |
|
||||
| Shell into container | `podman exec -it flyer-crawler-dev bash` |
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Key environment variables are set in `compose.dev.yml`:
|
||||
|
||||
| Variable | Value | Purpose |
|
||||
| ----------------- | ----------------------------- | -------------------- |
|
||||
| `NODE_ENV` | `development` | Environment mode |
|
||||
| `DB_HOST` | `postgres` | PostgreSQL hostname |
|
||||
| `REDIS_URL` | `redis://redis:6379` | Redis connection URL |
|
||||
| `FRONTEND_URL` | `https://localhost` | CORS origin |
|
||||
| `SENTRY_DSN` | `http://...@127.0.0.1:8000/1` | Backend Bugsink DSN |
|
||||
| `VITE_SENTRY_DSN` | `http://...@127.0.0.1:8000/2` | Frontend Bugsink DSN |
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### PM2 Process Not Starting
|
||||
|
||||
**Symptom**: `pm2 status` shows process as "errored" or "stopped"
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check process logs
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev --lines 50
|
||||
|
||||
# Check if port is in use
|
||||
podman exec -it flyer-crawler-dev netstat -tlnp | grep 3001
|
||||
|
||||
# Try manual start to see errors
|
||||
podman exec -it flyer-crawler-dev tsx server.ts
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Port already in use
|
||||
- Missing environment variables
|
||||
- Syntax error in code
|
||||
|
||||
### Logs Not Appearing in Bugsink
|
||||
|
||||
**Symptom**: Errors in application but nothing in Bugsink UI
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check Logstash is running
|
||||
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
|
||||
|
||||
# Check Logstash logs for errors
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log
|
||||
|
||||
# Verify PM2 logs exist
|
||||
podman exec flyer-crawler-dev ls -la /var/log/pm2/
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Logstash not started
|
||||
- Log file permissions
|
||||
- Bugsink not running
|
||||
|
||||
### Redis Logs Not Captured
|
||||
|
||||
**Symptom**: Redis warnings not appearing in Bugsink
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Verify Redis logs exist
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
|
||||
|
||||
# Verify shared volume is mounted
|
||||
podman exec flyer-crawler-dev ls -la /var/log/redis/
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- `redis_logs` volume not mounted
|
||||
- Redis not configured to write to file
|
||||
|
||||
### Hot Reload Not Working
|
||||
|
||||
**Symptom**: Code changes not reflected in running application
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check tsx watch is running
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
|
||||
|
||||
# Manually restart process
|
||||
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- File watcher limit reached
|
||||
- Volume mount issues on Windows
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [QUICKSTART.md](../getting-started/QUICKSTART.md) - Getting started guide
|
||||
- [DEBUGGING.md](DEBUGGING.md) - Debugging strategies
|
||||
- [LOGSTASH-QUICK-REF.md](../operations/LOGSTASH-QUICK-REF.md) - Logstash quick reference
|
||||
- [DEV-CONTAINER-BUGSINK.md](../DEV-CONTAINER-BUGSINK.md) - Bugsink setup in dev container
|
||||
- [ADR-014](../adr/0014-linux-only-platform.md) - Linux-only platform decision
|
||||
- [ADR-050](../adr/0050-postgresql-function-observability.md) - PostgreSQL function observability
|
||||
@@ -119,6 +119,31 @@ npm run dev
|
||||
- **Frontend**: http://localhost:5173
|
||||
- **Backend API**: http://localhost:3001
|
||||
|
||||
### Dev Container with HTTPS (Full Stack)
|
||||
|
||||
When using the full dev container stack with NGINX (via `compose.dev.yml`), access the application over HTTPS:
|
||||
|
||||
- **Frontend**: https://localhost or https://127.0.0.1
|
||||
- **Backend API**: http://localhost:3001
|
||||
|
||||
**SSL Certificate Notes:**
|
||||
|
||||
- The dev container uses self-signed certificates generated by mkcert
|
||||
- Both `localhost` and `127.0.0.1` are valid hostnames (certificate includes both as SANs)
|
||||
- If images fail to load with SSL errors, see [FLYER-URL-CONFIGURATION.md](../FLYER-URL-CONFIGURATION.md#ssl-certificate-configuration-dev-container)
|
||||
|
||||
**Eliminate SSL Warnings (Recommended):**
|
||||
|
||||
To avoid browser security warnings for self-signed certificates, install the mkcert CA certificate on your system. The CA certificate is located at `certs/mkcert-ca.crt` in the project root.
|
||||
|
||||
See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
|
||||
|
||||
After installation:
|
||||
|
||||
- Your browser will trust all mkcert certificates without warnings
|
||||
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
|
||||
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
|
||||
|
||||
### Managing the Container
|
||||
|
||||
| Action | Command |
|
||||
|
||||
@@ -86,6 +86,44 @@ npm run dev
|
||||
- **Backend API**: http://localhost:3001
|
||||
- **Health Check**: http://localhost:3001/health
|
||||
|
||||
### Dev Container (HTTPS)
|
||||
|
||||
When using the full dev container with NGINX, access via HTTPS:
|
||||
|
||||
- **Frontend**: https://localhost or https://127.0.0.1
|
||||
- **Backend API**: http://localhost:3001
|
||||
- **Bugsink**: `https://localhost:8443` (error tracking)
|
||||
|
||||
**Note:** The dev container accepts both `localhost` and `127.0.0.1` for HTTPS connections. The self-signed certificate is valid for both hostnames.
|
||||
|
||||
**SSL Certificate Warnings:** To eliminate browser security warnings for self-signed certificates, install the mkcert CA certificate. See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions. This is optional but recommended for a better development experience.
|
||||
|
||||
### Dev Container Architecture
|
||||
|
||||
The dev container uses PM2 for process management, matching production (ADR-014):
|
||||
|
||||
| Process | Description | Port |
|
||||
| -------------------------- | ------------------------ | ---- |
|
||||
| `flyer-crawler-api-dev` | API server (tsx watch) | 3001 |
|
||||
| `flyer-crawler-worker-dev` | Background job worker | - |
|
||||
| `flyer-crawler-vite-dev` | Vite frontend dev server | 5173 |
|
||||
|
||||
**PM2 Commands** (run inside container):
|
||||
|
||||
```bash
|
||||
# View process status
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
|
||||
# View logs (all processes)
|
||||
podman exec -it flyer-crawler-dev pm2 logs
|
||||
|
||||
# Restart all processes
|
||||
podman exec -it flyer-crawler-dev pm2 restart all
|
||||
|
||||
# Restart specific process
|
||||
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
|
||||
```
|
||||
|
||||
## Verify Installation
|
||||
|
||||
```bash
|
||||
|
||||
@@ -11,6 +11,22 @@ This guide covers deploying Flyer Crawler to a production server.
|
||||
- NGINX (reverse proxy)
|
||||
- PM2 (process manager)
|
||||
|
||||
## Dev Container Parity (ADR-014)
|
||||
|
||||
The development container now matches production architecture:
|
||||
|
||||
| Component | Production | Dev Container |
|
||||
| ------------ | ---------------- | -------------------------- |
|
||||
| Process Mgmt | PM2 | PM2 |
|
||||
| API Server | PM2 cluster mode | PM2 fork + tsx watch |
|
||||
| Worker | PM2 process | PM2 process + tsx watch |
|
||||
| Logs | PM2 -> Logstash | PM2 -> Logstash -> Bugsink |
|
||||
| HTTPS | Let's Encrypt | Self-signed (mkcert) |
|
||||
|
||||
This ensures issues caught in dev will behave the same in production.
|
||||
|
||||
**Dev Container Guide**: See [docs/development/DEV-CONTAINER.md](../development/DEV-CONTAINER.md)
|
||||
|
||||
---
|
||||
|
||||
## Server Setup
|
||||
|
||||
@@ -31,6 +31,8 @@ Aggregates logs from PostgreSQL, PM2, Redis, NGINX; forwards errors to Bugsink.
|
||||
|
||||
## Commands
|
||||
|
||||
### Production (Bare Metal)
|
||||
|
||||
```bash
|
||||
# Status and control
|
||||
systemctl status logstash
|
||||
@@ -55,6 +57,36 @@ tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
|
||||
du -sh /var/log/logstash/
|
||||
```
|
||||
|
||||
### Dev Container
|
||||
|
||||
```bash
|
||||
# View Logstash logs (inside container)
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log
|
||||
|
||||
# Check PM2 API logs are being processed (ADR-014)
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-api-$(date +%Y-%m-%d).log
|
||||
|
||||
# Check PM2 Worker logs are being processed
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-worker-$(date +%Y-%m-%d).log
|
||||
|
||||
# Check Redis logs are being processed
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
|
||||
|
||||
# View raw PM2 logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-out.log
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/worker-out.log
|
||||
|
||||
# View raw Redis logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
|
||||
|
||||
# Check Logstash stats
|
||||
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
|
||||
|
||||
# Verify shared volume mounts
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev ls -la /var/log/pm2/
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev ls -la /var/log/redis/
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Issue | Check | Solution |
|
||||
@@ -64,12 +96,15 @@ du -sh /var/log/logstash/
|
||||
| Grok pattern failures | Stats endpoint | `curl localhost:9600/_node/stats/pipelines?pretty \| jq '.pipelines.main.plugins.filters'` |
|
||||
| Wrong Bugsink project | Env detection | Check tags in logs match expected environment |
|
||||
| Permission denied | Logstash groups | `groups logstash` should include `postgres`, `adm` |
|
||||
| PM2 not captured | File paths | `ls /home/gitea-runner/.pm2/logs/flyer-crawler-worker-*.log` |
|
||||
| PM2 not captured | File paths | Dev: `ls /var/log/pm2/`; Prod: `ls /home/gitea-runner/.pm2/logs/` |
|
||||
| PM2 logs empty | PM2 running | Dev: `podman exec flyer-crawler-dev pm2 status`; Prod: `pm2 status` |
|
||||
| NGINX logs missing | Output directory | `ls -lh /var/log/logstash/nginx-access-*.log` |
|
||||
| Redis logs missing | Shared volume | Dev: Check `redis_logs` volume mounted; Prod: Check `/var/log/redis/redis-server.log` exists |
|
||||
| High disk usage | Log rotation | Verify `/etc/logrotate.d/logstash` configured |
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- **Dev Container Guide**: [DEV-CONTAINER.md](../development/DEV-CONTAINER.md) - PM2 and log aggregation in dev
|
||||
- **Full setup**: [BARE-METAL-SETUP.md](BARE-METAL-SETUP.md) - PostgreSQL Function Observability section
|
||||
- **Architecture**: [adr/0050-postgresql-function-observability.md](adr/0050-postgresql-function-observability.md)
|
||||
- **Architecture**: [adr/0050-postgresql-function-observability.md](../adr/0050-postgresql-function-observability.md)
|
||||
- **Troubleshooting details**: [LOGSTASH-TROUBLESHOOTING.md](LOGSTASH-TROUBLESHOOTING.md)
|
||||
|
||||
172
ecosystem.dev.config.cjs
Normal file
172
ecosystem.dev.config.cjs
Normal file
@@ -0,0 +1,172 @@
|
||||
// ecosystem.dev.config.cjs
|
||||
// ============================================================================
|
||||
// DEVELOPMENT PM2 CONFIGURATION
|
||||
// ============================================================================
|
||||
// This file mirrors the production ecosystem.config.cjs but with settings
|
||||
// appropriate for the development environment inside the dev container.
|
||||
//
|
||||
// Key differences from production:
|
||||
// - Single instance for API (not cluster mode) to allow debugger attachment
|
||||
// - tsx watch mode for hot reloading
|
||||
// - Development-specific environment variables
|
||||
// - Logs written to /var/log/pm2 for Logstash integration (ADR-050)
|
||||
//
|
||||
// Usage:
|
||||
// pm2 start ecosystem.dev.config.cjs
|
||||
// pm2 logs # View all logs
|
||||
// pm2 logs flyer-crawler-api-dev # View API logs only
|
||||
// pm2 restart all # Restart all processes
|
||||
// pm2 delete all # Stop all processes
|
||||
//
|
||||
// Related:
|
||||
// - ecosystem.config.cjs (production configuration)
|
||||
// - docs/adr/0014-linux-only-platform.md
|
||||
// - docs/adr/0050-postgresql-function-observability.md
|
||||
// ============================================================================
|
||||
|
||||
// --- Environment Variable Validation ---
|
||||
// In dev, we warn but don't fail - variables come from compose.dev.yml
|
||||
const requiredVars = ['DB_HOST', 'JWT_SECRET'];
|
||||
const missingVars = requiredVars.filter((key) => !process.env[key]);
|
||||
|
||||
if (missingVars.length > 0) {
|
||||
console.warn(
|
||||
'\n[ecosystem.dev.config.cjs] WARNING: The following environment variables are MISSING:',
|
||||
);
|
||||
missingVars.forEach((key) => console.warn(` - ${key}`));
|
||||
console.warn(
|
||||
'[ecosystem.dev.config.cjs] These should be set in compose.dev.yml or .env.local\n',
|
||||
);
|
||||
} else {
|
||||
console.log('[ecosystem.dev.config.cjs] Required environment variables are present.');
|
||||
}
|
||||
|
||||
// --- Shared Environment Variables ---
|
||||
// These come from compose.dev.yml environment section
|
||||
const sharedEnv = {
|
||||
NODE_ENV: 'development',
|
||||
DB_HOST: process.env.DB_HOST || 'postgres',
|
||||
DB_PORT: process.env.DB_PORT || '5432',
|
||||
DB_USER: process.env.DB_USER || 'postgres',
|
||||
DB_PASSWORD: process.env.DB_PASSWORD || 'postgres',
|
||||
DB_NAME: process.env.DB_NAME || 'flyer_crawler_dev',
|
||||
REDIS_URL: process.env.REDIS_URL || 'redis://redis:6379',
|
||||
REDIS_HOST: process.env.REDIS_HOST || 'redis',
|
||||
REDIS_PORT: process.env.REDIS_PORT || '6379',
|
||||
FRONTEND_URL: process.env.FRONTEND_URL || 'https://localhost',
|
||||
JWT_SECRET: process.env.JWT_SECRET || 'dev-jwt-secret-change-in-production',
|
||||
WORKER_LOCK_DURATION: process.env.WORKER_LOCK_DURATION || '120000',
|
||||
// Sentry/Bugsink (ADR-015)
|
||||
SENTRY_DSN: process.env.SENTRY_DSN,
|
||||
SENTRY_ENVIRONMENT: process.env.SENTRY_ENVIRONMENT || 'development',
|
||||
SENTRY_ENABLED: process.env.SENTRY_ENABLED || 'true',
|
||||
SENTRY_DEBUG: process.env.SENTRY_DEBUG || 'true',
|
||||
// Optional API keys (may not be set in dev)
|
||||
GEMINI_API_KEY: process.env.GEMINI_API_KEY,
|
||||
GOOGLE_MAPS_API_KEY: process.env.GOOGLE_MAPS_API_KEY,
|
||||
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
|
||||
GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
apps: [
|
||||
// =========================================================================
|
||||
// API SERVER (Development)
|
||||
// =========================================================================
|
||||
// Single instance with tsx watch for hot reloading.
|
||||
// Unlike production, we don't use cluster mode to allow:
|
||||
// - Debugger attachment
|
||||
// - Simpler log output
|
||||
// - Hot reloading via tsx watch
|
||||
{
|
||||
name: 'flyer-crawler-api-dev',
|
||||
script: './node_modules/.bin/tsx',
|
||||
args: 'watch server.ts',
|
||||
cwd: '/app',
|
||||
watch: false, // tsx watch handles this
|
||||
instances: 1, // Single instance for debugging
|
||||
exec_mode: 'fork', // Fork mode (not cluster) for debugging
|
||||
max_memory_restart: '1G', // More generous in dev
|
||||
kill_timeout: 5000,
|
||||
// PM2 log configuration (ADR-050 - Logstash integration)
|
||||
output: '/var/log/pm2/api-out.log',
|
||||
error: '/var/log/pm2/api-error.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
// Restart configuration
|
||||
max_restarts: 10,
|
||||
exp_backoff_restart_delay: 500,
|
||||
min_uptime: '5s',
|
||||
// Environment
|
||||
env: {
|
||||
...sharedEnv,
|
||||
PORT: '3001',
|
||||
},
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// BACKGROUND WORKER (Development)
|
||||
// =========================================================================
|
||||
// Processes background jobs (flyer processing, cleanup, etc.)
|
||||
// Runs as a separate process like in production.
|
||||
{
|
||||
name: 'flyer-crawler-worker-dev',
|
||||
script: './node_modules/.bin/tsx',
|
||||
args: 'watch src/services/worker.ts',
|
||||
cwd: '/app',
|
||||
watch: false, // tsx watch handles this
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
max_memory_restart: '1G',
|
||||
kill_timeout: 10000, // Workers need more time to finish jobs
|
||||
// PM2 log configuration (ADR-050)
|
||||
output: '/var/log/pm2/worker-out.log',
|
||||
error: '/var/log/pm2/worker-error.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
// Restart configuration
|
||||
max_restarts: 10,
|
||||
exp_backoff_restart_delay: 500,
|
||||
min_uptime: '5s',
|
||||
// Environment
|
||||
env: {
|
||||
...sharedEnv,
|
||||
},
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// VITE FRONTEND DEV SERVER (Development)
|
||||
// =========================================================================
|
||||
// Vite dev server for frontend hot module replacement.
|
||||
// Listens on 5173 and is proxied by nginx to 443.
|
||||
{
|
||||
name: 'flyer-crawler-vite-dev',
|
||||
script: './node_modules/.bin/vite',
|
||||
args: '--host',
|
||||
cwd: '/app',
|
||||
watch: false, // Vite handles its own watching
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
max_memory_restart: '2G', // Vite can use significant memory
|
||||
kill_timeout: 5000,
|
||||
// PM2 log configuration (ADR-050)
|
||||
output: '/var/log/pm2/vite-out.log',
|
||||
error: '/var/log/pm2/vite-error.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
// Restart configuration
|
||||
max_restarts: 10,
|
||||
exp_backoff_restart_delay: 500,
|
||||
min_uptime: '5s',
|
||||
// Environment
|
||||
env: {
|
||||
NODE_ENV: 'development',
|
||||
// Vite-specific env vars (VITE_ prefix)
|
||||
VITE_SENTRY_DSN: process.env.VITE_SENTRY_DSN,
|
||||
VITE_SENTRY_ENVIRONMENT: process.env.VITE_SENTRY_ENVIRONMENT || 'development',
|
||||
VITE_SENTRY_ENABLED: process.env.VITE_SENTRY_ENABLED || 'true',
|
||||
VITE_SENTRY_DEBUG: process.env.VITE_SENTRY_DEBUG || 'true',
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
4
package-lock.json
generated
4
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "flyer-crawler",
|
||||
"version": "0.12.7",
|
||||
"version": "0.12.8",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "flyer-crawler",
|
||||
"version": "0.12.7",
|
||||
"version": "0.12.8",
|
||||
"dependencies": {
|
||||
"@bull-board/api": "^6.14.2",
|
||||
"@bull-board/express": "^6.14.2",
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
{
|
||||
"name": "flyer-crawler",
|
||||
"private": true,
|
||||
"version": "0.12.7",
|
||||
"version": "0.12.8",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "concurrently \"npm:start:dev\" \"vite\"",
|
||||
"dev:container": "concurrently \"npm:start:dev\" \"vite --host\"",
|
||||
"dev:pm2": "pm2 start ecosystem.dev.config.cjs && pm2 logs",
|
||||
"dev:pm2:restart": "pm2 restart ecosystem.dev.config.cjs",
|
||||
"dev:pm2:stop": "pm2 delete all",
|
||||
"dev:pm2:status": "pm2 status",
|
||||
"dev:pm2:logs": "pm2 logs",
|
||||
"start": "npm run start:prod",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
|
||||
@@ -3,22 +3,28 @@
|
||||
# ============================================================================
|
||||
# Development Container Entrypoint
|
||||
# ============================================================================
|
||||
# This script starts the development server automatically when the container
|
||||
# starts, both with VS Code Dev Containers and with plain podman-compose.
|
||||
# This script starts all development services when the container starts,
|
||||
# both with VS Code Dev Containers and with plain podman-compose.
|
||||
#
|
||||
# Services started:
|
||||
# - Nginx (proxies Vite 5173 → 3000)
|
||||
# - Nginx (HTTPS proxy: Vite 5173 -> 443, Bugsink 8000 -> 8443, API 3001)
|
||||
# - Bugsink (error tracking) on port 8000
|
||||
# - Logstash (log aggregation)
|
||||
# - Node.js dev server (API + Frontend) on ports 3001 and 5173
|
||||
# - PM2 (process manager) running:
|
||||
# - flyer-crawler-api-dev: API server (port 3001)
|
||||
# - flyer-crawler-worker-dev: Background job worker
|
||||
# - flyer-crawler-vite-dev: Vite frontend dev server (port 5173)
|
||||
#
|
||||
# ADR-014: This architecture matches production (PM2 managing processes)
|
||||
# ADR-050: Logs are written to /var/log/pm2 for Logstash integration
|
||||
# ============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
echo "🚀 Starting Flyer Crawler Dev Container..."
|
||||
echo "Starting Flyer Crawler Dev Container..."
|
||||
|
||||
# Configure Bugsink HTTPS (ADR-015)
|
||||
echo "🔒 Configuring Bugsink HTTPS..."
|
||||
echo "Configuring Bugsink HTTPS..."
|
||||
mkdir -p /etc/bugsink/ssl
|
||||
if [ ! -f "/etc/bugsink/ssl/localhost+2.pem" ]; then
|
||||
cd /etc/bugsink/ssl && mkcert localhost 127.0.0.1 ::1 > /dev/null 2>&1
|
||||
@@ -53,52 +59,92 @@ NGINX_EOF
|
||||
ln -sf /etc/nginx/sites-available/bugsink /etc/nginx/sites-enabled/bugsink
|
||||
|
||||
# Copy the dev nginx config from mounted volume to nginx sites-available
|
||||
echo "📋 Copying nginx dev config..."
|
||||
echo "Copying nginx dev config..."
|
||||
cp /app/docker/nginx/dev.conf /etc/nginx/sites-available/default
|
||||
|
||||
# Ensure PM2 log directory exists with correct permissions
|
||||
echo "Setting up PM2 log directory..."
|
||||
mkdir -p /var/log/pm2
|
||||
chmod 755 /var/log/pm2
|
||||
|
||||
# Start nginx in background (if installed)
|
||||
if command -v nginx &> /dev/null; then
|
||||
echo "🌐 Starting nginx (HTTPS: Vite 5173 → 443, Bugsink 8000 → 8443, API 3001 → /api/)..."
|
||||
echo "Starting nginx (HTTPS: Vite 5173 -> 443, Bugsink 8000 -> 8443, API 3001 -> /api/)..."
|
||||
nginx &
|
||||
fi
|
||||
|
||||
# Start Bugsink in background
|
||||
echo "📊 Starting Bugsink error tracking..."
|
||||
echo "Starting Bugsink error tracking..."
|
||||
/usr/local/bin/start-bugsink.sh > /var/log/bugsink/server.log 2>&1 &
|
||||
|
||||
# Wait for Bugsink to initialize, then run snappea migrations
|
||||
echo "⏳ Waiting for Bugsink to initialize..."
|
||||
echo "Waiting for Bugsink to initialize..."
|
||||
sleep 5
|
||||
echo "🔧 Running Bugsink snappea database migrations..."
|
||||
echo "Running Bugsink snappea database migrations..."
|
||||
cd /opt/bugsink/conf && \
|
||||
export DATABASE_URL="postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink" && \
|
||||
export SECRET_KEY="dev-bugsink-secret-key-minimum-50-characters-for-security" && \
|
||||
/opt/bugsink/bin/bugsink-manage migrate --database=snappea > /dev/null 2>&1
|
||||
|
||||
# Start Snappea task worker
|
||||
echo "🔄 Starting Snappea task worker..."
|
||||
echo "Starting Snappea task worker..."
|
||||
cd /opt/bugsink/conf && \
|
||||
export DATABASE_URL="postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink" && \
|
||||
export SECRET_KEY="dev-bugsink-secret-key-minimum-50-characters-for-security" && \
|
||||
/opt/bugsink/bin/bugsink-manage runsnappea > /var/log/bugsink/snappea.log 2>&1 &
|
||||
|
||||
# Start Logstash in background
|
||||
echo "📝 Starting Logstash..."
|
||||
echo "Starting Logstash..."
|
||||
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/bugsink.conf > /var/log/logstash/logstash.log 2>&1 &
|
||||
|
||||
# Wait a few seconds for services to initialize
|
||||
# Wait for services to initialize
|
||||
sleep 3
|
||||
|
||||
# Change to app directory
|
||||
cd /app
|
||||
|
||||
# Start development server
|
||||
echo "💻 Starting development server..."
|
||||
echo " - Frontend: https://localhost (nginx HTTPS → Vite on 5173)"
|
||||
echo " - Backend API: http://localhost:3001"
|
||||
echo " - Bugsink: https://localhost:8443 (nginx HTTPS → Bugsink on 8000)"
|
||||
echo " - Note: Accept the self-signed certificate warnings in your browser"
|
||||
# ============================================================================
|
||||
# Start PM2 (ADR-014: Production-like process management)
|
||||
# ============================================================================
|
||||
# PM2 manages all Node.js processes:
|
||||
# - API server (tsx watch server.ts)
|
||||
# - Background worker (tsx watch src/services/worker.ts)
|
||||
# - Vite dev server (vite --host)
|
||||
#
|
||||
# Logs are written to /var/log/pm2 for Logstash integration (ADR-050)
|
||||
# ============================================================================
|
||||
|
||||
echo "Starting PM2 with development configuration..."
|
||||
echo " - API Server: http://localhost:3001 (proxied via nginx)"
|
||||
echo " - Frontend: https://localhost (nginx HTTPS -> Vite on 5173)"
|
||||
echo " - Bugsink: https://localhost:8443 (nginx HTTPS -> Bugsink on 8000)"
|
||||
echo " - PM2 Logs: /var/log/pm2/*.log"
|
||||
echo ""
|
||||
echo "Note: Accept the self-signed certificate warnings in your browser"
|
||||
echo ""
|
||||
|
||||
# Run npm dev server (this will block and keep container alive)
|
||||
exec npm run dev:container
|
||||
# Delete any existing PM2 processes (clean start)
|
||||
pm2 delete all 2>/dev/null || true
|
||||
|
||||
# Start PM2 with the dev ecosystem config
|
||||
pm2 start /app/ecosystem.dev.config.cjs
|
||||
|
||||
# Show PM2 status
|
||||
echo ""
|
||||
echo "--- PM2 Process Status ---"
|
||||
pm2 status
|
||||
echo "-------------------------"
|
||||
echo ""
|
||||
echo "PM2 Commands:"
|
||||
echo " pm2 logs # View all logs (tail -f style)"
|
||||
echo " pm2 logs api # View API logs only"
|
||||
echo " pm2 restart all # Restart all processes"
|
||||
echo " pm2 status # Show process status"
|
||||
echo ""
|
||||
|
||||
# Keep the container running by tailing PM2 logs
|
||||
# This ensures:
|
||||
# 1. Container stays alive
|
||||
# 2. Logs are visible in docker logs / podman logs
|
||||
# 3. PM2 processes continue running
|
||||
exec pm2 logs --raw
|
||||
|
||||
@@ -1,21 +1,31 @@
|
||||
-- Update flyer URLs from example.com to environment-specific URLs
|
||||
--
|
||||
-- This script should be run after determining the correct base URL for the environment:
|
||||
-- - Dev container: http://127.0.0.1
|
||||
-- - Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
|
||||
-- - Test environment: https://flyer-crawler-test.projectium.com
|
||||
-- - Production: https://flyer-crawler.projectium.com
|
||||
|
||||
-- For dev container (run in dev database):
|
||||
-- Uses 'localhost' instead of '127.0.0.1' to match how users access the site.
|
||||
-- This avoids ERR_CERT_AUTHORITY_INVALID errors when images are loaded from a
|
||||
-- different origin than the page.
|
||||
UPDATE flyers
|
||||
SET
|
||||
image_url = REPLACE(image_url, 'example.com', '127.0.0.1'),
|
||||
image_url = REPLACE(image_url, 'https://', 'http://'),
|
||||
icon_url = REPLACE(icon_url, 'example.com', '127.0.0.1'),
|
||||
icon_url = REPLACE(icon_url, 'https://', 'http://')
|
||||
image_url = REPLACE(image_url, 'example.com', 'localhost'),
|
||||
icon_url = REPLACE(icon_url, 'example.com', 'localhost')
|
||||
WHERE
|
||||
image_url LIKE '%example.com%'
|
||||
OR icon_url LIKE '%example.com%';
|
||||
|
||||
-- Also fix any existing 127.0.0.1 URLs to use localhost:
|
||||
UPDATE flyers
|
||||
SET
|
||||
image_url = REPLACE(image_url, '127.0.0.1', 'localhost'),
|
||||
icon_url = REPLACE(icon_url, '127.0.0.1', 'localhost')
|
||||
WHERE
|
||||
image_url LIKE '%127.0.0.1%'
|
||||
OR icon_url LIKE '%127.0.0.1%';
|
||||
|
||||
-- For test environment (run in test database):
|
||||
-- UPDATE flyers
|
||||
-- SET
|
||||
@@ -37,7 +47,7 @@ WHERE
|
||||
-- Verify the changes:
|
||||
SELECT flyer_id, image_url, icon_url
|
||||
FROM flyers
|
||||
WHERE image_url LIKE '%127.0.0.1%'
|
||||
OR icon_url LIKE '%127.0.0.1%'
|
||||
WHERE image_url LIKE '%localhost%'
|
||||
OR icon_url LIKE '%localhost%'
|
||||
OR image_url LIKE '%flyer-crawler%'
|
||||
OR icon_url LIKE '%flyer-crawler%';
|
||||
|
||||
@@ -12,8 +12,8 @@ import path from 'node:path';
|
||||
import bcrypt from 'bcrypt';
|
||||
import { logger } from '../services/logger.server';
|
||||
|
||||
// Determine base URL based on environment
|
||||
// Dev container: https://127.0.0.1
|
||||
// Determine base URL for flyer images based on environment
|
||||
// Dev container: https://127.0.0.1 (NGINX now accepts both localhost and 127.0.0.1)
|
||||
// Test: https://flyer-crawler-test.projectium.com
|
||||
// Production: https://flyer-crawler.projectium.com
|
||||
const BASE_URL =
|
||||
|
||||
@@ -159,7 +159,7 @@ export const MainLayout: React.FC<MainLayoutProps> = ({
|
||||
<>
|
||||
<PriceHistoryChart />
|
||||
<Leaderboard />
|
||||
{userProfile && (
|
||||
{userProfile?.role === 'admin' && (
|
||||
<ActivityLog userProfile={userProfile} onLogClick={handleActivityLogClick} />
|
||||
)}
|
||||
</>
|
||||
|
||||
@@ -21,6 +21,8 @@ const isProduction = process.env.NODE_ENV === 'production';
|
||||
const isTest = process.env.NODE_ENV === 'test';
|
||||
const isStaging = process.env.NODE_ENV === 'staging';
|
||||
const isDevelopment = !isProduction && !isTest && !isStaging;
|
||||
// PM2_HOME is set when running under PM2 process manager
|
||||
const isRunningUnderPM2 = !!process.env.PM2_HOME;
|
||||
|
||||
// Determine log directory based on environment
|
||||
// In production/test, use the application directory's logs folder
|
||||
@@ -73,8 +75,9 @@ const redactConfig = {
|
||||
const createLogger = (): pino.Logger => {
|
||||
const logDir = ensureLogDirectory();
|
||||
|
||||
// Development: Use pino-pretty for human-readable output
|
||||
if (isDevelopment) {
|
||||
// Development WITHOUT PM2: Use pino-pretty for human-readable console output
|
||||
// This is for local development outside of containers (rare case now)
|
||||
if (isDevelopment && !isRunningUnderPM2) {
|
||||
return pino({
|
||||
level: 'debug',
|
||||
transport: {
|
||||
@@ -89,6 +92,15 @@ const createLogger = (): pino.Logger => {
|
||||
});
|
||||
}
|
||||
|
||||
// Development WITH PM2 (ADR-014): Output JSON for Logstash integration
|
||||
// PM2 captures stdout to /var/log/pm2/*.log files which Logstash parses
|
||||
if (isDevelopment && isRunningUnderPM2) {
|
||||
return pino({
|
||||
level: 'debug',
|
||||
redact: redactConfig,
|
||||
});
|
||||
}
|
||||
|
||||
// Production/Test: Write to both stdout and file
|
||||
if (logDir) {
|
||||
const logFilePath = path.join(logDir, 'app.log');
|
||||
|
||||
@@ -15,7 +15,7 @@ export const getTestBaseUrl = (): string => {
|
||||
/**
|
||||
* Get the flyer base URL for test data based on environment.
|
||||
* Uses FLYER_BASE_URL if set, otherwise detects environment:
|
||||
* - Dev container: http://127.0.0.1
|
||||
* - Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
|
||||
* - Test: https://flyer-crawler-test.projectium.com
|
||||
* - Production: https://flyer-crawler.projectium.com
|
||||
* - Default: https://example.com (for unit tests)
|
||||
@@ -26,8 +26,10 @@ export const getFlyerBaseUrl = (): string => {
|
||||
}
|
||||
|
||||
// Check if we're in dev container (DB_HOST=postgres is typical indicator)
|
||||
// Use 'localhost' instead of '127.0.0.1' to match the hostname users access
|
||||
// This avoids SSL certificate mixed-origin issues in browsers
|
||||
if (process.env.DB_HOST === 'postgres' || process.env.DB_HOST === '127.0.0.1') {
|
||||
return 'http://127.0.0.1';
|
||||
return 'https://localhost';
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
|
||||
Reference in New Issue
Block a user