Compare commits

...

3 Commits

Author SHA1 Message Date
Gitea Actions
dd067183ed ci: Bump version to 0.12.8 [skip ci] 2026-01-23 04:50:12 +05:00
9f3a070612 have dev continer send more to bugsink
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 19m20s
2026-01-22 15:47:59 -08:00
8a38befb1c fix cert and image display issues 2026-01-22 12:46:28 -08:00
30 changed files with 1927 additions and 238 deletions

View File

@@ -114,7 +114,15 @@
"Bash(git rm:*)",
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" log -1 --format=\"%H %ci %s\")",
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" config --get remote.origin.url)",
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" fetch --dry-run -v)"
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" fetch --dry-run -v)",
"mcp__localerrors__get_project",
"mcp__localerrors__get_issue",
"mcp__localerrors__get_event"
]
}
},
"enabledMcpjsonServers": [
"localerrors",
"devdb",
"gitea-projectium"
]
}

View File

@@ -36,10 +36,10 @@ NODE_ENV=development
FRONTEND_URL=http://localhost:3000
# Flyer Base URL - used for seed data and flyer image URLs
# Dev container: http://127.0.0.1
# Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
# Test: https://flyer-crawler-test.projectium.com
# Production: https://flyer-crawler.projectium.com
FLYER_BASE_URL=http://127.0.0.1
FLYER_BASE_URL=https://localhost
# ===================
# Authentication

View File

@@ -15,6 +15,14 @@
"@modelcontextprotocol/server-postgres",
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
]
},
"gitea-projectium": {
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
"args": ["run", "-t", "stdio"],
"env": {
"GITEA_HOST": "https://gitea.projectium.com",
"GITEA_ACCESS_TOKEN": "b111259253aa3cadcb6a37618de03bf388f6235a"
}
}
}
}

103
CLAUDE.md
View File

@@ -40,6 +40,7 @@ Ask before assuming. Never assume:
1. **Memory**: `mcp__memory__read_graph` - Recall project context, credentials, known issues
2. **Git**: `git log --oneline -10` - Recent changes
3. **Containers**: `mcp__podman__container_list` - Running state
4. **PM2 Status**: `podman exec flyer-crawler-dev pm2 status` - Process health (API, Worker, Vite)
---
@@ -47,12 +48,15 @@ Ask before assuming. Never assume:
### Essential Commands
| Command | Description |
| ------------------------------------------------------------ | ----------------- |
| `podman exec -it flyer-crawler-dev npm test` | Run all tests |
| `podman exec -it flyer-crawler-dev npm run test:unit` | Unit tests only |
| `podman exec -it flyer-crawler-dev npm run type-check` | TypeScript check |
| `podman exec -it flyer-crawler-dev npm run test:integration` | Integration tests |
| Command | Description |
| ------------------------------------------------------------ | --------------------- |
| `podman exec -it flyer-crawler-dev npm test` | Run all tests |
| `podman exec -it flyer-crawler-dev npm run test:unit` | Unit tests only |
| `podman exec -it flyer-crawler-dev npm run type-check` | TypeScript check |
| `podman exec -it flyer-crawler-dev npm run test:integration` | Integration tests |
| `podman exec -it flyer-crawler-dev pm2 status` | PM2 process status |
| `podman exec -it flyer-crawler-dev pm2 logs` | View all PM2 logs |
| `podman exec -it flyer-crawler-dev pm2 restart all` | Restart all processes |
### Key Patterns (with file locations)
@@ -65,14 +69,16 @@ Ask before assuming. Never assume:
### Key Files Quick Access
| Purpose | File |
| ------------ | -------------------------------- |
| Express app | `server.ts` |
| Environment | `src/config/env.ts` |
| Routes | `src/routes/*.routes.ts` |
| Repositories | `src/services/db/*.db.ts` |
| Workers | `src/services/workers.server.ts` |
| Queues | `src/services/queues.server.ts` |
| Purpose | File |
| ----------------- | -------------------------------- |
| Express app | `server.ts` |
| Environment | `src/config/env.ts` |
| Routes | `src/routes/*.routes.ts` |
| Repositories | `src/services/db/*.db.ts` |
| Workers | `src/services/workers.server.ts` |
| Queues | `src/services/queues.server.ts` |
| PM2 Config (Dev) | `ecosystem.dev.config.cjs` |
| PM2 Config (Prod) | `ecosystem.config.cjs` |
---
@@ -96,6 +102,49 @@ Routes → Services → Repositories → Database
---
## Dev Container Architecture (ADR-014)
The dev container now matches production by using PM2 for process management.
### Process Management
| Component | Production | Dev Container |
| ---------- | ---------------------- | ------------------------- |
| API Server | PM2 cluster mode | PM2 fork mode + tsx watch |
| Worker | PM2 process | PM2 process + tsx watch |
| Frontend | Static files via NGINX | PM2 + Vite dev server |
| Logs | PM2 logs -> Logstash | PM2 logs -> Logstash |
**PM2 Processes in Dev Container**:
- `flyer-crawler-api-dev` - API server (port 3001)
- `flyer-crawler-worker-dev` - Background job worker
- `flyer-crawler-vite-dev` - Vite frontend dev server (port 5173)
### Log Aggregation (ADR-050)
All logs flow to Bugsink via Logstash:
| Source | Log Location | Status |
| ----------------- | --------------------------------- | ------ |
| Backend (Pino) | `/var/log/pm2/api-*.log` | Active |
| Worker (Pino) | `/var/log/pm2/worker-*.log` | Active |
| Vite | `/var/log/pm2/vite-*.log` | Active |
| PostgreSQL | `/var/log/postgresql/*.log` | Active |
| Redis | `/var/log/redis/redis-server.log` | Active |
| NGINX | `/var/log/nginx/*.log` | Active |
| Frontend (Sentry) | Browser -> Bugsink SDK | Active |
**Key Files**:
- `ecosystem.dev.config.cjs` - PM2 development configuration
- `scripts/dev-entrypoint.sh` - Container startup script
- `docker/logstash/bugsink.conf` - Logstash pipeline configuration
**Full Dev Container Guide**: See [docs/development/DEV-CONTAINER.md](docs/development/DEV-CONTAINER.md)
---
## Common Workflows
### Adding a New API Endpoint
@@ -251,18 +300,20 @@ Log aggregation: PostgreSQL + PM2 + Redis + NGINX → Bugsink (ADR-050)
## Documentation Quick Links
| Topic | Document |
| ------------------- | ----------------------------------------------------- |
| **Getting Started** | [QUICKSTART.md](docs/getting-started/QUICKSTART.md) |
| **Architecture** | [OVERVIEW.md](docs/architecture/OVERVIEW.md) |
| **Code Patterns** | [CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md) |
| **Testing** | [TESTING.md](docs/development/TESTING.md) |
| **Debugging** | [DEBUGGING.md](docs/development/DEBUGGING.md) |
| **Database** | [DATABASE.md](docs/architecture/DATABASE.md) |
| **Deployment** | [DEPLOYMENT.md](docs/operations/DEPLOYMENT.md) |
| **Monitoring** | [MONITORING.md](docs/operations/MONITORING.md) |
| **ADRs** | [docs/adr/index.md](docs/adr/index.md) |
| **All Docs** | [docs/README.md](docs/README.md) |
| Topic | Document |
| ------------------- | -------------------------------------------------------------- |
| **Getting Started** | [QUICKSTART.md](docs/getting-started/QUICKSTART.md) |
| **Dev Container** | [DEV-CONTAINER.md](docs/development/DEV-CONTAINER.md) |
| **Architecture** | [OVERVIEW.md](docs/architecture/OVERVIEW.md) |
| **Code Patterns** | [CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md) |
| **Testing** | [TESTING.md](docs/development/TESTING.md) |
| **Debugging** | [DEBUGGING.md](docs/development/DEBUGGING.md) |
| **Database** | [DATABASE.md](docs/architecture/DATABASE.md) |
| **Deployment** | [DEPLOYMENT.md](docs/operations/DEPLOYMENT.md) |
| **Monitoring** | [MONITORING.md](docs/operations/MONITORING.md) |
| **Logstash** | [LOGSTASH-QUICK-REF.md](docs/operations/LOGSTASH-QUICK-REF.md) |
| **ADRs** | [docs/adr/index.md](docs/adr/index.md) |
| **All Docs** | [docs/README.md](docs/README.md) |
---

View File

@@ -54,6 +54,13 @@ RUN apt-get update && apt-get install -y \
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
&& apt-get install -y nodejs
# ============================================================================
# Install PM2 Globally (ADR-014 Parity)
# ============================================================================
# Install PM2 to match production architecture. This allows dev container to
# run the same process management as production (cluster mode, workers, etc.)
RUN npm install -g pm2
# ============================================================================
# Install mkcert and Generate Self-Signed Certificates
# ============================================================================
@@ -63,6 +70,27 @@ RUN wget -O /usr/local/bin/mkcert https://github.com/FiloSottile/mkcert/releases
&& chmod +x /usr/local/bin/mkcert
# Create certificates directory and generate localhost certificates
# ============================================================================
# IMPORTANT: Certificate includes MULTIPLE hostnames (SANs)
# ============================================================================
# The certificate is generated for 'localhost', '127.0.0.1', AND '::1' because:
#
# 1. Users may access the site via https://localhost/ OR https://127.0.0.1/
# 2. Database stores image URLs using one hostname (typically 127.0.0.1)
# 3. The seed script uses https://127.0.0.1 for image URLs (database constraint)
# 4. NGINX is configured to accept BOTH hostnames (see docker/nginx/dev.conf)
#
# Without all hostnames in the certificate's Subject Alternative Names (SANs),
# browsers would show ERR_CERT_AUTHORITY_INVALID when loading images or other
# resources that use a different hostname than the one in the address bar.
#
# The mkcert command below creates a certificate valid for all three:
# - localhost (IPv4 hostname)
# - 127.0.0.1 (IPv4 address)
# - ::1 (IPv6 loopback)
#
# See also: docker/nginx/dev.conf, docs/FLYER-URL-CONFIGURATION.md
# ============================================================================
RUN mkdir -p /app/certs \
&& cd /app/certs \
&& mkcert -install \
@@ -211,119 +239,25 @@ exec /opt/bugsink/bin/gunicorn \\\n\
&& chmod +x /usr/local/bin/start-bugsink.sh
# ============================================================================
# Create Logstash Pipeline Configuration
# Copy Logstash Pipeline Configuration
# ============================================================================
# ADR-015: Pino and Redis logs → Bugsink
# ADR-015 + ADR-050: Multi-source log aggregation to Bugsink
# Configuration file includes:
# - Pino application logs (Backend API errors)
# - PostgreSQL logs (including fn_log() structured output)
# - NGINX access and error logs
# See docker/logstash/bugsink.conf for full configuration
RUN mkdir -p /etc/logstash/conf.d /app/logs
RUN echo 'input {\n\
# Pino application logs\n\
file {\n\
path => "/app/logs/*.log"\n\
codec => json\n\
type => "pino"\n\
tags => ["app"]\n\
start_position => "beginning"\n\
sincedb_path => "/var/lib/logstash/sincedb_pino"\n\
}\n\
\n\
# Redis logs\n\
file {\n\
path => "/var/log/redis/*.log"\n\
type => "redis"\n\
tags => ["redis"]\n\
start_position => "beginning"\n\
sincedb_path => "/var/lib/logstash/sincedb_redis"\n\
}\n\
\n\
# PostgreSQL function logs (ADR-050)\n\
file {\n\
path => "/var/log/postgresql/*.log"\n\
type => "postgres"\n\
tags => ["postgres", "database"]\n\
start_position => "beginning"\n\
sincedb_path => "/var/lib/logstash/sincedb_postgres"\n\
}\n\
}\n\
\n\
filter {\n\
# Pino error detection (level 50 = error, 60 = fatal)\n\
if [type] == "pino" and [level] >= 50 {\n\
mutate { add_tag => ["error"] }\n\
}\n\
\n\
# Redis log parsing\n\
if [type] == "redis" {\n\
grok {\n\
match => { "message" => "%%{POSINT:pid}:%%{WORD:role} %%{MONTHDAY} %%{MONTH} %%{TIME} %%{WORD:loglevel} %%{GREEDYDATA:redis_message}" }\n\
}\n\
\n\
# Tag errors (WARNING/ERROR) for Bugsink forwarding\n\
if [loglevel] in ["WARNING", "ERROR"] {\n\
mutate { add_tag => ["error"] }\n\
}\n\
# Tag INFO-level operational events (startup, config, persistence)\n\
else if [loglevel] == "INFO" {\n\
mutate { add_tag => ["redis_operational"] }\n\
}\n\
}\n\
\n\
# PostgreSQL function log parsing (ADR-050)\n\
if [type] == "postgres" {\n\
# Extract timestamp and process ID from PostgreSQL log prefix\n\
# Format: "2026-01-18 10:30:00 PST [12345] user@database "\n\
grok {\n\
match => { "message" => "%%{TIMESTAMP_ISO8601:pg_timestamp} \\\\[%%{POSINT:pg_pid}\\\\] %%{USERNAME:pg_user}@%%{WORD:pg_database} %%{GREEDYDATA:pg_message}" }\n\
}\n\
\n\
# Check if this is a structured JSON log from fn_log()\n\
# fn_log() emits JSON like: {"timestamp":"...","level":"WARNING","source":"postgresql","function":"award_achievement",...}\n\
if [pg_message] =~ /^\\{.*"source":"postgresql".*\\}$/ {\n\
json {\n\
source => "pg_message"\n\
target => "fn_log"\n\
}\n\
\n\
# Mark as error if level is WARNING or ERROR\n\
if [fn_log][level] in ["WARNING", "ERROR"] {\n\
mutate { add_tag => ["error", "db_function"] }\n\
}\n\
}\n\
\n\
# Also catch native PostgreSQL errors\n\
if [pg_message] =~ /^ERROR:/ or [pg_message] =~ /^FATAL:/ {\n\
mutate { add_tag => ["error", "postgres_native"] }\n\
}\n\
}\n\
}\n\
\n\
output {\n\
# Forward errors to Bugsink\n\
if "error" in [tags] {\n\
http {\n\
url => "http://localhost:8000/api/store/"\n\
http_method => "post"\n\
format => "json"\n\
}\n\
}\n\
\n\
# Store Redis operational logs (INFO level) to file\n\
if "redis_operational" in [tags] {\n\
file {\n\
path => "/var/log/logstash/redis-operational-%%{+YYYY-MM-dd}.log"\n\
codec => json_lines\n\
}\n\
}\n\
\n\
# Debug output (comment out in production)\n\
stdout { codec => rubydebug }\n\
}\n\
' > /etc/logstash/conf.d/bugsink.conf
COPY docker/logstash/bugsink.conf /etc/logstash/conf.d/bugsink.conf
# Create Logstash directories
RUN mkdir -p /var/lib/logstash && chown -R logstash:logstash /var/lib/logstash
RUN mkdir -p /var/log/logstash && chown -R logstash:logstash /var/log/logstash
# Create PM2 log directory (ADR-014 Parity)
# Logs written here will be picked up by Logstash (ADR-050)
RUN mkdir -p /var/log/pm2
# ============================================================================
# Configure Nginx
# ============================================================================

105
certs/README.md Normal file
View File

@@ -0,0 +1,105 @@
# Development SSL Certificates
This directory contains SSL certificates for the development container HTTPS setup.
## Files
| File | Purpose | Generated By |
| --------------- | ---------------------------------------------------- | -------------------------- |
| `localhost.crt` | SSL certificate for localhost and 127.0.0.1 | mkcert (in Dockerfile.dev) |
| `localhost.key` | Private key for localhost.crt | mkcert (in Dockerfile.dev) |
| `mkcert-ca.crt` | Root CA certificate for trusting mkcert certificates | mkcert |
## Certificate Details
The `localhost.crt` certificate includes the following Subject Alternative Names (SANs):
- `DNS:localhost`
- `IP Address:127.0.0.1`
- `IP Address:::1` (IPv6 localhost)
This allows the development server to be accessed via both `https://localhost/` and `https://127.0.0.1/` without SSL errors.
## Installing the CA Certificate (Recommended)
To avoid SSL certificate warnings in your browser, install the mkcert CA certificate on your system.
### Windows
1. Double-click `mkcert-ca.crt`
2. Click **"Install Certificate..."**
3. Select **"Local Machine"** > Next
4. Select **"Place all certificates in the following store"**
5. Click **Browse** > Select **"Trusted Root Certification Authorities"** > OK
6. Click **Next** > **Finish**
7. Restart your browser
### macOS
```bash
sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain certs/mkcert-ca.crt
```
### Linux
```bash
# Ubuntu/Debian
sudo cp certs/mkcert-ca.crt /usr/local/share/ca-certificates/mkcert-ca.crt
sudo update-ca-certificates
# Fedora/RHEL
sudo cp certs/mkcert-ca.crt /etc/pki/ca-trust/source/anchors/
sudo update-ca-trust
```
### Firefox (All Platforms)
Firefox uses its own certificate store:
1. Open Firefox Settings
2. Search for "Certificates"
3. Click **"View Certificates"**
4. Go to **"Authorities"** tab
5. Click **"Import..."**
6. Select `certs/mkcert-ca.crt`
7. Check **"Trust this CA to identify websites"**
8. Click **OK**
## After Installation
Once the CA certificate is installed:
- Your browser will trust all mkcert certificates without warnings
- Access `https://localhost/` with no security warnings
- Images from `https://127.0.0.1/flyer-images/` will load without SSL errors
## Regenerating Certificates
If you need to regenerate the certificates (e.g., after rebuilding the container):
```bash
# Inside the container
cd /app/certs
mkcert localhost 127.0.0.1 ::1
mv localhost+2.pem localhost.crt
mv localhost+2-key.pem localhost.key
nginx -s reload
# Copy the new CA to the host
podman cp flyer-crawler-dev:/app/certs/mkcert-ca.crt ./certs/mkcert-ca.crt
```
Then reinstall the CA certificate as described above.
## Security Note
**DO NOT** commit the private key (`localhost.key`) to version control in production projects. For this development-only project, the certificates are checked in for convenience since they're only used locally with self-signed certificates.
The certificates in this directory are automatically generated by the Dockerfile.dev and should not be used in production.
## See Also
- [Dockerfile.dev](../Dockerfile.dev) - Certificate generation (line ~69)
- [docker/nginx/dev.conf](../docker/nginx/dev.conf) - NGINX SSL configuration
- [docs/FLYER-URL-CONFIGURATION.md](../docs/FLYER-URL-CONFIGURATION.md) - URL configuration details
- [docs/development/DEBUGGING.md](../docs/development/DEBUGGING.md) - SSL troubleshooting

View File

@@ -1,19 +1,25 @@
-----BEGIN CERTIFICATE-----
MIIDCTCCAfGgAwIBAgIUHhZUK1vmww2wCepWPuVcU6d27hMwDQYJKoZIhvcNAQEL
BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDExODAyMzM0NFoXDTI3MDEx
ODAyMzM0NFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
AAOCAQ8AMIIBCgKCAQEAuUJGtSZzd+ZpLi+efjrkxJJNfVxVz2VLhknNM2WKeOYx
JTK/VaTYq5hrczy6fEUnMhDAJCgEPUFlOK3vn1gFJKNMN8m7arkLVk6PYtrx8CTw
w78Q06FLITr6hR0vlJNpN4MsmGxYwUoUpn1j5JdfZF7foxNAZRiwoopf7ZJxltDu
PIuFjmVZqdzR8c6vmqIqdawx/V6sL9fizZr+CDH3oTsTUirn2qM+1ibBtPDiBvfX
omUsr6MVOcTtvnMvAdy9NfV88qwF7MEWBGCjXkoT1bKCLD8hjn8l7GjRmPcmMFE2
GqWEvfJiFkBK0CgSHYEUwzo0UtVNeQr0k0qkDRub6QIDAQABo1MwUTAdBgNVHQ4E
FgQU5VeD67yFLV0QNYbHaJ6u9cM6UbkwHwYDVR0jBBgwFoAU5VeD67yFLV0QNYbH
aJ6u9cM6UbkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEABueA
8ujAD+yjeP5dTgqQH1G0hlriD5LmlJYnktaLarFU+y+EZlRFwjdORF/vLPwSG+y7
CLty/xlmKKQop70QzQ5jtJcsWzUjww8w1sO3AevfZlIF3HNhJmt51ihfvtJ7DVCv
CNyMeYO0pBqRKwOuhbG3EtJgyV7MF8J25UEtO4t+GzX3jcKKU4pWP+kyLBVfeDU3
MQuigd2LBwBQQFxZdpYpcXVKnAJJlHZIt68ycO1oSBEJO9fIF0CiAlC6ITxjtYtz
oCjd6cCLKMJiC6Zg7t1Q17vGl+FdGyQObSsiYsYO9N3CVaeDdpyGCH0Rfa0+oZzu
a5U9/l1FHlvpX980bw==
MIIEJDCCAoygAwIBAgIQfbdj1KSREvW82wxWZcDRsDANBgkqhkiG9w0BAQsFADBf
MR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExGjAYBgNVBAsMEXJvb3RA
OTIxZjA1YmRkNDk4MSEwHwYDVQQDDBhta2NlcnQgcm9vdEA5MjFmMDViZGQ0OTgw
HhcNMjYwMTIyMjAwMzIwWhcNMjgwNDIyMjAwMzIwWjBFMScwJQYDVQQKEx5ta2Nl
cnQgZGV2ZWxvcG1lbnQgY2VydGlmaWNhdGUxGjAYBgNVBAsMEXJvb3RANzk2OWU1
ZjA2MGM4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2u+HpnTr+Ecj
X6Ur4I8YHVKiEIgOFozJWWAeCPSZg0lr9/z3UDl7QPfRKaDk11OMN7DB0Y/ZAIHj
kMFdYc+ODOUAKGv/MvlM41OeRUXhrt0Gr/TJoDe9RLzs9ffASpqTHUNNrmyj/fLP
M5VZU+4Y2POFq8qji6Otkvqr6wp+2CTS/fkAtenFS2X+Z5u6BBq1/KBSWZhUFSuJ
sAGN9u+l20Cj/Sxp2nD1npvMEvPehFKEK2tZGgFr+X426jJ4Znd19EyZoI/xetG6
ybSzBdQk1KbhWFa3LPuOG814m9Qh21vaL0Hj2hpqC3KEQ2jiuCovjRS+RBGatltl
5m47Cj0UrQIDAQABo3YwdDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYB
BQUHAwEwHwYDVR0jBBgwFoAUjf0NOmUuoqSSD1Mbu/3G+wYxvjEwLAYDVR0RBCUw
I4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3DQEB
CwUAA4IBgQAorThUyu4FxHkOafMp/4CmvptfNvYeYP81DW0wpJxGL68Bz+idvXKv
JopX/ZAr+dDhS/TLeSnzt83W7TaBBHY+usvsiBx9+pZOVZIpreXRamPu7utmuC46
dictMNGlRNX9bwAApOJ24NCpOVSIIKtjHcjl4idwUHqLVGS+3wsmxIILYxigzkuT
fcK5vs0ItZWeuunsBAGb/U/Iu9zZ71rtmBejxNPyEvd8+bZC0m2mtV8C0Lpn58jZ
FiEf5OHiOdWG9O/uh3QeVWkuKLmaH6a8VdKRSIlOxEEZdkUYlwuVvzeWgFw4kjm8
rNWz0aIPovmcgLXoUG1d1D8klveGd3plF7N2p3xWqKmS6R6FJHx7MIxH6XmBATii
x/193Sgzqe8mwXQr14ulg2M/B3ZWleNdD6SeieADSgvRKjnlO7xwUmcFrffnEKEG
WcSPoGIuZ8V2pkYLh7ipPN+tFUbhmrWnsao5kQ0sqLlfscyO9hYQeQRtTjtC3P8r
FJYOdBMOCzE=
-----END CERTIFICATE-----

View File

@@ -1,28 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC5Qka1JnN35mku
L55+OuTEkk19XFXPZUuGSc0zZYp45jElMr9VpNirmGtzPLp8RScyEMAkKAQ9QWU4
re+fWAUko0w3ybtquQtWTo9i2vHwJPDDvxDToUshOvqFHS+Uk2k3gyyYbFjBShSm
fWPkl19kXt+jE0BlGLCiil/tknGW0O48i4WOZVmp3NHxzq+aoip1rDH9Xqwv1+LN
mv4IMfehOxNSKufaoz7WJsG08OIG99eiZSyvoxU5xO2+cy8B3L019XzyrAXswRYE
YKNeShPVsoIsPyGOfyXsaNGY9yYwUTYapYS98mIWQErQKBIdgRTDOjRS1U15CvST
SqQNG5vpAgMBAAECggEAAnv0Dw1Mv+rRy4ZyxtObEVPXPRzoxnDDXzHP4E16BTye
Fc/4pSBUIAUn2bPvLz0/X8bMOa4dlDcIv7Eu9Pvns8AY70vMaUReA80fmtHVD2xX
1PCT0X3InnxRAYKstSIUIGs+aHvV5Z+iJ8F82soOStN1MU56h+JLWElL5deCPHq3
tLZT8wM9aOZlNG72kJ71+DlcViahynQj8+VrionOLNjTJ2Jv/ByjM3GMIuSdBrgd
Sl4YAcdn6ontjJGoTgI+e+qkBAPwMZxHarNGQgbS0yNVIJe7Lq4zIKHErU/ZSmpD
GzhdVNzhrjADNIDzS7G+pxtz+aUxGtmRvOyopy8GAQKBgQDEPp2mRM+uZVVT4e1j
pkKO1c3O8j24I5mGKwFqhhNs3qGy051RXZa0+cQNx63GokXQan9DIXzc/Il7Y72E
z9bCFbcSWnlP8dBIpWiJm+UmqLXRyY4N8ecNnzL5x+Tuxm5Ij+ixJwXgdz/TLNeO
MBzu+Qy738/l/cAYxwcF7mR7AQKBgQDxq1F95HzCxBahRU9OGUO4s3naXqc8xKCC
m3vbbI8V0Exse2cuiwtlPPQWzTPabLCJVvCGXNru98sdeOu9FO9yicwZX0knOABK
QfPyDeITsh2u0C63+T9DNn6ixI/T68bTs7DHawEYbpS7bR50BnbHbQrrOAo6FSXF
yC7+Te+o6QKBgQCXEWSmo/4D0Dn5Usg9l7VQ40GFd3EPmUgLwntal0/I1TFAyiom
gpcLReIogXhCmpSHthO1h8fpDfZ/p+4ymRRHYBQH6uHMKugdpEdu9zVVpzYgArp5
/afSEqVZJwoSzWoELdQA23toqiPV2oUtDdiYFdw5nDccY1RHPp8nb7amAQKBgQDj
f4DhYDxKJMmg21xCiuoDb4DgHoaUYA0xpii8cL9pq4KmBK0nVWFO1kh5Robvsa2m
PB+EfNjkaIPepLxWbOTUEAAASoDU2JT9UoTQcl1GaUAkFnpEWfBB14TyuNMkjinH
lLpvn72SQFbm8VvfoU4jgfTrZP/LmajLPR1v6/IWMQKBgBh9qvOTax/GugBAWNj3
ZvF99rHOx0rfotEdaPcRN66OOiSWILR9yfMsTvwt1V0VEj7OqO9juMRFuIyB57gd
Hs/zgbkuggqjr1dW9r22P/UpzpodAEEN2d52RSX8nkMOkH61JXlH2MyRX65kdExA
VkTDq6KwomuhrU3z0+r/MSOn
MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDa74emdOv4RyNf
pSvgjxgdUqIQiA4WjMlZYB4I9JmDSWv3/PdQOXtA99EpoOTXU4w3sMHRj9kAgeOQ
wV1hz44M5QAoa/8y+UzjU55FReGu3Qav9MmgN71EvOz198BKmpMdQ02ubKP98s8z
lVlT7hjY84WryqOLo62S+qvrCn7YJNL9+QC16cVLZf5nm7oEGrX8oFJZmFQVK4mw
AY3276XbQKP9LGnacPWem8wS896EUoQra1kaAWv5fjbqMnhmd3X0TJmgj/F60brJ
tLMF1CTUpuFYVrcs+44bzXib1CHbW9ovQePaGmoLcoRDaOK4Ki+NFL5EEZq2W2Xm
bjsKPRStAgMBAAECggEBALFhpHwe+xh7OpPBlR0pkpYfXyMZuKBYjMIW9/61frM6
B3oywIWFLPFkV1js/Lvg+xgb48zQSTb6BdBAelJHAYY8+7XEWk2IYt1D4FWr2r/8
X/Cr2bgvsO9CSpK2mltXhZ4N66BIcU3NLkdS178Ch6svErwvP/ZhNL6Czktug3rG
S2fxpKqoVQhqWiEBV0vBWpw2oskvcpP2Btev2eaJeDmP1IkCaKIU9jJzmSg8UKj3
AxJJ8lJvlDi2Z8mfB0BVIFZSI1s44LqbuPdITsWvG49srdAhyLkifcbY2r7eH8+s
rFNswCaqqNwmzZXHMFedfvgVJHtCTGX/U8G55dcG/YECgYEA6PvcX+axT4aSaG+/
fQpyW8TmNmIwS/8HD8rA5dSBB2AE+RnqbxxB1SWWk4w47R/riYOJCy6C88XUZWLn
05cYotR9lA6tZOYqIPUIxfCHl4vDDBnOGAB0ga3z5vF3X7HV6uspPFcrGVJ9k42S
BK1gk7Kj8RxVQMqXSqkR/pXJ4l0CgYEA8JBkZ4MRi1D74SAU+TY1rnUCTs2EnP1U
TKAI9qHyvNJOFZaObWPzegkZryZTm+yTblwvYkryMjvsiuCdv6ro/VKc667F3OBs
dJ/8+ylO0lArP9a23QHViNSvQmyi3bsw2UpIGQRGq2C4LxDceo5cYoVOWIlLl8u3
LUuL0IfxdpECgYEAyMd0DPljyGLyfSoAXaPJFajDtA4+DOAEl+lk/yt43oAzCPD6
hTJW0XcJIrJuxHsDoohGa+pzU90iwxTPMBtAUeLJLfTQHOn1WF2SZ/J3B3ScbCs4
3ppVzQO580YYV9GLxl1ONf/w1muuaKBSO9GmLuJ+QeTm22U7qE23gixXxMkCgYEA
3R7cK4l+huBZpgUnQitiDInhJS4jx2nUItq3YnxZ8tYckBtjr4lAM9xJj4VbNOew
XLC/nUnmdeY+9yif153xq2hUdQ6hMPXYuxqUHwlJOmgWWQez7lHRRYS51ASnb8iw
jgqJWvVjQAQXSKvm/X/9y1FdQmRw54aJSUk3quZKPQECgYA5DqFXn/jYOUCGfCUD
RENWe+3fNZSJCWmr4yvjEy9NUT5VcA+/hZHbbLUnVHfAHxJmdGhTzFe+ZGWFAGwi
Wo62BDu1fqHHegCWaFFluDnz8iZTxXtHMEGSBZnXC4f5wylcxRtCPUbrLMRTPo4O
t85qeBu1dMq1XQtU6ab3w3W0nw==
-----END PRIVATE KEY-----

27
certs/mkcert-ca.crt Normal file
View File

@@ -0,0 +1,27 @@
-----BEGIN CERTIFICATE-----
MIIEjjCCAvagAwIBAgIRAJtq2Z0+W981Ct2dMVPb3bQwDQYJKoZIhvcNAQELBQAw
XzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMRowGAYDVQQLDBFyb290
QDkyMWYwNWJkZDQ5ODEhMB8GA1UEAwwYbWtjZXJ0IHJvb3RAOTIxZjA1YmRkNDk4
MB4XDTI2MDEyMjA5NDcxNVoXDTM2MDEyMjA5NDcxNVowXzEeMBwGA1UEChMVbWtj
ZXJ0IGRldmVsb3BtZW50IENBMRowGAYDVQQLDBFyb290QDkyMWYwNWJkZDQ5ODEh
MB8GA1UEAwwYbWtjZXJ0IHJvb3RAOTIxZjA1YmRkNDk4MIIBojANBgkqhkiG9w0B
AQEFAAOCAY8AMIIBigKCAYEAxXR5gXDwv5cfQSud1eEhwDuaSaf5kf8NtPnucZXY
AN+/QW1+OEKJFuawj/YrSbL/yIB8sUSJToEYNJ4LAgzIZ4+TPYVvOIPqQnimfj98
2AKCt73U/AMvoEpts7U0s37f5wF8o+BHXfChxyN//z96+wsQ+2+Q9QBGjirvScF+
8FRnupcygDeGZ8x3JQVaEfEV6iYyXFl/4tEDVr9QX4avyUlf0vp1Y90TG3L42JYQ
xDU37Ct9dqsxPCPOPjmkQi9HV5TeqLTs/4NdrEYOSk7bOVMzL8EHs2prRL7sWzYJ
gRT+VXFPpoSCkZs1gS3FNXukTGx5LNsstyJZRa99cGgDcqvNseig06KUzZrRnCig
kARLF/n8VTpHETEuTdxdnXJO3i2N/99mG/2/lej9HNDMaqg45ur5EhaFhHarXMtc
wy7nfGoThzscZvpFbVHorhgRxzsUqRHTMHa9mUOYtShMA0IwFccHcczY3CDxXLg9
hC+24pdiCxtRmi23JB10Th+nAgMBAAGjRTBDMA4GA1UdDwEB/wQEAwICBDASBgNV
HRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBSN/Q06ZS6ipJIPUxu7/cb7BjG+MTAN
BgkqhkiG9w0BAQsFAAOCAYEAghQL80XpPs70EbCmETYri5ryjJPbsxK3Z3QgBYoy
/P/021eK5woLx4EDeEUyrRayOtLlDWiIMdV7FmlY4GpEdonVbqFJY2gghkIuQpkw
lWqbk08F+iXe8PSGD1qz4y6enioCRAx+RaZ6jnVJtaC8AR257FVhGIzDBiOA+LDM
L1yq6Bxxij17q9s5HL9KuzxWRMuXACHmaGBXHpl/1n4dIxi2lXRgp+1xCR/VPNPt
/ZRy29kncd8Fxx+VEtc0muoJvRo4ttVWhBvAVJrkAeukjYKpcDDfRU6Y22o54jD/
mseDb+0UPwSxaKbnGJlCcRbbh6i14bA4KfPq93bZX+Tlq9VCp8LvQSl3oU+23RVc
KjBB9EJnoBBNGIY7VmRI+QowrnlP2wtg2fTNaPqULtjqA9frbMTP0xTlumLzGB+6
9Da7/+AE2in3Aa8Xyry4BiXbk2L6c1xz/Cd1ZpFrSXAOEi1Xt7so/Ck0yM3c2KWK
5aSfCjjOzONMPJyY1oxodJ1p
-----END CERTIFICATE-----

View File

@@ -46,6 +46,10 @@ services:
- node_modules_data:/app/node_modules
# Mount PostgreSQL logs for Logstash access (ADR-050)
- postgres_logs:/var/log/postgresql:ro
# Mount Redis logs for Logstash access (ADR-050)
- redis_logs:/var/log/redis:ro
# Mount PM2 logs for Logstash access (ADR-050, ADR-014)
- pm2_logs:/var/log/pm2
ports:
- '80:80' # HTTP redirect to HTTPS (matches production)
- '443:443' # Frontend HTTPS (nginx proxies Vite 5173 → 443)
@@ -166,10 +170,17 @@ services:
redis:
image: docker.io/library/redis:alpine
container_name: flyer-crawler-redis
# Run as root to allow entrypoint to set up log directory permissions
# Redis will drop privileges after setup via gosu in entrypoint
user: root
ports:
- '6379:6379'
volumes:
- redis_data:/data
# Create log volume for Logstash access (ADR-050)
- redis_logs:/var/log/redis
# Mount custom entrypoint for log directory setup (ADR-050)
- ./docker/redis/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh:ro
# Healthcheck ensures redis is ready before app starts
healthcheck:
test: ['CMD', 'redis-cli', 'ping']
@@ -177,8 +188,17 @@ services:
timeout: 5s
retries: 10
start_period: 5s
# Enable persistence for development data
command: redis-server --appendonly yes
# Enable persistence and file logging for Logstash (ADR-050)
# Redis log levels: debug, verbose, notice (default), warning
# Custom entrypoint sets up log directory with correct permissions
entrypoint: ['/usr/local/bin/docker-entrypoint.sh']
command:
- --appendonly
- 'yes'
- --logfile
- /var/log/redis/redis-server.log
- --loglevel
- notice
# ===================
# Named Volumes
@@ -190,6 +210,10 @@ volumes:
name: flyer-crawler-postgres-logs
redis_data:
name: flyer-crawler-redis-data
redis_logs:
name: flyer-crawler-redis-logs
pm2_logs:
name: flyer-crawler-pm2-logs
node_modules_data:
name: flyer-crawler-node-modules

View File

@@ -0,0 +1,461 @@
# docker/logstash/bugsink.conf
# ============================================================================
# Logstash Pipeline Configuration for Bugsink Error Tracking
# ============================================================================
# This configuration aggregates logs from multiple sources and forwards errors
# to Bugsink (Sentry-compatible error tracking) in the dev container.
#
# Sources:
# - PM2 managed logs (/var/log/pm2/*.log) - API, Worker, Vite (ADR-014)
# - Pino application logs (/app/logs/*.log) - JSON format (fallback)
# - PostgreSQL logs (/var/log/postgresql/*.log) - Including fn_log() output
# - NGINX logs (/var/log/nginx/*.log) - Access and error logs
# - Redis logs (/var/log/redis/*.log) - Via shared volume (ADR-050)
#
# Bugsink Projects:
# - Project 1: Backend API (Dev) - Pino errors, PostgreSQL errors
# - Project 2: Frontend (Dev) - Configured via Sentry SDK in browser
#
# Related Documentation:
# - docs/adr/0050-postgresql-function-observability.md
# - docs/adr/0015-application-performance-monitoring-and-error-tracking.md
# - docs/operations/LOGSTASH-QUICK-REF.md
# ============================================================================
input {
# ============================================================================
# PM2 Managed Process Logs (ADR-014, ADR-050)
# ============================================================================
# PM2 manages all Node.js processes in the dev container, matching production.
# Logs are written to /var/log/pm2 for Logstash integration.
#
# Process logs:
# - api-out.log / api-error.log: API server (Pino JSON format)
# - worker-out.log / worker-error.log: Background worker (Pino JSON format)
# - vite-out.log / vite-error.log: Vite dev server (plain text)
# ============================================================================
# PM2 API Server Logs (Pino JSON format)
file {
path => "/var/log/pm2/api-out.log"
codec => json_lines
type => "pm2_api"
tags => ["app", "backend", "pm2", "api"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_api_out"
}
file {
path => "/var/log/pm2/api-error.log"
codec => json_lines
type => "pm2_api"
tags => ["app", "backend", "pm2", "api", "stderr"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_api_error"
}
# PM2 Worker Logs (Pino JSON format)
file {
path => "/var/log/pm2/worker-out.log"
codec => json_lines
type => "pm2_worker"
tags => ["app", "worker", "pm2"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_worker_out"
}
file {
path => "/var/log/pm2/worker-error.log"
codec => json_lines
type => "pm2_worker"
tags => ["app", "worker", "pm2", "stderr"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_worker_error"
}
# PM2 Vite Logs (plain text format)
file {
path => "/var/log/pm2/vite-out.log"
codec => plain
type => "pm2_vite"
tags => ["app", "frontend", "pm2", "vite"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_vite_out"
}
file {
path => "/var/log/pm2/vite-error.log"
codec => plain
type => "pm2_vite"
tags => ["app", "frontend", "pm2", "vite", "stderr"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_vite_error"
}
# ============================================================================
# Pino Application Logs (Fallback Path)
# ============================================================================
# JSON-formatted logs from the Node.js application using Pino logger.
# Note: Primary logs now go to /var/log/pm2. This is a fallback.
# Log levels: 10=trace, 20=debug, 30=info, 40=warn, 50=error, 60=fatal
file {
path => "/app/logs/*.log"
codec => json
type => "pino"
tags => ["app", "backend"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pino"
}
# ============================================================================
# PostgreSQL Function Logs (ADR-050)
# ============================================================================
# Captures PostgreSQL log output including fn_log() structured JSON messages.
# PostgreSQL is configured to write logs to /var/log/postgresql/ (shared volume).
# Log format: "2026-01-22 00:00:00 UTC [5724] postgres@flyer_crawler_dev LOG: message"
file {
path => "/var/log/postgresql/*.log"
type => "postgres"
tags => ["postgres", "database"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_postgres"
}
# ============================================================================
# NGINX Logs
# ============================================================================
# Access logs for request monitoring and error logs for proxy/SSL issues.
file {
path => "/var/log/nginx/access.log"
type => "nginx_access"
tags => ["nginx", "access"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_nginx_access"
}
file {
path => "/var/log/nginx/error.log"
type => "nginx_error"
tags => ["nginx", "error"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_nginx_error"
}
# ============================================================================
# Redis Logs (ADR-050)
# ============================================================================
# Redis logs from the shared volume. Redis uses its own log format:
# "<pid>:<role> <day> <month> <time> <loglevel> <message>"
# Example: "1:M 22 Jan 2026 14:30:00.123 * Ready to accept connections"
# Log levels: . (debug), - (verbose), * (notice), # (warning)
file {
path => "/var/log/redis/redis-server.log"
type => "redis"
tags => ["redis", "infra"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_redis"
}
}
filter {
# ============================================================================
# PM2 API/Worker Log Processing (ADR-014)
# ============================================================================
# PM2 API and Worker logs are in Pino JSON format.
# Process them the same way as direct Pino logs.
if [type] in ["pm2_api", "pm2_worker"] {
# Tag errors (level 50 = error, 60 = fatal)
if [level] >= 50 {
mutate { add_tag => ["error", "pm2_pino_error"] }
# Map Pino level to Sentry level and set error_message field
if [level] == 60 {
mutate { add_field => { "sentry_level" => "fatal" } }
} else {
mutate { add_field => { "sentry_level" => "error" } }
}
# Copy msg to error_message for consistent access in output
mutate { add_field => { "error_message" => "%{msg}" } }
}
}
# ============================================================================
# PM2 Vite Log Processing (ADR-014)
# ============================================================================
# Vite logs are plain text. Look for error patterns.
if [type] == "pm2_vite" {
# Detect Vite build/compilation errors
if [message] =~ /(?i)(error|failed|exception|cannot|enoent|eperm|EACCES|ECONNREFUSED)/ {
mutate { add_tag => ["error", "vite_error"] }
mutate { add_field => { "sentry_level" => "error" } }
mutate { add_field => { "error_message" => "Vite: %{message}" } }
}
}
# ============================================================================
# Pino Log Processing (Fallback)
# ============================================================================
if [type] == "pino" {
# Tag errors (level 50 = error, 60 = fatal)
if [level] >= 50 {
mutate { add_tag => ["error", "pino_error"] }
# Map Pino level to Sentry level and set error_message field
if [level] == 60 {
mutate { add_field => { "sentry_level" => "fatal" } }
} else {
mutate { add_field => { "sentry_level" => "error" } }
}
# Copy msg to error_message for consistent access in output
mutate { add_field => { "error_message" => "%{msg}" } }
}
}
# ============================================================================
# PostgreSQL Log Processing (ADR-050)
# ============================================================================
# PostgreSQL log format in dev container:
# "2026-01-22 00:00:00 UTC [5724] postgres@flyer_crawler_dev LOG: message"
# "2026-01-22 07:06:03 UTC [19851] postgres@flyer_crawler_dev ERROR: column "id" does not exist"
if [type] == "postgres" {
# Parse PostgreSQL log prefix with UTC timezone
grok {
match => { "message" => "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME} %{WORD:pg_timezone} \[%{POSINT:pg_pid}\] %{DATA:pg_user}@%{DATA:pg_database} %{WORD:pg_level}: ?%{GREEDYDATA:pg_message}" }
tag_on_failure => ["_postgres_grok_failure"]
}
# Check if this is a structured JSON log from fn_log()
if [pg_message] =~ /^\{.*"source":"postgresql".*\}$/ {
json {
source => "pg_message"
target => "fn_log"
}
# Mark as error if level is WARNING or ERROR
if [fn_log][level] in ["WARNING", "ERROR"] {
mutate { add_tag => ["error", "db_function"] }
mutate { add_field => { "sentry_level" => "warning" } }
if [fn_log][level] == "ERROR" {
mutate { replace => { "sentry_level" => "error" } }
}
# Use fn_log message for error_message
mutate { add_field => { "error_message" => "%{[fn_log][message]}" } }
}
}
# Catch native PostgreSQL errors (ERROR: or FATAL: prefix in pg_level)
if [pg_level] == "ERROR" or [pg_level] == "FATAL" {
mutate { add_tag => ["error", "postgres_native"] }
if [pg_level] == "FATAL" {
mutate { add_field => { "sentry_level" => "fatal" } }
} else {
mutate { add_field => { "sentry_level" => "error" } }
}
# Use the full pg_message for error_message
mutate { add_field => { "error_message" => "PostgreSQL %{pg_level}: %{pg_message}" } }
}
}
# ============================================================================
# NGINX Access Log Processing
# ============================================================================
if [type] == "nginx_access" {
grok {
match => { "message" => '%{IPORHOST:client_ip} - %{DATA:user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:user_agent}"' }
tag_on_failure => ["_nginx_access_grok_failure"]
}
# Tag 5xx errors for Bugsink
if [status] =~ /^5/ {
mutate { add_tag => ["error", "nginx_5xx"] }
mutate { add_field => { "sentry_level" => "error" } }
mutate { add_field => { "error_message" => "HTTP %{status} %{method} %{request}" } }
}
}
# ============================================================================
# NGINX Error Log Processing
# ============================================================================
# NGINX error log format: "2026/01/22 17:55:01 [error] 16#16: *3 message..."
if [type] == "nginx_error" {
grok {
match => { "message" => "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME} \[%{LOGLEVEL:nginx_level}\] %{POSINT:pid}#%{POSINT}: (\*%{POSINT:connection} )?%{GREEDYDATA:nginx_message}" }
tag_on_failure => ["_nginx_error_grok_failure"]
}
# Only process actual errors, not notices (like "signal process started")
if [nginx_level] in ["error", "crit", "alert", "emerg"] {
mutate { add_tag => ["error", "nginx_error"] }
# Map NGINX log level to Sentry level
if [nginx_level] in ["crit", "alert", "emerg"] {
mutate { add_field => { "sentry_level" => "fatal" } }
} else {
mutate { add_field => { "sentry_level" => "error" } }
}
mutate { add_field => { "error_message" => "NGINX [%{nginx_level}]: %{nginx_message}" } }
}
}
# ============================================================================
# Redis Log Processing (ADR-050)
# ============================================================================
# Redis log format: "<pid>:<role> <day> <month> <time>.<ms> <level> <message>"
# Example: "1:M 22 Jan 14:30:00.123 * Ready to accept connections"
# Roles: M=master, S=slave, C=sentinel, X=cluster
# Levels: . (debug), - (verbose), * (notice), # (warning)
if [type] == "redis" {
# Parse Redis log format
grok {
match => { "message" => "%{POSINT:redis_pid}:%{WORD:redis_role} %{MONTHDAY} %{MONTH} %{YEAR}? ?%{TIME} %{DATA:redis_level_char} %{GREEDYDATA:redis_message}" }
tag_on_failure => ["_redis_grok_failure"]
}
# Map Redis level characters to human-readable levels
# . = debug, - = verbose, * = notice, # = warning
if [redis_level_char] == "#" {
mutate {
add_field => { "redis_level" => "warning" }
add_tag => ["error", "redis_warning"]
}
mutate { add_field => { "sentry_level" => "warning" } }
mutate { add_field => { "error_message" => "Redis WARNING: %{redis_message}" } }
} else if [redis_level_char] == "*" {
mutate { add_field => { "redis_level" => "notice" } }
} else if [redis_level_char] == "-" {
mutate { add_field => { "redis_level" => "verbose" } }
} else if [redis_level_char] == "." {
mutate { add_field => { "redis_level" => "debug" } }
}
# Also detect error keywords in message content (e.g., ECONNREFUSED, OOM, etc.)
if [redis_message] =~ /(?i)(error|failed|refused|denied|timeout|oom|crash|fatal|exception)/ {
if "error" not in [tags] {
mutate { add_tag => ["error", "redis_error"] }
mutate { add_field => { "sentry_level" => "error" } }
mutate { add_field => { "error_message" => "Redis ERROR: %{redis_message}" } }
}
}
}
# ============================================================================
# Generate Sentry Event ID for all errors
# ============================================================================
if "error" in [tags] {
ruby {
code => '
require "securerandom"
event.set("sentry_event_id", SecureRandom.hex(16))
'
}
# Ensure error_message has a fallback value
if ![error_message] {
mutate { add_field => { "error_message" => "%{message}" } }
}
}
}
output {
# ============================================================================
# Forward Errors to Bugsink (Backend API Project)
# ============================================================================
# Bugsink uses Sentry-compatible API. Events must include:
# - event_id: 32 hex characters (UUID without dashes)
# - message: Human-readable error description
# - level: fatal, error, warning, info, debug
# - timestamp: Unix epoch in seconds
# - platform: "node" for backend, "javascript" for frontend
#
# Authentication via X-Sentry-Auth header with project's public key.
# Dev container DSN: http://cea01396c56246adb5878fa5ee6b1d22@localhost:8000/1
# ============================================================================
if "error" in [tags] {
http {
url => "http://localhost:8000/api/1/store/"
http_method => "post"
format => "json"
headers => {
"X-Sentry-Auth" => "Sentry sentry_key=cea01396c56246adb5878fa5ee6b1d22, sentry_version=7"
"Content-Type" => "application/json"
}
# Transform event to Sentry format using regular fields (not @metadata)
mapping => {
"event_id" => "%{sentry_event_id}"
"timestamp" => "%{@timestamp}"
"level" => "%{sentry_level}"
"platform" => "node"
"logger" => "%{type}"
"message" => "%{error_message}"
"extra" => {
"hostname" => "%{[host][name]}"
"source_type" => "%{type}"
"tags" => "%{tags}"
"original_message" => "%{message}"
}
}
}
}
# ============================================================================
# Store Operational Logs to Files (for debugging/audit)
# ============================================================================
# NGINX access logs (all requests, not just errors)
if [type] == "nginx_access" and "error" not in [tags] {
file {
path => "/var/log/logstash/nginx-access-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# PostgreSQL operational logs (non-error)
if [type] == "postgres" and "error" not in [tags] {
file {
path => "/var/log/logstash/postgres-operational-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# Redis operational logs (non-error)
if [type] == "redis" and "error" not in [tags] {
file {
path => "/var/log/logstash/redis-operational-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# PM2 API operational logs (non-error) - ADR-014
if [type] == "pm2_api" and "error" not in [tags] {
file {
path => "/var/log/logstash/pm2-api-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# PM2 Worker operational logs (non-error) - ADR-014
if [type] == "pm2_worker" and "error" not in [tags] {
file {
path => "/var/log/logstash/pm2-worker-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# PM2 Vite operational logs (non-error) - ADR-014
if [type] == "pm2_vite" and "error" not in [tags] {
file {
path => "/var/log/logstash/pm2-vite-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# ============================================================================
# Debug Output (for development only)
# ============================================================================
# Uncomment to see all processed events in Logstash stdout
# stdout { codec => rubydebug }
}

View File

@@ -9,13 +9,39 @@
# - Frontend accessible on https://localhost (port 443)
# - Backend API on http://localhost:3001
# - Port 80 redirects to HTTPS
#
# IMPORTANT: Dual Hostname Configuration (localhost AND 127.0.0.1)
# ============================================================================
# The server_name directive includes BOTH 'localhost' and '127.0.0.1' to
# prevent SSL certificate errors when resources use different hostnames.
#
# Problem Scenario:
# 1. User accesses site via https://localhost/
# 2. Database stores image URLs as https://127.0.0.1/flyer-images/...
# 3. Browser treats these as different origins, showing ERR_CERT_AUTHORITY_INVALID
#
# Solution:
# - mkcert generates certificates valid for: localhost, 127.0.0.1, ::1
# - NGINX accepts requests to BOTH hostnames using the same certificate
# - Users can access via either hostname without SSL warnings
#
# The self-signed certificate is generated in Dockerfile.dev with:
# mkcert localhost 127.0.0.1 ::1
#
# This creates a certificate with Subject Alternative Names (SANs) for all
# three hostnames, allowing NGINX to serve valid HTTPS for each.
#
# See also:
# - Dockerfile.dev (certificate generation, ~line 69)
# - docs/FLYER-URL-CONFIGURATION.md (URL configuration details)
# - docs/development/DEBUGGING.md (SSL troubleshooting)
# ============================================================================
# HTTPS Server (main)
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name localhost;
server_name localhost 127.0.0.1;
# SSL Configuration (self-signed certificates from mkcert)
ssl_certificate /app/certs/localhost.crt;
@@ -80,7 +106,7 @@ server {
server {
listen 80;
listen [::]:80;
server_name localhost;
server_name localhost 127.0.0.1;
return 301 https://$host$request_uri;
}

View File

@@ -0,0 +1,24 @@
#!/bin/sh
# docker/redis/docker-entrypoint.sh
# ============================================================================
# Redis Entrypoint Script for Dev Container
# ============================================================================
# This script ensures the Redis log directory exists and is writable before
# starting Redis. This is needed because the redis_logs volume is mounted
# at /var/log/redis but Redis Alpine runs as a non-root user.
#
# Related: ADR-050 (Log aggregation via Logstash)
# ============================================================================
set -e
# Create log directory if it doesn't exist
mkdir -p /var/log/redis
# Ensure redis user can write to the log directory
# Note: In Alpine Redis image, redis runs as uid 999
chown -R redis:redis /var/log/redis
chmod 755 /var/log/redis
# Start Redis with the provided arguments
exec redis-server "$@"

View File

@@ -12,6 +12,87 @@ Flyer image and icon URLs are environment-specific to ensure they point to the c
| Test | `https://flyer-crawler-test.projectium.com` | `https://flyer-crawler-test.projectium.com/flyer-images/safeway-flyer.jpg` |
| Production | `https://flyer-crawler.projectium.com` | `https://flyer-crawler.projectium.com/flyer-images/safeway-flyer.jpg` |
**Note:** The dev container accepts connections to **both** `https://localhost/` and `https://127.0.0.1/` thanks to the SSL certificate and NGINX configuration. See [SSL Certificate Configuration](#ssl-certificate-configuration-dev-container) below.
## SSL Certificate Configuration (Dev Container)
The dev container uses self-signed certificates generated by [mkcert](https://github.com/FiloSottile/mkcert) to enable HTTPS locally. This configuration solves a common mixed-origin SSL issue.
### The Problem
When users access the site via `https://localhost/` but image URLs in the database use `https://127.0.0.1/...`, browsers treat these as different origins. This causes `ERR_CERT_AUTHORITY_INVALID` errors when loading images, even though both hostnames point to the same server.
### The Solution
1. **Certificate Generation** (`Dockerfile.dev`):
```bash
mkcert localhost 127.0.0.1 ::1
```
This creates a certificate with Subject Alternative Names (SANs) for all three hostnames.
2. **NGINX Configuration** (`docker/nginx/dev.conf`):
```nginx
server_name localhost 127.0.0.1;
```
NGINX accepts requests to both hostnames using the same SSL certificate.
### How It Works
| Component | Configuration |
| -------------------- | ---------------------------------------------------- |
| SSL Certificate SANs | `localhost`, `127.0.0.1`, `::1` |
| NGINX `server_name` | `localhost 127.0.0.1` |
| Seed Script URLs | Uses `https://127.0.0.1` (works with DB constraints) |
| User Access | Either `https://localhost/` or `https://127.0.0.1/` |
### Why This Matters
- **Database Constraints**: The `flyers` table has CHECK constraints requiring URLs to start with `http://` or `https://`. Relative URLs are not allowed.
- **Consistent Behavior**: Users can access the site using either hostname without SSL warnings.
- **Same Certificate**: Both hostnames use the same self-signed certificate, eliminating mixed-content errors.
### Verifying the Configuration
```bash
# Check certificate SANs
podman exec flyer-crawler-dev openssl x509 -in /app/certs/localhost.crt -text -noout | grep -A1 "Subject Alternative Name"
# Expected output:
# X509v3 Subject Alternative Name:
# DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1
# Test both hostnames respond
curl -k https://localhost/health
curl -k https://127.0.0.1/health
```
### Troubleshooting SSL Issues
If you encounter `ERR_CERT_AUTHORITY_INVALID`:
1. **Check NGINX is running**: `podman exec flyer-crawler-dev nginx -t`
2. **Verify certificate exists**: `podman exec flyer-crawler-dev ls -la /app/certs/`
3. **Ensure both hostnames are in server_name**: Check `/etc/nginx/sites-available/default`
4. **Rebuild container if needed**: The certificate is generated at build time
### Permanent Fix: Install CA Certificate (Recommended)
To permanently eliminate SSL certificate warnings, install the mkcert CA certificate on your system. This is optional but provides a better development experience.
The CA certificate is located at `certs/mkcert-ca.crt` in the project root. See [`certs/README.md`](../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
After installation:
- Your browser will trust all mkcert certificates without warnings
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
See also: [Debugging Guide - SSL Issues](development/DEBUGGING.md#ssl-certificate-issues)
## NGINX Static File Serving
All environments serve flyer images as static files with browser caching:
@@ -41,7 +122,7 @@ Set `FLYER_BASE_URL` in your environment configuration:
```bash
# Dev container (.env)
FLYER_BASE_URL=https://127.0.0.1
FLYER_BASE_URL=https://localhost
# Test environment
FLYER_BASE_URL=https://flyer-crawler-test.projectium.com
@@ -58,7 +139,7 @@ The seed script ([src/db/seed.ts](../src/db/seed.ts)) automatically uses the cor
2. `NODE_ENV` value:
- `production` → `https://flyer-crawler.projectium.com`
- `test` → `https://flyer-crawler-test.projectium.com`
- Default → `https://127.0.0.1`
- Default → `https://localhost`
The seed script also copies test images from `src/tests/assets/` to `public/flyer-images/`:
@@ -78,8 +159,8 @@ podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev
# Run the update (dev container uses HTTPS with self-signed certs)
UPDATE flyers
SET
image_url = REPLACE(image_url, 'example.com', '127.0.0.1'),
icon_url = REPLACE(icon_url, 'example.com', '127.0.0.1')
image_url = REPLACE(image_url, 'example.com', 'localhost'),
icon_url = REPLACE(icon_url, 'example.com', 'localhost')
WHERE
image_url LIKE '%example.com%'
OR icon_url LIKE '%example.com%';
@@ -131,8 +212,10 @@ export const getFlyerBaseUrl = (): string => {
}
// Check if we're in dev container (DB_HOST=postgres is typical indicator)
// Use 'localhost' instead of '127.0.0.1' to match the hostname users access
// This avoids SSL certificate mixed-origin issues in browsers
if (process.env.DB_HOST === 'postgres' || process.env.DB_HOST === '127.0.0.1') {
return 'https://127.0.0.1';
return 'https://localhost';
}
if (process.env.NODE_ENV === 'production') {
@@ -169,13 +252,13 @@ This approach ensures tests work correctly in all environments (dev container, C
| `docker/nginx/dev.conf` | Added `/flyer-images/` location block for static file serving |
| `.env.example` | Added `FLYER_BASE_URL` variable |
| `sql/update_flyer_urls.sql` | SQL script for updating existing data |
| Test files | Updated mock data to use `https://127.0.0.1` |
| Test files | Updated mock data to use `https://localhost` |
## Summary
- Seed script now uses environment-specific HTTPS URLs
- Seed script copies test images from `src/tests/assets/` to `public/flyer-images/`
- NGINX serves `/flyer-images/` as static files with 7-day cache
- Test files updated with `https://127.0.0.1`
- Test files updated with `https://localhost` (not `127.0.0.1` to avoid SSL mixed-origin issues)
- SQL script provided for updating existing data
- Documentation updated for each environment

View File

@@ -293,7 +293,23 @@ Google Gemini powers the AI extraction capabilities.
### Background Workers (BullMQ/PM2)
BullMQ workers handle asynchronous processing tasks.
BullMQ workers handle asynchronous processing tasks. PM2 manages both the API server and worker processes in production and development environments (ADR-014).
**Dev Container PM2 Processes**:
| Process | Script | Purpose |
| -------------------------- | ---------------------------------- | -------------------------- |
| `flyer-crawler-api-dev` | `tsx watch server.ts` | API server with hot reload |
| `flyer-crawler-worker-dev` | `tsx watch src/services/worker.ts` | Background job processing |
| `flyer-crawler-vite-dev` | `vite --host` | Frontend dev server |
**Production PM2 Processes**:
| Process | Script | Purpose |
| -------------------------------- | ------------------ | --------------------------- |
| `flyer-crawler-api` | `tsx server.ts` | API server (cluster mode) |
| `flyer-crawler-worker` | `tsx worker.ts` | Background job processing |
| `flyer-crawler-analytics-worker` | `tsx analytics.ts` | Analytics report generation |
**Job Queues**:
@@ -651,9 +667,11 @@ Attempt 4: 4x delay (e.g., 20 seconds)
| | Windows Host Machine | | Linux Dev Container | |
| | - VS Code | | (flyer-crawler-dev) | |
| | - Podman Desktop +---->+ - Node.js 22 | |
| | - Git | | - PostgreSQL 16 | |
| +-----------------------------------+ | - Redis 7 | |
| | - Git | | - PM2 (process manager) | |
| +-----------------------------------+ | - PostgreSQL 16 | |
| | - Redis 7 | |
| | - Bugsink (local) | |
| | - Logstash (log aggregation) | |
| +-----------------------------------+ |
+-----------------------------------------------------------------------------------+

View File

@@ -11,6 +11,7 @@ Common debugging strategies and troubleshooting patterns for Flyer Crawler.
- [API Errors](#api-errors)
- [Authentication Problems](#authentication-problems)
- [Background Job Issues](#background-job-issues)
- [SSL Certificate Issues](#ssl-certificate-issues)
- [Frontend Issues](#frontend-issues)
- [Performance Problems](#performance-problems)
- [Debugging Tools](#debugging-tools)
@@ -43,14 +44,24 @@ When something breaks, check these first:
4. **Are there recent errors in logs?**
```bash
# Application logs
# PM2 logs (dev container)
podman exec -it flyer-crawler-dev pm2 logs
# Container logs
podman logs -f flyer-crawler-dev
# PM2 logs (production)
pm2 logs flyer-crawler-api
```
5. **Is Redis accessible?**
5. **Are PM2 processes running?**
```bash
podman exec -it flyer-crawler-dev pm2 status
```
6. **Is Redis accessible?**
```bash
podman exec flyer-crawler-redis redis-cli ping
```
@@ -426,6 +437,92 @@ console.log('Expired:', decoded.exp < Date.now() / 1000);
---
## PM2 Process Issues (Dev Container)
### PM2 Process Not Starting
**Symptom**: `pm2 status` shows process as "errored" or "stopped"
**Debug**:
```bash
# Check process status
podman exec -it flyer-crawler-dev pm2 status
# View process logs
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev --lines 50
# Show detailed process info
podman exec -it flyer-crawler-dev pm2 show flyer-crawler-api-dev
# Check if port is in use
podman exec -it flyer-crawler-dev netstat -tlnp | grep 3001
```
**Common Causes**:
- Port already in use by another process
- Missing environment variables
- Syntax error in code
- Database connection failure
**Solutions**:
```bash
# Restart specific process
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
# Restart all processes
podman exec -it flyer-crawler-dev pm2 restart all
# Delete and recreate processes
podman exec -it flyer-crawler-dev pm2 delete all
podman exec -it flyer-crawler-dev pm2 start /app/ecosystem.dev.config.cjs
```
### PM2 Log Access
**View logs in real-time**:
```bash
# All processes
podman exec -it flyer-crawler-dev pm2 logs
# Specific process
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
# Last 100 lines
podman exec -it flyer-crawler-dev pm2 logs --lines 100
```
**View log files directly**:
```bash
# API logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-out.log
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-error.log
# Worker logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/worker-out.log
# Vite logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/vite-out.log
```
### Redis Log Access
**View Redis logs**:
```bash
# View Redis log file
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
# View from dev container (shared volume)
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/redis/redis-server.log
```
---
## Background Job Issues
### Jobs Not Processing
@@ -433,10 +530,16 @@ console.log('Expired:', decoded.exp < Date.now() / 1000);
**Debug**:
```bash
# Check if worker is running
# Check if worker is running (dev container)
podman exec -it flyer-crawler-dev pm2 status
# Check worker logs (dev container)
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-worker-dev
# Check if worker is running (production)
pm2 list
# Check worker logs
# Check worker logs (production)
pm2 logs flyer-crawler-worker
# Check Redis connection
@@ -494,6 +597,77 @@ pm2 logs flyer-crawler-worker --lines 100
---
## SSL Certificate Issues
### Images Not Loading (ERR_CERT_AUTHORITY_INVALID)
**Symptom**: Flyer images fail to load with `ERR_CERT_AUTHORITY_INVALID` in browser console
**Cause**: Mixed hostname origins - user accesses via `localhost` but images use `127.0.0.1` (or vice versa)
**Debug**:
```bash
# Check which hostname images are using
podman exec flyer-crawler-dev psql -U postgres -d flyer_crawler_dev \
-c "SELECT image_url FROM flyers LIMIT 1;"
# Verify certificate includes both hostnames
podman exec flyer-crawler-dev openssl x509 -in /app/certs/localhost.crt -text -noout | grep -A1 "Subject Alternative Name"
# Check NGINX accepts both hostnames
podman exec flyer-crawler-dev grep "server_name" /etc/nginx/sites-available/default
```
**Solution**: The dev container is configured to handle both hostnames:
1. Certificate includes SANs for `localhost`, `127.0.0.1`, and `::1`
2. NGINX `server_name` directive includes both `localhost` and `127.0.0.1`
If you still see errors:
```bash
# Rebuild container to regenerate certificate
podman-compose down
podman-compose build --no-cache flyer-crawler-dev
podman-compose up -d
```
See [FLYER-URL-CONFIGURATION.md](../FLYER-URL-CONFIGURATION.md#ssl-certificate-configuration-dev-container) for full details.
### Self-Signed Certificate Not Trusted
**Symptom**: Browser shows security warning for `https://localhost`
**Temporary Workaround**: Accept the warning by clicking "Advanced" > "Proceed to localhost"
**Permanent Fix (Recommended)**: Install the mkcert CA certificate to eliminate all SSL warnings.
The CA certificate is located at `certs/mkcert-ca.crt` in the project root. See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
After installation:
- Your browser will trust all mkcert certificates without warnings
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
### NGINX SSL Configuration Test
**Debug**:
```bash
# Test NGINX configuration
podman exec flyer-crawler-dev nginx -t
# Check if NGINX is listening on 443
podman exec flyer-crawler-dev netstat -tlnp | grep 443
# Verify certificate files exist
podman exec flyer-crawler-dev ls -la /app/certs/
```
---
## Frontend Issues
### Hot Reload Not Working
@@ -661,8 +835,10 @@ podman exec flyer-crawler-redis redis-cli info stats
## See Also
- [DEV-CONTAINER.md](DEV-CONTAINER.md) - Dev container guide (PM2, Logstash)
- [TESTING.md](TESTING.md) - Testing strategies
- [CODE-PATTERNS.md](CODE-PATTERNS.md) - Common patterns
- [MONITORING.md](../operations/MONITORING.md) - Production monitoring
- [LOGSTASH-QUICK-REF.md](../operations/LOGSTASH-QUICK-REF.md) - Log aggregation
- [Bugsink Setup](../tools/BUGSINK-SETUP.md) - Error tracking
- [DevOps Guide](../subagents/DEVOPS-GUIDE.md) - Container debugging

View File

@@ -0,0 +1,377 @@
# Dev Container Guide
Comprehensive documentation for the Flyer Crawler development container.
**Last Updated**: 2026-01-22
---
## Table of Contents
1. [Overview](#overview)
2. [Architecture](#architecture)
3. [PM2 Process Management](#pm2-process-management)
4. [Log Aggregation](#log-aggregation)
5. [Quick Reference](#quick-reference)
6. [Troubleshooting](#troubleshooting)
---
## Overview
The dev container provides a production-like environment for local development. Key features:
- **PM2 Process Management** - Matches production architecture (ADR-014)
- **Logstash Integration** - All logs forwarded to Bugsink (ADR-050)
- **HTTPS by Default** - Self-signed certificates via mkcert
- **Hot Reloading** - tsx watch mode for API and worker processes
### Container Services
| Service | Container Name | Purpose |
| ----------- | ------------------------ | ------------------------------ |
| Application | `flyer-crawler-dev` | Node.js app, Bugsink, Logstash |
| PostgreSQL | `flyer-crawler-postgres` | Primary database with PostGIS |
| Redis | `flyer-crawler-redis` | Cache and job queue backing |
### Access Points
| Service | URL | Notes |
| ----------- | ------------------------ | ------------------------- |
| Frontend | `https://localhost` | NGINX proxies Vite (5173) |
| Backend API | `http://localhost:3001` | Express server |
| Bugsink | `https://localhost:8443` | Error tracking UI |
| PostgreSQL | `localhost:5432` | Direct database access |
| Redis | `localhost:6379` | Direct Redis access |
---
## Architecture
### Production vs Development Parity
The dev container was updated to match production architecture (ADR-014):
| Component | Production | Dev Container (OLD) | Dev Container (NEW) |
| ------------ | ---------------------- | ---------------------- | ----------------------- |
| API Server | PM2 cluster mode | `npm run dev` (inline) | PM2 fork + tsx watch |
| Worker | PM2 process | Inline with API | PM2 process + tsx watch |
| Frontend | Static files via NGINX | Vite standalone | PM2 + Vite dev server |
| Logs | PM2 logs -> Logstash | Console only | PM2 logs -> Logstash |
| Process Mgmt | PM2 | None | PM2 |
### Container Startup Flow
When the container starts (`scripts/dev-entrypoint.sh`):
1. **NGINX** starts (HTTPS proxy for Vite and Bugsink)
2. **Bugsink** starts (error tracking on port 8000)
3. **Logstash** starts (log aggregation)
4. **PM2** starts with `ecosystem.dev.config.cjs`:
- `flyer-crawler-api-dev` - API server
- `flyer-crawler-worker-dev` - Background worker
- `flyer-crawler-vite-dev` - Vite dev server
5. Container tails PM2 logs to stay alive
### Key Configuration Files
| File | Purpose |
| ------------------------------ | ---------------------------------- |
| `ecosystem.dev.config.cjs` | PM2 development configuration |
| `ecosystem.config.cjs` | PM2 production configuration |
| `scripts/dev-entrypoint.sh` | Container startup script |
| `docker/logstash/bugsink.conf` | Logstash pipeline configuration |
| `docker/nginx/dev.conf` | NGINX development configuration |
| `compose.dev.yml` | Docker Compose service definitions |
| `Dockerfile.dev` | Container image definition |
---
## PM2 Process Management
### Process Overview
PM2 manages three processes in the dev container:
```
+--------------------+ +------------------------+ +--------------------+
| flyer-crawler- | | flyer-crawler- | | flyer-crawler- |
| api-dev | | worker-dev | | vite-dev |
+--------------------+ +------------------------+ +--------------------+
| tsx watch | | tsx watch | | vite --host |
| server.ts | | src/services/worker.ts | | |
| Port: 3001 | | No port | | Port: 5173 |
+--------------------+ +------------------------+ +--------------------+
| | |
v v v
+------------------------------------------------------------------------+
| /var/log/pm2/*.log |
| (Logstash picks up for Bugsink) |
+------------------------------------------------------------------------+
```
### PM2 Commands
All commands should be run inside the container:
```bash
# View process status
podman exec -it flyer-crawler-dev pm2 status
# View all logs (tail -f style)
podman exec -it flyer-crawler-dev pm2 logs
# View specific process logs
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
# Restart all processes
podman exec -it flyer-crawler-dev pm2 restart all
# Restart specific process
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
# Stop all processes
podman exec -it flyer-crawler-dev pm2 delete all
# Show detailed process info
podman exec -it flyer-crawler-dev pm2 show flyer-crawler-api-dev
# Monitor processes in real-time
podman exec -it flyer-crawler-dev pm2 monit
```
### PM2 Log Locations
| Process | stdout Log | stderr Log |
| -------------------------- | ----------------------------- | ------------------------------- |
| `flyer-crawler-api-dev` | `/var/log/pm2/api-out.log` | `/var/log/pm2/api-error.log` |
| `flyer-crawler-worker-dev` | `/var/log/pm2/worker-out.log` | `/var/log/pm2/worker-error.log` |
| `flyer-crawler-vite-dev` | `/var/log/pm2/vite-out.log` | `/var/log/pm2/vite-error.log` |
### NPM Scripts for PM2
The following npm scripts are available for PM2 management:
```bash
# Start PM2 with dev config (inside container)
npm run dev:pm2
# Restart all PM2 processes
npm run dev:pm2:restart
# Stop all PM2 processes
npm run dev:pm2:stop
# View PM2 status
npm run dev:pm2:status
# View PM2 logs
npm run dev:pm2:logs
```
---
## Log Aggregation
### Log Flow Architecture (ADR-050)
All application logs flow through Logstash to Bugsink:
```
+------------------+ +------------------+ +------------------+
| PM2 Logs | | PostgreSQL | | Redis Logs |
| /var/log/pm2/ | | /var/log/ | | /var/log/redis/ |
+--------+---------+ | postgresql/ | +--------+---------+
| +--------+---------+ |
| | |
v v v
+------------------------------------------------------------------------+
| LOGSTASH |
| /etc/logstash/conf.d/bugsink.conf |
+------------------------------------------------------------------------+
| | |
| +---------+---------+ |
| | | |
v v v v
+------------------+ +------------------+ +------------------+
| Errors -> | | Operational -> | | NGINX Logs -> |
| Bugsink API | | /var/log/ | | /var/log/ |
| (Project 1) | | logstash/*.log | | logstash/*.log |
+------------------+ +------------------+ +------------------+
```
### Log Sources
| Source | Log Path | Format | Errors To Bugsink |
| ------------ | --------------------------------- | ---------- | ----------------- |
| API Server | `/var/log/pm2/api-*.log` | Pino JSON | Yes |
| Worker | `/var/log/pm2/worker-*.log` | Pino JSON | Yes |
| Vite | `/var/log/pm2/vite-*.log` | Plain text | Yes (if error) |
| PostgreSQL | `/var/log/postgresql/*.log` | PostgreSQL | Yes (ERROR/FATAL) |
| Redis | `/var/log/redis/redis-server.log` | Redis | Yes (warnings) |
| NGINX Access | `/var/log/nginx/access.log` | Combined | Yes (5xx only) |
| NGINX Error | `/var/log/nginx/error.log` | NGINX | Yes |
### Viewing Logs
```bash
# View Logstash processed logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-api-$(date +%Y-%m-%d).log
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
# View raw Redis logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
# Check Logstash status
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
```
### Bugsink Access
- **URL**: `https://localhost:8443`
- **Login**: `admin@localhost` / `admin`
- **Projects**:
- Project 1: Backend API (errors from Pino, PostgreSQL, Redis)
- Project 2: Frontend (errors from Sentry SDK in browser)
---
## Quick Reference
### Starting the Dev Container
```bash
# Start all services
podman-compose -f compose.dev.yml up -d
# View container logs
podman-compose -f compose.dev.yml logs -f
# Stop all services
podman-compose -f compose.dev.yml down
```
### Common Tasks
| Task | Command |
| -------------------- | ------------------------------------------------------------------------------ |
| Run tests | `podman exec -it flyer-crawler-dev npm test` |
| Run type check | `podman exec -it flyer-crawler-dev npm run type-check` |
| View PM2 status | `podman exec -it flyer-crawler-dev pm2 status` |
| View PM2 logs | `podman exec -it flyer-crawler-dev pm2 logs` |
| Restart API | `podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev` |
| Access PostgreSQL | `podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev` |
| Access Redis CLI | `podman exec -it flyer-crawler-redis redis-cli` |
| Shell into container | `podman exec -it flyer-crawler-dev bash` |
### Environment Variables
Key environment variables are set in `compose.dev.yml`:
| Variable | Value | Purpose |
| ----------------- | ----------------------------- | -------------------- |
| `NODE_ENV` | `development` | Environment mode |
| `DB_HOST` | `postgres` | PostgreSQL hostname |
| `REDIS_URL` | `redis://redis:6379` | Redis connection URL |
| `FRONTEND_URL` | `https://localhost` | CORS origin |
| `SENTRY_DSN` | `http://...@127.0.0.1:8000/1` | Backend Bugsink DSN |
| `VITE_SENTRY_DSN` | `http://...@127.0.0.1:8000/2` | Frontend Bugsink DSN |
---
## Troubleshooting
### PM2 Process Not Starting
**Symptom**: `pm2 status` shows process as "errored" or "stopped"
**Debug**:
```bash
# Check process logs
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev --lines 50
# Check if port is in use
podman exec -it flyer-crawler-dev netstat -tlnp | grep 3001
# Try manual start to see errors
podman exec -it flyer-crawler-dev tsx server.ts
```
**Common Causes**:
- Port already in use
- Missing environment variables
- Syntax error in code
### Logs Not Appearing in Bugsink
**Symptom**: Errors in application but nothing in Bugsink UI
**Debug**:
```bash
# Check Logstash is running
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
# Check Logstash logs for errors
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log
# Verify PM2 logs exist
podman exec flyer-crawler-dev ls -la /var/log/pm2/
```
**Common Causes**:
- Logstash not started
- Log file permissions
- Bugsink not running
### Redis Logs Not Captured
**Symptom**: Redis warnings not appearing in Bugsink
**Debug**:
```bash
# Verify Redis logs exist
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
# Verify shared volume is mounted
podman exec flyer-crawler-dev ls -la /var/log/redis/
```
**Common Causes**:
- `redis_logs` volume not mounted
- Redis not configured to write to file
### Hot Reload Not Working
**Symptom**: Code changes not reflected in running application
**Debug**:
```bash
# Check tsx watch is running
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
# Manually restart process
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
```
**Common Causes**:
- File watcher limit reached
- Volume mount issues on Windows
---
## Related Documentation
- [QUICKSTART.md](../getting-started/QUICKSTART.md) - Getting started guide
- [DEBUGGING.md](DEBUGGING.md) - Debugging strategies
- [LOGSTASH-QUICK-REF.md](../operations/LOGSTASH-QUICK-REF.md) - Logstash quick reference
- [DEV-CONTAINER-BUGSINK.md](../DEV-CONTAINER-BUGSINK.md) - Bugsink setup in dev container
- [ADR-014](../adr/0014-linux-only-platform.md) - Linux-only platform decision
- [ADR-050](../adr/0050-postgresql-function-observability.md) - PostgreSQL function observability

View File

@@ -119,6 +119,31 @@ npm run dev
- **Frontend**: http://localhost:5173
- **Backend API**: http://localhost:3001
### Dev Container with HTTPS (Full Stack)
When using the full dev container stack with NGINX (via `compose.dev.yml`), access the application over HTTPS:
- **Frontend**: https://localhost or https://127.0.0.1
- **Backend API**: http://localhost:3001
**SSL Certificate Notes:**
- The dev container uses self-signed certificates generated by mkcert
- Both `localhost` and `127.0.0.1` are valid hostnames (certificate includes both as SANs)
- If images fail to load with SSL errors, see [FLYER-URL-CONFIGURATION.md](../FLYER-URL-CONFIGURATION.md#ssl-certificate-configuration-dev-container)
**Eliminate SSL Warnings (Recommended):**
To avoid browser security warnings for self-signed certificates, install the mkcert CA certificate on your system. The CA certificate is located at `certs/mkcert-ca.crt` in the project root.
See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
After installation:
- Your browser will trust all mkcert certificates without warnings
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
### Managing the Container
| Action | Command |

View File

@@ -86,6 +86,44 @@ npm run dev
- **Backend API**: http://localhost:3001
- **Health Check**: http://localhost:3001/health
### Dev Container (HTTPS)
When using the full dev container with NGINX, access via HTTPS:
- **Frontend**: https://localhost or https://127.0.0.1
- **Backend API**: http://localhost:3001
- **Bugsink**: `https://localhost:8443` (error tracking)
**Note:** The dev container accepts both `localhost` and `127.0.0.1` for HTTPS connections. The self-signed certificate is valid for both hostnames.
**SSL Certificate Warnings:** To eliminate browser security warnings for self-signed certificates, install the mkcert CA certificate. See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions. This is optional but recommended for a better development experience.
### Dev Container Architecture
The dev container uses PM2 for process management, matching production (ADR-014):
| Process | Description | Port |
| -------------------------- | ------------------------ | ---- |
| `flyer-crawler-api-dev` | API server (tsx watch) | 3001 |
| `flyer-crawler-worker-dev` | Background job worker | - |
| `flyer-crawler-vite-dev` | Vite frontend dev server | 5173 |
**PM2 Commands** (run inside container):
```bash
# View process status
podman exec -it flyer-crawler-dev pm2 status
# View logs (all processes)
podman exec -it flyer-crawler-dev pm2 logs
# Restart all processes
podman exec -it flyer-crawler-dev pm2 restart all
# Restart specific process
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
```
## Verify Installation
```bash

View File

@@ -11,6 +11,22 @@ This guide covers deploying Flyer Crawler to a production server.
- NGINX (reverse proxy)
- PM2 (process manager)
## Dev Container Parity (ADR-014)
The development container now matches production architecture:
| Component | Production | Dev Container |
| ------------ | ---------------- | -------------------------- |
| Process Mgmt | PM2 | PM2 |
| API Server | PM2 cluster mode | PM2 fork + tsx watch |
| Worker | PM2 process | PM2 process + tsx watch |
| Logs | PM2 -> Logstash | PM2 -> Logstash -> Bugsink |
| HTTPS | Let's Encrypt | Self-signed (mkcert) |
This ensures issues caught in dev will behave the same in production.
**Dev Container Guide**: See [docs/development/DEV-CONTAINER.md](../development/DEV-CONTAINER.md)
---
## Server Setup

View File

@@ -31,6 +31,8 @@ Aggregates logs from PostgreSQL, PM2, Redis, NGINX; forwards errors to Bugsink.
## Commands
### Production (Bare Metal)
```bash
# Status and control
systemctl status logstash
@@ -55,6 +57,36 @@ tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
du -sh /var/log/logstash/
```
### Dev Container
```bash
# View Logstash logs (inside container)
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log
# Check PM2 API logs are being processed (ADR-014)
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-api-$(date +%Y-%m-%d).log
# Check PM2 Worker logs are being processed
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-worker-$(date +%Y-%m-%d).log
# Check Redis logs are being processed
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
# View raw PM2 logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-out.log
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/worker-out.log
# View raw Redis logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
# Check Logstash stats
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
# Verify shared volume mounts
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev ls -la /var/log/pm2/
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev ls -la /var/log/redis/
```
## Troubleshooting
| Issue | Check | Solution |
@@ -64,12 +96,15 @@ du -sh /var/log/logstash/
| Grok pattern failures | Stats endpoint | `curl localhost:9600/_node/stats/pipelines?pretty \| jq '.pipelines.main.plugins.filters'` |
| Wrong Bugsink project | Env detection | Check tags in logs match expected environment |
| Permission denied | Logstash groups | `groups logstash` should include `postgres`, `adm` |
| PM2 not captured | File paths | `ls /home/gitea-runner/.pm2/logs/flyer-crawler-worker-*.log` |
| PM2 not captured | File paths | Dev: `ls /var/log/pm2/`; Prod: `ls /home/gitea-runner/.pm2/logs/` |
| PM2 logs empty | PM2 running | Dev: `podman exec flyer-crawler-dev pm2 status`; Prod: `pm2 status` |
| NGINX logs missing | Output directory | `ls -lh /var/log/logstash/nginx-access-*.log` |
| Redis logs missing | Shared volume | Dev: Check `redis_logs` volume mounted; Prod: Check `/var/log/redis/redis-server.log` exists |
| High disk usage | Log rotation | Verify `/etc/logrotate.d/logstash` configured |
## Related Documentation
- **Dev Container Guide**: [DEV-CONTAINER.md](../development/DEV-CONTAINER.md) - PM2 and log aggregation in dev
- **Full setup**: [BARE-METAL-SETUP.md](BARE-METAL-SETUP.md) - PostgreSQL Function Observability section
- **Architecture**: [adr/0050-postgresql-function-observability.md](adr/0050-postgresql-function-observability.md)
- **Architecture**: [adr/0050-postgresql-function-observability.md](../adr/0050-postgresql-function-observability.md)
- **Troubleshooting details**: [LOGSTASH-TROUBLESHOOTING.md](LOGSTASH-TROUBLESHOOTING.md)

172
ecosystem.dev.config.cjs Normal file
View File

@@ -0,0 +1,172 @@
// ecosystem.dev.config.cjs
// ============================================================================
// DEVELOPMENT PM2 CONFIGURATION
// ============================================================================
// This file mirrors the production ecosystem.config.cjs but with settings
// appropriate for the development environment inside the dev container.
//
// Key differences from production:
// - Single instance for API (not cluster mode) to allow debugger attachment
// - tsx watch mode for hot reloading
// - Development-specific environment variables
// - Logs written to /var/log/pm2 for Logstash integration (ADR-050)
//
// Usage:
// pm2 start ecosystem.dev.config.cjs
// pm2 logs # View all logs
// pm2 logs flyer-crawler-api-dev # View API logs only
// pm2 restart all # Restart all processes
// pm2 delete all # Stop all processes
//
// Related:
// - ecosystem.config.cjs (production configuration)
// - docs/adr/0014-linux-only-platform.md
// - docs/adr/0050-postgresql-function-observability.md
// ============================================================================
// --- Environment Variable Validation ---
// In dev, we warn but don't fail - variables come from compose.dev.yml
const requiredVars = ['DB_HOST', 'JWT_SECRET'];
const missingVars = requiredVars.filter((key) => !process.env[key]);
if (missingVars.length > 0) {
console.warn(
'\n[ecosystem.dev.config.cjs] WARNING: The following environment variables are MISSING:',
);
missingVars.forEach((key) => console.warn(` - ${key}`));
console.warn(
'[ecosystem.dev.config.cjs] These should be set in compose.dev.yml or .env.local\n',
);
} else {
console.log('[ecosystem.dev.config.cjs] Required environment variables are present.');
}
// --- Shared Environment Variables ---
// These come from compose.dev.yml environment section
const sharedEnv = {
NODE_ENV: 'development',
DB_HOST: process.env.DB_HOST || 'postgres',
DB_PORT: process.env.DB_PORT || '5432',
DB_USER: process.env.DB_USER || 'postgres',
DB_PASSWORD: process.env.DB_PASSWORD || 'postgres',
DB_NAME: process.env.DB_NAME || 'flyer_crawler_dev',
REDIS_URL: process.env.REDIS_URL || 'redis://redis:6379',
REDIS_HOST: process.env.REDIS_HOST || 'redis',
REDIS_PORT: process.env.REDIS_PORT || '6379',
FRONTEND_URL: process.env.FRONTEND_URL || 'https://localhost',
JWT_SECRET: process.env.JWT_SECRET || 'dev-jwt-secret-change-in-production',
WORKER_LOCK_DURATION: process.env.WORKER_LOCK_DURATION || '120000',
// Sentry/Bugsink (ADR-015)
SENTRY_DSN: process.env.SENTRY_DSN,
SENTRY_ENVIRONMENT: process.env.SENTRY_ENVIRONMENT || 'development',
SENTRY_ENABLED: process.env.SENTRY_ENABLED || 'true',
SENTRY_DEBUG: process.env.SENTRY_DEBUG || 'true',
// Optional API keys (may not be set in dev)
GEMINI_API_KEY: process.env.GEMINI_API_KEY,
GOOGLE_MAPS_API_KEY: process.env.GOOGLE_MAPS_API_KEY,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
};
module.exports = {
apps: [
// =========================================================================
// API SERVER (Development)
// =========================================================================
// Single instance with tsx watch for hot reloading.
// Unlike production, we don't use cluster mode to allow:
// - Debugger attachment
// - Simpler log output
// - Hot reloading via tsx watch
{
name: 'flyer-crawler-api-dev',
script: './node_modules/.bin/tsx',
args: 'watch server.ts',
cwd: '/app',
watch: false, // tsx watch handles this
instances: 1, // Single instance for debugging
exec_mode: 'fork', // Fork mode (not cluster) for debugging
max_memory_restart: '1G', // More generous in dev
kill_timeout: 5000,
// PM2 log configuration (ADR-050 - Logstash integration)
output: '/var/log/pm2/api-out.log',
error: '/var/log/pm2/api-error.log',
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
merge_logs: true,
// Restart configuration
max_restarts: 10,
exp_backoff_restart_delay: 500,
min_uptime: '5s',
// Environment
env: {
...sharedEnv,
PORT: '3001',
},
},
// =========================================================================
// BACKGROUND WORKER (Development)
// =========================================================================
// Processes background jobs (flyer processing, cleanup, etc.)
// Runs as a separate process like in production.
{
name: 'flyer-crawler-worker-dev',
script: './node_modules/.bin/tsx',
args: 'watch src/services/worker.ts',
cwd: '/app',
watch: false, // tsx watch handles this
instances: 1,
exec_mode: 'fork',
max_memory_restart: '1G',
kill_timeout: 10000, // Workers need more time to finish jobs
// PM2 log configuration (ADR-050)
output: '/var/log/pm2/worker-out.log',
error: '/var/log/pm2/worker-error.log',
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
merge_logs: true,
// Restart configuration
max_restarts: 10,
exp_backoff_restart_delay: 500,
min_uptime: '5s',
// Environment
env: {
...sharedEnv,
},
},
// =========================================================================
// VITE FRONTEND DEV SERVER (Development)
// =========================================================================
// Vite dev server for frontend hot module replacement.
// Listens on 5173 and is proxied by nginx to 443.
{
name: 'flyer-crawler-vite-dev',
script: './node_modules/.bin/vite',
args: '--host',
cwd: '/app',
watch: false, // Vite handles its own watching
instances: 1,
exec_mode: 'fork',
max_memory_restart: '2G', // Vite can use significant memory
kill_timeout: 5000,
// PM2 log configuration (ADR-050)
output: '/var/log/pm2/vite-out.log',
error: '/var/log/pm2/vite-error.log',
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
merge_logs: true,
// Restart configuration
max_restarts: 10,
exp_backoff_restart_delay: 500,
min_uptime: '5s',
// Environment
env: {
NODE_ENV: 'development',
// Vite-specific env vars (VITE_ prefix)
VITE_SENTRY_DSN: process.env.VITE_SENTRY_DSN,
VITE_SENTRY_ENVIRONMENT: process.env.VITE_SENTRY_ENVIRONMENT || 'development',
VITE_SENTRY_ENABLED: process.env.VITE_SENTRY_ENABLED || 'true',
VITE_SENTRY_DEBUG: process.env.VITE_SENTRY_DEBUG || 'true',
},
},
],
};

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "flyer-crawler",
"version": "0.12.7",
"version": "0.12.8",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "flyer-crawler",
"version": "0.12.7",
"version": "0.12.8",
"dependencies": {
"@bull-board/api": "^6.14.2",
"@bull-board/express": "^6.14.2",

View File

@@ -1,11 +1,16 @@
{
"name": "flyer-crawler",
"private": true,
"version": "0.12.7",
"version": "0.12.8",
"type": "module",
"scripts": {
"dev": "concurrently \"npm:start:dev\" \"vite\"",
"dev:container": "concurrently \"npm:start:dev\" \"vite --host\"",
"dev:pm2": "pm2 start ecosystem.dev.config.cjs && pm2 logs",
"dev:pm2:restart": "pm2 restart ecosystem.dev.config.cjs",
"dev:pm2:stop": "pm2 delete all",
"dev:pm2:status": "pm2 status",
"dev:pm2:logs": "pm2 logs",
"start": "npm run start:prod",
"build": "vite build",
"preview": "vite preview",

View File

@@ -3,22 +3,28 @@
# ============================================================================
# Development Container Entrypoint
# ============================================================================
# This script starts the development server automatically when the container
# starts, both with VS Code Dev Containers and with plain podman-compose.
# This script starts all development services when the container starts,
# both with VS Code Dev Containers and with plain podman-compose.
#
# Services started:
# - Nginx (proxies Vite 5173 → 3000)
# - Nginx (HTTPS proxy: Vite 5173 -> 443, Bugsink 8000 -> 8443, API 3001)
# - Bugsink (error tracking) on port 8000
# - Logstash (log aggregation)
# - Node.js dev server (API + Frontend) on ports 3001 and 5173
# - PM2 (process manager) running:
# - flyer-crawler-api-dev: API server (port 3001)
# - flyer-crawler-worker-dev: Background job worker
# - flyer-crawler-vite-dev: Vite frontend dev server (port 5173)
#
# ADR-014: This architecture matches production (PM2 managing processes)
# ADR-050: Logs are written to /var/log/pm2 for Logstash integration
# ============================================================================
set -e
echo "🚀 Starting Flyer Crawler Dev Container..."
echo "Starting Flyer Crawler Dev Container..."
# Configure Bugsink HTTPS (ADR-015)
echo "🔒 Configuring Bugsink HTTPS..."
echo "Configuring Bugsink HTTPS..."
mkdir -p /etc/bugsink/ssl
if [ ! -f "/etc/bugsink/ssl/localhost+2.pem" ]; then
cd /etc/bugsink/ssl && mkcert localhost 127.0.0.1 ::1 > /dev/null 2>&1
@@ -53,52 +59,92 @@ NGINX_EOF
ln -sf /etc/nginx/sites-available/bugsink /etc/nginx/sites-enabled/bugsink
# Copy the dev nginx config from mounted volume to nginx sites-available
echo "📋 Copying nginx dev config..."
echo "Copying nginx dev config..."
cp /app/docker/nginx/dev.conf /etc/nginx/sites-available/default
# Ensure PM2 log directory exists with correct permissions
echo "Setting up PM2 log directory..."
mkdir -p /var/log/pm2
chmod 755 /var/log/pm2
# Start nginx in background (if installed)
if command -v nginx &> /dev/null; then
echo "🌐 Starting nginx (HTTPS: Vite 5173 443, Bugsink 8000 8443, API 3001 /api/)..."
echo "Starting nginx (HTTPS: Vite 5173 -> 443, Bugsink 8000 -> 8443, API 3001 -> /api/)..."
nginx &
fi
# Start Bugsink in background
echo "📊 Starting Bugsink error tracking..."
echo "Starting Bugsink error tracking..."
/usr/local/bin/start-bugsink.sh > /var/log/bugsink/server.log 2>&1 &
# Wait for Bugsink to initialize, then run snappea migrations
echo "Waiting for Bugsink to initialize..."
echo "Waiting for Bugsink to initialize..."
sleep 5
echo "🔧 Running Bugsink snappea database migrations..."
echo "Running Bugsink snappea database migrations..."
cd /opt/bugsink/conf && \
export DATABASE_URL="postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink" && \
export SECRET_KEY="dev-bugsink-secret-key-minimum-50-characters-for-security" && \
/opt/bugsink/bin/bugsink-manage migrate --database=snappea > /dev/null 2>&1
# Start Snappea task worker
echo "🔄 Starting Snappea task worker..."
echo "Starting Snappea task worker..."
cd /opt/bugsink/conf && \
export DATABASE_URL="postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink" && \
export SECRET_KEY="dev-bugsink-secret-key-minimum-50-characters-for-security" && \
/opt/bugsink/bin/bugsink-manage runsnappea > /var/log/bugsink/snappea.log 2>&1 &
# Start Logstash in background
echo "📝 Starting Logstash..."
echo "Starting Logstash..."
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/bugsink.conf > /var/log/logstash/logstash.log 2>&1 &
# Wait a few seconds for services to initialize
# Wait for services to initialize
sleep 3
# Change to app directory
cd /app
# Start development server
echo "💻 Starting development server..."
echo " - Frontend: https://localhost (nginx HTTPS → Vite on 5173)"
echo " - Backend API: http://localhost:3001"
echo " - Bugsink: https://localhost:8443 (nginx HTTPS → Bugsink on 8000)"
echo " - Note: Accept the self-signed certificate warnings in your browser"
# ============================================================================
# Start PM2 (ADR-014: Production-like process management)
# ============================================================================
# PM2 manages all Node.js processes:
# - API server (tsx watch server.ts)
# - Background worker (tsx watch src/services/worker.ts)
# - Vite dev server (vite --host)
#
# Logs are written to /var/log/pm2 for Logstash integration (ADR-050)
# ============================================================================
echo "Starting PM2 with development configuration..."
echo " - API Server: http://localhost:3001 (proxied via nginx)"
echo " - Frontend: https://localhost (nginx HTTPS -> Vite on 5173)"
echo " - Bugsink: https://localhost:8443 (nginx HTTPS -> Bugsink on 8000)"
echo " - PM2 Logs: /var/log/pm2/*.log"
echo ""
echo "Note: Accept the self-signed certificate warnings in your browser"
echo ""
# Run npm dev server (this will block and keep container alive)
exec npm run dev:container
# Delete any existing PM2 processes (clean start)
pm2 delete all 2>/dev/null || true
# Start PM2 with the dev ecosystem config
pm2 start /app/ecosystem.dev.config.cjs
# Show PM2 status
echo ""
echo "--- PM2 Process Status ---"
pm2 status
echo "-------------------------"
echo ""
echo "PM2 Commands:"
echo " pm2 logs # View all logs (tail -f style)"
echo " pm2 logs api # View API logs only"
echo " pm2 restart all # Restart all processes"
echo " pm2 status # Show process status"
echo ""
# Keep the container running by tailing PM2 logs
# This ensures:
# 1. Container stays alive
# 2. Logs are visible in docker logs / podman logs
# 3. PM2 processes continue running
exec pm2 logs --raw

View File

@@ -1,21 +1,31 @@
-- Update flyer URLs from example.com to environment-specific URLs
--
-- This script should be run after determining the correct base URL for the environment:
-- - Dev container: http://127.0.0.1
-- - Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
-- - Test environment: https://flyer-crawler-test.projectium.com
-- - Production: https://flyer-crawler.projectium.com
-- For dev container (run in dev database):
-- Uses 'localhost' instead of '127.0.0.1' to match how users access the site.
-- This avoids ERR_CERT_AUTHORITY_INVALID errors when images are loaded from a
-- different origin than the page.
UPDATE flyers
SET
image_url = REPLACE(image_url, 'example.com', '127.0.0.1'),
image_url = REPLACE(image_url, 'https://', 'http://'),
icon_url = REPLACE(icon_url, 'example.com', '127.0.0.1'),
icon_url = REPLACE(icon_url, 'https://', 'http://')
image_url = REPLACE(image_url, 'example.com', 'localhost'),
icon_url = REPLACE(icon_url, 'example.com', 'localhost')
WHERE
image_url LIKE '%example.com%'
OR icon_url LIKE '%example.com%';
-- Also fix any existing 127.0.0.1 URLs to use localhost:
UPDATE flyers
SET
image_url = REPLACE(image_url, '127.0.0.1', 'localhost'),
icon_url = REPLACE(icon_url, '127.0.0.1', 'localhost')
WHERE
image_url LIKE '%127.0.0.1%'
OR icon_url LIKE '%127.0.0.1%';
-- For test environment (run in test database):
-- UPDATE flyers
-- SET
@@ -37,7 +47,7 @@ WHERE
-- Verify the changes:
SELECT flyer_id, image_url, icon_url
FROM flyers
WHERE image_url LIKE '%127.0.0.1%'
OR icon_url LIKE '%127.0.0.1%'
WHERE image_url LIKE '%localhost%'
OR icon_url LIKE '%localhost%'
OR image_url LIKE '%flyer-crawler%'
OR icon_url LIKE '%flyer-crawler%';

View File

@@ -12,8 +12,8 @@ import path from 'node:path';
import bcrypt from 'bcrypt';
import { logger } from '../services/logger.server';
// Determine base URL based on environment
// Dev container: https://127.0.0.1
// Determine base URL for flyer images based on environment
// Dev container: https://127.0.0.1 (NGINX now accepts both localhost and 127.0.0.1)
// Test: https://flyer-crawler-test.projectium.com
// Production: https://flyer-crawler.projectium.com
const BASE_URL =

View File

@@ -159,7 +159,7 @@ export const MainLayout: React.FC<MainLayoutProps> = ({
<>
<PriceHistoryChart />
<Leaderboard />
{userProfile && (
{userProfile?.role === 'admin' && (
<ActivityLog userProfile={userProfile} onLogClick={handleActivityLogClick} />
)}
</>

View File

@@ -21,6 +21,8 @@ const isProduction = process.env.NODE_ENV === 'production';
const isTest = process.env.NODE_ENV === 'test';
const isStaging = process.env.NODE_ENV === 'staging';
const isDevelopment = !isProduction && !isTest && !isStaging;
// PM2_HOME is set when running under PM2 process manager
const isRunningUnderPM2 = !!process.env.PM2_HOME;
// Determine log directory based on environment
// In production/test, use the application directory's logs folder
@@ -73,8 +75,9 @@ const redactConfig = {
const createLogger = (): pino.Logger => {
const logDir = ensureLogDirectory();
// Development: Use pino-pretty for human-readable output
if (isDevelopment) {
// Development WITHOUT PM2: Use pino-pretty for human-readable console output
// This is for local development outside of containers (rare case now)
if (isDevelopment && !isRunningUnderPM2) {
return pino({
level: 'debug',
transport: {
@@ -89,6 +92,15 @@ const createLogger = (): pino.Logger => {
});
}
// Development WITH PM2 (ADR-014): Output JSON for Logstash integration
// PM2 captures stdout to /var/log/pm2/*.log files which Logstash parses
if (isDevelopment && isRunningUnderPM2) {
return pino({
level: 'debug',
redact: redactConfig,
});
}
// Production/Test: Write to both stdout and file
if (logDir) {
const logFilePath = path.join(logDir, 'app.log');

View File

@@ -15,7 +15,7 @@ export const getTestBaseUrl = (): string => {
/**
* Get the flyer base URL for test data based on environment.
* Uses FLYER_BASE_URL if set, otherwise detects environment:
* - Dev container: http://127.0.0.1
* - Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
* - Test: https://flyer-crawler-test.projectium.com
* - Production: https://flyer-crawler.projectium.com
* - Default: https://example.com (for unit tests)
@@ -26,8 +26,10 @@ export const getFlyerBaseUrl = (): string => {
}
// Check if we're in dev container (DB_HOST=postgres is typical indicator)
// Use 'localhost' instead of '127.0.0.1' to match the hostname users access
// This avoids SSL certificate mixed-origin issues in browsers
if (process.env.DB_HOST === 'postgres' || process.env.DB_HOST === '127.0.0.1') {
return 'http://127.0.0.1';
return 'https://localhost';
}
if (process.env.NODE_ENV === 'production') {