Compare commits

...

16 Commits

Author SHA1 Message Date
Gitea Actions
dd067183ed ci: Bump version to 0.12.8 [skip ci] 2026-01-23 04:50:12 +05:00
9f3a070612 have dev continer send more to bugsink
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 19m20s
2026-01-22 15:47:59 -08:00
8a38befb1c fix cert and image display issues 2026-01-22 12:46:28 -08:00
Gitea Actions
7e460a11e4 ci: Bump version to 0.12.7 [skip ci] 2026-01-23 00:24:43 +05:00
eae0dbaa8e bugsink mcp and claude subagents - documentation and test fixes
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 19m11s
2026-01-22 11:23:45 -08:00
fac98f4c54 doc updates and test fixin 2026-01-22 11:23:43 -08:00
9f7b821760 bugsink mcp and claude subagents - documentation and test fixes 2026-01-22 11:23:42 -08:00
cd60178450 bugsink mcp and claude subagents 2026-01-22 11:23:40 -08:00
Gitea Actions
1fcb9fd5c7 ci: Bump version to 0.12.6 [skip ci] 2026-01-22 03:41:25 +05:00
8bd4e081ea e2e fixin, frontend + home page work
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 19m0s
2026-01-21 14:40:19 -08:00
Gitea Actions
6e13570deb ci: Bump version to 0.12.5 [skip ci] 2026-01-22 01:36:01 +05:00
2eba66fb71 make e2e actually e2e - sigh
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 19m9s
2026-01-21 12:34:46 -08:00
Gitea Actions
10cdd78e22 ci: Bump version to 0.12.4 [skip ci] 2026-01-22 00:47:30 +05:00
521943bec0 make e2e actually e2e - sigh
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 18m55s
2026-01-21 11:43:39 -08:00
Gitea Actions
810c0eb61b ci: Bump version to 0.12.3 [skip ci] 2026-01-21 23:08:48 +05:00
3314063e25 migration from react-joyride to driver.js:
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 18m52s
2026-01-21 10:07:38 -08:00
108 changed files with 16767 additions and 2293 deletions

View File

@@ -100,7 +100,29 @@
"Read(//d/gitea/bugsink-mcp/**)",
"Bash(d:/nodejs/npm.cmd install)",
"Bash(node node_modules/vitest/vitest.mjs run:*)",
"Bash(npm run test:e2e:*)"
"Bash(npm run test:e2e:*)",
"Bash(export BUGSINK_URL=http://localhost:8000)",
"Bash(export BUGSINK_TOKEN=a609c2886daa4e1e05f1517074d7779a5fb49056)",
"Bash(timeout 3 d:/nodejs/node.exe:*)",
"Bash(export BUGSINK_URL=https://bugsink.projectium.com)",
"Bash(export BUGSINK_API_TOKEN=77deaa5e2649ab0fbbca51bbd427ec4637d073a0)",
"Bash(export BUGSINK_TOKEN=77deaa5e2649ab0fbbca51bbd427ec4637d073a0)",
"Bash(where:*)",
"mcp__localerrors__test_connection",
"mcp__localerrors__list_projects",
"Bash(\"D:\\\\nodejs\\\\npx.cmd\" -y @modelcontextprotocol/server-postgres --help)",
"Bash(git rm:*)",
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" log -1 --format=\"%H %ci %s\")",
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" config --get remote.origin.url)",
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" fetch --dry-run -v)",
"mcp__localerrors__get_project",
"mcp__localerrors__get_issue",
"mcp__localerrors__get_event"
]
}
},
"enabledMcpjsonServers": [
"localerrors",
"devdb",
"gitea-projectium"
]
}

View File

@@ -35,6 +35,12 @@ NODE_ENV=development
# Frontend URL for CORS and email links
FRONTEND_URL=http://localhost:3000
# Flyer Base URL - used for seed data and flyer image URLs
# Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
# Test: https://flyer-crawler-test.projectium.com
# Production: https://flyer-crawler.projectium.com
FLYER_BASE_URL=https://localhost
# ===================
# Authentication
# ===================

View File

@@ -45,7 +45,7 @@ jobs:
cache-dependency-path: '**/package-lock.json'
- name: Install Dependencies
run: npm ci --legacy-peer-deps
run: npm ci
- name: Bump Minor Version and Push
run: |

View File

@@ -41,7 +41,7 @@ jobs:
# If dependencies are not found in cache, it will run 'npm ci' automatically.
# If they are found, it restores them. This is the standard, reliable way.
- name: Install Dependencies
run: npm ci --legacy-peer-deps # 'ci' is faster and safer for CI/CD than 'install'.
run: npm ci # 'ci' is faster and safer for CI/CD than 'install'.
- name: Bump Version and Push
run: |

28
.mcp.json Normal file
View File

@@ -0,0 +1,28 @@
{
"mcpServers": {
"localerrors": {
"command": "d:\\nodejs\\node.exe",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "http://127.0.0.1:8000",
"BUGSINK_TOKEN": "a609c2886daa4e1e05f1517074d7779a5fb49056"
}
},
"devdb": {
"command": "D:\\nodejs\\npx.cmd",
"args": [
"-y",
"@modelcontextprotocol/server-postgres",
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
]
},
"gitea-projectium": {
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
"args": ["run", "-t", "stdio"],
"env": {
"GITEA_HOST": "https://gitea.projectium.com",
"GITEA_ACCESS_TOKEN": "b111259253aa3cadcb6a37618de03bf388f6235a"
}
}
}
}

892
CLAUDE.md
View File

@@ -1,606 +1,372 @@
# Claude Code Project Instructions
## CRITICAL RULES (READ FIRST)
### Platform: Linux Only (ADR-014)
**ALL tests MUST run in dev container** - Windows results are unreliable.
| Test Result | Container | Windows | Status |
| ----------- | --------- | ------- | ------------------------ |
| Pass | Fail | = | **BROKEN** (must fix) |
| Fail | Pass | = | **PASSING** (acceptable) |
```bash
# Always test in container
podman exec -it flyer-crawler-dev npm test
podman exec -it flyer-crawler-dev npm run type-check
```
### Database Schema Sync
**CRITICAL**: Keep these files synchronized:
- `sql/master_schema_rollup.sql` (test DB, complete reference)
- `sql/initial_schema.sql` (fresh install, identical to rollup)
- `sql/migrations/*.sql` (production ALTER TABLE statements)
Out-of-sync = test failures.
### Communication Style
Ask before assuming. Never assume:
- Steps completed / User knowledge / External services configured / Secrets created
---
## Session Startup Checklist
**IMPORTANT**: At the start of every session, perform these steps:
1. **Check Memory First** - Use `mcp__memory__read_graph` or `mcp__memory__search_nodes` to recall:
- Project-specific configurations and credentials
- Previous work context and decisions
- Infrastructure details (URLs, ports, access patterns)
- Known issues and their solutions
2. **Review Recent Git History** - Check `git log --oneline -10` to understand recent changes
3. **Check Container Status** - Use `mcp__podman__container_list` to see what's running
1. **Memory**: `mcp__memory__read_graph` - Recall project context, credentials, known issues
2. **Git**: `git log --oneline -10` - Recent changes
3. **Containers**: `mcp__podman__container_list` - Running state
4. **PM2 Status**: `podman exec flyer-crawler-dev pm2 status` - Process health (API, Worker, Vite)
---
## Project Instructions
### Things to Remember
Before writing any code:
1. State how you will verify this change works (test, bash command, browser check, etc.)
2. Write the test or verification step first
3. Then implement the code
4. Run verification and iterate until it passes
## Git Bash / MSYS Path Conversion Issue (Windows Host)
**CRITICAL ISSUE**: Git Bash on Windows automatically converts Unix-style paths to Windows paths, which breaks Podman/Docker commands.
### Problem Examples:
```bash
# This FAILS in Git Bash:
podman exec container /usr/local/bin/script.sh
# Git Bash converts to: C:/Program Files/Git/usr/local/bin/script.sh
# This FAILS in Git Bash:
podman exec container bash -c "cat /tmp/file.sql"
# Git Bash converts /tmp to C:/Users/user/AppData/Local/Temp
```
### Solutions:
1. **Use `sh -c` instead of `bash -c`** for single-quoted commands:
```bash
podman exec container sh -c '/usr/local/bin/script.sh'
```
2. **Use double slashes** to escape path conversion:
```bash
podman exec container //usr//local//bin//script.sh
```
3. **Set MSYS_NO_PATHCONV** environment variable:
```bash
MSYS_NO_PATHCONV=1 podman exec container /usr/local/bin/script.sh
```
4. **Use Windows paths with forward slashes** when referencing host files:
```bash
podman cp "d:/path/to/file" container:/tmp/file
```
**ALWAYS use one of these workarounds when running Bash commands on Windows that involve Unix paths inside containers.**
## Communication Style: Ask Before Assuming
**IMPORTANT**: When helping with tasks, **ask clarifying questions before making assumptions**. Do not assume:
- What steps the user has or hasn't completed
- What the user already knows or has configured
- What external services (OAuth providers, APIs, etc.) are already set up
- What secrets or credentials have already been created
Instead, ask the user to confirm the current state before providing instructions or making recommendations. This prevents wasted effort and respects the user's existing work.
## Platform Requirement: Linux Only
**CRITICAL**: This application is designed to run **exclusively on Linux**. See [ADR-014](docs/adr/0014-containerization-and-deployment-strategy.md) for full details.
### Environment Terminology
- **Dev Container** (or just "dev"): The containerized Linux development environment (`flyer-crawler-dev`). This is where all development and testing should occur.
- **Host**: The Windows machine running Podman/Docker and VS Code.
When instructions say "run in dev" or "run in the dev container", they mean executing commands inside the `flyer-crawler-dev` container.
### Test Execution Rules
1. **ALL tests MUST be executed in the dev container** - the Linux container environment
2. **NEVER run tests directly on Windows host** - test results from Windows are unreliable
3. **Always use the dev container for testing** when developing on Windows
4. **TypeScript type-check MUST run in dev container** - `npm run type-check` on Windows does not reliably detect errors
See [docs/TESTING.md](docs/TESTING.md) for comprehensive testing documentation.
### How to Run Tests Correctly
```bash
# If on Windows, first open VS Code and "Reopen in Container"
# Then run tests inside the dev container:
npm test # Run all unit tests
npm run test:unit # Run unit tests only
npm run test:integration # Run integration tests (requires DB/Redis)
```
### Running Tests via Podman (from Windows host)
**Note:** This project has 2900+ unit tests. For AI-assisted development, pipe output to a file for easier processing.
The command to run unit tests in the dev container via podman:
```bash
# Basic (output to terminal)
podman exec -it flyer-crawler-dev npm run test:unit
# Recommended for AI processing: pipe to file
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
```
The command to run integration tests in the dev container via podman:
```bash
podman exec -it flyer-crawler-dev npm run test:integration
```
For running specific test files:
```bash
podman exec -it flyer-crawler-dev npm test -- --run src/hooks/useAuth.test.tsx
```
### Why Linux Only?
- Path separators: Code uses POSIX-style paths (`/`) which may break on Windows
- Shell scripts in `scripts/` directory are Linux-only
- External dependencies like `pdftocairo` assume Linux installation paths
- Unix-style file permissions are assumed throughout
### Test Result Interpretation
- Tests that **pass on Windows but fail on Linux** = **BROKEN tests** (must be fixed)
- Tests that **fail on Windows but pass on Linux** = **PASSING tests** (acceptable)
## Development Workflow
1. Open project in VS Code
2. Use "Reopen in Container" (Dev Containers extension required) to enter the dev environment
3. Wait for dev container initialization to complete
4. Run `npm test` to verify the dev environment is working
5. Make changes and run tests inside the dev container
## Code Change Verification
After making any code changes, **always run a type-check** to catch TypeScript errors before committing:
```bash
npm run type-check
```
This prevents linting/type errors from being introduced into the codebase.
## Quick Reference
| Command | Description |
| -------------------------- | ---------------------------- |
| `npm test` | Run all unit tests |
| `npm run test:unit` | Run unit tests only |
| `npm run test:integration` | Run integration tests |
| `npm run dev:container` | Start dev server (container) |
| `npm run build` | Build for production |
| `npm run type-check` | Run TypeScript type checking |
### Essential Commands
## Database Schema Files
| Command | Description |
| ------------------------------------------------------------ | --------------------- |
| `podman exec -it flyer-crawler-dev npm test` | Run all tests |
| `podman exec -it flyer-crawler-dev npm run test:unit` | Unit tests only |
| `podman exec -it flyer-crawler-dev npm run type-check` | TypeScript check |
| `podman exec -it flyer-crawler-dev npm run test:integration` | Integration tests |
| `podman exec -it flyer-crawler-dev pm2 status` | PM2 process status |
| `podman exec -it flyer-crawler-dev pm2 logs` | View all PM2 logs |
| `podman exec -it flyer-crawler-dev pm2 restart all` | Restart all processes |
**CRITICAL**: The database schema files must be kept in sync with each other. When making schema changes:
### Key Patterns (with file locations)
| File | Purpose |
| ------------------------------ | ----------------------------------------------------------- |
| `sql/master_schema_rollup.sql` | Complete schema used by test database setup and reference |
| `sql/initial_schema.sql` | Base schema without seed data, used as standalone reference |
| `sql/migrations/*.sql` | Incremental migrations for production database updates |
| Pattern | ADR | Implementation | File |
| ------------------ | ------- | ------------------------------------------------- | ----------------------------------- |
| Error Handling | ADR-001 | `handleDbError()`, throw `NotFoundError` | `src/services/db/errors.db.ts` |
| Repository Methods | ADR-034 | `get*` (throws), `find*` (null), `list*` (array) | `src/services/db/*.db.ts` |
| API Responses | ADR-028 | `sendSuccess()`, `sendPaginated()`, `sendError()` | `src/utils/apiResponse.ts` |
| Transactions | ADR-002 | `withTransaction(async (client) => {...})` | `src/services/db/transaction.db.ts` |
**Maintenance Rules:**
### Key Files Quick Access
1. **Keep `master_schema_rollup.sql` and `initial_schema.sql` in sync** - These files should contain the same table definitions
2. **When adding columns via migration**, also add them to both `master_schema_rollup.sql` and `initial_schema.sql`
3. **Migrations are for production deployments** - They use `ALTER TABLE` to add columns incrementally
4. **Schema files are for fresh installs** - They define the complete table structure
5. **Test database uses `master_schema_rollup.sql`** - If schema files are out of sync with migrations, tests will fail
| Purpose | File |
| ----------------- | -------------------------------- |
| Express app | `server.ts` |
| Environment | `src/config/env.ts` |
| Routes | `src/routes/*.routes.ts` |
| Repositories | `src/services/db/*.db.ts` |
| Workers | `src/services/workers.server.ts` |
| Queues | `src/services/queues.server.ts` |
| PM2 Config (Dev) | `ecosystem.dev.config.cjs` |
| PM2 Config (Prod) | `ecosystem.config.cjs` |
**Example:** When `002_expiry_tracking.sql` adds `purchase_date` to `pantry_items`, that column must also exist in the `CREATE TABLE` statements in both `master_schema_rollup.sql` and `initial_schema.sql`.
---
## Known Integration Test Issues and Solutions
## Application Overview
This section documents common test issues encountered in integration tests, their root causes, and solutions. These patterns recur frequently.
**Flyer Crawler** - AI-powered grocery deal extraction and analysis platform.
### 1. Vitest globalSetup Runs in Separate Node.js Context
**Data Flow**: Upload → AI extraction (Gemini) → PostgreSQL → Cache (Redis) → API → React display
**Problem:** Vitest's `globalSetup` runs in a completely separate Node.js context from test files. This means:
**Architecture** (ADR-035):
- Singletons created in globalSetup are NOT the same instances as those in test files
- `global`, `globalThis`, and `process` are all isolated between contexts
- `vi.spyOn()` on module exports doesn't work cross-context
- Dependency injection via setter methods fails across contexts
**Affected Tests:** Any test trying to inject mocks into BullMQ worker services (e.g., AI failure tests, DB failure tests)
**Solution Options:**
1. Mark tests as `.todo()` until an API-based mock injection mechanism is implemented
2. Create test-only API endpoints that allow setting mock behaviors via HTTP
3. Use file-based or Redis-based mock flags that services check at runtime
**Example of affected code pattern:**
```typescript
// This DOES NOT work - different module instances
const { flyerProcessingService } = await import('../../services/workers.server');
flyerProcessingService._getAiProcessor()._setExtractAndValidateData(mockFn);
// The worker uses a different flyerProcessingService instance!
```text
Routes → Services → Repositories → Database
External APIs (*.server.ts)
```
### 2. BullMQ Cleanup Queue Deleting Files Before Test Verification
**Key Entities**: Flyers, FlyerItems, Stores, StoreLocations, Users, Watchlists, ShoppingLists, Recipes, Achievements
**Problem:** The cleanup worker runs in the globalSetup context and processes cleanup jobs even when tests spy on `cleanupQueue.add()`. The spy intercepts calls in the test context, but jobs already queued run in the worker's context.
**Full Architecture**: See [docs/architecture/OVERVIEW.md](docs/architecture/OVERVIEW.md)
**Affected Tests:** EXIF/PNG metadata stripping tests that need to verify file contents before deletion
---
**Solution:** Drain and pause the cleanup queue before the test:
## Dev Container Architecture (ADR-014)
The dev container now matches production by using PM2 for process management.
### Process Management
| Component | Production | Dev Container |
| ---------- | ---------------------- | ------------------------- |
| API Server | PM2 cluster mode | PM2 fork mode + tsx watch |
| Worker | PM2 process | PM2 process + tsx watch |
| Frontend | Static files via NGINX | PM2 + Vite dev server |
| Logs | PM2 logs -> Logstash | PM2 logs -> Logstash |
**PM2 Processes in Dev Container**:
- `flyer-crawler-api-dev` - API server (port 3001)
- `flyer-crawler-worker-dev` - Background job worker
- `flyer-crawler-vite-dev` - Vite frontend dev server (port 5173)
### Log Aggregation (ADR-050)
All logs flow to Bugsink via Logstash:
| Source | Log Location | Status |
| ----------------- | --------------------------------- | ------ |
| Backend (Pino) | `/var/log/pm2/api-*.log` | Active |
| Worker (Pino) | `/var/log/pm2/worker-*.log` | Active |
| Vite | `/var/log/pm2/vite-*.log` | Active |
| PostgreSQL | `/var/log/postgresql/*.log` | Active |
| Redis | `/var/log/redis/redis-server.log` | Active |
| NGINX | `/var/log/nginx/*.log` | Active |
| Frontend (Sentry) | Browser -> Bugsink SDK | Active |
**Key Files**:
- `ecosystem.dev.config.cjs` - PM2 development configuration
- `scripts/dev-entrypoint.sh` - Container startup script
- `docker/logstash/bugsink.conf` - Logstash pipeline configuration
**Full Dev Container Guide**: See [docs/development/DEV-CONTAINER.md](docs/development/DEV-CONTAINER.md)
---
## Common Workflows
### Adding a New API Endpoint
1. Add route in `src/routes/{domain}.routes.ts`
2. Use `validateRequest(schema)` middleware for input validation
3. Call service layer (never access DB directly from routes)
4. Return via `sendSuccess()` or `sendPaginated()`
5. Add tests in `*.routes.test.ts`
**Example Pattern**: See [docs/development/CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md)
### Adding a New Database Operation
1. Add method to `src/services/db/{domain}.db.ts`
2. Follow naming: `get*` (throws), `find*` (returns null), `list*` (array)
3. Use `handleDbError()` for error handling
4. Accept optional `PoolClient` for transaction support
5. Add unit test
### Adding a Background Job
1. Define queue in `src/services/queues.server.ts`
2. Add worker in `src/services/workers.server.ts`
3. Call `queue.add()` from service layer
---
## Subagent Delegation Guide
**When to Delegate**: Complex work, specialized expertise, multi-domain tasks
### Decision Matrix
| Task Type | Subagent | Key Docs |
| --------------------- | ----------------------- | ----------------------------------------------------------------- |
| Write production code | coder | [CODER-GUIDE.md](docs/subagents/CODER-GUIDE.md) |
| Database changes | db-dev | [DATABASE-GUIDE.md](docs/subagents/DATABASE-GUIDE.md) |
| Create tests | testwriter | [TESTER-GUIDE.md](docs/subagents/TESTER-GUIDE.md) |
| Fix failing tests | tester | [TESTER-GUIDE.md](docs/subagents/TESTER-GUIDE.md) |
| Container/deployment | devops | [DEVOPS-GUIDE.md](docs/subagents/DEVOPS-GUIDE.md) |
| UI components | frontend-specialist | [FRONTEND-GUIDE.md](docs/subagents/FRONTEND-GUIDE.md) |
| External APIs | integrations-specialist | [INTEGRATIONS-GUIDE.md](docs/subagents/INTEGRATIONS-GUIDE.md) |
| Security review | security-engineer | [SECURITY-DEBUG-GUIDE.md](docs/subagents/SECURITY-DEBUG-GUIDE.md) |
| Production errors | log-debug | [SECURITY-DEBUG-GUIDE.md](docs/subagents/SECURITY-DEBUG-GUIDE.md) |
| AI/Gemini issues | ai-usage | [AI-USAGE-GUIDE.md](docs/subagents/AI-USAGE-GUIDE.md) |
| Planning features | planner | [DOCUMENTATION-GUIDE.md](docs/subagents/DOCUMENTATION-GUIDE.md) |
**All Subagents**: See [docs/subagents/OVERVIEW.md](docs/subagents/OVERVIEW.md)
**Launch Pattern**:
```
Use Task tool with subagent_type: "coder", "db-dev", "tester", etc.
```
---
## Known Issues & Gotchas
### Integration Test Issues (Summary)
Common issues with solutions:
1. **Vitest globalSetup context isolation** - Mocks/spies don't share instances → Mark `.todo()` or use Redis-based flags
2. **Cleanup queue interference** - Worker processes jobs during tests → `cleanupQueue.drain()` and `.pause()`
3. **Cache staleness** - Direct SQL bypasses cache → `cacheService.invalidateFlyers()` after inserts
4. **Filename collisions** - Multer predictable names → Use `${Date.now()}-${Math.round(Math.random() * 1e9)}`
5. **Response format mismatches** - API format changes → Log response bodies, update assertions
6. **External service failures** - PM2/Redis unavailable → try/catch with graceful degradation
**Full Details**: See test issues section at end of this document or [docs/development/TESTING.md](docs/development/TESTING.md)
### Git Bash Path Conversion (Windows)
Git Bash auto-converts Unix paths, breaking container commands.
**Solutions**:
```bash
# Use sh -c with single quotes
podman exec container sh -c '/usr/local/bin/script.sh'
# Use MSYS_NO_PATHCONV=1
MSYS_NO_PATHCONV=1 podman exec container /path/to/script
# Use Windows paths for host files
podman cp "d:/path/file" container:/tmp/file
```
---
## Configuration & Environment
### Environment Variables
**See**: [docs/getting-started/ENVIRONMENT.md](docs/getting-started/ENVIRONMENT.md) for complete reference.
**Quick Overview**:
- **Production**: Gitea CI/CD secrets only (no `.env` file)
- **Test**: Gitea secrets + `.env.test` overrides
- **Dev**: `.env.local` file (overrides `compose.dev.yml`)
**Key Variables**: `DB_HOST`, `DB_USER`, `DB_PASSWORD`, `JWT_SECRET`, `VITE_GOOGLE_GENAI_API_KEY`, `REDIS_URL`
**Adding Variables**: Update `src/config/env.ts`, Gitea Secrets, workflows, `ecosystem.config.cjs`, `.env.example`
### MCP Servers
**See**: [docs/tools/MCP-CONFIGURATION.md](docs/tools/MCP-CONFIGURATION.md) for setup.
**Quick Overview**:
| Server | Purpose | Config |
| -------------------------- | -------------------- | ---------------------- |
| gitea-projectium/torbonium | Gitea API | Global `settings.json` |
| podman | Container management | Global `settings.json` |
| memory | Knowledge graph | Global `settings.json` |
| redis | Cache access | Global `settings.json` |
| bugsink | Prod error tracking | Global `settings.json` |
| localerrors | Dev Bugsink | Project `.mcp.json` |
| devdb | Dev PostgreSQL | Project `.mcp.json` |
**Note**: Localhost servers use project `.mcp.json` due to Windows/loader issues.
### Bugsink Error Tracking
**See**: [docs/tools/BUGSINK-SETUP.md](docs/tools/BUGSINK-SETUP.md) for setup.
**Quick Access**:
- **Dev**: https://localhost:8443 (`admin@localhost`/`admin`)
- **Prod**: https://bugsink.projectium.com
**Token Creation** (required for MCP):
```bash
# Dev container
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
# Production (via SSH)
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
```
### Logstash
**See**: [docs/operations/LOGSTASH-QUICK-REF.md](docs/operations/LOGSTASH-QUICK-REF.md)
Log aggregation: PostgreSQL + PM2 + Redis + NGINX → Bugsink (ADR-050)
---
## Documentation Quick Links
| Topic | Document |
| ------------------- | -------------------------------------------------------------- |
| **Getting Started** | [QUICKSTART.md](docs/getting-started/QUICKSTART.md) |
| **Dev Container** | [DEV-CONTAINER.md](docs/development/DEV-CONTAINER.md) |
| **Architecture** | [OVERVIEW.md](docs/architecture/OVERVIEW.md) |
| **Code Patterns** | [CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md) |
| **Testing** | [TESTING.md](docs/development/TESTING.md) |
| **Debugging** | [DEBUGGING.md](docs/development/DEBUGGING.md) |
| **Database** | [DATABASE.md](docs/architecture/DATABASE.md) |
| **Deployment** | [DEPLOYMENT.md](docs/operations/DEPLOYMENT.md) |
| **Monitoring** | [MONITORING.md](docs/operations/MONITORING.md) |
| **Logstash** | [LOGSTASH-QUICK-REF.md](docs/operations/LOGSTASH-QUICK-REF.md) |
| **ADRs** | [docs/adr/index.md](docs/adr/index.md) |
| **All Docs** | [docs/README.md](docs/README.md) |
---
## Appendix: Integration Test Issues (Full Details)
### 1. Vitest globalSetup Context Isolation
Vitest's `globalSetup` runs in separate Node.js context. Singletons, spies, mocks do NOT share instances with test files.
**Affected**: BullMQ worker service mocks (AI/DB failure tests)
**Solutions**: Mark `.todo()`, create test-only API endpoints, use Redis-based mock flags
```typescript
// DOES NOT WORK - different instances
const { flyerProcessingService } = await import('../../services/workers.server');
flyerProcessingService._getAiProcessor()._setExtractAndValidateData(mockFn);
```
### 2. Cleanup Queue Deletes Before Verification
Cleanup worker processes jobs in globalSetup context, ignoring test spies.
**Solution**: Drain and pause queue:
```typescript
const { cleanupQueue } = await import('../../services/queues.server');
await cleanupQueue.drain(); // Remove existing jobs
await cleanupQueue.pause(); // Prevent new jobs from processing
// ... run test ...
await cleanupQueue.resume(); // Restore normal operation
await cleanupQueue.drain();
await cleanupQueue.pause();
// ... test ...
await cleanupQueue.resume();
```
### 3. Cache Invalidation After Direct Database Inserts
### 3. Cache Stale After Direct SQL
**Problem:** Tests that insert data directly via SQL (bypassing the service layer) don't trigger cache invalidation. Subsequent API calls return stale cached data.
Direct `pool.query()` inserts bypass cache invalidation.
**Affected Tests:** Any test using `pool.query()` to insert flyers, stores, or other cached entities
**Solution**: `await cacheService.invalidateFlyers();` after inserts
**Solution:** Manually invalidate the cache after direct inserts:
### 4. Test Filename Collisions
```typescript
await pool.query('INSERT INTO flyers ...');
await cacheService.invalidateFlyers(); // Clear stale cache
```
Multer predictable filenames cause race conditions.
### 4. Unique Filenames Required for Test Isolation
**Problem:** Multer generates predictable filenames in test environments, causing race conditions when multiple tests upload files concurrently or in sequence.
**Affected Tests:** Flyer processing tests, file upload tests
**Solution:** Always use unique filenames with timestamps:
```typescript
// In multer.middleware.ts
const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1e9)}`;
cb(null, `${file.fieldname}-${uniqueSuffix}-${sanitizedOriginalName}`);
```
**Solution**: Use unique suffix: `${Date.now()}-${Math.round(Math.random() * 1e9)}`
### 5. Response Format Mismatches
**Problem:** API response formats may change, causing tests to fail when expecting old formats.
API formats change: `data.jobId` vs `data.job.id`, nested vs flat, string vs number IDs.
**Common Issues:**
- `response.body.data.jobId` vs `response.body.data.job.id`
- Nested objects vs flat response structures
- Type coercion (string vs number for IDs)
**Solution:** Always log response bodies during debugging and update test assertions to match actual API contracts.
**Solution**: Log response bodies, update assertions
### 6. External Service Availability
**Problem:** Tests depending on external services (PM2, Redis health checks) fail when those services aren't available in the test environment.
PM2/Redis health checks fail when unavailable.
**Solution:** Use try/catch with graceful degradation or mock the external service checks.
## Secrets and Environment Variables
**CRITICAL**: This project uses **Gitea CI/CD secrets** for all sensitive configuration. There is NO `/etc/flyer-crawler/environment` file or similar local config file on the server.
### Server Directory Structure
| Path | Environment | Notes |
| --------------------------------------------- | ----------- | ------------------------------------------------ |
| `/var/www/flyer-crawler.projectium.com/` | Production | NO `.env` file - secrets injected via CI/CD only |
| `/var/www/flyer-crawler-test.projectium.com/` | Test | Has `.env.test` file for test-specific config |
### How Secrets Work
1. **Gitea Secrets**: All secrets are stored in Gitea repository settings (Settings → Secrets)
2. **CI/CD Injection**: Secrets are injected during deployment via `.gitea/workflows/deploy-to-prod.yml` and `deploy-to-test.yml`
3. **PM2 Environment**: The CI/CD workflow passes secrets to PM2 via environment variables, which are then available to the application
### Key Files for Configuration
| File | Purpose |
| ------------------------------------- | ---------------------------------------------------- |
| `src/config/env.ts` | Centralized config with Zod schema validation |
| `ecosystem.config.cjs` | PM2 process config - reads from `process.env` |
| `.gitea/workflows/deploy-to-prod.yml` | Production deployment with secret injection |
| `.gitea/workflows/deploy-to-test.yml` | Test deployment with secret injection |
| `.env.example` | Template showing all available environment variables |
| `.env.test` | Test environment overrides (only on test server) |
### Adding New Secrets
To add a new secret (e.g., `SENTRY_DSN`):
1. Add the secret to Gitea repository settings
2. Update the relevant workflow file (e.g., `deploy-to-prod.yml`) to inject it:
```yaml
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
```
3. Update `ecosystem.config.cjs` to read it from `process.env`
4. Update `src/config/env.ts` schema if validation is needed
5. Update `.env.example` to document the new variable
### Current Gitea Secrets
**Shared (used by both environments):**
- `DB_HOST` - Database host (shared PostgreSQL server)
- `JWT_SECRET` - Authentication
- `GOOGLE_MAPS_API_KEY` - Google Maps
- `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` - Google OAuth
- `GH_CLIENT_ID`, `GH_CLIENT_SECRET` - GitHub OAuth
- `SENTRY_AUTH_TOKEN` - Bugsink API token for source map uploads (create at Settings > API Keys in Bugsink)
**Production-specific:**
- `DB_USER_PROD`, `DB_PASSWORD_PROD` - Production database credentials (`flyer_crawler_prod`)
- `DB_DATABASE_PROD` - Production database name (`flyer-crawler`)
- `REDIS_PASSWORD_PROD` - Redis password (uses database 0)
- `VITE_GOOGLE_GENAI_API_KEY` - Gemini API key for production
- `SENTRY_DSN`, `VITE_SENTRY_DSN` - Bugsink error tracking DSNs (production projects)
**Test-specific:**
- `DB_USER_TEST`, `DB_PASSWORD_TEST` - Test database credentials (`flyer_crawler_test`)
- `DB_DATABASE_TEST` - Test database name (`flyer-crawler-test`)
- `REDIS_PASSWORD_TEST` - Redis password (uses database 1 for isolation)
- `VITE_GOOGLE_GENAI_API_KEY_TEST` - Gemini API key for test
- `SENTRY_DSN_TEST`, `VITE_SENTRY_DSN_TEST` - Bugsink error tracking DSNs (test projects)
### Test Environment
The test environment (`flyer-crawler-test.projectium.com`) uses **both** Gitea CI/CD secrets and a local `.env.test` file:
- **Gitea secrets**: Injected during deployment via `.gitea/workflows/deploy-to-test.yml`
- **`.env.test` file**: Located at `/var/www/flyer-crawler-test.projectium.com/.env.test` for local overrides
- **Redis database 1**: Isolates test job queues from production (which uses database 0)
- **PM2 process names**: Suffixed with `-test` (e.g., `flyer-crawler-api-test`)
### Database User Setup (Test Environment)
**CRITICAL**: The test database requires specific PostgreSQL permissions to be configured manually. Schema ownership alone is NOT sufficient - explicit privileges must be granted.
**Database Users:**
| User | Database | Purpose |
| -------------------- | -------------------- | ---------- |
| `flyer_crawler_prod` | `flyer-crawler-prod` | Production |
| `flyer_crawler_test` | `flyer-crawler-test` | Testing |
**Required Setup Commands** (run as `postgres` superuser):
```bash
# Connect as postgres superuser
sudo -u postgres psql
# Create the test database and user (if not exists)
CREATE DATABASE "flyer-crawler-test";
CREATE USER flyer_crawler_test WITH PASSWORD 'your-password-here';
# Grant ownership and privileges
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
\c "flyer-crawler-test"
ALTER SCHEMA public OWNER TO flyer_crawler_test;
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
# Create required extension (must be done by superuser)
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
```
**Why These Steps Are Necessary:**
1. **Schema ownership alone is insufficient** - PostgreSQL requires explicit `GRANT CREATE, USAGE` privileges even when the user owns the schema
2. **uuid-ossp extension** - Required by the application for UUID generation; must be created by a superuser before the app can use it
3. **Separate users for prod/test** - Prevents accidental cross-environment data access; each environment has its own credentials in Gitea secrets
**Verification:**
```bash
# Check schema privileges (should show 'UC' for flyer_crawler_test)
psql -d "flyer-crawler-test" -c "\dn+ public"
# Expected output:
# Name | Owner | Access privileges
# -------+--------------------+------------------------------------------
# public | flyer_crawler_test | flyer_crawler_test=UC/flyer_crawler_test
```
### Dev Container Environment
The dev container runs its own **local Bugsink instance** - it does NOT connect to the production Bugsink server:
- **Local Bugsink**: Runs at `http://localhost:8000` inside the container
- **Pre-configured DSNs**: Set in `compose.dev.yml`, pointing to local instance
- **Admin credentials**: `admin@localhost` / `admin`
- **Isolated**: Dev errors stay local, don't pollute production/test dashboards
- **No Gitea secrets needed**: Everything is self-contained in the container
---
## MCP Servers
The following MCP servers are configured for this project:
| Server | Purpose |
| --------------------- | ------------------------------------------- |
| gitea-projectium | Gitea API for gitea.projectium.com |
| gitea-torbonium | Gitea API for gitea.torbonium.com |
| podman | Container management |
| filesystem | File system access |
| fetch | Web fetching |
| markitdown | Convert documents to markdown |
| sequential-thinking | Step-by-step reasoning |
| memory | Knowledge graph persistence |
| postgres | Direct database queries (localhost:5432) |
| playwright | Browser automation and testing |
| redis | Redis cache inspection (localhost:6379) |
| sentry-selfhosted-mcp | Error tracking via Bugsink (localhost:8000) |
**Note:** MCP servers work in both **Claude CLI** and **Claude Code VS Code extension** (as of January 2026).
### Sentry/Bugsink MCP Server Setup (ADR-015)
To enable Claude Code to query and analyze application errors from Bugsink:
1. **Install the MCP server**:
```bash
# Clone the sentry-selfhosted-mcp repository
git clone https://github.com/ddfourtwo/sentry-selfhosted-mcp.git
cd sentry-selfhosted-mcp
npm install
```
2. **Configure Claude Code** (add to `.claude/mcp.json`):
```json
{
"sentry-selfhosted-mcp": {
"command": "node",
"args": ["/path/to/sentry-selfhosted-mcp/dist/index.js"],
"env": {
"SENTRY_URL": "http://localhost:8000",
"SENTRY_AUTH_TOKEN": "<get-from-bugsink-ui>",
"SENTRY_ORG_SLUG": "flyer-crawler"
}
}
}
```
3. **Get the auth token**:
- Navigate to Bugsink UI at `http://localhost:8000`
- Log in with admin credentials
- Go to Settings > API Keys
- Create a new API key with read access
4. **Available capabilities**:
- List projects and issues
- View detailed error events
- Search by error message or stack trace
- Update issue status (resolve, ignore)
- Add comments to issues
### SSH Server Access
Claude Code can execute commands on the production server via SSH:
```bash
# Basic command execution
ssh root@projectium.com "command here"
# Examples:
ssh root@projectium.com "systemctl status logstash"
ssh root@projectium.com "pm2 list"
ssh root@projectium.com "tail -50 /var/www/flyer-crawler.projectium.com/logs/app.log"
```
**Use cases:**
- Managing Logstash, PM2, NGINX, Redis services
- Viewing server logs
- Deploying configuration changes
- Checking service status
**Important:** SSH access requires the host machine to have SSH keys configured for `root@projectium.com`.
---
## Logstash Configuration (ADR-050)
The production server uses **Logstash** to aggregate logs from multiple sources and forward errors to Bugsink for centralized error tracking.
**Log Sources:**
- **PostgreSQL function logs** - Structured JSON logs from `fn_log()` helper function
- **PM2 worker logs** - Service logs from BullMQ job workers (stdout)
- **Redis logs** - Operational logs (INFO level) and errors
- **NGINX logs** - Access logs (all requests) and error logs
### Configuration Location
**Primary configuration file:**
- `/etc/logstash/conf.d/bugsink.conf` - Complete Logstash pipeline configuration
**Related files:**
- `/etc/postgresql/14/main/conf.d/observability.conf` - PostgreSQL logging configuration
- `/var/log/postgresql/*.log` - PostgreSQL log files
- `/home/gitea-runner/.pm2/logs/*.log` - PM2 worker logs
- `/var/log/redis/redis-server.log` - Redis logs
- `/var/log/nginx/access.log` - NGINX access logs
- `/var/log/nginx/error.log` - NGINX error logs
- `/var/log/logstash/*.log` - Logstash file outputs (operational logs)
- `/var/lib/logstash/sincedb_*` - Logstash position tracking files
### Key Features
1. **Multi-source aggregation**: Collects logs from PostgreSQL, PM2 workers, Redis, and NGINX
2. **Environment-based routing**: Automatically detects production vs test environments and routes errors to the correct Bugsink project
3. **Structured JSON parsing**: Extracts `fn_log()` function output from PostgreSQL logs and Pino JSON from PM2 workers
4. **Sentry-compatible format**: Transforms events to Sentry format with `event_id`, `timestamp`, `level`, `message`, and `extra` context
5. **Error filtering**: Only forwards WARNING and ERROR level messages to Bugsink
6. **Operational log storage**: Stores non-error logs (Redis INFO, NGINX access, PM2 operational) to `/var/log/logstash/` for analysis
7. **Request monitoring**: Categorizes NGINX requests by status code (2xx, 3xx, 4xx, 5xx) and identifies slow requests
### Common Maintenance Commands
```bash
# Check Logstash status
systemctl status logstash
# Restart Logstash after configuration changes
systemctl restart logstash
# Test configuration syntax
/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf
# View Logstash logs
journalctl -u logstash -f
# Check Logstash stats (events processed, failures)
curl -XGET 'localhost:9600/_node/stats/pipelines?pretty' | jq '.pipelines.main.plugins.filters'
# Monitor PostgreSQL logs being processed
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log
# View operational log outputs
tail -f /var/log/logstash/pm2-workers-$(date +%Y-%m-%d).log
tail -f /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
# Check disk usage of log files
du -sh /var/log/logstash/
```
### Troubleshooting
| Issue | Check | Solution |
| ------------------------------- | ---------------------------- | ---------------------------------------------------------------------------------------------- |
| Errors not appearing in Bugsink | Check Logstash is running | `systemctl status logstash` |
| Configuration syntax errors | Test config file | `/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf` |
| Grok pattern failures | Check Logstash stats | `curl localhost:9600/_node/stats/pipelines?pretty \| jq '.pipelines.main.plugins.filters'` |
| Wrong Bugsink project | Verify environment detection | Check tags in logs match expected environment (production/test) |
| Permission denied reading logs | Check Logstash permissions | `groups logstash` should include `postgres`, `adm` groups |
| PM2 logs not captured | Check file paths exist | `ls /home/gitea-runner/.pm2/logs/flyer-crawler-worker-*.log` |
| NGINX access logs not showing | Check file output directory | `ls -lh /var/log/logstash/nginx-access-*.log` |
| High disk usage | Check log rotation | Verify `/etc/logrotate.d/logstash` is configured and running daily |
**Full setup guide**: See [docs/BARE-METAL-SETUP.md](docs/BARE-METAL-SETUP.md) section "PostgreSQL Function Observability (ADR-050)"
**Architecture details**: See [docs/adr/0050-postgresql-function-observability.md](docs/adr/0050-postgresql-function-observability.md)
**Solution**: try/catch with graceful degradation or mock

660
CLAUDE.md.backup Normal file
View File

@@ -0,0 +1,660 @@
# Claude Code Project Instructions
## Session Startup Checklist
**IMPORTANT**: At the start of every session, perform these steps:
1. **Check Memory First** - Use `mcp__memory__read_graph` or `mcp__memory__search_nodes` to recall:
- Project-specific configurations and credentials
- Previous work context and decisions
- Infrastructure details (URLs, ports, access patterns)
- Known issues and their solutions
2. **Review Recent Git History** - Check `git log --oneline -10` to understand recent changes
3. **Check Container Status** - Use `mcp__podman__container_list` to see what's running
---
## Project Instructions
### Things to Remember
Before writing any code:
1. State how you will verify this change works (test, bash command, browser check, etc.)
2. Write the test or verification step first
3. Then implement the code
4. Run verification and iterate until it passes
## Git Bash / MSYS Path Conversion Issue (Windows Host)
**CRITICAL ISSUE**: Git Bash on Windows automatically converts Unix-style paths to Windows paths, which breaks Podman/Docker commands.
### Problem Examples:
```bash
# This FAILS in Git Bash:
podman exec container /usr/local/bin/script.sh
# Git Bash converts to: C:/Program Files/Git/usr/local/bin/script.sh
# This FAILS in Git Bash:
podman exec container bash -c "cat /tmp/file.sql"
# Git Bash converts /tmp to C:/Users/user/AppData/Local/Temp
```
### Solutions:
1. **Use `sh -c` instead of `bash -c`** for single-quoted commands:
```bash
podman exec container sh -c '/usr/local/bin/script.sh'
```
2. **Use double slashes** to escape path conversion:
```bash
podman exec container //usr//local//bin//script.sh
```
3. **Set MSYS_NO_PATHCONV** environment variable:
```bash
MSYS_NO_PATHCONV=1 podman exec container /usr/local/bin/script.sh
```
4. **Use Windows paths with forward slashes** when referencing host files:
```bash
podman cp "d:/path/to/file" container:/tmp/file
```
**ALWAYS use one of these workarounds when running Bash commands on Windows that involve Unix paths inside containers.**
## Communication Style: Ask Before Assuming
**IMPORTANT**: When helping with tasks, **ask clarifying questions before making assumptions**. Do not assume:
- What steps the user has or hasn't completed
- What the user already knows or has configured
- What external services (OAuth providers, APIs, etc.) are already set up
- What secrets or credentials have already been created
Instead, ask the user to confirm the current state before providing instructions or making recommendations. This prevents wasted effort and respects the user's existing work.
## Platform Requirement: Linux Only
**CRITICAL**: This application is designed to run **exclusively on Linux**. See [ADR-014](docs/adr/0014-containerization-and-deployment-strategy.md) for full details.
### Environment Terminology
- **Dev Container** (or just "dev"): The containerized Linux development environment (`flyer-crawler-dev`). This is where all development and testing should occur.
- **Host**: The Windows machine running Podman/Docker and VS Code.
When instructions say "run in dev" or "run in the dev container", they mean executing commands inside the `flyer-crawler-dev` container.
### Test Execution Rules
1. **ALL tests MUST be executed in the dev container** - the Linux container environment
2. **NEVER run tests directly on Windows host** - test results from Windows are unreliable
3. **Always use the dev container for testing** when developing on Windows
4. **TypeScript type-check MUST run in dev container** - `npm run type-check` on Windows does not reliably detect errors
See [docs/TESTING.md](docs/TESTING.md) for comprehensive testing documentation.
### How to Run Tests Correctly
```bash
# If on Windows, first open VS Code and "Reopen in Container"
# Then run tests inside the dev container:
npm test # Run all unit tests
npm run test:unit # Run unit tests only
npm run test:integration # Run integration tests (requires DB/Redis)
```
### Running Tests via Podman (from Windows host)
**Note:** This project has 2900+ unit tests. For AI-assisted development, pipe output to a file for easier processing.
The command to run unit tests in the dev container via podman:
```bash
# Basic (output to terminal)
podman exec -it flyer-crawler-dev npm run test:unit
# Recommended for AI processing: pipe to file
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
```
The command to run integration tests in the dev container via podman:
```bash
podman exec -it flyer-crawler-dev npm run test:integration
```
For running specific test files:
```bash
podman exec -it flyer-crawler-dev npm test -- --run src/hooks/useAuth.test.tsx
```
### Why Linux Only?
- Path separators: Code uses POSIX-style paths (`/`) which may break on Windows
- Shell scripts in `scripts/` directory are Linux-only
- External dependencies like `pdftocairo` assume Linux installation paths
- Unix-style file permissions are assumed throughout
### Test Result Interpretation
- Tests that **pass on Windows but fail on Linux** = **BROKEN tests** (must be fixed)
- Tests that **fail on Windows but pass on Linux** = **PASSING tests** (acceptable)
## Development Workflow
1. Open project in VS Code
2. Use "Reopen in Container" (Dev Containers extension required) to enter the dev environment
3. Wait for dev container initialization to complete
4. Run `npm test` to verify the dev environment is working
5. Make changes and run tests inside the dev container
## Code Change Verification
After making any code changes, **always run a type-check** to catch TypeScript errors before committing:
```bash
npm run type-check
```
This prevents linting/type errors from being introduced into the codebase.
## Quick Reference
| Command | Description |
| -------------------------- | ---------------------------- |
| `npm test` | Run all unit tests |
| `npm run test:unit` | Run unit tests only |
| `npm run test:integration` | Run integration tests |
| `npm run dev:container` | Start dev server (container) |
| `npm run build` | Build for production |
| `npm run type-check` | Run TypeScript type checking |
## Database Schema Files
**CRITICAL**: The database schema files must be kept in sync with each other. When making schema changes:
| File | Purpose |
| ------------------------------ | ----------------------------------------------------------- |
| `sql/master_schema_rollup.sql` | Complete schema used by test database setup and reference |
| `sql/initial_schema.sql` | Base schema without seed data, used as standalone reference |
| `sql/migrations/*.sql` | Incremental migrations for production database updates |
**Maintenance Rules:**
1. **Keep `master_schema_rollup.sql` and `initial_schema.sql` in sync** - These files should contain the same table definitions
2. **When adding columns via migration**, also add them to both `master_schema_rollup.sql` and `initial_schema.sql`
3. **Migrations are for production deployments** - They use `ALTER TABLE` to add columns incrementally
4. **Schema files are for fresh installs** - They define the complete table structure
5. **Test database uses `master_schema_rollup.sql`** - If schema files are out of sync with migrations, tests will fail
**Example:** When `002_expiry_tracking.sql` adds `purchase_date` to `pantry_items`, that column must also exist in the `CREATE TABLE` statements in both `master_schema_rollup.sql` and `initial_schema.sql`.
## Known Integration Test Issues and Solutions
This section documents common test issues encountered in integration tests, their root causes, and solutions. These patterns recur frequently.
### 1. Vitest globalSetup Runs in Separate Node.js Context
**Problem:** Vitest's `globalSetup` runs in a completely separate Node.js context from test files. This means:
- Singletons created in globalSetup are NOT the same instances as those in test files
- `global`, `globalThis`, and `process` are all isolated between contexts
- `vi.spyOn()` on module exports doesn't work cross-context
- Dependency injection via setter methods fails across contexts
**Affected Tests:** Any test trying to inject mocks into BullMQ worker services (e.g., AI failure tests, DB failure tests)
**Solution Options:**
1. Mark tests as `.todo()` until an API-based mock injection mechanism is implemented
2. Create test-only API endpoints that allow setting mock behaviors via HTTP
3. Use file-based or Redis-based mock flags that services check at runtime
**Example of affected code pattern:**
```typescript
// This DOES NOT work - different module instances
const { flyerProcessingService } = await import('../../services/workers.server');
flyerProcessingService._getAiProcessor()._setExtractAndValidateData(mockFn);
// The worker uses a different flyerProcessingService instance!
```
### 2. BullMQ Cleanup Queue Deleting Files Before Test Verification
**Problem:** The cleanup worker runs in the globalSetup context and processes cleanup jobs even when tests spy on `cleanupQueue.add()`. The spy intercepts calls in the test context, but jobs already queued run in the worker's context.
**Affected Tests:** EXIF/PNG metadata stripping tests that need to verify file contents before deletion
**Solution:** Drain and pause the cleanup queue before the test:
```typescript
const { cleanupQueue } = await import('../../services/queues.server');
await cleanupQueue.drain(); // Remove existing jobs
await cleanupQueue.pause(); // Prevent new jobs from processing
// ... run test ...
await cleanupQueue.resume(); // Restore normal operation
```
### 3. Cache Invalidation After Direct Database Inserts
**Problem:** Tests that insert data directly via SQL (bypassing the service layer) don't trigger cache invalidation. Subsequent API calls return stale cached data.
**Affected Tests:** Any test using `pool.query()` to insert flyers, stores, or other cached entities
**Solution:** Manually invalidate the cache after direct inserts:
```typescript
await pool.query('INSERT INTO flyers ...');
await cacheService.invalidateFlyers(); // Clear stale cache
```
### 4. Unique Filenames Required for Test Isolation
**Problem:** Multer generates predictable filenames in test environments, causing race conditions when multiple tests upload files concurrently or in sequence.
**Affected Tests:** Flyer processing tests, file upload tests
**Solution:** Always use unique filenames with timestamps:
```typescript
// In multer.middleware.ts
const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1e9)}`;
cb(null, `${file.fieldname}-${uniqueSuffix}-${sanitizedOriginalName}`);
```
### 5. Response Format Mismatches
**Problem:** API response formats may change, causing tests to fail when expecting old formats.
**Common Issues:**
- `response.body.data.jobId` vs `response.body.data.job.id`
- Nested objects vs flat response structures
- Type coercion (string vs number for IDs)
**Solution:** Always log response bodies during debugging and update test assertions to match actual API contracts.
### 6. External Service Availability
**Problem:** Tests depending on external services (PM2, Redis health checks) fail when those services aren't available in the test environment.
**Solution:** Use try/catch with graceful degradation or mock the external service checks.
## Secrets and Environment Variables
**CRITICAL**: This project uses **Gitea CI/CD secrets** for all sensitive configuration. There is NO `/etc/flyer-crawler/environment` file or similar local config file on the server.
### Server Directory Structure
| Path | Environment | Notes |
| --------------------------------------------- | ----------- | ------------------------------------------------ |
| `/var/www/flyer-crawler.projectium.com/` | Production | NO `.env` file - secrets injected via CI/CD only |
| `/var/www/flyer-crawler-test.projectium.com/` | Test | Has `.env.test` file for test-specific config |
### How Secrets Work
1. **Gitea Secrets**: All secrets are stored in Gitea repository settings (Settings → Secrets)
2. **CI/CD Injection**: Secrets are injected during deployment via `.gitea/workflows/deploy-to-prod.yml` and `deploy-to-test.yml`
3. **PM2 Environment**: The CI/CD workflow passes secrets to PM2 via environment variables, which are then available to the application
### Key Files for Configuration
| File | Purpose |
| ------------------------------------- | ---------------------------------------------------- |
| `src/config/env.ts` | Centralized config with Zod schema validation |
| `ecosystem.config.cjs` | PM2 process config - reads from `process.env` |
| `.gitea/workflows/deploy-to-prod.yml` | Production deployment with secret injection |
| `.gitea/workflows/deploy-to-test.yml` | Test deployment with secret injection |
| `.env.example` | Template showing all available environment variables |
| `.env.test` | Test environment overrides (only on test server) |
### Adding New Secrets
To add a new secret (e.g., `SENTRY_DSN`):
1. Add the secret to Gitea repository settings
2. Update the relevant workflow file (e.g., `deploy-to-prod.yml`) to inject it:
```yaml
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
```
3. Update `ecosystem.config.cjs` to read it from `process.env`
4. Update `src/config/env.ts` schema if validation is needed
5. Update `.env.example` to document the new variable
### Current Gitea Secrets
**Shared (used by both environments):**
- `DB_HOST` - Database host (shared PostgreSQL server)
- `JWT_SECRET` - Authentication
- `GOOGLE_MAPS_API_KEY` - Google Maps
- `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` - Google OAuth
- `GH_CLIENT_ID`, `GH_CLIENT_SECRET` - GitHub OAuth
- `SENTRY_AUTH_TOKEN` - Bugsink API token for source map uploads (create at Settings > API Keys in Bugsink)
**Production-specific:**
- `DB_USER_PROD`, `DB_PASSWORD_PROD` - Production database credentials (`flyer_crawler_prod`)
- `DB_DATABASE_PROD` - Production database name (`flyer-crawler`)
- `REDIS_PASSWORD_PROD` - Redis password (uses database 0)
- `VITE_GOOGLE_GENAI_API_KEY` - Gemini API key for production
- `SENTRY_DSN`, `VITE_SENTRY_DSN` - Bugsink error tracking DSNs (production projects)
**Test-specific:**
- `DB_USER_TEST`, `DB_PASSWORD_TEST` - Test database credentials (`flyer_crawler_test`)
- `DB_DATABASE_TEST` - Test database name (`flyer-crawler-test`)
- `REDIS_PASSWORD_TEST` - Redis password (uses database 1 for isolation)
- `VITE_GOOGLE_GENAI_API_KEY_TEST` - Gemini API key for test
- `SENTRY_DSN_TEST`, `VITE_SENTRY_DSN_TEST` - Bugsink error tracking DSNs (test projects)
### Test Environment
The test environment (`flyer-crawler-test.projectium.com`) uses **both** Gitea CI/CD secrets and a local `.env.test` file:
- **Gitea secrets**: Injected during deployment via `.gitea/workflows/deploy-to-test.yml`
- **`.env.test` file**: Located at `/var/www/flyer-crawler-test.projectium.com/.env.test` for local overrides
- **Redis database 1**: Isolates test job queues from production (which uses database 0)
- **PM2 process names**: Suffixed with `-test` (e.g., `flyer-crawler-api-test`)
### Database User Setup (Test Environment)
**CRITICAL**: The test database requires specific PostgreSQL permissions to be configured manually. Schema ownership alone is NOT sufficient - explicit privileges must be granted.
**Database Users:**
| User | Database | Purpose |
| -------------------- | -------------------- | ---------- |
| `flyer_crawler_prod` | `flyer-crawler-prod` | Production |
| `flyer_crawler_test` | `flyer-crawler-test` | Testing |
**Required Setup Commands** (run as `postgres` superuser):
```bash
# Connect as postgres superuser
sudo -u postgres psql
# Create the test database and user (if not exists)
CREATE DATABASE "flyer-crawler-test";
CREATE USER flyer_crawler_test WITH PASSWORD 'your-password-here';
# Grant ownership and privileges
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
\c "flyer-crawler-test"
ALTER SCHEMA public OWNER TO flyer_crawler_test;
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
# Create required extension (must be done by superuser)
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
```
**Why These Steps Are Necessary:**
1. **Schema ownership alone is insufficient** - PostgreSQL requires explicit `GRANT CREATE, USAGE` privileges even when the user owns the schema
2. **uuid-ossp extension** - Required by the application for UUID generation; must be created by a superuser before the app can use it
3. **Separate users for prod/test** - Prevents accidental cross-environment data access; each environment has its own credentials in Gitea secrets
**Verification:**
```bash
# Check schema privileges (should show 'UC' for flyer_crawler_test)
psql -d "flyer-crawler-test" -c "\dn+ public"
# Expected output:
# Name | Owner | Access privileges
# -------+--------------------+------------------------------------------
# public | flyer_crawler_test | flyer_crawler_test=UC/flyer_crawler_test
```
### Dev Container Environment
The dev container runs its own **local Bugsink instance** - it does NOT connect to the production Bugsink server:
- **Local Bugsink UI**: Accessible at `https://localhost:8443` (proxied from `http://localhost:8000` by nginx)
- **Admin credentials**: `admin@localhost` / `admin`
- **Bugsink Projects**: Backend (Dev) - Project ID 1, Frontend (Dev) - Project ID 2
- **Configuration Files**:
- `compose.dev.yml` - Sets default DSNs using `127.0.0.1:8000` protocol (for initial container setup)
- `.env.local` - **OVERRIDES** compose.dev.yml with `localhost:8000` protocol (this is what the app actually uses)
- **CRITICAL**: `.env.local` takes precedence over `compose.dev.yml` environment variables
- **DSN Configuration**:
- **Backend DSN** (Node.js/Express): Configured in `.env.local` as `SENTRY_DSN=http://<key>@localhost:8000/1`
- **Frontend DSN** (React/Browser): Configured in `.env.local` as `VITE_SENTRY_DSN=http://<key>@localhost:8000/2`
- **Why localhost instead of 127.0.0.1?** The `.env.local` file was created separately and uses `localhost` which works fine in practice
- **HTTPS Setup**: Self-signed certificates auto-generated with mkcert on container startup (for UI access only, not for Sentry SDK)
- **CSRF Protection**: Django configured with `SECURE_PROXY_SSL_HEADER` to trust `X-Forwarded-Proto` from nginx
- **Isolated**: Dev errors stay local, don't pollute production/test dashboards
- **No Gitea secrets needed**: Everything is self-contained in the container
- **Accessing Errors**:
- **Via Browser**: Open `https://localhost:8443` and login to view issues
- **Via MCP**: Configure a second Bugsink MCP server pointing to `http://localhost:8000` (see MCP Servers section below)
---
## MCP Servers
The following MCP servers are configured for this project:
| Server | Purpose |
| ------------------- | ---------------------------------------------------------------------------- |
| gitea-projectium | Gitea API for gitea.projectium.com |
| gitea-torbonium | Gitea API for gitea.torbonium.com |
| podman | Container management |
| filesystem | File system access |
| fetch | Web fetching |
| markitdown | Convert documents to markdown |
| sequential-thinking | Step-by-step reasoning |
| memory | Knowledge graph persistence |
| postgres | Direct database queries (localhost:5432) |
| playwright | Browser automation and testing |
| redis | Redis cache inspection (localhost:6379) |
| bugsink | Error tracking - production Bugsink (bugsink.projectium.com) - **PROD/TEST** |
| bugsink-dev | Error tracking - dev container Bugsink (localhost:8000) - **DEV CONTAINER** |
**Note:** MCP servers work in both **Claude CLI** and **Claude Code VS Code extension** (as of January 2026).
**CRITICAL**: There are **TWO separate Bugsink MCP servers**:
- **bugsink**: Connects to production Bugsink at `https://bugsink.projectium.com` for production and test server errors
- **bugsink-dev**: Connects to local dev container Bugsink at `http://localhost:8000` for local development errors
### Bugsink MCP Server Setup (ADR-015)
**IMPORTANT**: You need to configure **TWO separate MCP servers** - one for production/test, one for local dev.
#### Installation (shared for both servers)
```bash
# Clone the bugsink-mcp repository (NOT sentry-selfhosted-mcp)
git clone https://github.com/j-shelfwood/bugsink-mcp.git
cd bugsink-mcp
npm install
npm run build
```
#### Production/Test Bugsink MCP (bugsink)
Add to `.claude/mcp.json`:
```json
{
"bugsink": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "https://bugsink.projectium.com",
"BUGSINK_API_TOKEN": "<get-from-production-bugsink>",
"BUGSINK_ORG_SLUG": "sentry"
}
}
}
```
**Get the auth token**:
- Navigate to https://bugsink.projectium.com
- Log in with production credentials
- Go to Settings > API Keys
- Create a new API key with read access
#### Dev Container Bugsink MCP (bugsink-dev)
Add to `.claude/mcp.json`:
```json
{
"bugsink-dev": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "http://localhost:8000",
"BUGSINK_API_TOKEN": "<get-from-local-bugsink>",
"BUGSINK_ORG_SLUG": "sentry"
}
}
}
```
**Get the auth token**:
- Navigate to http://localhost:8000 (or https://localhost:8443)
- Log in with `admin@localhost` / `admin`
- Go to Settings > API Keys
- Create a new API key with read access
#### MCP Tool Usage
When using Bugsink MCP tools, remember:
- `mcp__bugsink__*` tools connect to **production/test** Bugsink
- `mcp__bugsink-dev__*` tools connect to **dev container** Bugsink
- Available capabilities for both:
- List projects and issues
- View detailed error events and stacktraces
- Search by error message or stack trace
- Update issue status (resolve, ignore)
- Create releases
### SSH Server Access
Claude Code can execute commands on the production server via SSH:
```bash
# Basic command execution
ssh root@projectium.com "command here"
# Examples:
ssh root@projectium.com "systemctl status logstash"
ssh root@projectium.com "pm2 list"
ssh root@projectium.com "tail -50 /var/www/flyer-crawler.projectium.com/logs/app.log"
```
**Use cases:**
- Managing Logstash, PM2, NGINX, Redis services
- Viewing server logs
- Deploying configuration changes
- Checking service status
**Important:** SSH access requires the host machine to have SSH keys configured for `root@projectium.com`.
---
## Logstash Configuration (ADR-050)
The production server uses **Logstash** to aggregate logs from multiple sources and forward errors to Bugsink for centralized error tracking.
**Log Sources:**
- **PostgreSQL function logs** - Structured JSON logs from `fn_log()` helper function
- **PM2 worker logs** - Service logs from BullMQ job workers (stdout)
- **Redis logs** - Operational logs (INFO level) and errors
- **NGINX logs** - Access logs (all requests) and error logs
### Configuration Location
**Primary configuration file:**
- `/etc/logstash/conf.d/bugsink.conf` - Complete Logstash pipeline configuration
**Related files:**
- `/etc/postgresql/14/main/conf.d/observability.conf` - PostgreSQL logging configuration
- `/var/log/postgresql/*.log` - PostgreSQL log files
- `/home/gitea-runner/.pm2/logs/*.log` - PM2 worker logs
- `/var/log/redis/redis-server.log` - Redis logs
- `/var/log/nginx/access.log` - NGINX access logs
- `/var/log/nginx/error.log` - NGINX error logs
- `/var/log/logstash/*.log` - Logstash file outputs (operational logs)
- `/var/lib/logstash/sincedb_*` - Logstash position tracking files
### Key Features
1. **Multi-source aggregation**: Collects logs from PostgreSQL, PM2 workers, Redis, and NGINX
2. **Environment-based routing**: Automatically detects production vs test environments and routes errors to the correct Bugsink project
3. **Structured JSON parsing**: Extracts `fn_log()` function output from PostgreSQL logs and Pino JSON from PM2 workers
4. **Sentry-compatible format**: Transforms events to Sentry format with `event_id`, `timestamp`, `level`, `message`, and `extra` context
5. **Error filtering**: Only forwards WARNING and ERROR level messages to Bugsink
6. **Operational log storage**: Stores non-error logs (Redis INFO, NGINX access, PM2 operational) to `/var/log/logstash/` for analysis
7. **Request monitoring**: Categorizes NGINX requests by status code (2xx, 3xx, 4xx, 5xx) and identifies slow requests
### Common Maintenance Commands
```bash
# Check Logstash status
systemctl status logstash
# Restart Logstash after configuration changes
systemctl restart logstash
# Test configuration syntax
/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf
# View Logstash logs
journalctl -u logstash -f
# Check Logstash stats (events processed, failures)
curl -XGET 'localhost:9600/_node/stats/pipelines?pretty' | jq '.pipelines.main.plugins.filters'
# Monitor PostgreSQL logs being processed
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log
# View operational log outputs
tail -f /var/log/logstash/pm2-workers-$(date +%Y-%m-%d).log
tail -f /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
# Check disk usage of log files
du -sh /var/log/logstash/
```
### Troubleshooting
| Issue | Check | Solution |
| ------------------------------- | ---------------------------- | ---------------------------------------------------------------------------------------------- |
| Errors not appearing in Bugsink | Check Logstash is running | `systemctl status logstash` |
| Configuration syntax errors | Test config file | `/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf` |
| Grok pattern failures | Check Logstash stats | `curl localhost:9600/_node/stats/pipelines?pretty \| jq '.pipelines.main.plugins.filters'` |
| Wrong Bugsink project | Verify environment detection | Check tags in logs match expected environment (production/test) |
| Permission denied reading logs | Check Logstash permissions | `groups logstash` should include `postgres`, `adm` groups |
| PM2 logs not captured | Check file paths exist | `ls /home/gitea-runner/.pm2/logs/flyer-crawler-worker-*.log` |
| NGINX access logs not showing | Check file output directory | `ls -lh /var/log/logstash/nginx-access-*.log` |
| High disk usage | Check log rotation | Verify `/etc/logrotate.d/logstash` is configured and running daily |
**Full setup guide**: See [docs/BARE-METAL-SETUP.md](docs/BARE-METAL-SETUP.md) section "PostgreSQL Function Observability (ADR-050)"
**Architecture details**: See [docs/adr/0050-postgresql-function-observability.md](docs/adr/0050-postgresql-function-observability.md)

348
CONTRIBUTING.md Normal file
View File

@@ -0,0 +1,348 @@
# Contributing to Flyer Crawler
Thank you for your interest in contributing to Flyer Crawler! This guide will help you understand our development workflow and coding standards.
## Table of Contents
- [Getting Started](#getting-started)
- [Development Workflow](#development-workflow)
- [Code Standards](#code-standards)
- [Testing Requirements](#testing-requirements)
- [Pull Request Process](#pull-request-process)
- [Architecture Decision Records](#architecture-decision-records)
- [Working with AI Agents](#working-with-ai-agents)
## Getting Started
### Prerequisites
1. **Windows with Podman Desktop**: See [docs/getting-started/INSTALL.md](docs/getting-started/INSTALL.md)
2. **Node.js 20+**: For running the application
3. **Git**: For version control
### Initial Setup
```bash
# Clone the repository
git clone <repository-url>
cd flyer-crawler.projectium.com
# Install dependencies
npm install
# Start development containers
podman start flyer-crawler-postgres flyer-crawler-redis
# Start development server
npm run dev
```
## Development Workflow
### Before Making Changes
1. **Read [CLAUDE.md](CLAUDE.md)** - Project guidelines and patterns
2. **Review relevant ADRs** in [docs/adr/](docs/adr/) - Understand architectural decisions
3. **Check existing issues** - Avoid duplicate work
4. **Create a feature branch** - Use descriptive names
```bash
git checkout -b feature/descriptive-name
# or
git checkout -b fix/issue-description
```
### Making Changes
#### Code Changes
Follow the established patterns from [docs/development/CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md):
1. **Routes****Services****Repositories****Database**
2. Never access the database directly from routes
3. Use Zod schemas for input validation
4. Follow ADR-034 repository naming conventions:
- `get*` - Throws NotFoundError if not found
- `find*` - Returns null if not found
- `list*` - Returns empty array if none found
#### Database Changes
When modifying the database schema:
1. Create migration: `sql/migrations/NNNN-description.sql`
2. Update `sql/master_schema_rollup.sql` (complete schema)
3. Update `sql/initial_schema.sql` (identical to rollup)
4. Test with integration tests
**CRITICAL**: Schema files must stay synchronized. See [CLAUDE.md#database-schema-sync](CLAUDE.md#database-schema-sync).
#### NGINX Configuration Changes
When modifying NGINX configurations on the production or test servers:
1. Make changes on the server at `/etc/nginx/sites-available/`
2. Test with `sudo nginx -t` and reload with `sudo systemctl reload nginx`
3. Update the reference copies in the repository root:
- `etc-nginx-sites-available-flyer-crawler.projectium.com` - Production
- `etc-nginx-sites-available-flyer-crawler-test-projectium-com.txt` - Test
4. Commit the updated reference files
These reference files serve as version-controlled documentation of the deployed configurations.
### Testing
**IMPORTANT**: All tests must run in the dev container.
```bash
# Run all tests
podman exec -it flyer-crawler-dev npm test
# Run specific test file
podman exec -it flyer-crawler-dev npm test -- --run src/path/to/file.test.ts
# Type check
podman exec -it flyer-crawler-dev npm run type-check
```
#### Before Committing
1. Write tests for new features
2. Update existing tests if behavior changes
3. Run full test suite
4. Run type check
5. Verify documentation is updated
See [docs/development/TESTING.md](docs/development/TESTING.md) for detailed testing guidelines.
## Code Standards
### TypeScript
- Use strict TypeScript mode
- Define types in `src/types/*.ts` files
- Avoid `any` - use `unknown` if type is truly unknown
- Follow ADR-027 for naming conventions
### Error Handling
```typescript
// Repository layer (ADR-001)
import { handleDbError, NotFoundError } from '../services/db/errors.db';
try {
const result = await client.query(query, values);
if (result.rows.length === 0) {
throw new NotFoundError('Flyer', id);
}
return result.rows[0];
} catch (error) {
throw handleDbError(error);
}
```
### API Responses
```typescript
// Route handlers (ADR-028)
import { sendSuccess, sendError } from '../utils/apiResponse';
// Success response
return sendSuccess(res, flyer, 'Flyer retrieved successfully');
// Paginated response
return sendPaginated(res, {
items: flyers,
total: count,
page: 1,
pageSize: 20,
});
```
### Transactions
```typescript
// Multi-operation changes (ADR-002)
import { withTransaction } from '../services/db/transaction.db';
await withTransaction(async (client) => {
await flyerDb.createFlyer(flyerData, client);
await flyerItemDb.createItems(items, client);
// Automatically commits on success, rolls back on error
});
```
## Testing Requirements
### Test Coverage
- **Unit tests**: All service functions, utilities, and helpers
- **Integration tests**: API endpoints, database operations
- **E2E tests**: Critical user flows
### Test Patterns
See [docs/subagents/TESTER-GUIDE.md](docs/subagents/TESTER-GUIDE.md) for:
- Test helper functions
- Mocking patterns
- Known testing issues and solutions
### Test Naming
```typescript
// Good test names
describe('FlyerService.createFlyer', () => {
it('should create flyer with valid data', async () => { ... });
it('should throw ValidationError when store_id is missing', async () => { ... });
it('should rollback transaction on item creation failure', async () => { ... });
});
```
## Pull Request Process
### 1. Prepare Your PR
- [ ] All tests pass in dev container
- [ ] Type check passes
- [ ] No console.log or debugging code
- [ ] Documentation updated (if applicable)
- [ ] ADR created (if architectural decision made)
### 2. Create Pull Request
**Title Format:**
- `feat: Add flyer bulk import endpoint`
- `fix: Resolve cache invalidation bug`
- `docs: Update testing guide`
- `refactor: Simplify transaction handling`
**Description Template:**
```markdown
## Summary
Brief description of changes
## Changes Made
- Added X
- Modified Y
- Fixed Z
## Related Issues
Fixes #123
## Testing
- [ ] Unit tests added/updated
- [ ] Integration tests pass
- [ ] Manual testing completed
## Documentation
- [ ] Code comments added where needed
- [ ] ADR created/updated (if applicable)
- [ ] User-facing docs updated
```
### 3. Code Review
- Address all reviewer feedback
- Keep discussions focused and constructive
- Update PR based on feedback
### 4. Merge
- Squash commits if requested
- Ensure CI passes
- Maintainer will merge when approved
## Architecture Decision Records
When making significant architectural decisions:
1. Create ADR in `docs/adr/`
2. Use template from existing ADRs
3. Number sequentially
4. Update `docs/adr/index.md`
**Examples of ADR-worthy decisions:**
- New design patterns
- Technology choices
- API design changes
- Database schema conventions
See [docs/adr/index.md](docs/adr/index.md) for existing decisions.
## Working with AI Agents
This project uses Claude Code with specialized subagents. See:
- [docs/subagents/OVERVIEW.md](docs/subagents/OVERVIEW.md) - Introduction
- [CLAUDE.md](CLAUDE.md) - AI agent instructions
### When to Use Subagents
| Task | Subagent |
| -------------------- | ------------------- |
| Writing code | `coder` |
| Creating tests | `testwriter` |
| Database changes | `db-dev` |
| Container/deployment | `devops` |
| Security review | `security-engineer` |
### Example
```
Use the coder subagent to implement the bulk flyer import endpoint with proper transaction handling and error responses.
```
## Git Conventions
### Commit Messages
Follow conventional commits:
```
feat: Add watchlist price alerts
fix: Resolve duplicate flyer upload bug
docs: Update deployment guide
refactor: Simplify auth middleware
test: Add integration tests for flyer API
```
### Branch Naming
```
feature/watchlist-alerts
fix/duplicate-upload-bug
docs/update-deployment-guide
refactor/auth-middleware
```
## Getting Help
- **Documentation**: Start with [docs/README.md](docs/README.md)
- **Testing Issues**: See [docs/development/TESTING.md](docs/development/TESTING.md)
- **Architecture Questions**: Review [docs/adr/index.md](docs/adr/index.md)
- **Debugging**: Check [docs/development/DEBUGGING.md](docs/development/DEBUGGING.md)
- **AI Agents**: Consult [docs/subagents/OVERVIEW.md](docs/subagents/OVERVIEW.md)
## Code of Conduct
- Be respectful and inclusive
- Welcome newcomers
- Focus on constructive feedback
- Assume good intentions
## License
By contributing, you agree that your contributions will be licensed under the same license as the project.
---
Thank you for contributing to Flyer Crawler! 🎉

View File

@@ -29,6 +29,7 @@ ENV DEBIAN_FRONTEND=noninteractive
# - nginx: for proxying Vite dev server with HTTPS
# - libnss3-tools: required by mkcert for installing CA certificates
# - wget: for downloading mkcert binary
# - tzdata: timezone data required by Bugsink/Django (uses Europe/Amsterdam)
RUN apt-get update && apt-get install -y \
curl \
git \
@@ -44,6 +45,7 @@ RUN apt-get update && apt-get install -y \
nginx \
libnss3-tools \
wget \
tzdata \
&& rm -rf /var/lib/apt/lists/*
# ============================================================================
@@ -52,6 +54,13 @@ RUN apt-get update && apt-get install -y \
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
&& apt-get install -y nodejs
# ============================================================================
# Install PM2 Globally (ADR-014 Parity)
# ============================================================================
# Install PM2 to match production architecture. This allows dev container to
# run the same process management as production (cluster mode, workers, etc.)
RUN npm install -g pm2
# ============================================================================
# Install mkcert and Generate Self-Signed Certificates
# ============================================================================
@@ -61,6 +70,27 @@ RUN wget -O /usr/local/bin/mkcert https://github.com/FiloSottile/mkcert/releases
&& chmod +x /usr/local/bin/mkcert
# Create certificates directory and generate localhost certificates
# ============================================================================
# IMPORTANT: Certificate includes MULTIPLE hostnames (SANs)
# ============================================================================
# The certificate is generated for 'localhost', '127.0.0.1', AND '::1' because:
#
# 1. Users may access the site via https://localhost/ OR https://127.0.0.1/
# 2. Database stores image URLs using one hostname (typically 127.0.0.1)
# 3. The seed script uses https://127.0.0.1 for image URLs (database constraint)
# 4. NGINX is configured to accept BOTH hostnames (see docker/nginx/dev.conf)
#
# Without all hostnames in the certificate's Subject Alternative Names (SANs),
# browsers would show ERR_CERT_AUTHORITY_INVALID when loading images or other
# resources that use a different hostname than the one in the address bar.
#
# The mkcert command below creates a certificate valid for all three:
# - localhost (IPv4 hostname)
# - 127.0.0.1 (IPv4 address)
# - ::1 (IPv6 loopback)
#
# See also: docker/nginx/dev.conf, docs/FLYER-URL-CONFIGURATION.md
# ============================================================================
RUN mkdir -p /app/certs \
&& cd /app/certs \
&& mkcert -install \
@@ -147,6 +177,9 @@ ALLOWED_HOSTS = deduce_allowed_hosts(BUGSINK["BASE_URL"])\n\
\n\
# Console email backend for dev\n\
EMAIL_BACKEND = "bugsink.email_backends.QuietConsoleEmailBackend"\n\
\n\
# HTTPS proxy support (nginx reverse proxy on port 8443)\n\
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")\n\
' > /opt/bugsink/conf/bugsink_conf.py
# Create Bugsink startup script
@@ -206,119 +239,25 @@ exec /opt/bugsink/bin/gunicorn \\\n\
&& chmod +x /usr/local/bin/start-bugsink.sh
# ============================================================================
# Create Logstash Pipeline Configuration
# Copy Logstash Pipeline Configuration
# ============================================================================
# ADR-015: Pino and Redis logs → Bugsink
# ADR-015 + ADR-050: Multi-source log aggregation to Bugsink
# Configuration file includes:
# - Pino application logs (Backend API errors)
# - PostgreSQL logs (including fn_log() structured output)
# - NGINX access and error logs
# See docker/logstash/bugsink.conf for full configuration
RUN mkdir -p /etc/logstash/conf.d /app/logs
RUN echo 'input {\n\
# Pino application logs\n\
file {\n\
path => "/app/logs/*.log"\n\
codec => json\n\
type => "pino"\n\
tags => ["app"]\n\
start_position => "beginning"\n\
sincedb_path => "/var/lib/logstash/sincedb_pino"\n\
}\n\
\n\
# Redis logs\n\
file {\n\
path => "/var/log/redis/*.log"\n\
type => "redis"\n\
tags => ["redis"]\n\
start_position => "beginning"\n\
sincedb_path => "/var/lib/logstash/sincedb_redis"\n\
}\n\
\n\
# PostgreSQL function logs (ADR-050)\n\
file {\n\
path => "/var/log/postgresql/*.log"\n\
type => "postgres"\n\
tags => ["postgres", "database"]\n\
start_position => "beginning"\n\
sincedb_path => "/var/lib/logstash/sincedb_postgres"\n\
}\n\
}\n\
\n\
filter {\n\
# Pino error detection (level 50 = error, 60 = fatal)\n\
if [type] == "pino" and [level] >= 50 {\n\
mutate { add_tag => ["error"] }\n\
}\n\
\n\
# Redis log parsing\n\
if [type] == "redis" {\n\
grok {\n\
match => { "message" => "%%{POSINT:pid}:%%{WORD:role} %%{MONTHDAY} %%{MONTH} %%{TIME} %%{WORD:loglevel} %%{GREEDYDATA:redis_message}" }\n\
}\n\
\n\
# Tag errors (WARNING/ERROR) for Bugsink forwarding\n\
if [loglevel] in ["WARNING", "ERROR"] {\n\
mutate { add_tag => ["error"] }\n\
}\n\
# Tag INFO-level operational events (startup, config, persistence)\n\
else if [loglevel] == "INFO" {\n\
mutate { add_tag => ["redis_operational"] }\n\
}\n\
}\n\
\n\
# PostgreSQL function log parsing (ADR-050)\n\
if [type] == "postgres" {\n\
# Extract timestamp and process ID from PostgreSQL log prefix\n\
# Format: "2026-01-18 10:30:00 PST [12345] user@database "\n\
grok {\n\
match => { "message" => "%%{TIMESTAMP_ISO8601:pg_timestamp} \\\\[%%{POSINT:pg_pid}\\\\] %%{USERNAME:pg_user}@%%{WORD:pg_database} %%{GREEDYDATA:pg_message}" }\n\
}\n\
\n\
# Check if this is a structured JSON log from fn_log()\n\
# fn_log() emits JSON like: {"timestamp":"...","level":"WARNING","source":"postgresql","function":"award_achievement",...}\n\
if [pg_message] =~ /^\\{.*"source":"postgresql".*\\}$/ {\n\
json {\n\
source => "pg_message"\n\
target => "fn_log"\n\
}\n\
\n\
# Mark as error if level is WARNING or ERROR\n\
if [fn_log][level] in ["WARNING", "ERROR"] {\n\
mutate { add_tag => ["error", "db_function"] }\n\
}\n\
}\n\
\n\
# Also catch native PostgreSQL errors\n\
if [pg_message] =~ /^ERROR:/ or [pg_message] =~ /^FATAL:/ {\n\
mutate { add_tag => ["error", "postgres_native"] }\n\
}\n\
}\n\
}\n\
\n\
output {\n\
# Forward errors to Bugsink\n\
if "error" in [tags] {\n\
http {\n\
url => "http://localhost:8000/api/store/"\n\
http_method => "post"\n\
format => "json"\n\
}\n\
}\n\
\n\
# Store Redis operational logs (INFO level) to file\n\
if "redis_operational" in [tags] {\n\
file {\n\
path => "/var/log/logstash/redis-operational-%%{+YYYY-MM-dd}.log"\n\
codec => json_lines\n\
}\n\
}\n\
\n\
# Debug output (comment out in production)\n\
stdout { codec => rubydebug }\n\
}\n\
' > /etc/logstash/conf.d/bugsink.conf
COPY docker/logstash/bugsink.conf /etc/logstash/conf.d/bugsink.conf
# Create Logstash directories
RUN mkdir -p /var/lib/logstash && chown -R logstash:logstash /var/lib/logstash
RUN mkdir -p /var/log/logstash && chown -R logstash:logstash /var/log/logstash
# Create PM2 log directory (ADR-014 Parity)
# Logs written here will be picked up by Logstash (ADR-050)
RUN mkdir -p /var/log/pm2
# ============================================================================
# Configure Nginx
# ============================================================================
@@ -340,8 +279,7 @@ WORKDIR /app
COPY package*.json ./
# Install all dependencies (including devDependencies for development)
# Use --legacy-peer-deps due to react-joyride peer dependency conflict with React 19
RUN npm install --legacy-peer-deps
RUN npm install
# ============================================================================
# Environment Configuration
@@ -371,6 +309,33 @@ ENV BUGSINK_ADMIN_PASSWORD=admin
# 8000 - Bugsink error tracking
EXPOSE 80 443 3001 8000
# ============================================================================
# Copy Application Code and Scripts
# ============================================================================
# Copy the scripts directory which contains the entrypoint script
COPY scripts/ /app/scripts/
# ============================================================================
# Fix Line Endings for Windows Compatibility
# ============================================================================
# Convert ALL text files from CRLF to LF (Windows to Unix)
# This ensures compatibility when building on Windows hosts
# We process: shell scripts, JS/TS files, JSON, config files, etc.
RUN find /app -type f \( \
-name "*.sh" -o \
-name "*.js" -o \
-name "*.ts" -o \
-name "*.tsx" -o \
-name "*.jsx" -o \
-name "*.json" -o \
-name "*.conf" -o \
-name "*.config" -o \
-name "*.yml" -o \
-name "*.yaml" \
\) -exec sed -i 's/\r$//' {} \; && \
find /etc/nginx -type f -name "*.conf" -exec sed -i 's/\r$//' {} \; && \
chmod +x /app/scripts/*.sh
# ============================================================================
# Default Command
# ============================================================================

View File

@@ -34,32 +34,80 @@ Flyer Crawler is a web application that uses Google Gemini AI to extract, analyz
## Quick Start
### Development with Podman Containers
```bash
# Install dependencies
# 1. Start PostgreSQL and Redis containers
podman start flyer-crawler-postgres flyer-crawler-redis
# 2. Install dependencies (first time only)
npm install
# Run in development mode
# 3. Run in development mode
npm run dev
```
See [INSTALL.md](INSTALL.md) for detailed setup instructions.
The application will be available at:
- **Frontend**: http://localhost:5173
- **Backend API**: http://localhost:3001
See [docs/getting-started/INSTALL.md](docs/getting-started/INSTALL.md) for detailed setup instructions including:
- Podman Desktop installation
- Container configuration
- Database initialization
- Environment variables
### Testing
**IMPORTANT**: All tests must run inside the dev container for reliable results.
```bash
# Run all tests in container
podman exec -it flyer-crawler-dev npm test
# Run only unit tests
podman exec -it flyer-crawler-dev npm run test:unit
# Run only integration tests
podman exec -it flyer-crawler-dev npm run test:integration
```
See [docs/development/TESTING.md](docs/development/TESTING.md) for testing guidelines.
---
## Documentation
| Document | Description |
| -------------------------------------- | ---------------------------------------- |
| [INSTALL.md](INSTALL.md) | Local development setup with Podman |
| [DATABASE.md](DATABASE.md) | PostgreSQL setup, schema, and extensions |
| [AUTHENTICATION.md](AUTHENTICATION.md) | OAuth configuration (Google, GitHub) |
| [DEPLOYMENT.md](DEPLOYMENT.md) | Production server setup, NGINX, PM2 |
### Core Documentation
| Document | Description |
| --------------------------------------------------------- | --------------------------------------- |
| [📚 Documentation Index](docs/README.md) | Navigate all documentation |
| [⚙️ Installation Guide](docs/getting-started/INSTALL.md) | Local development setup with Podman |
| [🏗️ Architecture Overview](docs/architecture/DATABASE.md) | System design, database, authentication |
| [💻 Development Guide](docs/development/TESTING.md) | Testing, debugging, code patterns |
| [🚀 Deployment Guide](docs/operations/DEPLOYMENT.md) | Production setup, NGINX, PM2 |
| [🤖 AI Agent Guides](docs/subagents/OVERVIEW.md) | Working with Claude Code subagents |
### Quick References
| Document | Description |
| -------------------------------------------------- | -------------------------------- |
| [CLAUDE.md](CLAUDE.md) | AI agent project instructions |
| [CONTRIBUTING.md](CONTRIBUTING.md) | Development workflow, PR process |
| [Architecture Decision Records](docs/adr/index.md) | Design decisions and rationale |
---
## Environment Variables
This project uses environment variables for configuration (no `.env` files). Key variables:
**Production/Test**: Uses Gitea CI/CD secrets injected during deployment (no local `.env` files)
**Dev Container**: Uses `.env.local` file which **overrides** the default DSNs in `compose.dev.yml`
Key variables:
| Variable | Description |
| -------------------------------------------- | -------------------------------- |

View File

@@ -1,3 +0,0 @@
using powershell on win10 use this command to run the integration tests only in the container
podman exec -i flyer-crawler-dev npm run test:integration 2>&1 | Tee-Object -FilePath test-output.txt

View File

@@ -1,303 +0,0 @@
# Flyer Crawler - Development Environment Setup
Quick start guide for getting the development environment running with Podman containers.
## Prerequisites
- **Windows with WSL 2**: Install WSL 2 by running `wsl --install` in an administrator PowerShell
- **Podman Desktop**: Download and install [Podman Desktop for Windows](https://podman-desktop.io/)
- **Node.js 20+**: Required for running the application
## Quick Start - Container Environment
### 1. Initialize Podman
```powershell
# Start Podman machine (do this once after installing Podman Desktop)
podman machine init
podman machine start
```
### 2. Start Required Services
Start PostgreSQL (with PostGIS) and Redis containers:
```powershell
# Navigate to project directory
cd D:\gitea\flyer-crawler.projectium.com\flyer-crawler.projectium.com
# Start PostgreSQL with PostGIS
podman run -d \
--name flyer-crawler-postgres \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=flyer_crawler_dev \
-p 5432:5432 \
docker.io/postgis/postgis:15-3.3
# Start Redis
podman run -d \
--name flyer-crawler-redis \
-e REDIS_PASSWORD="" \
-p 6379:6379 \
docker.io/library/redis:alpine
```
### 3. Wait for PostgreSQL to Initialize
```powershell
# Wait a few seconds, then check if PostgreSQL is ready
podman exec flyer-crawler-postgres pg_isready -U postgres
# Should output: /var/run/postgresql:5432 - accepting connections
```
### 4. Install Required PostgreSQL Extensions
```powershell
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS pg_trgm; CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";"
```
### 5. Apply Database Schema
```powershell
# Apply the complete schema with URL constraints enabled
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/master_schema_rollup.sql
```
### 6. Verify URL Constraints Are Enabled
```powershell
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "\d public.flyers" | grep -E "(image_url|icon_url|Check)"
```
You should see:
```
image_url | text | | not null |
icon_url | text | | not null |
Check constraints:
"flyers_icon_url_check" CHECK (icon_url ~* '^https?://.*'::text)
"flyers_image_url_check" CHECK (image_url ~* '^https?://.*'::text)
```
### 7. Set Environment Variables and Start Application
```powershell
# Set required environment variables
$env:NODE_ENV="development"
$env:DB_HOST="localhost"
$env:DB_USER="postgres"
$env:DB_PASSWORD="postgres"
$env:DB_NAME="flyer_crawler_dev"
$env:REDIS_URL="redis://localhost:6379"
$env:PORT="3001"
$env:FRONTEND_URL="http://localhost:5173"
# Install dependencies (first time only)
npm install
# Start the development server (runs both backend and frontend)
npm run dev
```
The application will be available at:
- **Frontend**: http://localhost:5173
- **Backend API**: http://localhost:3001
## Managing Containers
### View Running Containers
```powershell
podman ps
```
### Stop Containers
```powershell
podman stop flyer-crawler-postgres flyer-crawler-redis
```
### Start Containers (After They've Been Created)
```powershell
podman start flyer-crawler-postgres flyer-crawler-redis
```
### Remove Containers (Clean Slate)
```powershell
podman stop flyer-crawler-postgres flyer-crawler-redis
podman rm flyer-crawler-postgres flyer-crawler-redis
```
### View Container Logs
```powershell
podman logs flyer-crawler-postgres
podman logs flyer-crawler-redis
```
## Database Management
### Connect to PostgreSQL
```powershell
podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev
```
### Reset Database Schema
```powershell
# Drop all tables
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/drop_tables.sql
# Reapply schema
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/master_schema_rollup.sql
```
### Seed Development Data
```powershell
npm run db:reset:dev
```
## Running Tests
### Unit Tests
```powershell
npm run test:unit
```
### Integration Tests
**IMPORTANT**: Integration tests require the PostgreSQL and Redis containers to be running.
```powershell
# Make sure containers are running
podman ps
# Run integration tests
npm run test:integration
```
## Troubleshooting
### Podman Machine Issues
If you get "unable to connect to Podman socket" errors:
```powershell
podman machine start
```
### PostgreSQL Connection Refused
Make sure PostgreSQL is ready:
```powershell
podman exec flyer-crawler-postgres pg_isready -U postgres
```
### Port Already in Use
If ports 5432 or 6379 are already in use, you can either:
1. Stop the conflicting service
2. Change the port mapping when creating containers (e.g., `-p 5433:5432`)
### URL Validation Errors
The database now enforces URL constraints. All `image_url` and `icon_url` fields must:
- Start with `http://` or `https://`
- Match the regex pattern: `^https?://.*`
Make sure the `FRONTEND_URL` environment variable is set correctly to avoid URL validation errors.
## ADR Implementation Status
This development environment implements:
- **ADR-0002**: Transaction Management ✅
- All database operations use the `withTransaction` pattern
- Automatic rollback on errors
- No connection pool leaks
- **ADR-0003**: Input Validation ✅
- Zod schemas for URL validation
- Database constraints enabled
- Validation at API boundaries
## Development Workflow
1. **Start Containers** (once per development session)
```powershell
podman start flyer-crawler-postgres flyer-crawler-redis
```
2. **Start Application**
```powershell
npm run dev
```
3. **Make Changes** to code (auto-reloads via `tsx watch`)
4. **Run Tests** before committing
```powershell
npm run test:unit
npm run test:integration
```
5. **Stop Application** (Ctrl+C)
6. **Stop Containers** (optional, or leave running)
```powershell
podman stop flyer-crawler-postgres flyer-crawler-redis
```
## PM2 Worker Setup (Production-like)
To test with PM2 workers locally:
```powershell
# Install PM2 globally (once)
npm install -g pm2
# Start the worker
pm2 start npm --name "flyer-crawler-worker" -- run worker:prod
# View logs
pm2 logs flyer-crawler-worker
# Stop worker
pm2 stop flyer-crawler-worker
pm2 delete flyer-crawler-worker
```
## Next Steps
After getting the environment running:
1. Review [docs/adr/](docs/adr/) for architectural decisions
2. Check [sql/master_schema_rollup.sql](sql/master_schema_rollup.sql) for database schema
3. Explore [src/routes/](src/routes/) for API endpoints
4. Review [src/types.ts](src/types.ts) for TypeScript type definitions
## Common Environment Variables
Create these environment variables for development:
```powershell
# Database
$env:DB_HOST="localhost"
$env:DB_USER="postgres"
$env:DB_PASSWORD="postgres"
$env:DB_NAME="flyer_crawler_dev"
$env:DB_PORT="5432"
# Redis
$env:REDIS_URL="redis://localhost:6379"
# Application
$env:NODE_ENV="development"
$env:PORT="3001"
$env:FRONTEND_URL="http://localhost:5173"
# Authentication (generate your own secrets)
$env:JWT_SECRET="your-dev-jwt-secret-change-this"
$env:SESSION_SECRET="your-dev-session-secret-change-this"
# AI Services (get your own API keys)
$env:VITE_GOOGLE_GENAI_API_KEY="your-google-genai-api-key"
$env:GOOGLE_MAPS_API_KEY="your-google-maps-api-key"
```
## Resources
- [Podman Desktop Documentation](https://podman-desktop.io/docs)
- [PostGIS Documentation](https://postgis.net/documentation/)
- [Original README.md](README.md) for production setup

105
certs/README.md Normal file
View File

@@ -0,0 +1,105 @@
# Development SSL Certificates
This directory contains SSL certificates for the development container HTTPS setup.
## Files
| File | Purpose | Generated By |
| --------------- | ---------------------------------------------------- | -------------------------- |
| `localhost.crt` | SSL certificate for localhost and 127.0.0.1 | mkcert (in Dockerfile.dev) |
| `localhost.key` | Private key for localhost.crt | mkcert (in Dockerfile.dev) |
| `mkcert-ca.crt` | Root CA certificate for trusting mkcert certificates | mkcert |
## Certificate Details
The `localhost.crt` certificate includes the following Subject Alternative Names (SANs):
- `DNS:localhost`
- `IP Address:127.0.0.1`
- `IP Address:::1` (IPv6 localhost)
This allows the development server to be accessed via both `https://localhost/` and `https://127.0.0.1/` without SSL errors.
## Installing the CA Certificate (Recommended)
To avoid SSL certificate warnings in your browser, install the mkcert CA certificate on your system.
### Windows
1. Double-click `mkcert-ca.crt`
2. Click **"Install Certificate..."**
3. Select **"Local Machine"** > Next
4. Select **"Place all certificates in the following store"**
5. Click **Browse** > Select **"Trusted Root Certification Authorities"** > OK
6. Click **Next** > **Finish**
7. Restart your browser
### macOS
```bash
sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain certs/mkcert-ca.crt
```
### Linux
```bash
# Ubuntu/Debian
sudo cp certs/mkcert-ca.crt /usr/local/share/ca-certificates/mkcert-ca.crt
sudo update-ca-certificates
# Fedora/RHEL
sudo cp certs/mkcert-ca.crt /etc/pki/ca-trust/source/anchors/
sudo update-ca-trust
```
### Firefox (All Platforms)
Firefox uses its own certificate store:
1. Open Firefox Settings
2. Search for "Certificates"
3. Click **"View Certificates"**
4. Go to **"Authorities"** tab
5. Click **"Import..."**
6. Select `certs/mkcert-ca.crt`
7. Check **"Trust this CA to identify websites"**
8. Click **OK**
## After Installation
Once the CA certificate is installed:
- Your browser will trust all mkcert certificates without warnings
- Access `https://localhost/` with no security warnings
- Images from `https://127.0.0.1/flyer-images/` will load without SSL errors
## Regenerating Certificates
If you need to regenerate the certificates (e.g., after rebuilding the container):
```bash
# Inside the container
cd /app/certs
mkcert localhost 127.0.0.1 ::1
mv localhost+2.pem localhost.crt
mv localhost+2-key.pem localhost.key
nginx -s reload
# Copy the new CA to the host
podman cp flyer-crawler-dev:/app/certs/mkcert-ca.crt ./certs/mkcert-ca.crt
```
Then reinstall the CA certificate as described above.
## Security Note
**DO NOT** commit the private key (`localhost.key`) to version control in production projects. For this development-only project, the certificates are checked in for convenience since they're only used locally with self-signed certificates.
The certificates in this directory are automatically generated by the Dockerfile.dev and should not be used in production.
## See Also
- [Dockerfile.dev](../Dockerfile.dev) - Certificate generation (line ~69)
- [docker/nginx/dev.conf](../docker/nginx/dev.conf) - NGINX SSL configuration
- [docs/FLYER-URL-CONFIGURATION.md](../docs/FLYER-URL-CONFIGURATION.md) - URL configuration details
- [docs/development/DEBUGGING.md](../docs/development/DEBUGGING.md) - SSL troubleshooting

View File

@@ -1,19 +1,25 @@
-----BEGIN CERTIFICATE-----
MIIDCTCCAfGgAwIBAgIUHhZUK1vmww2wCepWPuVcU6d27hMwDQYJKoZIhvcNAQEL
BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDExODAyMzM0NFoXDTI3MDEx
ODAyMzM0NFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
AAOCAQ8AMIIBCgKCAQEAuUJGtSZzd+ZpLi+efjrkxJJNfVxVz2VLhknNM2WKeOYx
JTK/VaTYq5hrczy6fEUnMhDAJCgEPUFlOK3vn1gFJKNMN8m7arkLVk6PYtrx8CTw
w78Q06FLITr6hR0vlJNpN4MsmGxYwUoUpn1j5JdfZF7foxNAZRiwoopf7ZJxltDu
PIuFjmVZqdzR8c6vmqIqdawx/V6sL9fizZr+CDH3oTsTUirn2qM+1ibBtPDiBvfX
omUsr6MVOcTtvnMvAdy9NfV88qwF7MEWBGCjXkoT1bKCLD8hjn8l7GjRmPcmMFE2
GqWEvfJiFkBK0CgSHYEUwzo0UtVNeQr0k0qkDRub6QIDAQABo1MwUTAdBgNVHQ4E
FgQU5VeD67yFLV0QNYbHaJ6u9cM6UbkwHwYDVR0jBBgwFoAU5VeD67yFLV0QNYbH
aJ6u9cM6UbkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEABueA
8ujAD+yjeP5dTgqQH1G0hlriD5LmlJYnktaLarFU+y+EZlRFwjdORF/vLPwSG+y7
CLty/xlmKKQop70QzQ5jtJcsWzUjww8w1sO3AevfZlIF3HNhJmt51ihfvtJ7DVCv
CNyMeYO0pBqRKwOuhbG3EtJgyV7MF8J25UEtO4t+GzX3jcKKU4pWP+kyLBVfeDU3
MQuigd2LBwBQQFxZdpYpcXVKnAJJlHZIt68ycO1oSBEJO9fIF0CiAlC6ITxjtYtz
oCjd6cCLKMJiC6Zg7t1Q17vGl+FdGyQObSsiYsYO9N3CVaeDdpyGCH0Rfa0+oZzu
a5U9/l1FHlvpX980bw==
MIIEJDCCAoygAwIBAgIQfbdj1KSREvW82wxWZcDRsDANBgkqhkiG9w0BAQsFADBf
MR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExGjAYBgNVBAsMEXJvb3RA
OTIxZjA1YmRkNDk4MSEwHwYDVQQDDBhta2NlcnQgcm9vdEA5MjFmMDViZGQ0OTgw
HhcNMjYwMTIyMjAwMzIwWhcNMjgwNDIyMjAwMzIwWjBFMScwJQYDVQQKEx5ta2Nl
cnQgZGV2ZWxvcG1lbnQgY2VydGlmaWNhdGUxGjAYBgNVBAsMEXJvb3RANzk2OWU1
ZjA2MGM4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2u+HpnTr+Ecj
X6Ur4I8YHVKiEIgOFozJWWAeCPSZg0lr9/z3UDl7QPfRKaDk11OMN7DB0Y/ZAIHj
kMFdYc+ODOUAKGv/MvlM41OeRUXhrt0Gr/TJoDe9RLzs9ffASpqTHUNNrmyj/fLP
M5VZU+4Y2POFq8qji6Otkvqr6wp+2CTS/fkAtenFS2X+Z5u6BBq1/KBSWZhUFSuJ
sAGN9u+l20Cj/Sxp2nD1npvMEvPehFKEK2tZGgFr+X426jJ4Znd19EyZoI/xetG6
ybSzBdQk1KbhWFa3LPuOG814m9Qh21vaL0Hj2hpqC3KEQ2jiuCovjRS+RBGatltl
5m47Cj0UrQIDAQABo3YwdDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYB
BQUHAwEwHwYDVR0jBBgwFoAUjf0NOmUuoqSSD1Mbu/3G+wYxvjEwLAYDVR0RBCUw
I4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3DQEB
CwUAA4IBgQAorThUyu4FxHkOafMp/4CmvptfNvYeYP81DW0wpJxGL68Bz+idvXKv
JopX/ZAr+dDhS/TLeSnzt83W7TaBBHY+usvsiBx9+pZOVZIpreXRamPu7utmuC46
dictMNGlRNX9bwAApOJ24NCpOVSIIKtjHcjl4idwUHqLVGS+3wsmxIILYxigzkuT
fcK5vs0ItZWeuunsBAGb/U/Iu9zZ71rtmBejxNPyEvd8+bZC0m2mtV8C0Lpn58jZ
FiEf5OHiOdWG9O/uh3QeVWkuKLmaH6a8VdKRSIlOxEEZdkUYlwuVvzeWgFw4kjm8
rNWz0aIPovmcgLXoUG1d1D8klveGd3plF7N2p3xWqKmS6R6FJHx7MIxH6XmBATii
x/193Sgzqe8mwXQr14ulg2M/B3ZWleNdD6SeieADSgvRKjnlO7xwUmcFrffnEKEG
WcSPoGIuZ8V2pkYLh7ipPN+tFUbhmrWnsao5kQ0sqLlfscyO9hYQeQRtTjtC3P8r
FJYOdBMOCzE=
-----END CERTIFICATE-----

View File

@@ -1,28 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC5Qka1JnN35mku
L55+OuTEkk19XFXPZUuGSc0zZYp45jElMr9VpNirmGtzPLp8RScyEMAkKAQ9QWU4
re+fWAUko0w3ybtquQtWTo9i2vHwJPDDvxDToUshOvqFHS+Uk2k3gyyYbFjBShSm
fWPkl19kXt+jE0BlGLCiil/tknGW0O48i4WOZVmp3NHxzq+aoip1rDH9Xqwv1+LN
mv4IMfehOxNSKufaoz7WJsG08OIG99eiZSyvoxU5xO2+cy8B3L019XzyrAXswRYE
YKNeShPVsoIsPyGOfyXsaNGY9yYwUTYapYS98mIWQErQKBIdgRTDOjRS1U15CvST
SqQNG5vpAgMBAAECggEAAnv0Dw1Mv+rRy4ZyxtObEVPXPRzoxnDDXzHP4E16BTye
Fc/4pSBUIAUn2bPvLz0/X8bMOa4dlDcIv7Eu9Pvns8AY70vMaUReA80fmtHVD2xX
1PCT0X3InnxRAYKstSIUIGs+aHvV5Z+iJ8F82soOStN1MU56h+JLWElL5deCPHq3
tLZT8wM9aOZlNG72kJ71+DlcViahynQj8+VrionOLNjTJ2Jv/ByjM3GMIuSdBrgd
Sl4YAcdn6ontjJGoTgI+e+qkBAPwMZxHarNGQgbS0yNVIJe7Lq4zIKHErU/ZSmpD
GzhdVNzhrjADNIDzS7G+pxtz+aUxGtmRvOyopy8GAQKBgQDEPp2mRM+uZVVT4e1j
pkKO1c3O8j24I5mGKwFqhhNs3qGy051RXZa0+cQNx63GokXQan9DIXzc/Il7Y72E
z9bCFbcSWnlP8dBIpWiJm+UmqLXRyY4N8ecNnzL5x+Tuxm5Ij+ixJwXgdz/TLNeO
MBzu+Qy738/l/cAYxwcF7mR7AQKBgQDxq1F95HzCxBahRU9OGUO4s3naXqc8xKCC
m3vbbI8V0Exse2cuiwtlPPQWzTPabLCJVvCGXNru98sdeOu9FO9yicwZX0knOABK
QfPyDeITsh2u0C63+T9DNn6ixI/T68bTs7DHawEYbpS7bR50BnbHbQrrOAo6FSXF
yC7+Te+o6QKBgQCXEWSmo/4D0Dn5Usg9l7VQ40GFd3EPmUgLwntal0/I1TFAyiom
gpcLReIogXhCmpSHthO1h8fpDfZ/p+4ymRRHYBQH6uHMKugdpEdu9zVVpzYgArp5
/afSEqVZJwoSzWoELdQA23toqiPV2oUtDdiYFdw5nDccY1RHPp8nb7amAQKBgQDj
f4DhYDxKJMmg21xCiuoDb4DgHoaUYA0xpii8cL9pq4KmBK0nVWFO1kh5Robvsa2m
PB+EfNjkaIPepLxWbOTUEAAASoDU2JT9UoTQcl1GaUAkFnpEWfBB14TyuNMkjinH
lLpvn72SQFbm8VvfoU4jgfTrZP/LmajLPR1v6/IWMQKBgBh9qvOTax/GugBAWNj3
ZvF99rHOx0rfotEdaPcRN66OOiSWILR9yfMsTvwt1V0VEj7OqO9juMRFuIyB57gd
Hs/zgbkuggqjr1dW9r22P/UpzpodAEEN2d52RSX8nkMOkH61JXlH2MyRX65kdExA
VkTDq6KwomuhrU3z0+r/MSOn
MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDa74emdOv4RyNf
pSvgjxgdUqIQiA4WjMlZYB4I9JmDSWv3/PdQOXtA99EpoOTXU4w3sMHRj9kAgeOQ
wV1hz44M5QAoa/8y+UzjU55FReGu3Qav9MmgN71EvOz198BKmpMdQ02ubKP98s8z
lVlT7hjY84WryqOLo62S+qvrCn7YJNL9+QC16cVLZf5nm7oEGrX8oFJZmFQVK4mw
AY3276XbQKP9LGnacPWem8wS896EUoQra1kaAWv5fjbqMnhmd3X0TJmgj/F60brJ
tLMF1CTUpuFYVrcs+44bzXib1CHbW9ovQePaGmoLcoRDaOK4Ki+NFL5EEZq2W2Xm
bjsKPRStAgMBAAECggEBALFhpHwe+xh7OpPBlR0pkpYfXyMZuKBYjMIW9/61frM6
B3oywIWFLPFkV1js/Lvg+xgb48zQSTb6BdBAelJHAYY8+7XEWk2IYt1D4FWr2r/8
X/Cr2bgvsO9CSpK2mltXhZ4N66BIcU3NLkdS178Ch6svErwvP/ZhNL6Czktug3rG
S2fxpKqoVQhqWiEBV0vBWpw2oskvcpP2Btev2eaJeDmP1IkCaKIU9jJzmSg8UKj3
AxJJ8lJvlDi2Z8mfB0BVIFZSI1s44LqbuPdITsWvG49srdAhyLkifcbY2r7eH8+s
rFNswCaqqNwmzZXHMFedfvgVJHtCTGX/U8G55dcG/YECgYEA6PvcX+axT4aSaG+/
fQpyW8TmNmIwS/8HD8rA5dSBB2AE+RnqbxxB1SWWk4w47R/riYOJCy6C88XUZWLn
05cYotR9lA6tZOYqIPUIxfCHl4vDDBnOGAB0ga3z5vF3X7HV6uspPFcrGVJ9k42S
BK1gk7Kj8RxVQMqXSqkR/pXJ4l0CgYEA8JBkZ4MRi1D74SAU+TY1rnUCTs2EnP1U
TKAI9qHyvNJOFZaObWPzegkZryZTm+yTblwvYkryMjvsiuCdv6ro/VKc667F3OBs
dJ/8+ylO0lArP9a23QHViNSvQmyi3bsw2UpIGQRGq2C4LxDceo5cYoVOWIlLl8u3
LUuL0IfxdpECgYEAyMd0DPljyGLyfSoAXaPJFajDtA4+DOAEl+lk/yt43oAzCPD6
hTJW0XcJIrJuxHsDoohGa+pzU90iwxTPMBtAUeLJLfTQHOn1WF2SZ/J3B3ScbCs4
3ppVzQO580YYV9GLxl1ONf/w1muuaKBSO9GmLuJ+QeTm22U7qE23gixXxMkCgYEA
3R7cK4l+huBZpgUnQitiDInhJS4jx2nUItq3YnxZ8tYckBtjr4lAM9xJj4VbNOew
XLC/nUnmdeY+9yif153xq2hUdQ6hMPXYuxqUHwlJOmgWWQez7lHRRYS51ASnb8iw
jgqJWvVjQAQXSKvm/X/9y1FdQmRw54aJSUk3quZKPQECgYA5DqFXn/jYOUCGfCUD
RENWe+3fNZSJCWmr4yvjEy9NUT5VcA+/hZHbbLUnVHfAHxJmdGhTzFe+ZGWFAGwi
Wo62BDu1fqHHegCWaFFluDnz8iZTxXtHMEGSBZnXC4f5wylcxRtCPUbrLMRTPo4O
t85qeBu1dMq1XQtU6ab3w3W0nw==
-----END PRIVATE KEY-----

27
certs/mkcert-ca.crt Normal file
View File

@@ -0,0 +1,27 @@
-----BEGIN CERTIFICATE-----
MIIEjjCCAvagAwIBAgIRAJtq2Z0+W981Ct2dMVPb3bQwDQYJKoZIhvcNAQELBQAw
XzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMRowGAYDVQQLDBFyb290
QDkyMWYwNWJkZDQ5ODEhMB8GA1UEAwwYbWtjZXJ0IHJvb3RAOTIxZjA1YmRkNDk4
MB4XDTI2MDEyMjA5NDcxNVoXDTM2MDEyMjA5NDcxNVowXzEeMBwGA1UEChMVbWtj
ZXJ0IGRldmVsb3BtZW50IENBMRowGAYDVQQLDBFyb290QDkyMWYwNWJkZDQ5ODEh
MB8GA1UEAwwYbWtjZXJ0IHJvb3RAOTIxZjA1YmRkNDk4MIIBojANBgkqhkiG9w0B
AQEFAAOCAY8AMIIBigKCAYEAxXR5gXDwv5cfQSud1eEhwDuaSaf5kf8NtPnucZXY
AN+/QW1+OEKJFuawj/YrSbL/yIB8sUSJToEYNJ4LAgzIZ4+TPYVvOIPqQnimfj98
2AKCt73U/AMvoEpts7U0s37f5wF8o+BHXfChxyN//z96+wsQ+2+Q9QBGjirvScF+
8FRnupcygDeGZ8x3JQVaEfEV6iYyXFl/4tEDVr9QX4avyUlf0vp1Y90TG3L42JYQ
xDU37Ct9dqsxPCPOPjmkQi9HV5TeqLTs/4NdrEYOSk7bOVMzL8EHs2prRL7sWzYJ
gRT+VXFPpoSCkZs1gS3FNXukTGx5LNsstyJZRa99cGgDcqvNseig06KUzZrRnCig
kARLF/n8VTpHETEuTdxdnXJO3i2N/99mG/2/lej9HNDMaqg45ur5EhaFhHarXMtc
wy7nfGoThzscZvpFbVHorhgRxzsUqRHTMHa9mUOYtShMA0IwFccHcczY3CDxXLg9
hC+24pdiCxtRmi23JB10Th+nAgMBAAGjRTBDMA4GA1UdDwEB/wQEAwICBDASBgNV
HRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBSN/Q06ZS6ipJIPUxu7/cb7BjG+MTAN
BgkqhkiG9w0BAQsFAAOCAYEAghQL80XpPs70EbCmETYri5ryjJPbsxK3Z3QgBYoy
/P/021eK5woLx4EDeEUyrRayOtLlDWiIMdV7FmlY4GpEdonVbqFJY2gghkIuQpkw
lWqbk08F+iXe8PSGD1qz4y6enioCRAx+RaZ6jnVJtaC8AR257FVhGIzDBiOA+LDM
L1yq6Bxxij17q9s5HL9KuzxWRMuXACHmaGBXHpl/1n4dIxi2lXRgp+1xCR/VPNPt
/ZRy29kncd8Fxx+VEtc0muoJvRo4ttVWhBvAVJrkAeukjYKpcDDfRU6Y22o54jD/
mseDb+0UPwSxaKbnGJlCcRbbh6i14bA4KfPq93bZX+Tlq9VCp8LvQSl3oU+23RVc
KjBB9EJnoBBNGIY7VmRI+QowrnlP2wtg2fTNaPqULtjqA9frbMTP0xTlumLzGB+6
9Da7/+AE2in3Aa8Xyry4BiXbk2L6c1xz/Cd1ZpFrSXAOEi1Xt7so/Ck0yM3c2KWK
5aSfCjjOzONMPJyY1oxodJ1p
-----END CERTIFICATE-----

View File

@@ -46,11 +46,16 @@ services:
- node_modules_data:/app/node_modules
# Mount PostgreSQL logs for Logstash access (ADR-050)
- postgres_logs:/var/log/postgresql:ro
# Mount Redis logs for Logstash access (ADR-050)
- redis_logs:/var/log/redis:ro
# Mount PM2 logs for Logstash access (ADR-050, ADR-014)
- pm2_logs:/var/log/pm2
ports:
- '80:80' # HTTP redirect to HTTPS (matches production)
- '443:443' # Frontend HTTPS (nginx proxies Vite 5173 → 443)
- '3001:3001' # Backend API
- '8000:8000' # Bugsink error tracking (ADR-015)
- '8000:8000' # Bugsink error tracking HTTP (ADR-015)
- '8443:8443' # Bugsink error tracking HTTPS (ADR-015)
environment:
# Core settings
- NODE_ENV=development
@@ -77,13 +82,16 @@ services:
- BUGSINK_DB_USER=bugsink
- BUGSINK_DB_PASSWORD=bugsink_dev_password
- BUGSINK_PORT=8000
- BUGSINK_BASE_URL=http://localhost:8000
- BUGSINK_BASE_URL=https://localhost:8443
- BUGSINK_ADMIN_EMAIL=admin@localhost
- BUGSINK_ADMIN_PASSWORD=admin
- BUGSINK_SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security
# Sentry SDK configuration (points to local Bugsink)
- SENTRY_DSN=http://59a58583-e869-7697-f94a-cfa0337676a8@localhost:8000/1
- VITE_SENTRY_DSN=http://d5fc5221-4266-ff2f-9af8-5689696072f3@localhost:8000/2
# Sentry SDK configuration (points to local Bugsink HTTP)
# Note: Using HTTP with 127.0.0.1 instead of localhost because Sentry SDK
# doesn't accept 'localhost' as a valid hostname in DSN validation
# The browser accesses Bugsink at http://localhost:8000 (nginx proxies to HTTPS for the app)
- SENTRY_DSN=http://cea01396-c562-46ad-b587-8fa5ee6b1d22@127.0.0.1:8000/1
- VITE_SENTRY_DSN=http://d92663cb-73cf-4145-b677-b84029e4b762@127.0.0.1:8000/2
- SENTRY_ENVIRONMENT=development
- VITE_SENTRY_ENVIRONMENT=development
- SENTRY_ENABLED=true
@@ -162,10 +170,17 @@ services:
redis:
image: docker.io/library/redis:alpine
container_name: flyer-crawler-redis
# Run as root to allow entrypoint to set up log directory permissions
# Redis will drop privileges after setup via gosu in entrypoint
user: root
ports:
- '6379:6379'
volumes:
- redis_data:/data
# Create log volume for Logstash access (ADR-050)
- redis_logs:/var/log/redis
# Mount custom entrypoint for log directory setup (ADR-050)
- ./docker/redis/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh:ro
# Healthcheck ensures redis is ready before app starts
healthcheck:
test: ['CMD', 'redis-cli', 'ping']
@@ -173,8 +188,17 @@ services:
timeout: 5s
retries: 10
start_period: 5s
# Enable persistence for development data
command: redis-server --appendonly yes
# Enable persistence and file logging for Logstash (ADR-050)
# Redis log levels: debug, verbose, notice (default), warning
# Custom entrypoint sets up log directory with correct permissions
entrypoint: ['/usr/local/bin/docker-entrypoint.sh']
command:
- --appendonly
- 'yes'
- --logfile
- /var/log/redis/redis-server.log
- --loglevel
- notice
# ===================
# Named Volumes
@@ -186,6 +210,10 @@ volumes:
name: flyer-crawler-postgres-logs
redis_data:
name: flyer-crawler-redis-data
redis_logs:
name: flyer-crawler-redis-logs
pm2_logs:
name: flyer-crawler-pm2-logs
node_modules_data:
name: flyer-crawler-node-modules

View File

@@ -0,0 +1,461 @@
# docker/logstash/bugsink.conf
# ============================================================================
# Logstash Pipeline Configuration for Bugsink Error Tracking
# ============================================================================
# This configuration aggregates logs from multiple sources and forwards errors
# to Bugsink (Sentry-compatible error tracking) in the dev container.
#
# Sources:
# - PM2 managed logs (/var/log/pm2/*.log) - API, Worker, Vite (ADR-014)
# - Pino application logs (/app/logs/*.log) - JSON format (fallback)
# - PostgreSQL logs (/var/log/postgresql/*.log) - Including fn_log() output
# - NGINX logs (/var/log/nginx/*.log) - Access and error logs
# - Redis logs (/var/log/redis/*.log) - Via shared volume (ADR-050)
#
# Bugsink Projects:
# - Project 1: Backend API (Dev) - Pino errors, PostgreSQL errors
# - Project 2: Frontend (Dev) - Configured via Sentry SDK in browser
#
# Related Documentation:
# - docs/adr/0050-postgresql-function-observability.md
# - docs/adr/0015-application-performance-monitoring-and-error-tracking.md
# - docs/operations/LOGSTASH-QUICK-REF.md
# ============================================================================
input {
# ============================================================================
# PM2 Managed Process Logs (ADR-014, ADR-050)
# ============================================================================
# PM2 manages all Node.js processes in the dev container, matching production.
# Logs are written to /var/log/pm2 for Logstash integration.
#
# Process logs:
# - api-out.log / api-error.log: API server (Pino JSON format)
# - worker-out.log / worker-error.log: Background worker (Pino JSON format)
# - vite-out.log / vite-error.log: Vite dev server (plain text)
# ============================================================================
# PM2 API Server Logs (Pino JSON format)
file {
path => "/var/log/pm2/api-out.log"
codec => json_lines
type => "pm2_api"
tags => ["app", "backend", "pm2", "api"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_api_out"
}
file {
path => "/var/log/pm2/api-error.log"
codec => json_lines
type => "pm2_api"
tags => ["app", "backend", "pm2", "api", "stderr"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_api_error"
}
# PM2 Worker Logs (Pino JSON format)
file {
path => "/var/log/pm2/worker-out.log"
codec => json_lines
type => "pm2_worker"
tags => ["app", "worker", "pm2"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_worker_out"
}
file {
path => "/var/log/pm2/worker-error.log"
codec => json_lines
type => "pm2_worker"
tags => ["app", "worker", "pm2", "stderr"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_worker_error"
}
# PM2 Vite Logs (plain text format)
file {
path => "/var/log/pm2/vite-out.log"
codec => plain
type => "pm2_vite"
tags => ["app", "frontend", "pm2", "vite"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_vite_out"
}
file {
path => "/var/log/pm2/vite-error.log"
codec => plain
type => "pm2_vite"
tags => ["app", "frontend", "pm2", "vite", "stderr"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pm2_vite_error"
}
# ============================================================================
# Pino Application Logs (Fallback Path)
# ============================================================================
# JSON-formatted logs from the Node.js application using Pino logger.
# Note: Primary logs now go to /var/log/pm2. This is a fallback.
# Log levels: 10=trace, 20=debug, 30=info, 40=warn, 50=error, 60=fatal
file {
path => "/app/logs/*.log"
codec => json
type => "pino"
tags => ["app", "backend"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_pino"
}
# ============================================================================
# PostgreSQL Function Logs (ADR-050)
# ============================================================================
# Captures PostgreSQL log output including fn_log() structured JSON messages.
# PostgreSQL is configured to write logs to /var/log/postgresql/ (shared volume).
# Log format: "2026-01-22 00:00:00 UTC [5724] postgres@flyer_crawler_dev LOG: message"
file {
path => "/var/log/postgresql/*.log"
type => "postgres"
tags => ["postgres", "database"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_postgres"
}
# ============================================================================
# NGINX Logs
# ============================================================================
# Access logs for request monitoring and error logs for proxy/SSL issues.
file {
path => "/var/log/nginx/access.log"
type => "nginx_access"
tags => ["nginx", "access"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_nginx_access"
}
file {
path => "/var/log/nginx/error.log"
type => "nginx_error"
tags => ["nginx", "error"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_nginx_error"
}
# ============================================================================
# Redis Logs (ADR-050)
# ============================================================================
# Redis logs from the shared volume. Redis uses its own log format:
# "<pid>:<role> <day> <month> <time> <loglevel> <message>"
# Example: "1:M 22 Jan 2026 14:30:00.123 * Ready to accept connections"
# Log levels: . (debug), - (verbose), * (notice), # (warning)
file {
path => "/var/log/redis/redis-server.log"
type => "redis"
tags => ["redis", "infra"]
start_position => "beginning"
sincedb_path => "/var/lib/logstash/sincedb_redis"
}
}
filter {
# ============================================================================
# PM2 API/Worker Log Processing (ADR-014)
# ============================================================================
# PM2 API and Worker logs are in Pino JSON format.
# Process them the same way as direct Pino logs.
if [type] in ["pm2_api", "pm2_worker"] {
# Tag errors (level 50 = error, 60 = fatal)
if [level] >= 50 {
mutate { add_tag => ["error", "pm2_pino_error"] }
# Map Pino level to Sentry level and set error_message field
if [level] == 60 {
mutate { add_field => { "sentry_level" => "fatal" } }
} else {
mutate { add_field => { "sentry_level" => "error" } }
}
# Copy msg to error_message for consistent access in output
mutate { add_field => { "error_message" => "%{msg}" } }
}
}
# ============================================================================
# PM2 Vite Log Processing (ADR-014)
# ============================================================================
# Vite logs are plain text. Look for error patterns.
if [type] == "pm2_vite" {
# Detect Vite build/compilation errors
if [message] =~ /(?i)(error|failed|exception|cannot|enoent|eperm|EACCES|ECONNREFUSED)/ {
mutate { add_tag => ["error", "vite_error"] }
mutate { add_field => { "sentry_level" => "error" } }
mutate { add_field => { "error_message" => "Vite: %{message}" } }
}
}
# ============================================================================
# Pino Log Processing (Fallback)
# ============================================================================
if [type] == "pino" {
# Tag errors (level 50 = error, 60 = fatal)
if [level] >= 50 {
mutate { add_tag => ["error", "pino_error"] }
# Map Pino level to Sentry level and set error_message field
if [level] == 60 {
mutate { add_field => { "sentry_level" => "fatal" } }
} else {
mutate { add_field => { "sentry_level" => "error" } }
}
# Copy msg to error_message for consistent access in output
mutate { add_field => { "error_message" => "%{msg}" } }
}
}
# ============================================================================
# PostgreSQL Log Processing (ADR-050)
# ============================================================================
# PostgreSQL log format in dev container:
# "2026-01-22 00:00:00 UTC [5724] postgres@flyer_crawler_dev LOG: message"
# "2026-01-22 07:06:03 UTC [19851] postgres@flyer_crawler_dev ERROR: column "id" does not exist"
if [type] == "postgres" {
# Parse PostgreSQL log prefix with UTC timezone
grok {
match => { "message" => "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME} %{WORD:pg_timezone} \[%{POSINT:pg_pid}\] %{DATA:pg_user}@%{DATA:pg_database} %{WORD:pg_level}: ?%{GREEDYDATA:pg_message}" }
tag_on_failure => ["_postgres_grok_failure"]
}
# Check if this is a structured JSON log from fn_log()
if [pg_message] =~ /^\{.*"source":"postgresql".*\}$/ {
json {
source => "pg_message"
target => "fn_log"
}
# Mark as error if level is WARNING or ERROR
if [fn_log][level] in ["WARNING", "ERROR"] {
mutate { add_tag => ["error", "db_function"] }
mutate { add_field => { "sentry_level" => "warning" } }
if [fn_log][level] == "ERROR" {
mutate { replace => { "sentry_level" => "error" } }
}
# Use fn_log message for error_message
mutate { add_field => { "error_message" => "%{[fn_log][message]}" } }
}
}
# Catch native PostgreSQL errors (ERROR: or FATAL: prefix in pg_level)
if [pg_level] == "ERROR" or [pg_level] == "FATAL" {
mutate { add_tag => ["error", "postgres_native"] }
if [pg_level] == "FATAL" {
mutate { add_field => { "sentry_level" => "fatal" } }
} else {
mutate { add_field => { "sentry_level" => "error" } }
}
# Use the full pg_message for error_message
mutate { add_field => { "error_message" => "PostgreSQL %{pg_level}: %{pg_message}" } }
}
}
# ============================================================================
# NGINX Access Log Processing
# ============================================================================
if [type] == "nginx_access" {
grok {
match => { "message" => '%{IPORHOST:client_ip} - %{DATA:user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:user_agent}"' }
tag_on_failure => ["_nginx_access_grok_failure"]
}
# Tag 5xx errors for Bugsink
if [status] =~ /^5/ {
mutate { add_tag => ["error", "nginx_5xx"] }
mutate { add_field => { "sentry_level" => "error" } }
mutate { add_field => { "error_message" => "HTTP %{status} %{method} %{request}" } }
}
}
# ============================================================================
# NGINX Error Log Processing
# ============================================================================
# NGINX error log format: "2026/01/22 17:55:01 [error] 16#16: *3 message..."
if [type] == "nginx_error" {
grok {
match => { "message" => "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME} \[%{LOGLEVEL:nginx_level}\] %{POSINT:pid}#%{POSINT}: (\*%{POSINT:connection} )?%{GREEDYDATA:nginx_message}" }
tag_on_failure => ["_nginx_error_grok_failure"]
}
# Only process actual errors, not notices (like "signal process started")
if [nginx_level] in ["error", "crit", "alert", "emerg"] {
mutate { add_tag => ["error", "nginx_error"] }
# Map NGINX log level to Sentry level
if [nginx_level] in ["crit", "alert", "emerg"] {
mutate { add_field => { "sentry_level" => "fatal" } }
} else {
mutate { add_field => { "sentry_level" => "error" } }
}
mutate { add_field => { "error_message" => "NGINX [%{nginx_level}]: %{nginx_message}" } }
}
}
# ============================================================================
# Redis Log Processing (ADR-050)
# ============================================================================
# Redis log format: "<pid>:<role> <day> <month> <time>.<ms> <level> <message>"
# Example: "1:M 22 Jan 14:30:00.123 * Ready to accept connections"
# Roles: M=master, S=slave, C=sentinel, X=cluster
# Levels: . (debug), - (verbose), * (notice), # (warning)
if [type] == "redis" {
# Parse Redis log format
grok {
match => { "message" => "%{POSINT:redis_pid}:%{WORD:redis_role} %{MONTHDAY} %{MONTH} %{YEAR}? ?%{TIME} %{DATA:redis_level_char} %{GREEDYDATA:redis_message}" }
tag_on_failure => ["_redis_grok_failure"]
}
# Map Redis level characters to human-readable levels
# . = debug, - = verbose, * = notice, # = warning
if [redis_level_char] == "#" {
mutate {
add_field => { "redis_level" => "warning" }
add_tag => ["error", "redis_warning"]
}
mutate { add_field => { "sentry_level" => "warning" } }
mutate { add_field => { "error_message" => "Redis WARNING: %{redis_message}" } }
} else if [redis_level_char] == "*" {
mutate { add_field => { "redis_level" => "notice" } }
} else if [redis_level_char] == "-" {
mutate { add_field => { "redis_level" => "verbose" } }
} else if [redis_level_char] == "." {
mutate { add_field => { "redis_level" => "debug" } }
}
# Also detect error keywords in message content (e.g., ECONNREFUSED, OOM, etc.)
if [redis_message] =~ /(?i)(error|failed|refused|denied|timeout|oom|crash|fatal|exception)/ {
if "error" not in [tags] {
mutate { add_tag => ["error", "redis_error"] }
mutate { add_field => { "sentry_level" => "error" } }
mutate { add_field => { "error_message" => "Redis ERROR: %{redis_message}" } }
}
}
}
# ============================================================================
# Generate Sentry Event ID for all errors
# ============================================================================
if "error" in [tags] {
ruby {
code => '
require "securerandom"
event.set("sentry_event_id", SecureRandom.hex(16))
'
}
# Ensure error_message has a fallback value
if ![error_message] {
mutate { add_field => { "error_message" => "%{message}" } }
}
}
}
output {
# ============================================================================
# Forward Errors to Bugsink (Backend API Project)
# ============================================================================
# Bugsink uses Sentry-compatible API. Events must include:
# - event_id: 32 hex characters (UUID without dashes)
# - message: Human-readable error description
# - level: fatal, error, warning, info, debug
# - timestamp: Unix epoch in seconds
# - platform: "node" for backend, "javascript" for frontend
#
# Authentication via X-Sentry-Auth header with project's public key.
# Dev container DSN: http://cea01396c56246adb5878fa5ee6b1d22@localhost:8000/1
# ============================================================================
if "error" in [tags] {
http {
url => "http://localhost:8000/api/1/store/"
http_method => "post"
format => "json"
headers => {
"X-Sentry-Auth" => "Sentry sentry_key=cea01396c56246adb5878fa5ee6b1d22, sentry_version=7"
"Content-Type" => "application/json"
}
# Transform event to Sentry format using regular fields (not @metadata)
mapping => {
"event_id" => "%{sentry_event_id}"
"timestamp" => "%{@timestamp}"
"level" => "%{sentry_level}"
"platform" => "node"
"logger" => "%{type}"
"message" => "%{error_message}"
"extra" => {
"hostname" => "%{[host][name]}"
"source_type" => "%{type}"
"tags" => "%{tags}"
"original_message" => "%{message}"
}
}
}
}
# ============================================================================
# Store Operational Logs to Files (for debugging/audit)
# ============================================================================
# NGINX access logs (all requests, not just errors)
if [type] == "nginx_access" and "error" not in [tags] {
file {
path => "/var/log/logstash/nginx-access-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# PostgreSQL operational logs (non-error)
if [type] == "postgres" and "error" not in [tags] {
file {
path => "/var/log/logstash/postgres-operational-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# Redis operational logs (non-error)
if [type] == "redis" and "error" not in [tags] {
file {
path => "/var/log/logstash/redis-operational-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# PM2 API operational logs (non-error) - ADR-014
if [type] == "pm2_api" and "error" not in [tags] {
file {
path => "/var/log/logstash/pm2-api-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# PM2 Worker operational logs (non-error) - ADR-014
if [type] == "pm2_worker" and "error" not in [tags] {
file {
path => "/var/log/logstash/pm2-worker-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# PM2 Vite operational logs (non-error) - ADR-014
if [type] == "pm2_vite" and "error" not in [tags] {
file {
path => "/var/log/logstash/pm2-vite-%{+YYYY-MM-dd}.log"
codec => json_lines
}
}
# ============================================================================
# Debug Output (for development only)
# ============================================================================
# Uncomment to see all processed events in Logstash stdout
# stdout { codec => rubydebug }
}

View File

@@ -9,13 +9,39 @@
# - Frontend accessible on https://localhost (port 443)
# - Backend API on http://localhost:3001
# - Port 80 redirects to HTTPS
#
# IMPORTANT: Dual Hostname Configuration (localhost AND 127.0.0.1)
# ============================================================================
# The server_name directive includes BOTH 'localhost' and '127.0.0.1' to
# prevent SSL certificate errors when resources use different hostnames.
#
# Problem Scenario:
# 1. User accesses site via https://localhost/
# 2. Database stores image URLs as https://127.0.0.1/flyer-images/...
# 3. Browser treats these as different origins, showing ERR_CERT_AUTHORITY_INVALID
#
# Solution:
# - mkcert generates certificates valid for: localhost, 127.0.0.1, ::1
# - NGINX accepts requests to BOTH hostnames using the same certificate
# - Users can access via either hostname without SSL warnings
#
# The self-signed certificate is generated in Dockerfile.dev with:
# mkcert localhost 127.0.0.1 ::1
#
# This creates a certificate with Subject Alternative Names (SANs) for all
# three hostnames, allowing NGINX to serve valid HTTPS for each.
#
# See also:
# - Dockerfile.dev (certificate generation, ~line 69)
# - docs/FLYER-URL-CONFIGURATION.md (URL configuration details)
# - docs/development/DEBUGGING.md (SSL troubleshooting)
# ============================================================================
# HTTPS Server (main)
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name localhost;
server_name localhost 127.0.0.1;
# SSL Configuration (self-signed certificates from mkcert)
ssl_certificate /app/certs/localhost.crt;
@@ -24,7 +50,36 @@ server {
# Allow large file uploads (matches production)
client_max_body_size 100M;
# Proxy all requests to Vite dev server on port 5173
# Proxy API requests to Express server on port 3001
location /api/ {
proxy_pass http://localhost:3001;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# Proxy WebSocket connections for real-time notifications
location /ws {
proxy_pass http://localhost:3001;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# Serve flyer images from static storage
location /flyer-images/ {
alias /app/public/flyer-images/;
expires 7d;
add_header Cache-Control "public, immutable";
}
# Proxy all other requests to Vite dev server on port 5173
location / {
proxy_pass http://localhost:5173;
proxy_http_version 1.1;
@@ -51,7 +106,7 @@ server {
server {
listen 80;
listen [::]:80;
server_name localhost;
server_name localhost 127.0.0.1;
return 301 https://$host$request_uri;
}

View File

@@ -0,0 +1,24 @@
#!/bin/sh
# docker/redis/docker-entrypoint.sh
# ============================================================================
# Redis Entrypoint Script for Dev Container
# ============================================================================
# This script ensures the Redis log directory exists and is writable before
# starting Redis. This is needed because the redis_logs volume is mounted
# at /var/log/redis but Redis Alpine runs as a non-root user.
#
# Related: ADR-050 (Log aggregation via Logstash)
# ============================================================================
set -e
# Create log directory if it doesn't exist
mkdir -p /var/log/redis
# Ensure redis user can write to the log directory
# Note: In Alpine Redis image, redis runs as uid 999
chown -R redis:redis /var/log/redis
chmod 755 /var/log/redis
# Start Redis with the provided arguments
exec redis-server "$@"

View File

@@ -0,0 +1,267 @@
# Bugsink MCP Troubleshooting Guide
This document tracks known issues and solutions for the Bugsink MCP server integration with Claude Code.
## Issue History
### 2026-01-22: Server Name Prefix Collision (LATEST)
**Problem:**
- `bugsink-dev` MCP server never starts (tools not available)
- Production `bugsink` MCP works fine
- Manual test works: `BUGSINK_URL=http://localhost:8000 BUGSINK_TOKEN=<token> node d:/gitea/bugsink-mcp/dist/index.js`
- Configuration correct, environment variables correct, but server silently skipped
**Root Cause:**
Claude Code silently skips MCP servers when server names share prefixes (e.g., `bugsink` and `bugsink-dev`).
Debug logs showed `bugsink-dev` was NEVER started - no "Starting connection" message ever appeared.
**Discovery Method:**
- Analyzed Claude Code debug logs at `C:\Users\games3\.claude\debug\*.txt`
- Found that MCP startup messages only showed: `memory`, `bugsink`, `redis`, `gitea-projectium`, etc.
- `bugsink-dev` was completely absent from startup sequence
- No error was logged - server was silently filtered out
**Solution Attempts:**
**Attempt 1:** Renamed `bugsink-dev` to `devbugsink`
- New MCP tool prefix: `mcp__devbugsink__*`
- Changed URL from `http://localhost:8000` to `http://127.0.0.1:8000`
- **Result:** Still failed after full VS Code restart - server never loaded
**Attempt 2:** Renamed `devbugsink` to `localerrors` (completely different name)
- New MCP tool prefix: `mcp__localerrors__*`
- Uses completely unrelated name with no shared prefix
- Based on infra-architect research showing name collision issues
- **Result:** Still failed after full VS Code restart - server never loaded
**Attempt 3:** Created project-level `.mcp.json` file ✅ **SUCCESS**
- Location: `d:\gitea\flyer-crawler.projectium.com\flyer-crawler.projectium.com\.mcp.json`
- Contains `localerrors` server configuration
- Project-level config bypassed the global config loader issue
- **Result:** ✅ Working! Server loads successfully, 2 projects found
- Tool prefix: `mcp__localerrors__*`
**Alternative Solutions Researched:**
- Sentry MCP: Not compatible (different API endpoints `/api/canonical/0/` vs `/api/0/`)
- MCP-Proxy: Could work but requires separate process
- Project-level `.mcp.json`: Alternative if global config continues to fail
**Status:** ✅ RESOLVED - Project-level `.mcp.json` works successfully
**Root Cause Analysis:**
Claude Code's global settings.json has an issue loading localhost stdio MCP servers, even with completely distinct names. The exact cause is unknown, but may be related to:
- Multiple servers using the same package (bugsink-mcp)
- Localhost URL filtering in global config
- Internal MCP loader bug specific to Windows/localhost combinations
**Working Solution:**
Use **project-level `.mcp.json`** file instead of global `settings.json` for localhost MCP servers. This bypasses the global config loader issue entirely.
**Key Insights:**
1. Global config fails for localhost servers even with distinct names (`localerrors`)
2. Project-level `.mcp.json` successfully loads the same configuration
3. Production HTTPS servers work fine in global config
4. Both configs can coexist: global for production, project-level for dev
---
### 2026-01-21: Environment Variable Typo (RESOLVED)
**Problem:**
- `bugsink-dev` MCP server fails to start (tools not available)
- Production `bugsink` MCP works fine
- User experienced repeated troubleshooting loops without resolution
**Root Cause:**
Environment variable name mismatch:
- **Package expects:** `BUGSINK_TOKEN`
- **Configuration had:** `BUGSINK_API_TOKEN`
**Discovery Method:**
- infra-architect subagent examined `d:\gitea\bugsink-mcp\README.md` line 47
- Found correct variable name in package documentation
- Production MCP continued working because it was started before config change
**Solution Applied:**
1. Updated `C:\Users\games3\.claude\settings.json`:
- Changed `BUGSINK_API_TOKEN` to `BUGSINK_TOKEN` for both `bugsink` and `bugsink-dev`
- Removed unused `BUGSINK_ORG_SLUG` environment variable
2. Updated documentation:
- `CLAUDE.md` MCP Servers section (lines 307-327)
- `docs/BUGSINK-SYNC.md` environment variables (line 66, 108)
3. Required action:
- Restart Claude Code to reload MCP servers
**Status:** Fixed - but superseded by server name collision issue above
---
## Correct Configuration
### Required Environment Variables
The bugsink-mcp package requires exactly **TWO** environment variables:
```json
{
"bugsink": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "https://bugsink.projectium.com",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
},
"localerrors": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "http://127.0.0.1:8000",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
}
}
```
**Important:**
- Variable is `BUGSINK_TOKEN`, NOT `BUGSINK_API_TOKEN`
- `BUGSINK_ORG_SLUG` is NOT used by the package
- Works with both HTTPS (production) and HTTP (localhost)
- Use completely distinct name like `localerrors` (not `bugsink-dev` or `devbugsink`) to avoid any name collision
---
## Common Issues
### MCP Server Tools Not Available
**Symptoms:**
- `mcp__localerrors__*` tools return "No such tool available"
- Production `bugsink` MCP may work while `localerrors` fails
**Possible Causes:**
1. **Wrong environment variable name** (most common)
- Check: Variable must be `BUGSINK_TOKEN`, not `BUGSINK_API_TOKEN`
2. **Invalid API token**
- Check: Token must be 40-character lowercase hex
- Verify: Token created via Django management command
3. **Bugsink instance not accessible**
- Test: `curl -s -o /dev/null -w "%{http_code}" http://localhost:8000`
- Expected: `302` (redirect) or `200`
4. **MCP server crashed on startup**
- Check: Claude Code logs (if available)
- Test manually: `BUGSINK_URL=http://localhost:8000 BUGSINK_TOKEN=<token> node d:/gitea/bugsink-mcp/dist/index.js`
**Solution:**
1. Verify correct variable names in settings.json
2. Restart Claude Code
3. Test connection: Use `mcp__bugsink__test_connection` tool
---
## Testing MCP Server
### Manual Test
Run the MCP server directly to see error output:
```bash
# For localhost Bugsink
cd d:\gitea\bugsink-mcp
set BUGSINK_URL=http://localhost:8000
set BUGSINK_TOKEN=a609c2886daa4e1e05f1517074d7779a5fb49056
node dist/index.js
```
Expected output:
```
Bugsink MCP server started
Connected to: http://localhost:8000
```
### Test in Claude Code
After restart, verify both MCP servers work:
```typescript
// Production Bugsink
mcp__bugsink__test_connection();
// Expected: "Connection successful: Connected successfully. Found N project(s)."
// Dev Container Bugsink
mcp__localerrors__test_connection();
// Expected: "Connection successful: Connected successfully. Found N project(s)."
```
---
## Creating Bugsink API Tokens
Bugsink 2.0.11 does NOT have a "Settings > API Keys" menu in the UI. Tokens must be created via Django management command.
### For Dev Container (localhost:8000)
```bash
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
```
### For Production (bugsink.projectium.com via SSH)
```bash
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
```
Both commands output a 40-character hex token.
---
## Package Information
- **Repository:** https://github.com/j-shelfwood/bugsink-mcp.git
- **Local Installation:** `d:\gitea\bugsink-mcp`
- **Build Command:** `npm install && npm run build`
- **Main File:** `dist/index.js`
---
## Related Documentation
- [CLAUDE.md MCP Servers Section](../CLAUDE.md#mcp-servers)
- [DEV-CONTAINER-BUGSINK.md](./DEV-CONTAINER-BUGSINK.md)
- [BUGSINK-SYNC.md](./BUGSINK-SYNC.md)
---
## Failed Solutions (Do Not Retry)
These approaches were tried and did NOT work:
1. ❌ Regenerating API token multiple times
2. ❌ Restarting Claude Code without config changes
3. ❌ Checking Bugsink instance accessibility (was already working)
4. ❌ Adding `BUGSINK_ORG_SLUG` environment variable (not used by package)
**Lesson:** Always verify actual package requirements in source code/README before troubleshooting.

View File

@@ -63,7 +63,7 @@ Add these to **test environment only** (`deploy-to-test.yml`):
```bash
# Bugsink API
BUGSINK_URL=https://bugsink.projectium.com
BUGSINK_API_TOKEN=<from Bugsink Settings > API Keys>
BUGSINK_TOKEN=<create via Django management command - see below>
# Gitea API
GITEA_URL=https://gitea.projectium.com
@@ -76,15 +76,38 @@ BUGSINK_SYNC_ENABLED=true # Only set true in test env
BUGSINK_SYNC_INTERVAL=15 # Minutes between sync runs
```
## Creating Bugsink API Token
Bugsink 2.0.11 does not have a "Settings > API Keys" UI. Create API tokens via Django management command:
**On Production Server:**
```bash
sudo su - bugsink
source venv/bin/activate
cd ~
bugsink-manage shell -c "
from django.contrib.auth import get_user_model
from rest_framework.authtoken.models import Token
User = get_user_model()
user = User.objects.get(email='admin@yourdomain.com') # Use your admin email
token, created = Token.objects.get_or_create(user=user)
print(f'Token: {token.key}')
"
exit
```
This will output a 40-character lowercase hex token.
## Gitea Secrets to Add
Add these secrets in Gitea repository settings (Settings > Secrets):
| Secret Name | Value | Environment |
| ---------------------- | ---------------------- | ----------- |
| `BUGSINK_API_TOKEN` | API token from Bugsink | Test only |
| `GITEA_SYNC_TOKEN` | Personal access token | Test only |
| `BUGSINK_SYNC_ENABLED` | `true` | Test only |
| Secret Name | Value | Environment |
| ---------------------- | ------------------------ | ----------- |
| `BUGSINK_TOKEN` | Token from command above | Test only |
| `GITEA_SYNC_TOKEN` | Personal access token | Test only |
| `BUGSINK_SYNC_ENABLED` | `true` | Test only |
## Redis Configuration

View File

@@ -0,0 +1,81 @@
# Dev Container Bugsink Setup
Local Bugsink instance for development - NOT connected to production.
## Quick Reference
| Item | Value |
| ------------ | ----------------------------------------------------------- |
| UI | `https://localhost:8443` (nginx proxy from 8000) |
| Credentials | `admin@localhost` / `admin` |
| Projects | Backend (Dev) = Project ID 1, Frontend (Dev) = Project ID 2 |
| Backend DSN | `SENTRY_DSN=http://<key>@localhost:8000/1` |
| Frontend DSN | `VITE_SENTRY_DSN=http://<key>@localhost:8000/2` |
## Configuration Files
| File | Purpose |
| ----------------- | ----------------------------------------------------------------- |
| `compose.dev.yml` | Initial DSNs using `127.0.0.1:8000` (container startup) |
| `.env.local` | **OVERRIDES** compose.dev.yml with `localhost:8000` (app runtime) |
**CRITICAL**: `.env.local` takes precedence over `compose.dev.yml` environment variables.
## Why localhost vs 127.0.0.1?
The `.env.local` file uses `localhost` while `compose.dev.yml` uses `127.0.0.1`. Both work in practice - `localhost` was chosen when `.env.local` was created separately.
## HTTPS Setup
- Self-signed certificates auto-generated with mkcert on container startup
- CSRF Protection: Django configured with `SECURE_PROXY_SSL_HEADER` to trust `X-Forwarded-Proto` from nginx
- HTTPS is for UI access only - Sentry SDK uses HTTP directly
## Isolation Benefits
- Dev errors stay local, don't pollute production/test dashboards
- No Gitea secrets needed - everything self-contained
- Independent testing of error tracking without affecting metrics
## Accessing Errors
### Via Browser
1. Open `https://localhost:8443`
2. Login with credentials above
3. Navigate to Issues to view captured errors
### Via MCP (bugsink-dev)
Configure in `.claude/mcp.json`:
```json
{
"bugsink-dev": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "http://localhost:8000",
"BUGSINK_API_TOKEN": "<token-from-local-bugsink>",
"BUGSINK_ORG_SLUG": "sentry"
}
}
}
```
**Get auth token**:
API tokens must be created via Django management command (Bugsink 2.0.11 does not have a "Settings > API Keys" UI):
```bash
podman exec flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && \
DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink \
SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security \
DJANGO_SETTINGS_MODULE=bugsink_conf \
PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages \
/opt/bugsink/bin/python -m django create_auth_token'
```
This will output a 40-character lowercase hex token. Copy it to your MCP configuration.
**MCP Tools**: Use `mcp__bugsink-dev__*` tools (not `mcp__bugsink__*` which connects to production).

View File

@@ -0,0 +1,264 @@
# Flyer URL Configuration
## Overview
Flyer image and icon URLs are environment-specific to ensure they point to the correct server for each deployment. Images are served as static files by NGINX from the `/flyer-images/` path with 7-day browser caching enabled.
## Environment-Specific URLs
| Environment | Base URL | Example |
| ------------- | ------------------------------------------- | -------------------------------------------------------------------------- |
| Dev Container | `https://127.0.0.1` | `https://127.0.0.1/flyer-images/safeway-flyer.jpg` |
| Test | `https://flyer-crawler-test.projectium.com` | `https://flyer-crawler-test.projectium.com/flyer-images/safeway-flyer.jpg` |
| Production | `https://flyer-crawler.projectium.com` | `https://flyer-crawler.projectium.com/flyer-images/safeway-flyer.jpg` |
**Note:** The dev container accepts connections to **both** `https://localhost/` and `https://127.0.0.1/` thanks to the SSL certificate and NGINX configuration. See [SSL Certificate Configuration](#ssl-certificate-configuration-dev-container) below.
## SSL Certificate Configuration (Dev Container)
The dev container uses self-signed certificates generated by [mkcert](https://github.com/FiloSottile/mkcert) to enable HTTPS locally. This configuration solves a common mixed-origin SSL issue.
### The Problem
When users access the site via `https://localhost/` but image URLs in the database use `https://127.0.0.1/...`, browsers treat these as different origins. This causes `ERR_CERT_AUTHORITY_INVALID` errors when loading images, even though both hostnames point to the same server.
### The Solution
1. **Certificate Generation** (`Dockerfile.dev`):
```bash
mkcert localhost 127.0.0.1 ::1
```
This creates a certificate with Subject Alternative Names (SANs) for all three hostnames.
2. **NGINX Configuration** (`docker/nginx/dev.conf`):
```nginx
server_name localhost 127.0.0.1;
```
NGINX accepts requests to both hostnames using the same SSL certificate.
### How It Works
| Component | Configuration |
| -------------------- | ---------------------------------------------------- |
| SSL Certificate SANs | `localhost`, `127.0.0.1`, `::1` |
| NGINX `server_name` | `localhost 127.0.0.1` |
| Seed Script URLs | Uses `https://127.0.0.1` (works with DB constraints) |
| User Access | Either `https://localhost/` or `https://127.0.0.1/` |
### Why This Matters
- **Database Constraints**: The `flyers` table has CHECK constraints requiring URLs to start with `http://` or `https://`. Relative URLs are not allowed.
- **Consistent Behavior**: Users can access the site using either hostname without SSL warnings.
- **Same Certificate**: Both hostnames use the same self-signed certificate, eliminating mixed-content errors.
### Verifying the Configuration
```bash
# Check certificate SANs
podman exec flyer-crawler-dev openssl x509 -in /app/certs/localhost.crt -text -noout | grep -A1 "Subject Alternative Name"
# Expected output:
# X509v3 Subject Alternative Name:
# DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1
# Test both hostnames respond
curl -k https://localhost/health
curl -k https://127.0.0.1/health
```
### Troubleshooting SSL Issues
If you encounter `ERR_CERT_AUTHORITY_INVALID`:
1. **Check NGINX is running**: `podman exec flyer-crawler-dev nginx -t`
2. **Verify certificate exists**: `podman exec flyer-crawler-dev ls -la /app/certs/`
3. **Ensure both hostnames are in server_name**: Check `/etc/nginx/sites-available/default`
4. **Rebuild container if needed**: The certificate is generated at build time
### Permanent Fix: Install CA Certificate (Recommended)
To permanently eliminate SSL certificate warnings, install the mkcert CA certificate on your system. This is optional but provides a better development experience.
The CA certificate is located at `certs/mkcert-ca.crt` in the project root. See [`certs/README.md`](../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
After installation:
- Your browser will trust all mkcert certificates without warnings
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
See also: [Debugging Guide - SSL Issues](development/DEBUGGING.md#ssl-certificate-issues)
## NGINX Static File Serving
All environments serve flyer images as static files with browser caching:
```nginx
# Serve flyer images from static storage (7-day cache)
location /flyer-images/ {
alias /path/to/flyer-images/;
expires 7d;
add_header Cache-Control "public, immutable";
}
```
### Directory Paths by Environment
| Environment | NGINX Alias Path |
| ------------- | ---------------------------------------------------------- |
| Dev Container | `/app/public/flyer-images/` |
| Test | `/var/www/flyer-crawler-test.projectium.com/flyer-images/` |
| Production | `/var/www/flyer-crawler.projectium.com/flyer-images/` |
## Configuration
### Environment Variable
Set `FLYER_BASE_URL` in your environment configuration:
```bash
# Dev container (.env)
FLYER_BASE_URL=https://localhost
# Test environment
FLYER_BASE_URL=https://flyer-crawler-test.projectium.com
# Production
FLYER_BASE_URL=https://flyer-crawler.projectium.com
```
### Seed Script
The seed script ([src/db/seed.ts](../src/db/seed.ts)) automatically uses the correct base URL based on:
1. `FLYER_BASE_URL` environment variable (if set)
2. `NODE_ENV` value:
- `production` → `https://flyer-crawler.projectium.com`
- `test` → `https://flyer-crawler-test.projectium.com`
- Default → `https://localhost`
The seed script also copies test images from `src/tests/assets/` to `public/flyer-images/`:
- `test-flyer-image.jpg` - Sample flyer image
- `test-flyer-icon.png` - Sample 64x64 icon
## Updating Existing Data
If you need to update existing flyer URLs in the database, use the provided SQL script:
### Dev Container
```bash
# Connect to dev database
podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev
# Run the update (dev container uses HTTPS with self-signed certs)
UPDATE flyers
SET
image_url = REPLACE(image_url, 'example.com', 'localhost'),
icon_url = REPLACE(icon_url, 'example.com', 'localhost')
WHERE
image_url LIKE '%example.com%'
OR icon_url LIKE '%example.com%';
# Verify
SELECT flyer_id, image_url, icon_url FROM flyers;
```
### Test Environment
```bash
# Via SSH
ssh root@projectium.com "psql -U flyer_crawler_test -d flyer-crawler-test -c \"
UPDATE flyers
SET
image_url = REPLACE(image_url, 'example.com', 'flyer-crawler-test.projectium.com'),
icon_url = REPLACE(icon_url, 'example.com', 'flyer-crawler-test.projectium.com')
WHERE
image_url LIKE '%example.com%'
OR icon_url LIKE '%example.com%';
\""
```
### Production
```bash
# Via SSH
ssh root@projectium.com "psql -U flyer_crawler_prod -d flyer-crawler-prod -c \"
UPDATE flyers
SET
image_url = REPLACE(image_url, 'example.com', 'flyer-crawler.projectium.com'),
icon_url = REPLACE(icon_url, 'example.com', 'flyer-crawler.projectium.com')
WHERE
image_url LIKE '%example.com%'
OR icon_url LIKE '%example.com%';
\""
```
## Test Data Updates
### Test Helper Function
A helper function `getFlyerBaseUrl()` is available in [src/tests/utils/testHelpers.ts](../src/tests/utils/testHelpers.ts) that automatically detects the correct base URL for tests:
```typescript
export const getFlyerBaseUrl = (): string => {
if (process.env.FLYER_BASE_URL) {
return process.env.FLYER_BASE_URL;
}
// Check if we're in dev container (DB_HOST=postgres is typical indicator)
// Use 'localhost' instead of '127.0.0.1' to match the hostname users access
// This avoids SSL certificate mixed-origin issues in browsers
if (process.env.DB_HOST === 'postgres' || process.env.DB_HOST === '127.0.0.1') {
return 'https://localhost';
}
if (process.env.NODE_ENV === 'production') {
return 'https://flyer-crawler.projectium.com';
}
if (process.env.NODE_ENV === 'test') {
return 'https://flyer-crawler-test.projectium.com';
}
// Default for unit tests
return 'https://example.com';
};
```
### Updated Test Files
The following test files now use `getFlyerBaseUrl()` for environment-aware URL generation:
- [src/db/seed.ts](../src/db/seed.ts) - Main seed script (uses `FLYER_BASE_URL`)
- [src/tests/utils/testHelpers.ts](../src/tests/utils/testHelpers.ts) - `getFlyerBaseUrl()` helper function
- [src/hooks/useDataExtraction.test.ts](../src/hooks/useDataExtraction.test.ts) - Mock flyer factory
- [src/schemas/flyer.schemas.test.ts](../src/schemas/flyer.schemas.test.ts) - Schema validation tests
- [src/services/flyerProcessingService.server.test.ts](../src/services/flyerProcessingService.server.test.ts) - Processing service tests
- [src/tests/integration/flyer-processing.integration.test.ts](../src/tests/integration/flyer-processing.integration.test.ts) - Integration tests
This approach ensures tests work correctly in all environments (dev container, CI/CD, local development, test, production).
## Files Changed
| File | Change |
| --------------------------- | ------------------------------------------------------------------------------------------------- |
| `src/db/seed.ts` | Added `FLYER_BASE_URL` environment variable support, copies test images to `public/flyer-images/` |
| `docker/nginx/dev.conf` | Added `/flyer-images/` location block for static file serving |
| `.env.example` | Added `FLYER_BASE_URL` variable |
| `sql/update_flyer_urls.sql` | SQL script for updating existing data |
| Test files | Updated mock data to use `https://localhost` |
## Summary
- Seed script now uses environment-specific HTTPS URLs
- Seed script copies test images from `src/tests/assets/` to `public/flyer-images/`
- NGINX serves `/flyer-images/` as static files with 7-day cache
- Test files updated with `https://localhost` (not `127.0.0.1` to avoid SSL mixed-origin issues)
- SQL script provided for updating existing data
- Documentation updated for each environment

318
docs/POSTGRES-MCP-SETUP.md Normal file
View File

@@ -0,0 +1,318 @@
# PostgreSQL MCP Server Setup
This document describes the configuration and troubleshooting for the PostgreSQL MCP server integration with Claude Code.
## Status
**WORKING** - Successfully configured and tested on 2026-01-22
- **Server Name**: `devdb`
- **Database**: `flyer_crawler_dev` (68 tables)
- **Connection**: Verified working
- **Tool Prefix**: `mcp__devdb__*`
- **Configuration**: Project-level `.mcp.json`
## Overview
The PostgreSQL MCP server (`@modelcontextprotocol/server-postgres`) provides database query capabilities directly from Claude Code, enabling:
- Running SQL queries against the development database
- Exploring database schema and tables
- Testing queries before implementation
- Debugging data issues
## Configuration
### Project-Level Configuration (Recommended)
The PostgreSQL MCP server is configured in the project-level `.mcp.json` file:
```json
{
"mcpServers": {
"devdb": {
"command": "D:\\nodejs\\npx.cmd",
"args": [
"-y",
"@modelcontextprotocol/server-postgres",
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
]
}
}
}
```
**Key Configuration Details:**
| Parameter | Value | Notes |
| ----------- | --------------------------------------- | --------------------------------------- |
| Server Name | `devdb` | Distinct name to avoid collision issues |
| Package | `@modelcontextprotocol/server-postgres` | Official MCP PostgreSQL server |
| Host | `127.0.0.1` | Use IP address, not `localhost` |
| Port | `5432` | Default PostgreSQL port |
| Database | `flyer_crawler_dev` | Development database name |
| User | `postgres` | Default superuser for dev |
| Password | `postgres` | Default password for dev |
### Why Project-Level Configuration?
Based on troubleshooting experience with other MCP servers (documented in `BUGSINK-MCP-TROUBLESHOOTING.md`), **localhost MCP servers work more reliably in project-level `.mcp.json`** than in global `settings.json`.
Issues observed with global configuration:
- MCP servers silently not loading
- No error messages in logs
- Tools not appearing in available tool list
Project-level configuration bypasses these issues entirely.
### Connection String Format
```
postgresql://[user]:[password]@[host]:[port]/[database]
```
Examples:
```
# Development (local container)
postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev
# Test database (if needed)
postgresql://flyer_crawler_test:password@127.0.0.1:5432/flyer_crawler_test
```
## Available Tools
Once configured, the following tools become available (prefix `mcp__devdb__`):
| Tool | Description |
| ------- | -------------------------------------- |
| `query` | Execute SQL queries and return results |
## Usage Examples
### Basic Query
```typescript
// List all tables
mcp__devdb__query("SELECT tablename FROM pg_tables WHERE schemaname = 'public'");
// Count records in a table
mcp__devdb__query('SELECT COUNT(*) FROM flyers');
// Check table structure
mcp__devdb__query(
"SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'flyers'",
);
```
### Debugging Data Issues
```typescript
// Find recent flyers
mcp__devdb__query('SELECT id, name, created_at FROM flyers ORDER BY created_at DESC LIMIT 10');
// Check job queue status
mcp__devdb__query('SELECT state, COUNT(*) FROM bullmq_jobs GROUP BY state');
// Verify user data
mcp__devdb__query("SELECT id, email, created_at FROM users WHERE email LIKE '%test%'");
```
## Prerequisites
### 1. PostgreSQL Container Running
The PostgreSQL container must be running and healthy:
```bash
# Check container status
podman ps | grep flyer-crawler-postgres
# Expected output shows "healthy" status
# flyer-crawler-postgres ... Up N hours (healthy) ...
```
### 2. Port Accessible from Host
PostgreSQL port 5432 must be mapped to the host:
```bash
# Verify port mapping
podman port flyer-crawler-postgres
# Expected: 5432/tcp -> 0.0.0.0:5432
```
### 3. Database Exists
Verify the database exists:
```bash
podman exec flyer-crawler-postgres psql -U postgres -c "\l" | grep flyer_crawler_dev
```
## Troubleshooting
### Tools Not Available
**Symptoms:**
- `mcp__devdb__*` tools not in available tool list
- No error messages displayed
**Solutions:**
1. **Restart Claude Code** - MCP config changes require restart
2. **Check container status** - Ensure PostgreSQL container is running
3. **Verify port mapping** - Confirm port 5432 is accessible
4. **Test connection manually**:
```bash
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1"
```
### Connection Refused
**Symptoms:**
- Connection error when using tools
- "Connection refused" in error message
**Solutions:**
1. **Check container health**:
```bash
podman ps | grep flyer-crawler-postgres
```
2. **Restart the container**:
```bash
podman restart flyer-crawler-postgres
```
3. **Check for port conflicts**:
```bash
netstat -an | findstr 5432
```
### Authentication Failed
**Symptoms:**
- "password authentication failed" error
**Solutions:**
1. **Verify credentials** in container environment:
```bash
podman exec flyer-crawler-dev env | grep DB_
```
2. **Check PostgreSQL users**:
```bash
podman exec flyer-crawler-postgres psql -U postgres -c "\du"
```
3. **Update connection string** in `.mcp.json` if credentials differ
### Database Does Not Exist
**Symptoms:**
- "database does not exist" error
**Solutions:**
1. **List available databases**:
```bash
podman exec flyer-crawler-postgres psql -U postgres -c "\l"
```
2. **Create database if missing**:
```bash
podman exec flyer-crawler-postgres createdb -U postgres flyer_crawler_dev
```
## Security Considerations
### Development Only
The default credentials (`postgres:postgres`) are for **development only**. Never use these in production.
### Connection String in Config
The connection string includes the password in plain text. This is acceptable for:
- Local development
- Container environments
For production MCP access (if ever needed):
- Use environment variables
- Consider connection pooling
- Implement proper access controls
### Query Permissions
The MCP server executes queries as the configured user (`postgres` in dev). Be aware that:
- `postgres` is a superuser with full access
- For restricted access, create a dedicated MCP user with limited permissions:
```sql
-- Example: Create read-only MCP user
CREATE USER mcp_reader WITH PASSWORD 'secure_password';
GRANT CONNECT ON DATABASE flyer_crawler_dev TO mcp_reader;
GRANT USAGE ON SCHEMA public TO mcp_reader;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO mcp_reader;
```
## Database Information
### Development Environment
| Property | Value |
| --------------------- | ------------------------- |
| Container | `flyer-crawler-postgres` |
| Image | `postgis/postgis:15-3.4` |
| Host (from Windows) | `127.0.0.1` / `localhost` |
| Host (from container) | `postgres` |
| Port | `5432` |
| Database | `flyer_crawler_dev` |
| User | `postgres` |
| Password | `postgres` |
### Schema Reference
The database uses PostGIS for geographic data. Key tables include:
- `users` - User accounts
- `stores` - Store definitions
- `store_locations` - Store geographic locations
- `flyers` - Uploaded flyer metadata
- `flyer_items` - Extracted deal items
- `watchlists` - User watchlists
- `shopping_lists` - User shopping lists
- `recipes` - Recipe definitions
For complete schema, see `sql/master_schema_rollup.sql`.
## Related Documentation
- [CLAUDE.md - MCP Servers Section](../CLAUDE.md#mcp-servers)
- [BUGSINK-MCP-TROUBLESHOOTING.md](./BUGSINK-MCP-TROUBLESHOOTING.md) - Similar MCP setup patterns
- [sql/master_schema_rollup.sql](../sql/master_schema_rollup.sql) - Database schema
## Changelog
### 2026-01-21
- Initial configuration added to project-level `.mcp.json`
- Server named `devdb` to avoid naming collisions
- Using `127.0.0.1` instead of `localhost` based on Bugsink MCP experience
- Documentation created

130
docs/README.md Normal file
View File

@@ -0,0 +1,130 @@
# Flyer Crawler Documentation
Welcome to the Flyer Crawler documentation. This guide will help you navigate the various documentation resources available.
## Quick Links
- [Main README](../README.md) - Project overview and quick start
- [CLAUDE.md](../CLAUDE.md) - AI agent instructions and project guidelines
- [CONTRIBUTING.md](../CONTRIBUTING.md) - Development workflow and contribution guide
## Documentation Structure
### 🚀 Getting Started
New to the project? Start here:
- [Installation Guide](getting-started/INSTALL.md) - Complete setup instructions
- [Environment Configuration](getting-started/ENVIRONMENT.md) - Environment variables and secrets
### 🏗️ Architecture
Understand how the system works:
- [System Overview](architecture/OVERVIEW.md) - High-level architecture
- [Database Schema](architecture/DATABASE.md) - Database design and entities
- [Authentication](architecture/AUTHENTICATION.md) - OAuth and JWT authentication
- [WebSocket Usage](architecture/WEBSOCKET_USAGE.md) - Real-time communication patterns
### 💻 Development
Day-to-day development guides:
- [Testing Guide](development/TESTING.md) - Unit, integration, and E2E testing
- [Code Patterns](development/CODE-PATTERNS.md) - Common code patterns and ADR examples
- [Design Tokens](development/DESIGN_TOKENS.md) - UI design system and Neo-Brutalism
- [Debugging Guide](development/DEBUGGING.md) - Common debugging patterns
### 🔧 Operations
Production operations and deployment:
- [Deployment Guide](operations/DEPLOYMENT.md) - Deployment procedures
- [Bare Metal Setup](operations/BARE-METAL-SETUP.md) - Server provisioning
- [Logstash Quick Reference](operations/LOGSTASH-QUICK-REF.md) - Log aggregation
- [Logstash Troubleshooting](operations/LOGSTASH-TROUBLESHOOTING.md) - Debugging logs
- [Monitoring](operations/MONITORING.md) - Bugsink, health checks, observability
**NGINX Reference Configs** (in repository root):
- `etc-nginx-sites-available-flyer-crawler.projectium.com` - Production server config
- `etc-nginx-sites-available-flyer-crawler-test-projectium-com.txt` - Test server config
### 🛠️ Tools
External tool configuration:
- [MCP Configuration](tools/MCP-CONFIGURATION.md) - Model Context Protocol servers
- [Bugsink Setup](tools/BUGSINK-SETUP.md) - Error tracking configuration
- [VS Code Setup](tools/VSCODE-SETUP.md) - Editor configuration
### 🤖 AI Agents
Working with Claude Code subagents:
- [Subagent Overview](subagents/OVERVIEW.md) - Introduction to specialized agents
- [Coder Guide](subagents/CODER-GUIDE.md) - Code development patterns
- [Tester Guide](subagents/TESTER-GUIDE.md) - Testing strategies
- [Database Guide](subagents/DATABASE-GUIDE.md) - Database workflows
- [DevOps Guide](subagents/DEVOPS-GUIDE.md) - Deployment and infrastructure
- [AI Usage Guide](subagents/AI-USAGE-GUIDE.md) - Gemini integration
- [Frontend Guide](subagents/FRONTEND-GUIDE.md) - UI/UX development
- [Documentation Guide](subagents/DOCUMENTATION-GUIDE.md) - Writing docs
- [Security & Debug Guide](subagents/SECURITY-DEBUG-GUIDE.md) - Security and debugging
**AI-Optimized References** (token-efficient quick refs):
- [Coder Reference](SUBAGENT-CODER-REFERENCE.md)
- [Tester Reference](SUBAGENT-TESTER-REFERENCE.md)
- [DB Reference](SUBAGENT-DB-REFERENCE.md)
- [DevOps Reference](SUBAGENT-DEVOPS-REFERENCE.md)
- [Integrations Reference](SUBAGENT-INTEGRATIONS-REFERENCE.md)
### 📐 Architecture Decision Records (ADRs)
Design decisions and rationale:
- [ADR Index](adr/index.md) - Complete list of all ADRs
- 54+ ADRs covering patterns, conventions, and technical decisions
### 📦 Archive
Historical and completed documentation:
- [Session Notes](archive/sessions/) - Development session logs
- [Planning Documents](archive/plans/) - Feature plans and implementation status
- [Research Notes](archive/research/) - Investigation and research documents
## Documentation Conventions
- **File Names**: Use SCREAMING_SNAKE_CASE for human-readable docs (e.g., `INSTALL.md`)
- **Links**: Use relative paths from the document's location
- **Code Blocks**: Always specify language for syntax highlighting
- **Tables**: Use markdown tables for structured data
- **Cross-References**: Link to ADRs and other docs for detailed explanations
## Contributing to Documentation
See [CONTRIBUTING.md](../CONTRIBUTING.md) for guidelines on:
- Writing clear, concise documentation
- Updating docs when code changes
- Creating new ADRs for significant decisions
- Documenting new features and APIs
## Need Help?
- Check the [Testing Guide](development/TESTING.md) for test-related issues
- See [Debugging Guide](development/DEBUGGING.md) for troubleshooting
- Review [ADRs](adr/index.md) for architectural context
- Consult [Subagent Guides](subagents/OVERVIEW.md) for AI agent assistance
## Documentation Maintenance
This documentation is actively maintained. If you find:
- Broken links or outdated information
- Missing documentation for features
- Unclear or confusing sections
Please open an issue or submit a pull request with improvements.

View File

@@ -0,0 +1,265 @@
# Coder Subagent Reference
## Quick Navigation
| Category | Key Files |
| ------------ | ------------------------------------------------------------------ |
| Routes | `src/routes/*.routes.ts` |
| Services | `src/services/*.server.ts` (backend), `src/services/*.ts` (shared) |
| Repositories | `src/services/db/*.db.ts` |
| Types | `src/types.ts`, `src/types/*.ts` |
| Schemas | `src/schemas/*.schemas.ts` |
| Config | `src/config/env.ts` |
| Utils | `src/utils/*.ts` |
---
## Architecture Patterns (ADR Summary)
### Layer Flow
```
Route → validateRequest(schema) → Service → Repository → Database
External APIs
```
### Repository Naming Convention (ADR-034)
| Prefix | Behavior | Return |
| --------- | --------------------------------- | -------------- |
| `get*` | Throws `NotFoundError` if missing | Entity |
| `find*` | Returns `null` if missing | Entity \| null |
| `list*` | Returns empty array if none | Entity[] |
| `create*` | Creates new record | Entity |
| `update*` | Updates existing | Entity |
| `delete*` | Removes record | void |
### Error Handling (ADR-001)
```typescript
import { handleDbError, NotFoundError } from '../services/db/errors.db';
import { logger } from '../services/logger.server';
// Repository pattern
async function getById(id: string): Promise<Entity> {
try {
const result = await pool.query('SELECT * FROM table WHERE id = $1', [id]);
if (result.rows.length === 0) throw new NotFoundError('Entity not found');
return result.rows[0];
} catch (error) {
handleDbError(error, logger, 'Failed to get entity', { id });
}
}
```
### API Response Helpers (ADR-028)
```typescript
import {
sendSuccess,
sendPaginated,
sendError,
sendNoContent,
sendMessage,
ErrorCode,
} from '../utils/apiResponse';
// Success with data
sendSuccess(res, data); // 200
sendSuccess(res, data, 201); // 201 Created
// Paginated
sendPaginated(res, items, { page, limit, total });
// Error
sendError(res, ErrorCode.NOT_FOUND, 'User not found', 404);
sendError(res, ErrorCode.VALIDATION_ERROR, 'Invalid input', 400, validationErrors);
// No content / Message
sendNoContent(res); // 204
sendMessage(res, 'Password updated');
```
### Transaction Pattern (ADR-002)
```typescript
import { withTransaction } from '../services/db/connection.db';
const result = await withTransaction(async (client) => {
await client.query('INSERT INTO a ...');
await client.query('INSERT INTO b ...');
return { success: true };
});
```
---
## Adding New Features
### New API Endpoint Checklist
1. **Schema** (`src/schemas/{domain}.schemas.ts`)
```typescript
import { z } from 'zod';
export const createEntitySchema = z.object({
body: z.object({ name: z.string().min(1) }),
});
```
2. **Route** (`src/routes/{domain}.routes.ts`)
```typescript
import { validateRequest } from '../middleware/validation.middleware';
import { createEntitySchema } from '../schemas/{domain}.schemas';
router.post('/', validateRequest(createEntitySchema), async (req, res, next) => {
try {
const result = await entityService.create(req.body);
sendSuccess(res, result, 201);
} catch (error) {
next(error);
}
});
```
3. **Service** (`src/services/{domain}Service.server.ts`)
```typescript
export async function create(data: CreateInput): Promise<Entity> {
// Business logic here
return repository.create(data);
}
```
4. **Repository** (`src/services/db/{domain}.db.ts`)
```typescript
export async function create(data: CreateInput, client?: PoolClient): Promise<Entity> {
const pool = client || getPool();
try {
const result = await pool.query('INSERT INTO ...', [data.name]);
return result.rows[0];
} catch (error) {
handleDbError(error, logger, 'Failed to create entity', { data });
}
}
```
### New Background Job Checklist
1. **Queue** (`src/services/queues.server.ts`)
```typescript
export const myQueue = new Queue('my-queue', { connection: redisConnection });
```
2. **Worker** (`src/services/workers.server.ts`)
```typescript
new Worker(
'my-queue',
async (job) => {
// Process job
},
{ connection: redisConnection },
);
```
3. **Trigger** (in service)
```typescript
await myQueue.add('job-name', { data });
```
---
## Key Files Reference
### Database Repositories
| Repository | Purpose | Path |
| -------------------- | ----------------------------- | ------------------------------------ |
| `flyer.db.ts` | Flyer CRUD, processing status | `src/services/db/flyer.db.ts` |
| `store.db.ts` | Store management | `src/services/db/store.db.ts` |
| `user.db.ts` | User accounts | `src/services/db/user.db.ts` |
| `shopping.db.ts` | Shopping lists, watchlists | `src/services/db/shopping.db.ts` |
| `gamification.db.ts` | Achievements, points | `src/services/db/gamification.db.ts` |
| `category.db.ts` | Item categories | `src/services/db/category.db.ts` |
| `price.db.ts` | Price history, comparisons | `src/services/db/price.db.ts` |
| `recipe.db.ts` | Recipe management | `src/services/db/recipe.db.ts` |
### Services
| Service | Purpose | Path |
| ---------------------------------- | -------------------------------- | ----------------------------------------------- |
| `flyerProcessingService.server.ts` | Orchestrates flyer AI extraction | `src/services/flyerProcessingService.server.ts` |
| `flyerAiProcessor.server.ts` | Gemini AI integration | `src/services/flyerAiProcessor.server.ts` |
| `cacheService.server.ts` | Redis caching | `src/services/cacheService.server.ts` |
| `queues.server.ts` | BullMQ queue definitions | `src/services/queues.server.ts` |
| `workers.server.ts` | BullMQ workers | `src/services/workers.server.ts` |
| `emailService.server.ts` | Nodemailer integration | `src/services/emailService.server.ts` |
| `geocodingService.server.ts` | Address geocoding | `src/services/geocodingService.server.ts` |
### Routes
| Route | Base Path | Auth Required |
| ------------------ | ------------- | ------------- |
| `flyer.routes.ts` | `/api/flyers` | Mixed |
| `store.routes.ts` | `/api/stores` | Mixed |
| `user.routes.ts` | `/api/users` | Yes |
| `auth.routes.ts` | `/api/auth` | No |
| `admin.routes.ts` | `/api/admin` | Admin only |
| `deals.routes.ts` | `/api/deals` | No |
| `health.routes.ts` | `/api/health` | No |
---
## Error Types
| Error Class | HTTP Status | Use Case |
| --------------------------- | ----------- | ------------------------- |
| `NotFoundError` | 404 | Resource not found |
| `ForbiddenError` | 403 | Access denied |
| `ValidationError` | 400 | Input validation failed |
| `UniqueConstraintError` | 409 | Duplicate record |
| `ForeignKeyConstraintError` | 400 | Referenced record missing |
| `NotNullConstraintError` | 400 | Required field null |
Import: `import { NotFoundError, ... } from '../services/db/errors.db'`
---
## Middleware
| Middleware | Purpose | Usage |
| ------------------------- | -------------------- | ------------------------------------------------------------ |
| `validateRequest(schema)` | Zod validation | `router.post('/', validateRequest(schema), handler)` |
| `requireAuth` | JWT authentication | `router.get('/', requireAuth, handler)` |
| `requireAdmin` | Admin role check | `router.delete('/', requireAuth, requireAdmin, handler)` |
| `fileUpload` | Multer file handling | `router.post('/upload', fileUpload.single('file'), handler)` |
---
## Type Definitions
| File | Contains |
| --------------------------- | ----------------------------------------------- |
| `src/types.ts` | Main types: User, Flyer, FlyerItem, Store, etc. |
| `src/types/api.ts` | API response envelopes, pagination |
| `src/types/auth.ts` | Auth-related types |
| `src/types/gamification.ts` | Achievement types |
---
## Naming Conventions (ADR-027)
| Context | Convention | Example |
| ----------------- | ---------------- | ----------------------------------- |
| AI output types | `Ai*` prefix | `AiFlyerItem`, `AiExtractionResult` |
| Database types | `Db*` prefix | `DbFlyer`, `DbUser` |
| API types | No prefix | `Flyer`, `User` |
| Schema validation | `*Schema` suffix | `createFlyerSchema` |
| Routes | `*.routes.ts` | `flyer.routes.ts` |
| Repositories | `*.db.ts` | `flyer.db.ts` |
| Server services | `*.server.ts` | `aiService.server.ts` |
| Client services | `*.client.ts` | `logger.client.ts` |

View File

@@ -0,0 +1,377 @@
# Database Subagent Reference
## Quick Navigation
| Resource | Path |
| ------------------ | ---------------------------------------- |
| Master Schema | `sql/master_schema_rollup.sql` |
| Initial Schema | `sql/initial_schema.sql` |
| Migrations | `sql/migrations/*.sql` |
| Triggers/Functions | `sql/Initial_triggers_and_functions.sql` |
| Initial Data | `sql/initial_data.sql` |
| Drop Script | `sql/drop_tables.sql` |
| Repositories | `src/services/db/*.db.ts` |
| Connection | `src/services/db/connection.db.ts` |
| Errors | `src/services/db/errors.db.ts` |
---
## Database Credentials
### Environments
| Environment | User | Database | Host |
| ------------- | -------------------- | -------------------- | --------------------------- |
| Production | `flyer_crawler_prod` | `flyer-crawler-prod` | `DB_HOST` secret |
| Test | `flyer_crawler_test` | `flyer-crawler-test` | `DB_HOST` secret |
| Dev Container | `postgres` | `flyer_crawler_dev` | `postgres` (container name) |
### Connection (Dev Container)
```bash
# Inside container
psql -U postgres -d flyer_crawler_dev
# From Windows via Podman
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev
```
### Connection (Production/Test via SSH)
```bash
# SSH to server, then:
PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -U $DB_USER -d $DB_NAME
```
---
## Schema Tables (Core)
| Table | Purpose | Key Columns |
| --------------------- | -------------------- | --------------------------------------------------------------- |
| `users` | Authentication | `user_id` (UUID PK), `email`, `password_hash` |
| `profiles` | User data | `user_id` (FK), `full_name`, `role`, `points` |
| `addresses` | Normalized addresses | `address_id`, `address_line_1`, `city`, `latitude`, `longitude` |
| `stores` | Store chains | `store_id`, `name`, `logo_url` |
| `store_locations` | Physical locations | `store_location_id`, `store_id` (FK), `address_id` (FK) |
| `flyers` | Uploaded flyers | `flyer_id`, `store_id` (FK), `image_url`, `status` |
| `flyer_items` | Extracted deals | `flyer_item_id`, `flyer_id` (FK), `name`, `price` |
| `categories` | Item categories | `category_id`, `name` |
| `master_items` | Canonical items | `master_item_id`, `name`, `category_id` (FK) |
| `shopping_lists` | User lists | `shopping_list_id`, `user_id` (FK), `name` |
| `shopping_list_items` | List items | `shopping_list_item_id`, `shopping_list_id` (FK) |
| `watchlist` | Price alerts | `watchlist_id`, `user_id` (FK), `search_term` |
| `activity_log` | Audit trail | `activity_log_id`, `user_id`, `action`, `details` |
---
## Schema Sync Rule (CRITICAL)
**Both files MUST stay synchronized:**
- `sql/master_schema_rollup.sql` - Used by test DB setup
- `sql/initial_schema.sql` - Used for fresh installs
**When adding columns:**
1. Add migration in `sql/migrations/NNN_description.sql`
2. Add column to `master_schema_rollup.sql`
3. Add column to `initial_schema.sql`
4. Test DB uses `master_schema_rollup.sql` - out-of-sync = test failures
---
## Migration Pattern
### Creating a Migration
```sql
-- sql/migrations/NNN_descriptive_name.sql
-- Add column with default
ALTER TABLE public.flyers
ADD COLUMN IF NOT EXISTS new_column TEXT DEFAULT 'value';
-- Add index
CREATE INDEX IF NOT EXISTS idx_flyers_new_column
ON public.flyers(new_column);
-- Update schema_info
UPDATE public.schema_info
SET schema_hash = 'new_hash', updated_at = now()
WHERE environment = 'production';
```
### Running Migrations
```bash
# Via psql
PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -U $DB_USER -d $DB_NAME -f sql/migrations/NNN_description.sql
# In CI/CD - migrations are checked via schema hash
```
---
## Repository Pattern (ADR-034)
### Method Naming Convention
| Prefix | Behavior | Return Type |
| --------- | --------------------------------- | ---------------- |
| `get*` | Throws `NotFoundError` if missing | `Entity` |
| `find*` | Returns `null` if missing | `Entity \| null` |
| `list*` | Returns empty array if none | `Entity[]` |
| `create*` | Creates new record | `Entity` |
| `update*` | Updates existing record | `Entity` |
| `delete*` | Removes record | `void` |
| `count*` | Returns count | `number` |
### Repository Template
```typescript
// src/services/db/entity.db.ts
import { getPool } from './connection.db';
import { handleDbError, NotFoundError } from './errors.db';
import type { PoolClient } from 'pg';
import type { Logger } from 'pino';
export async function getEntityById(
id: string,
logger: Logger,
client?: PoolClient,
): Promise<Entity> {
const pool = client || getPool();
try {
const result = await pool.query('SELECT * FROM public.entities WHERE entity_id = $1', [id]);
if (result.rows.length === 0) {
throw new NotFoundError('Entity not found');
}
return result.rows[0];
} catch (error) {
handleDbError(error, logger, 'Failed to get entity', { id });
}
}
export async function findEntityByName(
name: string,
logger: Logger,
client?: PoolClient,
): Promise<Entity | null> {
const pool = client || getPool();
try {
const result = await pool.query('SELECT * FROM public.entities WHERE name = $1', [name]);
return result.rows[0] || null;
} catch (error) {
handleDbError(error, logger, 'Failed to find entity', { name });
}
}
export async function listEntities(logger: Logger, client?: PoolClient): Promise<Entity[]> {
const pool = client || getPool();
try {
const result = await pool.query('SELECT * FROM public.entities ORDER BY name');
return result.rows;
} catch (error) {
handleDbError(error, logger, 'Failed to list entities', {});
}
}
```
---
## Transaction Pattern (ADR-002)
```typescript
import { withTransaction } from './connection.db';
const result = await withTransaction(async (client) => {
// All queries use same client
const user = await userRepo.createUser(data, logger, client);
const profile = await profileRepo.createProfile(user.user_id, profileData, logger, client);
await activityRepo.logActivity('user_created', user.user_id, logger, client);
return { user, profile };
});
// Commits on success, rolls back on any error
```
---
## Error Handling (ADR-001)
### Error Types
| Error | PostgreSQL Code | HTTP Status | Use Case |
| -------------------------------- | --------------- | ----------- | ------------------------- |
| `UniqueConstraintError` | `23505` | 409 | Duplicate record |
| `ForeignKeyConstraintError` | `23503` | 400 | Referenced record missing |
| `NotNullConstraintError` | `23502` | 400 | Required field null |
| `CheckConstraintError` | `23514` | 400 | Check constraint violated |
| `InvalidTextRepresentationError` | `22P02` | 400 | Invalid type format |
| `NumericValueOutOfRangeError` | `22003` | 400 | Number out of range |
| `NotFoundError` | - | 404 | Record not found |
| `ForbiddenError` | - | 403 | Access denied |
### Using handleDbError
```typescript
import { handleDbError, NotFoundError } from './errors.db';
try {
const result = await pool.query('INSERT INTO ...', [data]);
if (result.rows.length === 0) throw new NotFoundError('Entity not found');
return result.rows[0];
} catch (error) {
handleDbError(
error,
logger,
'Failed to create entity',
{ data },
{
uniqueMessage: 'Entity with this name already exists',
fkMessage: 'Referenced category does not exist',
defaultMessage: 'Failed to create entity',
},
);
}
```
---
## Connection Pool
```typescript
import { getPool, getPoolStatus } from './connection.db';
// Get pool (singleton)
const pool = getPool();
// Check pool status
const status = getPoolStatus();
// { totalCount: 20, idleCount: 15, waitingCount: 0 }
```
### Pool Configuration
| Setting | Value | Purpose |
| ------------------------- | ----- | ------------------- |
| `max` | 20 | Max clients in pool |
| `idleTimeoutMillis` | 30000 | Idle client timeout |
| `connectionTimeoutMillis` | 2000 | Connection timeout |
---
## Common Queries
### Paginated List
```typescript
const result = await pool.query(
`SELECT * FROM public.flyers
ORDER BY created_at DESC
LIMIT $1 OFFSET $2`,
[limit, (page - 1) * limit],
);
const countResult = await pool.query('SELECT COUNT(*) FROM public.flyers');
const total = parseInt(countResult.rows[0].count, 10);
```
### Spatial Query (Find Nearby)
```typescript
const result = await pool.query(
`SELECT sl.*, a.*,
ST_Distance(a.location, ST_SetSRID(ST_MakePoint($1, $2), 4326)::geography) as distance
FROM public.store_locations sl
JOIN public.addresses a ON sl.address_id = a.address_id
WHERE ST_DWithin(a.location, ST_SetSRID(ST_MakePoint($1, $2), 4326)::geography, $3)
ORDER BY distance`,
[longitude, latitude, radiusMeters],
);
```
### Upsert Pattern
```typescript
const result = await pool.query(
`INSERT INTO public.stores (name, logo_url)
VALUES ($1, $2)
ON CONFLICT (name) DO UPDATE SET
logo_url = EXCLUDED.logo_url,
updated_at = now()
RETURNING *`,
[name, logoUrl],
);
```
---
## Database Reset Commands
### Dev Container
```bash
# Reset dev database (runs seed script)
podman exec -it flyer-crawler-dev npm run db:reset:dev
# Reset test database
podman exec -it flyer-crawler-dev npm run db:reset:test
```
### Manual SQL
```bash
# Drop all tables
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -f /app/sql/drop_tables.sql
# Recreate schema
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -f /app/sql/master_schema_rollup.sql
# Load initial data
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -f /app/sql/initial_data.sql
```
---
## Database Users Setup
```sql
-- Create database and user (as postgres superuser)
CREATE DATABASE "flyer-crawler-test";
CREATE USER flyer_crawler_test WITH PASSWORD 'password';
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
-- Grant permissions
\c "flyer-crawler-test"
ALTER SCHEMA public OWNER TO flyer_crawler_test;
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
-- Required extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS "postgis";
-- Verify permissions
\dn+ public
-- Should show 'UC' for the user
```
---
## Repository Files
| Repository | Domain | Path |
| --------------------- | -------------------------- | ------------------------------------- |
| `user.db.ts` | Users, profiles | `src/services/db/user.db.ts` |
| `flyer.db.ts` | Flyers, processing | `src/services/db/flyer.db.ts` |
| `store.db.ts` | Stores | `src/services/db/store.db.ts` |
| `storeLocation.db.ts` | Store locations | `src/services/db/storeLocation.db.ts` |
| `address.db.ts` | Addresses | `src/services/db/address.db.ts` |
| `category.db.ts` | Categories | `src/services/db/category.db.ts` |
| `shopping.db.ts` | Shopping lists, watchlists | `src/services/db/shopping.db.ts` |
| `price.db.ts` | Price history | `src/services/db/price.db.ts` |
| `gamification.db.ts` | Achievements, points | `src/services/db/gamification.db.ts` |
| `notification.db.ts` | Notifications | `src/services/db/notification.db.ts` |
| `recipe.db.ts` | Recipes | `src/services/db/recipe.db.ts` |
| `receipt.db.ts` | Receipts | `src/services/db/receipt.db.ts` |
| `admin.db.ts` | Admin operations | `src/services/db/admin.db.ts` |

View File

@@ -0,0 +1,357 @@
# DevOps Subagent Reference
## Critical Rule: Git Bash Path Conversion
Git Bash on Windows auto-converts Unix paths, breaking container commands.
| Solution | Example |
| ---------------------------- | -------------------------------------------------------- |
| `sh -c` with single quotes | `podman exec container sh -c '/usr/local/bin/script.sh'` |
| Double slashes | `podman exec container //usr//local//bin//script.sh` |
| MSYS_NO_PATHCONV=1 | `MSYS_NO_PATHCONV=1 podman exec ...` |
| Windows paths for host files | `podman cp "d:/path/file" container:/tmp/file` |
---
## Container Commands (Podman)
### Dev Container Operations
```bash
# List running containers
podman ps
# Container logs
podman logs flyer-crawler-dev
podman logs -f flyer-crawler-dev # Follow
# Execute in container
podman exec -it flyer-crawler-dev bash
podman exec -it flyer-crawler-dev npm run test:unit
# Restart container
podman restart flyer-crawler-dev
# Container resource usage
podman stats flyer-crawler-dev
```
### Test Execution (from Windows)
```bash
# Unit tests - pipe output for AI processing
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
# Integration tests
podman exec -it flyer-crawler-dev npm run test:integration
# Type check (CRITICAL before commit)
podman exec -it flyer-crawler-dev npm run type-check
# Specific test file
podman exec -it flyer-crawler-dev npm test -- --run src/hooks/useAuth.test.tsx
```
### Database Operations (from Windows)
```bash
# Reset dev database
podman exec -it flyer-crawler-dev npm run db:reset:dev
# Access PostgreSQL
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev
# Run SQL file (use MSYS_NO_PATHCONV to avoid path conversion)
MSYS_NO_PATHCONV=1 podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -f /app/sql/master_schema_rollup.sql
```
---
## PM2 Commands
### Production Server (via SSH)
```bash
# SSH to server
ssh root@projectium.com
# List all apps
pm2 list
# App status
pm2 show flyer-crawler-api
# Logs
pm2 logs flyer-crawler-api
pm2 logs --lines 100
# Restart apps
pm2 restart flyer-crawler-api
pm2 reload flyer-crawler-api # Zero-downtime
# Stop/Start
pm2 stop flyer-crawler-api
pm2 start flyer-crawler-api
# Delete and reload from config
pm2 delete all
pm2 start ecosystem.config.cjs
```
### PM2 Config: `ecosystem.config.cjs`
| App | Purpose | Memory | Mode |
| -------------------------------- | ---------------- | ------ | ------- |
| `flyer-crawler-api` | Express server | 500M | Cluster |
| `flyer-crawler-worker` | BullMQ worker | 1G | Fork |
| `flyer-crawler-analytics-worker` | Analytics worker | 1G | Fork |
Test variants: `*-test` suffix
---
## CI/CD Workflows
### Location: `.gitea/workflows/`
| Workflow | Trigger | Purpose |
| ----------------------------- | ------------ | ----------------------- |
| `deploy-to-test.yml` | Push to main | Auto-deploy to test env |
| `deploy-to-prod.yml` | Manual | Deploy to production |
| `manual-db-backup.yml` | Manual | Database backup |
| `manual-db-reset-test.yml` | Manual | Reset test database |
| `manual-db-reset-prod.yml` | Manual | Reset prod database |
| `manual-db-restore.yml` | Manual | Restore database |
| `manual-deploy-major.yml` | Manual | Major version release |
| `manual-redis-flush-prod.yml` | Manual | Flush Redis cache |
### Deploy to Test Pipeline Steps
1. Checkout code
2. Setup Node.js 20
3. `npm ci`
4. Bump patch version (creates git tag)
5. `npm run type-check`
6. `npm run test:unit`
7. Check schema hash against deployed DB
8. `npm run build`
9. Copy files to `/var/www/flyer-crawler-test.projectium.com/`
10. `pm2 reload ecosystem-test.config.cjs`
### Deploy to Production Pipeline Steps
1. Verify confirmation phrase ("deploy-to-prod")
2. Checkout `main` branch
3. `npm ci`
4. Bump minor version (creates git tag)
5. Check schema hash against prod DB
6. `npm run build` (with Sentry source maps)
7. Copy files to `/var/www/flyer-crawler.projectium.com/`
8. `pm2 reload ecosystem.config.cjs`
---
## Deployment Paths
| Environment | Path | Domain |
| ------------- | --------------------------------------------- | ----------------------------------- |
| Production | `/var/www/flyer-crawler.projectium.com/` | `flyer-crawler.projectium.com` |
| Test | `/var/www/flyer-crawler-test.projectium.com/` | `flyer-crawler-test.projectium.com` |
| Dev Container | `/app/` | `localhost:3000` |
---
## Environment Configuration
### Files
| File | Purpose |
| --------------------------- | ----------------------- |
| `ecosystem.config.cjs` | PM2 production config |
| `ecosystem-test.config.cjs` | PM2 test config |
| `src/config/env.ts` | Zod schema for env vars |
| `.env.example` | Template for env vars |
### Required Secrets (Gitea CI/CD)
| Category | Secrets |
| ---------- | -------------------------------------------------------------------------------------------- |
| Database | `DB_HOST`, `DB_USER_PROD`, `DB_PASSWORD_PROD`, `DB_DATABASE_PROD` |
| Test DB | `DB_USER_TEST`, `DB_PASSWORD_TEST`, `DB_DATABASE_TEST` |
| Redis | `REDIS_PASSWORD_PROD`, `REDIS_PASSWORD_TEST` |
| Auth | `JWT_SECRET`, `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, `GH_CLIENT_ID`, `GH_CLIENT_SECRET` |
| AI | `VITE_GOOGLE_GENAI_API_KEY`, `VITE_GOOGLE_GENAI_API_KEY_TEST` |
| Monitoring | `SENTRY_DSN`, `SENTRY_DSN_TEST`, `SENTRY_AUTH_TOKEN` |
| Maps | `GOOGLE_MAPS_API_KEY` |
### Adding New Secret
1. Add to Gitea Settings > Secrets
2. Update workflow YAML: `SENTRY_DSN: ${{ secrets.SENTRY_DSN }}`
3. Update `ecosystem.config.cjs`
4. Update `src/config/env.ts` Zod schema
5. Update `.env.example`
---
## Redis Commands
### Dev Container
```bash
# Access Redis CLI
podman exec -it flyer-crawler-dev redis-cli
# Common commands
KEYS *
FLUSHALL
INFO
```
### Production
```bash
# Via SSH
ssh root@projectium.com
redis-cli -a $REDIS_PASSWORD
# Flush cache (use with caution)
# Or use manual-redis-flush-prod.yml workflow
```
---
## Health Checks
### Endpoints
| Endpoint | Purpose |
| -------------------------- | -------------------------------------- |
| `GET /api/health` | Basic health check |
| `GET /api/health/detailed` | Full system status (DB, Redis, queues) |
### Manual Health Check
```bash
# From Windows
curl http://localhost:3000/api/health
# Or via Podman
podman exec -it flyer-crawler-dev curl http://localhost:3000/api/health
```
---
## Log Locations
### Production Server
```bash
# PM2 logs
~/.pm2/logs/
# NGINX logs
/var/log/nginx/access.log
/var/log/nginx/error.log
# Application logs (via PM2)
pm2 logs flyer-crawler-api --lines 200
```
### Dev Container
```bash
# View container logs
podman logs flyer-crawler-dev
# Follow logs
podman logs -f flyer-crawler-dev
```
---
## Backup/Restore
### Database Backup (Manual Workflow)
Trigger `manual-db-backup.yml` from Gitea Actions UI.
### Manual Backup
```bash
# SSH to server
ssh root@projectium.com
# Backup
PGPASSWORD=$DB_PASSWORD pg_dump -h $DB_HOST -U $DB_USER $DB_NAME > backup_$(date +%Y%m%d).sql
# Restore
PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -U $DB_USER $DB_NAME < backup.sql
```
---
## Bugsink (Error Tracking)
### Dev Container Token Generation
```bash
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
```
### Production Token Generation
```bash
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
```
---
## Common Troubleshooting
### Container Won't Start
```bash
# Check logs
podman logs flyer-crawler-dev
# Inspect container
podman inspect flyer-crawler-dev
# Remove and recreate
podman rm -f flyer-crawler-dev
# Then recreate with docker-compose or podman run
```
### Database Connection Issues
```bash
# Test connection inside container
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -c "SELECT 1"
# Check if PostgreSQL is running
podman exec -it flyer-crawler-dev pg_isready
```
### PM2 App Keeps Restarting
```bash
# Check logs
pm2 logs flyer-crawler-api --err --lines 100
# Check for memory issues
pm2 monit
# View app details
pm2 show flyer-crawler-api
```
### Redis Connection Issues
```bash
# Test Redis inside container
podman exec -it flyer-crawler-dev redis-cli ping
# Check Redis logs
podman logs flyer-crawler-redis
```

View File

@@ -0,0 +1,410 @@
# Integrations Subagent Reference
## MCP Servers Overview
| Server | Purpose | URL | Tools Prefix |
| ------------------ | ------------------------- | ----------------------------------------------------------------- | -------------------------- |
| `bugsink` | Production error tracking | `https://bugsink.projectium.com` | `mcp__bugsink__*` |
| `localerrors` | Dev container errors | `http://127.0.0.1:8000` | `mcp__localerrors__*` |
| `devdb` | Dev PostgreSQL | `postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev` | `mcp__devdb__*` |
| `gitea-projectium` | Gitea API | `gitea.projectium.com` | `mcp__gitea-projectium__*` |
| `gitea-torbonium` | Gitea API | `gitea.torbonium.com` | `mcp__gitea-torbonium__*` |
| `podman` | Container management | - | `mcp__podman__*` |
| `filesystem` | File system access | - | `mcp__filesystem__*` |
| `memory` | Knowledge graph | - | `mcp__memory__*` |
| `redis` | Cache management | `localhost:6379` | `mcp__redis__*` |
---
## MCP Server Configuration
### Global Config: `~/.claude/settings.json`
Used for production/remote servers (HTTPS works fine).
```json
{
"mcpServers": {
"bugsink": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "https://bugsink.projectium.com",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
}
}
}
```
### Project Config: `.mcp.json`
**CRITICAL:** Use project-level `.mcp.json` for localhost servers. Global config has issues loading localhost stdio MCP servers.
```json
{
"mcpServers": {
"localerrors": {
"command": "d:\\nodejs\\node.exe",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "http://127.0.0.1:8000",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
},
"devdb": {
"command": "D:\\nodejs\\npx.cmd",
"args": [
"-y",
"@modelcontextprotocol/server-postgres",
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
]
}
}
}
```
---
## Bugsink Integration
### API Token Generation
**Bugsink 2.0.11 has NO UI for API tokens.** Use Django management command.
#### Dev Container
```bash
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
```
#### Production
```bash
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
```
### Bugsink MCP Tools
| Tool | Purpose |
| ----------------- | ---------------------- |
| `test_connection` | Verify connection |
| `list_projects` | List all projects |
| `list_issues` | List issues by project |
| `get_issue` | Issue details |
| `list_events` | Events for an issue |
| `get_event` | Event details |
| `get_stacktrace` | Formatted stacktrace |
### Usage Example
```typescript
// Test connection
mcp__bugsink__test_connection();
// List production issues
mcp__bugsink__list_issues({ project_id: 1, status: 'unresolved', limit: 10 });
// Get stacktrace
mcp__bugsink__get_stacktrace({ event_id: 'uuid-here' });
```
---
## PostgreSQL MCP Integration
### Setup
```bash
# Uses @modelcontextprotocol/server-postgres package
# Connection string in .mcp.json
```
### Tools
| Tool | Purpose |
| ------- | --------------------- |
| `query` | Execute read-only SQL |
### Usage Example
```typescript
mcp__devdb__query({ sql: 'SELECT * FROM public.users LIMIT 5' });
mcp__devdb__query({ sql: 'SELECT COUNT(*) FROM public.flyers' });
```
---
## Gitea MCP Integration
### Common Tools
| Tool | Purpose |
| ------------------------- | ----------------- |
| `get_my_user_info` | Current user info |
| `list_my_repos` | List repositories |
| `get_issue_by_index` | Issue details |
| `list_repo_issues` | Repository issues |
| `create_issue` | Create new issue |
| `create_pull_request` | Create PR |
| `list_repo_pull_requests` | List PRs |
| `get_file_content` | Read file |
| `list_repo_commits` | Commit history |
### Usage Example
```typescript
// List issues
mcp__gitea -
projectium__list_repo_issues({
owner: 'james',
repo: 'flyer-crawler',
state: 'open',
});
// Create issue
mcp__gitea -
projectium__create_issue({
owner: 'james',
repo: 'flyer-crawler',
title: 'Bug: Description',
body: 'Details here',
});
// Get file content
mcp__gitea -
projectium__get_file_content({
owner: 'james',
repo: 'flyer-crawler',
ref: 'main',
filePath: 'CLAUDE.md',
});
```
---
## Redis MCP Integration
### Tools
| Tool | Purpose |
| -------- | -------------------- |
| `get` | Get key value |
| `set` | Set key value |
| `delete` | Delete key(s) |
| `list` | List keys by pattern |
### Usage Example
```typescript
// List cache keys
mcp__redis__list({ pattern: 'flyer:*' });
// Get cached value
mcp__redis__get({ key: 'flyer:123' });
// Set with expiration
mcp__redis__set({ key: 'test:key', value: 'data', expireSeconds: 3600 });
// Delete key
mcp__redis__delete({ key: 'test:key' });
```
---
## Podman MCP Integration
### Tools
| Tool | Purpose |
| ------------------- | ----------------- |
| `container_list` | List containers |
| `container_logs` | View logs |
| `container_inspect` | Container details |
| `container_stop` | Stop container |
| `container_remove` | Remove container |
| `container_run` | Run container |
| `image_list` | List images |
| `image_pull` | Pull image |
### Usage Example
```typescript
// List running containers
mcp__podman__container_list();
// View container logs
mcp__podman__container_logs({ name: 'flyer-crawler-dev' });
// Inspect container
mcp__podman__container_inspect({ name: 'flyer-crawler-dev' });
```
---
## Memory MCP (Knowledge Graph)
### Tools
| Tool | Purpose |
| ------------------ | -------------------- |
| `read_graph` | Read entire graph |
| `search_nodes` | Search by query |
| `open_nodes` | Get specific nodes |
| `create_entities` | Create entities |
| `create_relations` | Create relationships |
| `add_observations` | Add observations |
| `delete_entities` | Delete entities |
### Usage Example
```typescript
// Search for context
mcp__memory__search_nodes({ query: 'flyer-crawler' });
// Read full graph
mcp__memory__read_graph();
// Create entity
mcp__memory__create_entities({
entities: [
{
name: 'FlyCrawler',
entityType: 'Project',
observations: ['Uses PostgreSQL', 'Express backend'],
},
],
});
```
---
## Filesystem MCP
### Tools
| Tool | Purpose |
| ---------------- | --------------------- |
| `read_text_file` | Read file contents |
| `write_file` | Write file |
| `edit_file` | Edit file |
| `list_directory` | List directory |
| `directory_tree` | Tree view |
| `search_files` | Find files by pattern |
### Usage Example
```typescript
// Read file
mcp__filesystem__read_text_file({ path: 'd:\\gitea\\project\\README.md' });
// List directory
mcp__filesystem__list_directory({ path: 'd:\\gitea\\project\\src' });
// Search for files
mcp__filesystem__search_files({
path: 'd:\\gitea\\project',
pattern: '**/*.test.ts',
});
```
---
## Troubleshooting MCP Servers
### Server Not Loading
1. **Check server name** - Avoid shared prefixes (e.g., `bugsink` and `bugsink-dev`)
2. **Use project-level `.mcp.json`** for localhost servers
3. **Restart Claude Code** after config changes
### Test Connection Manually
```bash
# Bugsink
set BUGSINK_URL=http://localhost:8000
set BUGSINK_TOKEN=<token>
node d:\gitea\bugsink-mcp\dist\index.js
# PostgreSQL
npx -y @modelcontextprotocol/server-postgres "postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
```
### Check Claude Debug Logs
```
C:\Users\<username>\.claude\debug\*.txt
```
Look for "Starting connection" messages - missing server = never started.
---
## External API Integrations
### Gemini AI (Flyer Extraction)
| Config | Location |
| ------- | ---------------------------------------------- |
| API Key | `VITE_GOOGLE_GENAI_API_KEY` / `GEMINI_API_KEY` |
| Service | `src/services/flyerAiProcessor.server.ts` |
| Client | `@google/genai` package |
### Google OAuth
| Config | Location |
| ------------- | ------------------------ |
| Client ID | `GOOGLE_CLIENT_ID` |
| Client Secret | `GOOGLE_CLIENT_SECRET` |
| Service | `src/config/passport.ts` |
### GitHub OAuth
| Config | Location |
| ------------- | ------------------------------------------- |
| Client ID | `GH_CLIENT_ID` / `GITHUB_CLIENT_ID` |
| Client Secret | `GH_CLIENT_SECRET` / `GITHUB_CLIENT_SECRET` |
| Service | `src/config/passport.ts` |
### Google Maps (Geocoding)
| Config | Location |
| ------- | ----------------------------------------------- |
| API Key | `GOOGLE_MAPS_API_KEY` |
| Service | `src/services/googleGeocodingService.server.ts` |
### Nominatim (Fallback Geocoding)
| Config | Location |
| ------- | -------------------------------------------------- |
| URL | `https://nominatim.openstreetmap.org` |
| Service | `src/services/nominatimGeocodingService.server.ts` |
### Sentry (Error Tracking)
| Config | Location |
| -------------- | ------------------------------------------------- |
| DSN | `SENTRY_DSN` (server), `VITE_SENTRY_DSN` (client) |
| Auth Token | `SENTRY_AUTH_TOKEN` (source map upload) |
| Server Service | `src/services/sentry.server.ts` |
| Client Service | `src/services/sentry.client.ts` |
### SMTP (Email)
| Config | Location |
| ----------- | ------------------------------------- |
| Host | `SMTP_HOST` |
| Port | `SMTP_PORT` |
| Credentials | `SMTP_USER`, `SMTP_PASS` |
| Service | `src/services/emailService.server.ts` |
---
## Related Documentation
| Document | Purpose |
| -------------------------------- | ----------------------- |
| `BUGSINK-MCP-TROUBLESHOOTING.md` | MCP server issues |
| `POSTGRES-MCP-SETUP.md` | PostgreSQL MCP setup |
| `DEV-CONTAINER-BUGSINK.md` | Local Bugsink setup |
| `BUGSINK-SYNC.md` | Bugsink synchronization |

View File

@@ -0,0 +1,358 @@
# Tester Subagent Reference
## Critical Rule: Linux Only (ADR-014)
**ALL tests MUST run in the dev container.** Windows test results are unreliable.
| Result | Interpretation |
| ------------------------- | -------------------- |
| Pass Windows / Fail Linux | BROKEN - must fix |
| Fail Windows / Pass Linux | PASSING - acceptable |
---
## Test Commands
### From Windows Host (via Podman)
```bash
# Unit tests (~2900 tests) - pipe to file for AI processing
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
# Integration tests (requires DB/Redis)
podman exec -it flyer-crawler-dev npm run test:integration
# E2E tests (requires all services)
podman exec -it flyer-crawler-dev npm run test:e2e
# Specific test file
podman exec -it flyer-crawler-dev npm test -- --run src/hooks/useAuth.test.tsx
# Type checking (CRITICAL before commit)
podman exec -it flyer-crawler-dev npm run type-check
# Coverage report
podman exec -it flyer-crawler-dev npm run test:coverage
```
### Inside Dev Container
```bash
npm test # All tests
npm run test:unit # Unit tests only
npm run test:integration # Integration tests
npm run test:e2e # E2E tests
npm run type-check # TypeScript check
```
---
## Test File Locations
| Test Type | Location | Config |
| ----------- | --------------------------------------------- | ------------------------------ |
| Unit | `src/**/*.test.ts`, `src/**/*.test.tsx` | `vite.config.ts` |
| Integration | `src/tests/integration/*.integration.test.ts` | `vitest.config.integration.ts` |
| E2E | `src/tests/e2e/*.e2e.test.ts` | `vitest.config.e2e.ts` |
| Setup | `src/tests/setup/*.ts` | - |
| Helpers | `src/tests/utils/*.ts` | - |
---
## Test Helpers
### Location: `src/tests/utils/`
| Helper | Purpose | Import |
| ----------------------- | --------------------------------------------------------------- | ----------------------------------- |
| `testHelpers.ts` | `createAndLoginUser()`, `getTestBaseUrl()`, `getFlyerBaseUrl()` | `../tests/utils/testHelpers` |
| `cleanup.ts` | `cleanupDb({ userIds, flyerIds })` | `../tests/utils/cleanup` |
| `mockFactories.ts` | `createMockStore()`, `createMockAddress()`, `createMockFlyer()` | `../tests/utils/mockFactories` |
| `storeHelpers.ts` | `createStoreWithLocation()`, `cleanupStoreLocations()` | `../tests/utils/storeHelpers` |
| `poll.ts` | `poll(fn, predicate, options)` - wait for async conditions | `../tests/utils/poll` |
| `mockLogger.ts` | Mock pino logger for tests | `../tests/utils/mockLogger` |
| `createTestApp.ts` | Create Express app instance for route tests | `../tests/utils/createTestApp` |
| `createMockRequest.ts` | Create mock Express request objects | `../tests/utils/createMockRequest` |
| `cleanupFiles.ts` | Clean up test file uploads | `../tests/utils/cleanupFiles` |
| `websocketTestUtils.ts` | WebSocket testing utilities | `../tests/utils/websocketTestUtils` |
### Usage Examples
```typescript
// Create authenticated user for tests
import { createAndLoginUser, TEST_PASSWORD } from '../tests/utils/testHelpers';
const { user, token } = await createAndLoginUser({
email: `test-${Date.now()}@example.com`,
request: request(app), // For integration tests
role: 'admin', // Optional: make admin
});
// Cleanup after tests
import { cleanupDb } from '../tests/utils/cleanup';
afterEach(async () => {
await cleanupDb({ userIds: [user.user.user_id] });
});
// Wait for async operation
import { poll } from '../tests/utils/poll';
await poll(
() => db.userRepo.findUserByEmail(email, logger),
(user) => !!user,
{ timeout: 5000, interval: 500, description: 'user to be findable' },
);
// Create mock data
import { createMockStore, createMockFlyer } from '../tests/utils/mockFactories';
const mockStore = createMockStore({ name: 'Test Store' });
const mockFlyer = createMockFlyer({ store_id: mockStore.store_id });
```
---
## Test Setup Files
| File | Purpose |
| --------------------------------------------- | ---------------------------------------- |
| `src/tests/setup/tests-setup-unit.ts` | Unit test setup (mocks, DOM environment) |
| `src/tests/setup/tests-setup-integration.ts` | Integration test setup (DB connections) |
| `src/tests/setup/global-setup.ts` | Global setup for unit tests |
| `src/tests/setup/integration-global-setup.ts` | Global setup for integration tests |
| `src/tests/setup/e2e-global-setup.ts` | Global setup for E2E tests |
| `src/tests/setup/mockHooks.ts` | React hook mocking utilities |
| `src/tests/setup/mockUI.ts` | UI component mocking |
| `src/tests/setup/globalApiMock.ts` | API mocking setup |
---
## Known Integration Test Issues
### 1. Vitest globalSetup Context Isolation
**Problem**: globalSetup runs in separate Node.js context. Singletons/mocks don't share instances.
**Affected**: BullMQ worker mocks
**Solution**: Use `.todo()`, test-only API endpoints, or Redis-based flags.
### 2. Cleanup Queue Race Condition
**Problem**: Cleanup worker processes jobs before test verification.
**Solution**:
```typescript
const { cleanupQueue } = await import('../../services/queues.server');
await cleanupQueue.drain();
await cleanupQueue.pause();
// ... run test ...
await cleanupQueue.resume();
```
### 3. Cache Stale After Direct SQL
**Problem**: Direct `pool.query()` bypasses cache invalidation.
**Solution**:
```typescript
await pool.query('INSERT INTO flyers ...');
await cacheService.invalidateFlyers(); // Add this
```
### 4. File Upload Filename Collisions
**Problem**: Multer predictable filenames cause race conditions.
**Solution**:
```typescript
const filename = `test-${Date.now()}-${Math.round(Math.random() * 1e9)}.jpg`;
```
### 5. Response Format Mismatches
**Problem**: API response structure changes (`data.jobId` vs `data.job.id`).
**Solution**: Log response bodies, update assertions to match actual format.
---
## Test Patterns
### Unit Test Pattern
```typescript
import { describe, it, expect, vi, beforeEach } from 'vitest';
describe('MyService', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('should do something', () => {
// Arrange
const input = { data: 'test' };
// Act
const result = myService.process(input);
// Assert
expect(result).toBe('expected');
});
});
```
### Integration Test Pattern
```typescript
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import request from 'supertest';
import { createAndLoginUser } from '../utils/testHelpers';
import { cleanupDb } from '../utils/cleanup';
describe('API Integration', () => {
let user, token;
beforeEach(async () => {
const result = await createAndLoginUser({ request: request(app) });
user = result.user;
token = result.token;
});
afterEach(async () => {
await cleanupDb({ userIds: [user.user.user_id] });
});
it('GET /api/resource returns data', async () => {
const res = await request(app).get('/api/resource').set('Authorization', `Bearer ${token}`);
expect(res.status).toBe(200);
expect(res.body.success).toBe(true);
});
});
```
### Route Test Pattern
```typescript
import { describe, it, expect, vi, beforeEach } from 'vitest';
import request from 'supertest';
import { createTestApp } from '../tests/utils/createTestApp';
vi.mock('../services/db/flyer.db', () => ({
getFlyerById: vi.fn(),
}));
describe('Flyer Routes', () => {
let app;
beforeEach(() => {
vi.clearAllMocks();
app = createTestApp();
});
it('GET /api/flyers/:id returns flyer', async () => {
const mockFlyer = { id: '123', name: 'Test' };
vi.mocked(flyerRepo.getFlyerById).mockResolvedValue(mockFlyer);
const res = await request(app).get('/api/flyers/123');
expect(res.status).toBe(200);
expect(res.body.data).toEqual(mockFlyer);
});
});
```
---
## Mocking Patterns
### Mock Modules
```typescript
// At top of test file
vi.mock('../services/db/flyer.db', () => ({
getFlyerById: vi.fn(),
listFlyers: vi.fn(),
}));
// In test
import * as flyerDb from '../services/db/flyer.db';
vi.mocked(flyerDb.getFlyerById).mockResolvedValue(mockFlyer);
```
### Mock React Query
```typescript
vi.mock('@tanstack/react-query', async () => {
const actual = await vi.importActual('@tanstack/react-query');
return {
...actual,
useQuery: vi.fn().mockReturnValue({
data: mockData,
isLoading: false,
error: null,
}),
};
});
```
### Mock Pino Logger
```typescript
import { createMockLogger } from '../tests/utils/mockLogger';
const mockLogger = createMockLogger();
```
---
## Testing React Components
```typescript
import { render, screen, fireEvent, waitFor } from '@testing-library/react';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { BrowserRouter } from 'react-router-dom';
const createWrapper = () => {
const queryClient = new QueryClient({
defaultOptions: { queries: { retry: false } },
});
return ({ children }) => (
<QueryClientProvider client={queryClient}>
<BrowserRouter>{children}</BrowserRouter>
</QueryClientProvider>
);
};
it('renders component', () => {
render(<MyComponent />, { wrapper: createWrapper() });
expect(screen.getByText('Expected Text')).toBeInTheDocument();
});
```
---
## Test Coverage
```bash
# Generate coverage report
podman exec -it flyer-crawler-dev npm run test:coverage
# View HTML report
# Coverage reports generated in coverage/ directory
```
---
## Debugging Tests
```bash
# Verbose output
npm test -- --reporter=verbose
# Run single test with debugging
DEBUG=* npm test -- --run src/path/to/test.test.ts
# Vitest UI (interactive)
npm run test:ui
```

View File

@@ -76,16 +76,18 @@ This provides a secondary error capture path for:
- Database function errors and slow queries
- Historical error analysis from log files
### 5. MCP Server Integration: sentry-selfhosted-mcp
### 5. MCP Server Integration: bugsink-mcp
For AI tool integration (Claude Code, Cursor, etc.), we use the open-source [sentry-selfhosted-mcp](https://github.com/ddfourtwo/sentry-selfhosted-mcp) server:
For AI tool integration (Claude Code, Cursor, etc.), we use the open-source [bugsink-mcp](https://github.com/j-shelfwood/bugsink-mcp) server:
- **No code changes required**: Configurable via environment variables
- **Capabilities**: List projects, get issues, view events, update status, add comments
- **Capabilities**: List projects, get issues, view events, get stacktraces, manage releases
- **Configuration**:
- `SENTRY_URL`: Points to Bugsink instance
- `SENTRY_AUTH_TOKEN`: API token from Bugsink
- `SENTRY_ORG_SLUG`: Organization identifier
- `BUGSINK_URL`: Points to Bugsink instance (`http://localhost:8000` for dev, `https://bugsink.projectium.com` for prod)
- `BUGSINK_API_TOKEN`: API token from Bugsink (created via Django management command)
- `BUGSINK_ORG_SLUG`: Organization identifier (usually "sentry")
**Note:** Despite the name `sentry-selfhosted-mcp` mentioned in earlier drafts of this ADR, the actual MCP server used is `bugsink-mcp` which is specifically designed for Bugsink's API structure.
## Architecture
@@ -144,12 +146,12 @@ External (Developer Machine):
┌──────────────────────────────────────┐
│ Claude Code / Cursor / VS Code │
│ ┌────────────────────────────────┐ │
│ │ sentry-selfhosted-mcp │ │
│ │ bugsink-mcp │ │
│ │ (MCP Server) │ │
│ │ │ │
│ │ SENTRY_URL=http://localhost:8000
│ │ SENTRY_AUTH_TOKEN=... │ │
│ │ SENTRY_ORG_SLUG=... │ │
│ │ BUGSINK_URL=http://localhost:8000
│ │ BUGSINK_API_TOKEN=... │ │
│ │ BUGSINK_ORG_SLUG=... │ │
│ └────────────────────────────────┘ │
└──────────────────────────────────────┘
```
@@ -279,7 +281,7 @@ output {
- Configure Redis log monitoring (connection errors, slow commands)
7. **MCP server documentation**:
- Document `sentry-selfhosted-mcp` setup in CLAUDE.md
- Document `bugsink-mcp` setup in CLAUDE.md
8. **PostgreSQL function logging** (future):
- Configure PostgreSQL to log function execution errors
@@ -318,5 +320,5 @@ output {
- [Bugsink Docker Install](https://www.bugsink.com/docs/docker-install/)
- [@sentry/node Documentation](https://docs.sentry.io/platforms/javascript/guides/node/)
- [@sentry/react Documentation](https://docs.sentry.io/platforms/javascript/guides/react/)
- [sentry-selfhosted-mcp](https://github.com/ddfourtwo/sentry-selfhosted-mcp)
- [bugsink-mcp](https://github.com/j-shelfwood/bugsink-mcp)
- [Logstash Reference](https://www.elastic.co/guide/en/logstash/current/index.html)

View File

@@ -384,21 +384,21 @@ const AuthCallback = () => {
### Mitigation
- Document OAuth enablement steps clearly (see AUTHENTICATION.md).
- Document OAuth enablement steps clearly (see [../architecture/AUTHENTICATION.md](../architecture/AUTHENTICATION.md)).
- Consider adding OAuth provider ID columns for future account linking.
- Use URL fragment (`#token=`) instead of query parameter for callback.
## Key Files
| File | Purpose |
| ------------------------------- | ------------------------------------------------ |
| `src/routes/passport.routes.ts` | Passport strategies (local, JWT, OAuth) |
| `src/routes/auth.routes.ts` | Auth endpoints (login, register, refresh, OAuth) |
| `src/services/authService.ts` | Auth business logic |
| `src/services/db/user.db.ts` | User database operations |
| `src/config/env.ts` | Environment variable validation |
| `AUTHENTICATION.md` | OAuth setup guide |
| `.env.example` | Environment variable template |
| File | Purpose |
| ------------------------------------------------------ | ------------------------------------------------ |
| `src/routes/passport.routes.ts` | Passport strategies (local, JWT, OAuth) |
| `src/routes/auth.routes.ts` | Auth endpoints (login, register, refresh, OAuth) |
| `src/services/authService.ts` | Auth business logic |
| `src/services/db/user.db.ts` | User database operations |
| `src/config/env.ts` | Environment variable validation |
| [AUTHENTICATION.md](../architecture/AUTHENTICATION.md) | OAuth setup guide |
| `.env.example` | Environment variable template |
## Related ADRs

View File

@@ -149,7 +149,7 @@ All synced issues also receive the `source:bugsink` label.
```bash
# Bugsink Configuration
BUGSINK_URL=https://bugsink.projectium.com
BUGSINK_API_TOKEN=77deaa5e... # From Bugsink Settings > API Keys
BUGSINK_API_TOKEN=77deaa5e... # Created via Django management command (see BUGSINK-SYNC.md)
# Gitea Configuration
GITEA_URL=https://gitea.projectium.com

View File

@@ -0,0 +1,877 @@
# Flyer Crawler - System Architecture Overview
**Version**: 0.12.5
**Last Updated**: 2026-01-22
**Platform**: Linux (Production and Development)
---
## Table of Contents
1. [Executive Summary](#executive-summary)
2. [System Architecture Diagram](#system-architecture-diagram)
3. [Technology Stack](#technology-stack)
4. [System Components](#system-components)
5. [Data Flow](#data-flow)
6. [Architecture Layers](#architecture-layers)
7. [Key Entities](#key-entities)
8. [Authentication Flow](#authentication-flow)
9. [Background Processing](#background-processing)
10. [Deployment Architecture](#deployment-architecture)
11. [Design Principles and ADRs](#design-principles-and-adrs)
12. [Key Files Reference](#key-files-reference)
---
## Executive Summary
**Flyer Crawler** is a grocery deal extraction and analysis platform that uses AI-powered processing to extract deals from grocery store flyer images and PDFs. The system provides users with features including watchlists, price history tracking, shopping lists, deal alerts, and recipe management.
### Core Capabilities
| Domain | Description |
| ------------------------- | --------------------------------------------------------------------------------------- |
| **Deal Extraction** | AI-powered extraction of deals from grocery store flyer images/PDFs using Google Gemini |
| **Price Tracking** | Historical price data, trend analysis, and price alerts |
| **User Features** | Watchlists, shopping lists, recipes, pantry management, achievements |
| **Real-time Updates** | WebSocket-based notifications for price alerts and processing status |
| **Background Processing** | Asynchronous job queues for flyer processing, emails, and analytics |
---
## System Architecture Diagram
```
+-----------------------------------------------------------------------------------+
| CLIENT LAYER |
+-----------------------------------------------------------------------------------+
| |
| +-------------------+ +-------------------+ +-------------------+ |
| | Web Browser | | Mobile PWA | | API Clients | |
| | (React SPA) | | (React SPA) | | (REST/JSON) | |
| +--------+----------+ +--------+----------+ +--------+----------+ |
| | | | |
+-------------|-------------------------|-------------------------|------------------+
| | |
v v v
+-----------------------------------------------------------------------------------+
| NGINX REVERSE PROXY |
| - SSL/TLS Termination - Rate Limiting - Static Asset Serving |
| - Load Balancing - Compression - WebSocket Proxying |
| - Flyer Images (/flyer-images/) with 7-day cache |
+----------------------------------+------------------------------------------------+
|
v
+-----------------------------------------------------------------------------------+
| APPLICATION LAYER |
+-----------------------------------------------------------------------------------+
| |
| +-----------------------------------------------------------------------------+ |
| | EXPRESS.JS SERVER (Node.js) | |
| | | |
| | +-------------------------+ +-------------------------+ | |
| | | Routes Layer | | Middleware Chain | | |
| | | - API Endpoints | | - Authentication | | |
| | | - Request Validation | | - Rate Limiting | | |
| | | - Response Formatting | | - Logging | | |
| | +------------+------------+ | - Error Handling | | |
| | | +-------------------------+ | |
| | v | |
| | +-------------------------+ +-------------------------+ | |
| | | Services Layer | | External Services | | |
| | | - Business Logic | | - Google Gemini AI | | |
| | | - Transaction Coord. | | - Google Maps API | | |
| | | - Event Publishing | | - OAuth Providers | | |
| | +------------+------------+ | - Email (SMTP) | | |
| | | +-------------------------+ | |
| | v | |
| | +-------------------------+ | |
| | | Repository Layer | | |
| | | - Database Access | | |
| | | - Query Construction | | |
| | | - Entity Mapping | | |
| | +------------+------------+ | |
| | | | |
| +---------------|-------------------------------------------------------------+ |
| | |
+------------------|----------------------------------------------------------------+
|
v
+-----------------------------------------------------------------------------------+
| DATA LAYER |
+-----------------------------------------------------------------------------------+
| |
| +---------------------------+ +---------------------------+ |
| | PostgreSQL 16 | | Redis 7 | |
| | (with PostGIS) | | | |
| | | | - Session Cache | |
| | - Primary Data Store | | - Query Cache | |
| | - Geographic Queries | | - Job Queue Backing | |
| | - Full-Text Search | | - Rate Limit Counters | |
| | - Stored Functions | | - Real-time Pub/Sub | |
| +---------------------------+ +---------------------------+ |
| |
+-----------------------------------------------------------------------------------+
+-----------------------------------------------------------------------------------+
| BACKGROUND PROCESSING LAYER |
+-----------------------------------------------------------------------------------+
| |
| +---------------------------+ +---------------------------+ |
| | PM2 Process | | BullMQ Workers | |
| | Manager | | | |
| | | | - Flyer Processing | |
| | - Process Clustering | | - Receipt Processing | |
| | - Auto-restart | | - Email Sending | |
| | - Log Management | | - Analytics Reports | |
| | - Health Monitoring | | - File Cleanup | |
| +---------------------------+ | - Token Cleanup | |
| | - Expiry Alerts | |
| | - Barcode Detection | |
| +---------------------------+ |
| |
+-----------------------------------------------------------------------------------+
+-----------------------------------------------------------------------------------+
| OBSERVABILITY LAYER |
+-----------------------------------------------------------------------------------+
| |
| +------------------+ +------------------+ +------------------+ |
| | Bugsink/Sentry | | Pino Logger | | Logstash | |
| | (Error Track) | | (Structured) | | (Aggregation) | |
| +------------------+ +------------------+ +------------------+ |
| |
+-----------------------------------------------------------------------------------+
```
---
## Technology Stack
### Core Technologies
| Component | Technology | Version | Purpose |
| ---------------------- | ---------- | -------- | -------------------------------- |
| **Runtime** | Node.js | 22.x LTS | Server-side JavaScript runtime |
| **Language** | TypeScript | 5.9.x | Type-safe JavaScript superset |
| **Web Framework** | Express.js | 5.1.x | HTTP server and routing |
| **Frontend Framework** | React | 19.2.x | UI component library |
| **Build Tool** | Vite | 7.2.x | Frontend bundling and dev server |
### Data Storage
| Component | Technology | Version | Purpose |
| --------------------- | ---------------- | ------- | ---------------------------------------------- |
| **Primary Database** | PostgreSQL | 16.x | Relational data storage |
| **Spatial Extension** | PostGIS | 3.x | Geographic queries and store location features |
| **Cache & Queues** | Redis | 7.x | Caching, session storage, job queue backing |
| **File Storage** | Local Filesystem | - | Uploaded flyers and processed images |
### AI and External Services
| Component | Technology | Purpose |
| --------------- | --------------------------- | --------------------------------------- |
| **AI Provider** | Google Gemini | Flyer data extraction, image analysis |
| **Geocoding** | Google Maps API / Nominatim | Address geocoding and location services |
| **OAuth** | Google, GitHub | Social authentication |
| **Email** | Nodemailer (SMTP) | Transactional emails |
### Background Processing
| Component | Technology | Version | Purpose |
| ------------------- | ---------- | ------- | --------------------------------- |
| **Job Queues** | BullMQ | 5.65.x | Reliable async job processing |
| **Process Manager** | PM2 | Latest | Process management and clustering |
| **Scheduler** | node-cron | 4.2.x | Scheduled tasks |
### Frontend Stack
| Component | Technology | Version | Purpose |
| -------------------- | -------------- | ------- | ---------------------------------------- |
| **State Management** | TanStack Query | 5.90.x | Server state caching and synchronization |
| **Routing** | React Router | 7.9.x | Client-side routing |
| **Styling** | Tailwind CSS | 4.1.x | Utility-first CSS framework |
| **Icons** | Lucide React | 0.555.x | Icon components |
| **Charts** | Recharts | 3.4.x | Data visualization |
### Observability and Quality
| Component | Technology | Purpose |
| --------------------- | ---------------- | ----------------------------- |
| **Error Tracking** | Sentry / Bugsink | Error monitoring and alerting |
| **Logging** | Pino | Structured JSON logging |
| **Log Aggregation** | Logstash | Centralized log collection |
| **Testing** | Vitest | Unit and integration testing |
| **API Documentation** | Swagger/OpenAPI | Interactive API documentation |
---
## System Components
### Frontend (React/Vite)
The frontend is a single-page application (SPA) built with React 19 and Vite.
**Key Characteristics**:
- Server state management via TanStack Query
- Neo-Brutalism design system (ADR-012)
- Responsive design for mobile and desktop
- PWA-capable for offline access
**Directory Structure**:
```
src/
+-- components/ # Reusable UI components
+-- contexts/ # React context providers
+-- features/ # Feature-specific modules (ADR-047)
+-- hooks/ # Custom React hooks
+-- layouts/ # Page layout components
+-- pages/ # Route page components
+-- services/ # API client services
```
### Backend (Express/Node.js)
The backend is a RESTful API server built with Express.js 5.
**Key Characteristics**:
- Layered architecture (Routes -> Services -> Repositories)
- JWT-based authentication with OAuth support
- Request validation via Zod schemas
- Structured logging with Pino
- Standardized error handling (ADR-001)
**API Route Modules**:
| Route | Purpose |
|-------|---------|
| `/api/auth` | Authentication (login, register, OAuth) |
| `/api/users` | User profile management |
| `/api/flyers` | Flyer CRUD and processing |
| `/api/recipes` | Recipe management |
| `/api/deals` | Best prices and deal discovery |
| `/api/stores` | Store management |
| `/api/admin` | Administrative functions |
| `/api/health` | Health checks and monitoring |
### Database (PostgreSQL/PostGIS)
PostgreSQL serves as the primary data store with PostGIS extension for geographic queries.
**Key Features**:
- UUID primary keys for user data
- BIGINT IDENTITY for auto-incrementing IDs
- PostGIS geography types for store locations
- Stored functions for complex business logic
- Triggers for automated updates (e.g., `item_count` maintenance)
### Cache (Redis)
Redis provides caching and backing for the job queue system.
**Usage Patterns**:
- Query result caching (flyers, prices, stats)
- Rate limiting counters
- BullMQ job queue storage
- Session token storage
### AI (Google Gemini)
Google Gemini powers the AI extraction capabilities.
**Capabilities**:
- Flyer image analysis and data extraction
- Store name and logo detection
- Deal item parsing (name, price, quantity)
- Date range extraction
- Category classification
### Background Workers (BullMQ/PM2)
BullMQ workers handle asynchronous processing tasks. PM2 manages both the API server and worker processes in production and development environments (ADR-014).
**Dev Container PM2 Processes**:
| Process | Script | Purpose |
| -------------------------- | ---------------------------------- | -------------------------- |
| `flyer-crawler-api-dev` | `tsx watch server.ts` | API server with hot reload |
| `flyer-crawler-worker-dev` | `tsx watch src/services/worker.ts` | Background job processing |
| `flyer-crawler-vite-dev` | `vite --host` | Frontend dev server |
**Production PM2 Processes**:
| Process | Script | Purpose |
| -------------------------------- | ------------------ | --------------------------- |
| `flyer-crawler-api` | `tsx server.ts` | API server (cluster mode) |
| `flyer-crawler-worker` | `tsx worker.ts` | Background job processing |
| `flyer-crawler-analytics-worker` | `tsx analytics.ts` | Analytics report generation |
**Job Queues**:
| Queue | Purpose | Retry Strategy |
| ---------------------------- | -------------------------------- | ------------------------------------- |
| `flyer-processing` | Process uploaded flyers with AI | 3 attempts, exponential backoff (5s) |
| `receipt-processing` | OCR and parse receipts | 3 attempts, exponential backoff (10s) |
| `email-sending` | Send transactional emails | 5 attempts, exponential backoff (10s) |
| `analytics-reporting` | Generate daily analytics | 2 attempts, exponential backoff (60s) |
| `weekly-analytics-reporting` | Generate weekly reports | 2 attempts, exponential backoff (1h) |
| `file-cleanup` | Remove temporary files | 3 attempts, exponential backoff (30s) |
| `token-cleanup` | Expire old refresh tokens | 2 attempts, exponential backoff (1h) |
| `expiry-alerts` | Send pantry expiry notifications | 2 attempts, exponential backoff (5m) |
| `barcode-detection` | Process barcode scans | 2 attempts, exponential backoff (5s) |
---
## Data Flow
### Flyer Processing Pipeline
```
+-------------+ +----------------+ +------------------+ +---------------+
| User | | Express | | BullMQ | | PostgreSQL |
| Upload +---->+ Route +---->+ Queue +---->+ Storage |
+-------------+ +-------+--------+ +--------+---------+ +-------+-------+
| | |
v v v
+-------+--------+ +--------+---------+ +-------+-------+
| Validate | | Worker | | Cache |
| & Store | | Process | | Invalidate |
| Temp File | | | | |
+----------------+ +--------+---------+ +---------------+
|
v
+--------+---------+
| Google |
| Gemini AI |
| Extraction |
+--------+---------+
|
v
+--------+---------+
| Transform |
| & Validate |
| Data |
+--------+---------+
|
v
+--------+---------+
| Persist to |
| Database |
| (Transaction) |
+--------+---------+
|
v
+--------+---------+
| WebSocket |
| Notification |
+------------------+
```
### Detailed Processing Steps
1. **Upload**: User uploads flyer image via `/api/flyers/upload`
2. **Validation**: Server validates file type, size, and generates checksum
3. **Queueing**: Job added to `flyer-processing` queue with file path
4. **Worker Pickup**: BullMQ worker picks up job for processing
5. **AI Extraction**: Google Gemini analyzes image and extracts:
- Store name
- Valid date range
- Store address (if present)
- Deal items (name, price, quantity, category)
6. **Data Transformation**: Raw AI output transformed to database schema
7. **Persistence**: Transactional insert of flyer + items + store
8. **Cache Invalidation**: Redis cache cleared for affected queries
9. **Notification**: WebSocket message sent to user with results
10. **Cleanup**: Temporary files scheduled for deletion
---
## Architecture Layers
The application follows a strict layered architecture as defined in ADR-035.
```
+-----------------------------------------------------------------------+
| ROUTES LAYER |
| Responsibilities: |
| - HTTP request/response handling |
| - Input validation (via middleware) |
| - Authentication/authorization checks |
| - Rate limiting |
| - Response formatting (sendSuccess, sendPaginated, sendError) |
+----------------------------------+------------------------------------+
|
v
+-----------------------------------------------------------------------+
| SERVICES LAYER |
| Responsibilities: |
| - Business logic orchestration |
| - Transaction coordination (withTransaction) |
| - External API integration |
| - Cross-repository operations |
| - Event publishing |
+----------------------------------+------------------------------------+
|
v
+-----------------------------------------------------------------------+
| REPOSITORY LAYER |
| Responsibilities: |
| - Direct database access |
| - Query construction |
| - Entity mapping |
| - Error translation (handleDbError) |
+-----------------------------------------------------------------------+
```
### Layer Communication Rules
1. **Routes MUST NOT** directly access repositories (except simple CRUD)
2. **Repositories MUST NOT** call other repositories (use services)
3. **Services MAY** call other services
4. **Infrastructure services MAY** be called from any layer
### Service Types and Naming Conventions
| Type | Suffix | Example | Location |
| ------------------- | ------------- | --------------------- | ------------------ |
| Business Service | `*Service.ts` | `authService.ts` | `src/services/` |
| Server-Only Service | `*.server.ts` | `aiService.server.ts` | `src/services/` |
| Database Repository | `*.db.ts` | `user.db.ts` | `src/services/db/` |
| Infrastructure | Descriptive | `logger.server.ts` | `src/services/` |
### Repository Method Naming (ADR-034)
| Prefix | Behavior | Return Type |
| ------- | ----------------------------------- | -------------- |
| `get*` | Throws `NotFoundError` if not found | Entity |
| `find*` | Returns `null` if not found | Entity or null |
| `list*` | Returns empty array if none found | Entity[] |
---
## Key Entities
### Entity Relationship Overview
```
+------------------+ +------------------+ +------------------+
| users | | profiles | | addresses |
|------------------| |------------------| |------------------|
| user_id (PK) |<-------->| user_id (PK,FK) |--------->| address_id (PK) |
| email | | full_name | | address_line_1 |
| password_hash | | avatar_url | | city |
| refresh_token | | points | | province_state |
+--------+---------+ | role | | latitude |
| +------------------+ | longitude |
| | location (GIS) |
| +--------+---------+
| ^
v |
+--------+---------+ +------------------+ +--------+---------+
| stores |--------->| store_locations |--------->| |
|------------------| |------------------| | |
| store_id (PK) | | store_location_id| | |
| name | | store_id (FK) | | |
| logo_url | | address_id (FK) | | |
+--------+---------+ +------------------+ +------------------+
|
v
+--------+---------+ +------------------+ +------------------+
| flyers |--------->| flyer_items |--------->| master_grocery_ |
|------------------| |------------------| | items |
| flyer_id (PK) | | flyer_item_id | |------------------|
| store_id (FK) | | flyer_id (FK) | | master_grocery_ |
| file_name | | item | | item_id (PK) |
| image_url | | price_display | | name |
| valid_from | | price_in_cents | | category_id (FK) |
| valid_to | | quantity | | is_allergen |
| status | | master_item_id | +------------------+
| item_count | | category_id (FK) |
+------------------+ +------------------+
```
### Core Entities
| Entity | Table | Purpose |
| --------------------- | ---------------------- | --------------------------------------------- |
| **User** | `users` | Authentication credentials and login tracking |
| **Profile** | `profiles` | Public user data, preferences, points |
| **Store** | `stores` | Grocery store chains (Safeway, Kroger, etc.) |
| **StoreLocation** | `store_locations` | Physical store locations with addresses |
| **Address** | `addresses` | Normalized address storage with geocoding |
| **Flyer** | `flyers` | Uploaded flyer metadata and status |
| **FlyerItem** | `flyer_items` | Individual deals extracted from flyers |
| **MasterGroceryItem** | `master_grocery_items` | Canonical grocery item dictionary |
| **Category** | `categories` | Item categorization (Produce, Dairy, etc.) |
### User Feature Entities
| Entity | Table | Purpose |
| -------------------- | --------------------- | ------------------------------------ |
| **UserWatchedItem** | `user_watched_items` | Items user wants to track prices for |
| **UserAlert** | `user_alerts` | Price alert thresholds |
| **ShoppingList** | `shopping_lists` | User shopping lists |
| **ShoppingListItem** | `shopping_list_items` | Items on shopping lists |
| **Recipe** | `recipes` | User recipes with ingredients |
| **RecipeIngredient** | `recipe_ingredients` | Recipe ingredient list |
| **PantryItem** | `pantry_items` | User pantry inventory |
| **Receipt** | `receipts` | Scanned receipt data |
| **ReceiptItem** | `receipt_items` | Items parsed from receipts |
### Gamification Entities
| Entity | Table | Purpose |
| ------------------- | ------------------- | ------------------------------------- |
| **Achievement** | `achievements` | Defined achievements |
| **UserAchievement** | `user_achievements` | Achievements earned by users |
| **ActivityLog** | `activity_log` | User activity for feeds and analytics |
---
## Authentication Flow
### JWT Token Architecture
```
+-------------------+ +-------------------+ +-------------------+
| Login Request | | Server | | Database |
| (email/pass) +---->+ Validates +---->+ Verify User |
+-------------------+ +--------+----------+ +-------------------+
|
v
+--------+----------+
| Generate |
| JWT Tokens |
| - Access (15m) |
| - Refresh (7d) |
+--------+----------+
|
v
+-------------------+ +--------+----------+
| Client Storage |<----+ Return Tokens |
| - Access: Memory| | - Access: Body |
| - Refresh: HTTP | | - Refresh: Cookie|
| Only Cookie | +-------------------+
+-------------------+
```
### Authentication Methods
1. **Local Authentication**: Email/password with bcrypt hashing
2. **Google OAuth 2.0**: Social login via Google account
3. **GitHub OAuth 2.0**: Social login via GitHub account
### Security Features (ADR-016, ADR-048)
- **Rate Limiting**: Login attempts rate-limited per IP
- **Account Lockout**: 15-minute lockout after 5 failed attempts
- **Password Requirements**: Strength validation via zxcvbn
- **JWT Rotation**: Access tokens are short-lived, refresh tokens are rotated
- **HTTPS Only**: All production traffic encrypted
### Protected Route Flow
```
+-------------------+ +-------------------+ +-------------------+
| API Request | | requireAuth | | JWT Strategy |
| + Bearer Token +---->+ Middleware +---->+ Validate |
+-------------------+ +--------+----------+ +--------+----------+
| |
| +-------------------+
| |
v v
+--------+-----+----+
| req.user |
| populated |
+--------+----------+
|
v
+--------+----------+
| Route Handler |
| Executes |
+-------------------+
```
---
## Background Processing
### Worker Architecture
```
+-------------------+ +-------------------+ +-------------------+
| API Server | | Redis | | Worker Process |
| (Queue Producer)| | (Job Storage) | | (Consumer) |
+--------+----------+ +--------+----------+ +--------+----------+
| ^ |
| Add Job | Poll/Process |
+------------------------>+<------------------------+
|
|
+-------------------------+-------------------------+
| | |
v v v
+--------+----------+ +--------+----------+ +--------+----------+
| Flyer Worker | | Email Worker | | Analytics |
| Concurrency: 1 | | Concurrency: 10 | | Worker |
+-------------------+ +-------------------+ | Concurrency: 1 |
+-------------------+
```
### Job Lifecycle
1. **Queued**: Job added to queue with data payload
2. **Active**: Worker picks up job and begins processing
3. **Completed**: Job finishes successfully
4. **Failed**: Job encounters error, may retry
5. **Delayed**: Job waiting for retry backoff
### Retry Strategy
Jobs use exponential backoff for retries:
```
Attempt 1: Immediate
Attempt 2: Initial delay (e.g., 5 seconds)
Attempt 3: 2x delay (e.g., 10 seconds)
Attempt 4: 4x delay (e.g., 20 seconds)
...
```
### Scheduled Jobs (ADR-037)
| Schedule | Job | Purpose |
| --------------------- | ---------------- | ------------------------------------------ |
| Daily 2:00 AM | Analytics Report | Generate daily usage statistics |
| Weekly Sunday 3:00 AM | Weekly Analytics | Generate weekly summary reports |
| Every 6 hours | Token Cleanup | Remove expired refresh tokens |
| Every hour | Expiry Alerts | Check and send pantry expiry notifications |
---
## Deployment Architecture
### Environment Overview
```
+-----------------------------------------------------------------------------------+
| DEVELOPMENT |
+-----------------------------------------------------------------------------------+
| |
| +-----------------------------------+ +-----------------------------------+ |
| | Windows Host Machine | | Linux Dev Container | |
| | - VS Code | | (flyer-crawler-dev) | |
| | - Podman Desktop +---->+ - Node.js 22 | |
| | - Git | | - PM2 (process manager) | |
| +-----------------------------------+ | - PostgreSQL 16 | |
| | - Redis 7 | |
| | - Bugsink (local) | |
| | - Logstash (log aggregation) | |
| +-----------------------------------+ |
+-----------------------------------------------------------------------------------+
+-----------------------------------------------------------------------------------+
| TEST SERVER |
+-----------------------------------------------------------------------------------+
| |
| +-----------------------------------+ +-----------------------------------+ |
| | NGINX Reverse Proxy | | Application Server | |
| | flyer-crawler-test.projectium.com | - PM2 Process Manager | |
| | - SSL/TLS (Let's Encrypt) +---->+ - Node.js 22 | |
| | - Rate Limiting | | - PostgreSQL 16 | |
| +-----------------------------------+ | - Redis 7 | |
| +-----------------------------------+ |
+-----------------------------------------------------------------------------------+
+-----------------------------------------------------------------------------------+
| PRODUCTION |
+-----------------------------------------------------------------------------------+
| |
| +-----------------------------------+ +-----------------------------------+ |
| | NGINX Reverse Proxy | | Application Server | |
| | flyer-crawler.projectium.com | - PM2 Process Manager | |
| | - SSL/TLS (Let's Encrypt) +---->+ - Node.js 22 (Clustered) | |
| | - Rate Limiting | | - PostgreSQL 16 | |
| | - Gzip Compression | | - Redis 7 | |
| +-----------------------------------+ +-----------------------------------+ |
| |
| +-----------------------------------+ |
| | Monitoring | |
| | - Bugsink (Error Tracking) | |
| | - Logstash (Log Aggregation) | |
| +-----------------------------------+ |
+-----------------------------------------------------------------------------------+
```
### Deployment Pipeline (ADR-017)
```
+------------+ +------------+ +------------+ +------------+
| Push to | | Gitea | | Build & | | Deploy |
| main +---->+ Actions +---->+ Test +---->+ to Prod |
+------------+ +------------+ +------------+ +------------+
|
v
+------+------+
| Type |
| Check |
+------+------+
|
v
+------+------+
| Unit |
| Tests |
+------+------+
|
v
+------+------+
| Build |
| Assets |
+-------------+
```
### Server Paths
| Environment | Web Root | Data Storage | Flyer Images |
| ----------- | --------------------------------------------- | ----------------------------------------------------- | ---------------------------------------------------------- |
| Production | `/var/www/flyer-crawler.projectium.com/` | `/var/www/flyer-crawler.projectium.com/uploads/` | `/var/www/flyer-crawler.projectium.com/flyer-images/` |
| Test | `/var/www/flyer-crawler-test.projectium.com/` | `/var/www/flyer-crawler-test.projectium.com/uploads/` | `/var/www/flyer-crawler-test.projectium.com/flyer-images/` |
| Development | Container-local | Container-local | `/app/public/flyer-images/` |
Flyer images are served by NGINX as static files at `/flyer-images/` with 7-day browser caching.
---
## Design Principles and ADRs
The system architecture is governed by Architecture Decision Records (ADRs). Key decisions include:
### Core Infrastructure
| ADR | Title | Status |
| ------- | ------------------------------------ | -------- |
| ADR-001 | Standardized Error Handling | Accepted |
| ADR-002 | Standardized Transaction Management | Accepted |
| ADR-007 | Configuration and Secrets Management | Accepted |
| ADR-020 | Health Checks and Probes | Accepted |
### API and Integration
| ADR | Title | Status |
| ------- | ----------------------------- | ----------- |
| ADR-003 | Standardized Input Validation | Accepted |
| ADR-022 | Real-time Notification System | Proposed |
| ADR-028 | API Response Standardization | Implemented |
### Security
| ADR | Title | Status |
| ------- | ----------------------- | --------------------- |
| ADR-016 | API Security Hardening | Accepted |
| ADR-032 | Rate Limiting Strategy | Accepted |
| ADR-048 | Authentication Strategy | Partially Implemented |
### Architecture Patterns
| ADR | Title | Status |
| ------- | ---------------------------------- | -------- |
| ADR-034 | Repository Pattern Standards | Accepted |
| ADR-035 | Service Layer Architecture | Accepted |
| ADR-036 | Event Bus and Pub/Sub Pattern | Accepted |
| ADR-041 | AI/Gemini Integration Architecture | Accepted |
### Operations
| ADR | Title | Status |
| ------- | ------------------------------- | --------------------- |
| ADR-006 | Background Job Processing | Accepted |
| ADR-014 | Containerization and Deployment | Partially Implemented |
| ADR-037 | Scheduled Jobs and Cron Pattern | Accepted |
| ADR-038 | Graceful Shutdown Pattern | Accepted |
### Observability
| ADR | Title | Status |
| ------- | --------------------------------- | -------- |
| ADR-004 | Structured Logging | Accepted |
| ADR-015 | APM and Error Tracking | Proposed |
| ADR-050 | PostgreSQL Function Observability | Accepted |
**Full ADR Index**: [docs/adr/index.md](../adr/index.md)
---
## Key Files Reference
### Configuration Files
| File | Purpose |
| ------------------------ | ------------------------------------------------------ |
| `server.ts` | Express application setup and middleware configuration |
| `src/config/env.ts` | Environment variable validation (Zod schema) |
| `src/config/passport.ts` | Authentication strategies (Local, JWT, OAuth) |
| `ecosystem.config.cjs` | PM2 process manager configuration |
| `vite.config.ts` | Vite build and dev server configuration |
### Route Files
| File | API Prefix |
| ----------------------------- | -------------- |
| `src/routes/auth.routes.ts` | `/api/auth` |
| `src/routes/user.routes.ts` | `/api/users` |
| `src/routes/flyer.routes.ts` | `/api/flyers` |
| `src/routes/recipe.routes.ts` | `/api/recipes` |
| `src/routes/deals.routes.ts` | `/api/deals` |
| `src/routes/store.routes.ts` | `/api/stores` |
| `src/routes/admin.routes.ts` | `/api/admin` |
| `src/routes/health.routes.ts` | `/api/health` |
### Service Files
| File | Purpose |
| ----------------------------------------------- | --------------------------------------- |
| `src/services/flyerProcessingService.server.ts` | Flyer processing pipeline orchestration |
| `src/services/aiService.server.ts` | Google Gemini AI integration |
| `src/services/cacheService.server.ts` | Redis caching abstraction |
| `src/services/emailService.server.ts` | Email sending |
| `src/services/queues.server.ts` | BullMQ queue definitions |
| `src/services/workers.server.ts` | BullMQ worker definitions |
### Database Files
| File | Purpose |
| ---------------------------------- | -------------------------------------------- |
| `src/services/db/connection.db.ts` | Database pool and transaction management |
| `src/services/db/errors.db.ts` | Database error types |
| `src/services/db/user.db.ts` | User repository |
| `src/services/db/flyer.db.ts` | Flyer repository |
| `sql/master_schema_rollup.sql` | Complete database schema (for test DB setup) |
| `sql/initial_schema.sql` | Fresh installation schema |
### Type Definitions
| File | Purpose |
| ----------------------- | ---------------------------- |
| `src/types.ts` | Core entity type definitions |
| `src/types/job-data.ts` | BullMQ job payload types |
---
## Additional Resources
- **API Documentation**: Available at `/docs/api-docs` in development environments
- **Testing Guide**: [docs/tests/](../tests/)
- **Getting Started**: [docs/getting-started/](../getting-started/)
- **Operations Guide**: [docs/operations/](../operations/)
- **Authentication Details**: [docs/architecture/AUTHENTICATION.md](./AUTHENTICATION.md)
- **Database Schema**: [docs/architecture/DATABASE.md](./DATABASE.md)
- **WebSocket Usage**: [docs/architecture/WEBSOCKET_USAGE.md](./WEBSOCKET_USAGE.md)
---
_This document is maintained as part of the Flyer Crawler project documentation. For updates, contact the development team or submit a pull request._

View File

@@ -0,0 +1,534 @@
# Testing Session - UI/UX Improvements
**Date**: 2026-01-21
**Tester**: [Your Name]
**Session Start**: [Time]
**Environment**: Dev Container
---
## 🎯 Session Objective
Test all 4 critical UI/UX improvements:
1. Brand Colors (visual verification)
2. Button Component (functional testing)
3. Onboarding Tour (flow testing)
4. Mobile Navigation (responsive testing)
---
## ✅ Pre-Test Setup Checklist
### 1. Dev Server Status
- [ ] Dev server running at `http://localhost:5173`
- [ ] Browser open (Chrome/Edge recommended)
- [ ] DevTools open (F12)
**Command to start**:
```bash
podman exec -it flyer-crawler-dev npm run dev:container
```
**Server Status**: [ ] Running [ ] Not Running
---
### 2. Browser Setup
- [ ] Clear cache (Ctrl+Shift+Delete)
- [ ] Clear localStorage for localhost
- [ ] Enable responsive design mode (Ctrl+Shift+M)
**Browser Version**: **\*\*\*\***\_**\*\*\*\***
---
## 🧪 Test Execution
### TEST 1: Onboarding Tour ⭐ CRITICAL
**Priority**: 🔴 Must Pass
**Time**: 5 minutes
#### Steps:
1. Open DevTools → Application → Local Storage
2. Delete key: `flyer_crawler_onboarding_completed`
3. Refresh page (F5)
4. Observe if tour appears
#### Expected:
- ✅ Tour modal appears within 2 seconds
- ✅ Shows "Step 1 of 6"
- ✅ Points to Flyer Uploader section
- ✅ Skip button visible
- ✅ Next button visible
#### Actual Result:
```
[Record what you see here]
```
**Status**: [ ] ✅ PASS [ ] ❌ FAIL [ ] ⚠️ PARTIAL
**Screenshots**: [Attach if needed]
---
### TEST 2: Tour Navigation
**Time**: 5 minutes
#### Steps:
Click "Next" button 6 times, observe each step
#### Verification Table:
| Step | Target | Visible? | Correct Text? | Notes |
| ---- | -------------- | -------- | ------------- | ----- |
| 1 | Flyer Uploader | [ ] | [ ] | |
| 2 | Data Table | [ ] | [ ] | |
| 3 | Watch Button | [ ] | [ ] | |
| 4 | Watchlist | [ ] | [ ] | |
| 5 | Price Chart | [ ] | [ ] | |
| 6 | Shopping List | [ ] | [ ] | |
#### Additional Checks:
- [ ] Progress indicator updates (1/6 → 6/6)
- [ ] Can click "Previous" button
- [ ] Tour closes after step 6
- [ ] localStorage key saved
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
---
### TEST 3: Mobile Tab Bar ⭐ CRITICAL
**Priority**: 🔴 Must Pass
**Time**: 8 minutes
#### Part A: Mobile View (375px)
**Setup**: Toggle device toolbar → iPhone SE
#### Checks:
- [ ] Bottom tab bar visible
- [ ] 4 tabs present: Home, Deals, Lists, Profile
- [ ] Left sidebar (flyer list) HIDDEN
- [ ] Right sidebar (widgets) HIDDEN
- [ ] Main content uses full width
**Visual Check**:
```
Tab Bar Position: [ ] Bottom [ ] Other: _______
Number of Tabs: _______
Tab Bar Height: ~64px? [ ] Yes [ ] No
```
#### Part B: Tab Navigation
Click each tab and verify:
| Tab | URL | Page Loads? | Highlights? | Content Correct? |
| ------- | ---------- | ----------- | ----------- | ---------------- |
| Home | `/` | [ ] | [ ] | [ ] |
| Deals | `/deals` | [ ] | [ ] | [ ] |
| Lists | `/lists` | [ ] | [ ] | [ ] |
| Profile | `/profile` | [ ] | [ ] | [ ] |
#### Part C: Desktop View (1440px)
**Setup**: Exit device mode, maximize window
#### Checks:
- [ ] Tab bar HIDDEN (not visible)
- [ ] Left sidebar VISIBLE
- [ ] Right sidebar VISIBLE
- [ ] 3-column layout intact
- [ ] No layout regressions
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
---
### TEST 4: Dark Mode ⭐ CRITICAL
**Priority**: 🔴 Must Pass
**Time**: 5 minutes
#### Steps:
1. Click dark mode toggle in header
2. Navigate: Home → Deals → Lists → Profile
3. Observe colors and contrast
#### Visual Verification:
**Mobile Tab Bar**:
- [ ] Dark background (#111827 or similar)
- [ ] Dark border color
- [ ] Active tab: teal (#14b8a6)
- [ ] Inactive tabs: gray
**New Pages**:
- [ ] DealsPage: dark background, light text
- [ ] ShoppingListsPage: dark cards
- [ ] FlyersPage: dark theme
- [ ] No white boxes visible
**Button Component**:
- [ ] Primary buttons: teal background
- [ ] Secondary buttons: gray background
- [ ] Danger buttons: red background
- [ ] All text readable
#### Toggle Back:
- [ ] Light mode restores correctly
- [ ] No stuck dark elements
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
---
### TEST 5: Brand Colors Visual Check
**Time**: 3 minutes
#### Verification:
Navigate through app and check teal color consistency:
- [ ] Active tab: teal
- [ ] Primary buttons: teal
- [ ] Links on hover: teal
- [ ] Focus rings: teal
- [ ] All teal shades match (#14b8a6)
**Color Picker Check** (optional):
Use DevTools color picker on active tab:
- Expected: `#14b8a6` or `rgb(20, 184, 166)`
- Actual: **\*\*\*\***\_\_\_**\*\*\*\***
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
---
### TEST 6: Button Component
**Time**: 5 minutes
#### Find and Test Buttons:
**FlyerUploader Page**:
- [ ] "Upload Another Flyer" button (primary, teal)
- [ ] Button clickable
- [ ] Hover effect works
- [ ] Loading state (if applicable)
**ShoppingList Page** (navigate to /lists):
- [ ] "New List" button (secondary, gray)
- [ ] "Delete List" button (danger, red)
- [ ] Buttons functional
- [ ] Hover states work
**In Dark Mode**:
- [ ] All button variants visible
- [ ] Good contrast
- [ ] No white backgrounds
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
---
### TEST 7: Responsive Breakpoints
**Time**: 5 minutes
#### Test at each width:
**375px (Mobile)**:
```
Tab bar: [ ] Visible [ ] Hidden
Sidebars: [ ] Visible [ ] Hidden
Layout: [ ] Single column [ ] Multi-column
```
**768px (Tablet)**:
```
Tab bar: [ ] Visible [ ] Hidden
Sidebars: [ ] Visible [ ] Hidden
Layout: [ ] Single column [ ] Multi-column
```
**1024px (Desktop)**:
```
Tab bar: [ ] Visible [ ] Hidden
Sidebars: [ ] Visible [ ] Hidden
Layout: [ ] Single column [ ] Multi-column
```
**1440px (Large Desktop)**:
```
Layout: [ ] Unchanged [ ] Broken
All elements: [ ] Visible [ ] Hidden/Cut off
```
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
---
### TEST 8: Admin Routes (If Admin User)
**Time**: 3 minutes
**Skip if**: [ ] Not admin user
#### Steps:
1. Log in as admin
2. Navigate to `/admin`
3. Check for tab bar
#### Checks:
- [ ] Admin dashboard loads
- [ ] Tab bar NOT visible
- [ ] Layout looks correct
- [ ] Can navigate to subpages
- [ ] Subpages work in mobile view
**Status**: [ ] ✅ PASS [ ] ❌ FAIL [ ] ⏭️ SKIPPED
---
### TEST 9: Console Errors
**Time**: 2 minutes
#### Steps:
1. Open Console tab in DevTools
2. Clear console
3. Navigate through app: Home → Deals → Lists → Profile → Home
4. Check for red error messages
#### Results:
```
Errors Found: [ ] None [ ] Some (list below)
```
**React 19 warnings are OK** (peer dependencies)
**Status**: [ ] ✅ PASS (no errors) [ ] ❌ FAIL (errors present)
---
### TEST 10: Integration Flow
**Time**: 5 minutes
#### User Journey:
1. Start on Home page (mobile view)
2. Navigate to Deals tab
3. Navigate to Lists tab
4. Navigate to Profile tab
5. Navigate back to Home
6. Toggle dark mode
7. Navigate through tabs again
#### Checks:
- [ ] All navigation smooth
- [ ] No data loss
- [ ] Active tab always correct
- [ ] Browser back button works
- [ ] Dark mode persists across routes
- [ ] No JavaScript errors
- [ ] No layout shifting
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
---
## 📊 Test Results Summary
### Critical Tests Status
| Test | Status | Priority | Notes |
| ------------------- | ------ | ----------- | ----- |
| 1. Onboarding Tour | [ ] | 🔴 Critical | |
| 2. Tour Navigation | [ ] | 🟡 High | |
| 3. Mobile Tab Bar | [ ] | 🔴 Critical | |
| 4. Dark Mode | [ ] | 🔴 Critical | |
| 5. Brand Colors | [ ] | 🟡 High | |
| 6. Button Component | [ ] | 🟢 Medium | |
| 7. Responsive | [ ] | 🔴 Critical | |
| 8. Admin Routes | [ ] | 🟢 Medium | |
| 9. Console Errors | [ ] | 🔴 Critical | |
| 10. Integration | [ ] | 🟡 High | |
**Pass Rate**: **\_** / 10 tests passed
---
## 🐛 Issues Found
### Critical Issues (Blockers)
1. ***
2. ***
3. ***
### High Priority Issues
1. ***
2. ***
3. ***
### Medium/Low Priority Issues
1. ***
2. ***
3. ***
---
## 📸 Screenshots
Attach screenshots for:
- [ ] Onboarding tour (step 1)
- [ ] Mobile tab bar (375px)
- [ ] Desktop layout (1440px)
- [ ] Dark mode (tab bar)
- [ ] Any bugs/issues found
---
## 🎯 Final Decision
### Must-Pass Criteria
**Critical tests** (all must pass):
- [ ] Test 1: Onboarding Tour
- [ ] Test 3: Mobile Tab Bar
- [ ] Test 4: Dark Mode
- [ ] Test 7: Responsive
- [ ] Test 9: No Console Errors
**Result**: [ ] ALL CRITICAL PASS [ ] SOME FAIL
---
### Production Readiness
**Overall Assessment**:
[ ] ✅ READY FOR PRODUCTION
[ ] ⚠️ READY WITH MINOR ISSUES
[ ] ❌ NOT READY (critical issues)
**Blocking Issues** (must fix before deploy):
1. ***
2. ***
3. ***
**Recommended Fixes** (can deploy, fix later):
1. ***
2. ***
3. ***
---
## 🔐 Sign-Off
**Tester Name**: ******\*\*\*\*******\_\_\_******\*\*\*\*******
**Date/Time Completed**: ****\*\*\*\*****\_\_\_****\*\*\*\*****
**Total Testing Time**: **\_\_** minutes
**Recommended Action**:
[ ] Deploy to production
[ ] Deploy to staging first
[ ] Fix issues, re-test
[ ] Hold deployment
**Additional Notes**:
---
---
---
---
---
## 📋 Next Steps
**If PASS**:
1. [ ] Create commit with test results
2. [ ] Update CHANGELOG.md
3. [ ] Tag release (v0.12.4)
4. [ ] Deploy to staging
5. [ ] Monitor for 24 hours
6. [ ] Deploy to production
**If FAIL**:
1. [ ] Log issues in GitHub/Gitea
2. [ ] Assign to developer
3. [ ] Schedule re-test
4. [ ] Update test plan if needed
---
**Session End**: [Time]
**Session Duration**: **\_\_** minutes

View File

@@ -135,7 +135,7 @@ New users saw "Welcome to Flyer Crawler!" with no explanation of features or how
### Solution
Implemented interactive guided tour using `react-joyride`:
Implemented interactive guided tour using `driver.js` (framework-agnostic, React 19 compatible):
**Tour Steps** (6 total):
@@ -148,24 +148,27 @@ Implemented interactive guided tour using `react-joyride`:
**Features**:
- Auto-starts for first-time users
- Auto-starts for first-time users (500ms delay for DOM readiness)
- Persists completion in localStorage (`flyer_crawler_onboarding_completed`)
- Skip button for experienced users
- Progress indicator showing current step
- Styled with brand colors (#14b8a6)
- Custom styled with pastel colors, sharp borders (design system)
- Dark mode compatible
- Zero React peer dependencies (compatible with React 19)
### Deliverables
- **Created**: `src/hooks/useOnboardingTour.ts` (custom hook)
- **Created**: `src/hooks/useOnboardingTour.ts` (custom hook with Driver.js)
- **Modified**: Added `data-tour` attributes to 6 components:
- `src/features/flyer/FlyerUploader.tsx`
- `src/features/flyer/ExtractedDataTable.tsx`
- `src/features/shopping/WatchedItemsList.tsx`
- `src/features/charts/PriceChart.tsx`
- `src/features/shopping/ShoppingList.tsx`
- **Modified**: `src/layouts/MainLayout.tsx` - Integrated Joyride component
- **Installed**: `react-joyride@2.9.3`, `@types/react-joyride@2.0.2`
- **Modified**: `src/layouts/MainLayout.tsx` - Integrated tour via hook
- **Installed**: `driver.js@^1.3.1`
**Migration Note (2026-01-21)**: Originally implemented with `react-joyride@2.9.3`, but migrated to `driver.js` for React 19 compatibility.
### User Flow
@@ -360,13 +363,13 @@ npm test -- --run src/components/Button.test.tsx
1. `tailwind.config.js` - Brand colors
2. `src/App.tsx` - New routes, MobileTabBar
3. `src/layouts/MainLayout.tsx` - Joyride, responsive layout
3. `src/layouts/MainLayout.tsx` - Tour integration, responsive layout
4. `src/features/flyer/FlyerUploader.tsx` - Button, data-tour
5. `src/features/flyer/ExtractedDataTable.tsx` - data-tour
6. `src/features/shopping/WatchedItemsList.tsx` - Button, data-tour
7. `src/features/shopping/ShoppingList.tsx` - Button, data-tour
8. `src/features/charts/PriceChart.tsx` - data-tour
9. `package.json` - Dependencies (react-joyride)
9. `package.json` - Dependencies (driver.js)
10. `package-lock.json` - Dependency lock
### Statistics
@@ -383,7 +386,7 @@ npm test -- --run src/components/Button.test.tsx
### Bundle Size Impact
- `react-joyride`: ~30KB gzipped
- `driver.js`: ~10KB gzipped (lightweight, zero dependencies)
- `Button` component: <5KB (reduces duplication)
- Brand colors: 0KB (CSS utilities, tree-shaken)
- **Total increase**: ~25KB gzipped
@@ -461,8 +464,9 @@ If issues arise:
1. Revert commit containing `src/components/MobileTabBar.tsx`
2. Remove new routes from `src/App.tsx`
3. Restore previous `MainLayout.tsx` (remove Joyride)
3. Restore previous `MainLayout.tsx` (remove tour integration)
4. Keep Button component and brand colors (safe changes)
5. Remove `driver.js` and restore localStorage keys if needed
---

View File

@@ -0,0 +1,478 @@
# Code Patterns
Common code patterns extracted from Architecture Decision Records (ADRs). Use these as templates when writing new code.
## Table of Contents
- [Error Handling](#error-handling)
- [Repository Patterns](#repository-patterns)
- [API Response Patterns](#api-response-patterns)
- [Transaction Management](#transaction-management)
- [Input Validation](#input-validation)
- [Authentication](#authentication)
- [Caching](#caching)
- [Background Jobs](#background-jobs)
---
## Error Handling
**ADR**: [ADR-001](../adr/0001-standardized-error-handling-for-database-operations.md)
### Repository Layer Error Handling
```typescript
import { handleDbError, NotFoundError } from '../services/db/errors.db';
import { PoolClient } from 'pg';
export async function getFlyerById(id: number, client?: PoolClient): Promise<Flyer> {
const db = client || pool;
try {
const result = await db.query('SELECT * FROM flyers WHERE id = $1', [id]);
if (result.rows.length === 0) {
throw new NotFoundError('Flyer', id);
}
return result.rows[0];
} catch (error) {
throw handleDbError(error);
}
}
```
### Route Layer Error Handling
```typescript
import { sendError } from '../utils/apiResponse';
app.get('/api/flyers/:id', async (req, res) => {
try {
const flyer = await flyerDb.getFlyerById(parseInt(req.params.id));
return sendSuccess(res, flyer);
} catch (error) {
return sendError(res, error);
}
});
```
### Custom Error Types
```typescript
// NotFoundError - Entity not found
throw new NotFoundError('Flyer', id);
// ValidationError - Invalid input
throw new ValidationError('Invalid email format');
// DatabaseError - Database operation failed
throw new DatabaseError('Failed to insert flyer', originalError);
```
---
## Repository Patterns
**ADR**: [ADR-034](../adr/0034-repository-layer-method-naming-conventions.md)
### Method Naming Conventions
| Prefix | Returns | Not Found Behavior | Use Case |
| ------- | -------------- | -------------------- | ------------------------- |
| `get*` | Entity | Throws NotFoundError | When entity must exist |
| `find*` | Entity \| null | Returns null | When entity may not exist |
| `list*` | Array | Returns [] | When returning multiple |
### Get Method (Must Exist)
```typescript
/**
* Get a flyer by ID. Throws NotFoundError if not found.
*/
export async function getFlyerById(id: number, client?: PoolClient): Promise<Flyer> {
const db = client || pool;
try {
const result = await db.query('SELECT * FROM flyers WHERE id = $1', [id]);
if (result.rows.length === 0) {
throw new NotFoundError('Flyer', id);
}
return result.rows[0];
} catch (error) {
throw handleDbError(error);
}
}
```
### Find Method (May Not Exist)
```typescript
/**
* Find a flyer by ID. Returns null if not found.
*/
export async function findFlyerById(id: number, client?: PoolClient): Promise<Flyer | null> {
const db = client || pool;
try {
const result = await db.query('SELECT * FROM flyers WHERE id = $1', [id]);
return result.rows[0] || null;
} catch (error) {
throw handleDbError(error);
}
}
```
### List Method (Multiple Results)
```typescript
/**
* List all active flyers. Returns empty array if none found.
*/
export async function listActiveFlyers(client?: PoolClient): Promise<Flyer[]> {
const db = client || pool;
try {
const result = await db.query(
'SELECT * FROM flyers WHERE end_date >= CURRENT_DATE ORDER BY start_date DESC',
);
return result.rows;
} catch (error) {
throw handleDbError(error);
}
}
```
---
## API Response Patterns
**ADR**: [ADR-028](../adr/0028-consistent-api-response-format.md)
### Success Response
```typescript
import { sendSuccess } from '../utils/apiResponse';
app.post('/api/flyers', async (req, res) => {
const flyer = await flyerService.createFlyer(req.body);
return sendSuccess(res, flyer, 'Flyer created successfully', 201);
});
```
### Paginated Response
```typescript
import { sendPaginated } from '../utils/apiResponse';
app.get('/api/flyers', async (req, res) => {
const { page = 1, pageSize = 20 } = req.query;
const { items, total } = await flyerService.listFlyers(page, pageSize);
return sendPaginated(res, {
items,
total,
page: parseInt(page),
pageSize: parseInt(pageSize),
});
});
```
### Error Response
```typescript
import { sendError } from '../utils/apiResponse';
app.get('/api/flyers/:id', async (req, res) => {
try {
const flyer = await flyerDb.getFlyerById(parseInt(req.params.id));
return sendSuccess(res, flyer);
} catch (error) {
return sendError(res, error); // Automatically maps error to correct status
}
});
```
---
## Transaction Management
**ADR**: [ADR-002](../adr/0002-transaction-management-pattern.md)
### Basic Transaction
```typescript
import { withTransaction } from '../services/db/transaction.db';
export async function createFlyerWithItems(
flyerData: FlyerInput,
items: FlyerItemInput[],
): Promise<Flyer> {
return withTransaction(async (client) => {
// Create flyer
const flyer = await flyerDb.createFlyer(flyerData, client);
// Create items
const createdItems = await flyerItemDb.createItems(
items.map((item) => ({ ...item, flyer_id: flyer.id })),
client,
);
// Automatically commits on success, rolls back on error
return { ...flyer, items: createdItems };
});
}
```
### Nested Transactions
```typescript
export async function bulkImportFlyers(flyersData: FlyerInput[]): Promise<ImportResult> {
return withTransaction(async (client) => {
const results = [];
for (const flyerData of flyersData) {
try {
// Each flyer import is atomic
const flyer = await createFlyerWithItems(
flyerData,
flyerData.items,
client, // Pass transaction client
);
results.push({ success: true, flyer });
} catch (error) {
results.push({ success: false, error: error.message });
}
}
return results;
});
}
```
---
## Input Validation
**ADR**: [ADR-003](../adr/0003-input-validation-framework.md)
### Zod Schema Definition
```typescript
// src/schemas/flyer.schemas.ts
import { z } from 'zod';
export const createFlyerSchema = z.object({
store_id: z.number().int().positive(),
image_url: z
.string()
.url()
.regex(/^https?:\/\/.*/),
start_date: z.string().datetime(),
end_date: z.string().datetime(),
items: z
.array(
z.object({
name: z.string().min(1).max(255),
price: z.number().positive(),
quantity: z.string().optional(),
}),
)
.min(1),
});
export type CreateFlyerInput = z.infer<typeof createFlyerSchema>;
```
### Route Validation Middleware
```typescript
import { validateRequest } from '../middleware/validation';
import { createFlyerSchema } from '../schemas/flyer.schemas';
app.post('/api/flyers', validateRequest(createFlyerSchema), async (req, res) => {
// req.body is now type-safe and validated
const flyer = await flyerService.createFlyer(req.body);
return sendSuccess(res, flyer, 'Flyer created successfully', 201);
});
```
### Manual Validation
```typescript
import { createFlyerSchema } from '../schemas/flyer.schemas';
export async function processFlyer(data: unknown): Promise<Flyer> {
// Validate and parse input
const validated = createFlyerSchema.parse(data);
// Type-safe from here on
return flyerDb.createFlyer(validated);
}
```
---
## Authentication
**ADR**: [ADR-048](../adr/0048-authentication-strategy.md)
### Protected Route with JWT
```typescript
import { authenticateJWT } from '../middleware/auth';
app.get(
'/api/profile',
authenticateJWT, // Middleware adds req.user
async (req, res) => {
// req.user is guaranteed to exist
const user = await userDb.getUserById(req.user.id);
return sendSuccess(res, user);
},
);
```
### Optional Authentication
```typescript
import { optionalAuth } from '../middleware/auth';
app.get(
'/api/flyers',
optionalAuth, // req.user may or may not exist
async (req, res) => {
const flyers = req.user
? await flyerDb.listFlyersForUser(req.user.id)
: await flyerDb.listPublicFlyers();
return sendSuccess(res, flyers);
},
);
```
### Generate JWT Token
```typescript
import jwt from 'jsonwebtoken';
import { env } from '../config/env';
export function generateToken(user: User): string {
return jwt.sign({ id: user.id, email: user.email }, env.JWT_SECRET, { expiresIn: '7d' });
}
```
---
## Caching
**ADR**: [ADR-029](../adr/0029-redis-caching-strategy.md)
### Cache Pattern
```typescript
import { cacheService } from '../services/cache.server';
export async function getFlyer(id: number): Promise<Flyer> {
// Try cache first
const cached = await cacheService.get<Flyer>(`flyer:${id}`);
if (cached) return cached;
// Cache miss - fetch from database
const flyer = await flyerDb.getFlyerById(id);
// Store in cache (1 hour TTL)
await cacheService.set(`flyer:${id}`, flyer, 3600);
return flyer;
}
```
### Cache Invalidation
```typescript
export async function updateFlyer(id: number, data: UpdateFlyerInput): Promise<Flyer> {
const flyer = await flyerDb.updateFlyer(id, data);
// Invalidate cache
await cacheService.delete(`flyer:${id}`);
await cacheService.invalidatePattern('flyers:list:*');
return flyer;
}
```
---
## Background Jobs
**ADR**: [ADR-036](../adr/0036-background-job-processing-architecture.md)
### Queue Job
```typescript
import { flyerProcessingQueue } from '../services/queues.server';
export async function enqueueFlyerProcessing(flyerId: number): Promise<void> {
await flyerProcessingQueue.add(
'process-flyer',
{
flyerId,
timestamp: Date.now(),
},
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 2000,
},
},
);
}
```
### Process Job
```typescript
// src/services/workers.server.ts
import { Worker } from 'bullmq';
const flyerWorker = new Worker(
'flyer-processing',
async (job) => {
const { flyerId } = job.data;
try {
// Process flyer
const result = await aiService.extractFlyerData(flyerId);
await flyerDb.updateFlyerWithData(flyerId, result);
// Update progress
await job.updateProgress(100);
return { success: true, itemCount: result.items.length };
} catch (error) {
logger.error('Flyer processing failed', { flyerId, error });
throw error; // Will retry automatically
}
},
{
connection: redisConnection,
concurrency: 5,
},
);
```
---
## Related Documentation
- [ADR Index](../adr/index.md) - All architecture decision records
- [TESTING.md](TESTING.md) - Testing patterns
- [DEBUGGING.md](DEBUGGING.md) - Debugging strategies
- [Database Guide](../subagents/DATABASE-GUIDE.md) - Database patterns
- [Coder Reference](../SUBAGENT-CODER-REFERENCE.md) - Quick reference for AI agents

View File

@@ -0,0 +1,844 @@
# Debugging Guide
Common debugging strategies and troubleshooting patterns for Flyer Crawler.
## Table of Contents
- [Quick Debugging Checklist](#quick-debugging-checklist)
- [Container Issues](#container-issues)
- [Database Issues](#database-issues)
- [Test Failures](#test-failures)
- [API Errors](#api-errors)
- [Authentication Problems](#authentication-problems)
- [Background Job Issues](#background-job-issues)
- [SSL Certificate Issues](#ssl-certificate-issues)
- [Frontend Issues](#frontend-issues)
- [Performance Problems](#performance-problems)
- [Debugging Tools](#debugging-tools)
---
## Quick Debugging Checklist
When something breaks, check these first:
1. **Are containers running?**
```bash
podman ps
```
2. **Is the database accessible?**
```bash
podman exec flyer-crawler-postgres pg_isready -U postgres
```
3. **Are environment variables set?**
```bash
# Check .env.local exists
cat .env.local
```
4. **Are there recent errors in logs?**
```bash
# PM2 logs (dev container)
podman exec -it flyer-crawler-dev pm2 logs
# Container logs
podman logs -f flyer-crawler-dev
# PM2 logs (production)
pm2 logs flyer-crawler-api
```
5. **Are PM2 processes running?**
```bash
podman exec -it flyer-crawler-dev pm2 status
```
6. **Is Redis accessible?**
```bash
podman exec flyer-crawler-redis redis-cli ping
```
---
## Container Issues
### Container Won't Start
**Symptom**: `podman start` fails or container exits immediately
**Debug**:
```bash
# Check container status
podman ps -a
# View container logs
podman logs flyer-crawler-postgres
podman logs flyer-crawler-redis
podman logs flyer-crawler-dev
# Inspect container
podman inspect flyer-crawler-dev
```
**Common Causes**:
- Port already in use
- Insufficient resources
- Configuration error
**Solutions**:
```bash
# Check port usage
netstat -an | findstr "5432"
netstat -an | findstr "6379"
# Remove and recreate container
podman stop flyer-crawler-postgres
podman rm flyer-crawler-postgres
# ... recreate with podman run ...
```
### "Unable to connect to Podman socket"
**Symptom**: `Error: unable to connect to Podman socket`
**Solution**:
```bash
# Start Podman machine
podman machine start
# Verify it's running
podman machine list
```
### Port Already in Use
**Symptom**: `Error: port 5432 is already allocated`
**Solutions**:
**Option 1**: Stop conflicting service
```bash
# Find process using port
netstat -ano | findstr "5432"
# Stop the service or kill process
```
**Option 2**: Use different port
```bash
# Run container on different host port
podman run -d --name flyer-crawler-postgres -p 5433:5432 ...
# Update .env.local
DB_PORT=5433
```
---
## Database Issues
### Connection Refused
**Symptom**: `Error: connect ECONNREFUSED 127.0.0.1:5432`
**Debug**:
```bash
# 1. Check if PostgreSQL container is running
podman ps | grep postgres
# 2. Check if PostgreSQL is ready
podman exec flyer-crawler-postgres pg_isready -U postgres
# 3. Test connection
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1;"
```
**Common Causes**:
- Container not running
- PostgreSQL still initializing
- Wrong credentials in `.env.local`
**Solutions**:
```bash
# Start container
podman start flyer-crawler-postgres
# Wait for initialization (check logs)
podman logs -f flyer-crawler-postgres
# Verify credentials match .env.local
cat .env.local | grep DB_
```
### Schema Out of Sync
**Symptom**: Tests fail with missing column or table errors
**Cause**: `master_schema_rollup.sql` not in sync with migrations
**Solution**:
```bash
# Reset database with current schema
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/drop_tables.sql
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/master_schema_rollup.sql
# Verify schema
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "\dt"
```
### Query Performance Issues
**Debug**:
```sql
-- Enable query logging
ALTER DATABASE flyer_crawler_dev SET log_statement = 'all';
-- Check slow queries
SELECT query, mean_exec_time, calls
FROM pg_stat_statements
WHERE mean_exec_time > 100
ORDER BY mean_exec_time DESC
LIMIT 10;
-- Analyze query plan
EXPLAIN ANALYZE
SELECT * FROM flyers WHERE store_id = 1;
```
**Solutions**:
- Add missing indexes
- Optimize WHERE clauses
- Use connection pooling
- See [ADR-034](../adr/0034-repository-layer-method-naming-conventions.md)
---
## Test Failures
### Tests Pass on Windows, Fail in Container
**Cause**: Platform-specific behavior (ADR-014)
**Rule**: Container results are authoritative. Windows results are unreliable.
**Solution**:
```bash
# Always run tests in container
podman exec -it flyer-crawler-dev npm test
# For specific test
podman exec -it flyer-crawler-dev npm test -- --run src/path/to/test.test.ts
```
### Integration Tests Fail
**Common Issues**:
**1. Vitest globalSetup Context Isolation**
**Symptom**: Mocks or spies don't work in integration tests
**Cause**: `globalSetup` runs in separate Node.js context
**Solutions**:
- Mark test as `.todo()` and document limitation
- Create test-only API endpoints
- Use Redis-based mock flags
See [CLAUDE.md#integration-test-issues](../../CLAUDE.md#integration-test-issues) for details.
**2. Cache Stale After Direct SQL**
**Symptom**: Test reads stale data after direct database insert
**Cause**: Cache not invalidated
**Solution**:
```typescript
// After direct SQL insert
await cacheService.invalidateFlyers();
```
**3. Queue Interference**
**Symptom**: Cleanup worker processes test data before assertions
**Solution**:
```typescript
const { cleanupQueue } = await import('../../services/queues.server');
await cleanupQueue.drain();
await cleanupQueue.pause();
// ... test ...
await cleanupQueue.resume();
```
### Type Check Failures
**Symptom**: `npm run type-check` fails
**Debug**:
```bash
# Run type check in container
podman exec -it flyer-crawler-dev npm run type-check
# Check specific file
podman exec -it flyer-crawler-dev npx tsc --noEmit src/path/to/file.ts
```
**Common Causes**:
- Missing type definitions
- Incorrect imports
- Type mismatch in function calls
---
## API Errors
### 404 Not Found
**Debug**:
```bash
# Check route registration
grep -r "router.get" src/routes/
# Check route path matches request
# Verify middleware order
```
**Common Causes**:
- Route not registered in `server.ts`
- Typo in route path
- Middleware blocking request
### 500 Internal Server Error
**Debug**:
```bash
# Check application logs
podman logs -f flyer-crawler-dev
# Check Bugsink for errors
# Visit: http://localhost:8443 (dev) or https://bugsink.projectium.com (prod)
```
**Common Causes**:
- Unhandled exception
- Database error
- Missing environment variable
**Solution Pattern**:
```typescript
// Always wrap route handlers
app.get('/api/endpoint', async (req, res) => {
try {
const result = await service.doSomething();
return sendSuccess(res, result);
} catch (error) {
return sendError(res, error); // Handles error types automatically
}
});
```
### 401 Unauthorized
**Debug**:
```bash
# Check JWT token in request
# Verify token is valid and not expired
# Test token decoding
node -e "console.log(require('jsonwebtoken').decode('YOUR_TOKEN_HERE'))"
```
**Common Causes**:
- Token expired
- Invalid token format
- Missing Authorization header
- Wrong JWT_SECRET
---
## Authentication Problems
### OAuth Not Working
**Debug**:
```bash
# 1. Verify OAuth credentials
cat .env.local | grep GOOGLE_CLIENT
# 2. Check OAuth routes are registered
grep -r "passport.authenticate" src/routes/
# 3. Verify redirect URI matches Google Console
# Should be: http://localhost:3001/api/auth/google/callback
```
**Common Issues**:
- Redirect URI mismatch in Google Console
- OAuth not enabled (commented out in config)
- Wrong client ID/secret
See [AUTHENTICATION.md](../architecture/AUTHENTICATION.md) for setup.
### JWT Token Invalid
**Debug**:
```typescript
// Decode token to inspect
import jwt from 'jsonwebtoken';
const decoded = jwt.decode(token);
console.log('Token payload:', decoded);
console.log('Expired:', decoded.exp < Date.now() / 1000);
```
**Solutions**:
- Regenerate token
- Check JWT_SECRET matches between environments
- Verify token hasn't expired
---
## PM2 Process Issues (Dev Container)
### PM2 Process Not Starting
**Symptom**: `pm2 status` shows process as "errored" or "stopped"
**Debug**:
```bash
# Check process status
podman exec -it flyer-crawler-dev pm2 status
# View process logs
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev --lines 50
# Show detailed process info
podman exec -it flyer-crawler-dev pm2 show flyer-crawler-api-dev
# Check if port is in use
podman exec -it flyer-crawler-dev netstat -tlnp | grep 3001
```
**Common Causes**:
- Port already in use by another process
- Missing environment variables
- Syntax error in code
- Database connection failure
**Solutions**:
```bash
# Restart specific process
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
# Restart all processes
podman exec -it flyer-crawler-dev pm2 restart all
# Delete and recreate processes
podman exec -it flyer-crawler-dev pm2 delete all
podman exec -it flyer-crawler-dev pm2 start /app/ecosystem.dev.config.cjs
```
### PM2 Log Access
**View logs in real-time**:
```bash
# All processes
podman exec -it flyer-crawler-dev pm2 logs
# Specific process
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
# Last 100 lines
podman exec -it flyer-crawler-dev pm2 logs --lines 100
```
**View log files directly**:
```bash
# API logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-out.log
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-error.log
# Worker logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/worker-out.log
# Vite logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/vite-out.log
```
### Redis Log Access
**View Redis logs**:
```bash
# View Redis log file
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
# View from dev container (shared volume)
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/redis/redis-server.log
```
---
## Background Job Issues
### Jobs Not Processing
**Debug**:
```bash
# Check if worker is running (dev container)
podman exec -it flyer-crawler-dev pm2 status
# Check worker logs (dev container)
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-worker-dev
# Check if worker is running (production)
pm2 list
# Check worker logs (production)
pm2 logs flyer-crawler-worker
# Check Redis connection
podman exec flyer-crawler-redis redis-cli ping
# Check queue status
node -e "
const { flyerProcessingQueue } = require('./dist/services/queues.server.js');
flyerProcessingQueue.getJobCounts().then(console.log);
"
```
**Common Causes**:
- Worker not running
- Redis connection lost
- Queue paused
- Job stuck in failed state
**Solutions**:
```bash
# Restart worker
pm2 restart flyer-crawler-worker
# Clear failed jobs
node -e "
const { flyerProcessingQueue } = require('./dist/services/queues.server.js');
flyerProcessingQueue.clean(0, 1000, 'failed');
"
```
### Jobs Failing
**Debug**:
```bash
# Check failed jobs
node -e "
const { flyerProcessingQueue } = require('./dist/services/queues.server.js');
flyerProcessingQueue.getFailed().then(jobs => {
jobs.forEach(job => console.log(job.failedReason));
});
"
# Check worker logs for stack traces
pm2 logs flyer-crawler-worker --lines 100
```
**Common Causes**:
- Gemini API errors
- Database errors
- Invalid job data
---
## SSL Certificate Issues
### Images Not Loading (ERR_CERT_AUTHORITY_INVALID)
**Symptom**: Flyer images fail to load with `ERR_CERT_AUTHORITY_INVALID` in browser console
**Cause**: Mixed hostname origins - user accesses via `localhost` but images use `127.0.0.1` (or vice versa)
**Debug**:
```bash
# Check which hostname images are using
podman exec flyer-crawler-dev psql -U postgres -d flyer_crawler_dev \
-c "SELECT image_url FROM flyers LIMIT 1;"
# Verify certificate includes both hostnames
podman exec flyer-crawler-dev openssl x509 -in /app/certs/localhost.crt -text -noout | grep -A1 "Subject Alternative Name"
# Check NGINX accepts both hostnames
podman exec flyer-crawler-dev grep "server_name" /etc/nginx/sites-available/default
```
**Solution**: The dev container is configured to handle both hostnames:
1. Certificate includes SANs for `localhost`, `127.0.0.1`, and `::1`
2. NGINX `server_name` directive includes both `localhost` and `127.0.0.1`
If you still see errors:
```bash
# Rebuild container to regenerate certificate
podman-compose down
podman-compose build --no-cache flyer-crawler-dev
podman-compose up -d
```
See [FLYER-URL-CONFIGURATION.md](../FLYER-URL-CONFIGURATION.md#ssl-certificate-configuration-dev-container) for full details.
### Self-Signed Certificate Not Trusted
**Symptom**: Browser shows security warning for `https://localhost`
**Temporary Workaround**: Accept the warning by clicking "Advanced" > "Proceed to localhost"
**Permanent Fix (Recommended)**: Install the mkcert CA certificate to eliminate all SSL warnings.
The CA certificate is located at `certs/mkcert-ca.crt` in the project root. See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
After installation:
- Your browser will trust all mkcert certificates without warnings
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
### NGINX SSL Configuration Test
**Debug**:
```bash
# Test NGINX configuration
podman exec flyer-crawler-dev nginx -t
# Check if NGINX is listening on 443
podman exec flyer-crawler-dev netstat -tlnp | grep 443
# Verify certificate files exist
podman exec flyer-crawler-dev ls -la /app/certs/
```
---
## Frontend Issues
### Hot Reload Not Working
**Debug**:
```bash
# Check Vite is running
curl http://localhost:5173
# Check for port conflicts
netstat -an | findstr "5173"
```
**Solution**:
```bash
# Restart dev server
npm run dev
```
### API Calls Failing (CORS)
**Symptom**: `CORS policy: No 'Access-Control-Allow-Origin' header`
**Debug**:
```typescript
// Check CORS configuration in server.ts
import cors from 'cors';
app.use(
cors({
origin: env.FRONTEND_URL, // Should match http://localhost:5173 in dev
credentials: true,
}),
);
```
**Solution**: Verify `FRONTEND_URL` in `.env.local` matches the frontend URL
---
## Performance Problems
### Slow API Responses
**Debug**:
```typescript
// Add timing logs
const start = Date.now();
const result = await slowOperation();
console.log(`Operation took ${Date.now() - start}ms`);
```
**Common Causes**:
- N+1 query problem
- Missing database indexes
- Large payload size
- No caching
**Solutions**:
- Use JOINs instead of multiple queries
- Add indexes: `CREATE INDEX idx_name ON table(column);`
- Implement pagination
- Add Redis caching
### High Memory Usage
**Debug**:
```bash
# Check PM2 memory usage
pm2 monit
# Check container memory
podman stats flyer-crawler-dev
```
**Common Causes**:
- Memory leak
- Large in-memory cache
- Unbounded array growth
---
## Debugging Tools
### VS Code Debugger
**Launch Configuration** (`.vscode/launch.json`):
```json
{
"version": "0.2.0",
"configurations": [
{
"type": "node",
"request": "launch",
"name": "Debug Tests",
"program": "${workspaceFolder}/node_modules/vitest/vitest.mjs",
"args": ["--run", "${file}"],
"console": "integratedTerminal",
"internalConsoleOptions": "neverOpen"
}
]
}
```
### Logging
```typescript
import { logger } from './utils/logger';
// Structured logging
logger.info('Processing flyer', { flyerId, userId });
logger.error('Failed to process', { error, context });
logger.debug('Cache hit', { key, ttl });
```
### Database Query Logging
```typescript
// In development, log all queries
if (env.NODE_ENV === 'development') {
pool.on('connect', () => {
console.log('Database connected');
});
// Log slow queries
const originalQuery = pool.query.bind(pool);
pool.query = async (...args) => {
const start = Date.now();
const result = await originalQuery(...args);
const duration = Date.now() - start;
if (duration > 100) {
console.log(`Slow query (${duration}ms):`, args[0]);
}
return result;
};
}
```
### Redis Debugging
```bash
# Monitor Redis commands
podman exec -it flyer-crawler-redis redis-cli monitor
# Check keys
podman exec flyer-crawler-redis redis-cli keys "*"
# Get key value
podman exec flyer-crawler-redis redis-cli get "flyer:123"
# Check cache stats
podman exec flyer-crawler-redis redis-cli info stats
```
---
## See Also
- [DEV-CONTAINER.md](DEV-CONTAINER.md) - Dev container guide (PM2, Logstash)
- [TESTING.md](TESTING.md) - Testing strategies
- [CODE-PATTERNS.md](CODE-PATTERNS.md) - Common patterns
- [MONITORING.md](../operations/MONITORING.md) - Production monitoring
- [LOGSTASH-QUICK-REF.md](../operations/LOGSTASH-QUICK-REF.md) - Log aggregation
- [Bugsink Setup](../tools/BUGSINK-SETUP.md) - Error tracking
- [DevOps Guide](../subagents/DEVOPS-GUIDE.md) - Container debugging

View File

@@ -0,0 +1,377 @@
# Dev Container Guide
Comprehensive documentation for the Flyer Crawler development container.
**Last Updated**: 2026-01-22
---
## Table of Contents
1. [Overview](#overview)
2. [Architecture](#architecture)
3. [PM2 Process Management](#pm2-process-management)
4. [Log Aggregation](#log-aggregation)
5. [Quick Reference](#quick-reference)
6. [Troubleshooting](#troubleshooting)
---
## Overview
The dev container provides a production-like environment for local development. Key features:
- **PM2 Process Management** - Matches production architecture (ADR-014)
- **Logstash Integration** - All logs forwarded to Bugsink (ADR-050)
- **HTTPS by Default** - Self-signed certificates via mkcert
- **Hot Reloading** - tsx watch mode for API and worker processes
### Container Services
| Service | Container Name | Purpose |
| ----------- | ------------------------ | ------------------------------ |
| Application | `flyer-crawler-dev` | Node.js app, Bugsink, Logstash |
| PostgreSQL | `flyer-crawler-postgres` | Primary database with PostGIS |
| Redis | `flyer-crawler-redis` | Cache and job queue backing |
### Access Points
| Service | URL | Notes |
| ----------- | ------------------------ | ------------------------- |
| Frontend | `https://localhost` | NGINX proxies Vite (5173) |
| Backend API | `http://localhost:3001` | Express server |
| Bugsink | `https://localhost:8443` | Error tracking UI |
| PostgreSQL | `localhost:5432` | Direct database access |
| Redis | `localhost:6379` | Direct Redis access |
---
## Architecture
### Production vs Development Parity
The dev container was updated to match production architecture (ADR-014):
| Component | Production | Dev Container (OLD) | Dev Container (NEW) |
| ------------ | ---------------------- | ---------------------- | ----------------------- |
| API Server | PM2 cluster mode | `npm run dev` (inline) | PM2 fork + tsx watch |
| Worker | PM2 process | Inline with API | PM2 process + tsx watch |
| Frontend | Static files via NGINX | Vite standalone | PM2 + Vite dev server |
| Logs | PM2 logs -> Logstash | Console only | PM2 logs -> Logstash |
| Process Mgmt | PM2 | None | PM2 |
### Container Startup Flow
When the container starts (`scripts/dev-entrypoint.sh`):
1. **NGINX** starts (HTTPS proxy for Vite and Bugsink)
2. **Bugsink** starts (error tracking on port 8000)
3. **Logstash** starts (log aggregation)
4. **PM2** starts with `ecosystem.dev.config.cjs`:
- `flyer-crawler-api-dev` - API server
- `flyer-crawler-worker-dev` - Background worker
- `flyer-crawler-vite-dev` - Vite dev server
5. Container tails PM2 logs to stay alive
### Key Configuration Files
| File | Purpose |
| ------------------------------ | ---------------------------------- |
| `ecosystem.dev.config.cjs` | PM2 development configuration |
| `ecosystem.config.cjs` | PM2 production configuration |
| `scripts/dev-entrypoint.sh` | Container startup script |
| `docker/logstash/bugsink.conf` | Logstash pipeline configuration |
| `docker/nginx/dev.conf` | NGINX development configuration |
| `compose.dev.yml` | Docker Compose service definitions |
| `Dockerfile.dev` | Container image definition |
---
## PM2 Process Management
### Process Overview
PM2 manages three processes in the dev container:
```
+--------------------+ +------------------------+ +--------------------+
| flyer-crawler- | | flyer-crawler- | | flyer-crawler- |
| api-dev | | worker-dev | | vite-dev |
+--------------------+ +------------------------+ +--------------------+
| tsx watch | | tsx watch | | vite --host |
| server.ts | | src/services/worker.ts | | |
| Port: 3001 | | No port | | Port: 5173 |
+--------------------+ +------------------------+ +--------------------+
| | |
v v v
+------------------------------------------------------------------------+
| /var/log/pm2/*.log |
| (Logstash picks up for Bugsink) |
+------------------------------------------------------------------------+
```
### PM2 Commands
All commands should be run inside the container:
```bash
# View process status
podman exec -it flyer-crawler-dev pm2 status
# View all logs (tail -f style)
podman exec -it flyer-crawler-dev pm2 logs
# View specific process logs
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
# Restart all processes
podman exec -it flyer-crawler-dev pm2 restart all
# Restart specific process
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
# Stop all processes
podman exec -it flyer-crawler-dev pm2 delete all
# Show detailed process info
podman exec -it flyer-crawler-dev pm2 show flyer-crawler-api-dev
# Monitor processes in real-time
podman exec -it flyer-crawler-dev pm2 monit
```
### PM2 Log Locations
| Process | stdout Log | stderr Log |
| -------------------------- | ----------------------------- | ------------------------------- |
| `flyer-crawler-api-dev` | `/var/log/pm2/api-out.log` | `/var/log/pm2/api-error.log` |
| `flyer-crawler-worker-dev` | `/var/log/pm2/worker-out.log` | `/var/log/pm2/worker-error.log` |
| `flyer-crawler-vite-dev` | `/var/log/pm2/vite-out.log` | `/var/log/pm2/vite-error.log` |
### NPM Scripts for PM2
The following npm scripts are available for PM2 management:
```bash
# Start PM2 with dev config (inside container)
npm run dev:pm2
# Restart all PM2 processes
npm run dev:pm2:restart
# Stop all PM2 processes
npm run dev:pm2:stop
# View PM2 status
npm run dev:pm2:status
# View PM2 logs
npm run dev:pm2:logs
```
---
## Log Aggregation
### Log Flow Architecture (ADR-050)
All application logs flow through Logstash to Bugsink:
```
+------------------+ +------------------+ +------------------+
| PM2 Logs | | PostgreSQL | | Redis Logs |
| /var/log/pm2/ | | /var/log/ | | /var/log/redis/ |
+--------+---------+ | postgresql/ | +--------+---------+
| +--------+---------+ |
| | |
v v v
+------------------------------------------------------------------------+
| LOGSTASH |
| /etc/logstash/conf.d/bugsink.conf |
+------------------------------------------------------------------------+
| | |
| +---------+---------+ |
| | | |
v v v v
+------------------+ +------------------+ +------------------+
| Errors -> | | Operational -> | | NGINX Logs -> |
| Bugsink API | | /var/log/ | | /var/log/ |
| (Project 1) | | logstash/*.log | | logstash/*.log |
+------------------+ +------------------+ +------------------+
```
### Log Sources
| Source | Log Path | Format | Errors To Bugsink |
| ------------ | --------------------------------- | ---------- | ----------------- |
| API Server | `/var/log/pm2/api-*.log` | Pino JSON | Yes |
| Worker | `/var/log/pm2/worker-*.log` | Pino JSON | Yes |
| Vite | `/var/log/pm2/vite-*.log` | Plain text | Yes (if error) |
| PostgreSQL | `/var/log/postgresql/*.log` | PostgreSQL | Yes (ERROR/FATAL) |
| Redis | `/var/log/redis/redis-server.log` | Redis | Yes (warnings) |
| NGINX Access | `/var/log/nginx/access.log` | Combined | Yes (5xx only) |
| NGINX Error | `/var/log/nginx/error.log` | NGINX | Yes |
### Viewing Logs
```bash
# View Logstash processed logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-api-$(date +%Y-%m-%d).log
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
# View raw Redis logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
# Check Logstash status
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
```
### Bugsink Access
- **URL**: `https://localhost:8443`
- **Login**: `admin@localhost` / `admin`
- **Projects**:
- Project 1: Backend API (errors from Pino, PostgreSQL, Redis)
- Project 2: Frontend (errors from Sentry SDK in browser)
---
## Quick Reference
### Starting the Dev Container
```bash
# Start all services
podman-compose -f compose.dev.yml up -d
# View container logs
podman-compose -f compose.dev.yml logs -f
# Stop all services
podman-compose -f compose.dev.yml down
```
### Common Tasks
| Task | Command |
| -------------------- | ------------------------------------------------------------------------------ |
| Run tests | `podman exec -it flyer-crawler-dev npm test` |
| Run type check | `podman exec -it flyer-crawler-dev npm run type-check` |
| View PM2 status | `podman exec -it flyer-crawler-dev pm2 status` |
| View PM2 logs | `podman exec -it flyer-crawler-dev pm2 logs` |
| Restart API | `podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev` |
| Access PostgreSQL | `podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev` |
| Access Redis CLI | `podman exec -it flyer-crawler-redis redis-cli` |
| Shell into container | `podman exec -it flyer-crawler-dev bash` |
### Environment Variables
Key environment variables are set in `compose.dev.yml`:
| Variable | Value | Purpose |
| ----------------- | ----------------------------- | -------------------- |
| `NODE_ENV` | `development` | Environment mode |
| `DB_HOST` | `postgres` | PostgreSQL hostname |
| `REDIS_URL` | `redis://redis:6379` | Redis connection URL |
| `FRONTEND_URL` | `https://localhost` | CORS origin |
| `SENTRY_DSN` | `http://...@127.0.0.1:8000/1` | Backend Bugsink DSN |
| `VITE_SENTRY_DSN` | `http://...@127.0.0.1:8000/2` | Frontend Bugsink DSN |
---
## Troubleshooting
### PM2 Process Not Starting
**Symptom**: `pm2 status` shows process as "errored" or "stopped"
**Debug**:
```bash
# Check process logs
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev --lines 50
# Check if port is in use
podman exec -it flyer-crawler-dev netstat -tlnp | grep 3001
# Try manual start to see errors
podman exec -it flyer-crawler-dev tsx server.ts
```
**Common Causes**:
- Port already in use
- Missing environment variables
- Syntax error in code
### Logs Not Appearing in Bugsink
**Symptom**: Errors in application but nothing in Bugsink UI
**Debug**:
```bash
# Check Logstash is running
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
# Check Logstash logs for errors
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log
# Verify PM2 logs exist
podman exec flyer-crawler-dev ls -la /var/log/pm2/
```
**Common Causes**:
- Logstash not started
- Log file permissions
- Bugsink not running
### Redis Logs Not Captured
**Symptom**: Redis warnings not appearing in Bugsink
**Debug**:
```bash
# Verify Redis logs exist
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
# Verify shared volume is mounted
podman exec flyer-crawler-dev ls -la /var/log/redis/
```
**Common Causes**:
- `redis_logs` volume not mounted
- Redis not configured to write to file
### Hot Reload Not Working
**Symptom**: Code changes not reflected in running application
**Debug**:
```bash
# Check tsx watch is running
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
# Manually restart process
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
```
**Common Causes**:
- File watcher limit reached
- Volume mount issues on Windows
---
## Related Documentation
- [QUICKSTART.md](../getting-started/QUICKSTART.md) - Getting started guide
- [DEBUGGING.md](DEBUGGING.md) - Debugging strategies
- [LOGSTASH-QUICK-REF.md](../operations/LOGSTASH-QUICK-REF.md) - Logstash quick reference
- [DEV-CONTAINER-BUGSINK.md](../DEV-CONTAINER-BUGSINK.md) - Bugsink setup in dev container
- [ADR-014](../adr/0014-linux-only-platform.md) - Linux-only platform decision
- [ADR-050](../adr/0050-postgresql-function-observability.md) - PostgreSQL function observability

View File

@@ -187,6 +187,17 @@ const mockStoreWithLocations = createMockStoreWithLocations({
});
```
### Test Assets
Test images and other assets are located in `src/tests/assets/`:
| File | Purpose |
| ---------------------- | ---------------------------------------------- |
| `test-flyer-image.jpg` | Sample flyer image for upload/processing tests |
| `test-flyer-icon.png` | Sample flyer icon (64x64) for thumbnail tests |
These images are copied to `public/flyer-images/` by the seed script (`npm run seed`) and served via NGINX at `/flyer-images/`.
## Known Integration Test Issues
See `CLAUDE.md` for documentation of common integration test issues and their solutions, including:

View File

@@ -0,0 +1,271 @@
# Environment Variables Reference
Complete guide to environment variables used in Flyer Crawler.
## Configuration by Environment
### Production
**Location**: Gitea CI/CD secrets injected during deployment
**Path**: `/var/www/flyer-crawler.projectium.com/`
**Note**: No `.env` file exists - all variables come from CI/CD
### Test
**Location**: Gitea CI/CD secrets + `.env.test` file
**Path**: `/var/www/flyer-crawler-test.projectium.com/`
**Note**: `.env.test` overrides for test-specific values
### Development Container
**Location**: `.env.local` file in project root
**Note**: Overrides default DSNs in `compose.dev.yml`
## Required Variables
### Database
| Variable | Description | Example |
| ------------------ | ---------------------------- | ------------------------------------------ |
| `DB_HOST` | PostgreSQL host | `localhost` (dev), `projectium.com` (prod) |
| `DB_PORT` | PostgreSQL port | `5432` |
| `DB_USER_PROD` | Production database user | `flyer_crawler_prod` |
| `DB_PASSWORD_PROD` | Production database password | (secret) |
| `DB_DATABASE_PROD` | Production database name | `flyer-crawler-prod` |
| `DB_USER_TEST` | Test database user | `flyer_crawler_test` |
| `DB_PASSWORD_TEST` | Test database password | (secret) |
| `DB_DATABASE_TEST` | Test database name | `flyer-crawler-test` |
| `DB_USER` | Dev database user | `postgres` |
| `DB_PASSWORD` | Dev database password | `postgres` |
| `DB_NAME` | Dev database name | `flyer_crawler_dev` |
**Note**: Production and test use separate `_PROD` and `_TEST` suffixed variables. Development uses unsuffixed variables.
### Redis
| Variable | Description | Example |
| --------------------- | ------------------------- | ------------------------------ |
| `REDIS_URL` | Redis connection URL | `redis://localhost:6379` (dev) |
| `REDIS_PASSWORD_PROD` | Production Redis password | (secret) |
| `REDIS_PASSWORD_TEST` | Test Redis password | (secret) |
### Authentication
| Variable | Description | Example |
| ---------------------- | -------------------------- | -------------------------------- |
| `JWT_SECRET` | JWT token signing key | (minimum 32 characters) |
| `SESSION_SECRET` | Session encryption key | (minimum 32 characters) |
| `GOOGLE_CLIENT_ID` | Google OAuth client ID | `xxx.apps.googleusercontent.com` |
| `GOOGLE_CLIENT_SECRET` | Google OAuth client secret | (secret) |
| `GH_CLIENT_ID` | GitHub OAuth client ID | `xxx` |
| `GH_CLIENT_SECRET` | GitHub OAuth client secret | (secret) |
### AI Services
| Variable | Description | Example |
| -------------------------------- | ---------------------------- | ----------- |
| `VITE_GOOGLE_GENAI_API_KEY` | Google Gemini API key (prod) | `AIzaSy...` |
| `VITE_GOOGLE_GENAI_API_KEY_TEST` | Google Gemini API key (test) | `AIzaSy...` |
| `GOOGLE_MAPS_API_KEY` | Google Maps Geocoding API | `AIzaSy...` |
### Application
| Variable | Description | Example |
| -------------- | ------------------------ | ----------------------------------- |
| `NODE_ENV` | Environment mode | `development`, `test`, `production` |
| `PORT` | Backend server port | `3001` |
| `FRONTEND_URL` | Frontend application URL | `http://localhost:5173` (dev) |
### Error Tracking
| Variable | Description | Example |
| ---------------------- | -------------------------------- | --------------------------- |
| `SENTRY_DSN` | Sentry DSN (production) | `https://xxx@sentry.io/xxx` |
| `VITE_SENTRY_DSN` | Frontend Sentry DSN (production) | `https://xxx@sentry.io/xxx` |
| `SENTRY_DSN_TEST` | Sentry DSN (test) | `https://xxx@sentry.io/xxx` |
| `VITE_SENTRY_DSN_TEST` | Frontend Sentry DSN (test) | `https://xxx@sentry.io/xxx` |
| `SENTRY_AUTH_TOKEN` | Sentry API token for releases | (secret) |
## Optional Variables
| Variable | Description | Default |
| ------------------- | ----------------------- | ----------------- |
| `LOG_LEVEL` | Logging verbosity | `info` |
| `REDIS_TTL` | Cache TTL in seconds | `3600` |
| `MAX_UPLOAD_SIZE` | Max file upload size | `10mb` |
| `RATE_LIMIT_WINDOW` | Rate limit window (ms) | `900000` (15 min) |
| `RATE_LIMIT_MAX` | Max requests per window | `100` |
## Configuration Files
| File | Purpose |
| ------------------------------------- | ------------------------------------------- |
| `src/config/env.ts` | Zod schema validation - **source of truth** |
| `ecosystem.config.cjs` | PM2 process manager config |
| `.gitea/workflows/deploy-to-prod.yml` | Production deployment workflow |
| `.gitea/workflows/deploy-to-test.yml` | Test deployment workflow |
| `.env.example` | Template with all variables |
| `.env.local` | Dev container overrides (not in git) |
| `.env.test` | Test environment overrides (not in git) |
## Adding New Variables
### 1. Update Zod Schema
Edit `src/config/env.ts`:
```typescript
const envSchema = z.object({
// ... existing variables ...
NEW_VARIABLE: z.string().min(1),
});
```
### 2. Add to Gitea Secrets
For prod/test environments:
1. Go to Gitea repository Settings > Secrets
2. Add `NEW_VARIABLE` with value
3. Add `NEW_VARIABLE_TEST` if test needs different value
### 3. Update Deployment Workflows
Edit `.gitea/workflows/deploy-to-prod.yml`:
```yaml
env:
NEW_VARIABLE: ${{ secrets.NEW_VARIABLE }}
```
Edit `.gitea/workflows/deploy-to-test.yml`:
```yaml
env:
NEW_VARIABLE: ${{ secrets.NEW_VARIABLE_TEST }}
```
### 4. Update PM2 Config
Edit `ecosystem.config.cjs`:
```javascript
module.exports = {
apps: [
{
env: {
NEW_VARIABLE: process.env.NEW_VARIABLE,
},
},
],
};
```
### 5. Update Documentation
- Add to `.env.example`
- Update this document
- Document in relevant feature docs
## Security Best Practices
### Secrets Management
- **NEVER** commit secrets to git
- Use Gitea Secrets for prod/test
- Use `.env.local` for dev (gitignored)
- Rotate secrets regularly
### Secret Generation
```bash
# Generate secure random secrets
node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
```
### Database Users
Each environment has its own PostgreSQL user:
| Environment | User | Database |
| ----------- | -------------------- | -------------------- |
| Production | `flyer_crawler_prod` | `flyer-crawler-prod` |
| Test | `flyer_crawler_test` | `flyer-crawler-test` |
| Development | `postgres` | `flyer_crawler_dev` |
**Setup Commands** (as postgres superuser):
```sql
-- Production
CREATE DATABASE "flyer-crawler-prod";
CREATE USER flyer_crawler_prod WITH PASSWORD 'secure-password';
ALTER DATABASE "flyer-crawler-prod" OWNER TO flyer_crawler_prod;
\c "flyer-crawler-prod"
ALTER SCHEMA public OWNER TO flyer_crawler_prod;
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_prod;
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS postgis;
CREATE EXTENSION IF NOT EXISTS pg_trgm;
-- Test (similar commands with _test suffix)
```
## Validation
Environment variables are validated at startup via `src/config/env.ts`. If validation fails:
1. Check the error message for missing/invalid variables
2. Verify `.env.local` (dev) or Gitea Secrets (prod/test)
3. Ensure values match schema requirements (min length, format, etc.)
## Troubleshooting
### Variable Not Found
```
Error: Missing required environment variable: JWT_SECRET
```
**Solution**: Add the variable to your environment configuration.
### Invalid Value
```
Error: JWT_SECRET must be at least 32 characters
```
**Solution**: Generate a longer secret value.
### Wrong Environment Selected
Check `NODE_ENV` is set correctly:
- `development` - Local dev container
- `test` - CI/CD test server
- `production` - Production server
### Database Connection Issues
Verify database credentials:
```bash
# Development
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1;"
# Production (via SSH)
ssh root@projectium.com "psql -U flyer_crawler_prod -d flyer-crawler-prod -c 'SELECT 1;'"
```
## Reference
- **Validation Schema**: [src/config/env.ts](../../src/config/env.ts)
- **Template**: [.env.example](../../.env.example)
- **Deployment Workflows**: [.gitea/workflows/](../../.gitea/workflows/)
- **PM2 Config**: [ecosystem.config.cjs](../../ecosystem.config.cjs)
## See Also
- [QUICKSTART.md](QUICKSTART.md) - Quick setup guide
- [INSTALL.md](INSTALL.md) - Detailed installation
- [DEPLOYMENT.md](../operations/DEPLOYMENT.md) - Production deployment
- [AUTHENTICATION.md](../architecture/AUTHENTICATION.md) - OAuth setup

View File

@@ -119,6 +119,31 @@ npm run dev
- **Frontend**: http://localhost:5173
- **Backend API**: http://localhost:3001
### Dev Container with HTTPS (Full Stack)
When using the full dev container stack with NGINX (via `compose.dev.yml`), access the application over HTTPS:
- **Frontend**: https://localhost or https://127.0.0.1
- **Backend API**: http://localhost:3001
**SSL Certificate Notes:**
- The dev container uses self-signed certificates generated by mkcert
- Both `localhost` and `127.0.0.1` are valid hostnames (certificate includes both as SANs)
- If images fail to load with SSL errors, see [FLYER-URL-CONFIGURATION.md](../FLYER-URL-CONFIGURATION.md#ssl-certificate-configuration-dev-container)
**Eliminate SSL Warnings (Recommended):**
To avoid browser security warnings for self-signed certificates, install the mkcert CA certificate on your system. The CA certificate is located at `certs/mkcert-ca.crt` in the project root.
See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
After installation:
- Your browser will trust all mkcert certificates without warnings
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
### Managing the Container
| Action | Command |
@@ -149,14 +174,24 @@ For local development, you can export these in your shell or use your IDE's envi
---
## Seeding Development Users
## Seeding Development Data
To create initial test accounts (`admin@example.com` and `user@example.com`):
To create initial test accounts (`admin@example.com` and `user@example.com`) and sample data:
```bash
npm run seed
```
The seed script performs the following actions:
1. Rebuilds the database schema from `sql/master_schema_rollup.sql`
2. Creates test user accounts (admin and regular user)
3. Copies test flyer images from `src/tests/assets/` to `public/flyer-images/`
4. Creates a sample flyer with items linked to the test images
5. Seeds watched items and a shopping list for the test user
**Test Images**: The seed script copies `test-flyer-image.jpg` and `test-flyer-icon.png` to the `public/flyer-images/` directory, which is served by NGINX at `/flyer-images/`.
After running, you may need to restart your IDE's TypeScript server to pick up any generated types.
---

View File

@@ -0,0 +1,186 @@
# Quick Start Guide
Get Flyer Crawler running in 5 minutes.
## Prerequisites
- **Windows 10/11** with WSL 2
- **Podman Desktop** installed
- **Node.js 20+** installed
## 1. Start Containers (1 minute)
```bash
# Start PostgreSQL and Redis
podman start flyer-crawler-postgres flyer-crawler-redis
# If containers don't exist yet, create them:
podman run -d --name flyer-crawler-postgres \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=flyer_crawler_dev \
-p 5432:5432 \
docker.io/postgis/postgis:15-3.3
podman run -d --name flyer-crawler-redis \
-p 6379:6379 \
docker.io/library/redis:alpine
```
## 2. Initialize Database (2 minutes)
```bash
# Wait for PostgreSQL to be ready
podman exec flyer-crawler-postgres pg_isready -U postgres
# Install extensions
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev \
-c "CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS pg_trgm; CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";"
# Apply schema
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/master_schema_rollup.sql
```
## 3. Configure Environment (1 minute)
Create `.env.local` in the project root:
```bash
# Database
DB_HOST=localhost
DB_USER=postgres
DB_PASSWORD=postgres
DB_NAME=flyer_crawler_dev
DB_PORT=5432
# Redis
REDIS_URL=redis://localhost:6379
# Application
NODE_ENV=development
PORT=3001
FRONTEND_URL=http://localhost:5173
# Secrets (generate your own)
JWT_SECRET=your-dev-jwt-secret-at-least-32-chars-long
SESSION_SECRET=your-dev-session-secret-at-least-32-chars-long
# AI Services (get your own keys)
VITE_GOOGLE_GENAI_API_KEY=your-google-genai-api-key
GOOGLE_MAPS_API_KEY=your-google-maps-api-key
```
## 4. Install & Run (1 minute)
```bash
# Install dependencies (first time only)
npm install
# Start development server
npm run dev
```
## 5. Access Application
- **Frontend**: http://localhost:5173
- **Backend API**: http://localhost:3001
- **Health Check**: http://localhost:3001/health
### Dev Container (HTTPS)
When using the full dev container with NGINX, access via HTTPS:
- **Frontend**: https://localhost or https://127.0.0.1
- **Backend API**: http://localhost:3001
- **Bugsink**: `https://localhost:8443` (error tracking)
**Note:** The dev container accepts both `localhost` and `127.0.0.1` for HTTPS connections. The self-signed certificate is valid for both hostnames.
**SSL Certificate Warnings:** To eliminate browser security warnings for self-signed certificates, install the mkcert CA certificate. See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions. This is optional but recommended for a better development experience.
### Dev Container Architecture
The dev container uses PM2 for process management, matching production (ADR-014):
| Process | Description | Port |
| -------------------------- | ------------------------ | ---- |
| `flyer-crawler-api-dev` | API server (tsx watch) | 3001 |
| `flyer-crawler-worker-dev` | Background job worker | - |
| `flyer-crawler-vite-dev` | Vite frontend dev server | 5173 |
**PM2 Commands** (run inside container):
```bash
# View process status
podman exec -it flyer-crawler-dev pm2 status
# View logs (all processes)
podman exec -it flyer-crawler-dev pm2 logs
# Restart all processes
podman exec -it flyer-crawler-dev pm2 restart all
# Restart specific process
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
```
## Verify Installation
```bash
# Check containers are running
podman ps
# Test database connection
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT version();"
# Run tests (in dev container)
podman exec -it flyer-crawler-dev npm run test:unit
```
## Common Issues
### "Unable to connect to Podman socket"
```bash
podman machine start
```
### "Connection refused" to PostgreSQL
Wait a few seconds for PostgreSQL to initialize:
```bash
podman exec flyer-crawler-postgres pg_isready -U postgres
```
### Port 5432 or 6379 already in use
Stop conflicting services or change port mappings:
```bash
# Use different host port
podman run -d --name flyer-crawler-postgres -p 5433:5432 ...
```
Then update `DB_PORT=5433` in `.env.local`.
## Next Steps
- **Read the docs**: [docs/README.md](../README.md)
- **Understand the architecture**: [docs/architecture/DATABASE.md](../architecture/DATABASE.md)
- **Learn testing**: [docs/development/TESTING.md](../development/TESTING.md)
- **Explore ADRs**: [docs/adr/index.md](../adr/index.md)
- **Contributing**: [CONTRIBUTING.md](../../CONTRIBUTING.md)
## Development Workflow
```bash
# Daily workflow
podman start flyer-crawler-postgres flyer-crawler-redis
npm run dev
# ... make changes ...
npm test
git commit
```
For detailed setup instructions, see [INSTALL.md](INSTALL.md).

View File

@@ -369,6 +369,17 @@ pm2 delete flyer-crawler-api-test flyer-crawler-worker-test flyer-crawler-analyt
sudo apt install -y nginx
```
### Reference Configuration Files
The repository contains reference copies of the actual production NGINX configurations at the project root:
- `etc-nginx-sites-available-flyer-crawler.projectium.com` - Production config
- `etc-nginx-sites-available-flyer-crawler-test-projectium-com.txt` - Test config
These reference files document the exact configuration deployed on the server, including SSL settings managed by Certbot. Use them as a reference when setting up new servers or troubleshooting configuration issues.
**Note:** The simplified example below shows the basic structure. For the complete production configuration with SSL, security headers, and all location blocks, refer to the reference files in the repository root.
### Create Site Configuration
Create `/etc/nginx/sites-available/flyer-crawler.projectium.com`:
@@ -408,6 +419,13 @@ server {
client_max_body_size 50M;
}
# Serve flyer images from static storage (7-day cache)
location /flyer-images/ {
alias /var/www/flyer-crawler.projectium.com/flyer-images/;
expires 7d;
add_header Cache-Control "public, immutable";
}
# MIME type fix for .mjs files
types {
application/javascript js mjs;
@@ -415,6 +433,26 @@ server {
}
```
### Static Flyer Images Directory
Create the directory for storing flyer images:
```bash
# Production
sudo mkdir -p /var/www/flyer-crawler.projectium.com/flyer-images
sudo chown www-data:www-data /var/www/flyer-crawler.projectium.com/flyer-images
# Test environment
sudo mkdir -p /var/www/flyer-crawler-test.projectium.com/flyer-images
sudo chown www-data:www-data /var/www/flyer-crawler-test.projectium.com/flyer-images
```
The `/flyer-images/` location serves static images with:
- **7-day browser cache** (`expires 7d`)
- **Immutable cache header** for optimal CDN/browser caching
- Direct file serving (no proxy overhead)
### Enable the Site
```bash

View File

@@ -11,6 +11,22 @@ This guide covers deploying Flyer Crawler to a production server.
- NGINX (reverse proxy)
- PM2 (process manager)
## Dev Container Parity (ADR-014)
The development container now matches production architecture:
| Component | Production | Dev Container |
| ------------ | ---------------- | -------------------------- |
| Process Mgmt | PM2 | PM2 |
| API Server | PM2 cluster mode | PM2 fork + tsx watch |
| Worker | PM2 process | PM2 process + tsx watch |
| Logs | PM2 -> Logstash | PM2 -> Logstash -> Bugsink |
| HTTPS | Let's Encrypt | Self-signed (mkcert) |
This ensures issues caught in dev will behave the same in production.
**Dev Container Guide**: See [docs/development/DEV-CONTAINER.md](../development/DEV-CONTAINER.md)
---
## Server Setup
@@ -80,6 +96,22 @@ For deployments using Gitea CI/CD workflows, configure these as **repository sec
## NGINX Configuration
### Reference Configuration Files
The repository contains reference copies of the production NGINX configurations for documentation and version control purposes:
| File | Server Config |
| ----------------------------------------------------------------- | ------------------------- |
| `etc-nginx-sites-available-flyer-crawler.projectium.com` | Production NGINX config |
| `etc-nginx-sites-available-flyer-crawler-test-projectium-com.txt` | Test/staging NGINX config |
**Important Notes:**
- These are **reference copies only** - they are not used directly by NGINX
- The actual live configurations reside on the server at `/etc/nginx/sites-available/`
- When modifying server NGINX configs, update these reference files to keep them in sync
- Use the dev container config at `docker/nginx/dev.conf` for local development
### Reverse Proxy Setup
Create a site configuration at `/etc/nginx/sites-available/flyer-crawler.projectium.com`:
@@ -106,9 +138,35 @@ server {
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
# Serve flyer images from static storage (7-day cache)
location /flyer-images/ {
alias /var/www/flyer-crawler.projectium.com/flyer-images/;
expires 7d;
add_header Cache-Control "public, immutable";
}
}
```
### Static Flyer Images
Flyer images are served as static files from the `/flyer-images/` path with browser caching enabled:
| Environment | Directory | URL Pattern |
| ------------- | ---------------------------------------------------------- | --------------------------------------------------------- |
| Production | `/var/www/flyer-crawler.projectium.com/flyer-images/` | `https://flyer-crawler.projectium.com/flyer-images/` |
| Test | `/var/www/flyer-crawler-test.projectium.com/flyer-images/` | `https://flyer-crawler-test.projectium.com/flyer-images/` |
| Dev Container | `/app/public/flyer-images/` | `https://localhost/flyer-images/` |
**Cache Settings**: Files are served with `expires 7d` and `Cache-Control: public, immutable` headers for optimal browser caching.
Create the flyer images directory if it does not exist:
```bash
sudo mkdir -p /var/www/flyer-crawler.projectium.com/flyer-images
sudo chown www-data:www-data /var/www/flyer-crawler.projectium.com/flyer-images
```
Enable the site:
```bash

View File

@@ -0,0 +1,110 @@
# Logstash Quick Reference (ADR-050)
Aggregates logs from PostgreSQL, PM2, Redis, NGINX; forwards errors to Bugsink.
## Configuration
**Primary config**: `/etc/logstash/conf.d/bugsink.conf`
### Related Files
| Path | Purpose |
| --------------------------------------------------- | ------------------------- |
| `/etc/postgresql/14/main/conf.d/observability.conf` | PostgreSQL logging config |
| `/var/log/postgresql/*.log` | PostgreSQL logs |
| `/home/gitea-runner/.pm2/logs/*.log` | PM2 worker logs |
| `/var/log/redis/redis-server.log` | Redis logs |
| `/var/log/nginx/access.log` | NGINX access logs |
| `/var/log/nginx/error.log` | NGINX error logs |
| `/var/log/logstash/*.log` | Logstash file outputs |
| `/var/lib/logstash/sincedb_*` | Position tracking files |
## Features
- **Multi-source aggregation**: PostgreSQL, PM2 workers, Redis, NGINX
- **Environment routing**: Auto-detects prod/test, routes to correct Bugsink project
- **JSON parsing**: Extracts `fn_log()` from PostgreSQL, Pino JSON from PM2
- **Sentry format**: Transforms to `event_id`, `timestamp`, `level`, `message`, `extra`
- **Error filtering**: Only forwards WARNING/ERROR to Bugsink
- **Operational storage**: Non-error logs saved to `/var/log/logstash/`
- **Request monitoring**: NGINX requests categorized by status, slow request detection
## Commands
### Production (Bare Metal)
```bash
# Status and control
systemctl status logstash
systemctl restart logstash
# Test configuration
/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf
# View logs
journalctl -u logstash -f
# Check stats (events processed, failures)
curl -XGET 'localhost:9600/_node/stats/pipelines?pretty' | jq '.pipelines.main.plugins.filters'
# Monitor sources
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log
tail -f /var/log/logstash/pm2-workers-$(date +%Y-%m-%d).log
tail -f /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
# Check disk usage
du -sh /var/log/logstash/
```
### Dev Container
```bash
# View Logstash logs (inside container)
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log
# Check PM2 API logs are being processed (ADR-014)
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-api-$(date +%Y-%m-%d).log
# Check PM2 Worker logs are being processed
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-worker-$(date +%Y-%m-%d).log
# Check Redis logs are being processed
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
# View raw PM2 logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-out.log
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/worker-out.log
# View raw Redis logs
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
# Check Logstash stats
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
# Verify shared volume mounts
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev ls -la /var/log/pm2/
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev ls -la /var/log/redis/
```
## Troubleshooting
| Issue | Check | Solution |
| --------------------- | ---------------- | ---------------------------------------------------------------------------------------------- |
| No Bugsink errors | Logstash running | `systemctl status logstash` |
| Config syntax error | Test config | `/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf` |
| Grok pattern failures | Stats endpoint | `curl localhost:9600/_node/stats/pipelines?pretty \| jq '.pipelines.main.plugins.filters'` |
| Wrong Bugsink project | Env detection | Check tags in logs match expected environment |
| Permission denied | Logstash groups | `groups logstash` should include `postgres`, `adm` |
| PM2 not captured | File paths | Dev: `ls /var/log/pm2/`; Prod: `ls /home/gitea-runner/.pm2/logs/` |
| PM2 logs empty | PM2 running | Dev: `podman exec flyer-crawler-dev pm2 status`; Prod: `pm2 status` |
| NGINX logs missing | Output directory | `ls -lh /var/log/logstash/nginx-access-*.log` |
| Redis logs missing | Shared volume | Dev: Check `redis_logs` volume mounted; Prod: Check `/var/log/redis/redis-server.log` exists |
| High disk usage | Log rotation | Verify `/etc/logrotate.d/logstash` configured |
## Related Documentation
- **Dev Container Guide**: [DEV-CONTAINER.md](../development/DEV-CONTAINER.md) - PM2 and log aggregation in dev
- **Full setup**: [BARE-METAL-SETUP.md](BARE-METAL-SETUP.md) - PostgreSQL Function Observability section
- **Architecture**: [adr/0050-postgresql-function-observability.md](../adr/0050-postgresql-function-observability.md)
- **Troubleshooting details**: [LOGSTASH-TROUBLESHOOTING.md](LOGSTASH-TROUBLESHOOTING.md)

View File

@@ -0,0 +1,896 @@
# Monitoring Guide
This guide covers all aspects of monitoring the Flyer Crawler application across development, test, and production environments.
## Table of Contents
1. [Health Checks](#health-checks)
2. [Bugsink Error Tracking](#bugsink-error-tracking)
3. [Logstash Log Aggregation](#logstash-log-aggregation)
4. [PM2 Process Monitoring](#pm2-process-monitoring)
5. [Database Monitoring](#database-monitoring)
6. [Redis Monitoring](#redis-monitoring)
7. [Production Alerts and On-Call](#production-alerts-and-on-call)
---
## Health Checks
The application exposes health check endpoints at `/api/health/*` implementing ADR-020.
### Endpoint Reference
| Endpoint | Purpose | Use Case |
| ----------------------- | ---------------------- | --------------------------------------- |
| `/api/health/ping` | Simple connectivity | Quick "is it running?" check |
| `/api/health/live` | Liveness probe | Container orchestration restart trigger |
| `/api/health/ready` | Readiness probe | Load balancer traffic routing |
| `/api/health/startup` | Startup probe | Initial container readiness |
| `/api/health/db-schema` | Schema verification | Deployment validation |
| `/api/health/db-pool` | Connection pool status | Performance diagnostics |
| `/api/health/redis` | Redis connectivity | Cache/queue health |
| `/api/health/storage` | File storage access | Upload capability |
| `/api/health/time` | Server time sync | Time-sensitive operations |
### Liveness Probe (`/api/health/live`)
Returns 200 OK if the Node.js process is running. No external dependencies.
```bash
# Check liveness
curl -s https://flyer-crawler.projectium.com/api/health/live | jq .
# Expected response
{
"success": true,
"data": {
"status": "ok",
"timestamp": "2026-01-22T10:00:00.000Z"
}
}
```
**Usage**: If this endpoint fails, restart the application immediately.
### Readiness Probe (`/api/health/ready`)
Comprehensive check of all critical dependencies: database, Redis, and storage.
```bash
# Check readiness
curl -s https://flyer-crawler.projectium.com/api/health/ready | jq .
# Expected healthy response (200)
{
"success": true,
"data": {
"status": "healthy",
"timestamp": "2026-01-22T10:00:00.000Z",
"uptime": 3600.5,
"services": {
"database": {
"status": "healthy",
"latency": 5,
"details": {
"totalConnections": 10,
"idleConnections": 8,
"waitingConnections": 0
}
},
"redis": {
"status": "healthy",
"latency": 2
},
"storage": {
"status": "healthy",
"latency": 1,
"details": {
"path": "/var/www/flyer-crawler.projectium.com/flyer-images"
}
}
}
}
}
```
**Status Values**:
| Status | Meaning | Action |
| ----------- | ------------------------------------------------ | ------------------------- |
| `healthy` | All critical services operational | None required |
| `degraded` | Non-critical issues (e.g., high connection wait) | Monitor closely |
| `unhealthy` | Critical service unavailable (returns 503) | Remove from load balancer |
### Database Health Thresholds
| Metric | Healthy | Degraded | Unhealthy |
| ------------------- | ------------------- | -------- | ---------------- |
| Query response | `SELECT 1` succeeds | N/A | Connection fails |
| Waiting connections | 0-3 | 4+ | N/A |
### Verifying Services from CLI
**Production**:
```bash
# Quick health check
curl -s https://flyer-crawler.projectium.com/api/health/ready | jq '.data.status'
# Database pool status
curl -s https://flyer-crawler.projectium.com/api/health/db-pool | jq .
# Redis health
curl -s https://flyer-crawler.projectium.com/api/health/redis | jq .
```
**Test Environment**:
```bash
# Test environment runs on port 3002
curl -s https://flyer-crawler-test.projectium.com/api/health/ready | jq .
```
**Dev Container**:
```bash
# From inside the container
curl -s http://localhost:3001/api/health/ready | jq .
# From Windows host (via port mapping)
curl -s http://localhost:3001/api/health/ready | jq .
```
### Admin System Check UI
The admin dashboard at `/admin` includes a **System Check** component that runs all health checks with a visual interface:
1. Navigate to `https://flyer-crawler.projectium.com/admin`
2. Login with admin credentials
3. View the "System Check" section
4. Click "Re-run Checks" to verify all services
Checks include:
- Backend Server Connection
- PM2 Process Status
- Database Connection Pool
- Redis Connection
- Database Schema
- Default Admin User
- Assets Storage Directory
- Gemini API Key
---
## Bugsink Error Tracking
Bugsink is our self-hosted, Sentry-compatible error tracking system (ADR-015).
### Access Points
| Environment | URL | Purpose |
| ----------------- | -------------------------------- | -------------------------- |
| **Production** | `https://bugsink.projectium.com` | Production and test errors |
| **Dev Container** | `https://localhost:8443` | Local development errors |
### Credentials
**Production Bugsink**:
- Credentials stored in password manager
- Admin account created during initial deployment
**Dev Container Bugsink**:
- Email: `admin@localhost`
- Password: `admin`
### Projects
| Project ID | Name | Environment | Error Source |
| ---------- | --------------------------------- | ----------- | ------------------------------- |
| 1 | flyer-crawler-backend | Production | Backend Node.js errors |
| 2 | flyer-crawler-frontend | Production | Frontend JavaScript errors |
| 3 | flyer-crawler-backend-test | Test | Test environment backend |
| 4 | flyer-crawler-frontend-test | Test | Test environment frontend |
| 5 | flyer-crawler-infrastructure | Production | PostgreSQL, Redis, NGINX errors |
| 6 | flyer-crawler-test-infrastructure | Test | Test infra errors |
**Dev Container Projects** (localhost:8000):
- Project 1: Backend (Dev)
- Project 2: Frontend (Dev)
### Accessing Errors via Web UI
1. Navigate to the Bugsink URL
2. Login with credentials
3. Select project from the sidebar
4. Click on an issue to view details
**Issue Details Include**:
- Exception type and message
- Full stack trace
- Request context (URL, method, headers)
- User context (if authenticated)
- Occurrence statistics (first seen, last seen, count)
- Release/version information
### Accessing Errors via MCP
Claude Code and other AI tools can access Bugsink via MCP servers.
**Available MCP Tools**:
```bash
# List all projects
mcp__bugsink__list_projects
# List unresolved issues for a project
mcp__bugsink__list_issues --project_id 1 --status unresolved
# Get issue details
mcp__bugsink__get_issue --issue_id <uuid>
# Get stacktrace (pre-rendered Markdown)
mcp__bugsink__get_stacktrace --event_id <uuid>
# List events for an issue
mcp__bugsink__list_events --issue_id <uuid>
```
**MCP Server Configuration**:
Production (in `~/.claude/settings.json`):
```json
{
"bugsink": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "https://bugsink.projectium.com",
"BUGSINK_TOKEN": "<token>"
}
}
}
```
Dev Container (in `.mcp.json`):
```json
{
"localerrors": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "http://127.0.0.1:8000",
"BUGSINK_TOKEN": "<token>"
}
}
}
```
### Creating API Tokens
Bugsink 2.0.11 does not have a UI for API tokens. Create via Django management command.
**Production**:
```bash
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
```
**Dev Container**:
```bash
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
```
The command outputs a 40-character hex token.
### Interpreting Errors
**Error Anatomy**:
```
TypeError: Cannot read properties of undefined (reading 'map')
├── Exception Type: TypeError
├── Message: Cannot read properties of undefined (reading 'map')
├── Where: FlyerItemsList.tsx:45:23
├── When: 2026-01-22T10:30:00.000Z
├── Count: 12 occurrences
└── Context:
├── URL: GET /api/flyers/123/items
├── User: user@example.com
└── Release: v0.12.5
```
**Common Error Patterns**:
| Pattern | Likely Cause | Investigation |
| ----------------------------------- | ------------------------------------------------- | -------------------------------------------------- |
| `TypeError: ... undefined` | Missing null check, API returned unexpected shape | Check API response, add defensive coding |
| `DatabaseError: Connection timeout` | Pool exhaustion, slow queries | Check `/api/health/db-pool`, review slow query log |
| `RedisConnectionError` | Redis unavailable | Check Redis service, network connectivity |
| `ValidationError: ...` | Invalid input, schema mismatch | Review request payload, update validation |
| `NotFoundError: ...` | Missing resource | Verify resource exists, check ID format |
### Error Triage Workflow
1. **Review new issues daily** in Bugsink
2. **Categorize by severity**:
- **Critical**: Data corruption, security, payment failures
- **High**: Core feature broken for many users
- **Medium**: Feature degraded, workaround available
- **Low**: Minor UX issues, cosmetic bugs
3. **Check occurrence count** - frequent errors need urgent attention
4. **Review stack trace** - identify root cause
5. **Check recent deployments** - did a release introduce this?
6. **Create Gitea issue** if not auto-synced
### Bugsink-to-Gitea Sync
The test environment automatically syncs Bugsink issues to Gitea (see `docs/BUGSINK-SYNC.md`).
**Sync Workflow**:
1. Runs every 15 minutes on test server
2. Fetches unresolved issues from all Bugsink projects
3. Creates Gitea issues with appropriate labels
4. Marks synced issues as resolved in Bugsink
**Manual Sync**:
```bash
# Trigger sync via API (test environment only)
curl -X POST https://flyer-crawler-test.projectium.com/api/admin/bugsink/sync \
-H "Authorization: Bearer <admin_jwt>"
```
---
## Logstash Log Aggregation
Logstash aggregates logs from multiple sources and forwards errors to Bugsink (ADR-050).
### Architecture
```
Log Sources Logstash Outputs
┌──────────────┐ ┌─────────────┐ ┌─────────────┐
│ PostgreSQL │──────────────│ │───────────│ Bugsink │
│ PM2 Workers │──────────────│ Filter │───────────│ (errors) │
│ Redis │──────────────│ & Route │───────────│ │
│ NGINX │──────────────│ │───────────│ File Logs │
└──────────────┘ └─────────────┘ │ (all logs) │
└─────────────┘
```
### Configuration Files
| Path | Purpose |
| --------------------------------------------------- | --------------------------- |
| `/etc/logstash/conf.d/bugsink.conf` | Main pipeline configuration |
| `/etc/postgresql/14/main/conf.d/observability.conf` | PostgreSQL logging settings |
| `/var/log/logstash/` | Logstash file outputs |
| `/var/lib/logstash/sincedb_*` | File position tracking |
### Log Sources
| Source | Path | Contents |
| ----------- | -------------------------------------------------- | ----------------------------------- |
| PostgreSQL | `/var/log/postgresql/*.log` | Function logs, slow queries, errors |
| PM2 Workers | `/home/gitea-runner/.pm2/logs/flyer-crawler-*.log` | Worker stdout/stderr |
| Redis | `/var/log/redis/redis-server.log` | Connection errors, memory warnings |
| NGINX | `/var/log/nginx/access.log`, `error.log` | HTTP requests, upstream errors |
### Pipeline Status
**Check Logstash Service**:
```bash
ssh root@projectium.com
# Service status
systemctl status logstash
# Recent logs
journalctl -u logstash -n 50 --no-pager
# Pipeline statistics
curl -s http://localhost:9600/_node/stats/pipelines?pretty | jq '.pipelines.main.events'
# Events processed today
curl -s http://localhost:9600/_node/stats/pipelines?pretty | jq '{
in: .pipelines.main.events.in,
out: .pipelines.main.events.out,
filtered: .pipelines.main.events.filtered
}'
```
**Check Filter Performance**:
```bash
# Grok pattern success/failure rates
curl -s http://localhost:9600/_node/stats/pipelines?pretty | \
jq '.pipelines.main.plugins.filters[] | select(.name == "grok") | {name, events_in: .events.in, events_out: .events.out, failures}'
```
### Viewing Aggregated Logs
```bash
# PM2 worker logs (all workers combined)
tail -f /var/log/logstash/pm2-workers-$(date +%Y-%m-%d).log
# Redis operational logs
tail -f /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
# NGINX access logs (parsed)
tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
# PostgreSQL function logs
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log
```
### Troubleshooting Logstash
| Issue | Diagnostic | Solution |
| --------------------- | --------------------------- | ------------------------------- |
| No events processed | `systemctl status logstash` | Start/restart service |
| Config syntax error | Test config command | Fix config file |
| Grok failures | Check stats endpoint | Update grok patterns |
| Wrong Bugsink project | Check environment tags | Verify tag routing |
| Permission denied | `groups logstash` | Add to `postgres`, `adm` groups |
| PM2 logs not captured | Check file paths | Verify log file existence |
| High disk usage | Check log rotation | Configure logrotate |
**Test Configuration**:
```bash
/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf
```
**Restart After Config Change**:
```bash
systemctl restart logstash
journalctl -u logstash -f # Watch for startup errors
```
---
## PM2 Process Monitoring
PM2 manages the Node.js application processes in production.
### Process Overview
**Production Processes** (`ecosystem.config.cjs`):
| Process Name | Script | Purpose | Instances |
| -------------------------------- | ----------- | -------------------- | ------------------ |
| `flyer-crawler-api` | `server.ts` | Express API server | Cluster (max CPUs) |
| `flyer-crawler-worker` | `worker.ts` | BullMQ job processor | 1 |
| `flyer-crawler-analytics-worker` | `worker.ts` | Analytics jobs | 1 |
**Test Processes** (`ecosystem-test.config.cjs`):
| Process Name | Script | Port | Instances |
| ------------------------------------- | ----------- | ---- | ------------- |
| `flyer-crawler-api-test` | `server.ts` | 3002 | 1 (fork mode) |
| `flyer-crawler-worker-test` | `worker.ts` | N/A | 1 |
| `flyer-crawler-analytics-worker-test` | `worker.ts` | N/A | 1 |
### Basic Commands
```bash
ssh root@projectium.com
su - gitea-runner # PM2 runs under this user
# List all processes
pm2 list
# Process details
pm2 show flyer-crawler-api
# Monitor in real-time
pm2 monit
# View logs
pm2 logs flyer-crawler-api
pm2 logs flyer-crawler-worker --lines 100
# View all logs
pm2 logs
# Restart processes
pm2 restart flyer-crawler-api
pm2 restart all
# Reload without downtime (cluster mode only)
pm2 reload flyer-crawler-api
# Stop processes
pm2 stop flyer-crawler-api
```
### Health Indicators
**Healthy Process**:
```
┌─────────────────────┬────┬─────────┬─────────┬───────┬────────┬─────────┬──────────┐
│ Name │ id │ mode │ status │ cpu │ mem │ uptime │ restarts │
├─────────────────────┼────┼─────────┼─────────┼───────┼────────┼─────────┼──────────┤
│ flyer-crawler-api │ 0 │ cluster │ online │ 0.5% │ 150MB │ 5d │ 0 │
│ flyer-crawler-api │ 1 │ cluster │ online │ 0.3% │ 145MB │ 5d │ 0 │
│ flyer-crawler-worker│ 2 │ fork │ online │ 0.1% │ 200MB │ 5d │ 0 │
└─────────────────────┴────┴─────────┴─────────┴───────┴────────┴─────────┴──────────┘
```
**Warning Signs**:
- `status: errored` - Process crashed
- High `restarts` count - Instability
- High `mem` (>500MB for API, >1GB for workers) - Memory leak
- Low `uptime` with high restarts - Repeated crashes
### Log File Locations
| Process | stdout | stderr |
| ---------------------- | ----------------------------------------------------------- | --------------- |
| `flyer-crawler-api` | `/home/gitea-runner/.pm2/logs/flyer-crawler-api-out.log` | `...-error.log` |
| `flyer-crawler-worker` | `/home/gitea-runner/.pm2/logs/flyer-crawler-worker-out.log` | `...-error.log` |
### Memory Management
PM2 is configured to restart processes when they exceed memory limits:
| Process | Memory Limit | Action |
| ---------------- | ------------ | ------------ |
| API | 500MB | Auto-restart |
| Worker | 1GB | Auto-restart |
| Analytics Worker | 1GB | Auto-restart |
**Check Memory Usage**:
```bash
pm2 show flyer-crawler-api | grep memory
pm2 show flyer-crawler-worker | grep memory
```
### Restart Strategies
PM2 uses exponential backoff for restarts:
```javascript
{
max_restarts: 40,
exp_backoff_restart_delay: 100, // Start at 100ms, exponentially increase
min_uptime: '10s', // Must run 10s to be considered "started"
}
```
**Force Restart After Repeated Failures**:
```bash
pm2 delete flyer-crawler-api
pm2 start ecosystem.config.cjs --only flyer-crawler-api
```
---
## Database Monitoring
### Connection Pool Status
The application uses a PostgreSQL connection pool with these defaults:
| Setting | Value | Purpose |
| ------------------------- | ----- | -------------------------------- |
| `max` | 20 | Maximum concurrent connections |
| `idleTimeoutMillis` | 30000 | Close idle connections after 30s |
| `connectionTimeoutMillis` | 2000 | Fail if connection takes >2s |
**Check Pool Status via API**:
```bash
curl -s https://flyer-crawler.projectium.com/api/health/db-pool | jq .
# Response
{
"success": true,
"data": {
"message": "Pool Status: 10 total, 8 idle, 0 waiting.",
"totalCount": 10,
"idleCount": 8,
"waitingCount": 0
}
}
```
**Pool Health Thresholds**:
| Metric | Healthy | Warning | Critical |
| ------------------- | ------- | ------- | ---------- |
| Waiting Connections | 0-2 | 3-4 | 5+ |
| Total Connections | 1-15 | 16-19 | 20 (maxed) |
### Slow Query Logging
PostgreSQL is configured to log slow queries:
```ini
# /etc/postgresql/14/main/conf.d/observability.conf
log_min_duration_statement = 1000 # Log queries over 1 second
```
**View Slow Queries**:
```bash
ssh root@projectium.com
grep "duration:" /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log | tail -20
```
### Database Size Monitoring
```bash
# Connect to production database
psql -h localhost -U flyer_crawler_prod -d flyer-crawler-prod
# Database size
SELECT pg_size_pretty(pg_database_size('flyer-crawler-prod'));
# Table sizes
SELECT
relname AS table,
pg_size_pretty(pg_total_relation_size(relid)) AS total_size,
pg_size_pretty(pg_relation_size(relid)) AS data_size,
pg_size_pretty(pg_indexes_size(relid)) AS index_size
FROM pg_catalog.pg_statio_user_tables
ORDER BY pg_total_relation_size(relid) DESC
LIMIT 10;
# Check for bloat
SELECT schemaname, relname, n_dead_tup, n_live_tup,
round(n_dead_tup * 100.0 / nullif(n_live_tup + n_dead_tup, 0), 2) as dead_pct
FROM pg_stat_user_tables
WHERE n_dead_tup > 1000
ORDER BY n_dead_tup DESC;
```
### Disk Space Monitoring
```bash
# Check PostgreSQL data directory
du -sh /var/lib/postgresql/14/main/
# Check available disk space
df -h /var/lib/postgresql/
# Estimate growth rate
psql -c "SELECT date_trunc('day', created_at) as day, count(*)
FROM flyer_items
WHERE created_at > now() - interval '7 days'
GROUP BY 1 ORDER BY 1;"
```
### Database Health via MCP
```bash
# Query database directly
mcp__devdb__query --sql "SELECT count(*) FROM flyers WHERE created_at > now() - interval '1 day'"
# Check connection count
mcp__devdb__query --sql "SELECT count(*) FROM pg_stat_activity WHERE datname = 'flyer_crawler_dev'"
```
---
## Redis Monitoring
### Basic Health Check
```bash
# Via API endpoint
curl -s https://flyer-crawler.projectium.com/api/health/redis | jq .
# Direct Redis check (on server)
redis-cli ping # Should return PONG
```
### Memory Usage
```bash
redis-cli info memory | grep -E "used_memory_human|maxmemory_human|mem_fragmentation_ratio"
# Expected output
used_memory_human:50.00M
maxmemory_human:256.00M
mem_fragmentation_ratio:1.05
```
**Memory Thresholds**:
| Metric | Healthy | Warning | Critical |
| ------------------- | ----------- | ------- | -------- |
| Used Memory | <70% of max | 70-85% | >85% |
| Fragmentation Ratio | 1.0-1.5 | 1.5-2.0 | >2.0 |
### Cache Statistics
```bash
redis-cli info stats | grep -E "keyspace_hits|keyspace_misses|evicted_keys"
# Calculate hit rate
# Hit Rate = keyspace_hits / (keyspace_hits + keyspace_misses) * 100
```
**Cache Hit Rate Targets**:
- Excellent: >95%
- Good: 85-95%
- Needs attention: <85%
### Queue Monitoring
BullMQ queues are stored in Redis:
```bash
# List all queues
redis-cli keys "bull:*:id"
# Check queue depths
redis-cli llen "bull:flyer-processing:wait"
redis-cli llen "bull:email-sending:wait"
redis-cli llen "bull:analytics-reporting:wait"
# Check failed jobs
redis-cli llen "bull:flyer-processing:failed"
```
**Queue Depth Thresholds**:
| Queue | Normal | Warning | Critical |
| ------------------- | ------ | ------- | -------- |
| flyer-processing | 0-10 | 11-50 | >50 |
| email-sending | 0-100 | 101-500 | >500 |
| analytics-reporting | 0-5 | 6-20 | >20 |
### Bull Board UI
Access the job queue dashboard:
- **Production**: `https://flyer-crawler.projectium.com/api/admin/jobs` (requires admin auth)
- **Test**: `https://flyer-crawler-test.projectium.com/api/admin/jobs`
- **Dev**: `http://localhost:3001/api/admin/jobs`
Features:
- View all queues and job counts
- Inspect job data and errors
- Retry failed jobs
- Clean completed jobs
### Redis Database Allocation
| Database | Purpose |
| -------- | ------------------------ |
| 0 | BullMQ production queues |
| 1 | BullMQ test queues |
| 15 | Bugsink sync state |
---
## Production Alerts and On-Call
### Critical Monitoring Targets
| Service | Check | Interval | Alert Threshold |
| ---------- | ------------------- | -------- | ---------------------- |
| API Server | `/api/health/ready` | 1 min | 2 consecutive failures |
| Database | Pool waiting count | 1 min | >5 waiting |
| Redis | Memory usage | 5 min | >85% of maxmemory |
| Disk Space | `/var/log` | 15 min | <10GB free |
| Worker | Queue depth | 5 min | >50 jobs waiting |
| Error Rate | Bugsink issue count | 15 min | >10 new issues/hour |
### Alert Channels
Configure alerts in your monitoring tool (UptimeRobot, Datadog, etc.):
1. **Slack channel**: `#flyer-crawler-alerts`
2. **Email**: On-call rotation email
3. **PagerDuty**: Critical issues only
### On-Call Response Procedures
**P1 - Critical (Site Down)**:
1. Acknowledge alert within 5 minutes
2. Check `/api/health/ready` - identify failing service
3. Check PM2 status: `pm2 list`
4. Check recent deploys: `git log -5 --oneline`
5. If database: check pool, restart if needed
6. If Redis: check memory, flush if critical
7. If application: restart PM2 processes
8. Document in incident channel
**P2 - High (Degraded Service)**:
1. Acknowledge within 15 minutes
2. Review Bugsink for error patterns
3. Check system resources (CPU, memory, disk)
4. Identify root cause
5. Plan remediation
6. Create Gitea issue if not auto-created
**P3 - Medium (Non-Critical)**:
1. Acknowledge within 1 hour
2. Review during business hours
3. Create Gitea issue for tracking
### Quick Diagnostic Commands
```bash
# Full system health check
ssh root@projectium.com << 'EOF'
echo "=== Service Status ==="
systemctl status pm2-gitea-runner --no-pager
systemctl status logstash --no-pager
systemctl status redis --no-pager
systemctl status postgresql --no-pager
echo "=== PM2 Processes ==="
su - gitea-runner -c "pm2 list"
echo "=== Disk Space ==="
df -h / /var
echo "=== Memory ==="
free -h
echo "=== Recent Errors ==="
journalctl -p err -n 20 --no-pager
EOF
```
### Runbook Quick Reference
| Symptom | First Action | If That Fails |
| --------------- | ---------------- | --------------------- |
| 503 errors | Restart PM2 | Check database, Redis |
| Slow responses | Check DB pool | Review slow query log |
| High error rate | Check Bugsink | Review recent deploys |
| Queue backlog | Restart worker | Scale workers |
| Out of memory | Restart process | Increase PM2 limit |
| Disk full | Clean old logs | Expand volume |
| Redis OOM | Flush cache keys | Increase maxmemory |
### Post-Incident Review
After any P1/P2 incident:
1. Write incident report within 24 hours
2. Identify root cause
3. Document timeline of events
4. List action items to prevent recurrence
5. Schedule review meeting if needed
6. Update runbooks if new procedures discovered
---
## Related Documentation
- [ADR-015: Application Performance Monitoring](../adr/0015-application-performance-monitoring-and-error-tracking.md)
- [ADR-020: Health Checks](../adr/0020-health-checks-and-liveness-readiness-probes.md)
- [ADR-050: PostgreSQL Function Observability](../adr/0050-postgresql-function-observability.md)
- [ADR-053: Worker Health Checks](../adr/0053-worker-health-checks.md)
- [DEV-CONTAINER-BUGSINK.md](../DEV-CONTAINER-BUGSINK.md)
- [BUGSINK-SYNC.md](../BUGSINK-SYNC.md)
- [LOGSTASH-QUICK-REF.md](LOGSTASH-QUICK-REF.md)
- [LOGSTASH-TROUBLESHOOTING.md](LOGSTASH-TROUBLESHOOTING.md)
- [LOGSTASH_DEPLOYMENT_CHECKLIST.md](../LOGSTASH_DEPLOYMENT_CHECKLIST.md)

View File

@@ -0,0 +1,300 @@
# AI Usage Subagent Guide
The **ai-usage** subagent specializes in LLM APIs (Gemini, Claude), prompt engineering, and AI-powered features in the Flyer Crawler project.
## When to Use
Use the **ai-usage** subagent when you need to:
- Integrate with the Gemini API for flyer extraction
- Debug AI extraction failures
- Optimize prompts for better accuracy
- Handle rate limiting and API errors
- Implement new AI-powered features
- Fine-tune extraction schemas
## What ai-usage Knows
The ai-usage subagent understands:
- Google Generative AI (Gemini) API
- Flyer extraction prompts and schemas
- Error handling for AI services
- Rate limiting strategies
- Token optimization
- AI service architecture (ADR-041)
## AI Architecture Overview
```
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
│ Flyer Upload │───►│ AI Service │───►│ Gemini API │
│ │ │ │ │ │
└─────────────────┘ │ - Preprocessing │ │ - Vision model │
│ - Prompt build │ │ - JSON output │
│ - Response parse│ │ │
└─────────────────┘ └─────────────────┘
┌─────────────────┐
│ Validation & │
│ Normalization │
└─────────────────┘
```
## Key Files
| File | Purpose |
| ----------------------------------------------- | ------------------------------------ |
| `src/services/aiService.server.ts` | Gemini API integration |
| `src/services/flyerProcessingService.server.ts` | Flyer extraction pipeline |
| `src/schemas/flyer.schemas.ts` | Zod schemas for AI output validation |
| `src/types/ai.types.ts` | TypeScript types for AI responses |
## Example Requests
### Debugging Extraction Failures
```
"Use ai-usage to debug why flyer extractions are failing for
multi-page PDFs. The error logs show 'Invalid JSON response'
but only for certain stores."
```
### Optimizing Prompts
```
"Use ai-usage to improve the item extraction prompt. Currently
it's missing unit prices when items show 'X for $Y' pricing
(e.g., '3 for $5')."
```
### Handling Rate Limits
```
"Use ai-usage to implement exponential backoff for Gemini API
rate limits. We're seeing 429 errors during high-volume uploads."
```
### Adding New AI Features
```
"Use ai-usage to add a feature that uses Gemini to categorize
extracted items into grocery categories (produce, dairy, meat, etc.)."
```
## Extraction Pipeline
### 1. Image Preprocessing
```typescript
// Convert PDF to images, resize large images
const processedImages = await imageProcessor.prepareForAI(uploadedFile);
```
### 2. Prompt Construction
The extraction prompt includes:
- System instructions for the AI model
- Expected output schema (JSON)
- Examples of correct extraction
- Handling instructions for edge cases
### 3. API Call
```typescript
const response = await aiService.extractFlyerData(processedImages, storeContext, extractionOptions);
```
### 4. Response Validation
```typescript
// Validate against Zod schema
const validatedItems = flyerItemsSchema.parse(response.items);
```
### 5. Normalization
```typescript
// Normalize prices, units, quantities
const normalizedItems = normalizeExtractedItems(validatedItems);
```
## Common Issues and Solutions
### Issue: Inconsistent Price Extraction
**Symptoms**: Same item priced differently on different extractions.
**Solution**: Improve prompt with explicit price format examples:
```
"Price formats to recognize:
- $X.XX (regular price)
- X for $Y.YY (multi-buy)
- $X.XX/lb or $X.XX/kg (unit price)
- $X.XX each (per item)
- SAVE $X.XX (discount amount, not item price)"
```
### Issue: Missing Items from Dense Flyers
**Symptoms**: Flyers with many items on one page have missing extractions.
**Solution**:
1. Split page into quadrants for separate extraction
2. Increase token limit for response
3. Use structured grid-based prompting
### Issue: Rate Limit Errors (429)
**Symptoms**: `429 Too Many Requests` errors during bulk uploads.
**Solution**: Implement request queuing:
```typescript
// Add to job queue instead of direct call
await flyerQueue.add(
'extract',
{
flyerId,
images,
},
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 2000,
},
},
);
```
### Issue: Hallucinated Items
**Symptoms**: Items extracted that don't exist in the flyer.
**Solution**:
1. Add confidence scoring to extraction
2. Request bounding box coordinates for verification
3. Add post-extraction validation against image
## Prompt Engineering Best Practices
### 1. Be Specific About Output Format
```
Output MUST be valid JSON matching this schema:
{
"items": [
{
"name": "string (product name as shown)",
"brand": "string or null",
"price": number (in dollars),
"unit": "string (each, lb, kg, etc.)",
"quantity": number (default 1)
}
]
}
```
### 2. Provide Examples
```
Example extractions:
- "Chicken Breast $4.99/lb" -> {"name": "Chicken Breast", "price": 4.99, "unit": "lb"}
- "Coca-Cola 12pk $5.99" -> {"name": "Coca-Cola", "quantity": 12, "price": 5.99, "unit": "each"}
```
### 3. Handle Edge Cases Explicitly
```
Special cases:
- If "LIMIT X" shown, add to notes, don't affect price
- If "SAVE $X" shown without base price, mark price as null
- If item is "FREE with purchase", set price to 0
```
### 4. Request Structured Thinking
```
For each item:
1. Identify the product name and brand
2. Find the associated price
3. Determine if price is per-unit or total
4. Extract any quantity information
```
## Monitoring AI Performance
### Metrics to Track
| Metric | Description | Target |
| ----------------------- | --------------------------------------- | --------------- |
| Extraction success rate | % of flyers processed without error | >95% |
| Items per flyer | Average items extracted | Varies by store |
| Price accuracy | Match rate vs manual verification | >98% |
| Response time | Time from upload to extraction complete | <30s |
### Logging
```typescript
log.info(
{
flyerId,
itemCount: extractedItems.length,
processingTime: duration,
modelVersion: response.model,
tokenUsage: response.usage,
},
'Flyer extraction completed',
);
```
## Environment Configuration
| Variable | Purpose |
| -------------------------------- | --------------------------- |
| `VITE_GOOGLE_GENAI_API_KEY` | Gemini API key (production) |
| `VITE_GOOGLE_GENAI_API_KEY_TEST` | Gemini API key (test) |
**Note**: Use separate API keys for production and test to avoid rate limit conflicts and enable separate billing tracking.
## Testing AI Features
### Unit Tests
Mock the Gemini API response:
```typescript
vi.mock('@google/generative-ai', () => ({
GoogleGenerativeAI: vi.fn().mockImplementation(() => ({
getGenerativeModel: () => ({
generateContent: vi.fn().mockResolvedValue({
response: {
text: () => JSON.stringify({ items: mockItems }),
},
}),
}),
})),
}));
```
### Integration Tests
Use recorded responses for deterministic testing:
```typescript
// Save real API responses to fixtures
const fixtureResponse = await fs.readFile('fixtures/gemini-response.json');
```
## Related Documentation
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
- [../adr/0041-ai-gemini-integration-architecture.md](../adr/0041-ai-gemini-integration-architecture.md) - AI integration ADR
- [../adr/0046-image-processing-pipeline.md](../adr/0046-image-processing-pipeline.md) - Image processing
- [CODER-GUIDE.md](./CODER-GUIDE.md) - For implementing AI features

View File

@@ -0,0 +1,312 @@
# Coder Subagent Guide
The **coder** subagent is your primary tool for writing and modifying production Node.js/TypeScript code in the Flyer Crawler project. This guide explains how to work effectively with the coder subagent.
## When to Use the Coder Subagent
Use the coder subagent when you need to:
- Implement new features or functionality
- Fix bugs in existing code
- Refactor existing code
- Add new API endpoints
- Create new React components
- Write service layer logic
- Implement business rules
## What the Coder Subagent Knows
The coder subagent has deep knowledge of:
### Project Architecture
```
Routes -> Services -> Repositories -> Database
|
External APIs (*.server.ts)
```
- **Routes Layer**: Request/response handling, validation, authentication
- **Services Layer**: Business logic, transaction coordination, external APIs
- **Repositories Layer**: Database access, query construction, error translation
### Key Patterns
| Pattern | ADR | Implementation |
| ------------------ | ------- | ---------------------------------------------------------- |
| Error Handling | ADR-001 | `handleDbError()`, throw `NotFoundError` |
| Repository Methods | ADR-034 | `get*` throws, `find*` returns null, `list*` returns array |
| API Responses | ADR-028 | `sendSuccess()`, `sendPaginated()`, `sendError()` |
| Transactions | ADR-002 | `withTransaction(async (client) => {...})` |
### File Naming Conventions
| Pattern | Location | Purpose |
| ------------- | ------------------ | -------------------------------- |
| `*.db.ts` | `src/services/db/` | Database repositories |
| `*.server.ts` | `src/services/` | Server-only code (external APIs) |
| `*.routes.ts` | `src/routes/` | Express route handlers |
| `*.test.ts` | Colocated | Unit tests |
## How to Request Code Changes
### Good Request Examples
**Specific and contextual:**
```
"Use the coder subagent to add a new endpoint GET /api/stores/:id/locations
that returns all locations for a store, following the existing patterns
in stores.routes.ts"
```
**With acceptance criteria:**
```
"Use the coder subagent to implement the shopping list sharing feature:
- Add a share_token column to shopping_lists table
- Create POST /api/shopping-lists/:id/share endpoint
- Return a shareable link with the token
- Allow anonymous users to view shared lists"
```
**Bug fix with reproduction steps:**
```
"Use the coder subagent to fix the issue where flyer items are not
sorted by price on the deals page. The expected behavior is lowest
price first, but currently they appear in insertion order."
```
### Less Effective Request Examples
**Too vague:**
```
"Make the code better"
```
**Missing context:**
```
"Add a feature to search things"
```
**Multiple unrelated tasks:**
```
"Fix the login bug, add a new table, and update the homepage"
```
## Common Workflows
### Adding a New API Endpoint
The coder subagent will follow this workflow:
1. **Add route** in `src/routes/{domain}.routes.ts`
2. **Use `validateRequest(schema)`** middleware for input validation
3. **Call service layer** (never access DB directly from routes)
4. **Return via** `sendSuccess()` or `sendPaginated()`
5. **Add tests** in `*.routes.test.ts`
**Example Code Pattern:**
```typescript
// src/routes/stores.routes.ts
router.get('/:id/locations', validateRequest(getStoreLocationsSchema), async (req, res, next) => {
try {
const { id } = req.params;
const locations = await storeService.getLocationsForStore(parseInt(id, 10), req.log);
sendSuccess(res, { locations });
} catch (error) {
next(error);
}
});
```
### Adding a New Database Operation
The coder subagent will:
1. **Add method** to `src/services/db/{domain}.db.ts`
2. **Follow naming**: `get*` (throws), `find*` (returns null), `list*` (array)
3. **Use `handleDbError()`** for error handling
4. **Accept optional `PoolClient`** for transaction support
5. **Add unit test**
**Example Code Pattern:**
```typescript
// src/services/db/store.db.ts
export async function listLocationsByStoreId(
storeId: number,
client?: PoolClient,
): Promise<StoreLocation[]> {
const queryable = client || getPool();
try {
const result = await queryable.query<StoreLocation>(
`SELECT * FROM store_locations WHERE store_id = $1 ORDER BY created_at`,
[storeId],
);
return result.rows;
} catch (error) {
handleDbError(
error,
log,
'Database error in listLocationsByStoreId',
{ storeId },
{
entityName: 'StoreLocation',
defaultMessage: 'Failed to list store locations.',
},
);
}
}
```
### Adding a New React Component
The coder subagent will:
1. **Create component** in `src/components/` or feature-specific folder
2. **Follow Neo-Brutalism design** patterns (ADR-012)
3. **Use existing design tokens** from `src/styles/`
4. **Add unit tests** using Testing Library
**Example Code Pattern:**
```typescript
// src/components/StoreCard.tsx
import { Store } from '@/types';
interface StoreCardProps {
store: Store;
onSelect?: (store: Store) => void;
}
export function StoreCard({ store, onSelect }: StoreCardProps) {
return (
<div
className="brutal-card p-4 cursor-pointer hover:translate-x-1 hover:-translate-y-1 transition-transform"
onClick={() => onSelect?.(store)}
>
<h3 className="text-lg font-bold">{store.name}</h3>
<p className="text-sm text-gray-600">{store.location_count} locations</p>
</div>
);
}
```
## Code Quality Standards
The coder subagent adheres to these standards:
### TypeScript
- Strict TypeScript mode enabled
- No `any` types unless absolutely necessary
- Explicit return types for functions
- Proper interface/type definitions
### Error Handling
- Use custom error classes from `src/services/db/errors.db.ts`
- Never swallow errors silently
- Log errors with appropriate context
- Return meaningful error messages to API consumers
### Logging
- Use Pino logger (`src/services/logger.server.ts`)
- Include module context in log child
- Log at appropriate levels (info, warn, error)
- Include relevant data in structured format
### Testing
- All new code should have corresponding tests
- Follow testing patterns in ADR-010
- Use mock factories from `src/tests/utils/mockFactories.ts`
- Run tests in the dev container
## Platform Considerations
### Linux-Only Development
The coder subagent knows that this application runs exclusively on Linux:
- Uses POSIX-style paths (`/`)
- Assumes Linux shell commands
- References dev container environment
**Important**: Any code changes should be tested in the dev container:
```bash
podman exec -it flyer-crawler-dev npm run test:unit
```
### Database Schema Synchronization
When the coder subagent modifies database-related code, it will remind you:
> **Schema files must stay synchronized:**
>
> - `sql/master_schema_rollup.sql` - Test DB setup
> - `sql/initial_schema.sql` - Fresh install schema
> - `sql/migrations/*.sql` - Production changes
## Working with the Coder Subagent
### Before Starting
1. **Identify the scope** - What exactly needs to change?
2. **Check existing patterns** - Is there similar code to follow?
3. **Consider tests** - Will you need the testwriter subagent too?
### During Development
1. **Review changes incrementally** - Don't wait until the end
2. **Ask for explanations** - Understand why certain approaches are chosen
3. **Provide feedback** - Tell the coder if something doesn't look right
### After Completion
1. **Run tests** in the dev container
2. **Run type-check**: `npm run type-check`
3. **Review the changes** before committing
4. **Consider code review** with the code-reviewer subagent
## Common Issues and Solutions
### Issue: Code Doesn't Follow Project Patterns
**Solution**: Provide examples of existing code that follows the desired pattern. The coder will align with it.
### Issue: Missing Error Handling
**Solution**: Explicitly request comprehensive error handling:
```
"Include proper error handling using handleDbError and the project's
error classes for all database operations"
```
### Issue: Tests Not Included
**Solution**: Either:
1. Ask the coder to include tests: "Include unit tests for all new code"
2. Use the testwriter subagent separately for comprehensive test coverage
### Issue: Code Works on Windows but Fails on Linux
**Solution**: Always test in the dev container. The coder subagent writes Linux-compatible code, but IDE tooling might behave differently.
## Related Documentation
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
- [TESTER-GUIDE.md](./TESTER-GUIDE.md) - Testing strategies
- [../adr/0034-repository-pattern-standards.md](../adr/0034-repository-pattern-standards.md) - Repository patterns
- [../adr/0035-service-layer-architecture.md](../adr/0035-service-layer-architecture.md) - Service layer architecture
- [../adr/0028-api-response-standardization.md](../adr/0028-api-response-standardization.md) - API response patterns

View File

@@ -0,0 +1,419 @@
# Database Subagent Guide
This guide covers two database-focused subagents:
- **db-dev**: Database development - schemas, queries, migrations, optimization
- **db-admin**: Database administration - PostgreSQL/Redis admin, security, backups
## Understanding the Difference
| Aspect | db-dev | db-admin |
| --------------- | ----------------------------------- | ------------------------------------- |
| **Focus** | Application database code | Infrastructure and operations |
| **Tasks** | Queries, migrations, repositories | Performance tuning, backups, security |
| **Output** | SQL migrations, repository methods | Configuration, monitoring scripts |
| **When to Use** | Adding features, optimizing queries | Production issues, capacity planning |
## The db-dev Subagent
### When to Use
Use the **db-dev** subagent when you need to:
- Design new database tables or modify existing ones
- Write SQL queries or optimize existing ones
- Create database migrations
- Implement repository pattern methods
- Fix N+1 query problems
- Add indexes for performance
- Work with PostGIS spatial queries
### What db-dev Knows
The db-dev subagent has deep knowledge of:
- Project database schema (`sql/master_schema_rollup.sql`)
- Repository pattern standards (ADR-034)
- Transaction management (ADR-002)
- PostgreSQL-specific features (PostGIS, pg_trgm, etc.)
- Schema synchronization requirements
### Schema Synchronization (Critical)
> **Schema files MUST stay synchronized:**
>
> | File | Purpose |
> | ------------------------------ | --------------------------------- |
> | `sql/master_schema_rollup.sql` | Test DB setup, complete reference |
> | `sql/initial_schema.sql` | Fresh install schema |
> | `sql/migrations/*.sql` | Production incremental changes |
When db-dev creates a migration, it will also update the schema files.
### Example Requests
**Adding a new table:**
```
"Use db-dev to design a table for storing user recipe reviews.
Include fields for rating (1-5), review text, and relationships
to users and recipes. Create the migration and update schema files."
```
**Optimizing a slow query:**
```
"Use db-dev to optimize the query that lists flyers with their
item counts. It's currently doing N+1 queries and takes too long
with many flyers."
```
**Adding spatial search:**
```
"Use db-dev to add the ability to search stores within a radius
of a given location using PostGIS. Include the migration for
adding the geography column."
```
### Repository Pattern Standards
The db-dev subagent follows these naming conventions:
| Prefix | Returns | Behavior on Not Found |
| --------- | ------------------- | ------------------------------------ |
| `get*` | Single entity | Throws `NotFoundError` |
| `find*` | Entity or `null` | Returns `null` |
| `list*` | Array | Returns `[]` |
| `create*` | Created entity | Throws on constraint violation |
| `update*` | Updated entity | Throws `NotFoundError` if not exists |
| `delete*` | `void` or `boolean` | Throws `NotFoundError` if not exists |
### Example Migration
```sql
-- sql/migrations/20260121_add_recipe_reviews.sql
-- Create recipe_reviews table
CREATE TABLE IF NOT EXISTS recipe_reviews (
review_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
recipe_id UUID NOT NULL REFERENCES recipes(recipe_id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
rating INTEGER NOT NULL CHECK (rating >= 1 AND rating <= 5),
review_text TEXT,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
UNIQUE (recipe_id, user_id)
);
-- Add indexes
CREATE INDEX idx_recipe_reviews_recipe_id ON recipe_reviews(recipe_id);
CREATE INDEX idx_recipe_reviews_user_id ON recipe_reviews(user_id);
CREATE INDEX idx_recipe_reviews_rating ON recipe_reviews(rating);
-- Add trigger for updated_at
CREATE TRIGGER update_recipe_reviews_updated_at
BEFORE UPDATE ON recipe_reviews
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
```
### Example Repository Method
```typescript
// src/services/db/recipeReview.db.ts
import { handleDbError, NotFoundError } from './errors.db';
export async function getReviewById(reviewId: string, client?: PoolClient): Promise<RecipeReview> {
const queryable = client || getPool();
try {
const result = await queryable.query<RecipeReview>(
`SELECT * FROM recipe_reviews WHERE review_id = $1`,
[reviewId],
);
if (result.rows.length === 0) {
throw new NotFoundError(`Review with ID ${reviewId} not found.`);
}
return result.rows[0];
} catch (error) {
handleDbError(
error,
log,
'Database error in getReviewById',
{ reviewId },
{
entityName: 'RecipeReview',
defaultMessage: 'Failed to fetch review.',
},
);
}
}
export async function listReviewsByRecipeId(
recipeId: string,
options: { limit?: number; offset?: number } = {},
client?: PoolClient,
): Promise<RecipeReview[]> {
const queryable = client || getPool();
const { limit = 50, offset = 0 } = options;
try {
const result = await queryable.query<RecipeReview>(
`SELECT * FROM recipe_reviews
WHERE recipe_id = $1
ORDER BY created_at DESC
LIMIT $2 OFFSET $3`,
[recipeId, limit, offset],
);
return result.rows;
} catch (error) {
handleDbError(
error,
log,
'Database error in listReviewsByRecipeId',
{ recipeId, limit, offset },
{
entityName: 'RecipeReview',
defaultMessage: 'Failed to list reviews.',
},
);
}
}
```
## The db-admin Subagent
### When to Use
Use the **db-admin** subagent when you need to:
- Debug production database issues
- Configure PostgreSQL settings
- Set up database backups
- Analyze slow query logs
- Configure Redis for production
- Plan database capacity
- Manage database users and permissions
- Handle replication or failover
### What db-admin Knows
The db-admin subagent understands:
- PostgreSQL configuration and tuning
- Redis configuration for BullMQ queues
- Backup and recovery strategies (ADR-019)
- Connection pooling settings
- Production deployment setup
- Bugsink PostgreSQL observability (ADR-050)
### Example Requests
**Performance tuning:**
```
"Use db-admin to analyze why the database is running slow.
Check connection pool settings, identify slow queries, and
recommend PostgreSQL configuration changes."
```
**Backup configuration:**
```
"Use db-admin to set up daily automated backups for the
production database with 30-day retention."
```
**User management:**
```
"Use db-admin to create a read-only database user for
reporting purposes that can only SELECT from specific tables."
```
### Database Users
| User | Database | Purpose |
| -------------------- | -------------------- | ---------------------- |
| `flyer_crawler_prod` | `flyer-crawler-prod` | Production |
| `flyer_crawler_test` | `flyer-crawler-test` | Testing |
| `postgres` | All | Superuser (admin only) |
### Creating Database Users
```sql
-- As postgres superuser
CREATE DATABASE "flyer-crawler-test";
CREATE USER flyer_crawler_test WITH PASSWORD 'secure_password';
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
\c "flyer-crawler-test"
ALTER SCHEMA public OWNER TO flyer_crawler_test;
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
```
### PostgreSQL Configuration Guidance
For production, db-admin may recommend settings like:
```ini
# /etc/postgresql/14/main/conf.d/performance.conf
# Connection settings
max_connections = 100
shared_buffers = 256MB
# Query optimization
effective_cache_size = 768MB
random_page_cost = 1.1
# Write performance
wal_buffers = 16MB
checkpoint_completion_target = 0.9
# Logging
log_min_duration_statement = 1000 # Log queries over 1 second
```
### Redis Configuration Guidance
For BullMQ queues:
```ini
# /etc/redis/redis.conf
# Memory management
maxmemory 256mb
maxmemory-policy noeviction # BullMQ requires this
# Persistence
appendonly yes
appendfsync everysec
# Security
requirepass your_redis_password
```
## Common Database Tasks
### Running Migrations in Production
```bash
# SSH to production server
ssh root@projectium.com
# Run migration
cd /var/www/flyer-crawler.projectium.com
npm run db:migrate
```
### Checking Database Health
```bash
# Connection count
psql -c "SELECT count(*) FROM pg_stat_activity WHERE datname = 'flyer-crawler-prod';"
# Table sizes
psql -d "flyer-crawler-prod" -c "
SELECT
tablename,
pg_size_pretty(pg_total_relation_size(schemaname || '.' || tablename)) as size
FROM pg_tables
WHERE schemaname = 'public'
ORDER BY pg_total_relation_size(schemaname || '.' || tablename) DESC
LIMIT 10;"
# Slow queries
psql -d "flyer-crawler-prod" -c "
SELECT
calls,
mean_exec_time::numeric(10,2) as avg_ms,
query
FROM pg_stat_statements
ORDER BY mean_exec_time DESC
LIMIT 5;"
```
### Database Backup Commands
```bash
# Manual backup
pg_dump -U flyer_crawler_prod -h localhost "flyer-crawler-prod" > backup_$(date +%Y%m%d).sql
# Restore from backup
psql -U flyer_crawler_prod -h localhost "flyer-crawler-prod" < backup_20260121.sql
```
## N+1 Query Detection
The db-dev subagent is particularly skilled at identifying N+1 query problems:
**Problematic Pattern:**
```typescript
// BAD: N+1 queries
const flyers = await listFlyers();
for (const flyer of flyers) {
flyer.items = await listItemsByFlyerId(flyer.flyer_id); // N queries!
}
```
**Optimized Pattern:**
```typescript
// GOOD: Single query with JOIN or separate batch query
const flyersWithItems = await listFlyersWithItems(); // 1 query
// Or with batching:
const flyers = await listFlyers();
const flyerIds = flyers.map((f) => f.flyer_id);
const allItems = await listItemsByFlyerIds(flyerIds); // 1 query
// Group items by flyer_id in application code
```
## Working with PostGIS
The project uses PostGIS for spatial queries. Example:
```sql
-- Find stores within 10km of a location
SELECT
s.store_id,
s.name,
ST_Distance(
sl.location::geography,
ST_MakePoint(-79.3832, 43.6532)::geography
) / 1000 as distance_km
FROM stores s
JOIN store_locations sl ON s.store_id = sl.store_id
WHERE ST_DWithin(
sl.location::geography,
ST_MakePoint(-79.3832, 43.6532)::geography,
10000 -- 10km in meters
)
ORDER BY distance_km;
```
## MCP Database Access
For direct database queries during development, use the MCP server:
```
// Query the dev database
mcp__devdb__query("SELECT * FROM flyers LIMIT 5")
```
This is useful for:
- Verifying data during debugging
- Checking schema state
- Testing queries before implementing
## Related Documentation
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
- [CODER-GUIDE.md](./CODER-GUIDE.md) - Working with the coder subagent
- [../adr/0034-repository-pattern-standards.md](../adr/0034-repository-pattern-standards.md) - Repository patterns
- [../adr/0002-standardized-transaction-management.md](../adr/0002-standardized-transaction-management.md) - Transaction management
- [../adr/0019-data-backup-and-recovery-strategy.md](../adr/0019-data-backup-and-recovery-strategy.md) - Backup strategy
- [../adr/0050-postgresql-function-observability.md](../adr/0050-postgresql-function-observability.md) - Database observability
- [../BARE-METAL-SETUP.md](../BARE-METAL-SETUP.md) - Production database setup

View File

@@ -0,0 +1,475 @@
# DevOps Subagent Guide
This guide covers DevOps-related subagents for deployment, infrastructure, and operations:
- **devops**: Containers, services, CI/CD pipelines, deployments
- **infra-architect**: Resource optimization, capacity planning
- **bg-worker**: Background jobs, PM2 workers, BullMQ queues
## The devops Subagent
### When to Use
Use the **devops** subagent when you need to:
- Debug container issues in development
- Modify CI/CD pipelines
- Configure PM2 for production
- Update deployment workflows
- Troubleshoot service startup issues
- Configure NGINX or reverse proxy
- Set up SSL/TLS certificates
### What devops Knows
The devops subagent understands:
- Podman/Docker container management
- Dev container configuration (`.devcontainer/`)
- Compose files (`compose.dev.yml`)
- PM2 ecosystem configuration
- Gitea Actions CI/CD workflows
- NGINX configuration
- Systemd service management
### Development Environment
**Container Architecture:**
```
┌─────────────────────────────────────────────────────────────┐
│ Development Environment │
├─────────────────────────────────────────────────────────────┤
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
│ │ app │ │ postgres │ │ redis │ │
│ │ (Node.js) │───►│ (PostGIS) │ │ (Cache) │ │
│ │ │───►│ │ │ │ │
│ └─────────────┘ └─────────────┘ └─────────────┘ │
│ :3000/:3001 :5432 :6379 │
└─────────────────────────────────────────────────────────────┘
```
**Container Services:**
| Service | Image | Purpose | Port |
| ---------- | ----------------------- | ---------------------- | ---------- |
| `app` | Custom (Dockerfile.dev) | Node.js application | 3000, 3001 |
| `postgres` | postgis/postgis:15-3.4 | Database with PostGIS | 5432 |
| `redis` | redis:alpine | Caching and job queues | 6379 |
### Example Requests
**Container debugging:**
```
"Use devops to debug why the dev container fails to start.
The postgres service shows as unhealthy and the app can't connect."
```
**CI/CD pipeline update:**
```
"Use devops to add a step to the deploy-to-test.yml workflow
that runs database migrations before restarting the app."
```
**PM2 configuration:**
```
"Use devops to update the PM2 ecosystem config to use cluster
mode with 4 instances instead of max for the API server."
```
### Container Commands Reference
```bash
# Start development environment
podman-compose -f compose.dev.yml up -d
# View container logs
podman-compose -f compose.dev.yml logs -f app
# Restart specific service
podman-compose -f compose.dev.yml restart app
# Rebuild container (after Dockerfile changes)
podman-compose -f compose.dev.yml build app
# Reset everything
podman-compose -f compose.dev.yml down -v
podman-compose -f compose.dev.yml up -d --build
# Enter container shell
podman exec -it flyer-crawler-dev bash
# Run tests in container (from Windows)
podman exec -it flyer-crawler-dev npm run test:unit
```
### Git Bash Path Conversion (Windows)
When running commands from Git Bash on Windows, paths may be incorrectly converted:
| Solution | Example |
| -------------------------- | -------------------------------------------------------- |
| `sh -c` with single quotes | `podman exec container sh -c '/usr/local/bin/script.sh'` |
| Double slashes | `podman exec container //usr//local//bin//script.sh` |
| MSYS_NO_PATHCONV=1 | `MSYS_NO_PATHCONV=1 podman exec ...` |
### PM2 Production Configuration
**ecosystem.config.cjs Structure:**
```javascript
module.exports = {
apps: [
{
name: 'flyer-crawler-api',
script: './node_modules/.bin/tsx',
args: 'server.ts',
instances: 'max', // Use all CPU cores
exec_mode: 'cluster', // Enable cluster mode
max_memory_restart: '500M',
kill_timeout: 5000, // Graceful shutdown
env_production: {
NODE_ENV: 'production',
cwd: '/var/www/flyer-crawler.projectium.com',
},
},
{
name: 'flyer-crawler-worker',
script: './node_modules/.bin/tsx',
args: 'src/services/worker.ts',
instances: 1, // Single instance for workers
max_memory_restart: '1G',
kill_timeout: 10000, // Workers need more time
},
],
};
```
**PM2 Commands:**
```bash
# Start/reload with environment
pm2 startOrReload ecosystem.config.cjs --env production --update-env
# Save process list
pm2 save
# View logs
pm2 logs flyer-crawler-api --lines 50
# Monitor processes
pm2 monit
# Describe process
pm2 describe flyer-crawler-api
```
### CI/CD Workflow Files
| File | Purpose |
| ------------------------------------- | --------------------------- |
| `.gitea/workflows/deploy-to-prod.yml` | Production deployment |
| `.gitea/workflows/deploy-to-test.yml` | Test environment deployment |
**Deployment Flow:**
1. Push to `main` branch
2. Gitea Actions triggered
3. SSH to production server
4. Pull latest code
5. Install dependencies
6. Run build
7. Run migrations
8. Restart PM2 processes
### Directory Structure (Production)
```
/var/www/
├── flyer-crawler.projectium.com/ # Production
│ ├── server.ts
│ ├── ecosystem.config.cjs
│ ├── package.json
│ ├── flyer-images/
│ │ ├── icons/
│ │ └── archive/
│ └── logs/
│ └── app.log
└── flyer-crawler-test.projectium.com/ # Test environment
└── ... (same structure)
```
## The infra-architect Subagent
### When to Use
Use the **infra-architect** subagent when you need to:
- Analyze resource usage and optimize
- Plan for scaling
- Reduce infrastructure costs
- Configure memory limits
- Analyze disk usage
- Plan capacity for growth
### What infra-architect Knows
The infra-architect subagent understands:
- Node.js memory management
- PostgreSQL resource tuning
- Redis memory configuration
- Container resource limits
- PM2 process monitoring
- Disk and storage management
### Example Requests
**Memory optimization:**
```
"Use infra-architect to analyze memory usage of the worker
processes. They're frequently hitting the 1GB limit and restarting."
```
**Capacity planning:**
```
"Use infra-architect to estimate resource requirements for
handling 10x current traffic. Include database, Redis, and
application server recommendations."
```
**Cost optimization:**
```
"Use infra-architect to identify opportunities to reduce
infrastructure costs without impacting performance."
```
### Resource Limits Reference
| Process | Memory Limit | Notes |
| ---------------- | ------------ | --------------------- |
| API Server | 500MB | Per cluster instance |
| Worker | 1GB | Single instance |
| Analytics Worker | 1GB | Single instance |
| PostgreSQL | System RAM | Tune `shared_buffers` |
| Redis | 256MB | `maxmemory` setting |
## The bg-worker Subagent
### When to Use
Use the **bg-worker** subagent when you need to:
- Debug BullMQ queue issues
- Add new background job types
- Configure job retry logic
- Analyze job processing failures
- Optimize worker performance
- Handle job timeouts
### What bg-worker Knows
The bg-worker subagent understands:
- BullMQ queue patterns
- PM2 worker configuration
- Job retry and backoff strategies
- Queue monitoring and debugging
- Redis connection for queues
- Worker health checks (ADR-053)
### Queue Architecture
```
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
│ API Server │───►│ Redis (BullMQ) │◄───│ Worker │
│ │ │ │ │ │
│ queue.add() │ │ flyerQueue │ │ process jobs │
│ │ │ cleanupQueue │ │ │
└─────────────────┘ │ analyticsQueue │ └─────────────────┘
└─────────────────┘
```
### Example Requests
**Debugging stuck jobs:**
```
"Use bg-worker to debug why jobs are stuck in the flyer processing
queue. Check for failed jobs, worker status, and Redis connectivity."
```
**Adding retry logic:**
```
"Use bg-worker to add exponential backoff retry logic to the
AI extraction job. It should retry up to 3 times with increasing
delays for rate limit errors."
```
**Queue monitoring:**
```
"Use bg-worker to add health check endpoints for monitoring
queue depth and worker status."
```
### Queue Configuration
```typescript
// src/services/queues.server.ts
export const flyerQueue = new Queue('flyer-processing', {
connection: redisConnection,
defaultJobOptions: {
attempts: 3,
backoff: {
type: 'exponential',
delay: 1000,
},
removeOnComplete: { count: 100 },
removeOnFail: { count: 1000 },
},
});
```
### Worker Configuration
```typescript
// src/services/workers.server.ts
export const flyerWorker = new Worker(
'flyer-processing',
async (job) => {
// Process job
},
{
connection: redisConnection,
concurrency: 5,
limiter: {
max: 10,
duration: 1000,
},
},
);
```
### Monitoring Queues
```bash
# Check queue status via Redis
redis-cli -a $REDIS_PASSWORD
> KEYS bull:*
> LLEN bull:flyer-processing:wait
> ZRANGE bull:flyer-processing:failed 0 -1
```
## Service Management Commands
### PM2 Commands
```bash
# Start/reload
pm2 startOrReload ecosystem.config.cjs --env production --update-env && pm2 save
# View status
pm2 list
pm2 status
# View logs
pm2 logs
pm2 logs flyer-crawler-api --lines 100
# Restart specific process
pm2 restart flyer-crawler-api
pm2 restart flyer-crawler-worker
# Stop all
pm2 stop all
# Delete all
pm2 delete all
```
### Systemd Services (Production)
| Service | Command |
| ---------- | ---------------------- | ---- | ------------------------- |
| PostgreSQL | `sudo systemctl {start | stop | status} postgresql` |
| Redis | `sudo systemctl {start | stop | status} redis-server` |
| NGINX | `sudo systemctl {start | stop | status} nginx` |
| Bugsink | `sudo systemctl {start | stop | status} gunicorn-bugsink` |
| Logstash | `sudo systemctl {start | stop | status} logstash` |
### Health Checks
```bash
# API health check
curl http://localhost:3001/api/health
# PM2 health
pm2 list
# PostgreSQL health
pg_isready -h localhost -p 5432
# Redis health
redis-cli -a $REDIS_PASSWORD ping
```
## Troubleshooting Guide
### Container Won't Start
1. Check container logs: `podman-compose logs app`
2. Verify services are healthy: `podman-compose ps`
3. Check environment variables in `compose.dev.yml`
4. Try rebuilding: `podman-compose build --no-cache app`
### Tests Fail in Container but Pass Locally
Tests must run in the Linux container environment:
```bash
# Wrong (Windows)
npm test
# Correct (in container)
podman exec -it flyer-crawler-dev npm test
```
### PM2 Process Keeps Restarting
1. Check logs: `pm2 logs <process-name>`
2. Check memory usage: `pm2 monit`
3. Verify environment variables: `pm2 env <process-id>`
4. Check for unhandled errors in application code
### Database Connection Refused
1. Verify PostgreSQL is running
2. Check connection string in environment
3. Verify database user has permissions
4. Check `pg_hba.conf` for allowed connections
### Redis Connection Issues
1. Verify Redis is running: `redis-cli ping`
2. Check password in environment variables
3. Verify Redis is listening on expected port
4. Check `maxmemory` setting if queue operations fail
## Related Documentation
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
- [../BARE-METAL-SETUP.md](../BARE-METAL-SETUP.md) - Production setup guide
- [../adr/0014-containerization-and-deployment-strategy.md](../adr/0014-containerization-and-deployment-strategy.md) - Containerization ADR
- [../adr/0006-background-job-processing-and-task-queues.md](../adr/0006-background-job-processing-and-task-queues.md) - Background jobs ADR
- [../adr/0017-ci-cd-and-branching-strategy.md](../adr/0017-ci-cd-and-branching-strategy.md) - CI/CD strategy
- [../adr/0053-worker-health-checks.md](../adr/0053-worker-health-checks.md) - Worker health checks

View File

@@ -0,0 +1,442 @@
# Documentation Subagent Guide
This guide covers documentation-focused subagents:
- **documenter**: User docs, API specs, feature documentation
- **describer-for-ai**: Technical docs for AI, ADRs, system overviews
- **planner**: Feature breakdown, roadmaps, scope management
- **product-owner**: Requirements, user stories, backlog prioritization
## The documenter Subagent
### When to Use
Use the **documenter** subagent when you need to:
- Write user-facing documentation
- Create API endpoint documentation
- Document feature usage guides
- Write setup or installation guides
- Create troubleshooting guides
### What documenter Knows
The documenter subagent understands:
- Markdown formatting and best practices
- API documentation standards
- User documentation patterns
- Project-specific terminology
- Existing documentation structure
### Example Requests
**API Documentation:**
```
"Use documenter to create API documentation for the shopping
list endpoints. Include request/response schemas, authentication
requirements, and example curl commands."
```
**Feature Guide:**
```
"Use documenter to write a user guide for the price watchlist
feature. Explain how to add items, set price alerts, and view
price history."
```
**Troubleshooting Guide:**
```
"Use documenter to create a troubleshooting guide for common
flyer upload issues, including file format errors, size limits,
and processing failures."
```
### Documentation Standards
#### API Documentation Format
````markdown
### [METHOD] /api/endpoint
**Description**: Brief purpose of the endpoint
**Authentication**: Required (Bearer token)
**Request**:
- Headers: `Content-Type: application/json`, `Authorization: Bearer {token}`
- Body:
```json
{
"field": "string (required) - Description",
"optional_field": "number (optional) - Description"
}
```
````
**Response**:
- Success (200):
```json
{
"success": true,
"data": { ... }
}
```
- Error (400):
```json
{
"success": false,
"error": {
"code": "VALIDATION_ERROR",
"message": "Description of error"
}
}
```
**Example**:
```bash
curl -X POST https://api.example.com/api/endpoint \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: application/json" \
-d '{"field": "value"}'
```
````
## The describer-for-ai Subagent
### When to Use
Use the **describer-for-ai** subagent when you need to:
- Write Architecture Decision Records (ADRs)
- Create technical specifications for AI consumption
- Document system architecture for context
- Write CLAUDE.md updates
- Create technical overviews
### What describer-for-ai Knows
The describer-for-ai subagent understands:
- ADR format and conventions
- Technical documentation for AI assistants
- System architecture patterns
- Project conventions and patterns
- How to provide context efficiently for AI
### ADR Format
```markdown
# ADR-NNN: Title of Decision
**Date**: YYYY-MM-DD
**Status**: Proposed | Accepted | Implemented | Superseded
## Context
Describe the problem space and constraints that led to this decision.
## Decision
The chosen solution and its rationale.
## Consequences
### Positive
- Benefits of this decision
### Negative
- Trade-offs or limitations
### Neutral
- Other notable effects
## Implementation Details
Technical details, code examples, configuration.
## Key Files
- `path/to/file.ts` - Description
- `path/to/other.ts` - Description
## Related ADRs
- [ADR-XXX](./XXXX-title.md) - Related decision
````
### Example Requests
**Creating an ADR:**
```
"Use describer-for-ai to create an ADR for adding websocket
support for real-time price alerts. Include the technical
approach, alternatives considered, and implementation details."
```
**CLAUDE.md Update:**
```
"Use describer-for-ai to update CLAUDE.md with the new
authentication flow and any new patterns developers should
be aware of."
```
**Technical Overview:**
```
"Use describer-for-ai to create a technical overview of the
caching layer for future AI context, including how Redis is
used, cache invalidation patterns, and key prefixes."
```
## The planner Subagent
### When to Use
Use the **planner** subagent when you need to:
- Break down a feature into tasks
- Create implementation roadmaps
- Scope work for sprints
- Identify dependencies
- Estimate effort
### What planner Knows
The planner subagent understands:
- Project architecture and conventions
- Existing codebase structure
- Common implementation patterns
- Task estimation heuristics
- Dependency identification
### Example Requests
**Feature Breakdown:**
```
"Use planner to break down the 'store comparison' feature
into implementable tasks. Include frontend, backend, and
database work. Identify dependencies between tasks."
```
**Roadmap Planning:**
```
"Use planner to create a roadmap for the Q2 features:
recipe integration, mobile app preparation, and store
notifications. Identify what can be parallelized."
```
**Scope Assessment:**
```
"Use planner to assess the scope of adding multi-language
support. What systems would need to change? What's the
minimum viable implementation?"
```
### Planning Output Format
```markdown
# Feature: [Feature Name]
## Overview
Brief description of the feature and its value.
## Tasks
### Phase 1: Foundation
1. **[Task Name]** (S/M/L)
- Description
- Files: `path/to/file.ts`
- Dependencies: None
- Acceptance: What "done" looks like
2. **[Task Name]** (S/M/L)
- Description
- Files: `path/to/file.ts`
- Dependencies: Task 1
- Acceptance: What "done" looks like
### Phase 2: Core Implementation
...
### Phase 3: Polish & Testing
...
## Dependencies
- External: Third-party services, APIs
- Internal: Other features that must be complete first
## Risks
- Risk 1: Mitigation strategy
- Risk 2: Mitigation strategy
## Estimates
- Phase 1: X days
- Phase 2: Y days
- Phase 3: Z days
- Total: X+Y+Z days
```
## The product-owner Subagent
### When to Use
Use the **product-owner** subagent when you need to:
- Write user stories
- Define acceptance criteria
- Prioritize backlog items
- Validate requirements
- Clarify feature scope
### What product-owner Knows
The product-owner subagent understands:
- User story format
- Acceptance criteria patterns
- Feature prioritization frameworks
- User research interpretation
- Business value assessment
### Example Requests
**User Story Writing:**
```
"Use product-owner to write user stories for the meal planning
feature. Consider different user personas: budget shoppers,
health-conscious users, and busy families."
```
**Acceptance Criteria:**
```
"Use product-owner to define acceptance criteria for the price
alert feature. What conditions must be met for this feature
to be considered complete?"
```
**Prioritization:**
```
"Use product-owner to prioritize these feature requests based
on user value and development effort:
1. Dark mode
2. Recipe suggestions based on deals
3. Store location search
4. Price history graphs"
```
### User Story Format
```markdown
## User Story: [Short Title]
**As a** [type of user]
**I want to** [goal/desire]
**So that** [benefit/value]
### Acceptance Criteria
**Given** [context/starting state]
**When** [action taken]
**Then** [expected outcome]
### Additional Notes
- Edge cases to consider
- Related features
- Out of scope items
### Technical Notes
- API endpoints needed
- Database changes
- Third-party integrations
```
## Documentation Organization
The project organizes documentation as follows:
```
docs/
├── adr/ # Architecture Decision Records
│ ├── index.md # ADR index
│ └── NNNN-title.md # Individual ADRs
├── subagents/ # Subagent guides (this directory)
├── plans/ # Implementation plans
├── tests/ # Test documentation
├── TESTING.md # Testing guide
├── BARE-METAL-SETUP.md # Production setup
├── DESIGN_TOKENS.md # Design system tokens
└── ... # Other documentation
```
## Best Practices
### 1. Keep Documentation Current
Documentation should be updated alongside code changes. The `describer-for-ai` subagent can help identify what documentation needs updating after code changes.
### 2. Use Consistent Terminology
Refer to entities and concepts consistently:
- "Flyer" not "Ad" or "Circular"
- "Store" not "Retailer" or "Shop"
- "Deal" not "Offer" or "Sale"
### 3. Include Examples
All documentation should include concrete examples:
- API docs: Include curl commands and JSON payloads
- User guides: Include screenshots or step-by-step instructions
- Technical docs: Include code snippets
### 4. Cross-Reference Related Documentation
Use relative links to connect related documentation:
```markdown
See [Testing Guide](../TESTING.md) for test execution details.
```
### 5. Date and Version Documentation
Include dates on documentation that may become stale:
```markdown
**Last Updated**: 2026-01-21
**Applies to**: v0.12.x
```
## Related Documentation
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
- [../adr/index.md](../adr/index.md) - ADR index
- [../TESTING.md](../TESTING.md) - Testing guide
- [../../CLAUDE.md](../../CLAUDE.md) - AI instructions

View File

@@ -0,0 +1,412 @@
# Frontend Subagent Guide
This guide covers frontend-focused subagents:
- **frontend-specialist**: UI components, Neo-Brutalism, Core Web Vitals, accessibility
- **uiux-designer**: UI/UX decisions, component design, user experience
## The frontend-specialist Subagent
### When to Use
Use the **frontend-specialist** subagent when you need to:
- Build new React components
- Fix CSS/styling issues
- Improve Core Web Vitals performance
- Implement accessibility features
- Debug React rendering issues
- Optimize bundle size
### What frontend-specialist Knows
The frontend-specialist subagent understands:
- React 18+ patterns and hooks
- TanStack Query for server state
- Zustand for client state
- Tailwind CSS with custom design tokens
- Neo-Brutalism design system
- Accessibility standards (WCAG)
- Performance optimization
## Design System: Neo-Brutalism
The project uses a Neo-Brutalism design aesthetic characterized by:
- Bold, black borders
- High contrast colors
- Shadow offsets for depth
- Raw, honest UI elements
- Playful but functional
### Design Tokens
Located in `src/styles/` and documented in `docs/DESIGN_TOKENS.md`:
```css
/* Core colors */
--color-primary: #ff6b35;
--color-secondary: #004e89;
--color-accent: #f7c548;
--color-background: #fffdf7;
--color-text: #1a1a1a;
/* Borders */
--border-width: 3px;
--border-color: #1a1a1a;
/* Shadows (offset style) */
--shadow-sm: 2px 2px 0 0 #1a1a1a;
--shadow-md: 4px 4px 0 0 #1a1a1a;
--shadow-lg: 6px 6px 0 0 #1a1a1a;
```
### Component Patterns
**Brutal Card:**
```tsx
<div className="border-3 border-black bg-white p-4 shadow-[4px_4px_0_0_#1A1A1A] hover:shadow-[6px_6px_0_0_#1A1A1A] hover:translate-x-[-2px] hover:translate-y-[-2px] transition-all">
{children}
</div>
```
**Brutal Button:**
```tsx
<button className="border-3 border-black bg-primary px-4 py-2 font-bold shadow-[4px_4px_0_0_#1A1A1A] hover:shadow-[2px_2px_0_0_#1A1A1A] hover:translate-x-[2px] hover:translate-y-[2px] active:shadow-none active:translate-x-[4px] active:translate-y-[4px] transition-all">
Click Me
</button>
```
## Example Requests
### Building New Components
```
"Use frontend-specialist to create a PriceTag component that
displays the current price and original price (if discounted)
in the Neo-Brutalism style with a 'SALE' badge when applicable."
```
### Performance Optimization
```
"Use frontend-specialist to optimize the deals list page.
It's showing poor Largest Contentful Paint scores and the
initial load feels sluggish."
```
### Accessibility Fix
```
"Use frontend-specialist to audit and fix accessibility issues
on the shopping list page. Screen reader users report that
the checkbox states aren't being announced correctly."
```
### Responsive Design
```
"Use frontend-specialist to make the store search component
work better on mobile. The dropdown menu is getting cut off
on smaller screens."
```
## State Management
### Server State (TanStack Query)
```tsx
// Fetching data with caching
const {
data: deals,
isLoading,
error,
} = useQuery({
queryKey: ['deals', storeId],
queryFn: () => dealsApi.getByStore(storeId),
staleTime: 5 * 60 * 1000, // 5 minutes
});
// Mutations with optimistic updates
const mutation = useMutation({
mutationFn: dealsApi.favorite,
onMutate: async (dealId) => {
await queryClient.cancelQueries(['deals']);
const previous = queryClient.getQueryData(['deals']);
queryClient.setQueryData(['deals'], (old) =>
old.map((d) => (d.id === dealId ? { ...d, isFavorite: true } : d)),
);
return { previous };
},
onError: (err, dealId, context) => {
queryClient.setQueryData(['deals'], context.previous);
},
});
```
### Client State (Zustand)
```tsx
// Simple client-only state
const useUIStore = create((set) => ({
sidebarOpen: false,
toggleSidebar: () => set((s) => ({ sidebarOpen: !s.sidebarOpen })),
}));
```
## The uiux-designer Subagent
### When to Use
Use the **uiux-designer** subagent when you need to:
- Make design decisions for new features
- Improve user flows
- Design component layouts
- Choose appropriate UI patterns
- Plan information architecture
### Example Requests
**Design new feature:**
```
"Use uiux-designer to design the user flow for adding items
to a shopping list from the deals page. Consider both desktop
and mobile experiences."
```
**Improve existing UX:**
```
"Use uiux-designer to improve the flyer upload experience.
Users are confused about which file types are supported and
don't understand the processing status."
```
**Component design:**
```
"Use uiux-designer to design a price comparison component
that shows the same item across multiple stores."
```
## Component Structure
### Feature-Based Organization
```
src/
├── components/ # Shared UI components
│ ├── ui/ # Basic UI primitives
│ │ ├── Button.tsx
│ │ ├── Card.tsx
│ │ └── Input.tsx
│ ├── layout/ # Layout components
│ │ ├── Header.tsx
│ │ └── Sidebar.tsx
│ └── shared/ # Complex shared components
│ └── PriceDisplay.tsx
├── features/ # Feature-specific components
│ ├── deals/
│ │ ├── components/
│ │ ├── hooks/
│ │ └── api/
│ └── shopping-list/
│ ├── components/
│ ├── hooks/
│ └── api/
└── pages/ # Route page components
├── DealsPage.tsx
└── ShoppingListPage.tsx
```
### Component Pattern
```tsx
// src/components/PriceTag.tsx
import { cn } from '@/utils/cn';
interface PriceTagProps {
currentPrice: number;
originalPrice?: number;
currency?: string;
className?: string;
}
export function PriceTag({
currentPrice,
originalPrice,
currency = '$',
className,
}: PriceTagProps) {
const isOnSale = originalPrice && originalPrice > currentPrice;
const discount = isOnSale ? Math.round((1 - currentPrice / originalPrice) * 100) : 0;
return (
<div className={cn('flex items-baseline gap-2', className)}>
<span className="text-2xl font-bold text-primary">
{currency}
{currentPrice.toFixed(2)}
</span>
{isOnSale && (
<>
<span className="text-sm text-gray-500 line-through">
{currency}
{originalPrice.toFixed(2)}
</span>
<span className="border-2 border-black bg-accent px-1 text-xs font-bold">
-{discount}%
</span>
</>
)}
</div>
);
}
```
## Testing React Components
### Component Test Pattern
```tsx
import { describe, it, expect, vi } from 'vitest';
import { renderWithProviders, screen } from '@/tests/utils/renderWithProviders';
import userEvent from '@testing-library/user-event';
import { PriceTag } from './PriceTag';
describe('PriceTag', () => {
it('displays current price', () => {
renderWithProviders(<PriceTag currentPrice={9.99} />);
expect(screen.getByText('$9.99')).toBeInTheDocument();
});
it('shows discount when original price is higher', () => {
renderWithProviders(<PriceTag currentPrice={7.99} originalPrice={9.99} />);
expect(screen.getByText('$7.99')).toBeInTheDocument();
expect(screen.getByText('$9.99')).toBeInTheDocument();
expect(screen.getByText('-20%')).toBeInTheDocument();
});
});
```
### Hook Test Pattern
```tsx
import { renderHook, waitFor } from '@testing-library/react';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { useDeals } from './useDeals';
describe('useDeals', () => {
it('fetches deals for store', async () => {
const queryClient = new QueryClient({
defaultOptions: { queries: { retry: false } },
});
const { result } = renderHook(() => useDeals('store-123'), {
wrapper: ({ children }) => (
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
),
});
await waitFor(() => expect(result.current.isSuccess).toBe(true));
expect(result.current.data).toHaveLength(10);
});
});
```
## Accessibility Guidelines
### ARIA Patterns
```tsx
// Proper button with loading state
<button
aria-busy={isLoading}
aria-label={isLoading ? 'Loading...' : 'Add to cart'}
disabled={isLoading}
>
{isLoading ? <Spinner /> : 'Add to Cart'}
</button>
// Proper form field
<label htmlFor="email">Email Address</label>
<input
id="email"
type="email"
aria-describedby="email-error"
aria-invalid={!!errors.email}
/>
{errors.email && (
<span id="email-error" role="alert">
{errors.email}
</span>
)}
```
### Keyboard Navigation
- All interactive elements must be focusable
- Focus order should be logical
- Focus traps for modals
- Skip links for main content
### Color Contrast
- Normal text: minimum 4.5:1 contrast ratio
- Large text: minimum 3:1 contrast ratio
- Use the Neo-Brutalism palette which is designed for high contrast
## Performance Optimization
### Code Splitting
```tsx
// Lazy load heavy components
const PdfViewer = lazy(() => import('./PdfViewer'));
function FlyerPage() {
return (
<Suspense fallback={<LoadingSpinner />}>
<PdfViewer />
</Suspense>
);
}
```
### Image Optimization
```tsx
// Use appropriate sizes and formats
<img
src={imageUrl}
srcSet={`${imageUrl}?w=400 400w, ${imageUrl}?w=800 800w`}
sizes="(max-width: 600px) 400px, 800px"
loading="lazy"
alt={itemName}
/>
```
### Memoization
```tsx
// Memoize expensive computations
const sortedDeals = useMemo(() => deals.slice().sort((a, b) => a.price - b.price), [deals]);
// Memoize callbacks passed to children
const handleSelect = useCallback((id: string) => {
setSelectedId(id);
}, []);
```
## Related Documentation
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
- [CODER-GUIDE.md](./CODER-GUIDE.md) - For implementing features
- [../DESIGN_TOKENS.md](../DESIGN_TOKENS.md) - Design token reference
- [../adr/0012-frontend-component-library-and-design-system.md](../adr/0012-frontend-component-library-and-design-system.md) - Design system ADR
- [../adr/0005-frontend-state-management-and-server-cache-strategy.md](../adr/0005-frontend-state-management-and-server-cache-strategy.md) - State management ADR
- [../adr/0044-frontend-feature-organization.md](../adr/0044-frontend-feature-organization.md) - Feature organization

217
docs/subagents/OVERVIEW.md Normal file
View File

@@ -0,0 +1,217 @@
# Claude Code Subagent System Overview
This document provides a comprehensive guide to the subagent system used in the Flyer Crawler project. Subagents are specialized AI assistants that focus on specific domains, allowing for more targeted and effective development workflows.
## What Are Subagents?
Subagents are task-specific Claude instances that can be launched using the `Task` tool in Claude Code. Each subagent has specialized knowledge and instructions tailored to a particular domain, such as coding, testing, database work, or DevOps.
**Why Use Subagents?**
- **Focused Expertise**: Each subagent has domain-specific knowledge and instructions
- **Better Context Management**: Subagents can work on isolated tasks without polluting the main conversation
- **Parallel Work**: Multiple subagents can work on independent tasks simultaneously
- **Consistency**: Subagents follow project-specific patterns and conventions automatically
## Available Subagents
The following subagents are available for use in this project:
### Core Development
| Subagent | Purpose | When to Use |
| --------- | --------------------------------------------------------------- | ---------------------------------------------------------------- |
| **plan** | Design implementation plans, identify files, analyze trade-offs | Starting new features, major refactoring, architecture decisions |
| **coder** | Write and modify production Node.js/TypeScript code | Implementing features, fixing bugs, writing new modules |
### Testing and Quality
| Subagent | Purpose | When to Use |
| ----------------- | ----------------------------------------------------------------- | ---------------------------------------------------- |
| **tester** | Adversarial testing: edge cases, race conditions, vulnerabilities | Finding bugs, security testing, stress testing |
| **testwriter** | Create comprehensive tests for features and fixes | Writing unit tests, integration tests, test coverage |
| **code-reviewer** | Review code quality, security, best practices | Code review, PR reviews, architecture review |
### Database and Infrastructure
| Subagent | Purpose | When to Use |
| ------------------- | --------------------------------------------------------- | -------------------------------------------------------------- |
| **db-dev** | Schemas, queries, migrations, optimization, N+1 problems | Database development, query optimization, schema changes |
| **db-admin** | PostgreSQL/Redis admin, security, backups, infrastructure | Database administration, performance tuning, backup strategies |
| **devops** | Containers, services, CI/CD pipelines, deployments | Deployment issues, container configuration, CI/CD pipelines |
| **infra-architect** | Resource optimization: RAM, CPU, disk, storage | Capacity planning, performance optimization, cost reduction |
### Specialized Technical
| Subagent | Purpose | When to Use |
| --------------------------- | ---------------------------------------------------------- | --------------------------------------------------------- |
| **bg-worker** | Background jobs: PM2 workers, BullMQ queues, async tasks | Queue management, worker debugging, job scheduling |
| **ai-usage** | LLM APIs (Gemini, Claude), prompt engineering, AI features | AI integration, prompt optimization, Gemini API issues |
| **security-engineer** | Security audits, vulnerability scanning, OWASP, pentesting | Security reviews, vulnerability assessments, compliance |
| **log-debug** | Production errors, observability, Bugsink/Sentry analysis | Debugging production issues, log analysis, error tracking |
| **integrations-specialist** | Third-party services, webhooks, external APIs | External API integration, webhook implementation |
### Frontend and Design
| Subagent | Purpose | When to Use |
| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------- |
| **frontend-specialist** | UI components, Neo-Brutalism, Core Web Vitals, accessibility | Frontend development, performance optimization, accessibility |
| **uiux-designer** | UI/UX decisions, component design, Neo-Brutalism compliance | Design decisions, user experience improvements |
### Documentation and Planning
| Subagent | Purpose | When to Use |
| -------------------- | ----------------------------------------------------------- | ---------------------------------------------------------- |
| **documenter** | User docs, API specs, feature documentation | Writing documentation, API specs, user guides |
| **describer-for-ai** | Technical docs for AI: ADRs, system overviews, context docs | Writing ADRs, technical specifications, context documents |
| **planner** | Break down features, roadmaps, scope management | Project planning, feature breakdown, roadmap development |
| **product-owner** | Feature requirements, user stories, validation, backlog | Requirements gathering, user story writing, prioritization |
### Support
| Subagent | Purpose | When to Use |
| -------------------------------- | ---------------------------------------- | ---------------------------------------------------- |
| **tools-integration-specialist** | Bugsink, Gitea, OAuth, operational tools | Tool configuration, OAuth setup, operational tooling |
## How to Launch a Subagent
Subagents are launched using the `Task` tool in Claude Code. Simply ask Claude to use a specific subagent for a task:
```
"Use the coder subagent to implement the new store search feature"
```
Or:
```
"Launch the db-dev subagent to optimize the flyer items query"
```
Claude will automatically invoke the appropriate subagent with the relevant context.
## Subagent Selection Guide
### Which Subagent Should I Use?
**For Writing Code:**
- New features or modules: `coder`
- Complex architectural changes: `plan` first, then `coder`
- Database-related code: `db-dev`
- Frontend components: `frontend-specialist`
- Background job code: `bg-worker`
**For Testing:**
- Writing new tests: `testwriter`
- Finding edge cases and bugs: `tester`
- Reviewing test coverage: `code-reviewer`
**For Infrastructure:**
- Container issues: `devops`
- CI/CD pipelines: `devops`
- Database administration: `db-admin`
- Performance optimization: `infra-architect`
**For Debugging:**
- Production errors: `log-debug`
- Database issues: `db-admin` or `db-dev`
- AI/Gemini issues: `ai-usage`
**For Documentation:**
- API documentation: `documenter`
- Architecture decisions: `describer-for-ai`
- Planning and requirements: `planner` or `product-owner`
## Best Practices
### 1. Start with Planning
For complex features, always start with the `plan` subagent to:
- Identify affected files
- Understand architectural implications
- Break down the work into manageable tasks
### 2. Use Specialized Subagents for Specialized Work
Avoid using `coder` for database migrations. Use `db-dev` instead - it understands:
- The project's migration patterns
- Schema synchronization requirements
- PostgreSQL-specific optimizations
### 3. Let Subagents Follow Project Conventions
All subagents are pre-configured with knowledge of project conventions:
- ADR patterns (see [docs/adr/index.md](../adr/index.md))
- Repository pattern standards (ADR-034)
- Service layer architecture (ADR-035)
- Testing standards (ADR-010)
### 4. Combine Subagents for Complex Tasks
Some tasks benefit from multiple subagents:
1. **New API endpoint**: `plan` -> `coder` -> `testwriter` -> `code-reviewer`
2. **Database optimization**: `db-dev` -> `tester` -> `infra-architect`
3. **Security fix**: `security-engineer` -> `coder` -> `testwriter`
### 5. Always Run Tests in the Dev Container
Regardless of which subagent you use, remember:
> **ALL tests MUST be run in the dev container (Linux environment)**
The subagents know this, but as a developer, ensure you verify test results in the correct environment:
```bash
podman exec -it flyer-crawler-dev npm run test:unit
```
## Subagent Communication
Subagents can pass information back to the main conversation and to each other through:
1. **Direct Output**: Results and recommendations returned to the conversation
2. **File Changes**: Code, documentation, and configuration changes
3. **Todo Lists**: Task tracking and progress updates
## Related Documentation
- [CODER-GUIDE.md](./CODER-GUIDE.md) - Working with the coder subagent
- [TESTER-GUIDE.md](./TESTER-GUIDE.md) - Testing strategies and patterns
- [DATABASE-GUIDE.md](./DATABASE-GUIDE.md) - Database development workflows
- [DEVOPS-GUIDE.md](./DEVOPS-GUIDE.md) - DevOps and deployment workflows
- [../adr/index.md](../adr/index.md) - Architecture Decision Records
- [../TESTING.md](../TESTING.md) - Testing guide
## Troubleshooting
### Subagent Not Available
If a subagent fails to launch, it may be due to:
- Incorrect subagent name (check the list above)
- Network or API issues
- Context length limitations
### Subagent Gives Incorrect Advice
All subagents follow the CLAUDE.md instructions. If advice seems wrong:
1. Verify the project context is correct
2. Check if the advice conflicts with an ADR
3. Provide additional context to the subagent
### Subagent Takes Too Long
For complex tasks, subagents may take time. Consider:
- Breaking the task into smaller pieces
- Using the `plan` subagent first to scope the work
- Running simpler queries first to verify understanding

View File

@@ -0,0 +1,439 @@
# Security and Debugging Subagent Guide
This guide covers security and debugging-focused subagents:
- **security-engineer**: Security audits, vulnerability scanning, OWASP, pentesting
- **log-debug**: Production errors, observability, Bugsink/Sentry analysis
- **code-reviewer**: Code quality, security review, best practices
## The security-engineer Subagent
### When to Use
Use the **security-engineer** subagent when you need to:
- Conduct security audits of code or features
- Review authentication/authorization flows
- Identify vulnerabilities (OWASP Top 10)
- Review API security
- Assess data protection measures
- Plan security improvements
### What security-engineer Knows
The security-engineer subagent understands:
- OWASP Top 10 vulnerabilities
- Node.js/Express security best practices
- JWT authentication security
- SQL injection prevention
- XSS and CSRF protection
- Rate limiting strategies (ADR-032)
- API security hardening (ADR-016)
### Example Requests
**Security Audit:**
```
"Use security-engineer to audit the user registration and
login flow for security vulnerabilities. Check for common
issues like credential stuffing, brute force, and session
management problems."
```
**API Security Review:**
```
"Use security-engineer to review the flyer upload endpoint
for security issues. Consider file type validation, size
limits, malicious file handling, and authorization."
```
**Vulnerability Assessment:**
```
"Use security-engineer to assess our exposure to the OWASP
Top 10 vulnerabilities. Identify any gaps in our current
security measures."
```
### Security Checklist
The security-engineer subagent uses this checklist:
#### Authentication & Authorization
- [ ] Password hashing with bcrypt (cost factor >= 10)
- [ ] JWT tokens with appropriate expiration
- [ ] Refresh token rotation
- [ ] Session invalidation on password change
- [ ] Role-based access control (RBAC)
#### Input Validation
- [ ] All user input validated with Zod schemas
- [ ] SQL queries use parameterized statements
- [ ] File uploads validated for type and size
- [ ] Path traversal prevention
#### Data Protection
- [ ] Sensitive data encrypted at rest
- [ ] HTTPS enforced in production
- [ ] No secrets in source code
- [ ] Proper error messages (no stack traces to users)
#### Rate Limiting
- [ ] Login attempts limited
- [ ] API endpoints rate limited
- [ ] File upload rate limited
#### Headers & CORS
- [ ] Security headers set (Helmet.js)
- [ ] CORS configured appropriately
- [ ] Content-Security-Policy defined
### Security Patterns in This Project
**Rate Limiting (ADR-032):**
```typescript
// src/config/rateLimiters.ts
export const loginLimiter = rateLimit({
windowMs: 15 * 60 * 1000, // 15 minutes
max: 5, // 5 attempts per window
message: 'Too many login attempts',
});
```
**Input Validation (ADR-003):**
```typescript
// src/middleware/validation.middleware.ts
router.post(
'/register',
validateRequest(registerSchema),
async (req, res, next) => { ... }
);
```
**Authentication (ADR-048):**
```typescript
// JWT with refresh tokens
const accessToken = jwt.sign(payload, secret, { expiresIn: '15m' });
const refreshToken = jwt.sign({ userId }, refreshSecret, { expiresIn: '7d' });
```
## The log-debug Subagent
### When to Use
Use the **log-debug** subagent when you need to:
- Debug production errors
- Analyze Bugsink/Sentry error reports
- Investigate performance issues
- Trace request flows through logs
- Identify patterns in error occurrences
### What log-debug Knows
The log-debug subagent understands:
- Pino structured logging
- Bugsink/Sentry error tracking
- Log aggregation with Logstash
- PostgreSQL function observability (ADR-050)
- Request tracing patterns
- Error correlation
### MCP Tools for Debugging
The log-debug subagent can use MCP tools to access error tracking:
```
// Check Bugsink for production errors
mcp__bugsink__list_projects()
mcp__bugsink__list_issues({ project_id: 1 })
mcp__bugsink__get_event({ event_id: "..." })
mcp__bugsink__get_stacktrace({ event_id: "..." })
// Check local dev errors
mcp__localerrors__list_issues({ project_id: 1 })
```
### Example Requests
**Production Error Investigation:**
```
"Use log-debug to investigate the spike in 500 errors on the
flyer processing endpoint yesterday. Check Bugsink for error
patterns and identify the root cause."
```
**Performance Analysis:**
```
"Use log-debug to analyze the slow response times on the deals
page. Check logs for database query timing and identify any
bottlenecks."
```
**Error Pattern Analysis:**
```
"Use log-debug to identify patterns in the authentication
failures over the past week. Are they coming from specific
IPs or affecting specific users?"
```
### Log Analysis Patterns
**Structured Log Format (Pino):**
```json
{
"level": 50,
"time": 1704067200000,
"pid": 1234,
"hostname": "server1",
"module": "flyerService",
"requestId": "abc-123",
"userId": "user-456",
"msg": "Flyer processing failed",
"err": {
"type": "AIExtractionError",
"message": "Rate limit exceeded",
"stack": "..."
}
}
```
**Request Tracing:**
```typescript
// Each request gets a unique ID for tracing
app.use((req, res, next) => {
req.requestId = crypto.randomUUID();
req.log = logger.child({ requestId: req.requestId });
next();
});
```
**Error Correlation:**
- Same `requestId` across all logs for a request
- Same `userId` for user-related errors
- Same `flyerId` for flyer processing errors
### Bugsink Error Tracking
**Production Bugsink Projects:**
| Project | ID | Purpose |
| ---------------------------- | --- | --------------- |
| flyer-crawler-backend | 1 | Backend errors |
| flyer-crawler-frontend | 2 | Frontend errors |
| flyer-crawler-backend-test | 3 | Test backend |
| flyer-crawler-frontend-test | 4 | Test frontend |
| flyer-crawler-infrastructure | 5 | Infra errors |
**Accessing Bugsink:**
- Production: https://bugsink.projectium.com
- Dev Container: http://localhost:8000
### Log File Locations
| Environment | Log Path |
| ------------- | --------------------------------------------------------- |
| Production | `/var/www/flyer-crawler.projectium.com/logs/app.log` |
| Test | `/var/www/flyer-crawler-test.projectium.com/logs/app.log` |
| Dev Container | `/app/logs/app.log` |
## The code-reviewer Subagent
### When to Use
Use the **code-reviewer** subagent when you need to:
- Review code quality before merging
- Identify potential issues in implementations
- Check adherence to project patterns
- Review security implications
- Assess test coverage
### What code-reviewer Knows
The code-reviewer subagent understands:
- Project architecture patterns (ADRs)
- Repository pattern standards (ADR-034)
- Service layer architecture (ADR-035)
- Testing standards (ADR-010)
- TypeScript best practices
- Security considerations
### Example Requests
**Code Review:**
```
"Use code-reviewer to review the changes in the shopping list
feature branch. Check for adherence to project patterns,
potential bugs, and security issues."
```
**Architecture Review:**
```
"Use code-reviewer to review the proposed changes to the
caching layer. Does it follow our patterns? Are there
potential issues with cache invalidation?"
```
**Security-Focused Review:**
```
"Use code-reviewer to review the new file upload handling
code with a focus on security. Check for path traversal,
file type validation, and size limits."
```
### Code Review Checklist
The code-reviewer subagent checks:
#### Code Quality
- [ ] Follows TypeScript strict mode
- [ ] No `any` types without justification
- [ ] Proper error handling
- [ ] Meaningful variable names
- [ ] Appropriate comments
#### Architecture
- [ ] Follows layer separation (Routes -> Services -> Repositories)
- [ ] Uses correct file naming conventions
- [ ] Repository methods follow naming patterns
- [ ] Transactions used for multi-operation changes
#### Testing
- [ ] New code has corresponding tests
- [ ] Tests follow project patterns
- [ ] Edge cases covered
- [ ] Mocks used appropriately
#### Security
- [ ] Input validation present
- [ ] Authorization checks in place
- [ ] No secrets in code
- [ ] Error messages don't leak information
#### Performance
- [ ] No obvious N+1 queries
- [ ] Appropriate use of caching
- [ ] Large data sets paginated
- [ ] Expensive operations async/queued
### Review Output Format
```markdown
## Code Review: [Feature/PR Name]
### Summary
Brief overview of the changes reviewed.
### Issues Found
#### Critical
- **[File:Line]** Description of critical issue
- Impact: What could go wrong
- Suggestion: How to fix
#### High Priority
- **[File:Line]** Description
#### Medium Priority
- **[File:Line]** Description
#### Low Priority / Suggestions
- **[File:Line]** Description
### Positive Observations
- Good patterns followed
- Well-tested areas
- Clean implementations
### Recommendations
1. Priority items to address before merge
2. Items for follow-up tickets
```
## Debugging Workflow
### 1. Error Investigation
```
1. Identify the error in Bugsink
mcp__bugsink__list_issues({ project_id: 1, status: "unresolved" })
2. Get error details
mcp__bugsink__get_issue({ issue_id: "..." })
3. Get full stacktrace
mcp__bugsink__get_stacktrace({ event_id: "..." })
4. Check for patterns across events
mcp__bugsink__list_events({ issue_id: "..." })
```
### 2. Log Correlation
```bash
# Find related logs by request ID
grep "requestId\":\"abc-123\"" /var/www/flyer-crawler.projectium.com/logs/app.log
# Find all errors in a time range
jq 'select(.level >= 50 and .time >= 1704067200000)' app.log
```
### 3. Database Query Analysis
```bash
# Check slow query log
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log | grep "duration:"
```
### 4. Root Cause Analysis
- Correlate error timing with deployments
- Check for resource constraints (memory, connections)
- Review recent code changes
- Check external service status
## Related Documentation
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
- [DEVOPS-GUIDE.md](./DEVOPS-GUIDE.md) - Infrastructure debugging
- [../adr/0016-api-security-hardening.md](../adr/0016-api-security-hardening.md) - Security ADR
- [../adr/0032-rate-limiting-strategy.md](../adr/0032-rate-limiting-strategy.md) - Rate limiting
- [../adr/0015-application-performance-monitoring-and-error-tracking.md](../adr/0015-application-performance-monitoring-and-error-tracking.md) - Monitoring ADR
- [../adr/0050-postgresql-function-observability.md](../adr/0050-postgresql-function-observability.md) - Database observability
- [../BARE-METAL-SETUP.md](../BARE-METAL-SETUP.md) - Production setup

View File

@@ -0,0 +1,404 @@
# Tester and Testwriter Subagent Guide
This guide covers two related but distinct subagents for testing in the Flyer Crawler project:
- **tester**: Adversarial testing to find edge cases, race conditions, and vulnerabilities
- **testwriter**: Creating comprehensive test suites for features and fixes
## Understanding the Difference
| Aspect | tester | testwriter |
| --------------- | ------------------------------- | ------------------------------- |
| **Purpose** | Find bugs and weaknesses | Create test coverage |
| **Approach** | Adversarial, exploratory | Systematic, comprehensive |
| **Output** | Bug reports, security findings | Test files, test utilities |
| **When to Use** | Before release, security review | During development, refactoring |
## The tester Subagent
### When to Use
Use the **tester** subagent when you need to:
- Find edge cases that might cause failures
- Identify race conditions in async code
- Test security vulnerabilities
- Stress test APIs or database queries
- Validate error handling paths
- Find memory leaks or performance issues
### What the tester Knows
The tester subagent understands:
- Common vulnerability patterns (SQL injection, XSS, CSRF)
- Race condition scenarios in Node.js
- Edge cases in data validation
- Authentication and authorization bypasses
- BullMQ queue edge cases
- Database transaction isolation issues
### Example Requests
**Finding edge cases:**
```
"Use the tester subagent to find edge cases in the flyer upload
endpoint. Consider file types, sizes, concurrent uploads, and
invalid data scenarios."
```
**Security testing:**
```
"Use the tester subagent to review the authentication flow for
security vulnerabilities, including JWT handling, session management,
and OAuth integration."
```
**Race condition analysis:**
```
"Use the tester subagent to identify potential race conditions in
the shopping list sharing feature where multiple users might modify
the same list simultaneously."
```
### Sample Output from tester
The tester subagent typically produces:
1. **Vulnerability Reports**
- Issue description
- Reproduction steps
- Severity assessment
- Recommended fix
2. **Edge Case Catalog**
- Input combinations to test
- Expected vs actual behavior
- Priority for fixing
3. **Test Scenarios**
- Detailed test cases for the testwriter
- Setup and teardown requirements
- Assertions to verify
## The testwriter Subagent
### When to Use
Use the **testwriter** subagent when you need to:
- Write unit tests for new features
- Add integration tests for API endpoints
- Create end-to-end test scenarios
- Improve test coverage for existing code
- Write regression tests for bug fixes
- Create test utilities and factories
### What the testwriter Knows
The testwriter subagent understands:
- Project testing stack (Vitest, Testing Library, Supertest)
- Mock factory patterns (`src/tests/utils/mockFactories.ts`)
- Test helper utilities (`src/tests/utils/testHelpers.ts`)
- Database cleanup patterns
- Integration test setup with globalSetup
- Known testing issues documented in CLAUDE.md
### Testing Framework Stack
| Tool | Version | Purpose |
| ------------------------- | ------- | ----------------- |
| Vitest | 4.0.15 | Test runner |
| @testing-library/react | 16.3.0 | Component testing |
| @testing-library/jest-dom | 6.9.1 | DOM assertions |
| supertest | 7.1.4 | API testing |
| msw | 2.12.3 | Network mocking |
### Test File Organization
```
src/
├── components/
│ └── *.test.tsx # Component tests (colocated)
├── hooks/
│ └── *.test.ts # Hook tests (colocated)
├── services/
│ └── *.test.ts # Service tests (colocated)
├── routes/
│ └── *.test.ts # Route handler tests (colocated)
└── tests/
├── integration/ # Integration tests
└── e2e/ # End-to-end tests
```
### Example Requests
**Unit tests for a new feature:**
```
"Use the testwriter subagent to create comprehensive unit tests
for the new StoreSearchService in src/services/storeSearchService.ts.
Include edge cases for empty results, partial matches, and pagination."
```
**Integration tests for API:**
```
"Use the testwriter subagent to add integration tests for the
POST /api/flyers endpoint, covering successful uploads, validation
errors, authentication requirements, and file size limits."
```
**Regression test for bug fix:**
```
"Use the testwriter subagent to create a regression test that
verifies the fix for issue #123 where duplicate flyer items were
created when uploading certain PDFs."
```
### Test Patterns the testwriter Uses
#### Unit Test Pattern
```typescript
// src/services/storeSearchService.test.ts
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { createMockStore, resetMockIds } from '@/tests/utils/mockFactories';
describe('StoreSearchService', () => {
beforeEach(() => {
resetMockIds(); // Ensure deterministic IDs
vi.clearAllMocks();
});
describe('searchByName', () => {
it('returns matching stores when query matches', async () => {
const mockStore = createMockStore({ name: 'Test Mart' });
// ... test implementation
});
it('returns empty array when no matches found', async () => {
// ... test implementation
});
it('handles special characters in search query', async () => {
// ... test implementation
});
});
});
```
#### Integration Test Pattern
```typescript
// src/tests/integration/stores.integration.test.ts
import supertest from 'supertest';
import { createAndLoginUser, cleanupDb } from '@/tests/utils/testHelpers';
describe('Stores API', () => {
let request: ReturnType<typeof supertest>;
let authToken: string;
let testUserId: string;
beforeAll(async () => {
const app = (await import('../../../server')).default;
request = supertest(app);
const { token, userId } = await createAndLoginUser(request);
authToken = token;
testUserId = userId;
});
afterAll(async () => {
await cleanupDb({ users: [testUserId] });
});
describe('GET /api/stores', () => {
it('returns list of stores', async () => {
const response = await request.get('/api/stores').set('Authorization', `Bearer ${authToken}`);
expect(response.status).toBe(200);
expect(response.body.data.stores).toBeInstanceOf(Array);
});
});
});
```
#### Component Test Pattern
```typescript
// src/components/StoreCard.test.tsx
import { describe, it, expect, vi } from 'vitest';
import { renderWithProviders, screen } from '@/tests/utils/renderWithProviders';
import { createMockStore } from '@/tests/utils/mockFactories';
import { StoreCard } from './StoreCard';
describe('StoreCard', () => {
it('renders store name and location count', () => {
const store = createMockStore({
name: 'Test Store',
location_count: 5
});
renderWithProviders(<StoreCard store={store} />);
expect(screen.getByText('Test Store')).toBeInTheDocument();
expect(screen.getByText('5 locations')).toBeInTheDocument();
});
it('calls onSelect when clicked', async () => {
const store = createMockStore();
const handleSelect = vi.fn();
renderWithProviders(<StoreCard store={store} onSelect={handleSelect} />);
await userEvent.click(screen.getByText(store.name));
expect(handleSelect).toHaveBeenCalledWith(store);
});
});
```
## Test Execution Environment
### Critical Requirement
> **ALL tests MUST be executed inside the dev container (Linux environment)**
Tests that pass on Windows but fail on Linux are considered **broken tests**.
### Running Tests
```bash
# From Windows host - run in container
podman exec -it flyer-crawler-dev npm run test:unit
podman exec -it flyer-crawler-dev npm run test:integration
# Inside dev container
npm run test:unit
npm run test:integration
# Run specific test file
npm test -- --run src/services/storeService.test.ts
```
### Test Commands Reference
| Command | Description |
| -------------------------- | ------------------------------------- |
| `npm test` | All unit tests |
| `npm run test:unit` | Unit tests only |
| `npm run test:integration` | Integration tests (requires DB/Redis) |
| `npm run test:coverage` | Tests with coverage report |
## Known Testing Issues
The testwriter subagent is aware of these documented issues:
### 1. Vitest globalSetup Context Isolation
Vitest's `globalSetup` runs in a separate Node.js context. Mocks and spies do NOT share instances with test files.
**Impact**: BullMQ worker service mocks don't work in integration tests.
**Solution**: Use `.todo()` for affected tests or create test-only API endpoints.
### 2. Cleanup Queue Timing
The cleanup worker may process jobs before tests can verify them.
**Solution**:
```typescript
const { cleanupQueue } = await import('../../services/queues.server');
await cleanupQueue.drain();
await cleanupQueue.pause();
// ... run test ...
await cleanupQueue.resume();
```
### 3. Cache Stale After Direct SQL
Direct database inserts bypass cache invalidation.
**Solution**:
```typescript
await cacheService.invalidateFlyers();
```
### 4. Unique Filenames Required
File upload tests need unique filenames to avoid collisions.
**Solution**:
```typescript
const filename = `test-${Date.now()}-${Math.round(Math.random() * 1e9)}.jpg`;
```
## Test Coverage Guidelines
### When Writing Tests
1. **Unit Tests** (required for all new code):
- Pure functions and utilities
- React components
- Custom hooks
- Service methods
- Repository methods
2. **Integration Tests** (required for API changes):
- New API endpoints
- Authentication flows
- Middleware behavior
3. **E2E Tests** (for critical paths):
- User registration/login
- Flyer upload workflow
- Admin operations
### Test Isolation
1. Reset mock IDs in `beforeEach()`
2. Use unique test data (timestamps, UUIDs)
3. Clean up after tests with `cleanupDb()`
4. Don't share state between tests
## Combining tester and testwriter
A typical workflow for thorough testing:
1. **Development**: Write code with basic tests using `testwriter`
2. **Edge Cases**: Use `tester` to identify edge cases and vulnerabilities
3. **Coverage**: Use `testwriter` to add tests for identified edge cases
4. **Review**: Use `code-reviewer` to verify test quality
### Example Combined Workflow
```
1. "Use testwriter to create initial tests for the new discount
calculation feature"
2. "Use tester to find edge cases in the discount calculation -
consider rounding errors, negative values, percentage limits,
and currency precision"
3. "Use testwriter to add tests for the edge cases identified:
- Rounding to 2 decimal places
- Negative discount values
- Discounts over 100%
- Very small amounts (under $0.01)"
```
## Related Documentation
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
- [CODER-GUIDE.md](./CODER-GUIDE.md) - Working with the coder subagent
- [../TESTING.md](../TESTING.md) - Testing guide
- [../adr/0010-testing-strategy-and-standards.md](../adr/0010-testing-strategy-and-standards.md) - Testing ADR
- [../adr/0040-testing-economics-and-priorities.md](../adr/0040-testing-economics-and-priorities.md) - Testing priorities

757
docs/tools/BUGSINK-SETUP.md Normal file
View File

@@ -0,0 +1,757 @@
# Bugsink Error Tracking Setup and Usage Guide
This document covers the complete setup and usage of Bugsink for error tracking in the Flyer Crawler application.
## Table of Contents
- [What is Bugsink](#what-is-bugsink)
- [Environments](#environments)
- [Token Creation](#token-creation)
- [MCP Integration](#mcp-integration)
- [Application Integration](#application-integration)
- [Logstash Integration](#logstash-integration)
- [Using Bugsink](#using-bugsink)
- [Common Workflows](#common-workflows)
- [Troubleshooting](#troubleshooting)
---
## What is Bugsink
Bugsink is a lightweight, self-hosted error tracking platform that is fully compatible with the Sentry SDK ecosystem. We use Bugsink instead of Sentry SaaS or self-hosted Sentry for several reasons:
| Aspect | Bugsink | Self-Hosted Sentry |
| ----------------- | -------------------------- | -------------------------------- |
| Resource Usage | Single process, ~256MB RAM | 16GB+ RAM, Kafka, ClickHouse |
| Deployment | Simple pip/binary install | Docker Compose with 20+ services |
| SDK Compatibility | Full Sentry SDK support | Full Sentry SDK support |
| Database | PostgreSQL or SQLite | PostgreSQL + ClickHouse |
| Cost | Free, self-hosted | Free, self-hosted |
| Maintenance | Minimal | Significant |
**Key Benefits:**
1. **Sentry SDK Compatibility**: Uses the same `@sentry/node` and `@sentry/react` SDKs as Sentry
2. **Self-Hosted**: All error data stays on our infrastructure
3. **Lightweight**: Runs alongside the application without significant overhead
4. **MCP Integration**: AI tools (Claude Code) can query errors via the bugsink-mcp server
**Architecture Decision**: See [ADR-015: Application Performance Monitoring and Error Tracking](../adr/0015-application-performance-monitoring-and-error-tracking.md) for the full rationale.
---
## Environments
### Dev Container (Local Development)
| Item | Value |
| ---------------- | ----------------------------------------------------------------- |
| Web UI | `https://localhost:8443` (nginx proxy) |
| Internal URL | `http://localhost:8000` (direct) |
| Credentials | `admin@localhost` / `admin` |
| Backend Project | Project ID 1 - `flyer-crawler-dev-backend` |
| Frontend Project | Project ID 2 - `flyer-crawler-dev-frontend` |
| Backend DSN | `http://<key>@localhost:8000/1` |
| Frontend DSN | `http://<key>@localhost:8000/2` |
| Database | `postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink` |
**Configuration Files:**
| File | Purpose |
| ----------------- | ----------------------------------------------------------------- |
| `compose.dev.yml` | Initial DSNs using `127.0.0.1:8000` (container startup) |
| `.env.local` | **OVERRIDES** compose.dev.yml with `localhost:8000` (app runtime) |
**Note:** `.env.local` takes precedence over `compose.dev.yml` environment variables.
### Production
| Item | Value |
| ---------------- | --------------------------------------- |
| Web UI | `https://bugsink.projectium.com` |
| Credentials | Managed separately (not shared in docs) |
| Backend Project | `flyer-crawler-backend` |
| Frontend Project | `flyer-crawler-frontend` |
| Infra Project | `flyer-crawler-infrastructure` |
**Bugsink Projects:**
| Project Slug | Type | Environment |
| --------------------------------- | -------- | ----------- |
| flyer-crawler-backend | Backend | Production |
| flyer-crawler-backend-test | Backend | Test |
| flyer-crawler-frontend | Frontend | Production |
| flyer-crawler-frontend-test | Frontend | Test |
| flyer-crawler-infrastructure | Infra | Production |
| flyer-crawler-test-infrastructure | Infra | Test |
---
## Token Creation
Bugsink 2.0.11 does **NOT** have a "Settings > API Keys" menu in the UI. API tokens must be created via Django management command.
### Dev Container Token
Run this command from the Windows host (Git Bash or PowerShell):
```bash
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
```
**Output:** A 40-character lowercase hex token (e.g., `a609c2886daa4e1e05f1517074d7779a5fb49056`)
### Production Token
SSH into the production server:
```bash
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
```
**Output:** Same format - 40-character hex token.
### Token Storage
| Environment | Storage Location | Notes |
| ----------- | ----------------------------- | ---------------------- |
| Dev | `.mcp.json` (project-level) | Not committed to git |
| Production | Gitea secrets + settings.json | `BUGSINK_TOKEN` secret |
---
## MCP Integration
The bugsink-mcp server allows Claude Code and other AI tools to query Bugsink for error information.
### Installation
```bash
# Clone the MCP server
cd d:\gitea
git clone https://github.com/j-shelfwood/bugsink-mcp.git
cd bugsink-mcp
npm install
npm run build
```
### Configuration
**IMPORTANT:** Localhost MCP servers must use project-level `.mcp.json` due to a known Claude Code loader issue. See [BUGSINK-MCP-TROUBLESHOOTING.md](../BUGSINK-MCP-TROUBLESHOOTING.md) for details.
#### Production (Global settings.json)
Location: `~/.claude/settings.json` (or `C:\Users\<username>\.claude\settings.json`)
```json
{
"mcpServers": {
"bugsink": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "https://bugsink.projectium.com",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
}
}
}
```
#### Dev Container (Project-level .mcp.json)
Location: Project root `.mcp.json`
```json
{
"mcpServers": {
"localerrors": {
"command": "node",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "http://127.0.0.1:8000",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
}
}
}
```
### Environment Variables
The bugsink-mcp package requires exactly TWO environment variables:
| Variable | Description | Required |
| --------------- | ----------------------- | -------- |
| `BUGSINK_URL` | Bugsink instance URL | Yes |
| `BUGSINK_TOKEN` | API token (40-char hex) | Yes |
**Common Mistakes:**
- Using `BUGSINK_API_TOKEN` (wrong - use `BUGSINK_TOKEN`)
- Including `BUGSINK_ORG_SLUG` (not used by the package)
### Available MCP Tools
| Tool | Purpose |
| ----------------- | ------------------------------------ |
| `test_connection` | Verify MCP server can reach Bugsink |
| `list_projects` | List all projects in the instance |
| `get_project` | Get project details including DSN |
| `list_issues` | List issues for a project |
| `get_issue` | Get detailed issue information |
| `list_events` | List individual error occurrences |
| `get_event` | Get full event details with context |
| `get_stacktrace` | Get pre-rendered Markdown stacktrace |
| `list_releases` | List releases for a project |
| `create_release` | Create a new release |
**Tool Prefixes:**
- Production: `mcp__bugsink__*`
- Dev Container: `mcp__localerrors__*`
### Verifying MCP Connection
After configuration, restart Claude Code and test:
```typescript
// Production
mcp__bugsink__test_connection();
// Expected: "Connection successful: Connected successfully. Found N project(s)."
// Dev Container
mcp__localerrors__test_connection();
// Expected: "Connection successful: Connected successfully. Found N project(s)."
```
---
## Application Integration
### Backend (Express/Node.js)
**File:** `src/services/sentry.server.ts`
The backend uses `@sentry/node` SDK v8+ to capture errors:
```typescript
import * as Sentry from '@sentry/node';
import { config, isSentryConfigured, isProduction, isTest } from '../config/env';
export function initSentry(): void {
if (!isSentryConfigured || isTest) return;
Sentry.init({
dsn: config.sentry.dsn,
environment: config.sentry.environment || config.server.nodeEnv,
debug: config.sentry.debug,
tracesSampleRate: 0, // Performance monitoring disabled
beforeSend(event, hint) {
// Custom filtering logic
return event;
},
});
}
```
**Key Functions:**
| Function | Purpose |
| ----------------------- | -------------------------------------------- |
| `initSentry()` | Initialize SDK at application startup |
| `captureException()` | Manually capture caught errors |
| `captureMessage()` | Log non-exception events |
| `setUser()` | Set user context after authentication |
| `addBreadcrumb()` | Add navigation/action breadcrumbs |
| `getSentryMiddleware()` | Get Express middleware for automatic capture |
**Integration in server.ts:**
```typescript
// At the very top of server.ts, before other imports
import { initSentry, getSentryMiddleware } from './services/sentry.server';
initSentry();
// After Express app creation
const { requestHandler, errorHandler } = getSentryMiddleware();
app.use(requestHandler);
// ... routes ...
// Before final error handler
app.use(errorHandler);
```
### Frontend (React)
**File:** `src/services/sentry.client.ts`
The frontend uses `@sentry/react` SDK:
```typescript
import * as Sentry from '@sentry/react';
import config from '../config';
export function initSentry(): void {
if (!config.sentry.dsn || !config.sentry.enabled) return;
Sentry.init({
dsn: config.sentry.dsn,
environment: config.sentry.environment,
debug: config.sentry.debug,
tracesSampleRate: 0,
integrations: [
Sentry.breadcrumbsIntegration({
console: true,
dom: true,
fetch: true,
history: true,
xhr: true,
}),
],
beforeSend(event) {
// Filter browser extension errors
if (
event.exception?.values?.[0]?.stacktrace?.frames?.some((frame) =>
frame.filename?.includes('extension://'),
)
) {
return null;
}
return event;
},
});
}
```
**Client Configuration (src/config.ts):**
```typescript
const config = {
sentry: {
dsn: import.meta.env.VITE_SENTRY_DSN,
environment: import.meta.env.VITE_SENTRY_ENVIRONMENT || import.meta.env.MODE,
debug: import.meta.env.VITE_SENTRY_DEBUG === 'true',
enabled: import.meta.env.VITE_SENTRY_ENABLED !== 'false',
},
};
```
### Environment Variables
**Backend (src/config/env.ts):**
| Variable | Description | Default |
| -------------------- | ------------------------------ | ---------- |
| `SENTRY_DSN` | Sentry-compatible DSN | (optional) |
| `SENTRY_ENABLED` | Enable/disable error reporting | `true` |
| `SENTRY_ENVIRONMENT` | Environment tag | NODE_ENV |
| `SENTRY_DEBUG` | Enable SDK debug logging | `false` |
**Frontend (Vite):**
| Variable | Description |
| ------------------------- | ------------------------------- |
| `VITE_SENTRY_DSN` | Frontend DSN (separate project) |
| `VITE_SENTRY_ENVIRONMENT` | Environment tag |
| `VITE_SENTRY_DEBUG` | Enable SDK debug logging |
| `VITE_SENTRY_ENABLED` | Enable/disable error reporting |
---
## Logstash Integration
Logstash aggregates logs from multiple sources and forwards error patterns to Bugsink.
**Note:** See [ADR-015](../adr/0015-application-performance-monitoring-and-error-tracking.md) for the full architecture.
### Log Sources
| Source | Log Path | Error Detection |
| ---------- | ---------------------- | ------------------------- |
| Pino (app) | `/app/logs/*.log` | level >= 50 (error/fatal) |
| Redis | `/var/log/redis/*.log` | WARNING/ERROR log levels |
| PostgreSQL | (future) | ERROR/FATAL log levels |
### Pipeline Configuration
**Location:** `/etc/logstash/conf.d/bugsink.conf`
```conf
# === INPUTS ===
input {
file {
path => "/app/logs/*.log"
codec => json
type => "pino"
tags => ["app"]
}
file {
path => "/var/log/redis/*.log"
type => "redis"
tags => ["redis"]
}
}
# === FILTERS ===
filter {
if [type] == "pino" and [level] >= 50 {
mutate { add_tag => ["error"] }
}
if [type] == "redis" {
grok {
match => { "message" => "%{POSINT:pid}:%{WORD:role} %{MONTHDAY} %{MONTH} %{TIME} %{WORD:loglevel} %{GREEDYDATA:redis_message}" }
}
if [loglevel] in ["WARNING", "ERROR"] {
mutate { add_tag => ["error"] }
}
}
}
# === OUTPUT ===
output {
if "error" in [tags] {
http {
url => "http://localhost:8000/api/store/"
http_method => "post"
format => "json"
}
}
}
```
### Benefits
1. **Secondary Capture Path**: Catches errors before SDK initialization
2. **Log-Based Errors**: Captures errors that don't throw exceptions
3. **Infrastructure Monitoring**: Redis connection issues, slow commands
4. **Historical Analysis**: Process existing log files
---
## Using Bugsink
### Accessing the Web UI
**Dev Container:**
1. Open `https://localhost:8443` in your browser
2. Accept the self-signed certificate warning
3. Login with `admin@localhost` / `admin`
**Production:**
1. Open `https://bugsink.projectium.com`
2. Login with your credentials
### Projects and Teams
Bugsink organizes errors into projects:
| Concept | Description |
| ------- | ---------------------------------------------- |
| Team | Group of projects (e.g., "Flyer Crawler") |
| Project | Single application/service |
| DSN | Data Source Name - unique key for each project |
To view projects:
1. Click the project dropdown in the top navigation
2. Or use MCP: `mcp__bugsink__list_projects()`
### Viewing Issues
**Issues** represent grouped error occurrences. Multiple identical errors are deduplicated into a single issue.
**Issue List View:**
- Navigate to a project
- Issues are sorted by last occurrence
- Each issue shows: title, count, first/last seen
**Issue Detail View:**
- Click an issue to see full details
- View aggregated statistics
- See list of individual events
- Access full stacktrace
### Viewing Events
**Events** are individual error occurrences.
**Event Information:**
- Full stacktrace
- Request context (URL, method, headers)
- User context (if set)
- Breadcrumbs (actions leading to error)
- Tags and extra data
**Via MCP:**
```typescript
// List events for an issue
mcp__bugsink__list_events({ issue_id: 'uuid-here' });
// Get full event details
mcp__bugsink__get_event({ event_id: 'uuid-here' });
// Get readable stacktrace
mcp__bugsink__get_stacktrace({ event_id: 'uuid-here' });
```
### Stacktraces and Context
Stacktraces show the call stack at the time of error:
**Via Web UI:**
- Open an event
- Expand the "Exception" section
- Click frames to see source code context
**Via MCP:**
- `get_stacktrace` returns pre-rendered Markdown
- Includes file paths, line numbers, function names
### Filtering and Searching
**Web UI Filters:**
- By status: unresolved, resolved, muted
- By date range
- By release version
- By environment
**MCP Filtering:**
```typescript
// Filter by status
mcp__bugsink__list_issues({
project_id: 1,
status: 'unresolved',
limit: 25,
});
// Sort options
mcp__bugsink__list_issues({
project_id: 1,
sort: 'last_seen', // or "digest_order"
order: 'desc', // or "asc"
});
```
### Release Tracking
Releases help identify which version introduced or fixed issues.
**Creating Releases:**
```typescript
mcp__bugsink__create_release({
project_id: 1,
version: '1.2.3',
});
```
**Viewing Releases:**
```typescript
mcp__bugsink__list_releases({ project_id: 1 });
```
---
## Common Workflows
### Investigating Production Errors
1. **Check for new errors** (via MCP):
```typescript
mcp__bugsink__list_issues({
project_id: 1,
status: 'unresolved',
sort: 'last_seen',
limit: 10,
});
```
2. **Get issue details**:
```typescript
mcp__bugsink__get_issue({ issue_id: 'uuid' });
```
3. **View stacktrace**:
```typescript
mcp__bugsink__list_events({ issue_id: 'uuid', limit: 1 });
mcp__bugsink__get_stacktrace({ event_id: 'event-uuid' });
```
4. **Examine the code**: Use the file path and line numbers from the stacktrace to locate the issue in the codebase.
### Tracking Down Bugs
1. **Identify error patterns**:
- Group similar errors by message or location
- Check occurrence counts and frequency
2. **Examine request context**:
```typescript
mcp__bugsink__get_event({ event_id: 'uuid' });
```
Look for: URL, HTTP method, request body, user info
3. **Review breadcrumbs**: Understand the sequence of actions leading to the error.
4. **Correlate with logs**: Use the request ID from the event to search application logs.
### Monitoring Error Rates
1. **Check issue counts**: Compare event counts over time
2. **Watch for regressions**: Resolved issues that reopen
3. **Track new issues**: Filter by "first seen" date
### Dev Container Debugging
1. **Access local Bugsink**: `https://localhost:8443`
2. **Trigger a test error**:
```bash
curl -X POST http://localhost:3001/api/test/error
```
3. **View in Bugsink**: Check the dev project for the captured error
4. **Query via MCP**:
```typescript
mcp__localerrors__list_issues({ project_id: 1 });
```
---
## Troubleshooting
### MCP Server Not Available
**Symptoms:**
- `mcp__localerrors__*` tools return "No such tool available"
- `mcp__bugsink__*` works but `mcp__localerrors__*` fails
**Solutions:**
1. **Check configuration location**: Localhost servers must use project-level `.mcp.json`, not global settings.json
2. **Verify token variable name**: Use `BUGSINK_TOKEN`, not `BUGSINK_API_TOKEN`
3. **Test manually**:
```bash
cd d:\gitea\bugsink-mcp
set BUGSINK_URL=http://localhost:8000
set BUGSINK_TOKEN=<your-token>
node dist/index.js
```
Expected: `Bugsink MCP server started`
4. **Full restart**: Close VS Code completely, restart
See [BUGSINK-MCP-TROUBLESHOOTING.md](../BUGSINK-MCP-TROUBLESHOOTING.md) for detailed troubleshooting.
### Connection Refused to localhost:8000
**Cause:** Dev container Bugsink service not running
**Solutions:**
1. **Check container status**:
```bash
podman exec flyer-crawler-dev systemctl status bugsink
```
2. **Start the service**:
```bash
podman exec flyer-crawler-dev systemctl start bugsink
```
3. **Check logs**:
```bash
podman exec flyer-crawler-dev journalctl -u bugsink -n 50
```
### Errors Not Appearing in Bugsink
**Backend:**
1. **Check DSN**: Verify `SENTRY_DSN` environment variable is set
2. **Check enabled flag**: `SENTRY_ENABLED` should be `true`
3. **Check test environment**: Sentry is disabled in `NODE_ENV=test`
**Frontend:**
1. **Check Vite env**: `VITE_SENTRY_DSN` must be set
2. **Verify initialization**: Check browser console for Sentry init message
3. **Check filtering**: `beforeSend` may be filtering the error
### HTTPS Certificate Warnings
**Dev Container:** Self-signed certificates are expected. Accept the warning.
**Production:** Should use valid certificates. If warnings appear, check certificate expiration.
### Token Invalid or Expired
**Symptoms:** MCP returns authentication errors
**Solutions:**
1. **Regenerate token**: Use Django management command (see [Token Creation](#token-creation))
2. **Update configuration**: Put new token in `.mcp.json` or `settings.json`
3. **Restart Claude Code**: Required after config changes
### Bugsink Database Issues
**Symptoms:** 500 errors in Bugsink UI, connection refused
**Dev Container:**
```bash
# Check PostgreSQL
podman exec flyer-crawler-dev pg_isready -U bugsink -d bugsink -h postgres
# Check database exists
podman exec flyer-crawler-dev psql -U postgres -h postgres -c "\l" | grep bugsink
```
**Production:**
```bash
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage check"
```
---
## Related Documentation
- [ADR-015: Application Performance Monitoring and Error Tracking](../adr/0015-application-performance-monitoring-and-error-tracking.md)
- [BUGSINK-MCP-TROUBLESHOOTING.md](../BUGSINK-MCP-TROUBLESHOOTING.md)
- [DEV-CONTAINER-BUGSINK.md](../DEV-CONTAINER-BUGSINK.md)
- [BUGSINK-SYNC.md](../BUGSINK-SYNC.md) - Bugsink to Gitea issue synchronization
- [bugsink-mcp Repository](https://github.com/j-shelfwood/bugsink-mcp)
- [Bugsink Documentation](https://www.bugsink.com/docs/)
- [@sentry/node Documentation](https://docs.sentry.io/platforms/javascript/guides/node/)
- [@sentry/react Documentation](https://docs.sentry.io/platforms/javascript/guides/react/)

View File

@@ -0,0 +1,892 @@
# MCP Configuration Guide
This document provides comprehensive guidance for configuring Model Context Protocol (MCP) servers with Claude Code for the Flyer Crawler project.
## Table of Contents
1. [What is MCP](#what-is-mcp)
2. [Server Overview](#server-overview)
3. [Configuration Locations](#configuration-locations)
4. [Global Settings Configuration](#global-settings-configuration)
5. [Project-Level Configuration](#project-level-configuration)
6. [Server Setup Instructions](#server-setup-instructions)
7. [Bugsink MCP](#bugsink-mcp)
8. [PostgreSQL MCP](#postgresql-mcp)
9. [Gitea MCP](#gitea-mcp)
10. [Other MCP Servers](#other-mcp-servers)
11. [Troubleshooting](#troubleshooting)
12. [Best Practices](#best-practices)
---
## What is MCP
Model Context Protocol (MCP) is a standardized protocol that allows AI assistants like Claude to interact with external tools and services. MCP servers expose capabilities (tools) that Claude can invoke to:
- Query databases
- Manage containers
- Access file systems
- Interact with APIs (Gitea, Bugsink, etc.)
- Store and retrieve knowledge graph data
- Inspect caches and key-value stores
**Why We Use MCP:**
| Benefit | Description |
| ------------------ | ------------------------------------------------------------------------ |
| Direct Integration | Claude can directly query databases, inspect containers, and access APIs |
| Context Awareness | Tools provide real-time information without manual copy-paste |
| Automation | Complex workflows can be executed through tool chains |
| Consistency | Standardized interface across different services |
---
## Server Overview
The Flyer Crawler project uses the following MCP servers:
| Server | Tool Prefix | Purpose | Config Location |
| ------------------ | -------------------------- | -------------------------------------------------- | --------------- |
| `gitea-projectium` | `mcp__gitea-projectium__*` | Gitea API at gitea.projectium.com | Global |
| `gitea-torbonium` | `mcp__gitea-torbonium__*` | Gitea API at gitea.torbonium.com | Global |
| `podman` | `mcp__podman__*` | Container management | Global |
| `filesystem` | `mcp__filesystem__*` | File system access | Global |
| `memory` | `mcp__memory__*` | Knowledge graph persistence | Global |
| `redis` | `mcp__redis__*` | Redis cache inspection | Global |
| `bugsink` | `mcp__bugsink__*` | Production error tracking (bugsink.projectium.com) | Global |
| `localerrors` | `mcp__localerrors__*` | Dev container error tracking (localhost:8000) | Project |
| `devdb` | `mcp__devdb__*` | Development PostgreSQL database | Project |
---
## Configuration Locations
Claude Code uses **two separate configuration systems** for MCP servers:
### Global Configuration
**Location (Windows):**
```text
C:\Users\<username>\.claude\settings.json
```
**Used For:**
- Production services (HTTPS endpoints)
- Servers shared across all projects
- Container management (Podman)
- Knowledge graph (Memory)
### Project-Level Configuration
**Location:**
```text
<project-root>/.mcp.json
```
**Used For:**
- Localhost services (HTTP endpoints)
- Development databases
- Project-specific tools
### When to Use Each
| Scenario | Configuration |
| --------------------------------- | ---------------------- |
| Production APIs (HTTPS) | Global `settings.json` |
| Shared tools (memory, filesystem) | Global `settings.json` |
| Localhost services (HTTP) | Project `.mcp.json` |
| Development databases | Project `.mcp.json` |
| Per-project customization | Project `.mcp.json` |
**Important:** Localhost MCP servers work more reliably in project-level `.mcp.json` than in global `settings.json`. See [Troubleshooting](#localhost-servers-not-loading) for details.
---
## Global Settings Configuration
### File Format
```json
{
"mcpServers": {
"server-name": {
"command": "path/to/executable",
"args": ["arg1", "arg2"],
"env": {
"ENV_VAR": "value"
},
"disabled": true
}
}
}
```
**Configuration Options:**
| Field | Required | Description |
| ---------- | -------- | ----------------------------------------- |
| `command` | Yes | Path to executable or command |
| `args` | No | Array of command-line arguments |
| `env` | No | Environment variables for the server |
| `disabled` | No | Set to `true` to disable without removing |
### Example Global Configuration
```json
{
"mcpServers": {
"memory": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-memory"]
},
"filesystem": {
"command": "d:\\nodejs\\node.exe",
"args": [
"c:\\Users\\<user>\\AppData\\Roaming\\npm\\node_modules\\@modelcontextprotocol\\server-filesystem\\dist\\index.js",
"d:\\gitea"
]
},
"podman": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "podman-mcp-server@latest"],
"env": {
"DOCKER_HOST": "npipe:////./pipe/podman-machine-default"
}
},
"redis": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-redis", "redis://localhost:6379"]
},
"bugsink": {
"command": "d:\\nodejs\\node.exe",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "https://bugsink.projectium.com",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
},
"gitea-projectium": {
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
"args": ["run", "-t", "stdio"],
"env": {
"GITEA_HOST": "https://gitea.projectium.com",
"GITEA_ACCESS_TOKEN": "<your-token>"
}
},
"gitea-torbonium": {
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
"args": ["run", "-t", "stdio"],
"env": {
"GITEA_HOST": "https://gitea.torbonium.com",
"GITEA_ACCESS_TOKEN": "<your-token>"
}
}
}
}
```
---
## Project-Level Configuration
### File Location
Create `.mcp.json` in the project root:
```text
d:\gitea\flyer-crawler.projectium.com\flyer-crawler.projectium.com\.mcp.json
```
### File Format
```json
{
"mcpServers": {
"server-name": {
"command": "path/to/executable",
"args": ["arg1", "arg2"],
"env": {
"ENV_VAR": "value"
}
}
}
}
```
### Current Project Configuration
```json
{
"mcpServers": {
"localerrors": {
"command": "d:\\nodejs\\node.exe",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "http://127.0.0.1:8000",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
},
"devdb": {
"command": "D:\\nodejs\\npx.cmd",
"args": [
"-y",
"@modelcontextprotocol/server-postgres",
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
]
}
}
}
```
---
## Server Setup Instructions
### Memory (Knowledge Graph)
**Package:** `@modelcontextprotocol/server-memory`
**Purpose:** Persists knowledge across sessions - project context, credentials, known issues.
**Configuration:**
```json
"memory": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-memory"]
}
```
**Key Tools:**
- `mcp__memory__read_graph` - Read entire knowledge graph
- `mcp__memory__search_nodes` - Search for specific entities
- `mcp__memory__create_entities` - Add new knowledge
### Filesystem
**Package:** `@modelcontextprotocol/server-filesystem`
**Purpose:** Provides file system access to specified directories.
**Configuration:**
```json
"filesystem": {
"command": "d:\\nodejs\\node.exe",
"args": [
"c:\\Users\\<user>\\AppData\\Roaming\\npm\\node_modules\\@modelcontextprotocol\\server-filesystem\\dist\\index.js",
"d:\\gitea"
]
}
```
**Note:** The last argument(s) specify allowed directories.
### Podman/Docker
**Package:** `podman-mcp-server`
**Purpose:** Container management - list, start, stop, inspect containers.
**Configuration:**
```json
"podman": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "podman-mcp-server@latest"],
"env": {
"DOCKER_HOST": "npipe:////./pipe/podman-machine-default"
}
}
```
**Key Tools:**
- `mcp__podman__container_list` - List running containers
- `mcp__podman__container_logs` - View container logs
- `mcp__podman__container_inspect` - Detailed container info
- `mcp__podman__image_list` - List images
### Redis
**Package:** `@modelcontextprotocol/server-redis`
**Purpose:** Inspect Redis cache, set/get values, list keys.
**Configuration:**
```json
"redis": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-redis", "redis://localhost:6379"]
}
```
**Key Tools:**
- `mcp__redis__get` - Get value by key
- `mcp__redis__set` - Set key-value pair
- `mcp__redis__list` - List keys matching pattern
- `mcp__redis__delete` - Delete key(s)
---
## Bugsink MCP
Bugsink is a self-hosted error tracking service. We run two instances:
| Instance | URL | MCP Server | Purpose |
| ----------- | -------------------------------- | ------------- | ---------------------------- |
| Production | `https://bugsink.projectium.com` | `bugsink` | Production error tracking |
| Development | `http://localhost:8000` | `localerrors` | Dev container error tracking |
### Installation
The `bugsink-mcp` package is **NOT published to npm**. Clone and build from source:
```bash
# Clone the repository
git clone https://github.com/j-shelfwood/bugsink-mcp.git d:\gitea\bugsink-mcp
# Install and build
cd d:\gitea\bugsink-mcp
npm install
npm run build
```
**Repository:** https://github.com/j-shelfwood/bugsink-mcp
### Configuration
**Production (Global `settings.json`):**
```json
"bugsink": {
"command": "d:\\nodejs\\node.exe",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "https://bugsink.projectium.com",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
}
```
**Development (Project `.mcp.json`):**
```json
"localerrors": {
"command": "d:\\nodejs\\node.exe",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "http://127.0.0.1:8000",
"BUGSINK_TOKEN": "<40-char-hex-token>"
}
}
```
**Required Environment Variables:**
| Variable | Description |
| --------------- | -------------------------------------------- |
| `BUGSINK_URL` | Full URL to Bugsink instance (with protocol) |
| `BUGSINK_TOKEN` | 40-character hex API token |
**Important:**
- Variable is `BUGSINK_TOKEN`, NOT `BUGSINK_API_TOKEN`
- Do NOT use `npx` - the package is not on npm
- Use `http://127.0.0.1:8000` not `http://localhost:8000` for localhost
### Creating API Tokens
Bugsink 2.0.11 does NOT have a "Settings > API Keys" menu in the UI. Tokens must be created via Django management command.
**For Dev Container (localhost:8000):**
```bash
MSYS_NO_PATHCONV=1 podman exec \
-e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink \
-e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security \
flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && \
DJANGO_SETTINGS_MODULE=bugsink_conf \
PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages \
/opt/bugsink/bin/python -m django create_auth_token'
```
**For Production (via SSH):**
```bash
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
```
Both commands output a 40-character lowercase hex token (e.g., `a609c2886daa4e1e05f1517074d7779a5fb49056`).
### Key Tools
- `mcp__bugsink__test_connection` / `mcp__localerrors__test_connection` - Verify connection
- `mcp__bugsink__list_projects` - List all projects
- `mcp__bugsink__list_issues` - List issues for a project
- `mcp__bugsink__get_issue` - Get issue details
- `mcp__bugsink__get_stacktrace` - Get event stacktrace as Markdown
### Testing Connection
```typescript
// Production
mcp__bugsink__test_connection();
// Expected: "Connection successful: Connected successfully. Found N project(s)."
// Development
mcp__localerrors__test_connection();
// Expected: "Connection successful: Connected successfully. Found N project(s)."
```
---
## PostgreSQL MCP
**Package:** `@modelcontextprotocol/server-postgres`
**Purpose:** Execute SQL queries against the development database.
### Configuration
Add to project-level `.mcp.json`:
```json
"devdb": {
"command": "D:\\nodejs\\npx.cmd",
"args": [
"-y",
"@modelcontextprotocol/server-postgres",
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
]
}
```
### Connection String Format
```text
postgresql://[user]:[password]@[host]:[port]/[database]
```
**Examples:**
```text
# Development (local container)
postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev
# Test database
postgresql://flyer_crawler_test:password@127.0.0.1:5432/flyer_crawler_test
```
### Database Information
| Property | Value |
| --------------------- | ------------------------ |
| Container | `flyer-crawler-postgres` |
| Image | `postgis/postgis:15-3.4` |
| Host (from Windows) | `127.0.0.1` |
| Host (from container) | `postgres` |
| Port | `5432` |
| Database | `flyer_crawler_dev` |
| User | `postgres` |
| Password | `postgres` |
### Usage Examples
```typescript
// List all tables
mcp__devdb__query({ sql: "SELECT tablename FROM pg_tables WHERE schemaname = 'public'" });
// Count records
mcp__devdb__query({ sql: 'SELECT COUNT(*) FROM flyers' });
// Check table structure
mcp__devdb__query({
sql: "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'flyers'",
});
// Find recent records
mcp__devdb__query({
sql: 'SELECT id, name, created_at FROM flyers ORDER BY created_at DESC LIMIT 10',
});
```
### Prerequisites
1. **PostgreSQL container must be running:**
```bash
podman ps | grep flyer-crawler-postgres
```
2. **Port 5432 must be mapped:**
```bash
podman port flyer-crawler-postgres
# Expected: 5432/tcp -> 0.0.0.0:5432
```
3. **Database must exist:**
```bash
podman exec flyer-crawler-postgres psql -U postgres -c "\l" | grep flyer_crawler_dev
```
---
## Gitea MCP
**Binary:** `gitea-mcp` (compiled Go binary)
**Purpose:** Interact with Gitea repositories, issues, pull requests.
### Configuration
```json
"gitea-projectium": {
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
"args": ["run", "-t", "stdio"],
"env": {
"GITEA_HOST": "https://gitea.projectium.com",
"GITEA_ACCESS_TOKEN": "<your-token>"
}
}
```
### Getting Access Token
1. Log in to Gitea web interface
2. Go to **Settings > Applications**
3. Under **Generate New Token**, enter a name
4. Select required scopes (typically `read:user`, `write:repository`, `write:issue`)
5. Click **Generate Token**
6. Copy the token immediately (shown only once)
### Key Tools
- `mcp__gitea-projectium__list_my_repos` - List accessible repositories
- `mcp__gitea-projectium__list_repo_issues` - List issues in a repo
- `mcp__gitea-projectium__get_issue_by_index` - Get issue details
- `mcp__gitea-projectium__create_issue` - Create new issue
- `mcp__gitea-projectium__create_pull_request` - Create PR
- `mcp__gitea-projectium__get_file_content` - Read file from repo
- `mcp__gitea-projectium__list_branches` - List branches
### Example Operations
```typescript
// List repositories
mcp__gitea - projectium__list_my_repos({ page: 1, pageSize: 20 });
// Get issue
mcp__gitea -
projectium__get_issue_by_index({
owner: 'username',
repo: 'repository-name',
index: 42,
});
// Create issue
mcp__gitea -
projectium__create_issue({
owner: 'username',
repo: 'repository-name',
title: 'Bug: Something is broken',
body: '## Description\n\nSteps to reproduce...',
});
```
---
## Other MCP Servers
### Sequential Thinking
**Package:** `@modelcontextprotocol/server-sequential-thinking`
**Purpose:** Structured step-by-step reasoning for complex problems.
```json
"sequential-thinking": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-sequential-thinking"]
}
```
### Playwright (Browser Automation)
**Package:** `@anthropics/mcp-server-playwright`
**Purpose:** Browser automation for testing and scraping.
```json
"playwright": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@anthropics/mcp-server-playwright"]
}
```
### Sentry (Cloud Error Tracking)
**Package:** `@sentry/mcp-server`
**Purpose:** Error tracking for Sentry instances (NOT Bugsink).
```json
"sentry": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@sentry/mcp-server"],
"env": {
"SENTRY_AUTH_TOKEN": "<your-sentry-token>"
}
}
```
**Note:** Bugsink has a different API than Sentry. Use `bugsink-mcp` for Bugsink instances.
---
## Troubleshooting
### Localhost Servers Not Loading
**Symptoms:**
- `mcp__localerrors__*` or `mcp__devdb__*` tools not available
- No error messages in logs
- Server silently skipped during startup
**Root Cause:**
Claude Code's global `settings.json` has issues loading localhost stdio MCP servers on Windows. The exact cause may be related to:
- Multiple servers using the same underlying package
- Localhost URL filtering
- Windows-specific MCP loader bugs
**Solution:**
Use **project-level `.mcp.json`** for all localhost MCP servers. This bypasses the global config loader entirely.
**Working Pattern:**
- Global `settings.json`: Production HTTPS servers
- Project `.mcp.json`: Localhost HTTP servers
### Server Name Collision
**Symptoms:**
- Second server with similar name never starts
- No error logged - server silently filtered out
**Root Cause:**
Claude Code may skip MCP servers when names share prefixes (e.g., `bugsink` and `bugsink-dev`).
**Solution:**
Use completely distinct names:
- `bugsink` for production
- `localerrors` for development (NOT `bugsink-dev` or `devbugsink`)
### Connection Timed Out
**Error:** `Connection timed out after 30000ms`
**Causes:**
- Server takes too long to start
- npx download is slow
- Server crashes during initialization
**Solutions:**
1. Move important servers earlier in config
2. Use pre-installed packages instead of npx:
```json
"command": "d:\\nodejs\\node.exe",
"args": ["path/to/installed/package/dist/index.js"]
```
3. Check server can start manually
### Environment Variable Issues
**Common Mistakes:**
| Wrong | Correct |
| ----------------------- | ----------------------- |
| `BUGSINK_API_TOKEN` | `BUGSINK_TOKEN` |
| `http://localhost:8000` | `http://127.0.0.1:8000` |
**Verification:**
Test server manually with environment variables:
```bash
cd d:\gitea\bugsink-mcp
set BUGSINK_URL=http://127.0.0.1:8000
set BUGSINK_TOKEN=<your-token>
node dist/index.js
```
Expected output:
```
Bugsink MCP server started
Connected to: http://127.0.0.1:8000
```
### PostgreSQL Connection Refused
**Solutions:**
1. Check container is running:
```bash
podman ps | grep flyer-crawler-postgres
```
2. Verify port mapping:
```bash
podman port flyer-crawler-postgres
```
3. Test connection:
```bash
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1"
```
4. Check for port conflicts:
```bash
netstat -an | findstr 5432
```
### Verifying Configuration
**List loaded servers:**
```bash
claude mcp list
```
**Check debug logs (Windows):**
```text
C:\Users\<username>\.claude\debug\*.txt
```
Look for MCP server startup messages. Missing servers indicate configuration problems.
---
## Best Practices
### 1. Keep Configs Organized
- **Global config:** Shared/production servers
- **Project config:** Local development servers
- **Never duplicate** the same server in both
### 2. Order Servers by Importance
Place essential servers first in configuration:
1. `memory` - Knowledge persistence
2. `filesystem` - File access
3. `podman` - Container management
4. Other servers...
### 3. Use Direct Node Execution
For faster startup, avoid npx and use direct node execution:
```json
// Slow (npx downloads on each start)
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "package-name"]
// Fast (pre-installed)
"command": "d:\\nodejs\\node.exe",
"args": ["path/to/installed/dist/index.js"]
```
### 4. Disable Instead of Delete
Use `"disabled": true` to troubleshoot without losing configuration:
```json
"problem-server": {
"command": "...",
"disabled": true
}
```
### 5. Test Manually First
Before adding to config, verify server works:
```bash
cd /path/to/mcp-server
set ENV_VAR=value
node dist/index.js
```
### 6. Store Secrets Securely
- Use the memory MCP to store API tokens for future sessions
- Never commit `.mcp.json` with real tokens (add to `.gitignore`)
- Use environment variables where possible
### 7. Restart After Changes
MCP configuration changes require a full VS Code restart (not just window reload).
---
## Quick Reference
### Available MCP Packages
| Server | Package/Source | npm? |
| ------------------- | -------------------------------------------------- | ---------------------- |
| memory | `@modelcontextprotocol/server-memory` | Yes |
| filesystem | `@modelcontextprotocol/server-filesystem` | Yes |
| redis | `@modelcontextprotocol/server-redis` | Yes |
| postgres | `@modelcontextprotocol/server-postgres` | Yes |
| sequential-thinking | `@modelcontextprotocol/server-sequential-thinking` | Yes |
| podman | `podman-mcp-server` | Yes |
| gitea | `gitea-mcp` (binary) | No |
| bugsink | `j-shelfwood/bugsink-mcp` | No (build from source) |
| sentry | `@sentry/mcp-server` | Yes |
| playwright | `@anthropics/mcp-server-playwright` | Yes |
### Common Tool Prefixes
| Server | Tool Prefix | Example |
| -------------- | ------------------------- | -------------------------------------- |
| Memory | `mcp__memory__` | `mcp__memory__read_graph` |
| Filesystem | `mcp__filesystem__` | `mcp__filesystem__read_file` |
| Podman | `mcp__podman__` | `mcp__podman__container_list` |
| Redis | `mcp__redis__` | `mcp__redis__get` |
| Bugsink (prod) | `mcp__bugsink__` | `mcp__bugsink__list_issues` |
| Bugsink (dev) | `mcp__localerrors__` | `mcp__localerrors__list_issues` |
| PostgreSQL | `mcp__devdb__` | `mcp__devdb__query` |
| Gitea | `mcp__gitea-projectium__` | `mcp__gitea-projectium__list_my_repos` |
---
## Related Documentation
- [CLAUDE.md - MCP Servers Section](../../CLAUDE.md#mcp-servers)
- [DEV-CONTAINER-BUGSINK.md](../DEV-CONTAINER-BUGSINK.md)
- [BUGSINK-SYNC.md](../BUGSINK-SYNC.md)
- [sql/master_schema_rollup.sql](../../sql/master_schema_rollup.sql)
---
_Last updated: January 2026_

View File

@@ -22,6 +22,7 @@ MCP (Model Context Protocol) allows AI assistants to interact with external tool
Access to multiple Gitea instances for repository management, code search, issue tracking, and CI/CD workflows.
#### Gitea Projectium (Primary)
- **Host**: `https://gitea.projectium.com`
- **Purpose**: Main production Gitea server
- **Capabilities**:
@@ -31,11 +32,13 @@ Access to multiple Gitea instances for repository management, code search, issue
- Repository cloning and management
#### Gitea Torbonium
- **Host**: `https://gitea.torbonium.com`
- **Purpose**: Development/testing Gitea instance
- **Capabilities**: Same as Gitea Projectium
#### Gitea LAN
- **Host**: `https://gitea.torbolan.com`
- **Purpose**: Local network Gitea instance
- **Status**: Disabled (requires token configuration)
@@ -43,6 +46,7 @@ Access to multiple Gitea instances for repository management, code search, issue
**Executable Location**: `d:\gitea-mcp\gitea-mcp.exe`
**Configuration Example** (Gemini Code - mcp.json):
```json
{
"servers": {
@@ -59,6 +63,7 @@ Access to multiple Gitea instances for repository management, code search, issue
```
**Configuration Example** (Claude Code - settings.json):
```json
{
"mcpServers": {
@@ -87,10 +92,12 @@ Manages local containers via Podman Desktop (using Docker-compatible API).
- Inspect container status and configuration
**Current Containers** (for this project):
- `flyer-crawler-postgres` - PostgreSQL 15 + PostGIS on port 5432
- `flyer-crawler-redis` - Redis on port 6379
**Configuration** (Gemini Code - mcp.json):
```json
{
"servers": {
@@ -106,6 +113,7 @@ Manages local containers via Podman Desktop (using Docker-compatible API).
```
**Configuration** (Claude Code):
```json
{
"mcpServers": {
@@ -133,6 +141,7 @@ Direct file system access to the project directory.
- Search files
**Configuration** (Gemini Code - mcp.json):
```json
{
"servers": {
@@ -149,6 +158,7 @@ Direct file system access to the project directory.
```
**Configuration** (Claude Code):
```json
{
"mcpServers": {
@@ -175,6 +185,7 @@ Web request capabilities for documentation lookups and API testing.
- Test endpoints
**Configuration** (Gemini Code - mcp.json):
```json
{
"servers": {
@@ -187,6 +198,7 @@ Web request capabilities for documentation lookups and API testing.
```
**Configuration** (Claude Code):
```json
{
"mcpServers": {
@@ -211,6 +223,7 @@ Browser automation and debugging capabilities.
- Network monitoring
**Configuration** (when enabled):
```json
{
"mcpServers": {
@@ -218,9 +231,12 @@ Browser automation and debugging capabilities.
"command": "npx",
"args": [
"chrome-devtools-mcp@latest",
"--headless", "false",
"--isolated", "false",
"--channel", "stable"
"--headless",
"false",
"--isolated",
"false",
"--channel",
"stable"
]
}
}
@@ -240,6 +256,7 @@ Document conversion capabilities.
- Convert other document formats
**Configuration** (when enabled):
```json
{
"mcpServers": {
@@ -254,6 +271,7 @@ Document conversion capabilities.
## Prerequisites
### For Podman MCP
1. **Podman Desktop** installed and running
2. Podman machine initialized and started:
```powershell
@@ -262,6 +280,7 @@ Document conversion capabilities.
```
### For Gitea MCP
1. **Gitea MCP executable** at `d:\gitea-mcp\gitea-mcp.exe`
2. **Gitea Access Tokens** with appropriate permissions:
- `repo` - Full repository access
@@ -269,10 +288,12 @@ Document conversion capabilities.
- `read:organization` - Organization access
### For Chrome DevTools MCP
1. **Chrome browser** installed (stable channel)
2. **Node.js 18+** for npx execution
### For Markitdown MCP
1. **Python 3.8+** installed
2. **uvx** (universal virtualenv executor):
```powershell
@@ -282,39 +303,160 @@ Document conversion capabilities.
## Testing MCP Servers
### Test Podman Connection
```powershell
podman ps
# Should list running containers
```
### Test Gitea API Access
```powershell
curl -H "Authorization: token YOUR_TOKEN" https://gitea.projectium.com/api/v1/user
# Should return your user information
```
### Test Database Container
```powershell
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT version();"
# Should return PostgreSQL version
### Claude Code Extension Auto-Update Issues
#### Problem: Version 2.1.15 Crashes on CPUs Without AVX Support
Claude Code version 2.1.15 introduced a regression that causes crashes on CPUs that do not support AVX (Advanced Vector Extensions) instructions. The error manifests as:
```
Illegal instruction (core dumped)
```
or similar AVX-related illegal instruction errors when the extension tries to start.
**Affected**: CPUs without AVX support (typically older processors or certain VMs)
**Working Version**: 2.1.11 (and earlier)
**Broken Version**: 2.1.15+
#### Solution: Disable Auto-Updates for Claude Code Extension
The VS Code right-click menu option "Disable Auto Update" for extensions may be greyed out and non-functional. Use the settings.json workaround instead.
**Step 1: Open VS Code Settings JSON**
Press `Ctrl+Shift+P` and type "Preferences: Open User Settings (JSON)" or manually edit:
```
C:\Users\<username>\AppData\Roaming\Code\User\settings.json
````
**Step 2: Add the Extension to Ignore List**
Add the following setting to your `settings.json`:
```json
{
"extensions.ignoreAutoUpdate": ["anthropic.claude-code"]
}
````
If you already have other settings, add it within the existing JSON object:
```json
{
"editor.fontSize": 14,
"extensions.ignoreAutoUpdate": ["anthropic.claude-code"],
"other.settings": "value"
}
```
**Step 3: Downgrade to Working Version**
If you're already on 2.1.15, you need to downgrade:
1. Open the Extensions view (`Ctrl+Shift+X`)
2. Find "Claude Code" in the list
3. Click the gear icon next to the extension
4. Select "Install Another Version..."
5. Choose version **2.1.11** from the list
6. Wait for installation to complete
7. Reload VS Code when prompted
**Step 4: Verify Configuration**
To verify the setting is working:
1. Open VS Code Settings JSON and confirm `extensions.ignoreAutoUpdate` includes `anthropic.claude-code`
2. Check the Extensions view - Claude Code should show version 2.1.11
3. VS Code should no longer prompt to update Claude Code automatically
#### Updating Later When Bug is Fixed
Once Anthropic releases a fixed version:
1. **Remove the ignore setting** from `settings.json`:
```json
// Remove or comment out:
// "extensions.ignoreAutoUpdate": ["anthropic.claude-code"]
```
2. **Manually update** the extension:
- Open Extensions view (`Ctrl+Shift+X`)
- Find Claude Code
- Click "Update" or use the gear menu to install a specific version
3. **Or re-enable auto-updates** by removing the extension from the ignore list, then:
- Reload VS Code
- The extension will update automatically
#### Alternative: Pin to Specific Version
If you prefer to pin to a specific version rather than just disabling auto-updates:
```json
{
"extensions.autoUpdate": "onlyEnabledExtensions",
"extensions.ignoreAutoUpdate": ["anthropic.claude-code"]
}
```
This allows other extensions to update automatically while keeping Claude Code locked.
#### Checking Current Extension Version
To verify which version is installed:
1. Open Extensions view (`Ctrl+Shift+X`)
2. Find "Claude Code" by Anthropic
3. The version number appears below the extension name
4. Or click on the extension to see full details including version history
````
## Security Notes
### Token Management
- **Never commit tokens** to version control
- Store tokens in environment variables or secure password managers
- Rotate tokens periodically
- Use minimal required permissions
### Access Tokens in Configuration Files
The configuration files (`mcp.json` and `settings.json`) contain sensitive access tokens. These files should:
- Be added to `.gitignore`
- Have restricted file permissions
- Be backed up securely
- Be updated when tokens are rotated
### Current Security Setup
- `%APPDATA%\Code\User\mcp.json` - Gitea tokens embedded
- `%USERPROFILE%\.claude\settings.json` - Gitea tokens embedded
- Both files are in user-specific directories with appropriate Windows ACLs
@@ -322,10 +464,12 @@ The configuration files (`mcp.json` and `settings.json`) contain sensitive acces
## Troubleshooting
### Podman MCP Not Working
1. Check Podman machine status:
```powershell
podman machine list
```
````
2. Ensure Podman Desktop is running
3. Verify Docker socket is accessible:
```powershell
@@ -333,6 +477,7 @@ The configuration files (`mcp.json` and `settings.json`) contain sensitive acces
```
### Gitea MCP Connection Issues
1. Verify token has correct permissions
2. Check network connectivity to Gitea server:
```powershell
@@ -341,11 +486,13 @@ The configuration files (`mcp.json` and `settings.json`) contain sensitive acces
3. Ensure `gitea-mcp.exe` is not blocked by antivirus/firewall
### VS Code Extension Issues
1. **Reload Window**: Press `Ctrl+Shift+P` → "Developer: Reload Window"
2. **Check Extension Logs**: View → Output → Select extension from dropdown
3. **Verify JSON Syntax**: Ensure both config files have valid JSON
### MCP Server Not Loading
1. Check config file syntax with JSON validator
2. Verify executable paths are correct (use forward slashes or escaped backslashes)
3. Ensure required dependencies are installed (Node.js, Python, etc.)
@@ -356,11 +503,13 @@ The configuration files (`mcp.json` and `settings.json`) contain sensitive acces
To add a new MCP server to both Gemini Code and Claude Code:
1. **Install the MCP server** (if it's an npm package):
```powershell
npm install -g @modelcontextprotocol/server-YOUR-SERVER
```
2. **Add to Gemini Code** (`mcp.json`):
```json
{
"servers": {
@@ -375,6 +524,7 @@ To add a new MCP server to both Gemini Code and Claude Code:
```
3. **Add to Claude Code** (`settings.json`):
```json
{
"mcpServers": {
@@ -392,10 +542,12 @@ To add a new MCP server to both Gemini Code and Claude Code:
## Current Project Integration
### ADR Implementation Status
- **ADR-0002**: Transaction Management ✅ Enforced
- **ADR-0003**: Input Validation ✅ Enforced with URL validation
### Database Setup
- PostgreSQL 15 + PostGIS running in container
- 63 tables created
- URL constraints active:
@@ -403,6 +555,7 @@ To add a new MCP server to both Gemini Code and Claude Code:
- `flyers_icon_url_check` enforces `^https?://.*`
### Development Workflow
1. Start containers: `podman start flyer-crawler-postgres flyer-crawler-redis`
2. Use MCP servers to manage development environment
3. AI assistants can:
@@ -421,6 +574,7 @@ To add a new MCP server to both Gemini Code and Claude Code:
## Maintenance
### Regular Tasks
- **Monthly**: Rotate Gitea access tokens
- **Weekly**: Update MCP server packages:
```powershell
@@ -429,7 +583,9 @@ To add a new MCP server to both Gemini Code and Claude Code:
- **As Needed**: Update Gitea MCP executable when new version is released
### Backup Configuration
Recommended to backup these files regularly:
- `%APPDATA%\Code\User\mcp.json`
- `%USERPROFILE%\.claude\settings.json`
@@ -442,6 +598,7 @@ This project uses Gitea Actions for continuous integration and deployment. The w
#### Automated Workflows
**deploy-to-test.yml** - Automated deployment to test environment
- **Trigger**: Automatically on every push to `main` branch
- **Runner**: `projectium.com` (self-hosted)
- **Process**:
@@ -459,6 +616,7 @@ This project uses Gitea Actions for continuous integration and deployment. The w
#### Manual Workflows
**deploy-to-prod.yml** - Manual deployment to production
- **Trigger**: Manual via workflow_dispatch
- **Confirmation Required**: Must type "deploy-to-prod"
- **Process**:
@@ -471,28 +629,34 @@ This project uses Gitea Actions for continuous integration and deployment. The w
- **Optional**: Force PM2 reload even if version matches
**manual-db-backup.yml** - Database backup workflow
- Creates timestamped backup of production database
- Stored in `/var/backups/postgres/`
**manual-db-restore.yml** - Database restore workflow
- Restores production database from backup file
- Requires confirmation and backup filename
**manual-db-reset-test.yml** - Reset test database
- Drops and recreates test database schema
- Used for testing schema migrations
**manual-db-reset-prod.yml** - Reset production database
- **DANGER**: Drops and recreates production database
- Requires multiple confirmations
**manual-deploy-major.yml** - Major version deployment
- Similar to deploy-to-prod but bumps major version
- For breaking changes or major releases
### Accessing Workflows via Gitea MCP
With the Gitea MCP server configured, AI assistants can:
- View workflow files
- Monitor workflow runs
- Check deployment status
@@ -500,6 +664,7 @@ With the Gitea MCP server configured, AI assistants can:
- Trigger manual workflows (via API)
**Example MCP Operations**:
```bash
# Via Gitea MCP, you can:
# - List recent workflow runs
@@ -514,6 +679,7 @@ With the Gitea MCP server configured, AI assistants can:
The workflows use these Gitea repository secrets:
**Database**:
- `DB_HOST` - PostgreSQL host
- `DB_USER` - Database user
- `DB_PASSWORD` - Database password
@@ -521,15 +687,18 @@ The workflows use these Gitea repository secrets:
- `DB_DATABASE_TEST` - Test database name
**Redis**:
- `REDIS_PASSWORD_PROD` - Production Redis password
- `REDIS_PASSWORD_TEST` - Test Redis password
**API Keys**:
- `VITE_GOOGLE_GENAI_API_KEY` - Production Gemini API key
- `VITE_GOOGLE_GENAI_API_KEY_TEST` - Test Gemini API key
- `GOOGLE_MAPS_API_KEY` - Google Maps Geocoding API key
**Authentication**:
- `JWT_SECRET` - JWT signing secret
### Schema Migration Process
@@ -542,6 +711,7 @@ The workflows use a schema hash comparison system:
4. **Protection**: Deployment fails if schemas don't match
**Manual Migration Steps** (when schema changes):
1. Update `sql/master_schema_rollup.sql`
2. Run manual migration workflow or:
```bash
@@ -554,16 +724,23 @@ The workflows use a schema hash comparison system:
The workflows manage three PM2 processes per environment:
**Production** (`ecosystem.config.cjs --env production`):
- `flyer-crawler-api` - Express API server
- `flyer-crawler-worker` - Background job worker
- `flyer-crawler-analytics-worker` - Analytics processor
**Test** (`ecosystem.config.cjs --env test`):
- `flyer-crawler-api-test` - Test Express API server
- `flyer-crawler-worker-test` - Test background worker
- `flyer-crawler-analytics-worker-test` - Test analytics worker
**Process Cleanup**:
- **2026-01-22**: Added Claude Code extension auto-update troubleshooting
- Documented AVX CPU crash bug in version 2.1.15
- Added workaround using `extensions.ignoreAutoUpdate` setting
- Included instructions for downgrading and re-enabling updates
- Workflows automatically delete errored/stopped processes
- Version comparison prevents unnecessary reloads
- Force reload option available for production
@@ -605,17 +782,20 @@ Using Gitea MCP, you can monitor deployments in real-time:
With the configured MCP servers, you can:
**Via Gitea MCP**:
- Trigger manual workflows
- View deployment history
- Monitor test results
- Access workflow logs
**Via Podman MCP**:
- Inspect container logs (for local testing)
- Manage local database containers
- Test migrations locally
**Via Filesystem MCP**:
- Review workflow files
- Edit deployment scripts
- Update ecosystem config

172
ecosystem.dev.config.cjs Normal file
View File

@@ -0,0 +1,172 @@
// ecosystem.dev.config.cjs
// ============================================================================
// DEVELOPMENT PM2 CONFIGURATION
// ============================================================================
// This file mirrors the production ecosystem.config.cjs but with settings
// appropriate for the development environment inside the dev container.
//
// Key differences from production:
// - Single instance for API (not cluster mode) to allow debugger attachment
// - tsx watch mode for hot reloading
// - Development-specific environment variables
// - Logs written to /var/log/pm2 for Logstash integration (ADR-050)
//
// Usage:
// pm2 start ecosystem.dev.config.cjs
// pm2 logs # View all logs
// pm2 logs flyer-crawler-api-dev # View API logs only
// pm2 restart all # Restart all processes
// pm2 delete all # Stop all processes
//
// Related:
// - ecosystem.config.cjs (production configuration)
// - docs/adr/0014-linux-only-platform.md
// - docs/adr/0050-postgresql-function-observability.md
// ============================================================================
// --- Environment Variable Validation ---
// In dev, we warn but don't fail - variables come from compose.dev.yml
const requiredVars = ['DB_HOST', 'JWT_SECRET'];
const missingVars = requiredVars.filter((key) => !process.env[key]);
if (missingVars.length > 0) {
console.warn(
'\n[ecosystem.dev.config.cjs] WARNING: The following environment variables are MISSING:',
);
missingVars.forEach((key) => console.warn(` - ${key}`));
console.warn(
'[ecosystem.dev.config.cjs] These should be set in compose.dev.yml or .env.local\n',
);
} else {
console.log('[ecosystem.dev.config.cjs] Required environment variables are present.');
}
// --- Shared Environment Variables ---
// These come from compose.dev.yml environment section
const sharedEnv = {
NODE_ENV: 'development',
DB_HOST: process.env.DB_HOST || 'postgres',
DB_PORT: process.env.DB_PORT || '5432',
DB_USER: process.env.DB_USER || 'postgres',
DB_PASSWORD: process.env.DB_PASSWORD || 'postgres',
DB_NAME: process.env.DB_NAME || 'flyer_crawler_dev',
REDIS_URL: process.env.REDIS_URL || 'redis://redis:6379',
REDIS_HOST: process.env.REDIS_HOST || 'redis',
REDIS_PORT: process.env.REDIS_PORT || '6379',
FRONTEND_URL: process.env.FRONTEND_URL || 'https://localhost',
JWT_SECRET: process.env.JWT_SECRET || 'dev-jwt-secret-change-in-production',
WORKER_LOCK_DURATION: process.env.WORKER_LOCK_DURATION || '120000',
// Sentry/Bugsink (ADR-015)
SENTRY_DSN: process.env.SENTRY_DSN,
SENTRY_ENVIRONMENT: process.env.SENTRY_ENVIRONMENT || 'development',
SENTRY_ENABLED: process.env.SENTRY_ENABLED || 'true',
SENTRY_DEBUG: process.env.SENTRY_DEBUG || 'true',
// Optional API keys (may not be set in dev)
GEMINI_API_KEY: process.env.GEMINI_API_KEY,
GOOGLE_MAPS_API_KEY: process.env.GOOGLE_MAPS_API_KEY,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
};
module.exports = {
apps: [
// =========================================================================
// API SERVER (Development)
// =========================================================================
// Single instance with tsx watch for hot reloading.
// Unlike production, we don't use cluster mode to allow:
// - Debugger attachment
// - Simpler log output
// - Hot reloading via tsx watch
{
name: 'flyer-crawler-api-dev',
script: './node_modules/.bin/tsx',
args: 'watch server.ts',
cwd: '/app',
watch: false, // tsx watch handles this
instances: 1, // Single instance for debugging
exec_mode: 'fork', // Fork mode (not cluster) for debugging
max_memory_restart: '1G', // More generous in dev
kill_timeout: 5000,
// PM2 log configuration (ADR-050 - Logstash integration)
output: '/var/log/pm2/api-out.log',
error: '/var/log/pm2/api-error.log',
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
merge_logs: true,
// Restart configuration
max_restarts: 10,
exp_backoff_restart_delay: 500,
min_uptime: '5s',
// Environment
env: {
...sharedEnv,
PORT: '3001',
},
},
// =========================================================================
// BACKGROUND WORKER (Development)
// =========================================================================
// Processes background jobs (flyer processing, cleanup, etc.)
// Runs as a separate process like in production.
{
name: 'flyer-crawler-worker-dev',
script: './node_modules/.bin/tsx',
args: 'watch src/services/worker.ts',
cwd: '/app',
watch: false, // tsx watch handles this
instances: 1,
exec_mode: 'fork',
max_memory_restart: '1G',
kill_timeout: 10000, // Workers need more time to finish jobs
// PM2 log configuration (ADR-050)
output: '/var/log/pm2/worker-out.log',
error: '/var/log/pm2/worker-error.log',
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
merge_logs: true,
// Restart configuration
max_restarts: 10,
exp_backoff_restart_delay: 500,
min_uptime: '5s',
// Environment
env: {
...sharedEnv,
},
},
// =========================================================================
// VITE FRONTEND DEV SERVER (Development)
// =========================================================================
// Vite dev server for frontend hot module replacement.
// Listens on 5173 and is proxied by nginx to 443.
{
name: 'flyer-crawler-vite-dev',
script: './node_modules/.bin/vite',
args: '--host',
cwd: '/app',
watch: false, // Vite handles its own watching
instances: 1,
exec_mode: 'fork',
max_memory_restart: '2G', // Vite can use significant memory
kill_timeout: 5000,
// PM2 log configuration (ADR-050)
output: '/var/log/pm2/vite-out.log',
error: '/var/log/pm2/vite-error.log',
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
merge_logs: true,
// Restart configuration
max_restarts: 10,
exp_backoff_restart_delay: 500,
min_uptime: '5s',
// Environment
env: {
NODE_ENV: 'development',
// Vite-specific env vars (VITE_ prefix)
VITE_SENTRY_DSN: process.env.VITE_SENTRY_DSN,
VITE_SENTRY_ENVIRONMENT: process.env.VITE_SENTRY_ENVIRONMENT || 'development',
VITE_SENTRY_ENABLED: process.env.VITE_SENTRY_ENABLED || 'true',
VITE_SENTRY_DEBUG: process.env.VITE_SENTRY_DEBUG || 'true',
},
},
],
};

View File

@@ -47,6 +47,13 @@ server {
proxy_cache_bypass $http_upgrade;
}
# Serve flyer images from static storage
location /flyer-images/ {
alias /var/www/flyer-crawler-test.projectium.com/flyer-images/;
expires 7d;
add_header Cache-Control "public, immutable";
}
# Correct MIME type for .mjs files
location ~ \.mjs$ {
include /etc/nginx/mime.types;

View File

@@ -51,6 +51,13 @@ server {
proxy_cache_bypass $http_upgrade;
}
# Serve flyer images from static storage
location /flyer-images/ {
alias /var/www/flyer-crawler.projectium.com/flyer-images/;
expires 7d;
add_header Cache-Control "public, immutable";
}
# This block specifically targets requests for .mjs files.
location ~ \.mjs$ {
# It ensures that these files are served with the correct JavaScript MIME type.
@@ -65,7 +72,7 @@ server {
add_header X-XSS-Protection "1; mode=block" always;
add_header X-Content-Type-Options "nosniff" always;
listen [::]:443 ssl ipv6only=on; # managed by Certbot
listen [::]:443 ssl; # managed by Certbot
listen 443 ssl; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/flyer-crawler.projectium.com/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/flyer-crawler.projectium.com/privkey.pem; # managed by Certbot

186
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "flyer-crawler",
"version": "0.12.2",
"version": "0.12.8",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "flyer-crawler",
"version": "0.12.2",
"version": "0.12.8",
"dependencies": {
"@bull-board/api": "^6.14.2",
"@bull-board/express": "^6.14.2",
@@ -15,12 +15,12 @@
"@sentry/react": "^10.32.1",
"@tanstack/react-query": "^5.90.12",
"@types/connect-timeout": "^1.9.0",
"@types/react-joyride": "^2.0.2",
"bcrypt": "^5.1.1",
"bullmq": "^5.65.1",
"connect-timeout": "^1.9.1",
"cookie-parser": "^1.4.7",
"date-fns": "^4.1.0",
"driver.js": "^1.3.1",
"exif-parser": "^0.1.12",
"express": "^5.1.0",
"express-list-endpoints": "^7.1.1",
@@ -45,7 +45,6 @@
"react": "^19.2.0",
"react-dom": "^19.2.0",
"react-hot-toast": "^2.6.0",
"react-joyride": "^2.9.3",
"react-router-dom": "^7.9.6",
"recharts": "^3.4.1",
"sharp": "^0.34.5",
@@ -2144,12 +2143,6 @@
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
}
},
"node_modules/@gilbarbara/deep-equal": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/@gilbarbara/deep-equal/-/deep-equal-0.3.1.tgz",
"integrity": "sha512-I7xWjLs2YSVMc5gGx1Z3ZG1lgFpITPndpi8Ku55GeEIKpACCPQNS/OTqQbxgTCfq0Ncvcc+CrFov96itVh6Qvw==",
"license": "MIT"
},
"node_modules/@google/genai": {
"version": "1.34.0",
"resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.34.0.tgz",
@@ -6603,6 +6596,7 @@
"version": "19.2.7",
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.7.tgz",
"integrity": "sha512-MWtvHrGZLFttgeEj28VXHxpmwYbor/ATPYbBfSFZEIRK0ecCFLl2Qo55z52Hss+UV9CRN7trSeq1zbgx7YDWWg==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"csstype": "^3.2.2"
@@ -6618,15 +6612,6 @@
"@types/react": "^19.2.0"
}
},
"node_modules/@types/react-joyride": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@types/react-joyride/-/react-joyride-2.0.2.tgz",
"integrity": "sha512-RbixI8KE4K4B4bVzigT765oiQMCbWqlb9vj5qz1pFvkOvynkiAGurGVVf+nGszGGa89WrQhUnAwd0t1tqxeoDw==",
"license": "MIT",
"dependencies": {
"@types/react": "*"
}
},
"node_modules/@types/send": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz",
@@ -9354,13 +9339,6 @@
"integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==",
"license": "MIT"
},
"node_modules/deep-diff": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/deep-diff/-/deep-diff-1.0.2.tgz",
"integrity": "sha512-aWS3UIVH+NPGCD1kki+DCU9Dua032iSsO43LqQpcs4R3+dVv7tX0qBGjiVHJHjplsoUM2XRO/KB92glqc68awg==",
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
"license": "MIT"
},
"node_modules/deep-is": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
@@ -9368,15 +9346,6 @@
"dev": true,
"license": "MIT"
},
"node_modules/deepmerge": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
"integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/default-require-extensions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-3.0.1.tgz",
@@ -9625,6 +9594,12 @@
"url": "https://dotenvx.com"
}
},
"node_modules/driver.js": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.4.0.tgz",
"integrity": "sha512-Gm64jm6PmcU+si21sQhBrTAM1JvUrR0QhNmjkprNLxohOBzul9+pNHXgQaT9lW84gwg9GMLB3NZGuGolsz5uew==",
"license": "MIT"
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -12186,12 +12161,6 @@
"node": ">=0.10.0"
}
},
"node_modules/is-lite": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/is-lite/-/is-lite-1.2.1.tgz",
"integrity": "sha512-pgF+L5bxC+10hLBgf6R2P4ZZUBOQIIacbdo8YvuCP8/JvsWxG7aZ9p10DYuLtifFci4l3VITphhMlMV4Y+urPw==",
"license": "MIT"
},
"node_modules/is-map": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.3.tgz",
@@ -12731,6 +12700,7 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"dev": true,
"license": "MIT"
},
"node_modules/js-yaml": {
@@ -13629,6 +13599,7 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
"integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"js-tokens": "^3.0.0 || ^4.0.0"
@@ -14792,6 +14763,13 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/openapi-types": {
"version": "12.1.3",
"resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz",
"integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==",
"license": "MIT",
"peer": true
},
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@@ -15420,17 +15398,6 @@
"node": ">=8"
}
},
"node_modules/popper.js": {
"version": "1.16.1",
"resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz",
"integrity": "sha512-Wb4p1J4zyFTbM+u6WuO4XstYx4Ky9Cewe4DWrel7B0w6VVICvPwdOpotjzcf6eD8TsckVnIMNONQyPIUFOUbCQ==",
"deprecated": "You can find the new Popper v2 at @popperjs/core, this package is dedicated to the legacy v1",
"license": "MIT",
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/popperjs"
}
},
"node_modules/possible-typed-array-names": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz",
@@ -15630,6 +15597,7 @@
"version": "15.8.1",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
"dev": true,
"license": "MIT",
"dependencies": {
"loose-envify": "^1.4.0",
@@ -15641,6 +15609,7 @@
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
"integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
"dev": true,
"license": "MIT"
},
"node_modules/proper-lockfile": {
@@ -15856,45 +15825,6 @@
"react": "^19.2.3"
}
},
"node_modules/react-floater": {
"version": "0.7.9",
"resolved": "https://registry.npmjs.org/react-floater/-/react-floater-0.7.9.tgz",
"integrity": "sha512-NXqyp9o8FAXOATOEo0ZpyaQ2KPb4cmPMXGWkx377QtJkIXHlHRAGer7ai0r0C1kG5gf+KJ6Gy+gdNIiosvSicg==",
"license": "MIT",
"dependencies": {
"deepmerge": "^4.3.1",
"is-lite": "^0.8.2",
"popper.js": "^1.16.0",
"prop-types": "^15.8.1",
"tree-changes": "^0.9.1"
},
"peerDependencies": {
"react": "15 - 18",
"react-dom": "15 - 18"
}
},
"node_modules/react-floater/node_modules/@gilbarbara/deep-equal": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/@gilbarbara/deep-equal/-/deep-equal-0.1.2.tgz",
"integrity": "sha512-jk+qzItoEb0D0xSSmrKDDzf9sheQj/BAPxlgNxgmOaA3mxpUa6ndJLYGZKsJnIVEQSD8zcTbyILz7I0HcnBCRA==",
"license": "MIT"
},
"node_modules/react-floater/node_modules/is-lite": {
"version": "0.8.2",
"resolved": "https://registry.npmjs.org/is-lite/-/is-lite-0.8.2.tgz",
"integrity": "sha512-JZfH47qTsslwaAsqbMI3Q6HNNjUuq6Cmzzww50TdP5Esb6e1y2sK2UAaZZuzfAzpoI2AkxoPQapZdlDuP6Vlsw==",
"license": "MIT"
},
"node_modules/react-floater/node_modules/tree-changes": {
"version": "0.9.3",
"resolved": "https://registry.npmjs.org/tree-changes/-/tree-changes-0.9.3.tgz",
"integrity": "sha512-vvvS+O6kEeGRzMglTKbc19ltLWNtmNt1cpBoSYLj/iEcPVvpJasemKOlxBrmZaCtDJoF+4bwv3m01UKYi8mukQ==",
"license": "MIT",
"dependencies": {
"@gilbarbara/deep-equal": "^0.1.1",
"is-lite": "^0.8.2"
}
},
"node_modules/react-hot-toast": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/react-hot-toast/-/react-hot-toast-2.6.0.tgz",
@@ -15912,64 +15842,12 @@
"react-dom": ">=16"
}
},
"node_modules/react-innertext": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/react-innertext/-/react-innertext-1.1.5.tgz",
"integrity": "sha512-PWAqdqhxhHIv80dT9znP2KvS+hfkbRovFp4zFYHFFlOoQLRiawIic81gKb3U1wEyJZgMwgs3JoLtwryASRWP3Q==",
"license": "MIT",
"peerDependencies": {
"@types/react": ">=0.0.0 <=99",
"react": ">=0.0.0 <=99"
}
},
"node_modules/react-is": {
"version": "17.0.2",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",
"integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==",
"dev": true,
"license": "MIT"
},
"node_modules/react-joyride": {
"version": "2.9.3",
"resolved": "https://registry.npmjs.org/react-joyride/-/react-joyride-2.9.3.tgz",
"integrity": "sha512-1+Mg34XK5zaqJ63eeBhqdbk7dlGCFp36FXwsEvgpjqrtyywX2C6h9vr3jgxP0bGHCw8Ilsp/nRDzNVq6HJ3rNw==",
"license": "MIT",
"dependencies": {
"@gilbarbara/deep-equal": "^0.3.1",
"deep-diff": "^1.0.2",
"deepmerge": "^4.3.1",
"is-lite": "^1.2.1",
"react-floater": "^0.7.9",
"react-innertext": "^1.1.5",
"react-is": "^16.13.1",
"scroll": "^3.0.1",
"scrollparent": "^2.1.0",
"tree-changes": "^0.11.2",
"type-fest": "^4.27.0"
},
"peerDependencies": {
"react": "15 - 18",
"react-dom": "15 - 18"
}
},
"node_modules/react-joyride/node_modules/react-is": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
"integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
"license": "MIT"
},
"node_modules/react-joyride/node_modules/type-fest": {
"version": "4.41.0",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz",
"integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==",
"license": "(MIT OR CC0-1.0)",
"engines": {
"node": ">=16"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/react-redux": {
"version": "9.2.0",
"resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz",
@@ -16603,18 +16481,6 @@
"integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==",
"license": "MIT"
},
"node_modules/scroll": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/scroll/-/scroll-3.0.1.tgz",
"integrity": "sha512-pz7y517OVls1maEzlirKO5nPYle9AXsFzTMNJrRGmT951mzpIBy7sNHOg5o/0MQd/NqliCiWnAi0kZneMPFLcg==",
"license": "MIT"
},
"node_modules/scrollparent": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/scrollparent/-/scrollparent-2.1.0.tgz",
"integrity": "sha512-bnnvJL28/Rtz/kz2+4wpBjHzWoEzXhVg/TE8BeVGJHUqE8THNIRnDxDWMktwM+qahvlRdvlLdsQfYe+cuqfZeA==",
"license": "ISC"
},
"node_modules/secure-json-parse": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz",
@@ -18065,16 +17931,6 @@
"node": ">=20"
}
},
"node_modules/tree-changes": {
"version": "0.11.3",
"resolved": "https://registry.npmjs.org/tree-changes/-/tree-changes-0.11.3.tgz",
"integrity": "sha512-r14mvDZ6tqz8PRQmlFKjhUVngu4VZ9d92ON3tp0EGpFBE6PAHOq8Bx8m8ahbNoGE3uI/npjYcJiqVydyOiYXag==",
"license": "MIT",
"dependencies": {
"@gilbarbara/deep-equal": "^0.3.1",
"is-lite": "^1.2.1"
}
},
"node_modules/tree-kill": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",

View File

@@ -1,11 +1,16 @@
{
"name": "flyer-crawler",
"private": true,
"version": "0.12.2",
"version": "0.12.8",
"type": "module",
"scripts": {
"dev": "concurrently \"npm:start:dev\" \"vite\"",
"dev:container": "concurrently \"npm:start:dev\" \"vite --host\"",
"dev:pm2": "pm2 start ecosystem.dev.config.cjs && pm2 logs",
"dev:pm2:restart": "pm2 restart ecosystem.dev.config.cjs",
"dev:pm2:stop": "pm2 delete all",
"dev:pm2:status": "pm2 status",
"dev:pm2:logs": "pm2 logs",
"start": "npm run start:prod",
"build": "vite build",
"preview": "vite preview",
@@ -36,7 +41,6 @@
"@sentry/react": "^10.32.1",
"@tanstack/react-query": "^5.90.12",
"@types/connect-timeout": "^1.9.0",
"@types/react-joyride": "^2.0.2",
"bcrypt": "^5.1.1",
"bullmq": "^5.65.1",
"connect-timeout": "^1.9.1",
@@ -66,7 +70,7 @@
"react": "^19.2.0",
"react-dom": "^19.2.0",
"react-hot-toast": "^2.6.0",
"react-joyride": "^2.9.3",
"driver.js": "^1.3.1",
"react-router-dom": "^7.9.6",
"recharts": "^3.4.1",
"sharp": "^0.34.5",

View File

@@ -0,0 +1,26 @@
#!/usr/bin/env node
/**
* Creates a 64x64 icon from test-flyer-image.png
* Run from container: node scripts/create-test-icon.js
*/
import sharp from 'sharp';
import path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const inputPath = path.join(__dirname, '../src/tests/assets/test-flyer-image.png');
const outputPath = path.join(__dirname, '../src/tests/assets/test-flyer-icon.png');
sharp(inputPath)
.resize(64, 64, { fit: 'cover' })
.toFile(outputPath)
.then(() => {
console.log(`✓ Created icon: ${outputPath}`);
})
.catch((err) => {
console.error('Error creating icon:', err);
process.exit(1);
});

View File

@@ -3,47 +3,148 @@
# ============================================================================
# Development Container Entrypoint
# ============================================================================
# This script starts the development server automatically when the container
# starts, both with VS Code Dev Containers and with plain podman-compose.
# This script starts all development services when the container starts,
# both with VS Code Dev Containers and with plain podman-compose.
#
# Services started:
# - Nginx (proxies Vite 5173 → 3000)
# - Nginx (HTTPS proxy: Vite 5173 -> 443, Bugsink 8000 -> 8443, API 3001)
# - Bugsink (error tracking) on port 8000
# - Logstash (log aggregation)
# - Node.js dev server (API + Frontend) on ports 3001 and 5173
# - PM2 (process manager) running:
# - flyer-crawler-api-dev: API server (port 3001)
# - flyer-crawler-worker-dev: Background job worker
# - flyer-crawler-vite-dev: Vite frontend dev server (port 5173)
#
# ADR-014: This architecture matches production (PM2 managing processes)
# ADR-050: Logs are written to /var/log/pm2 for Logstash integration
# ============================================================================
set -e
echo "🚀 Starting Flyer Crawler Dev Container..."
echo "Starting Flyer Crawler Dev Container..."
# Configure Bugsink HTTPS (ADR-015)
echo "Configuring Bugsink HTTPS..."
mkdir -p /etc/bugsink/ssl
if [ ! -f "/etc/bugsink/ssl/localhost+2.pem" ]; then
cd /etc/bugsink/ssl && mkcert localhost 127.0.0.1 ::1 > /dev/null 2>&1
fi
# Create nginx config for Bugsink HTTPS
cat > /etc/nginx/sites-available/bugsink <<'NGINX_EOF'
server {
listen 8443 ssl http2;
listen [::]:8443 ssl http2;
server_name localhost;
ssl_certificate /etc/bugsink/ssl/localhost+2.pem;
ssl_certificate_key /etc/bugsink/ssl/localhost+2-key.pem;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
location / {
proxy_pass http://127.0.0.1:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_redirect off;
proxy_buffering off;
client_max_body_size 20M;
}
}
NGINX_EOF
ln -sf /etc/nginx/sites-available/bugsink /etc/nginx/sites-enabled/bugsink
# Copy the dev nginx config from mounted volume to nginx sites-available
echo "Copying nginx dev config..."
cp /app/docker/nginx/dev.conf /etc/nginx/sites-available/default
# Ensure PM2 log directory exists with correct permissions
echo "Setting up PM2 log directory..."
mkdir -p /var/log/pm2
chmod 755 /var/log/pm2
# Start nginx in background (if installed)
if command -v nginx &> /dev/null; then
echo "🌐 Starting nginx (HTTPS proxy: Vite 5173 → port 443)..."
echo "Starting nginx (HTTPS: Vite 5173 -> 443, Bugsink 8000 -> 8443, API 3001 -> /api/)..."
nginx &
fi
# Start Bugsink in background
echo "📊 Starting Bugsink error tracking..."
echo "Starting Bugsink error tracking..."
/usr/local/bin/start-bugsink.sh > /var/log/bugsink/server.log 2>&1 &
# Wait for Bugsink to initialize, then run snappea migrations
echo "Waiting for Bugsink to initialize..."
sleep 5
echo "Running Bugsink snappea database migrations..."
cd /opt/bugsink/conf && \
export DATABASE_URL="postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink" && \
export SECRET_KEY="dev-bugsink-secret-key-minimum-50-characters-for-security" && \
/opt/bugsink/bin/bugsink-manage migrate --database=snappea > /dev/null 2>&1
# Start Snappea task worker
echo "Starting Snappea task worker..."
cd /opt/bugsink/conf && \
export DATABASE_URL="postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink" && \
export SECRET_KEY="dev-bugsink-secret-key-minimum-50-characters-for-security" && \
/opt/bugsink/bin/bugsink-manage runsnappea > /var/log/bugsink/snappea.log 2>&1 &
# Start Logstash in background
echo "📝 Starting Logstash..."
echo "Starting Logstash..."
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/bugsink.conf > /var/log/logstash/logstash.log 2>&1 &
# Wait a few seconds for services to initialize
# Wait for services to initialize
sleep 3
# Change to app directory
cd /app
# Start development server
echo "💻 Starting development server..."
echo " - Frontend: https://localhost (nginx HTTPS → Vite on 5173)"
echo " - Backend API: http://localhost:3001"
echo " - Bugsink: http://localhost:8000"
echo " - Note: Accept the self-signed certificate warning in your browser"
# ============================================================================
# Start PM2 (ADR-014: Production-like process management)
# ============================================================================
# PM2 manages all Node.js processes:
# - API server (tsx watch server.ts)
# - Background worker (tsx watch src/services/worker.ts)
# - Vite dev server (vite --host)
#
# Logs are written to /var/log/pm2 for Logstash integration (ADR-050)
# ============================================================================
echo "Starting PM2 with development configuration..."
echo " - API Server: http://localhost:3001 (proxied via nginx)"
echo " - Frontend: https://localhost (nginx HTTPS -> Vite on 5173)"
echo " - Bugsink: https://localhost:8443 (nginx HTTPS -> Bugsink on 8000)"
echo " - PM2 Logs: /var/log/pm2/*.log"
echo ""
echo "Note: Accept the self-signed certificate warnings in your browser"
echo ""
# Run npm dev server (this will block and keep container alive)
exec npm run dev:container
# Delete any existing PM2 processes (clean start)
pm2 delete all 2>/dev/null || true
# Start PM2 with the dev ecosystem config
pm2 start /app/ecosystem.dev.config.cjs
# Show PM2 status
echo ""
echo "--- PM2 Process Status ---"
pm2 status
echo "-------------------------"
echo ""
echo "PM2 Commands:"
echo " pm2 logs # View all logs (tail -f style)"
echo " pm2 logs api # View API logs only"
echo " pm2 restart all # Restart all processes"
echo " pm2 status # Show process status"
echo ""
# Keep the container running by tailing PM2 logs
# This ensures:
# 1. Container stays alive
# 2. Logs are visible in docker logs / podman logs
# 3. PM2 processes continue running
exec pm2 logs --raw

53
sql/update_flyer_urls.sql Normal file
View File

@@ -0,0 +1,53 @@
-- Update flyer URLs from example.com to environment-specific URLs
--
-- This script should be run after determining the correct base URL for the environment:
-- - Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
-- - Test environment: https://flyer-crawler-test.projectium.com
-- - Production: https://flyer-crawler.projectium.com
-- For dev container (run in dev database):
-- Uses 'localhost' instead of '127.0.0.1' to match how users access the site.
-- This avoids ERR_CERT_AUTHORITY_INVALID errors when images are loaded from a
-- different origin than the page.
UPDATE flyers
SET
image_url = REPLACE(image_url, 'example.com', 'localhost'),
icon_url = REPLACE(icon_url, 'example.com', 'localhost')
WHERE
image_url LIKE '%example.com%'
OR icon_url LIKE '%example.com%';
-- Also fix any existing 127.0.0.1 URLs to use localhost:
UPDATE flyers
SET
image_url = REPLACE(image_url, '127.0.0.1', 'localhost'),
icon_url = REPLACE(icon_url, '127.0.0.1', 'localhost')
WHERE
image_url LIKE '%127.0.0.1%'
OR icon_url LIKE '%127.0.0.1%';
-- For test environment (run in test database):
-- UPDATE flyers
-- SET
-- image_url = REPLACE(image_url, 'example.com', 'flyer-crawler-test.projectium.com'),
-- icon_url = REPLACE(icon_url, 'example.com', 'flyer-crawler-test.projectium.com')
-- WHERE
-- image_url LIKE '%example.com%'
-- OR icon_url LIKE '%example.com%';
-- For production (run in production database):
-- UPDATE flyers
-- SET
-- image_url = REPLACE(image_url, 'example.com', 'flyer-crawler.projectium.com'),
-- icon_url = REPLACE(icon_url, 'example.com', 'flyer-crawler.projectium.com')
-- WHERE
-- image_url LIKE '%example.com%'
-- OR icon_url LIKE '%example.com%';
-- Verify the changes:
SELECT flyer_id, image_url, icon_url
FROM flyers
WHERE image_url LIKE '%localhost%'
OR icon_url LIKE '%localhost%'
OR image_url LIKE '%flyer-crawler%'
OR icon_url LIKE '%flyer-crawler%';

View File

@@ -15,7 +15,7 @@ export const Dashboard: React.FC = () => {
<RecipeSuggester />
{/* Other Dashboard Widgets */}
<div className="bg-white dark:bg-gray-800 shadow rounded-lg p-6">
<div className="bg-white dark:bg-gray-800 shadow rounded-lg p-6 transition-colors hover:bg-gray-50 dark:hover:bg-gray-800/80">
<h2 className="text-lg font-medium text-gray-900 dark:text-white mb-4">Your Flyers</h2>
<FlyerCountDisplay />
</div>

View File

@@ -31,7 +31,7 @@ export const Header: React.FC<HeaderProps> = ({
// The state and handlers for the old AuthModal and SignUpModal have been removed.
return (
<>
<header className="bg-white dark:bg-gray-900 shadow-md sticky top-0 z-20">
<header className="bg-white dark:bg-gray-900 shadow-md sticky top-0 z-20 border-b-2 border-brand-primary dark:border-brand-secondary">
<div className="max-w-screen-2xl mx-auto px-4 sm:px-6 lg:px-8">
<div className="flex items-center justify-between h-16">
<div className="flex items-center">

View File

@@ -43,7 +43,7 @@ export const Leaderboard: React.FC = () => {
}
return (
<div className="bg-white dark:bg-gray-800 shadow-lg rounded-lg p-6">
<div className="bg-white dark:bg-gray-800 shadow-lg rounded-lg p-6 transition-colors hover:bg-gray-50 dark:hover:bg-gray-800/80">
<h2 className="text-2xl font-bold text-gray-900 dark:text-white mb-4 flex items-center">
<Award className="w-6 h-6 mr-2 text-blue-500" />
Top Users
@@ -57,7 +57,7 @@ export const Leaderboard: React.FC = () => {
{leaderboard.map((user) => (
<li
key={user.user_id}
className="flex items-center space-x-4 p-3 bg-gray-50 dark:bg-gray-700 rounded-lg transition hover:bg-gray-100 dark:hover:bg-gray-600"
className="flex items-center space-x-4 p-3 bg-gray-50 dark:bg-gray-700 rounded-lg transition-colors hover:bg-brand-light/30 dark:hover:bg-brand-dark/20"
>
<div className="shrink-0 w-8 text-center">{getRankIcon(user.rank)}</div>
<img

View File

@@ -48,7 +48,7 @@ export const RecipeSuggester: React.FC = () => {
);
return (
<div className="bg-white dark:bg-gray-800 shadow rounded-lg p-6">
<div className="bg-white dark:bg-gray-800 shadow rounded-lg p-6 transition-colors hover:bg-gray-50 dark:hover:bg-gray-800/80">
<h2 className="text-xl font-semibold text-gray-900 dark:text-white mb-2">
Get a Recipe Suggestion
</h2>

View File

@@ -12,6 +12,17 @@ import path from 'node:path';
import bcrypt from 'bcrypt';
import { logger } from '../services/logger.server';
// Determine base URL for flyer images based on environment
// Dev container: https://127.0.0.1 (NGINX now accepts both localhost and 127.0.0.1)
// Test: https://flyer-crawler-test.projectium.com
// Production: https://flyer-crawler.projectium.com
const BASE_URL =
process.env.FLYER_BASE_URL || process.env.NODE_ENV === 'production'
? 'https://flyer-crawler.projectium.com'
: process.env.NODE_ENV === 'test'
? 'https://flyer-crawler-test.projectium.com'
: 'https://127.0.0.1';
const pool = new Pool({
user: process.env.DB_USER,
host: process.env.DB_HOST,
@@ -101,7 +112,21 @@ async function main() {
const userId = userRes.rows[0].user_id;
logger.info('Seeded regular user (user@example.com / userpass)');
// 4. Seed a Flyer
// 4. Copy test images to flyer-images directory
logger.info('--- Copying test flyer images... ---');
const flyerImagesDir = path.resolve(process.cwd(), 'public/flyer-images');
await fs.mkdir(flyerImagesDir, { recursive: true });
const testImageSource = path.resolve(process.cwd(), 'src/tests/assets/test-flyer-image.jpg');
const testIconSource = path.resolve(process.cwd(), 'src/tests/assets/test-flyer-icon.png');
const testImageDest = path.join(flyerImagesDir, 'test-flyer-image.jpg');
const testIconDest = path.join(flyerImagesDir, 'test-flyer-icon.png');
await fs.copyFile(testImageSource, testImageDest);
await fs.copyFile(testIconSource, testIconDest);
logger.info(`Copied test images to ${flyerImagesDir}`);
// 5. Seed a Flyer
logger.info('--- Seeding a Sample Flyer... ---');
const today = new Date();
const validFrom = new Date(today);
@@ -111,7 +136,7 @@ async function main() {
const flyerQuery = `
INSERT INTO public.flyers (file_name, image_url, icon_url, checksum, store_id, valid_from, valid_to)
VALUES ('safeway-flyer.jpg', 'https://example.com/flyer-images/safeway-flyer.jpg', 'https://example.com/flyer-images/icons/safeway-flyer.jpg', 'a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0', ${storeMap.get('Safeway')}, $1, $2)
VALUES ('test-flyer-image.jpg', '${BASE_URL}/flyer-images/test-flyer-image.jpg', '${BASE_URL}/flyer-images/test-flyer-icon.png', 'a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0', ${storeMap.get('Safeway')}, $1, $2)
RETURNING flyer_id;
`;
const flyerRes = await client.query<{ flyer_id: number }>(flyerQuery, [
@@ -121,7 +146,7 @@ async function main() {
const flyerId = flyerRes.rows[0].flyer_id;
logger.info(`Seeded flyer for Safeway (ID: ${flyerId}).`);
// 5. Seed Flyer Items
// 6. Seed Flyer Items
logger.info('--- Seeding Flyer Items... ---');
const flyerItems = [
{
@@ -169,7 +194,7 @@ async function main() {
}
logger.info(`Seeded ${flyerItems.length} items for the Safeway flyer.`);
// 6. Seed Watched Items for the user
// 7. Seed Watched Items for the user
logger.info('--- Seeding Watched Items... ---');
const watchedItemIds = [
masterItemMap.get('chicken breast'),
@@ -186,7 +211,7 @@ async function main() {
}
logger.info(`Seeded ${watchedItemIds.length} watched items for Test User.`);
// 7. Seed a Shopping List
// 8. Seed a Shopping List
logger.info('--- Seeding a Shopping List... ---');
const listRes = await client.query<{ shopping_list_id: number }>(
'INSERT INTO public.shopping_lists (user_id, name) VALUES ($1, $2) RETURNING shopping_list_id',

View File

@@ -3,6 +3,7 @@ import { useMutation } from '@tanstack/react-query';
import * as apiClient from '../../services/apiClient';
import { notifyError } from '../../services/notificationService';
import type { UserProfile } from '../../types';
import { logger } from '../../services/logger.client';
interface AuthResponse {
userprofile: UserProfile;
@@ -29,7 +30,9 @@ export const useLoginMutation = () => {
password: string;
rememberMe: boolean;
}): Promise<AuthResponse> => {
logger.info('[useLoginMutation] MUTATION STARTED', { email, rememberMe: rememberMe });
const response = await apiClient.loginUser(email, password, rememberMe);
logger.info('[useLoginMutation] Got response', { status: response.status });
if (!response.ok) {
const error = await response.json().catch(() => ({
@@ -38,7 +41,18 @@ export const useLoginMutation = () => {
throw new Error(error.message || 'Failed to login');
}
return response.json();
const result = await response.json();
// DEBUG: Log the actual response structure
logger.debug('[useLoginMutation] Raw API response', { result });
logger.debug('[useLoginMutation] result.data', { data: result.data });
logger.debug('[useLoginMutation] result.data.token', { token: result.data?.token });
// The API returns {success, data: {userprofile, token}}, so extract the data
const extracted = result.data;
logger.info('[useLoginMutation] Returning extracted data', {
hasUserprofile: !!extracted?.userprofile,
hasToken: !!extracted?.token,
});
return extracted;
},
onError: (error: Error) => {
notifyError(error.message || 'Failed to login');
@@ -75,7 +89,9 @@ export const useRegisterMutation = () => {
throw new Error(error.message || 'Failed to register');
}
return response.json();
const result = await response.json();
// The API returns {success, data: {userprofile, token}}, so extract the data
return result.data;
},
onError: (error: Error) => {
notifyError(error.message || 'Failed to register');

View File

@@ -3,6 +3,9 @@ import { renderHook, act } from '@testing-library/react';
import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest';
import { useDataExtraction } from './useDataExtraction';
import type { Flyer } from '../types';
import { getFlyerBaseUrl } from '../tests/utils/testHelpers';
const FLYER_BASE_URL = getFlyerBaseUrl();
// Create a mock flyer for testing
const createMockFlyer = (id: number, storeName: string = `Store ${id}`): Flyer => ({
@@ -14,8 +17,8 @@ const createMockFlyer = (id: number, storeName: string = `Store ${id}`): Flyer =
updated_at: '2024-01-01T00:00:00Z',
},
file_name: `flyer${id}.jpg`,
image_url: `https://example.com/flyer${id}.jpg`,
icon_url: `https://example.com/flyer${id}_icon.jpg`,
image_url: `${FLYER_BASE_URL}/flyer${id}.jpg`,
icon_url: `${FLYER_BASE_URL}/flyer${id}_icon.jpg`,
status: 'processed',
item_count: 0,
created_at: '2024-01-01T00:00:00Z',
@@ -76,7 +79,7 @@ describe('useDataExtraction Hook', () => {
expect(updatedFlyer.store?.name).toBe('New Store Name');
// Ensure other properties are preserved
expect(updatedFlyer.flyer_id).toBe(1);
expect(updatedFlyer.image_url).toBe('https://example.com/flyer1.jpg');
expect(updatedFlyer.image_url).toBe(`${FLYER_BASE_URL}/flyer1.jpg`);
});
it('should preserve store_id when updating store name', () => {

View File

@@ -1,87 +1,286 @@
import { useState, useEffect, useCallback } from 'react';
import type { Step, CallBackProps } from 'react-joyride';
import { useEffect, useCallback, useRef } from 'react';
import { driver, Driver, DriveStep } from 'driver.js';
import 'driver.js/dist/driver.css';
const ONBOARDING_STORAGE_KEY = 'flyer_crawler_onboarding_completed';
export const useOnboardingTour = () => {
const [runTour, setRunTour] = useState(false);
const [stepIndex, setStepIndex] = useState(0);
// Custom CSS to match design system: pastel colors, sharp borders, minimalist
const DRIVER_CSS = `
.driver-popover {
background-color: #f0fdfa !important;
border: 2px solid #0d9488 !important;
border-radius: 0 !important;
box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06) !important;
max-width: 320px !important;
}
.driver-popover-title {
color: #134e4a !important;
font-size: 1rem !important;
font-weight: 600 !important;
margin-bottom: 0.5rem !important;
}
.driver-popover-description {
color: #1f2937 !important;
font-size: 0.875rem !important;
line-height: 1.5 !important;
}
.driver-popover-progress-text {
color: #0d9488 !important;
font-size: 0.75rem !important;
font-weight: 500 !important;
}
.driver-popover-navigation-btns {
gap: 0.5rem !important;
}
.driver-popover-prev-btn,
.driver-popover-next-btn {
background-color: #14b8a6 !important;
color: white !important;
border: 1px solid #0d9488 !important;
border-radius: 0 !important;
padding: 0.5rem 1rem !important;
font-size: 0.875rem !important;
font-weight: 500 !important;
transition: background-color 0.15s ease !important;
}
.driver-popover-prev-btn:hover,
.driver-popover-next-btn:hover {
background-color: #115e59 !important;
}
.driver-popover-prev-btn {
background-color: #ccfbf1 !important;
color: #134e4a !important;
}
.driver-popover-prev-btn:hover {
background-color: #99f6e4 !important;
}
.driver-popover-close-btn {
color: #0d9488 !important;
font-size: 1.25rem !important;
}
.driver-popover-close-btn:hover {
color: #115e59 !important;
}
.driver-popover-arrow-side-left,
.driver-popover-arrow-side-right,
.driver-popover-arrow-side-top,
.driver-popover-arrow-side-bottom {
border-color: #0d9488 !important;
}
.driver-popover-arrow-side-left::after,
.driver-popover-arrow-side-right::after,
.driver-popover-arrow-side-top::after,
.driver-popover-arrow-side-bottom::after {
border-color: transparent !important;
}
.driver-popover-arrow-side-left::before {
border-right-color: #f0fdfa !important;
}
.driver-popover-arrow-side-right::before {
border-left-color: #f0fdfa !important;
}
.driver-popover-arrow-side-top::before {
border-bottom-color: #f0fdfa !important;
}
.driver-popover-arrow-side-bottom::before {
border-top-color: #f0fdfa !important;
}
.driver-overlay {
background-color: rgba(0, 0, 0, 0.5) !important;
}
.driver-active-element {
box-shadow: 0 0 0 4px #14b8a6 !important;
}
/* Dark mode support */
@media (prefers-color-scheme: dark) {
.driver-popover {
background-color: #1f2937 !important;
border-color: #14b8a6 !important;
}
.driver-popover-title {
color: #ccfbf1 !important;
}
.driver-popover-description {
color: #e5e7eb !important;
}
.driver-popover-arrow-side-left::before {
border-right-color: #1f2937 !important;
}
.driver-popover-arrow-side-right::before {
border-left-color: #1f2937 !important;
}
.driver-popover-arrow-side-top::before {
border-bottom-color: #1f2937 !important;
}
.driver-popover-arrow-side-bottom::before {
border-top-color: #1f2937 !important;
}
}
`;
const tourSteps: DriveStep[] = [
{
element: '[data-tour="flyer-uploader"]',
popover: {
title: 'Upload Flyers',
description:
'Upload a grocery flyer here by clicking or dragging a PDF/image file. Our AI will extract prices and items automatically.',
side: 'bottom',
align: 'start',
},
},
{
element: '[data-tour="extracted-data-table"]',
popover: {
title: 'Extracted Items',
description:
'View all extracted items from your flyers here. You can watch items to track price changes and deals.',
side: 'top',
align: 'start',
},
},
{
element: '[data-tour="watch-button"]',
popover: {
title: 'Watch Items',
description:
'Click the eye icon to watch items and get notified when prices drop or deals appear.',
side: 'left',
align: 'start',
},
},
{
element: '[data-tour="watched-items"]',
popover: {
title: 'Watched Items',
description:
'Your watched items appear here. Track prices across different stores and get deal alerts.',
side: 'left',
align: 'start',
},
},
{
element: '[data-tour="price-chart"]',
popover: {
title: 'Active Deals',
description:
'Active deals show here with price comparisons. See which store has the best price!',
side: 'left',
align: 'start',
},
},
{
element: '[data-tour="shopping-list"]',
popover: {
title: 'Shopping Lists',
description:
'Create shopping lists from your watched items and get the best prices automatically.',
side: 'left',
align: 'start',
},
},
];
// Inject custom styles into the document head
const injectStyles = () => {
const styleId = 'driver-js-custom-styles';
if (!document.getElementById(styleId)) {
const style = document.createElement('style');
style.id = styleId;
style.textContent = DRIVER_CSS;
document.head.appendChild(style);
}
};
export const useOnboardingTour = () => {
const driverRef = useRef<Driver | null>(null);
const markTourComplete = useCallback(() => {
localStorage.setItem(ONBOARDING_STORAGE_KEY, 'true');
}, []);
const startTour = useCallback(() => {
injectStyles();
if (driverRef.current) {
driverRef.current.destroy();
}
driverRef.current = driver({
showProgress: true,
steps: tourSteps,
nextBtnText: 'Next',
prevBtnText: 'Previous',
doneBtnText: 'Done',
progressText: 'Step {{current}} of {{total}}',
onDestroyed: () => {
markTourComplete();
},
});
driverRef.current.drive();
}, [markTourComplete]);
const skipTour = useCallback(() => {
if (driverRef.current) {
driverRef.current.destroy();
}
markTourComplete();
}, [markTourComplete]);
const replayTour = useCallback(() => {
startTour();
}, [startTour]);
// Auto-start tour on mount if not completed
useEffect(() => {
const hasCompletedOnboarding = localStorage.getItem(ONBOARDING_STORAGE_KEY);
if (!hasCompletedOnboarding) {
setRunTour(true);
// Small delay to ensure DOM elements are mounted
const timer = setTimeout(() => {
startTour();
}, 500);
return () => clearTimeout(timer);
}
}, []);
}, [startTour]);
const steps: Step[] = [
{
target: '[data-tour="flyer-uploader"]',
content:
'Upload a grocery flyer here by clicking or dragging a PDF/image file. Our AI will extract prices and items automatically.',
disableBeacon: true,
placement: 'bottom',
},
{
target: '[data-tour="extracted-data-table"]',
content:
'View all extracted items from your flyers here. You can watch items to track price changes and deals.',
placement: 'top',
},
{
target: '[data-tour="watch-button"]',
content:
'Click the eye icon to watch items and get notified when prices drop or deals appear.',
placement: 'left',
},
{
target: '[data-tour="watched-items"]',
content:
'Your watched items appear here. Track prices across different stores and get deal alerts.',
placement: 'left',
},
{
target: '[data-tour="price-chart"]',
content: 'Active deals show here with price comparisons. See which store has the best price!',
placement: 'left',
},
{
target: '[data-tour="shopping-list"]',
content:
'Create shopping lists from your watched items and get the best prices automatically.',
placement: 'left',
},
];
const handleJoyrideCallback = useCallback((data: CallBackProps) => {
const { status, index } = data;
if (status === 'finished' || status === 'skipped') {
localStorage.setItem(ONBOARDING_STORAGE_KEY, 'true');
setRunTour(false);
setStepIndex(0);
} else if (data.action === 'next' || data.action === 'prev') {
setStepIndex(index + (data.action === 'next' ? 1 : 0));
}
}, []);
const skipTour = useCallback(() => {
localStorage.setItem(ONBOARDING_STORAGE_KEY, 'true');
setRunTour(false);
setStepIndex(0);
}, []);
const replayTour = useCallback(() => {
setStepIndex(0);
setRunTour(true);
// Cleanup on unmount
useEffect(() => {
return () => {
if (driverRef.current) {
driverRef.current.destroy();
}
};
}, []);
return {
runTour,
steps,
stepIndex,
handleJoyrideCallback,
skipTour,
replayTour,
startTour,
};
};

View File

@@ -195,7 +195,7 @@ describe('MainLayout Component', () => {
onOpenProfile: mockOnOpenProfile,
};
it('renders all main sections and the Outlet content', () => {
it('renders all main sections and the Outlet content for unauthenticated users', () => {
renderWithRouter(<MainLayout {...defaultProps} />);
expect(screen.getByTestId('flyer-list')).toBeInTheDocument();
@@ -203,9 +203,10 @@ describe('MainLayout Component', () => {
expect(screen.getByTestId('shopping-list')).toBeInTheDocument();
expect(screen.getByTestId('watched-items-list')).toBeInTheDocument();
expect(screen.getByTestId('price-chart')).toBeInTheDocument();
expect(screen.getByTestId('price-history-chart')).toBeInTheDocument();
expect(screen.getByTestId('leaderboard')).toBeInTheDocument();
expect(screen.getByTestId('activity-log')).toBeInTheDocument();
// Auth-gated components should NOT be present for unauthenticated users
expect(screen.queryByTestId('price-history-chart')).not.toBeInTheDocument();
expect(screen.queryByTestId('leaderboard')).not.toBeInTheDocument();
expect(screen.queryByTestId('activity-log')).not.toBeInTheDocument();
expect(screen.getByText('Outlet Content')).toBeInTheDocument();
});
@@ -235,6 +236,44 @@ describe('MainLayout Component', () => {
renderWithRouter(<MainLayout {...defaultProps} />);
expect(screen.queryByTestId('anonymous-banner')).not.toBeInTheDocument();
});
it('renders auth-gated components (PriceHistoryChart, Leaderboard, ActivityLog)', () => {
renderWithRouter(<MainLayout {...defaultProps} />);
expect(screen.getByTestId('price-history-chart')).toBeInTheDocument();
expect(screen.getByTestId('leaderboard')).toBeInTheDocument();
expect(screen.getByTestId('activity-log')).toBeInTheDocument();
});
it('calls setActiveListId when a list is shared via ActivityLog and the list exists', () => {
mockedUseShoppingLists.mockReturnValue({
...defaultUseShoppingListsReturn,
shoppingLists: [
createMockShoppingList({ shopping_list_id: 1, name: 'My List', user_id: 'user-123' }),
],
});
renderWithRouter(<MainLayout {...defaultProps} />);
const activityLog = screen.getByTestId('activity-log');
fireEvent.click(activityLog);
expect(mockSetActiveListId).toHaveBeenCalledWith(1);
});
it('does not call setActiveListId for actions other than list_shared', () => {
renderWithRouter(<MainLayout {...defaultProps} />);
const otherLogAction = screen.getByTestId('activity-log-other');
fireEvent.click(otherLogAction);
expect(mockSetActiveListId).not.toHaveBeenCalled();
});
it('does not call setActiveListId if the shared list does not exist', () => {
renderWithRouter(<MainLayout {...defaultProps} />);
const activityLog = screen.getByTestId('activity-log');
fireEvent.click(activityLog); // Mock click simulates sharing list with id 1
expect(mockSetActiveListId).not.toHaveBeenCalled();
});
});
describe('Error Handling', () => {
@@ -285,37 +324,6 @@ describe('MainLayout Component', () => {
});
describe('Event Handlers', () => {
it('calls setActiveListId when a list is shared via ActivityLog and the list exists', () => {
mockedUseShoppingLists.mockReturnValue({
...defaultUseShoppingListsReturn,
shoppingLists: [
createMockShoppingList({ shopping_list_id: 1, name: 'My List', user_id: 'user-123' }),
],
});
renderWithRouter(<MainLayout {...defaultProps} />);
const activityLog = screen.getByTestId('activity-log');
fireEvent.click(activityLog);
expect(mockSetActiveListId).toHaveBeenCalledWith(1);
});
it('does not call setActiveListId for actions other than list_shared', () => {
renderWithRouter(<MainLayout {...defaultProps} />);
const otherLogAction = screen.getByTestId('activity-log-other');
fireEvent.click(otherLogAction);
expect(mockSetActiveListId).not.toHaveBeenCalled();
});
it('does not call setActiveListId if the shared list does not exist', () => {
renderWithRouter(<MainLayout {...defaultProps} />);
const activityLog = screen.getByTestId('activity-log');
fireEvent.click(activityLog); // Mock click simulates sharing list with id 1
expect(mockSetActiveListId).not.toHaveBeenCalled();
});
it('calls addItemToList when an item is added from ShoppingListComponent and a list is active', () => {
const mockAddItemToList = vi.fn();
mockedUseShoppingLists.mockReturnValue({

View File

@@ -1,7 +1,6 @@
// src/layouts/MainLayout.tsx
import React, { useCallback } from 'react';
import { Outlet } from 'react-router-dom';
import Joyride from 'react-joyride';
import { useAuth } from '../hooks/useAuth';
import { useOnboardingTour } from '../hooks/useOnboardingTour';
import { useFlyers } from '../hooks/useFlyers';
@@ -34,7 +33,8 @@ export const MainLayout: React.FC<MainLayoutProps> = ({
}) => {
const { userProfile, authStatus } = useAuth();
const user = userProfile?.user ?? null;
const { runTour, steps, stepIndex, handleJoyrideCallback } = useOnboardingTour();
// Driver.js tour is initialized and managed imperatively inside the hook
useOnboardingTour();
const { flyers, refetchFlyers, flyersError } = useFlyers();
const { masterItems, error: masterItemsError } = useMasterItems();
const {
@@ -99,22 +99,6 @@ export const MainLayout: React.FC<MainLayoutProps> = ({
return (
<main className="max-w-screen-2xl mx-auto py-4 px-2.5 sm:py-6 lg:py-8">
<Joyride
steps={steps}
run={runTour}
stepIndex={stepIndex}
callback={handleJoyrideCallback}
continuous
showProgress
showSkipButton
styles={{
options: {
primaryColor: '#14b8a6',
textColor: '#1f2937',
zIndex: 10000,
},
}}
/>
{shouldShowBanner && (
<div className="max-w-5xl mx-auto mb-6 px-4 lg:px-0">
<AnonymousUserBanner onOpenProfile={onOpenProfile} />
@@ -171,9 +155,15 @@ export const MainLayout: React.FC<MainLayoutProps> = ({
unitSystem={'imperial'} // This can be passed down or sourced from a context
user={user}
/>
<PriceHistoryChart />
<Leaderboard />
<ActivityLog userProfile={userProfile} onLogClick={handleActivityLogClick} />
{user && (
<>
<PriceHistoryChart />
<Leaderboard />
{userProfile?.role === 'admin' && (
<ActivityLog userProfile={userProfile} onLogClick={handleActivityLogClick} />
)}
</>
)}
</>
</div>
</div>

View File

@@ -71,6 +71,8 @@ export const AuthProvider: React.FC<{ children: ReactNode }> = ({ children }) =>
const login = useCallback(
async (token: string, profileData?: UserProfile) => {
logger.info(`[AuthProvider-Login] Attempting login.`);
console.log('[AuthProvider-Login] Received token:', token);
console.log('[AuthProvider-Login] Received profileData:', profileData);
setToken(token);
if (profileData) {

View File

@@ -1,12 +1,15 @@
// src/schemas/flyer.schemas.test.ts
import { describe, it, expect } from 'vitest';
import { flyerInsertSchema, flyerDbInsertSchema } from './flyer.schemas';
import { getFlyerBaseUrl } from '../tests/utils/testHelpers';
const FLYER_BASE_URL = getFlyerBaseUrl();
describe('flyerInsertSchema', () => {
const validFlyer = {
file_name: 'flyer.jpg',
image_url: 'https://example.com/flyer.jpg',
icon_url: 'https://example.com/icon.jpg',
image_url: `${FLYER_BASE_URL}/flyer.jpg`,
icon_url: `${FLYER_BASE_URL}/icon.jpg`,
checksum: 'a'.repeat(64),
store_name: 'Test Store',
valid_from: '2023-01-01T00:00:00Z',
@@ -128,8 +131,8 @@ describe('flyerInsertSchema', () => {
describe('flyerDbInsertSchema', () => {
const validDbFlyer = {
file_name: 'flyer.jpg',
image_url: 'https://example.com/flyer.jpg',
icon_url: 'https://example.com/icon.jpg',
image_url: `${FLYER_BASE_URL}/flyer.jpg`,
icon_url: `${FLYER_BASE_URL}/icon.jpg`,
checksum: 'a'.repeat(64),
store_id: 1,
valid_from: '2023-01-01T00:00:00Z',

View File

@@ -62,16 +62,21 @@ const _performTokenRefresh = async (): Promise<string> => {
// This endpoint relies on the HttpOnly cookie, so no body is needed.
headers: { 'Content-Type': 'application/json' },
});
const data = await response.json();
const result = await response.json();
if (!response.ok) {
throw new Error(data.message || 'Failed to refresh token.');
throw new Error(result.error?.message || result.message || 'Failed to refresh token.');
}
// The API returns {success, data: {token}}, so extract the token
const token = result.data?.token;
if (!token) {
throw new Error('No token received from refresh endpoint.');
}
// On successful refresh, store the new access token.
if (typeof window !== 'undefined') {
localStorage.setItem('authToken', data.token);
localStorage.setItem('authToken', token);
}
logger.info('Successfully refreshed access token.');
return data.token;
return token;
} catch (error) {
logger.error({ error }, 'Failed to refresh token. User session has expired.');
// Only perform browser-specific actions if in the browser environment.

View File

@@ -4,6 +4,9 @@ import { Job, UnrecoverableError } from 'bullmq';
import path from 'node:path';
import type { FlyerInsert } from '../types';
import type { CleanupJobData, FlyerJobData } from '../types/job-data';
import { getFlyerBaseUrl } from '../tests/utils/testHelpers';
const FLYER_BASE_URL = getFlyerBaseUrl();
// 1. Create hoisted mocks FIRST
const mocks = vi.hoisted(() => ({
@@ -840,8 +843,8 @@ describe('FlyerProcessingService', () => {
const job = createMockCleanupJob({ flyerId: 1, paths: [] }); // Empty paths
const mockFlyer = createMockFlyer({
image_url: 'https://example.com/flyer-images/flyer-abc.jpg',
icon_url: 'https://example.com/flyer-images/icons/icon-flyer-abc.webp',
image_url: `${FLYER_BASE_URL}/flyer-images/flyer-abc.jpg`,
icon_url: `${FLYER_BASE_URL}/flyer-images/icons/icon-flyer-abc.webp`,
});
// Mock DB call to return a flyer
vi.mocked(mockedDb.flyerRepo.getFlyerById).mockResolvedValue(mockFlyer);

View File

@@ -21,6 +21,8 @@ const isProduction = process.env.NODE_ENV === 'production';
const isTest = process.env.NODE_ENV === 'test';
const isStaging = process.env.NODE_ENV === 'staging';
const isDevelopment = !isProduction && !isTest && !isStaging;
// PM2_HOME is set when running under PM2 process manager
const isRunningUnderPM2 = !!process.env.PM2_HOME;
// Determine log directory based on environment
// In production/test, use the application directory's logs folder
@@ -73,8 +75,9 @@ const redactConfig = {
const createLogger = (): pino.Logger => {
const logDir = ensureLogDirectory();
// Development: Use pino-pretty for human-readable output
if (isDevelopment) {
// Development WITHOUT PM2: Use pino-pretty for human-readable console output
// This is for local development outside of containers (rare case now)
if (isDevelopment && !isRunningUnderPM2) {
return pino({
level: 'debug',
transport: {
@@ -89,6 +92,15 @@ const createLogger = (): pino.Logger => {
});
}
// Development WITH PM2 (ADR-014): Output JSON for Logstash integration
// PM2 captures stdout to /var/log/pm2/*.log files which Logstash parses
if (isDevelopment && isRunningUnderPM2) {
return pino({
level: 'debug',
redact: redactConfig,
});
}
// Production/Test: Write to both stdout and file
if (logDir) {
const logFilePath = path.join(logDir, 'app.log');

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

View File

@@ -1,14 +1,18 @@
// src/tests/e2e/admin-authorization.e2e.test.ts
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import * as apiClient from '../../services/apiClient';
import supertest from 'supertest';
import { cleanupDb } from '../utils/cleanup';
import { createAndLoginUser } from '../utils/testHelpers';
import type { UserProfile } from '../../types';
import { getServerUrl } from '../setup/e2e-global-setup';
/**
* @vitest-environment node
*/
describe('Admin Route Authorization', () => {
// Create a getter function that returns supertest instance with the app
const getRequest = () => supertest(getServerUrl());
let regularUser: UserProfile;
let regularUserAuthToken: string;
@@ -17,6 +21,7 @@ describe('Admin Route Authorization', () => {
const { user, token } = await createAndLoginUser({
email: `e2e-authz-user-${Date.now()}@example.com`,
fullName: 'E2E AuthZ User',
request: getRequest(),
});
regularUser = user;
regularUserAuthToken = token;
@@ -33,48 +38,42 @@ describe('Admin Route Authorization', () => {
const adminEndpoints = [
{
method: 'GET',
path: '/admin/stats',
action: (token: string) => apiClient.getApplicationStats(token),
path: '/api/admin/stats',
},
{
method: 'GET',
path: '/admin/users',
action: (token: string) => apiClient.authedGet('/admin/users', { tokenOverride: token }),
path: '/api/admin/users',
},
{
method: 'GET',
path: '/admin/corrections',
action: (token: string) => apiClient.getSuggestedCorrections(token),
path: '/api/admin/corrections',
},
{
method: 'POST',
path: '/admin/corrections/1/approve',
action: (token: string) => apiClient.approveCorrection(1, token),
path: '/api/admin/corrections/1/approve',
},
{
method: 'POST',
path: '/admin/trigger/daily-deal-check',
action: (token: string) =>
apiClient.authedPostEmpty('/admin/trigger/daily-deal-check', { tokenOverride: token }),
path: '/api/admin/trigger/daily-deal-check',
},
{
method: 'GET',
path: '/admin/queues/status',
action: (token: string) =>
apiClient.authedGet('/admin/queues/status', { tokenOverride: token }),
path: '/api/admin/queues/status',
},
];
it.each(adminEndpoints)(
'should return 403 Forbidden for a regular user trying to access $method $path',
async ({ action }) => {
async ({ method, path }) => {
// Act: Attempt to access the admin endpoint with the regular user's token
const response = await action(regularUserAuthToken);
const requestBuilder = method === 'GET' ? getRequest().get(path) : getRequest().post(path);
const response = await requestBuilder
.set('Authorization', `Bearer ${regularUserAuthToken}`)
.send();
// Assert: The request should be forbidden
expect(response.status).toBe(403);
const responseBody = await response.json();
expect(responseBody.error.message).toBe('Forbidden: Administrator access required.');
expect(response.body.error.message).toBe('Forbidden: Administrator access required.');
},
);
});

View File

@@ -1,14 +1,18 @@
// src/tests/e2e/admin-dashboard.e2e.test.ts
import { describe, it, expect, afterAll } from 'vitest';
import * as apiClient from '../../services/apiClient';
import supertest from 'supertest';
import { getPool } from '../../services/db/connection.db';
import { cleanupDb } from '../utils/cleanup';
import { getServerUrl } from '../setup/e2e-global-setup';
/**
* @vitest-environment node
*/
describe('E2E Admin Dashboard Flow', () => {
// Create a getter function that returns supertest instance with the app
const getRequest = () => supertest(getServerUrl());
// Use a unique email for every run to avoid collisions
const uniqueId = Date.now();
const adminEmail = `e2e-admin-${uniqueId}@example.com`;
@@ -26,15 +30,12 @@ describe('E2E Admin Dashboard Flow', () => {
it('should allow an admin to log in and access dashboard features', async () => {
// 1. Register a new user (initially a regular user)
const registerResponse = await apiClient.registerUser(
adminEmail,
adminPassword,
'E2E Admin User',
);
const registerResponse = await getRequest()
.post('/api/auth/register')
.send({ email: adminEmail, password: adminPassword, full_name: 'E2E Admin User' });
expect(registerResponse.status).toBe(201);
const registerResponseBody = await registerResponse.json();
const registeredUser = registerResponseBody.data.userprofile.user;
const registeredUser = registerResponse.body.data.userprofile.user;
adminUserId = registeredUser.user_id;
expect(adminUserId).toBeDefined();
@@ -49,49 +50,47 @@ describe('E2E Admin Dashboard Flow', () => {
// and to provide a buffer for any rate limits from previous tests.
await new Promise((resolve) => setTimeout(resolve, 2000));
const loginResponse = await apiClient.loginUser(adminEmail, adminPassword, false);
if (!loginResponse.ok) {
const errorText = await loginResponse.text();
throw new Error(`Failed to log in as admin: ${loginResponse.status} ${errorText}`);
}
const loginResponseBody = await loginResponse.json();
const loginResponse = await getRequest()
.post('/api/auth/login')
.send({ email: adminEmail, password: adminPassword, rememberMe: false });
expect(loginResponse.status).toBe(200);
authToken = loginResponseBody.data.token;
authToken = loginResponse.body.data.token;
expect(authToken).toBeDefined();
// Verify the role returned in the login response is now 'admin'
expect(loginResponseBody.data.userprofile.role).toBe('admin');
expect(loginResponse.body.data.userprofile.role).toBe('admin');
// 4. Fetch System Stats (Protected Admin Route)
const statsResponse = await apiClient.getApplicationStats(authToken);
const statsResponse = await getRequest()
.get('/api/admin/stats')
.set('Authorization', `Bearer ${authToken}`);
expect(statsResponse.status).toBe(200);
const statsResponseBody = await statsResponse.json();
expect(statsResponseBody.data).toHaveProperty('userCount');
expect(statsResponseBody.data).toHaveProperty('flyerCount');
expect(statsResponse.body.data).toHaveProperty('userCount');
expect(statsResponse.body.data).toHaveProperty('flyerCount');
// 5. Fetch User List (Protected Admin Route)
const usersResponse = await apiClient.authedGet('/admin/users', { tokenOverride: authToken });
const usersResponse = await getRequest()
.get('/api/admin/users')
.set('Authorization', `Bearer ${authToken}`);
expect(usersResponse.status).toBe(200);
const usersResponseBody = await usersResponse.json();
expect(usersResponseBody.data).toHaveProperty('users');
expect(usersResponseBody.data).toHaveProperty('total');
expect(Array.isArray(usersResponseBody.data.users)).toBe(true);
expect(usersResponse.body.data).toHaveProperty('users');
expect(usersResponse.body.data).toHaveProperty('total');
expect(Array.isArray(usersResponse.body.data.users)).toBe(true);
// The list should contain the admin user we just created
const self = usersResponseBody.data.users.find((u: any) => u.user_id === adminUserId);
const self = usersResponse.body.data.users.find((u: any) => u.user_id === adminUserId);
expect(self).toBeDefined();
// 6. Check Queue Status (Protected Admin Route)
const queueResponse = await apiClient.authedGet('/admin/queues/status', {
tokenOverride: authToken,
});
const queueResponse = await getRequest()
.get('/api/admin/queues/status')
.set('Authorization', `Bearer ${authToken}`);
expect(queueResponse.status).toBe(200);
const queueResponseBody = await queueResponse.json();
expect(Array.isArray(queueResponseBody.data)).toBe(true);
expect(Array.isArray(queueResponse.body.data)).toBe(true);
// Verify that the 'flyer-processing' queue is present in the status report
const flyerQueue = queueResponseBody.data.find((q: any) => q.name === 'flyer-processing');
const flyerQueue = queueResponse.body.data.find((q: any) => q.name === 'flyer-processing');
expect(flyerQueue).toBeDefined();
expect(flyerQueue.counts).toBeDefined();
});

View File

@@ -1,16 +1,19 @@
// src/tests/e2e/auth.e2e.test.ts
import { describe, it, expect, afterAll, beforeAll } from 'vitest';
import * as apiClient from '../../services/apiClient';
import supertest from 'supertest';
import { cleanupDb } from '../utils/cleanup';
import { poll } from '../utils/poll';
import { createAndLoginUser, TEST_PASSWORD } from '../utils/testHelpers';
import type { UserProfile } from '../../types';
import { getServerUrl } from '../setup/e2e-global-setup';
/**
* @vitest-environment node
*/
describe('Authentication E2E Flow', () => {
// Create a getter function that returns supertest instance with the app
const getRequest = () => supertest(getServerUrl());
let testUser: UserProfile;
let testUserAuthToken: string;
const createdUserIds: string[] = [];
@@ -21,6 +24,7 @@ describe('Authentication E2E Flow', () => {
const { user, token } = await createAndLoginUser({
email: `e2e-login-user-${Date.now()}@example.com`,
fullName: 'E2E Login User',
request: getRequest(),
});
testUserAuthToken = token;
testUser = user;
@@ -43,18 +47,19 @@ describe('Authentication E2E Flow', () => {
const fullName = 'E2E Register User';
// Act
const response = await apiClient.registerUser(email, TEST_PASSWORD, fullName);
const responseBody = await response.json();
const response = await getRequest()
.post('/api/auth/register')
.send({ email, password: TEST_PASSWORD, full_name: fullName });
// Assert
expect(response.status).toBe(201);
expect(responseBody.data.message).toBe('User registered successfully!');
expect(responseBody.data.userprofile).toBeDefined();
expect(responseBody.data.userprofile.user.email).toBe(email);
expect(responseBody.data.token).toBeTypeOf('string');
expect(response.body.data.message).toBe('User registered successfully!');
expect(response.body.data.userprofile).toBeDefined();
expect(response.body.data.userprofile.user.email).toBe(email);
expect(response.body.data.token).toBeTypeOf('string');
// Add to cleanup
createdUserIds.push(responseBody.data.userprofile.user.user_id);
createdUserIds.push(response.body.data.userprofile.user.user_id);
});
it('should fail to register a user with a weak password', async () => {
@@ -62,12 +67,13 @@ describe('Authentication E2E Flow', () => {
const weakPassword = '123';
// Act
const response = await apiClient.registerUser(email, weakPassword, 'Weak Pass User');
const responseBody = await response.json();
const response = await getRequest()
.post('/api/auth/register')
.send({ email, password: weakPassword, full_name: 'Weak Pass User' });
// Assert
expect(response.status).toBe(400);
expect(responseBody.error.details[0].message).toContain(
expect(response.body.error.details[0].message).toContain(
'Password must be at least 8 characters long.',
);
});
@@ -76,18 +82,20 @@ describe('Authentication E2E Flow', () => {
const email = `e2e-register-duplicate-${Date.now()}@example.com`;
// Act 1: Register the user successfully
const firstResponse = await apiClient.registerUser(email, TEST_PASSWORD, 'Duplicate User');
const firstResponseBody = await firstResponse.json();
const firstResponse = await getRequest()
.post('/api/auth/register')
.send({ email, password: TEST_PASSWORD, full_name: 'Duplicate User' });
expect(firstResponse.status).toBe(201);
createdUserIds.push(firstResponseBody.data.userprofile.user.user_id);
createdUserIds.push(firstResponse.body.data.userprofile.user.user_id);
// Act 2: Attempt to register the same user again
const secondResponse = await apiClient.registerUser(email, TEST_PASSWORD, 'Duplicate User');
const secondResponseBody = await secondResponse.json();
const secondResponse = await getRequest()
.post('/api/auth/register')
.send({ email, password: TEST_PASSWORD, full_name: 'Duplicate User' });
// Assert
expect(secondResponse.status).toBe(409); // Conflict
expect(secondResponseBody.error.message).toContain(
expect(secondResponse.body.error.message).toContain(
'A user with this email address already exists.',
);
});
@@ -96,32 +104,35 @@ describe('Authentication E2E Flow', () => {
describe('Login Flow', () => {
it('should successfully log in a registered user', async () => {
// Act: Attempt to log in with the user created in beforeAll
const response = await apiClient.loginUser(testUser.user.email, TEST_PASSWORD, false);
const responseBody = await response.json();
const response = await getRequest()
.post('/api/auth/login')
.send({ email: testUser.user.email, password: TEST_PASSWORD, rememberMe: false });
// Assert
expect(response.status).toBe(200);
expect(responseBody.data.userprofile).toBeDefined();
expect(responseBody.data.userprofile.user.email).toBe(testUser.user.email);
expect(responseBody.data.token).toBeTypeOf('string');
expect(response.body.data.userprofile).toBeDefined();
expect(response.body.data.userprofile.user.email).toBe(testUser.user.email);
expect(response.body.data.token).toBeTypeOf('string');
});
it('should fail to log in with an incorrect password', async () => {
// Act: Attempt to log in with the wrong password
const response = await apiClient.loginUser(testUser.user.email, 'wrong-password', false);
const responseBody = await response.json();
const response = await getRequest()
.post('/api/auth/login')
.send({ email: testUser.user.email, password: 'wrong-password', rememberMe: false });
// Assert
expect(response.status).toBe(401);
expect(responseBody.error.message).toBe('Incorrect email or password.');
expect(response.body.error.message).toBe('Incorrect email or password.');
});
it('should fail to log in with a non-existent email', async () => {
const response = await apiClient.loginUser('no-one-here@example.com', TEST_PASSWORD, false);
const responseBody = await response.json();
const response = await getRequest()
.post('/api/auth/login')
.send({ email: 'no-one-here@example.com', password: TEST_PASSWORD, rememberMe: false });
expect(response.status).toBe(401);
expect(responseBody.error.message).toBe('Incorrect email or password.');
expect(response.body.error.message).toBe('Incorrect email or password.');
});
it('should be able to access a protected route after logging in', async () => {
@@ -130,15 +141,16 @@ describe('Authentication E2E Flow', () => {
expect(token).toBeDefined();
// Act: Use the token to access a protected route
const profileResponse = await apiClient.getAuthenticatedUserProfile({ tokenOverride: token });
const responseBody = await profileResponse.json();
const response = await getRequest()
.get('/api/users/profile')
.set('Authorization', `Bearer ${token}`);
// Assert
expect(profileResponse.status).toBe(200);
expect(responseBody.data).toBeDefined();
expect(responseBody.data.user.user_id).toBe(testUser.user.user_id);
expect(responseBody.data.user.email).toBe(testUser.user.email);
expect(responseBody.data.role).toBe('user');
expect(response.status).toBe(200);
expect(response.body.data).toBeDefined();
expect(response.body.data.user.user_id).toBe(testUser.user.user_id);
expect(response.body.data.user.email).toBe(testUser.user.email);
expect(response.body.data.role).toBe('user');
});
it('should allow an authenticated user to update their profile', async () => {
@@ -152,23 +164,24 @@ describe('Authentication E2E Flow', () => {
};
// Act: Call the update endpoint
const updateResponse = await apiClient.updateUserProfile(profileUpdates, {
tokenOverride: token,
});
const updateResponseBody = await updateResponse.json();
const updateResponse = await getRequest()
.put('/api/users/profile')
.set('Authorization', `Bearer ${token}`)
.send(profileUpdates);
// Assert: Check the response from the update call
expect(updateResponse.status).toBe(200);
expect(updateResponseBody.data.full_name).toBe(profileUpdates.full_name);
expect(updateResponseBody.data.avatar_url).toBe(profileUpdates.avatar_url);
expect(updateResponse.body.data.full_name).toBe(profileUpdates.full_name);
expect(updateResponse.body.data.avatar_url).toBe(profileUpdates.avatar_url);
// Act 2: Fetch the profile again to verify persistence
const verifyResponse = await apiClient.getAuthenticatedUserProfile({ tokenOverride: token });
const verifyResponseBody = await verifyResponse.json();
const verifyResponse = await getRequest()
.get('/api/users/profile')
.set('Authorization', `Bearer ${token}`);
// Assert 2: Check the fetched data
expect(verifyResponseBody.data.full_name).toBe(profileUpdates.full_name);
expect(verifyResponseBody.data.avatar_url).toBe(profileUpdates.avatar_url);
expect(verifyResponse.body.data.full_name).toBe(profileUpdates.full_name);
expect(verifyResponse.body.data.avatar_url).toBe(profileUpdates.avatar_url);
});
});
@@ -176,27 +189,29 @@ describe('Authentication E2E Flow', () => {
it('should allow a user to reset their password and log in with the new one', async () => {
// Arrange: Create a user to reset the password for
const email = `e2e-reset-pass-${Date.now()}@example.com`;
const registerResponse = await apiClient.registerUser(
email,
TEST_PASSWORD,
'Reset Pass User',
);
const registerResponseBody = await registerResponse.json();
const registerResponse = await getRequest()
.post('/api/auth/register')
.send({ email, password: TEST_PASSWORD, full_name: 'Reset Pass User' });
expect(registerResponse.status).toBe(201);
createdUserIds.push(registerResponseBody.data.userprofile.user.user_id);
createdUserIds.push(registerResponse.body.data.userprofile.user.user_id);
// Poll until the user can log in, confirming the record has propagated.
await poll(
() => apiClient.loginUser(email, TEST_PASSWORD, false),
(response) => response.ok,
{ timeout: 10000, interval: 1000, description: 'user login after registration' },
);
// Verify user can log in (confirming registration completed)
let loginAttempts = 0;
let loginResponse;
while (loginAttempts < 10) {
loginResponse = await getRequest()
.post('/api/auth/login')
.send({ email, password: TEST_PASSWORD, rememberMe: false });
if (loginResponse.status === 200) break;
await new Promise((resolve) => setTimeout(resolve, 1000));
loginAttempts++;
}
expect(loginResponse?.status).toBe(200);
// Request password reset (do not poll, as this endpoint is rate-limited)
const forgotResponse = await apiClient.requestPasswordReset(email);
const forgotResponse = await getRequest().post('/api/auth/forgot-password').send({ email });
expect(forgotResponse.status).toBe(200);
const forgotResponseBody = await forgotResponse.json();
const resetToken = forgotResponseBody.data.token;
const resetToken = forgotResponse.body.data.token;
// Assert 1: Check that we received a token.
expect(
@@ -207,20 +222,22 @@ describe('Authentication E2E Flow', () => {
// Act 2: Use the token to set a new password.
const newPassword = 'my-new-e2e-password-!@#$';
const resetResponse = await apiClient.resetPassword(resetToken, newPassword);
const resetResponseBody = await resetResponse.json();
const resetResponse = await getRequest()
.post('/api/auth/reset-password')
.send({ token: resetToken, newPassword });
// Assert 2: Check for a successful password reset message.
expect(resetResponse.status).toBe(200);
expect(resetResponseBody.data.message).toBe('Password has been reset successfully.');
expect(resetResponse.body.data.message).toBe('Password has been reset successfully.');
// Act 3: Log in with the NEW password
const loginResponse = await apiClient.loginUser(email, newPassword, false);
const loginResponseBody = await loginResponse.json();
const newLoginResponse = await getRequest()
.post('/api/auth/login')
.send({ email, password: newPassword, rememberMe: false });
expect(loginResponse.status).toBe(200);
expect(loginResponseBody.data.userprofile).toBeDefined();
expect(loginResponseBody.data.userprofile.user.email).toBe(email);
expect(newLoginResponse.status).toBe(200);
expect(newLoginResponse.body.data.userprofile).toBeDefined();
expect(newLoginResponse.body.data.userprofile.user.email).toBe(email);
});
it('should return a generic success message for a non-existent email to prevent enumeration', async () => {
@@ -228,73 +245,71 @@ describe('Authentication E2E Flow', () => {
await new Promise((resolve) => setTimeout(resolve, 2000));
const nonExistentEmail = `non-existent-e2e-${Date.now()}@example.com`;
const response = await apiClient.requestPasswordReset(nonExistentEmail);
const response = await getRequest()
.post('/api/auth/forgot-password')
.send({ email: nonExistentEmail });
// Check for rate limiting or other errors before parsing JSON to avoid SyntaxError
if (!response.ok) {
const text = await response.text();
throw new Error(`Request failed with status ${response.status}: ${text}`);
}
const responseBody = await response.json();
expect(response.status).toBe(200);
expect(responseBody.data.message).toBe(
expect(response.body.data.message).toBe(
'If an account with that email exists, a password reset link has been sent.',
);
expect(responseBody.data.token).toBeUndefined();
expect(response.body.data.token).toBeUndefined();
});
});
describe('Token Refresh Flow', () => {
it('should allow an authenticated user to refresh their access token and use it', async () => {
// 1. Log in to get the refresh token cookie and an initial access token.
const loginResponse = await apiClient.loginUser(testUser.user.email, TEST_PASSWORD, false);
const loginResponse = await getRequest()
.post('/api/auth/login')
.send({ email: testUser.user.email, password: TEST_PASSWORD, rememberMe: false });
expect(loginResponse.status).toBe(200);
const loginResponseBody = await loginResponse.json();
const initialAccessToken = loginResponseBody.data.token;
const initialAccessToken = loginResponse.body.data.token;
// 2. Extract the refresh token from the 'set-cookie' header.
const setCookieHeader = loginResponse.headers.get('set-cookie');
const setCookieHeaders = loginResponse.headers['set-cookie'];
expect(
setCookieHeader,
setCookieHeaders,
'Set-Cookie header should be present in login response',
).toBeDefined();
// A typical Set-Cookie header might be 'refreshToken=...; Path=/; HttpOnly; Max-Age=...'. We just need the 'refreshToken=...' part.
const refreshTokenCookie = setCookieHeader!.split(';')[0];
// Find the refreshToken cookie
const refreshTokenCookie = Array.isArray(setCookieHeaders)
? setCookieHeaders.find((cookie: string) => cookie.startsWith('refreshToken='))
: setCookieHeaders;
expect(refreshTokenCookie).toBeDefined();
// Wait for >1 second to ensure the 'iat' (Issued At) claim in the new JWT changes.
// JWT timestamps have second-level precision.
await new Promise((resolve) => setTimeout(resolve, 1100));
// 3. Call the refresh token endpoint, passing the cookie.
// This assumes a new method in apiClient to handle this specific request.
const refreshResponse = await apiClient.refreshToken(refreshTokenCookie);
const refreshResponse = await getRequest()
.post('/api/auth/refresh-token')
.set('Cookie', refreshTokenCookie!);
// 4. Assert the refresh was successful and we got a new token.
expect(refreshResponse.status).toBe(200);
const refreshResponseBody = await refreshResponse.json();
const newAccessToken = refreshResponseBody.data.token;
const newAccessToken = refreshResponse.body.data.token;
expect(newAccessToken).toBeDefined();
expect(newAccessToken).not.toBe(initialAccessToken);
// 5. Use the new access token to access a protected route.
const profileResponse = await apiClient.getAuthenticatedUserProfile({
tokenOverride: newAccessToken,
});
const profileResponse = await getRequest()
.get('/api/users/profile')
.set('Authorization', `Bearer ${newAccessToken}`);
expect(profileResponse.status).toBe(200);
const profileResponseBody = await profileResponse.json();
expect(profileResponseBody.data.user.user_id).toBe(testUser.user.user_id);
expect(profileResponse.body.data.user.user_id).toBe(testUser.user.user_id);
});
it('should fail to refresh with an invalid or missing token', async () => {
// Case 1: No cookie provided. This assumes refreshToken can handle an empty string.
const noCookieResponse = await apiClient.refreshToken('');
// Case 1: No cookie provided
const noCookieResponse = await getRequest().post('/api/auth/refresh-token');
expect(noCookieResponse.status).toBe(401);
// Case 2: Invalid cookie provided
const invalidCookieResponse = await apiClient.refreshToken(
'refreshToken=invalid-garbage-token',
);
const invalidCookieResponse = await getRequest()
.post('/api/auth/refresh-token')
.set('Cookie', 'refreshToken=invalid-garbage-token');
expect(invalidCookieResponse.status).toBe(403);
});
});

View File

@@ -4,7 +4,7 @@
* Tests the complete flow from user registration to creating budgets, tracking spending, and managing finances.
*/
import { describe, it, expect, afterAll } from 'vitest';
import * as apiClient from '../../services/apiClient';
import supertest from 'supertest';
import { cleanupDb } from '../utils/cleanup';
import { poll } from '../utils/poll';
import { getPool } from '../../services/db/connection.db';
@@ -13,35 +13,16 @@ import {
cleanupStoreLocations,
type CreatedStoreLocation,
} from '../utils/storeHelpers';
import { getServerUrl } from '../setup/e2e-global-setup';
/**
* @vitest-environment node
*/
const API_BASE_URL = process.env.VITE_API_BASE_URL || 'http://localhost:3000/api';
// Helper to make authenticated API calls
const authedFetch = async (
path: string,
options: RequestInit & { token?: string } = {},
): Promise<Response> => {
const { token, ...fetchOptions } = options;
const headers: Record<string, string> = {
'Content-Type': 'application/json',
...(fetchOptions.headers as Record<string, string>),
};
if (token) {
headers['Authorization'] = `Bearer ${token}`;
}
return fetch(`${API_BASE_URL}${path}`, {
...fetchOptions,
headers,
});
};
describe('E2E Budget Management Journey', () => {
// Create a getter function that returns supertest instance with the app
const getRequest = () => supertest(getServerUrl());
const uniqueId = Date.now();
const userEmail = `budget-e2e-${uniqueId}@example.com`;
const userPassword = 'StrongBudgetPassword123!';
@@ -83,21 +64,23 @@ describe('E2E Budget Management Journey', () => {
it('should complete budget journey: Register -> Create Budget -> Track Spending -> Update -> Delete', async () => {
// Step 1: Register a new user
const registerResponse = await apiClient.registerUser(
userEmail,
userPassword,
'Budget E2E User',
);
const registerResponse = await getRequest().post('/api/auth/register').send({
email: userEmail,
password: userPassword,
full_name: 'Budget E2E User',
});
expect(registerResponse.status).toBe(201);
// Step 2: Login to get auth token
const { response: loginResponse, responseBody: loginResponseBody } = await poll(
async () => {
const response = await apiClient.loginUser(userEmail, userPassword, false);
const responseBody = response.ok ? await response.clone().json() : {};
const response = await getRequest()
.post('/api/auth/login')
.send({ email: userEmail, password: userPassword, rememberMe: false });
const responseBody = response.status === 200 ? response.body : {};
return { response, responseBody };
},
(result) => result.response.ok,
(result) => result.response.status === 200,
{ timeout: 10000, interval: 1000, description: 'user login after registration' },
);
@@ -111,73 +94,65 @@ describe('E2E Budget Management Journey', () => {
const startOfMonth = new Date(today.getFullYear(), today.getMonth(), 1);
const formatDate = (d: Date) => d.toISOString().split('T')[0];
const createBudgetResponse = await authedFetch('/budgets', {
method: 'POST',
token: authToken,
body: JSON.stringify({
const createBudgetResponse = await getRequest()
.post('/api/budgets')
.set('Authorization', `Bearer ${authToken}`)
.send({
name: 'Monthly Groceries',
amount_cents: 50000, // $500.00
period: 'monthly',
start_date: formatDate(startOfMonth),
}),
});
});
expect(createBudgetResponse.status).toBe(201);
const createBudgetData = await createBudgetResponse.json();
expect(createBudgetData.data.name).toBe('Monthly Groceries');
expect(createBudgetData.data.amount_cents).toBe(50000);
expect(createBudgetData.data.period).toBe('monthly');
const budgetId = createBudgetData.data.budget_id;
expect(createBudgetResponse.body.data.name).toBe('Monthly Groceries');
expect(createBudgetResponse.body.data.amount_cents).toBe(50000);
expect(createBudgetResponse.body.data.period).toBe('monthly');
const budgetId = createBudgetResponse.body.data.budget_id;
createdBudgetIds.push(budgetId);
// Step 4: Create a weekly budget
const weeklyBudgetResponse = await authedFetch('/budgets', {
method: 'POST',
token: authToken,
body: JSON.stringify({
const weeklyBudgetResponse = await getRequest()
.post('/api/budgets')
.set('Authorization', `Bearer ${authToken}`)
.send({
name: 'Weekly Dining Out',
amount_cents: 10000, // $100.00
period: 'weekly',
start_date: formatDate(today),
}),
});
});
expect(weeklyBudgetResponse.status).toBe(201);
const weeklyBudgetData = await weeklyBudgetResponse.json();
expect(weeklyBudgetData.data.period).toBe('weekly');
createdBudgetIds.push(weeklyBudgetData.data.budget_id);
expect(weeklyBudgetResponse.body.data.period).toBe('weekly');
createdBudgetIds.push(weeklyBudgetResponse.body.data.budget_id);
// Step 5: View all budgets
const listBudgetsResponse = await authedFetch('/budgets', {
method: 'GET',
token: authToken,
});
const listBudgetsResponse = await getRequest()
.get('/api/budgets')
.set('Authorization', `Bearer ${authToken}`);
expect(listBudgetsResponse.status).toBe(200);
const listBudgetsData = await listBudgetsResponse.json();
expect(listBudgetsData.data.length).toBe(2);
expect(listBudgetsResponse.body.data.length).toBe(2);
// Find our budgets
const monthlyBudget = listBudgetsData.data.find(
const monthlyBudget = listBudgetsResponse.body.data.find(
(b: { name: string }) => b.name === 'Monthly Groceries',
);
expect(monthlyBudget).toBeDefined();
expect(monthlyBudget.amount_cents).toBe(50000);
// Step 6: Update a budget
const updateBudgetResponse = await authedFetch(`/budgets/${budgetId}`, {
method: 'PUT',
token: authToken,
body: JSON.stringify({
const updateBudgetResponse = await getRequest()
.put(`/api/budgets/${budgetId}`)
.set('Authorization', `Bearer ${authToken}`)
.send({
amount_cents: 55000, // Increase to $550.00
name: 'Monthly Groceries (Updated)',
}),
});
});
expect(updateBudgetResponse.status).toBe(200);
const updateBudgetData = await updateBudgetResponse.json();
expect(updateBudgetData.data.amount_cents).toBe(55000);
expect(updateBudgetData.data.name).toBe('Monthly Groceries (Updated)');
expect(updateBudgetResponse.body.data.amount_cents).toBe(55000);
expect(updateBudgetResponse.body.data.name).toBe('Monthly Groceries (Updated)');
// Step 7: Create test spending data (receipts) to track against budget
const pool = getPool();
@@ -212,69 +187,67 @@ describe('E2E Budget Management Journey', () => {
// Step 8: Check spending analysis
const endOfMonth = new Date(today.getFullYear(), today.getMonth() + 1, 0);
const spendingResponse = await authedFetch(
`/budgets/spending-analysis?startDate=${formatDate(startOfMonth)}&endDate=${formatDate(endOfMonth)}`,
{
method: 'GET',
token: authToken,
},
);
const spendingResponse = await getRequest()
.get(
`/api/budgets/spending-analysis?startDate=${formatDate(startOfMonth)}&endDate=${formatDate(endOfMonth)}`,
)
.set('Authorization', `Bearer ${authToken}`);
expect(spendingResponse.status).toBe(200);
const spendingData = await spendingResponse.json();
expect(spendingData.success).toBe(true);
expect(Array.isArray(spendingData.data)).toBe(true);
expect(spendingResponse.body.success).toBe(true);
expect(Array.isArray(spendingResponse.body.data)).toBe(true);
// Verify we have spending data
// Note: The spending might be $0 or have data depending on how the backend calculates spending
// The test is mainly verifying the endpoint works
// Step 9: Test budget validation - try to create invalid budget
const invalidBudgetResponse = await authedFetch('/budgets', {
method: 'POST',
token: authToken,
body: JSON.stringify({
const invalidBudgetResponse = await getRequest()
.post('/api/budgets')
.set('Authorization', `Bearer ${authToken}`)
.send({
name: 'Invalid Budget',
amount_cents: -100, // Negative amount should be rejected
period: 'monthly',
start_date: formatDate(today),
}),
});
});
expect(invalidBudgetResponse.status).toBe(400);
// Step 10: Test budget validation - missing required fields
const missingFieldsResponse = await authedFetch('/budgets', {
method: 'POST',
token: authToken,
body: JSON.stringify({
const missingFieldsResponse = await getRequest()
.post('/api/budgets')
.set('Authorization', `Bearer ${authToken}`)
.send({
name: 'Incomplete Budget',
// Missing amount_cents, period, start_date
}),
});
});
expect(missingFieldsResponse.status).toBe(400);
// Step 11: Test update validation - empty update
const emptyUpdateResponse = await authedFetch(`/budgets/${budgetId}`, {
method: 'PUT',
token: authToken,
body: JSON.stringify({}), // No fields to update
});
const emptyUpdateResponse = await getRequest()
.put(`/api/budgets/${budgetId}`)
.set('Authorization', `Bearer ${authToken}`)
.send({}); // No fields to update
expect(emptyUpdateResponse.status).toBe(400);
// Step 12: Verify another user cannot access our budgets
const otherUserEmail = `other-budget-e2e-${uniqueId}@example.com`;
await apiClient.registerUser(otherUserEmail, userPassword, 'Other Budget User');
await getRequest()
.post('/api/auth/register')
.send({ email: otherUserEmail, password: userPassword, full_name: 'Other Budget User' });
const { responseBody: otherLoginData } = await poll(
async () => {
const response = await apiClient.loginUser(otherUserEmail, userPassword, false);
const responseBody = response.ok ? await response.clone().json() : {};
const response = await getRequest()
.post('/api/auth/login')
.send({ email: otherUserEmail, password: userPassword, rememberMe: false });
const responseBody = response.status === 200 ? response.body : {};
return { response, responseBody };
},
(result) => result.response.ok,
(result) => result.response.status === 200,
{ timeout: 10000, interval: 1000, description: 'other user login' },
);
@@ -282,31 +255,27 @@ describe('E2E Budget Management Journey', () => {
const otherUserId = otherLoginData.data.userprofile.user.user_id;
// Other user should not see our budgets
const otherBudgetsResponse = await authedFetch('/budgets', {
method: 'GET',
token: otherToken,
});
const otherBudgetsResponse = await getRequest()
.get('/api/budgets')
.set('Authorization', `Bearer ${otherToken}`);
expect(otherBudgetsResponse.status).toBe(200);
const otherBudgetsData = await otherBudgetsResponse.json();
expect(otherBudgetsData.data.length).toBe(0);
expect(otherBudgetsResponse.body.data.length).toBe(0);
// Other user should not be able to update our budget
const otherUpdateResponse = await authedFetch(`/budgets/${budgetId}`, {
method: 'PUT',
token: otherToken,
body: JSON.stringify({
const otherUpdateResponse = await getRequest()
.put(`/api/budgets/${budgetId}`)
.set('Authorization', `Bearer ${otherToken}`)
.send({
amount_cents: 99999,
}),
});
});
expect(otherUpdateResponse.status).toBe(404); // Should not find the budget
// Other user should not be able to delete our budget
const otherDeleteAttemptResponse = await authedFetch(`/budgets/${budgetId}`, {
method: 'DELETE',
token: otherToken,
});
const otherDeleteAttemptResponse = await getRequest()
.delete(`/api/budgets/${budgetId}`)
.set('Authorization', `Bearer ${otherToken}`);
expect(otherDeleteAttemptResponse.status).toBe(404);
@@ -314,38 +283,36 @@ describe('E2E Budget Management Journey', () => {
await cleanupDb({ userIds: [otherUserId] });
// Step 13: Delete the weekly budget
const deleteBudgetResponse = await authedFetch(`/budgets/${weeklyBudgetData.data.budget_id}`, {
method: 'DELETE',
token: authToken,
});
const deleteBudgetResponse = await getRequest()
.delete(`/api/budgets/${weeklyBudgetResponse.body.data.budget_id}`)
.set('Authorization', `Bearer ${authToken}`);
expect(deleteBudgetResponse.status).toBe(204);
// Remove from cleanup list
const deleteIndex = createdBudgetIds.indexOf(weeklyBudgetData.data.budget_id);
const deleteIndex = createdBudgetIds.indexOf(weeklyBudgetResponse.body.data.budget_id);
if (deleteIndex > -1) {
createdBudgetIds.splice(deleteIndex, 1);
}
// Step 14: Verify deletion
const verifyDeleteResponse = await authedFetch('/budgets', {
method: 'GET',
token: authToken,
});
const verifyDeleteResponse = await getRequest()
.get('/api/budgets')
.set('Authorization', `Bearer ${authToken}`);
expect(verifyDeleteResponse.status).toBe(200);
const verifyDeleteData = await verifyDeleteResponse.json();
expect(verifyDeleteData.data.length).toBe(1); // Only monthly budget remains
expect(verifyDeleteResponse.body.data.length).toBe(1); // Only monthly budget remains
const deletedBudget = verifyDeleteData.data.find(
(b: { budget_id: number }) => b.budget_id === weeklyBudgetData.data.budget_id,
const deletedBudget = verifyDeleteResponse.body.data.find(
(b: { budget_id: number }) => b.budget_id === weeklyBudgetResponse.body.data.budget_id,
);
expect(deletedBudget).toBeUndefined();
// Step 15: Delete account
const deleteAccountResponse = await apiClient.deleteUserAccount(userPassword, {
tokenOverride: authToken,
});
const deleteAccountResponse = await getRequest()
.delete('/api/users/account')
.set('Authorization', `Bearer ${authToken}`)
.send({ password: userPassword });
expect(deleteAccountResponse.status).toBe(200);
userId = null;

View File

@@ -4,7 +4,7 @@
* Tests the complete flow from user registration to watching items and viewing best prices.
*/
import { describe, it, expect, afterAll } from 'vitest';
import * as apiClient from '../../services/apiClient';
import supertest from 'supertest';
import { cleanupDb } from '../utils/cleanup';
import { poll } from '../utils/poll';
import { getPool } from '../../services/db/connection.db';
@@ -13,35 +13,16 @@ import {
cleanupStoreLocations,
type CreatedStoreLocation,
} from '../utils/storeHelpers';
import { getServerUrl } from '../setup/e2e-global-setup';
/**
* @vitest-environment node
*/
const API_BASE_URL = process.env.VITE_API_BASE_URL || 'http://localhost:3000/api';
// Helper to make authenticated API calls
const authedFetch = async (
path: string,
options: RequestInit & { token?: string } = {},
): Promise<Response> => {
const { token, ...fetchOptions } = options;
const headers: Record<string, string> = {
'Content-Type': 'application/json',
...(fetchOptions.headers as Record<string, string>),
};
if (token) {
headers['Authorization'] = `Bearer ${token}`;
}
return fetch(`${API_BASE_URL}${path}`, {
...fetchOptions,
headers,
});
};
describe('E2E Deals and Price Tracking Journey', () => {
// Create a getter function that returns supertest instance with the app
const getRequest = () => supertest(getServerUrl());
const uniqueId = Date.now();
const userEmail = `deals-e2e-${uniqueId}@example.com`;
const userPassword = 'StrongDealsPassword123!';
@@ -98,87 +79,70 @@ describe('E2E Deals and Price Tracking Journey', () => {
// will support both category names and IDs in the watched items API.
// Get all available categories
const categoriesResponse = await authedFetch('/categories', {
method: 'GET',
});
const categoriesResponse = await getRequest().get('/api/categories');
expect(categoriesResponse.status).toBe(200);
const categoriesData = await categoriesResponse.json();
expect(categoriesData.success).toBe(true);
expect(categoriesData.data.length).toBeGreaterThan(0);
expect(categoriesResponse.body.success).toBe(true);
expect(categoriesResponse.body.data.length).toBeGreaterThan(0);
// Find "Dairy & Eggs" category by name using the lookup endpoint
const categoryLookupResponse = await authedFetch(
'/categories/lookup?name=' + encodeURIComponent('Dairy & Eggs'),
{
method: 'GET',
},
const categoryLookupResponse = await getRequest().get(
'/api/categories/lookup?name=' + encodeURIComponent('Dairy & Eggs'),
);
expect(categoryLookupResponse.status).toBe(200);
const categoryLookupData = await categoryLookupResponse.json();
expect(categoryLookupData.success).toBe(true);
expect(categoryLookupData.data.name).toBe('Dairy & Eggs');
expect(categoryLookupResponse.body.success).toBe(true);
expect(categoryLookupResponse.body.data.name).toBe('Dairy & Eggs');
const dairyEggsCategoryId = categoryLookupData.data.category_id;
const dairyEggsCategoryId = categoryLookupResponse.body.data.category_id;
expect(dairyEggsCategoryId).toBeGreaterThan(0);
// Verify we can retrieve the category by ID
const categoryByIdResponse = await authedFetch(`/categories/${dairyEggsCategoryId}`, {
method: 'GET',
});
const categoryByIdResponse = await getRequest().get(`/api/categories/${dairyEggsCategoryId}`);
expect(categoryByIdResponse.status).toBe(200);
const categoryByIdData = await categoryByIdResponse.json();
expect(categoryByIdData.success).toBe(true);
expect(categoryByIdData.data.category_id).toBe(dairyEggsCategoryId);
expect(categoryByIdData.data.name).toBe('Dairy & Eggs');
expect(categoryByIdResponse.body.success).toBe(true);
expect(categoryByIdResponse.body.data.category_id).toBe(dairyEggsCategoryId);
expect(categoryByIdResponse.body.data.name).toBe('Dairy & Eggs');
// Look up other category IDs we'll need
const bakeryResponse = await authedFetch(
'/categories/lookup?name=' + encodeURIComponent('Bakery & Bread'),
{ method: 'GET' },
const bakeryResponse = await getRequest().get(
'/api/categories/lookup?name=' + encodeURIComponent('Bakery & Bread'),
);
const bakeryData = await bakeryResponse.json();
const bakeryCategoryId = bakeryData.data.category_id;
const bakeryCategoryId = bakeryResponse.body.data.category_id;
const beveragesResponse = await authedFetch('/categories/lookup?name=Beverages', {
method: 'GET',
});
const beveragesData = await beveragesResponse.json();
const beveragesCategoryId = beveragesData.data.category_id;
const beveragesResponse = await getRequest().get('/api/categories/lookup?name=Beverages');
const beveragesCategoryId = beveragesResponse.body.data.category_id;
const produceResponse = await authedFetch(
'/categories/lookup?name=' + encodeURIComponent('Fruits & Vegetables'),
{ method: 'GET' },
const produceResponse = await getRequest().get(
'/api/categories/lookup?name=' + encodeURIComponent('Fruits & Vegetables'),
);
const produceData = await produceResponse.json();
const produceCategoryId = produceData.data.category_id;
const produceCategoryId = produceResponse.body.data.category_id;
const meatResponse = await authedFetch(
'/categories/lookup?name=' + encodeURIComponent('Meat & Seafood'),
{ method: 'GET' },
const meatResponse = await getRequest().get(
'/api/categories/lookup?name=' + encodeURIComponent('Meat & Seafood'),
);
const meatData = await meatResponse.json();
const meatCategoryId = meatData.data.category_id;
const meatCategoryId = meatResponse.body.data.category_id;
// NOTE: The watched items API now uses category_id (number) as of Phase 3.
// Category names are no longer accepted. Use the category discovery endpoints
// to look up category IDs before creating watched items.
// Step 1: Register a new user
const registerResponse = await apiClient.registerUser(
userEmail,
userPassword,
'Deals E2E User',
);
const registerResponse = await getRequest().post('/api/auth/register').send({
email: userEmail,
password: userPassword,
full_name: 'Deals E2E User',
});
expect(registerResponse.status).toBe(201);
// Step 2: Login to get auth token
const { response: loginResponse, responseBody: loginResponseBody } = await poll(
async () => {
const response = await apiClient.loginUser(userEmail, userPassword, false);
const responseBody = response.ok ? await response.clone().json() : {};
const response = await getRequest()
.post('/api/auth/login')
.send({ email: userEmail, password: userPassword, rememberMe: false });
const responseBody = response.status === 200 ? response.body : {};
return { response, responseBody };
},
(result) => result.response.ok,
(result) => result.response.status === 200,
{ timeout: 10000, interval: 1000, description: 'user login after registration' },
);
@@ -280,18 +244,16 @@ describe('E2E Deals and Price Tracking Journey', () => {
);
// Step 4: Add items to watch list (using category_id from lookups above)
const watchItem1Response = await authedFetch('/users/watched-items', {
method: 'POST',
token: authToken,
body: JSON.stringify({
const watchItem1Response = await getRequest()
.post('/api/users/watched-items')
.set('Authorization', `Bearer ${authToken}`)
.send({
itemName: 'E2E Milk 2%',
category_id: dairyEggsCategoryId,
}),
});
});
expect(watchItem1Response.status).toBe(201);
const watchItem1Data = await watchItem1Response.json();
expect(watchItem1Data.data.name).toBe('E2E Milk 2%');
expect(watchItem1Response.body.data.name).toBe('E2E Milk 2%');
// Add more items to watch list
const itemsToWatch = [
@@ -300,47 +262,42 @@ describe('E2E Deals and Price Tracking Journey', () => {
];
for (const item of itemsToWatch) {
const response = await authedFetch('/users/watched-items', {
method: 'POST',
token: authToken,
body: JSON.stringify(item),
});
const response = await getRequest()
.post('/api/users/watched-items')
.set('Authorization', `Bearer ${authToken}`)
.send(item);
expect(response.status).toBe(201);
}
// Step 5: View all watched items
const watchedListResponse = await authedFetch('/users/watched-items', {
method: 'GET',
token: authToken,
});
const watchedListResponse = await getRequest()
.get('/api/users/watched-items')
.set('Authorization', `Bearer ${authToken}`);
expect(watchedListResponse.status).toBe(200);
const watchedListData = await watchedListResponse.json();
expect(watchedListData.data.length).toBeGreaterThanOrEqual(3);
expect(watchedListResponse.body.data.length).toBeGreaterThanOrEqual(3);
// Find our watched items
const watchedMilk = watchedListData.data.find(
const watchedMilk = watchedListResponse.body.data.find(
(item: { name: string }) => item.name === 'E2E Milk 2%',
);
expect(watchedMilk).toBeDefined();
expect(watchedMilk.category_id).toBe(dairyEggsCategoryId);
// Step 6: Get best prices for watched items
const bestPricesResponse = await authedFetch('/deals/best-watched-prices', {
method: 'GET',
token: authToken,
});
const bestPricesResponse = await getRequest()
.get('/api/deals/best-watched-prices')
.set('Authorization', `Bearer ${authToken}`);
expect(bestPricesResponse.status).toBe(200);
const bestPricesData = await bestPricesResponse.json();
expect(bestPricesData.success).toBe(true);
expect(bestPricesResponse.body.success).toBe(true);
// Verify we got deals for our watched items
expect(Array.isArray(bestPricesData.data)).toBe(true);
expect(Array.isArray(bestPricesResponse.body.data)).toBe(true);
// Find the milk deal and verify it's the best price (Store 2 at $4.99)
if (bestPricesData.data.length > 0) {
const milkDeal = bestPricesData.data.find(
if (bestPricesResponse.body.data.length > 0) {
const milkDeal = bestPricesResponse.body.data.find(
(deal: { item_name: string }) => deal.item_name === 'E2E Milk 2%',
);
@@ -356,38 +313,39 @@ describe('E2E Deals and Price Tracking Journey', () => {
// Step 8: Remove an item from watch list
const milkMasterItemId = createdMasterItemIds[0];
const removeResponse = await authedFetch(`/users/watched-items/${milkMasterItemId}`, {
method: 'DELETE',
token: authToken,
});
const removeResponse = await getRequest()
.delete(`/api/users/watched-items/${milkMasterItemId}`)
.set('Authorization', `Bearer ${authToken}`);
expect(removeResponse.status).toBe(204);
// Step 9: Verify item was removed
const updatedWatchedListResponse = await authedFetch('/users/watched-items', {
method: 'GET',
token: authToken,
});
const updatedWatchedListResponse = await getRequest()
.get('/api/users/watched-items')
.set('Authorization', `Bearer ${authToken}`);
expect(updatedWatchedListResponse.status).toBe(200);
const updatedWatchedListData = await updatedWatchedListResponse.json();
const milkStillWatched = updatedWatchedListData.data.find(
const milkStillWatched = updatedWatchedListResponse.body.data.find(
(item: { item_name: string }) => item.item_name === 'E2E Milk 2%',
);
expect(milkStillWatched).toBeUndefined();
// Step 10: Verify another user cannot see our watched items
const otherUserEmail = `other-deals-e2e-${uniqueId}@example.com`;
await apiClient.registerUser(otherUserEmail, userPassword, 'Other Deals User');
await getRequest()
.post('/api/auth/register')
.send({ email: otherUserEmail, password: userPassword, full_name: 'Other Deals User' });
const { responseBody: otherLoginData } = await poll(
async () => {
const response = await apiClient.loginUser(otherUserEmail, userPassword, false);
const responseBody = response.ok ? await response.clone().json() : {};
const response = await getRequest()
.post('/api/auth/login')
.send({ email: otherUserEmail, password: userPassword, rememberMe: false });
const responseBody = response.status === 200 ? response.body : {};
return { response, responseBody };
},
(result) => result.response.ok,
(result) => result.response.status === 200,
{ timeout: 10000, interval: 1000, description: 'other user login' },
);
@@ -395,32 +353,29 @@ describe('E2E Deals and Price Tracking Journey', () => {
const otherUserId = otherLoginData.data.userprofile.user.user_id;
// Other user's watched items should be empty
const otherWatchedResponse = await authedFetch('/users/watched-items', {
method: 'GET',
token: otherToken,
});
const otherWatchedResponse = await getRequest()
.get('/api/users/watched-items')
.set('Authorization', `Bearer ${otherToken}`);
expect(otherWatchedResponse.status).toBe(200);
const otherWatchedData = await otherWatchedResponse.json();
expect(otherWatchedData.data.length).toBe(0);
expect(otherWatchedResponse.body.data.length).toBe(0);
// Other user's deals should be empty
const otherDealsResponse = await authedFetch('/deals/best-watched-prices', {
method: 'GET',
token: otherToken,
});
const otherDealsResponse = await getRequest()
.get('/api/deals/best-watched-prices')
.set('Authorization', `Bearer ${otherToken}`);
expect(otherDealsResponse.status).toBe(200);
const otherDealsData = await otherDealsResponse.json();
expect(otherDealsData.data.length).toBe(0);
expect(otherDealsResponse.body.data.length).toBe(0);
// Clean up other user
await cleanupDb({ userIds: [otherUserId] });
// Step 11: Delete account
const deleteAccountResponse = await apiClient.deleteUserAccount(userPassword, {
tokenOverride: authToken,
});
const deleteAccountResponse = await getRequest()
.delete('/api/users/account')
.set('Authorization', `Bearer ${authToken}`)
.send({ password: userPassword });
expect(deleteAccountResponse.status).toBe(200);
userId = null;

View File

@@ -1,18 +1,22 @@
// src/tests/e2e/flyer-upload.e2e.test.ts
import { describe, it, expect, afterAll } from 'vitest';
import supertest from 'supertest';
import crypto from 'crypto';
import * as apiClient from '../../services/apiClient';
import { getPool } from '../../services/db/connection.db';
import path from 'path';
import fs from 'fs';
import { cleanupDb } from '../utils/cleanup';
import { poll } from '../utils/poll';
import { getServerUrl } from '../setup/e2e-global-setup';
/**
* @vitest-environment node
*/
describe('E2E Flyer Upload and Processing Workflow', () => {
// Create a getter function that returns supertest instance with the app
const getRequest = () => supertest(getServerUrl());
const uniqueId = Date.now();
const userEmail = `e2e-uploader-${uniqueId}@example.com`;
const userPassword = 'StrongPassword123!';
@@ -33,19 +37,20 @@ describe('E2E Flyer Upload and Processing Workflow', () => {
it('should allow a user to upload a flyer and wait for processing to complete', async () => {
// 1. Register a new user
const registerResponse = await apiClient.registerUser(
userEmail,
userPassword,
'E2E Flyer Uploader',
);
const registerResponse = await getRequest().post('/api/auth/register').send({
email: userEmail,
password: userPassword,
full_name: 'E2E Flyer Uploader',
});
expect(registerResponse.status).toBe(201);
// 2. Login to get the access token
const loginResponse = await apiClient.loginUser(userEmail, userPassword, false);
const loginResponse = await getRequest()
.post('/api/auth/login')
.send({ email: userEmail, password: userPassword, rememberMe: false });
expect(loginResponse.status).toBe(200);
const loginResponseBody = await loginResponse.json();
authToken = loginResponseBody.data.token;
userId = loginResponseBody.data.userprofile.user.user_id;
authToken = loginResponse.body.data.token;
userId = loginResponse.body.data.userprofile.user.user_id;
expect(authToken).toBeDefined();
// 3. Prepare the flyer file
@@ -69,29 +74,27 @@ describe('E2E Flyer Upload and Processing Workflow', () => {
]);
}
// Create a File object for the apiClient
// FIX: The Node.js `Buffer` type can be incompatible with the web `File` API's
// expected `BlobPart` type in some TypeScript configurations. Explicitly creating
// a `Uint8Array` from the buffer ensures compatibility and resolves the type error.
// `Uint8Array` is a valid `BufferSource`, which is a valid `BlobPart`.
const flyerFile = new File([new Uint8Array(fileBuffer)], fileName, { type: 'image/jpeg' });
// Calculate checksum (required by the API)
const checksum = crypto.createHash('sha256').update(fileBuffer).digest('hex');
// 4. Upload the flyer
const uploadResponse = await apiClient.uploadAndProcessFlyer(flyerFile, checksum, authToken);
const uploadResponse = await getRequest()
.post('/api/flyers/upload')
.set('Authorization', `Bearer ${authToken}`)
.attach('flyer', fileBuffer, fileName)
.field('checksum', checksum);
expect(uploadResponse.status).toBe(202);
const uploadResponseBody = await uploadResponse.json();
const jobId = uploadResponseBody.data.jobId;
const jobId = uploadResponse.body.data.jobId;
expect(jobId).toBeDefined();
// 5. Poll for job completion using the new utility
const jobStatusResponse = await poll(
async () => {
const statusResponse = await apiClient.getJobStatus(jobId, authToken);
return statusResponse.json();
const statusResponse = await getRequest()
.get(`/api/jobs/${jobId}`)
.set('Authorization', `Bearer ${authToken}`);
return statusResponse.body;
},
(responseBody) =>
responseBody.data.state === 'completed' || responseBody.data.state === 'failed',

View File

@@ -4,39 +4,20 @@
* Tests the complete flow from adding inventory items to tracking expiry and alerts.
*/
import { describe, it, expect, afterAll } from 'vitest';
import * as apiClient from '../../services/apiClient';
import supertest from 'supertest';
import { cleanupDb } from '../utils/cleanup';
import { poll } from '../utils/poll';
import { getPool } from '../../services/db/connection.db';
import { getServerUrl } from '../setup/e2e-global-setup';
/**
* @vitest-environment node
*/
const API_BASE_URL = process.env.VITE_API_BASE_URL || 'http://localhost:3000/api';
// Helper to make authenticated API calls
const authedFetch = async (
path: string,
options: RequestInit & { token?: string } = {},
): Promise<Response> => {
const { token, ...fetchOptions } = options;
const headers: Record<string, string> = {
'Content-Type': 'application/json',
...(fetchOptions.headers as Record<string, string>),
};
if (token) {
headers['Authorization'] = `Bearer ${token}`;
}
return fetch(`${API_BASE_URL}${path}`, {
...fetchOptions,
headers,
});
};
describe('E2E Inventory/Expiry Management Journey', () => {
// Create a getter function that returns supertest instance with the app
const getRequest = () => supertest(getServerUrl());
const uniqueId = Date.now();
const userEmail = `inventory-e2e-${uniqueId}@example.com`;
const userPassword = 'StrongInventoryPassword123!';
@@ -76,21 +57,23 @@ describe('E2E Inventory/Expiry Management Journey', () => {
it('should complete inventory journey: Register -> Add Items -> Track Expiry -> Consume -> Configure Alerts', async () => {
// Step 1: Register a new user
const registerResponse = await apiClient.registerUser(
userEmail,
userPassword,
'Inventory E2E User',
);
const registerResponse = await getRequest().post('/api/auth/register').send({
email: userEmail,
password: userPassword,
full_name: 'Inventory E2E User',
});
expect(registerResponse.status).toBe(201);
// Step 2: Login to get auth token
const { response: loginResponse, responseBody: loginResponseBody } = await poll(
async () => {
const response = await apiClient.loginUser(userEmail, userPassword, false);
const responseBody = response.ok ? await response.clone().json() : {};
const response = await getRequest()
.post('/api/auth/login')
.send({ email: userEmail, password: userPassword, rememberMe: false });
const responseBody = response.status === 200 ? response.body : {};
return { response, responseBody };
},
(result) => result.response.ok,
(result) => result.response.status === 200,
{ timeout: 10000, interval: 1000, description: 'user login after registration' },
);
@@ -172,16 +155,14 @@ describe('E2E Inventory/Expiry Management Journey', () => {
];
for (const item of items) {
const addResponse = await authedFetch('/inventory', {
method: 'POST',
token: authToken,
body: JSON.stringify(item),
});
const addResponse = await getRequest()
.post('/api/inventory')
.set('Authorization', `Bearer ${authToken}`)
.send(item);
expect(addResponse.status).toBe(201);
const addData = await addResponse.json();
expect(addData.data.item_name).toBe(item.item_name);
createdInventoryIds.push(addData.data.inventory_id);
expect(addResponse.body.data.item_name).toBe(item.item_name);
createdInventoryIds.push(addResponse.body.data.inventory_id);
}
// Add an expired item directly to the database for testing expired endpoint
@@ -217,159 +198,135 @@ describe('E2E Inventory/Expiry Management Journey', () => {
createdInventoryIds.push(expiredResult.rows[0].pantry_item_id);
// Step 4: View all inventory
const listResponse = await authedFetch('/inventory', {
method: 'GET',
token: authToken,
});
const listResponse = await getRequest()
.get('/api/inventory')
.set('Authorization', `Bearer ${authToken}`);
expect(listResponse.status).toBe(200);
const listData = await listResponse.json();
expect(listData.data.items.length).toBe(6); // All our items
expect(listData.data.total).toBe(6);
expect(listResponse.body.data.items.length).toBe(6); // All our items
expect(listResponse.body.data.total).toBe(6);
// Step 5: Filter by location
const fridgeResponse = await authedFetch('/inventory?location=fridge', {
method: 'GET',
token: authToken,
});
const fridgeResponse = await getRequest()
.get('/api/inventory?location=fridge')
.set('Authorization', `Bearer ${authToken}`);
expect(fridgeResponse.status).toBe(200);
const fridgeData = await fridgeResponse.json();
fridgeData.data.items.forEach((item: { location: string }) => {
fridgeResponse.body.data.items.forEach((item: { location: string }) => {
expect(item.location).toBe('fridge');
});
expect(fridgeData.data.items.length).toBe(3); // Milk, Apples, Expired Yogurt
expect(fridgeResponse.body.data.items.length).toBe(3); // Milk, Apples, Expired Yogurt
// Step 6: View expiring items
const expiringResponse = await authedFetch('/inventory/expiring?days=3', {
method: 'GET',
token: authToken,
});
const expiringResponse = await getRequest()
.get('/api/inventory/expiring?days=3')
.set('Authorization', `Bearer ${authToken}`);
expect(expiringResponse.status).toBe(200);
const expiringData = await expiringResponse.json();
// Should include the Milk (tomorrow)
expect(expiringData.data.items.length).toBeGreaterThanOrEqual(1);
expect(expiringResponse.body.data.items.length).toBeGreaterThanOrEqual(1);
// Step 7: View expired items
const expiredResponse = await authedFetch('/inventory/expired', {
method: 'GET',
token: authToken,
});
const expiredResponse = await getRequest()
.get('/api/inventory/expired')
.set('Authorization', `Bearer ${authToken}`);
expect(expiredResponse.status).toBe(200);
const expiredData = await expiredResponse.json();
expect(expiredData.data.items.length).toBeGreaterThanOrEqual(1);
expect(expiredResponse.body.data.items.length).toBeGreaterThanOrEqual(1);
// Find the expired yogurt
const expiredYogurt = expiredData.data.items.find(
const expiredYogurt = expiredResponse.body.data.items.find(
(i: { item_name: string }) => i.item_name === 'Expired Yogurt E2E',
);
expect(expiredYogurt).toBeDefined();
// Step 8: Get specific item details
const milkId = createdInventoryIds[0];
const detailResponse = await authedFetch(`/inventory/${milkId}`, {
method: 'GET',
token: authToken,
});
const detailResponse = await getRequest()
.get(`/api/inventory/${milkId}`)
.set('Authorization', `Bearer ${authToken}`);
expect(detailResponse.status).toBe(200);
const detailData = await detailResponse.json();
expect(detailData.data.item_name).toBe('E2E Milk');
expect(detailData.data.quantity).toBe(2);
expect(detailResponse.body.data.item_name).toBe('E2E Milk');
expect(detailResponse.body.data.quantity).toBe(2);
// Step 9: Update item quantity and location
const updateResponse = await authedFetch(`/inventory/${milkId}`, {
method: 'PUT',
token: authToken,
body: JSON.stringify({
const updateResponse = await getRequest()
.put(`/api/inventory/${milkId}`)
.set('Authorization', `Bearer ${authToken}`)
.send({
quantity: 1,
notes: 'One bottle used',
}),
});
});
expect(updateResponse.status).toBe(200);
const updateData = await updateResponse.json();
expect(updateData.data.quantity).toBe(1);
expect(updateResponse.body.data.quantity).toBe(1);
// Step 10: Consume some apples (partial consume via update, then mark fully consumed)
// First, reduce quantity via update
const applesId = createdInventoryIds[3];
const partialConsumeResponse = await authedFetch(`/inventory/${applesId}`, {
method: 'PUT',
token: authToken,
body: JSON.stringify({ quantity: 4 }), // 6 - 2 = 4
});
const partialConsumeResponse = await getRequest()
.put(`/api/inventory/${applesId}`)
.set('Authorization', `Bearer ${authToken}`)
.send({ quantity: 4 }); // 6 - 2 = 4
expect(partialConsumeResponse.status).toBe(200);
const partialConsumeData = await partialConsumeResponse.json();
expect(partialConsumeData.data.quantity).toBe(4);
expect(partialConsumeResponse.body.data.quantity).toBe(4);
// Step 11: Configure alert settings for email
// The API uses PUT /inventory/alerts/:alertMethod with days_before_expiry and is_enabled
const alertSettingsResponse = await authedFetch('/inventory/alerts/email', {
method: 'PUT',
token: authToken,
body: JSON.stringify({
const alertSettingsResponse = await getRequest()
.put('/api/inventory/alerts/email')
.set('Authorization', `Bearer ${authToken}`)
.send({
is_enabled: true,
days_before_expiry: 3,
}),
});
});
expect(alertSettingsResponse.status).toBe(200);
const alertSettingsData = await alertSettingsResponse.json();
expect(alertSettingsData.data.is_enabled).toBe(true);
expect(alertSettingsData.data.days_before_expiry).toBe(3);
expect(alertSettingsResponse.body.data.is_enabled).toBe(true);
expect(alertSettingsResponse.body.data.days_before_expiry).toBe(3);
// Step 12: Verify alert settings were saved
const getSettingsResponse = await authedFetch('/inventory/alerts', {
method: 'GET',
token: authToken,
});
const getSettingsResponse = await getRequest()
.get('/api/inventory/alerts')
.set('Authorization', `Bearer ${authToken}`);
expect(getSettingsResponse.status).toBe(200);
const getSettingsData = await getSettingsResponse.json();
// Should have email alerts enabled
const emailAlert = getSettingsData.data.find(
const emailAlert = getSettingsResponse.body.data.find(
(s: { alert_method: string }) => s.alert_method === 'email',
);
expect(emailAlert?.is_enabled).toBe(true);
// Step 13: Get recipe suggestions based on expiring items
const suggestionsResponse = await authedFetch('/inventory/recipes/suggestions', {
method: 'GET',
token: authToken,
});
const suggestionsResponse = await getRequest()
.get('/api/inventory/recipes/suggestions')
.set('Authorization', `Bearer ${authToken}`);
expect(suggestionsResponse.status).toBe(200);
const suggestionsData = await suggestionsResponse.json();
expect(Array.isArray(suggestionsData.data.recipes)).toBe(true);
expect(Array.isArray(suggestionsResponse.body.data.recipes)).toBe(true);
// Step 14: Fully consume an item (marks as consumed, returns 204)
const breadId = createdInventoryIds[2];
const fullConsumeResponse = await authedFetch(`/inventory/${breadId}/consume`, {
method: 'POST',
token: authToken,
});
const fullConsumeResponse = await getRequest()
.post(`/api/inventory/${breadId}/consume`)
.set('Authorization', `Bearer ${authToken}`);
expect(fullConsumeResponse.status).toBe(204);
// Verify the item is now marked as consumed
const consumedItemResponse = await authedFetch(`/inventory/${breadId}`, {
method: 'GET',
token: authToken,
});
const consumedItemResponse = await getRequest()
.get(`/api/inventory/${breadId}`)
.set('Authorization', `Bearer ${authToken}`);
expect(consumedItemResponse.status).toBe(200);
const consumedItemData = await consumedItemResponse.json();
expect(consumedItemData.data.is_consumed).toBe(true);
expect(consumedItemResponse.body.data.is_consumed).toBe(true);
// Step 15: Delete an item
const riceId = createdInventoryIds[4];
const deleteResponse = await authedFetch(`/inventory/${riceId}`, {
method: 'DELETE',
token: authToken,
});
const deleteResponse = await getRequest()
.delete(`/api/inventory/${riceId}`)
.set('Authorization', `Bearer ${authToken}`);
expect(deleteResponse.status).toBe(204);
@@ -380,24 +337,27 @@ describe('E2E Inventory/Expiry Management Journey', () => {
}
// Step 16: Verify deletion
const verifyDeleteResponse = await authedFetch(`/inventory/${riceId}`, {
method: 'GET',
token: authToken,
});
const verifyDeleteResponse = await getRequest()
.get(`/api/inventory/${riceId}`)
.set('Authorization', `Bearer ${authToken}`);
expect(verifyDeleteResponse.status).toBe(404);
// Step 17: Verify another user cannot access our inventory
const otherUserEmail = `other-inventory-e2e-${uniqueId}@example.com`;
await apiClient.registerUser(otherUserEmail, userPassword, 'Other Inventory User');
await getRequest()
.post('/api/auth/register')
.send({ email: otherUserEmail, password: userPassword, full_name: 'Other Inventory User' });
const { responseBody: otherLoginData } = await poll(
async () => {
const response = await apiClient.loginUser(otherUserEmail, userPassword, false);
const responseBody = response.ok ? await response.clone().json() : {};
const response = await getRequest()
.post('/api/auth/login')
.send({ email: otherUserEmail, password: userPassword, rememberMe: false });
const responseBody = response.status === 200 ? response.body : {};
return { response, responseBody };
},
(result) => result.response.ok,
(result) => result.response.status === 200,
{ timeout: 10000, interval: 1000, description: 'other user login' },
);
@@ -405,58 +365,52 @@ describe('E2E Inventory/Expiry Management Journey', () => {
const otherUserId = otherLoginData.data.userprofile.user.user_id;
// Other user should not see our inventory
const otherDetailResponse = await authedFetch(`/inventory/${milkId}`, {
method: 'GET',
token: otherToken,
});
const otherDetailResponse = await getRequest()
.get(`/api/inventory/${milkId}`)
.set('Authorization', `Bearer ${otherToken}`);
expect(otherDetailResponse.status).toBe(404);
// Other user's inventory should be empty
const otherListResponse = await authedFetch('/inventory', {
method: 'GET',
token: otherToken,
});
const otherListResponse = await getRequest()
.get('/api/inventory')
.set('Authorization', `Bearer ${otherToken}`);
expect(otherListResponse.status).toBe(200);
const otherListData = await otherListResponse.json();
expect(otherListData.data.total).toBe(0);
expect(otherListResponse.body.data.total).toBe(0);
// Clean up other user
await cleanupDb({ userIds: [otherUserId] });
// Step 18: Move frozen item to fridge (simulating thawing)
const pizzaId = createdInventoryIds[1];
const moveResponse = await authedFetch(`/inventory/${pizzaId}`, {
method: 'PUT',
token: authToken,
body: JSON.stringify({
const moveResponse = await getRequest()
.put(`/api/inventory/${pizzaId}`)
.set('Authorization', `Bearer ${authToken}`)
.send({
location: 'fridge',
expiry_date: formatDate(nextWeek), // Update expiry since thawed
notes: 'Thawed for dinner',
}),
});
});
expect(moveResponse.status).toBe(200);
const moveData = await moveResponse.json();
expect(moveData.data.location).toBe('fridge');
expect(moveResponse.body.data.location).toBe('fridge');
// Step 19: Final inventory check
const finalListResponse = await authedFetch('/inventory', {
method: 'GET',
token: authToken,
});
const finalListResponse = await getRequest()
.get('/api/inventory')
.set('Authorization', `Bearer ${authToken}`);
expect(finalListResponse.status).toBe(200);
const finalListData = await finalListResponse.json();
// We should have: Milk (1), Pizza (thawed, 3), Bread (consumed), Apples (4), Expired Yogurt (1)
// Rice was deleted, Bread was consumed
expect(finalListData.data.total).toBeLessThanOrEqual(5);
expect(finalListResponse.body.data.total).toBeLessThanOrEqual(5);
// Step 20: Delete account
const deleteAccountResponse = await apiClient.deleteUserAccount(userPassword, {
tokenOverride: authToken,
});
const deleteAccountResponse = await getRequest()
.delete('/api/users/account')
.set('Authorization', `Bearer ${authToken}`)
.send({ password: userPassword });
expect(deleteAccountResponse.status).toBe(200);
userId = null;

Some files were not shown because too many files have changed in this diff Show More