Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de263f74b0 | ||
| a71e41302b | |||
|
|
3575803252 | ||
| d03900cefe | |||
|
|
6d49639845 | ||
| d4543cf4b9 | |||
|
|
4f08238698 | ||
| 38b35f87aa | |||
|
|
dd067183ed | ||
| 9f3a070612 | |||
| 8a38befb1c | |||
|
|
7e460a11e4 | ||
| eae0dbaa8e | |||
| fac98f4c54 | |||
| 9f7b821760 | |||
| cd60178450 | |||
|
|
1fcb9fd5c7 | ||
| 8bd4e081ea | |||
|
|
6e13570deb | ||
| 2eba66fb71 | |||
|
|
10cdd78e22 | ||
| 521943bec0 | |||
|
|
810c0eb61b | ||
| 3314063e25 | |||
|
|
65c38765c6 | ||
| 4ddd9bb220 |
@@ -100,7 +100,30 @@
|
||||
"Read(//d/gitea/bugsink-mcp/**)",
|
||||
"Bash(d:/nodejs/npm.cmd install)",
|
||||
"Bash(node node_modules/vitest/vitest.mjs run:*)",
|
||||
"Bash(npm run test:e2e:*)"
|
||||
"Bash(npm run test:e2e:*)",
|
||||
"Bash(export BUGSINK_URL=http://localhost:8000)",
|
||||
"Bash(export BUGSINK_TOKEN=a609c2886daa4e1e05f1517074d7779a5fb49056)",
|
||||
"Bash(timeout 3 d:/nodejs/node.exe:*)",
|
||||
"Bash(export BUGSINK_URL=https://bugsink.projectium.com)",
|
||||
"Bash(export BUGSINK_API_TOKEN=77deaa5e2649ab0fbbca51bbd427ec4637d073a0)",
|
||||
"Bash(export BUGSINK_TOKEN=77deaa5e2649ab0fbbca51bbd427ec4637d073a0)",
|
||||
"Bash(where:*)",
|
||||
"mcp__localerrors__test_connection",
|
||||
"mcp__localerrors__list_projects",
|
||||
"Bash(\"D:\\\\nodejs\\\\npx.cmd\" -y @modelcontextprotocol/server-postgres --help)",
|
||||
"Bash(git rm:*)",
|
||||
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" log -1 --format=\"%H %ci %s\")",
|
||||
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" config --get remote.origin.url)",
|
||||
"Bash(git -C \"C:\\\\Users\\\\games3\\\\.claude\\\\plugins\\\\marketplaces\\\\claude-plugins-official\" fetch --dry-run -v)",
|
||||
"mcp__localerrors__get_project",
|
||||
"mcp__localerrors__get_issue",
|
||||
"mcp__localerrors__get_event",
|
||||
"mcp__localerrors__list_teams"
|
||||
]
|
||||
}
|
||||
},
|
||||
"enabledMcpjsonServers": [
|
||||
"localerrors",
|
||||
"devdb",
|
||||
"gitea-projectium"
|
||||
]
|
||||
}
|
||||
|
||||
23
.env.example
23
.env.example
@@ -35,6 +35,12 @@ NODE_ENV=development
|
||||
# Frontend URL for CORS and email links
|
||||
FRONTEND_URL=http://localhost:3000
|
||||
|
||||
# Flyer Base URL - used for seed data and flyer image URLs
|
||||
# Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
|
||||
# Test: https://flyer-crawler-test.projectium.com
|
||||
# Production: https://flyer-crawler.projectium.com
|
||||
FLYER_BASE_URL=https://localhost
|
||||
|
||||
# ===================
|
||||
# Authentication
|
||||
# ===================
|
||||
@@ -88,11 +94,18 @@ WORKER_LOCK_DURATION=120000
|
||||
# Error Tracking (ADR-015)
|
||||
# ===================
|
||||
# Sentry-compatible error tracking via Bugsink (self-hosted)
|
||||
# DSNs are created in Bugsink UI at http://localhost:8000 (dev) or your production URL
|
||||
# Backend DSN - for Express/Node.js errors
|
||||
SENTRY_DSN=
|
||||
# Frontend DSN - for React/browser errors (uses VITE_ prefix)
|
||||
VITE_SENTRY_DSN=
|
||||
# DSNs are created in Bugsink UI at https://localhost:8443 (dev) or your production URL
|
||||
#
|
||||
# Dev container projects:
|
||||
# - Project 1: Backend API (Dev) - receives Pino, PostgreSQL errors
|
||||
# - Project 2: Frontend (Dev) - receives browser errors via Sentry SDK
|
||||
# - Project 4: Infrastructure (Dev) - receives Redis, NGINX, Vite errors
|
||||
#
|
||||
# Backend DSN - for Express/Node.js errors (internal container URL)
|
||||
SENTRY_DSN=http://<key>@localhost:8000/1
|
||||
# Frontend DSN - for React/browser errors (uses nginx proxy for browser access)
|
||||
# Note: Browsers cannot reach localhost:8000 directly, so we use nginx proxy at /bugsink-api/
|
||||
VITE_SENTRY_DSN=https://<key>@localhost/bugsink-api/2
|
||||
# Environment name for error grouping (defaults to NODE_ENV)
|
||||
SENTRY_ENVIRONMENT=development
|
||||
VITE_SENTRY_ENVIRONMENT=development
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
cache-dependency-path: '**/package-lock.json'
|
||||
|
||||
- name: Install Dependencies
|
||||
run: npm ci --legacy-peer-deps
|
||||
run: npm ci
|
||||
|
||||
- name: Bump Minor Version and Push
|
||||
run: |
|
||||
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
# If dependencies are not found in cache, it will run 'npm ci' automatically.
|
||||
# If they are found, it restores them. This is the standard, reliable way.
|
||||
- name: Install Dependencies
|
||||
run: npm ci --legacy-peer-deps # 'ci' is faster and safer for CI/CD than 'install'.
|
||||
run: npm ci # 'ci' is faster and safer for CI/CD than 'install'.
|
||||
|
||||
- name: Bump Version and Push
|
||||
run: |
|
||||
|
||||
28
.mcp.json
Normal file
28
.mcp.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"localerrors": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "http://127.0.0.1:8000",
|
||||
"BUGSINK_TOKEN": "a609c2886daa4e1e05f1517074d7779a5fb49056"
|
||||
}
|
||||
},
|
||||
"devdb": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": [
|
||||
"-y",
|
||||
"@modelcontextprotocol/server-postgres",
|
||||
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
|
||||
]
|
||||
},
|
||||
"gitea-projectium": {
|
||||
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
|
||||
"args": ["run", "-t", "stdio"],
|
||||
"env": {
|
||||
"GITEA_HOST": "https://gitea.projectium.com",
|
||||
"GITEA_ACCESS_TOKEN": "b111259253aa3cadcb6a37618de03bf388f6235a"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
905
CLAUDE.md
905
CLAUDE.md
@@ -1,606 +1,405 @@
|
||||
# Claude Code Project Instructions
|
||||
|
||||
## CRITICAL RULES (READ FIRST)
|
||||
|
||||
### Platform: Linux Only (ADR-014)
|
||||
|
||||
**ALL tests MUST run in dev container** - Windows results are unreliable.
|
||||
|
||||
| Test Result | Container | Windows | Status |
|
||||
| ----------- | --------- | ------- | ------------------------ |
|
||||
| Pass | Fail | = | **BROKEN** (must fix) |
|
||||
| Fail | Pass | = | **PASSING** (acceptable) |
|
||||
|
||||
```bash
|
||||
# Always test in container
|
||||
podman exec -it flyer-crawler-dev npm test
|
||||
podman exec -it flyer-crawler-dev npm run type-check
|
||||
```
|
||||
|
||||
### Database Schema Sync
|
||||
|
||||
**CRITICAL**: Keep these files synchronized:
|
||||
|
||||
- `sql/master_schema_rollup.sql` (test DB, complete reference)
|
||||
- `sql/initial_schema.sql` (fresh install, identical to rollup)
|
||||
- `sql/migrations/*.sql` (production ALTER TABLE statements)
|
||||
|
||||
Out-of-sync = test failures.
|
||||
|
||||
### Communication Style
|
||||
|
||||
Ask before assuming. Never assume:
|
||||
|
||||
- Steps completed / User knowledge / External services configured / Secrets created
|
||||
|
||||
---
|
||||
|
||||
## Session Startup Checklist
|
||||
|
||||
**IMPORTANT**: At the start of every session, perform these steps:
|
||||
|
||||
1. **Check Memory First** - Use `mcp__memory__read_graph` or `mcp__memory__search_nodes` to recall:
|
||||
- Project-specific configurations and credentials
|
||||
- Previous work context and decisions
|
||||
- Infrastructure details (URLs, ports, access patterns)
|
||||
- Known issues and their solutions
|
||||
|
||||
2. **Review Recent Git History** - Check `git log --oneline -10` to understand recent changes
|
||||
|
||||
3. **Check Container Status** - Use `mcp__podman__container_list` to see what's running
|
||||
1. **Memory**: `mcp__memory__read_graph` - Recall project context, credentials, known issues
|
||||
2. **Git**: `git log --oneline -10` - Recent changes
|
||||
3. **Containers**: `mcp__podman__container_list` - Running state
|
||||
4. **PM2 Status**: `podman exec flyer-crawler-dev pm2 status` - Process health (API, Worker, Vite)
|
||||
|
||||
---
|
||||
|
||||
## Project Instructions
|
||||
|
||||
### Things to Remember
|
||||
|
||||
Before writing any code:
|
||||
|
||||
1. State how you will verify this change works (test, bash command, browser check, etc.)
|
||||
|
||||
2. Write the test or verification step first
|
||||
|
||||
3. Then implement the code
|
||||
|
||||
4. Run verification and iterate until it passes
|
||||
|
||||
## Git Bash / MSYS Path Conversion Issue (Windows Host)
|
||||
|
||||
**CRITICAL ISSUE**: Git Bash on Windows automatically converts Unix-style paths to Windows paths, which breaks Podman/Docker commands.
|
||||
|
||||
### Problem Examples:
|
||||
|
||||
```bash
|
||||
# This FAILS in Git Bash:
|
||||
podman exec container /usr/local/bin/script.sh
|
||||
# Git Bash converts to: C:/Program Files/Git/usr/local/bin/script.sh
|
||||
|
||||
# This FAILS in Git Bash:
|
||||
podman exec container bash -c "cat /tmp/file.sql"
|
||||
# Git Bash converts /tmp to C:/Users/user/AppData/Local/Temp
|
||||
```
|
||||
|
||||
### Solutions:
|
||||
|
||||
1. **Use `sh -c` instead of `bash -c`** for single-quoted commands:
|
||||
|
||||
```bash
|
||||
podman exec container sh -c '/usr/local/bin/script.sh'
|
||||
```
|
||||
|
||||
2. **Use double slashes** to escape path conversion:
|
||||
|
||||
```bash
|
||||
podman exec container //usr//local//bin//script.sh
|
||||
```
|
||||
|
||||
3. **Set MSYS_NO_PATHCONV** environment variable:
|
||||
|
||||
```bash
|
||||
MSYS_NO_PATHCONV=1 podman exec container /usr/local/bin/script.sh
|
||||
```
|
||||
|
||||
4. **Use Windows paths with forward slashes** when referencing host files:
|
||||
```bash
|
||||
podman cp "d:/path/to/file" container:/tmp/file
|
||||
```
|
||||
|
||||
**ALWAYS use one of these workarounds when running Bash commands on Windows that involve Unix paths inside containers.**
|
||||
|
||||
## Communication Style: Ask Before Assuming
|
||||
|
||||
**IMPORTANT**: When helping with tasks, **ask clarifying questions before making assumptions**. Do not assume:
|
||||
|
||||
- What steps the user has or hasn't completed
|
||||
- What the user already knows or has configured
|
||||
- What external services (OAuth providers, APIs, etc.) are already set up
|
||||
- What secrets or credentials have already been created
|
||||
|
||||
Instead, ask the user to confirm the current state before providing instructions or making recommendations. This prevents wasted effort and respects the user's existing work.
|
||||
|
||||
## Platform Requirement: Linux Only
|
||||
|
||||
**CRITICAL**: This application is designed to run **exclusively on Linux**. See [ADR-014](docs/adr/0014-containerization-and-deployment-strategy.md) for full details.
|
||||
|
||||
### Environment Terminology
|
||||
|
||||
- **Dev Container** (or just "dev"): The containerized Linux development environment (`flyer-crawler-dev`). This is where all development and testing should occur.
|
||||
- **Host**: The Windows machine running Podman/Docker and VS Code.
|
||||
|
||||
When instructions say "run in dev" or "run in the dev container", they mean executing commands inside the `flyer-crawler-dev` container.
|
||||
|
||||
### Test Execution Rules
|
||||
|
||||
1. **ALL tests MUST be executed in the dev container** - the Linux container environment
|
||||
2. **NEVER run tests directly on Windows host** - test results from Windows are unreliable
|
||||
3. **Always use the dev container for testing** when developing on Windows
|
||||
4. **TypeScript type-check MUST run in dev container** - `npm run type-check` on Windows does not reliably detect errors
|
||||
|
||||
See [docs/TESTING.md](docs/TESTING.md) for comprehensive testing documentation.
|
||||
|
||||
### How to Run Tests Correctly
|
||||
|
||||
```bash
|
||||
# If on Windows, first open VS Code and "Reopen in Container"
|
||||
# Then run tests inside the dev container:
|
||||
npm test # Run all unit tests
|
||||
npm run test:unit # Run unit tests only
|
||||
npm run test:integration # Run integration tests (requires DB/Redis)
|
||||
```
|
||||
|
||||
### Running Tests via Podman (from Windows host)
|
||||
|
||||
**Note:** This project has 2900+ unit tests. For AI-assisted development, pipe output to a file for easier processing.
|
||||
|
||||
The command to run unit tests in the dev container via podman:
|
||||
|
||||
```bash
|
||||
# Basic (output to terminal)
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
|
||||
# Recommended for AI processing: pipe to file
|
||||
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
|
||||
```
|
||||
|
||||
The command to run integration tests in the dev container via podman:
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev npm run test:integration
|
||||
```
|
||||
|
||||
For running specific test files:
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev npm test -- --run src/hooks/useAuth.test.tsx
|
||||
```
|
||||
|
||||
### Why Linux Only?
|
||||
|
||||
- Path separators: Code uses POSIX-style paths (`/`) which may break on Windows
|
||||
- Shell scripts in `scripts/` directory are Linux-only
|
||||
- External dependencies like `pdftocairo` assume Linux installation paths
|
||||
- Unix-style file permissions are assumed throughout
|
||||
|
||||
### Test Result Interpretation
|
||||
|
||||
- Tests that **pass on Windows but fail on Linux** = **BROKEN tests** (must be fixed)
|
||||
- Tests that **fail on Windows but pass on Linux** = **PASSING tests** (acceptable)
|
||||
|
||||
## Development Workflow
|
||||
|
||||
1. Open project in VS Code
|
||||
2. Use "Reopen in Container" (Dev Containers extension required) to enter the dev environment
|
||||
3. Wait for dev container initialization to complete
|
||||
4. Run `npm test` to verify the dev environment is working
|
||||
5. Make changes and run tests inside the dev container
|
||||
|
||||
## Code Change Verification
|
||||
|
||||
After making any code changes, **always run a type-check** to catch TypeScript errors before committing:
|
||||
|
||||
```bash
|
||||
npm run type-check
|
||||
```
|
||||
|
||||
This prevents linting/type errors from being introduced into the codebase.
|
||||
|
||||
## Quick Reference
|
||||
|
||||
| Command | Description |
|
||||
| -------------------------- | ---------------------------- |
|
||||
| `npm test` | Run all unit tests |
|
||||
| `npm run test:unit` | Run unit tests only |
|
||||
| `npm run test:integration` | Run integration tests |
|
||||
| `npm run dev:container` | Start dev server (container) |
|
||||
| `npm run build` | Build for production |
|
||||
| `npm run type-check` | Run TypeScript type checking |
|
||||
### Essential Commands
|
||||
|
||||
## Database Schema Files
|
||||
| Command | Description |
|
||||
| ------------------------------------------------------------ | --------------------- |
|
||||
| `podman exec -it flyer-crawler-dev npm test` | Run all tests |
|
||||
| `podman exec -it flyer-crawler-dev npm run test:unit` | Unit tests only |
|
||||
| `podman exec -it flyer-crawler-dev npm run type-check` | TypeScript check |
|
||||
| `podman exec -it flyer-crawler-dev npm run test:integration` | Integration tests |
|
||||
| `podman exec -it flyer-crawler-dev pm2 status` | PM2 process status |
|
||||
| `podman exec -it flyer-crawler-dev pm2 logs` | View all PM2 logs |
|
||||
| `podman exec -it flyer-crawler-dev pm2 restart all` | Restart all processes |
|
||||
|
||||
**CRITICAL**: The database schema files must be kept in sync with each other. When making schema changes:
|
||||
### Key Patterns (with file locations)
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------ | ----------------------------------------------------------- |
|
||||
| `sql/master_schema_rollup.sql` | Complete schema used by test database setup and reference |
|
||||
| `sql/initial_schema.sql` | Base schema without seed data, used as standalone reference |
|
||||
| `sql/migrations/*.sql` | Incremental migrations for production database updates |
|
||||
| Pattern | ADR | Implementation | File |
|
||||
| ------------------ | ------- | ------------------------------------------------- | ----------------------------------- |
|
||||
| Error Handling | ADR-001 | `handleDbError()`, throw `NotFoundError` | `src/services/db/errors.db.ts` |
|
||||
| Repository Methods | ADR-034 | `get*` (throws), `find*` (null), `list*` (array) | `src/services/db/*.db.ts` |
|
||||
| API Responses | ADR-028 | `sendSuccess()`, `sendPaginated()`, `sendError()` | `src/utils/apiResponse.ts` |
|
||||
| Transactions | ADR-002 | `withTransaction(async (client) => {...})` | `src/services/db/transaction.db.ts` |
|
||||
|
||||
**Maintenance Rules:**
|
||||
### Key Files Quick Access
|
||||
|
||||
1. **Keep `master_schema_rollup.sql` and `initial_schema.sql` in sync** - These files should contain the same table definitions
|
||||
2. **When adding columns via migration**, also add them to both `master_schema_rollup.sql` and `initial_schema.sql`
|
||||
3. **Migrations are for production deployments** - They use `ALTER TABLE` to add columns incrementally
|
||||
4. **Schema files are for fresh installs** - They define the complete table structure
|
||||
5. **Test database uses `master_schema_rollup.sql`** - If schema files are out of sync with migrations, tests will fail
|
||||
| Purpose | File |
|
||||
| ----------------- | -------------------------------- |
|
||||
| Express app | `server.ts` |
|
||||
| Environment | `src/config/env.ts` |
|
||||
| Routes | `src/routes/*.routes.ts` |
|
||||
| Repositories | `src/services/db/*.db.ts` |
|
||||
| Workers | `src/services/workers.server.ts` |
|
||||
| Queues | `src/services/queues.server.ts` |
|
||||
| PM2 Config (Dev) | `ecosystem.dev.config.cjs` |
|
||||
| PM2 Config (Prod) | `ecosystem.config.cjs` |
|
||||
|
||||
**Example:** When `002_expiry_tracking.sql` adds `purchase_date` to `pantry_items`, that column must also exist in the `CREATE TABLE` statements in both `master_schema_rollup.sql` and `initial_schema.sql`.
|
||||
---
|
||||
|
||||
## Known Integration Test Issues and Solutions
|
||||
## Application Overview
|
||||
|
||||
This section documents common test issues encountered in integration tests, their root causes, and solutions. These patterns recur frequently.
|
||||
**Flyer Crawler** - AI-powered grocery deal extraction and analysis platform.
|
||||
|
||||
### 1. Vitest globalSetup Runs in Separate Node.js Context
|
||||
**Data Flow**: Upload → AI extraction (Gemini) → PostgreSQL → Cache (Redis) → API → React display
|
||||
|
||||
**Problem:** Vitest's `globalSetup` runs in a completely separate Node.js context from test files. This means:
|
||||
**Architecture** (ADR-035):
|
||||
|
||||
- Singletons created in globalSetup are NOT the same instances as those in test files
|
||||
- `global`, `globalThis`, and `process` are all isolated between contexts
|
||||
- `vi.spyOn()` on module exports doesn't work cross-context
|
||||
- Dependency injection via setter methods fails across contexts
|
||||
|
||||
**Affected Tests:** Any test trying to inject mocks into BullMQ worker services (e.g., AI failure tests, DB failure tests)
|
||||
|
||||
**Solution Options:**
|
||||
|
||||
1. Mark tests as `.todo()` until an API-based mock injection mechanism is implemented
|
||||
2. Create test-only API endpoints that allow setting mock behaviors via HTTP
|
||||
3. Use file-based or Redis-based mock flags that services check at runtime
|
||||
|
||||
**Example of affected code pattern:**
|
||||
|
||||
```typescript
|
||||
// This DOES NOT work - different module instances
|
||||
const { flyerProcessingService } = await import('../../services/workers.server');
|
||||
flyerProcessingService._getAiProcessor()._setExtractAndValidateData(mockFn);
|
||||
// The worker uses a different flyerProcessingService instance!
|
||||
```text
|
||||
Routes → Services → Repositories → Database
|
||||
↓
|
||||
External APIs (*.server.ts)
|
||||
```
|
||||
|
||||
### 2. BullMQ Cleanup Queue Deleting Files Before Test Verification
|
||||
**Key Entities**: Flyers, FlyerItems, Stores, StoreLocations, Users, Watchlists, ShoppingLists, Recipes, Achievements
|
||||
|
||||
**Problem:** The cleanup worker runs in the globalSetup context and processes cleanup jobs even when tests spy on `cleanupQueue.add()`. The spy intercepts calls in the test context, but jobs already queued run in the worker's context.
|
||||
**Full Architecture**: See [docs/architecture/OVERVIEW.md](docs/architecture/OVERVIEW.md)
|
||||
|
||||
**Affected Tests:** EXIF/PNG metadata stripping tests that need to verify file contents before deletion
|
||||
---
|
||||
|
||||
**Solution:** Drain and pause the cleanup queue before the test:
|
||||
## Dev Container Architecture (ADR-014)
|
||||
|
||||
The dev container now matches production by using PM2 for process management.
|
||||
|
||||
### Process Management
|
||||
|
||||
| Component | Production | Dev Container |
|
||||
| ---------- | ---------------------- | ------------------------- |
|
||||
| API Server | PM2 cluster mode | PM2 fork mode + tsx watch |
|
||||
| Worker | PM2 process | PM2 process + tsx watch |
|
||||
| Frontend | Static files via NGINX | PM2 + Vite dev server |
|
||||
| Logs | PM2 logs -> Logstash | PM2 logs -> Logstash |
|
||||
|
||||
**PM2 Processes in Dev Container**:
|
||||
|
||||
- `flyer-crawler-api-dev` - API server (port 3001)
|
||||
- `flyer-crawler-worker-dev` - Background job worker
|
||||
- `flyer-crawler-vite-dev` - Vite frontend dev server (port 5173)
|
||||
|
||||
### Log Aggregation (ADR-050)
|
||||
|
||||
All logs flow to Bugsink via Logstash with 3-project routing:
|
||||
|
||||
| Source | Log Location | Bugsink Project |
|
||||
| ----------------- | --------------------------------- | ------------------ |
|
||||
| Backend (Pino) | `/var/log/pm2/api-*.log` | Backend API (1) |
|
||||
| Worker (Pino) | `/var/log/pm2/worker-*.log` | Backend API (1) |
|
||||
| PostgreSQL | `/var/log/postgresql/*.log` | Backend API (1) |
|
||||
| Vite | `/var/log/pm2/vite-*.log` | Infrastructure (4) |
|
||||
| Redis | `/var/log/redis/redis-server.log` | Infrastructure (4) |
|
||||
| NGINX | `/var/log/nginx/*.log` | Infrastructure (4) |
|
||||
| Frontend (Sentry) | Browser -> nginx proxy | Frontend (2) |
|
||||
|
||||
**Bugsink Projects (Dev Container)**:
|
||||
|
||||
- Project 1: Backend API (Dev) - Application errors
|
||||
- Project 2: Frontend (Dev) - Browser errors via nginx proxy
|
||||
- Project 4: Infrastructure (Dev) - Redis, NGINX, Vite errors
|
||||
|
||||
**Key Files**:
|
||||
|
||||
- `ecosystem.dev.config.cjs` - PM2 development configuration
|
||||
- `scripts/dev-entrypoint.sh` - Container startup script
|
||||
- `docker/logstash/bugsink.conf` - Logstash pipeline configuration
|
||||
- `docker/nginx/dev.conf` - NGINX config with Bugsink API proxy
|
||||
|
||||
**Full Dev Container Guide**: See [docs/development/DEV-CONTAINER.md](docs/development/DEV-CONTAINER.md)
|
||||
|
||||
---
|
||||
|
||||
## Common Workflows
|
||||
|
||||
### Adding a New API Endpoint
|
||||
|
||||
1. Add route in `src/routes/{domain}.routes.ts`
|
||||
2. Use `validateRequest(schema)` middleware for input validation
|
||||
3. Call service layer (never access DB directly from routes)
|
||||
4. Return via `sendSuccess()` or `sendPaginated()`
|
||||
5. Add tests in `*.routes.test.ts`
|
||||
|
||||
**Example Pattern**: See [docs/development/CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md)
|
||||
|
||||
### Adding a New Database Operation
|
||||
|
||||
1. Add method to `src/services/db/{domain}.db.ts`
|
||||
2. Follow naming: `get*` (throws), `find*` (returns null), `list*` (array)
|
||||
3. Use `handleDbError()` for error handling
|
||||
4. Accept optional `PoolClient` for transaction support
|
||||
5. Add unit test
|
||||
|
||||
### Adding a Background Job
|
||||
|
||||
1. Define queue in `src/services/queues.server.ts`
|
||||
2. Add worker in `src/services/workers.server.ts`
|
||||
3. Call `queue.add()` from service layer
|
||||
|
||||
---
|
||||
|
||||
## Subagent Delegation Guide
|
||||
|
||||
**When to Delegate**: Complex work, specialized expertise, multi-domain tasks
|
||||
|
||||
### Decision Matrix
|
||||
|
||||
| Task Type | Subagent | Key Docs |
|
||||
| --------------------- | ----------------------- | ----------------------------------------------------------------- |
|
||||
| Write production code | coder | [CODER-GUIDE.md](docs/subagents/CODER-GUIDE.md) |
|
||||
| Database changes | db-dev | [DATABASE-GUIDE.md](docs/subagents/DATABASE-GUIDE.md) |
|
||||
| Create tests | testwriter | [TESTER-GUIDE.md](docs/subagents/TESTER-GUIDE.md) |
|
||||
| Fix failing tests | tester | [TESTER-GUIDE.md](docs/subagents/TESTER-GUIDE.md) |
|
||||
| Container/deployment | devops | [DEVOPS-GUIDE.md](docs/subagents/DEVOPS-GUIDE.md) |
|
||||
| UI components | frontend-specialist | [FRONTEND-GUIDE.md](docs/subagents/FRONTEND-GUIDE.md) |
|
||||
| External APIs | integrations-specialist | [INTEGRATIONS-GUIDE.md](docs/subagents/INTEGRATIONS-GUIDE.md) |
|
||||
| Security review | security-engineer | [SECURITY-DEBUG-GUIDE.md](docs/subagents/SECURITY-DEBUG-GUIDE.md) |
|
||||
| Production errors | log-debug | [SECURITY-DEBUG-GUIDE.md](docs/subagents/SECURITY-DEBUG-GUIDE.md) |
|
||||
| AI/Gemini issues | ai-usage | [AI-USAGE-GUIDE.md](docs/subagents/AI-USAGE-GUIDE.md) |
|
||||
| Planning features | planner | [DOCUMENTATION-GUIDE.md](docs/subagents/DOCUMENTATION-GUIDE.md) |
|
||||
|
||||
**All Subagents**: See [docs/subagents/OVERVIEW.md](docs/subagents/OVERVIEW.md)
|
||||
|
||||
**Launch Pattern**:
|
||||
|
||||
```
|
||||
Use Task tool with subagent_type: "coder", "db-dev", "tester", etc.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Known Issues & Gotchas
|
||||
|
||||
### Integration Test Issues (Summary)
|
||||
|
||||
Common issues with solutions:
|
||||
|
||||
1. **Vitest globalSetup context isolation** - Mocks/spies don't share instances → Mark `.todo()` or use Redis-based flags
|
||||
2. **Cleanup queue interference** - Worker processes jobs during tests → `cleanupQueue.drain()` and `.pause()`
|
||||
3. **Cache staleness** - Direct SQL bypasses cache → `cacheService.invalidateFlyers()` after inserts
|
||||
4. **Filename collisions** - Multer predictable names → Use `${Date.now()}-${Math.round(Math.random() * 1e9)}`
|
||||
5. **Response format mismatches** - API format changes → Log response bodies, update assertions
|
||||
6. **External service failures** - PM2/Redis unavailable → try/catch with graceful degradation
|
||||
7. **TZ environment variable breaks async hooks** - `TZ=America/Los_Angeles` causes `RangeError: Invalid triggerAsyncId value: NaN` → Tests now explicitly set `TZ=` (empty) in package.json scripts
|
||||
|
||||
**Full Details**: See test issues section at end of this document or [docs/development/TESTING.md](docs/development/TESTING.md)
|
||||
|
||||
### Git Bash Path Conversion (Windows)
|
||||
|
||||
Git Bash auto-converts Unix paths, breaking container commands.
|
||||
|
||||
**Solutions**:
|
||||
|
||||
```bash
|
||||
# Use sh -c with single quotes
|
||||
podman exec container sh -c '/usr/local/bin/script.sh'
|
||||
|
||||
# Use MSYS_NO_PATHCONV=1
|
||||
MSYS_NO_PATHCONV=1 podman exec container /path/to/script
|
||||
|
||||
# Use Windows paths for host files
|
||||
podman cp "d:/path/file" container:/tmp/file
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration & Environment
|
||||
|
||||
### Environment Variables
|
||||
|
||||
**See**: [docs/getting-started/ENVIRONMENT.md](docs/getting-started/ENVIRONMENT.md) for complete reference.
|
||||
|
||||
**Quick Overview**:
|
||||
|
||||
- **Production**: Gitea CI/CD secrets only (no `.env` file)
|
||||
- **Test**: Gitea secrets + `.env.test` overrides
|
||||
- **Dev**: `.env.local` file (overrides `compose.dev.yml`)
|
||||
|
||||
**Key Variables**: `DB_HOST`, `DB_USER`, `DB_PASSWORD`, `JWT_SECRET`, `VITE_GOOGLE_GENAI_API_KEY`, `REDIS_URL`
|
||||
|
||||
**Adding Variables**: Update `src/config/env.ts`, Gitea Secrets, workflows, `ecosystem.config.cjs`, `.env.example`
|
||||
|
||||
### MCP Servers
|
||||
|
||||
**See**: [docs/tools/MCP-CONFIGURATION.md](docs/tools/MCP-CONFIGURATION.md) for setup.
|
||||
|
||||
**Quick Overview**:
|
||||
|
||||
| Server | Purpose | Config |
|
||||
| -------------------------- | -------------------- | ---------------------- |
|
||||
| gitea-projectium/torbonium | Gitea API | Global `settings.json` |
|
||||
| podman | Container management | Global `settings.json` |
|
||||
| memory | Knowledge graph | Global `settings.json` |
|
||||
| redis | Cache access | Global `settings.json` |
|
||||
| bugsink | Prod error tracking | Global `settings.json` |
|
||||
| localerrors | Dev Bugsink | Project `.mcp.json` |
|
||||
| devdb | Dev PostgreSQL | Project `.mcp.json` |
|
||||
|
||||
**Note**: Localhost servers use project `.mcp.json` due to Windows/loader issues.
|
||||
|
||||
### Bugsink Error Tracking
|
||||
|
||||
**See**: [docs/tools/BUGSINK-SETUP.md](docs/tools/BUGSINK-SETUP.md) for setup.
|
||||
|
||||
**Quick Access**:
|
||||
|
||||
- **Dev**: https://localhost:8443 (`admin@localhost`/`admin`)
|
||||
- **Prod**: https://bugsink.projectium.com
|
||||
|
||||
**Token Creation** (required for MCP):
|
||||
|
||||
```bash
|
||||
# Dev container
|
||||
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
|
||||
|
||||
# Production (via SSH)
|
||||
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
|
||||
```
|
||||
|
||||
### Logstash
|
||||
|
||||
**See**: [docs/operations/LOGSTASH-QUICK-REF.md](docs/operations/LOGSTASH-QUICK-REF.md)
|
||||
|
||||
Log aggregation: PostgreSQL + PM2 + Redis + NGINX → Bugsink (ADR-050)
|
||||
|
||||
---
|
||||
|
||||
## Documentation Quick Links
|
||||
|
||||
| Topic | Document |
|
||||
| ------------------- | -------------------------------------------------------------- |
|
||||
| **Getting Started** | [QUICKSTART.md](docs/getting-started/QUICKSTART.md) |
|
||||
| **Dev Container** | [DEV-CONTAINER.md](docs/development/DEV-CONTAINER.md) |
|
||||
| **Architecture** | [OVERVIEW.md](docs/architecture/OVERVIEW.md) |
|
||||
| **Code Patterns** | [CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md) |
|
||||
| **Testing** | [TESTING.md](docs/development/TESTING.md) |
|
||||
| **Debugging** | [DEBUGGING.md](docs/development/DEBUGGING.md) |
|
||||
| **Database** | [DATABASE.md](docs/architecture/DATABASE.md) |
|
||||
| **Deployment** | [DEPLOYMENT.md](docs/operations/DEPLOYMENT.md) |
|
||||
| **Monitoring** | [MONITORING.md](docs/operations/MONITORING.md) |
|
||||
| **Logstash** | [LOGSTASH-QUICK-REF.md](docs/operations/LOGSTASH-QUICK-REF.md) |
|
||||
| **ADRs** | [docs/adr/index.md](docs/adr/index.md) |
|
||||
| **All Docs** | [docs/README.md](docs/README.md) |
|
||||
|
||||
---
|
||||
|
||||
## Appendix: Integration Test Issues (Full Details)
|
||||
|
||||
### 1. Vitest globalSetup Context Isolation
|
||||
|
||||
Vitest's `globalSetup` runs in separate Node.js context. Singletons, spies, mocks do NOT share instances with test files.
|
||||
|
||||
**Affected**: BullMQ worker service mocks (AI/DB failure tests)
|
||||
|
||||
**Solutions**: Mark `.todo()`, create test-only API endpoints, use Redis-based mock flags
|
||||
|
||||
```typescript
|
||||
// DOES NOT WORK - different instances
|
||||
const { flyerProcessingService } = await import('../../services/workers.server');
|
||||
flyerProcessingService._getAiProcessor()._setExtractAndValidateData(mockFn);
|
||||
```
|
||||
|
||||
### 2. Cleanup Queue Deletes Before Verification
|
||||
|
||||
Cleanup worker processes jobs in globalSetup context, ignoring test spies.
|
||||
|
||||
**Solution**: Drain and pause queue:
|
||||
|
||||
```typescript
|
||||
const { cleanupQueue } = await import('../../services/queues.server');
|
||||
await cleanupQueue.drain(); // Remove existing jobs
|
||||
await cleanupQueue.pause(); // Prevent new jobs from processing
|
||||
// ... run test ...
|
||||
await cleanupQueue.resume(); // Restore normal operation
|
||||
await cleanupQueue.drain();
|
||||
await cleanupQueue.pause();
|
||||
// ... test ...
|
||||
await cleanupQueue.resume();
|
||||
```
|
||||
|
||||
### 3. Cache Invalidation After Direct Database Inserts
|
||||
### 3. Cache Stale After Direct SQL
|
||||
|
||||
**Problem:** Tests that insert data directly via SQL (bypassing the service layer) don't trigger cache invalidation. Subsequent API calls return stale cached data.
|
||||
Direct `pool.query()` inserts bypass cache invalidation.
|
||||
|
||||
**Affected Tests:** Any test using `pool.query()` to insert flyers, stores, or other cached entities
|
||||
**Solution**: `await cacheService.invalidateFlyers();` after inserts
|
||||
|
||||
**Solution:** Manually invalidate the cache after direct inserts:
|
||||
### 4. Test Filename Collisions
|
||||
|
||||
```typescript
|
||||
await pool.query('INSERT INTO flyers ...');
|
||||
await cacheService.invalidateFlyers(); // Clear stale cache
|
||||
```
|
||||
Multer predictable filenames cause race conditions.
|
||||
|
||||
### 4. Unique Filenames Required for Test Isolation
|
||||
|
||||
**Problem:** Multer generates predictable filenames in test environments, causing race conditions when multiple tests upload files concurrently or in sequence.
|
||||
|
||||
**Affected Tests:** Flyer processing tests, file upload tests
|
||||
|
||||
**Solution:** Always use unique filenames with timestamps:
|
||||
|
||||
```typescript
|
||||
// In multer.middleware.ts
|
||||
const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1e9)}`;
|
||||
cb(null, `${file.fieldname}-${uniqueSuffix}-${sanitizedOriginalName}`);
|
||||
```
|
||||
**Solution**: Use unique suffix: `${Date.now()}-${Math.round(Math.random() * 1e9)}`
|
||||
|
||||
### 5. Response Format Mismatches
|
||||
|
||||
**Problem:** API response formats may change, causing tests to fail when expecting old formats.
|
||||
API formats change: `data.jobId` vs `data.job.id`, nested vs flat, string vs number IDs.
|
||||
|
||||
**Common Issues:**
|
||||
|
||||
- `response.body.data.jobId` vs `response.body.data.job.id`
|
||||
- Nested objects vs flat response structures
|
||||
- Type coercion (string vs number for IDs)
|
||||
|
||||
**Solution:** Always log response bodies during debugging and update test assertions to match actual API contracts.
|
||||
**Solution**: Log response bodies, update assertions
|
||||
|
||||
### 6. External Service Availability
|
||||
|
||||
**Problem:** Tests depending on external services (PM2, Redis health checks) fail when those services aren't available in the test environment.
|
||||
PM2/Redis health checks fail when unavailable.
|
||||
|
||||
**Solution:** Use try/catch with graceful degradation or mock the external service checks.
|
||||
**Solution**: try/catch with graceful degradation or mock
|
||||
|
||||
## Secrets and Environment Variables
|
||||
### 7. TZ Environment Variable Breaking Async Hooks
|
||||
|
||||
**CRITICAL**: This project uses **Gitea CI/CD secrets** for all sensitive configuration. There is NO `/etc/flyer-crawler/environment` file or similar local config file on the server.
|
||||
**Problem**: When `TZ=America/Los_Angeles` (or other timezone values) is set in the environment, Node.js async_hooks module can produce `RangeError: Invalid triggerAsyncId value: NaN`. This breaks React Testing Library's `render()` function which uses async hooks internally.
|
||||
|
||||
### Server Directory Structure
|
||||
**Root Cause**: Setting `TZ` to certain timezone values interferes with Node.js's internal async tracking mechanism, causing invalid async IDs to be generated.
|
||||
|
||||
| Path | Environment | Notes |
|
||||
| --------------------------------------------- | ----------- | ------------------------------------------------ |
|
||||
| `/var/www/flyer-crawler.projectium.com/` | Production | NO `.env` file - secrets injected via CI/CD only |
|
||||
| `/var/www/flyer-crawler-test.projectium.com/` | Test | Has `.env.test` file for test-specific config |
|
||||
**Symptoms**:
|
||||
|
||||
### How Secrets Work
|
||||
|
||||
1. **Gitea Secrets**: All secrets are stored in Gitea repository settings (Settings → Secrets)
|
||||
2. **CI/CD Injection**: Secrets are injected during deployment via `.gitea/workflows/deploy-to-prod.yml` and `deploy-to-test.yml`
|
||||
3. **PM2 Environment**: The CI/CD workflow passes secrets to PM2 via environment variables, which are then available to the application
|
||||
|
||||
### Key Files for Configuration
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------------- | ---------------------------------------------------- |
|
||||
| `src/config/env.ts` | Centralized config with Zod schema validation |
|
||||
| `ecosystem.config.cjs` | PM2 process config - reads from `process.env` |
|
||||
| `.gitea/workflows/deploy-to-prod.yml` | Production deployment with secret injection |
|
||||
| `.gitea/workflows/deploy-to-test.yml` | Test deployment with secret injection |
|
||||
| `.env.example` | Template showing all available environment variables |
|
||||
| `.env.test` | Test environment overrides (only on test server) |
|
||||
|
||||
### Adding New Secrets
|
||||
|
||||
To add a new secret (e.g., `SENTRY_DSN`):
|
||||
|
||||
1. Add the secret to Gitea repository settings
|
||||
2. Update the relevant workflow file (e.g., `deploy-to-prod.yml`) to inject it:
|
||||
|
||||
```yaml
|
||||
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
|
||||
```
|
||||
|
||||
3. Update `ecosystem.config.cjs` to read it from `process.env`
|
||||
4. Update `src/config/env.ts` schema if validation is needed
|
||||
5. Update `.env.example` to document the new variable
|
||||
|
||||
### Current Gitea Secrets
|
||||
|
||||
**Shared (used by both environments):**
|
||||
|
||||
- `DB_HOST` - Database host (shared PostgreSQL server)
|
||||
- `JWT_SECRET` - Authentication
|
||||
- `GOOGLE_MAPS_API_KEY` - Google Maps
|
||||
- `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` - Google OAuth
|
||||
- `GH_CLIENT_ID`, `GH_CLIENT_SECRET` - GitHub OAuth
|
||||
- `SENTRY_AUTH_TOKEN` - Bugsink API token for source map uploads (create at Settings > API Keys in Bugsink)
|
||||
|
||||
**Production-specific:**
|
||||
|
||||
- `DB_USER_PROD`, `DB_PASSWORD_PROD` - Production database credentials (`flyer_crawler_prod`)
|
||||
- `DB_DATABASE_PROD` - Production database name (`flyer-crawler`)
|
||||
- `REDIS_PASSWORD_PROD` - Redis password (uses database 0)
|
||||
- `VITE_GOOGLE_GENAI_API_KEY` - Gemini API key for production
|
||||
- `SENTRY_DSN`, `VITE_SENTRY_DSN` - Bugsink error tracking DSNs (production projects)
|
||||
|
||||
**Test-specific:**
|
||||
|
||||
- `DB_USER_TEST`, `DB_PASSWORD_TEST` - Test database credentials (`flyer_crawler_test`)
|
||||
- `DB_DATABASE_TEST` - Test database name (`flyer-crawler-test`)
|
||||
- `REDIS_PASSWORD_TEST` - Redis password (uses database 1 for isolation)
|
||||
- `VITE_GOOGLE_GENAI_API_KEY_TEST` - Gemini API key for test
|
||||
- `SENTRY_DSN_TEST`, `VITE_SENTRY_DSN_TEST` - Bugsink error tracking DSNs (test projects)
|
||||
|
||||
### Test Environment
|
||||
|
||||
The test environment (`flyer-crawler-test.projectium.com`) uses **both** Gitea CI/CD secrets and a local `.env.test` file:
|
||||
|
||||
- **Gitea secrets**: Injected during deployment via `.gitea/workflows/deploy-to-test.yml`
|
||||
- **`.env.test` file**: Located at `/var/www/flyer-crawler-test.projectium.com/.env.test` for local overrides
|
||||
- **Redis database 1**: Isolates test job queues from production (which uses database 0)
|
||||
- **PM2 process names**: Suffixed with `-test` (e.g., `flyer-crawler-api-test`)
|
||||
|
||||
### Database User Setup (Test Environment)
|
||||
|
||||
**CRITICAL**: The test database requires specific PostgreSQL permissions to be configured manually. Schema ownership alone is NOT sufficient - explicit privileges must be granted.
|
||||
|
||||
**Database Users:**
|
||||
|
||||
| User | Database | Purpose |
|
||||
| -------------------- | -------------------- | ---------- |
|
||||
| `flyer_crawler_prod` | `flyer-crawler-prod` | Production |
|
||||
| `flyer_crawler_test` | `flyer-crawler-test` | Testing |
|
||||
|
||||
**Required Setup Commands** (run as `postgres` superuser):
|
||||
|
||||
```bash
|
||||
# Connect as postgres superuser
|
||||
sudo -u postgres psql
|
||||
|
||||
# Create the test database and user (if not exists)
|
||||
CREATE DATABASE "flyer-crawler-test";
|
||||
CREATE USER flyer_crawler_test WITH PASSWORD 'your-password-here';
|
||||
|
||||
# Grant ownership and privileges
|
||||
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
|
||||
\c "flyer-crawler-test"
|
||||
ALTER SCHEMA public OWNER TO flyer_crawler_test;
|
||||
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
|
||||
|
||||
# Create required extension (must be done by superuser)
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
```text
|
||||
RangeError: Invalid triggerAsyncId value: NaN
|
||||
❯ process.env.NODE_ENV.queueSeveralMicrotasks node_modules/react/cjs/react.development.js:751:15
|
||||
❯ process.env.NODE_ENV.exports.act node_modules/react/cjs/react.development.js:886:11
|
||||
❯ node_modules/@testing-library/react/dist/act-compat.js:46:25
|
||||
❯ renderRoot node_modules/@testing-library/react/dist/pure.js:189:26
|
||||
```
|
||||
|
||||
**Why These Steps Are Necessary:**
|
||||
**Solution**: Explicitly unset `TZ` in all test scripts by adding `TZ=` (empty value) to cross-env:
|
||||
|
||||
1. **Schema ownership alone is insufficient** - PostgreSQL requires explicit `GRANT CREATE, USAGE` privileges even when the user owns the schema
|
||||
2. **uuid-ossp extension** - Required by the application for UUID generation; must be created by a superuser before the app can use it
|
||||
3. **Separate users for prod/test** - Prevents accidental cross-environment data access; each environment has its own credentials in Gitea secrets
|
||||
|
||||
**Verification:**
|
||||
|
||||
```bash
|
||||
# Check schema privileges (should show 'UC' for flyer_crawler_test)
|
||||
psql -d "flyer-crawler-test" -c "\dn+ public"
|
||||
|
||||
# Expected output:
|
||||
# Name | Owner | Access privileges
|
||||
# -------+--------------------+------------------------------------------
|
||||
# public | flyer_crawler_test | flyer_crawler_test=UC/flyer_crawler_test
|
||||
```json
|
||||
"test:unit": "cross-env NODE_ENV=test TZ= tsx ..."
|
||||
"test:integration": "cross-env NODE_ENV=test TZ= tsx ..."
|
||||
```
|
||||
|
||||
### Dev Container Environment
|
||||
|
||||
The dev container runs its own **local Bugsink instance** - it does NOT connect to the production Bugsink server:
|
||||
|
||||
- **Local Bugsink**: Runs at `http://localhost:8000` inside the container
|
||||
- **Pre-configured DSNs**: Set in `compose.dev.yml`, pointing to local instance
|
||||
- **Admin credentials**: `admin@localhost` / `admin`
|
||||
- **Isolated**: Dev errors stay local, don't pollute production/test dashboards
|
||||
- **No Gitea secrets needed**: Everything is self-contained in the container
|
||||
|
||||
---
|
||||
|
||||
## MCP Servers
|
||||
|
||||
The following MCP servers are configured for this project:
|
||||
|
||||
| Server | Purpose |
|
||||
| --------------------- | ------------------------------------------- |
|
||||
| gitea-projectium | Gitea API for gitea.projectium.com |
|
||||
| gitea-torbonium | Gitea API for gitea.torbonium.com |
|
||||
| podman | Container management |
|
||||
| filesystem | File system access |
|
||||
| fetch | Web fetching |
|
||||
| markitdown | Convert documents to markdown |
|
||||
| sequential-thinking | Step-by-step reasoning |
|
||||
| memory | Knowledge graph persistence |
|
||||
| postgres | Direct database queries (localhost:5432) |
|
||||
| playwright | Browser automation and testing |
|
||||
| redis | Redis cache inspection (localhost:6379) |
|
||||
| sentry-selfhosted-mcp | Error tracking via Bugsink (localhost:8000) |
|
||||
|
||||
**Note:** MCP servers work in both **Claude CLI** and **Claude Code VS Code extension** (as of January 2026).
|
||||
|
||||
### Sentry/Bugsink MCP Server Setup (ADR-015)
|
||||
|
||||
To enable Claude Code to query and analyze application errors from Bugsink:
|
||||
|
||||
1. **Install the MCP server**:
|
||||
|
||||
```bash
|
||||
# Clone the sentry-selfhosted-mcp repository
|
||||
git clone https://github.com/ddfourtwo/sentry-selfhosted-mcp.git
|
||||
cd sentry-selfhosted-mcp
|
||||
npm install
|
||||
```
|
||||
|
||||
2. **Configure Claude Code** (add to `.claude/mcp.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"sentry-selfhosted-mcp": {
|
||||
"command": "node",
|
||||
"args": ["/path/to/sentry-selfhosted-mcp/dist/index.js"],
|
||||
"env": {
|
||||
"SENTRY_URL": "http://localhost:8000",
|
||||
"SENTRY_AUTH_TOKEN": "<get-from-bugsink-ui>",
|
||||
"SENTRY_ORG_SLUG": "flyer-crawler"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. **Get the auth token**:
|
||||
- Navigate to Bugsink UI at `http://localhost:8000`
|
||||
- Log in with admin credentials
|
||||
- Go to Settings > API Keys
|
||||
- Create a new API key with read access
|
||||
|
||||
4. **Available capabilities**:
|
||||
- List projects and issues
|
||||
- View detailed error events
|
||||
- Search by error message or stack trace
|
||||
- Update issue status (resolve, ignore)
|
||||
- Add comments to issues
|
||||
|
||||
### SSH Server Access
|
||||
|
||||
Claude Code can execute commands on the production server via SSH:
|
||||
|
||||
```bash
|
||||
# Basic command execution
|
||||
ssh root@projectium.com "command here"
|
||||
|
||||
# Examples:
|
||||
ssh root@projectium.com "systemctl status logstash"
|
||||
ssh root@projectium.com "pm2 list"
|
||||
ssh root@projectium.com "tail -50 /var/www/flyer-crawler.projectium.com/logs/app.log"
|
||||
```
|
||||
|
||||
**Use cases:**
|
||||
|
||||
- Managing Logstash, PM2, NGINX, Redis services
|
||||
- Viewing server logs
|
||||
- Deploying configuration changes
|
||||
- Checking service status
|
||||
|
||||
**Important:** SSH access requires the host machine to have SSH keys configured for `root@projectium.com`.
|
||||
|
||||
---
|
||||
|
||||
## Logstash Configuration (ADR-050)
|
||||
|
||||
The production server uses **Logstash** to aggregate logs from multiple sources and forward errors to Bugsink for centralized error tracking.
|
||||
|
||||
**Log Sources:**
|
||||
|
||||
- **PostgreSQL function logs** - Structured JSON logs from `fn_log()` helper function
|
||||
- **PM2 worker logs** - Service logs from BullMQ job workers (stdout)
|
||||
- **Redis logs** - Operational logs (INFO level) and errors
|
||||
- **NGINX logs** - Access logs (all requests) and error logs
|
||||
|
||||
### Configuration Location
|
||||
|
||||
**Primary configuration file:**
|
||||
|
||||
- `/etc/logstash/conf.d/bugsink.conf` - Complete Logstash pipeline configuration
|
||||
|
||||
**Related files:**
|
||||
|
||||
- `/etc/postgresql/14/main/conf.d/observability.conf` - PostgreSQL logging configuration
|
||||
- `/var/log/postgresql/*.log` - PostgreSQL log files
|
||||
- `/home/gitea-runner/.pm2/logs/*.log` - PM2 worker logs
|
||||
- `/var/log/redis/redis-server.log` - Redis logs
|
||||
- `/var/log/nginx/access.log` - NGINX access logs
|
||||
- `/var/log/nginx/error.log` - NGINX error logs
|
||||
- `/var/log/logstash/*.log` - Logstash file outputs (operational logs)
|
||||
- `/var/lib/logstash/sincedb_*` - Logstash position tracking files
|
||||
|
||||
### Key Features
|
||||
|
||||
1. **Multi-source aggregation**: Collects logs from PostgreSQL, PM2 workers, Redis, and NGINX
|
||||
2. **Environment-based routing**: Automatically detects production vs test environments and routes errors to the correct Bugsink project
|
||||
3. **Structured JSON parsing**: Extracts `fn_log()` function output from PostgreSQL logs and Pino JSON from PM2 workers
|
||||
4. **Sentry-compatible format**: Transforms events to Sentry format with `event_id`, `timestamp`, `level`, `message`, and `extra` context
|
||||
5. **Error filtering**: Only forwards WARNING and ERROR level messages to Bugsink
|
||||
6. **Operational log storage**: Stores non-error logs (Redis INFO, NGINX access, PM2 operational) to `/var/log/logstash/` for analysis
|
||||
7. **Request monitoring**: Categorizes NGINX requests by status code (2xx, 3xx, 4xx, 5xx) and identifies slow requests
|
||||
|
||||
### Common Maintenance Commands
|
||||
|
||||
```bash
|
||||
# Check Logstash status
|
||||
systemctl status logstash
|
||||
|
||||
# Restart Logstash after configuration changes
|
||||
systemctl restart logstash
|
||||
|
||||
# Test configuration syntax
|
||||
/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf
|
||||
|
||||
# View Logstash logs
|
||||
journalctl -u logstash -f
|
||||
|
||||
# Check Logstash stats (events processed, failures)
|
||||
curl -XGET 'localhost:9600/_node/stats/pipelines?pretty' | jq '.pipelines.main.plugins.filters'
|
||||
|
||||
# Monitor PostgreSQL logs being processed
|
||||
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log
|
||||
|
||||
# View operational log outputs
|
||||
tail -f /var/log/logstash/pm2-workers-$(date +%Y-%m-%d).log
|
||||
tail -f /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
|
||||
tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
|
||||
|
||||
# Check disk usage of log files
|
||||
du -sh /var/log/logstash/
|
||||
```
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
| Issue | Check | Solution |
|
||||
| ------------------------------- | ---------------------------- | ---------------------------------------------------------------------------------------------- |
|
||||
| Errors not appearing in Bugsink | Check Logstash is running | `systemctl status logstash` |
|
||||
| Configuration syntax errors | Test config file | `/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf` |
|
||||
| Grok pattern failures | Check Logstash stats | `curl localhost:9600/_node/stats/pipelines?pretty \| jq '.pipelines.main.plugins.filters'` |
|
||||
| Wrong Bugsink project | Verify environment detection | Check tags in logs match expected environment (production/test) |
|
||||
| Permission denied reading logs | Check Logstash permissions | `groups logstash` should include `postgres`, `adm` groups |
|
||||
| PM2 logs not captured | Check file paths exist | `ls /home/gitea-runner/.pm2/logs/flyer-crawler-worker-*.log` |
|
||||
| NGINX access logs not showing | Check file output directory | `ls -lh /var/log/logstash/nginx-access-*.log` |
|
||||
| High disk usage | Check log rotation | Verify `/etc/logrotate.d/logstash` is configured and running daily |
|
||||
|
||||
**Full setup guide**: See [docs/BARE-METAL-SETUP.md](docs/BARE-METAL-SETUP.md) section "PostgreSQL Function Observability (ADR-050)"
|
||||
|
||||
**Architecture details**: See [docs/adr/0050-postgresql-function-observability.md](docs/adr/0050-postgresql-function-observability.md)
|
||||
**Context**: This issue was introduced in commit `d03900c` which added `TZ: 'America/Los_Angeles'` to PM2 ecosystem configs for consistent log timestamps in production/dev environments. Tests must explicitly override this to prevent the async hooks error.
|
||||
|
||||
660
CLAUDE.md.backup
Normal file
660
CLAUDE.md.backup
Normal file
@@ -0,0 +1,660 @@
|
||||
# Claude Code Project Instructions
|
||||
|
||||
## Session Startup Checklist
|
||||
|
||||
**IMPORTANT**: At the start of every session, perform these steps:
|
||||
|
||||
1. **Check Memory First** - Use `mcp__memory__read_graph` or `mcp__memory__search_nodes` to recall:
|
||||
- Project-specific configurations and credentials
|
||||
- Previous work context and decisions
|
||||
- Infrastructure details (URLs, ports, access patterns)
|
||||
- Known issues and their solutions
|
||||
|
||||
2. **Review Recent Git History** - Check `git log --oneline -10` to understand recent changes
|
||||
|
||||
3. **Check Container Status** - Use `mcp__podman__container_list` to see what's running
|
||||
|
||||
---
|
||||
|
||||
## Project Instructions
|
||||
|
||||
### Things to Remember
|
||||
|
||||
Before writing any code:
|
||||
|
||||
1. State how you will verify this change works (test, bash command, browser check, etc.)
|
||||
|
||||
2. Write the test or verification step first
|
||||
|
||||
3. Then implement the code
|
||||
|
||||
4. Run verification and iterate until it passes
|
||||
|
||||
## Git Bash / MSYS Path Conversion Issue (Windows Host)
|
||||
|
||||
**CRITICAL ISSUE**: Git Bash on Windows automatically converts Unix-style paths to Windows paths, which breaks Podman/Docker commands.
|
||||
|
||||
### Problem Examples:
|
||||
|
||||
```bash
|
||||
# This FAILS in Git Bash:
|
||||
podman exec container /usr/local/bin/script.sh
|
||||
# Git Bash converts to: C:/Program Files/Git/usr/local/bin/script.sh
|
||||
|
||||
# This FAILS in Git Bash:
|
||||
podman exec container bash -c "cat /tmp/file.sql"
|
||||
# Git Bash converts /tmp to C:/Users/user/AppData/Local/Temp
|
||||
```
|
||||
|
||||
### Solutions:
|
||||
|
||||
1. **Use `sh -c` instead of `bash -c`** for single-quoted commands:
|
||||
|
||||
```bash
|
||||
podman exec container sh -c '/usr/local/bin/script.sh'
|
||||
```
|
||||
|
||||
2. **Use double slashes** to escape path conversion:
|
||||
|
||||
```bash
|
||||
podman exec container //usr//local//bin//script.sh
|
||||
```
|
||||
|
||||
3. **Set MSYS_NO_PATHCONV** environment variable:
|
||||
|
||||
```bash
|
||||
MSYS_NO_PATHCONV=1 podman exec container /usr/local/bin/script.sh
|
||||
```
|
||||
|
||||
4. **Use Windows paths with forward slashes** when referencing host files:
|
||||
```bash
|
||||
podman cp "d:/path/to/file" container:/tmp/file
|
||||
```
|
||||
|
||||
**ALWAYS use one of these workarounds when running Bash commands on Windows that involve Unix paths inside containers.**
|
||||
|
||||
## Communication Style: Ask Before Assuming
|
||||
|
||||
**IMPORTANT**: When helping with tasks, **ask clarifying questions before making assumptions**. Do not assume:
|
||||
|
||||
- What steps the user has or hasn't completed
|
||||
- What the user already knows or has configured
|
||||
- What external services (OAuth providers, APIs, etc.) are already set up
|
||||
- What secrets or credentials have already been created
|
||||
|
||||
Instead, ask the user to confirm the current state before providing instructions or making recommendations. This prevents wasted effort and respects the user's existing work.
|
||||
|
||||
## Platform Requirement: Linux Only
|
||||
|
||||
**CRITICAL**: This application is designed to run **exclusively on Linux**. See [ADR-014](docs/adr/0014-containerization-and-deployment-strategy.md) for full details.
|
||||
|
||||
### Environment Terminology
|
||||
|
||||
- **Dev Container** (or just "dev"): The containerized Linux development environment (`flyer-crawler-dev`). This is where all development and testing should occur.
|
||||
- **Host**: The Windows machine running Podman/Docker and VS Code.
|
||||
|
||||
When instructions say "run in dev" or "run in the dev container", they mean executing commands inside the `flyer-crawler-dev` container.
|
||||
|
||||
### Test Execution Rules
|
||||
|
||||
1. **ALL tests MUST be executed in the dev container** - the Linux container environment
|
||||
2. **NEVER run tests directly on Windows host** - test results from Windows are unreliable
|
||||
3. **Always use the dev container for testing** when developing on Windows
|
||||
4. **TypeScript type-check MUST run in dev container** - `npm run type-check` on Windows does not reliably detect errors
|
||||
|
||||
See [docs/TESTING.md](docs/TESTING.md) for comprehensive testing documentation.
|
||||
|
||||
### How to Run Tests Correctly
|
||||
|
||||
```bash
|
||||
# If on Windows, first open VS Code and "Reopen in Container"
|
||||
# Then run tests inside the dev container:
|
||||
npm test # Run all unit tests
|
||||
npm run test:unit # Run unit tests only
|
||||
npm run test:integration # Run integration tests (requires DB/Redis)
|
||||
```
|
||||
|
||||
### Running Tests via Podman (from Windows host)
|
||||
|
||||
**Note:** This project has 2900+ unit tests. For AI-assisted development, pipe output to a file for easier processing.
|
||||
|
||||
The command to run unit tests in the dev container via podman:
|
||||
|
||||
```bash
|
||||
# Basic (output to terminal)
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
|
||||
# Recommended for AI processing: pipe to file
|
||||
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
|
||||
```
|
||||
|
||||
The command to run integration tests in the dev container via podman:
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev npm run test:integration
|
||||
```
|
||||
|
||||
For running specific test files:
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev npm test -- --run src/hooks/useAuth.test.tsx
|
||||
```
|
||||
|
||||
### Why Linux Only?
|
||||
|
||||
- Path separators: Code uses POSIX-style paths (`/`) which may break on Windows
|
||||
- Shell scripts in `scripts/` directory are Linux-only
|
||||
- External dependencies like `pdftocairo` assume Linux installation paths
|
||||
- Unix-style file permissions are assumed throughout
|
||||
|
||||
### Test Result Interpretation
|
||||
|
||||
- Tests that **pass on Windows but fail on Linux** = **BROKEN tests** (must be fixed)
|
||||
- Tests that **fail on Windows but pass on Linux** = **PASSING tests** (acceptable)
|
||||
|
||||
## Development Workflow
|
||||
|
||||
1. Open project in VS Code
|
||||
2. Use "Reopen in Container" (Dev Containers extension required) to enter the dev environment
|
||||
3. Wait for dev container initialization to complete
|
||||
4. Run `npm test` to verify the dev environment is working
|
||||
5. Make changes and run tests inside the dev container
|
||||
|
||||
## Code Change Verification
|
||||
|
||||
After making any code changes, **always run a type-check** to catch TypeScript errors before committing:
|
||||
|
||||
```bash
|
||||
npm run type-check
|
||||
```
|
||||
|
||||
This prevents linting/type errors from being introduced into the codebase.
|
||||
|
||||
## Quick Reference
|
||||
|
||||
| Command | Description |
|
||||
| -------------------------- | ---------------------------- |
|
||||
| `npm test` | Run all unit tests |
|
||||
| `npm run test:unit` | Run unit tests only |
|
||||
| `npm run test:integration` | Run integration tests |
|
||||
| `npm run dev:container` | Start dev server (container) |
|
||||
| `npm run build` | Build for production |
|
||||
| `npm run type-check` | Run TypeScript type checking |
|
||||
|
||||
## Database Schema Files
|
||||
|
||||
**CRITICAL**: The database schema files must be kept in sync with each other. When making schema changes:
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------ | ----------------------------------------------------------- |
|
||||
| `sql/master_schema_rollup.sql` | Complete schema used by test database setup and reference |
|
||||
| `sql/initial_schema.sql` | Base schema without seed data, used as standalone reference |
|
||||
| `sql/migrations/*.sql` | Incremental migrations for production database updates |
|
||||
|
||||
**Maintenance Rules:**
|
||||
|
||||
1. **Keep `master_schema_rollup.sql` and `initial_schema.sql` in sync** - These files should contain the same table definitions
|
||||
2. **When adding columns via migration**, also add them to both `master_schema_rollup.sql` and `initial_schema.sql`
|
||||
3. **Migrations are for production deployments** - They use `ALTER TABLE` to add columns incrementally
|
||||
4. **Schema files are for fresh installs** - They define the complete table structure
|
||||
5. **Test database uses `master_schema_rollup.sql`** - If schema files are out of sync with migrations, tests will fail
|
||||
|
||||
**Example:** When `002_expiry_tracking.sql` adds `purchase_date` to `pantry_items`, that column must also exist in the `CREATE TABLE` statements in both `master_schema_rollup.sql` and `initial_schema.sql`.
|
||||
|
||||
## Known Integration Test Issues and Solutions
|
||||
|
||||
This section documents common test issues encountered in integration tests, their root causes, and solutions. These patterns recur frequently.
|
||||
|
||||
### 1. Vitest globalSetup Runs in Separate Node.js Context
|
||||
|
||||
**Problem:** Vitest's `globalSetup` runs in a completely separate Node.js context from test files. This means:
|
||||
|
||||
- Singletons created in globalSetup are NOT the same instances as those in test files
|
||||
- `global`, `globalThis`, and `process` are all isolated between contexts
|
||||
- `vi.spyOn()` on module exports doesn't work cross-context
|
||||
- Dependency injection via setter methods fails across contexts
|
||||
|
||||
**Affected Tests:** Any test trying to inject mocks into BullMQ worker services (e.g., AI failure tests, DB failure tests)
|
||||
|
||||
**Solution Options:**
|
||||
|
||||
1. Mark tests as `.todo()` until an API-based mock injection mechanism is implemented
|
||||
2. Create test-only API endpoints that allow setting mock behaviors via HTTP
|
||||
3. Use file-based or Redis-based mock flags that services check at runtime
|
||||
|
||||
**Example of affected code pattern:**
|
||||
|
||||
```typescript
|
||||
// This DOES NOT work - different module instances
|
||||
const { flyerProcessingService } = await import('../../services/workers.server');
|
||||
flyerProcessingService._getAiProcessor()._setExtractAndValidateData(mockFn);
|
||||
// The worker uses a different flyerProcessingService instance!
|
||||
```
|
||||
|
||||
### 2. BullMQ Cleanup Queue Deleting Files Before Test Verification
|
||||
|
||||
**Problem:** The cleanup worker runs in the globalSetup context and processes cleanup jobs even when tests spy on `cleanupQueue.add()`. The spy intercepts calls in the test context, but jobs already queued run in the worker's context.
|
||||
|
||||
**Affected Tests:** EXIF/PNG metadata stripping tests that need to verify file contents before deletion
|
||||
|
||||
**Solution:** Drain and pause the cleanup queue before the test:
|
||||
|
||||
```typescript
|
||||
const { cleanupQueue } = await import('../../services/queues.server');
|
||||
await cleanupQueue.drain(); // Remove existing jobs
|
||||
await cleanupQueue.pause(); // Prevent new jobs from processing
|
||||
// ... run test ...
|
||||
await cleanupQueue.resume(); // Restore normal operation
|
||||
```
|
||||
|
||||
### 3. Cache Invalidation After Direct Database Inserts
|
||||
|
||||
**Problem:** Tests that insert data directly via SQL (bypassing the service layer) don't trigger cache invalidation. Subsequent API calls return stale cached data.
|
||||
|
||||
**Affected Tests:** Any test using `pool.query()` to insert flyers, stores, or other cached entities
|
||||
|
||||
**Solution:** Manually invalidate the cache after direct inserts:
|
||||
|
||||
```typescript
|
||||
await pool.query('INSERT INTO flyers ...');
|
||||
await cacheService.invalidateFlyers(); // Clear stale cache
|
||||
```
|
||||
|
||||
### 4. Unique Filenames Required for Test Isolation
|
||||
|
||||
**Problem:** Multer generates predictable filenames in test environments, causing race conditions when multiple tests upload files concurrently or in sequence.
|
||||
|
||||
**Affected Tests:** Flyer processing tests, file upload tests
|
||||
|
||||
**Solution:** Always use unique filenames with timestamps:
|
||||
|
||||
```typescript
|
||||
// In multer.middleware.ts
|
||||
const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1e9)}`;
|
||||
cb(null, `${file.fieldname}-${uniqueSuffix}-${sanitizedOriginalName}`);
|
||||
```
|
||||
|
||||
### 5. Response Format Mismatches
|
||||
|
||||
**Problem:** API response formats may change, causing tests to fail when expecting old formats.
|
||||
|
||||
**Common Issues:**
|
||||
|
||||
- `response.body.data.jobId` vs `response.body.data.job.id`
|
||||
- Nested objects vs flat response structures
|
||||
- Type coercion (string vs number for IDs)
|
||||
|
||||
**Solution:** Always log response bodies during debugging and update test assertions to match actual API contracts.
|
||||
|
||||
### 6. External Service Availability
|
||||
|
||||
**Problem:** Tests depending on external services (PM2, Redis health checks) fail when those services aren't available in the test environment.
|
||||
|
||||
**Solution:** Use try/catch with graceful degradation or mock the external service checks.
|
||||
|
||||
## Secrets and Environment Variables
|
||||
|
||||
**CRITICAL**: This project uses **Gitea CI/CD secrets** for all sensitive configuration. There is NO `/etc/flyer-crawler/environment` file or similar local config file on the server.
|
||||
|
||||
### Server Directory Structure
|
||||
|
||||
| Path | Environment | Notes |
|
||||
| --------------------------------------------- | ----------- | ------------------------------------------------ |
|
||||
| `/var/www/flyer-crawler.projectium.com/` | Production | NO `.env` file - secrets injected via CI/CD only |
|
||||
| `/var/www/flyer-crawler-test.projectium.com/` | Test | Has `.env.test` file for test-specific config |
|
||||
|
||||
### How Secrets Work
|
||||
|
||||
1. **Gitea Secrets**: All secrets are stored in Gitea repository settings (Settings → Secrets)
|
||||
2. **CI/CD Injection**: Secrets are injected during deployment via `.gitea/workflows/deploy-to-prod.yml` and `deploy-to-test.yml`
|
||||
3. **PM2 Environment**: The CI/CD workflow passes secrets to PM2 via environment variables, which are then available to the application
|
||||
|
||||
### Key Files for Configuration
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------------- | ---------------------------------------------------- |
|
||||
| `src/config/env.ts` | Centralized config with Zod schema validation |
|
||||
| `ecosystem.config.cjs` | PM2 process config - reads from `process.env` |
|
||||
| `.gitea/workflows/deploy-to-prod.yml` | Production deployment with secret injection |
|
||||
| `.gitea/workflows/deploy-to-test.yml` | Test deployment with secret injection |
|
||||
| `.env.example` | Template showing all available environment variables |
|
||||
| `.env.test` | Test environment overrides (only on test server) |
|
||||
|
||||
### Adding New Secrets
|
||||
|
||||
To add a new secret (e.g., `SENTRY_DSN`):
|
||||
|
||||
1. Add the secret to Gitea repository settings
|
||||
2. Update the relevant workflow file (e.g., `deploy-to-prod.yml`) to inject it:
|
||||
|
||||
```yaml
|
||||
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
|
||||
```
|
||||
|
||||
3. Update `ecosystem.config.cjs` to read it from `process.env`
|
||||
4. Update `src/config/env.ts` schema if validation is needed
|
||||
5. Update `.env.example` to document the new variable
|
||||
|
||||
### Current Gitea Secrets
|
||||
|
||||
**Shared (used by both environments):**
|
||||
|
||||
- `DB_HOST` - Database host (shared PostgreSQL server)
|
||||
- `JWT_SECRET` - Authentication
|
||||
- `GOOGLE_MAPS_API_KEY` - Google Maps
|
||||
- `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` - Google OAuth
|
||||
- `GH_CLIENT_ID`, `GH_CLIENT_SECRET` - GitHub OAuth
|
||||
- `SENTRY_AUTH_TOKEN` - Bugsink API token for source map uploads (create at Settings > API Keys in Bugsink)
|
||||
|
||||
**Production-specific:**
|
||||
|
||||
- `DB_USER_PROD`, `DB_PASSWORD_PROD` - Production database credentials (`flyer_crawler_prod`)
|
||||
- `DB_DATABASE_PROD` - Production database name (`flyer-crawler`)
|
||||
- `REDIS_PASSWORD_PROD` - Redis password (uses database 0)
|
||||
- `VITE_GOOGLE_GENAI_API_KEY` - Gemini API key for production
|
||||
- `SENTRY_DSN`, `VITE_SENTRY_DSN` - Bugsink error tracking DSNs (production projects)
|
||||
|
||||
**Test-specific:**
|
||||
|
||||
- `DB_USER_TEST`, `DB_PASSWORD_TEST` - Test database credentials (`flyer_crawler_test`)
|
||||
- `DB_DATABASE_TEST` - Test database name (`flyer-crawler-test`)
|
||||
- `REDIS_PASSWORD_TEST` - Redis password (uses database 1 for isolation)
|
||||
- `VITE_GOOGLE_GENAI_API_KEY_TEST` - Gemini API key for test
|
||||
- `SENTRY_DSN_TEST`, `VITE_SENTRY_DSN_TEST` - Bugsink error tracking DSNs (test projects)
|
||||
|
||||
### Test Environment
|
||||
|
||||
The test environment (`flyer-crawler-test.projectium.com`) uses **both** Gitea CI/CD secrets and a local `.env.test` file:
|
||||
|
||||
- **Gitea secrets**: Injected during deployment via `.gitea/workflows/deploy-to-test.yml`
|
||||
- **`.env.test` file**: Located at `/var/www/flyer-crawler-test.projectium.com/.env.test` for local overrides
|
||||
- **Redis database 1**: Isolates test job queues from production (which uses database 0)
|
||||
- **PM2 process names**: Suffixed with `-test` (e.g., `flyer-crawler-api-test`)
|
||||
|
||||
### Database User Setup (Test Environment)
|
||||
|
||||
**CRITICAL**: The test database requires specific PostgreSQL permissions to be configured manually. Schema ownership alone is NOT sufficient - explicit privileges must be granted.
|
||||
|
||||
**Database Users:**
|
||||
|
||||
| User | Database | Purpose |
|
||||
| -------------------- | -------------------- | ---------- |
|
||||
| `flyer_crawler_prod` | `flyer-crawler-prod` | Production |
|
||||
| `flyer_crawler_test` | `flyer-crawler-test` | Testing |
|
||||
|
||||
**Required Setup Commands** (run as `postgres` superuser):
|
||||
|
||||
```bash
|
||||
# Connect as postgres superuser
|
||||
sudo -u postgres psql
|
||||
|
||||
# Create the test database and user (if not exists)
|
||||
CREATE DATABASE "flyer-crawler-test";
|
||||
CREATE USER flyer_crawler_test WITH PASSWORD 'your-password-here';
|
||||
|
||||
# Grant ownership and privileges
|
||||
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
|
||||
\c "flyer-crawler-test"
|
||||
ALTER SCHEMA public OWNER TO flyer_crawler_test;
|
||||
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
|
||||
|
||||
# Create required extension (must be done by superuser)
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
```
|
||||
|
||||
**Why These Steps Are Necessary:**
|
||||
|
||||
1. **Schema ownership alone is insufficient** - PostgreSQL requires explicit `GRANT CREATE, USAGE` privileges even when the user owns the schema
|
||||
2. **uuid-ossp extension** - Required by the application for UUID generation; must be created by a superuser before the app can use it
|
||||
3. **Separate users for prod/test** - Prevents accidental cross-environment data access; each environment has its own credentials in Gitea secrets
|
||||
|
||||
**Verification:**
|
||||
|
||||
```bash
|
||||
# Check schema privileges (should show 'UC' for flyer_crawler_test)
|
||||
psql -d "flyer-crawler-test" -c "\dn+ public"
|
||||
|
||||
# Expected output:
|
||||
# Name | Owner | Access privileges
|
||||
# -------+--------------------+------------------------------------------
|
||||
# public | flyer_crawler_test | flyer_crawler_test=UC/flyer_crawler_test
|
||||
```
|
||||
|
||||
### Dev Container Environment
|
||||
|
||||
The dev container runs its own **local Bugsink instance** - it does NOT connect to the production Bugsink server:
|
||||
|
||||
- **Local Bugsink UI**: Accessible at `https://localhost:8443` (proxied from `http://localhost:8000` by nginx)
|
||||
- **Admin credentials**: `admin@localhost` / `admin`
|
||||
- **Bugsink Projects**: Backend (Dev) - Project ID 1, Frontend (Dev) - Project ID 2
|
||||
- **Configuration Files**:
|
||||
- `compose.dev.yml` - Sets default DSNs using `127.0.0.1:8000` protocol (for initial container setup)
|
||||
- `.env.local` - **OVERRIDES** compose.dev.yml with `localhost:8000` protocol (this is what the app actually uses)
|
||||
- **CRITICAL**: `.env.local` takes precedence over `compose.dev.yml` environment variables
|
||||
- **DSN Configuration**:
|
||||
- **Backend DSN** (Node.js/Express): Configured in `.env.local` as `SENTRY_DSN=http://<key>@localhost:8000/1`
|
||||
- **Frontend DSN** (React/Browser): Configured in `.env.local` as `VITE_SENTRY_DSN=http://<key>@localhost:8000/2`
|
||||
- **Why localhost instead of 127.0.0.1?** The `.env.local` file was created separately and uses `localhost` which works fine in practice
|
||||
- **HTTPS Setup**: Self-signed certificates auto-generated with mkcert on container startup (for UI access only, not for Sentry SDK)
|
||||
- **CSRF Protection**: Django configured with `SECURE_PROXY_SSL_HEADER` to trust `X-Forwarded-Proto` from nginx
|
||||
- **Isolated**: Dev errors stay local, don't pollute production/test dashboards
|
||||
- **No Gitea secrets needed**: Everything is self-contained in the container
|
||||
- **Accessing Errors**:
|
||||
- **Via Browser**: Open `https://localhost:8443` and login to view issues
|
||||
- **Via MCP**: Configure a second Bugsink MCP server pointing to `http://localhost:8000` (see MCP Servers section below)
|
||||
|
||||
---
|
||||
|
||||
## MCP Servers
|
||||
|
||||
The following MCP servers are configured for this project:
|
||||
|
||||
| Server | Purpose |
|
||||
| ------------------- | ---------------------------------------------------------------------------- |
|
||||
| gitea-projectium | Gitea API for gitea.projectium.com |
|
||||
| gitea-torbonium | Gitea API for gitea.torbonium.com |
|
||||
| podman | Container management |
|
||||
| filesystem | File system access |
|
||||
| fetch | Web fetching |
|
||||
| markitdown | Convert documents to markdown |
|
||||
| sequential-thinking | Step-by-step reasoning |
|
||||
| memory | Knowledge graph persistence |
|
||||
| postgres | Direct database queries (localhost:5432) |
|
||||
| playwright | Browser automation and testing |
|
||||
| redis | Redis cache inspection (localhost:6379) |
|
||||
| bugsink | Error tracking - production Bugsink (bugsink.projectium.com) - **PROD/TEST** |
|
||||
| bugsink-dev | Error tracking - dev container Bugsink (localhost:8000) - **DEV CONTAINER** |
|
||||
|
||||
**Note:** MCP servers work in both **Claude CLI** and **Claude Code VS Code extension** (as of January 2026).
|
||||
|
||||
**CRITICAL**: There are **TWO separate Bugsink MCP servers**:
|
||||
|
||||
- **bugsink**: Connects to production Bugsink at `https://bugsink.projectium.com` for production and test server errors
|
||||
- **bugsink-dev**: Connects to local dev container Bugsink at `http://localhost:8000` for local development errors
|
||||
|
||||
### Bugsink MCP Server Setup (ADR-015)
|
||||
|
||||
**IMPORTANT**: You need to configure **TWO separate MCP servers** - one for production/test, one for local dev.
|
||||
|
||||
#### Installation (shared for both servers)
|
||||
|
||||
```bash
|
||||
# Clone the bugsink-mcp repository (NOT sentry-selfhosted-mcp)
|
||||
git clone https://github.com/j-shelfwood/bugsink-mcp.git
|
||||
cd bugsink-mcp
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
#### Production/Test Bugsink MCP (bugsink)
|
||||
|
||||
Add to `.claude/mcp.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"bugsink": {
|
||||
"command": "node",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "https://bugsink.projectium.com",
|
||||
"BUGSINK_API_TOKEN": "<get-from-production-bugsink>",
|
||||
"BUGSINK_ORG_SLUG": "sentry"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Get the auth token**:
|
||||
|
||||
- Navigate to https://bugsink.projectium.com
|
||||
- Log in with production credentials
|
||||
- Go to Settings > API Keys
|
||||
- Create a new API key with read access
|
||||
|
||||
#### Dev Container Bugsink MCP (bugsink-dev)
|
||||
|
||||
Add to `.claude/mcp.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"bugsink-dev": {
|
||||
"command": "node",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "http://localhost:8000",
|
||||
"BUGSINK_API_TOKEN": "<get-from-local-bugsink>",
|
||||
"BUGSINK_ORG_SLUG": "sentry"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Get the auth token**:
|
||||
|
||||
- Navigate to http://localhost:8000 (or https://localhost:8443)
|
||||
- Log in with `admin@localhost` / `admin`
|
||||
- Go to Settings > API Keys
|
||||
- Create a new API key with read access
|
||||
|
||||
#### MCP Tool Usage
|
||||
|
||||
When using Bugsink MCP tools, remember:
|
||||
|
||||
- `mcp__bugsink__*` tools connect to **production/test** Bugsink
|
||||
- `mcp__bugsink-dev__*` tools connect to **dev container** Bugsink
|
||||
- Available capabilities for both:
|
||||
- List projects and issues
|
||||
- View detailed error events and stacktraces
|
||||
- Search by error message or stack trace
|
||||
- Update issue status (resolve, ignore)
|
||||
- Create releases
|
||||
|
||||
### SSH Server Access
|
||||
|
||||
Claude Code can execute commands on the production server via SSH:
|
||||
|
||||
```bash
|
||||
# Basic command execution
|
||||
ssh root@projectium.com "command here"
|
||||
|
||||
# Examples:
|
||||
ssh root@projectium.com "systemctl status logstash"
|
||||
ssh root@projectium.com "pm2 list"
|
||||
ssh root@projectium.com "tail -50 /var/www/flyer-crawler.projectium.com/logs/app.log"
|
||||
```
|
||||
|
||||
**Use cases:**
|
||||
|
||||
- Managing Logstash, PM2, NGINX, Redis services
|
||||
- Viewing server logs
|
||||
- Deploying configuration changes
|
||||
- Checking service status
|
||||
|
||||
**Important:** SSH access requires the host machine to have SSH keys configured for `root@projectium.com`.
|
||||
|
||||
---
|
||||
|
||||
## Logstash Configuration (ADR-050)
|
||||
|
||||
The production server uses **Logstash** to aggregate logs from multiple sources and forward errors to Bugsink for centralized error tracking.
|
||||
|
||||
**Log Sources:**
|
||||
|
||||
- **PostgreSQL function logs** - Structured JSON logs from `fn_log()` helper function
|
||||
- **PM2 worker logs** - Service logs from BullMQ job workers (stdout)
|
||||
- **Redis logs** - Operational logs (INFO level) and errors
|
||||
- **NGINX logs** - Access logs (all requests) and error logs
|
||||
|
||||
### Configuration Location
|
||||
|
||||
**Primary configuration file:**
|
||||
|
||||
- `/etc/logstash/conf.d/bugsink.conf` - Complete Logstash pipeline configuration
|
||||
|
||||
**Related files:**
|
||||
|
||||
- `/etc/postgresql/14/main/conf.d/observability.conf` - PostgreSQL logging configuration
|
||||
- `/var/log/postgresql/*.log` - PostgreSQL log files
|
||||
- `/home/gitea-runner/.pm2/logs/*.log` - PM2 worker logs
|
||||
- `/var/log/redis/redis-server.log` - Redis logs
|
||||
- `/var/log/nginx/access.log` - NGINX access logs
|
||||
- `/var/log/nginx/error.log` - NGINX error logs
|
||||
- `/var/log/logstash/*.log` - Logstash file outputs (operational logs)
|
||||
- `/var/lib/logstash/sincedb_*` - Logstash position tracking files
|
||||
|
||||
### Key Features
|
||||
|
||||
1. **Multi-source aggregation**: Collects logs from PostgreSQL, PM2 workers, Redis, and NGINX
|
||||
2. **Environment-based routing**: Automatically detects production vs test environments and routes errors to the correct Bugsink project
|
||||
3. **Structured JSON parsing**: Extracts `fn_log()` function output from PostgreSQL logs and Pino JSON from PM2 workers
|
||||
4. **Sentry-compatible format**: Transforms events to Sentry format with `event_id`, `timestamp`, `level`, `message`, and `extra` context
|
||||
5. **Error filtering**: Only forwards WARNING and ERROR level messages to Bugsink
|
||||
6. **Operational log storage**: Stores non-error logs (Redis INFO, NGINX access, PM2 operational) to `/var/log/logstash/` for analysis
|
||||
7. **Request monitoring**: Categorizes NGINX requests by status code (2xx, 3xx, 4xx, 5xx) and identifies slow requests
|
||||
|
||||
### Common Maintenance Commands
|
||||
|
||||
```bash
|
||||
# Check Logstash status
|
||||
systemctl status logstash
|
||||
|
||||
# Restart Logstash after configuration changes
|
||||
systemctl restart logstash
|
||||
|
||||
# Test configuration syntax
|
||||
/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf
|
||||
|
||||
# View Logstash logs
|
||||
journalctl -u logstash -f
|
||||
|
||||
# Check Logstash stats (events processed, failures)
|
||||
curl -XGET 'localhost:9600/_node/stats/pipelines?pretty' | jq '.pipelines.main.plugins.filters'
|
||||
|
||||
# Monitor PostgreSQL logs being processed
|
||||
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log
|
||||
|
||||
# View operational log outputs
|
||||
tail -f /var/log/logstash/pm2-workers-$(date +%Y-%m-%d).log
|
||||
tail -f /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
|
||||
tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
|
||||
|
||||
# Check disk usage of log files
|
||||
du -sh /var/log/logstash/
|
||||
```
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
| Issue | Check | Solution |
|
||||
| ------------------------------- | ---------------------------- | ---------------------------------------------------------------------------------------------- |
|
||||
| Errors not appearing in Bugsink | Check Logstash is running | `systemctl status logstash` |
|
||||
| Configuration syntax errors | Test config file | `/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf` |
|
||||
| Grok pattern failures | Check Logstash stats | `curl localhost:9600/_node/stats/pipelines?pretty \| jq '.pipelines.main.plugins.filters'` |
|
||||
| Wrong Bugsink project | Verify environment detection | Check tags in logs match expected environment (production/test) |
|
||||
| Permission denied reading logs | Check Logstash permissions | `groups logstash` should include `postgres`, `adm` groups |
|
||||
| PM2 logs not captured | Check file paths exist | `ls /home/gitea-runner/.pm2/logs/flyer-crawler-worker-*.log` |
|
||||
| NGINX access logs not showing | Check file output directory | `ls -lh /var/log/logstash/nginx-access-*.log` |
|
||||
| High disk usage | Check log rotation | Verify `/etc/logrotate.d/logstash` is configured and running daily |
|
||||
|
||||
**Full setup guide**: See [docs/BARE-METAL-SETUP.md](docs/BARE-METAL-SETUP.md) section "PostgreSQL Function Observability (ADR-050)"
|
||||
|
||||
**Architecture details**: See [docs/adr/0050-postgresql-function-observability.md](docs/adr/0050-postgresql-function-observability.md)
|
||||
348
CONTRIBUTING.md
Normal file
348
CONTRIBUTING.md
Normal file
@@ -0,0 +1,348 @@
|
||||
# Contributing to Flyer Crawler
|
||||
|
||||
Thank you for your interest in contributing to Flyer Crawler! This guide will help you understand our development workflow and coding standards.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Getting Started](#getting-started)
|
||||
- [Development Workflow](#development-workflow)
|
||||
- [Code Standards](#code-standards)
|
||||
- [Testing Requirements](#testing-requirements)
|
||||
- [Pull Request Process](#pull-request-process)
|
||||
- [Architecture Decision Records](#architecture-decision-records)
|
||||
- [Working with AI Agents](#working-with-ai-agents)
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Prerequisites
|
||||
|
||||
1. **Windows with Podman Desktop**: See [docs/getting-started/INSTALL.md](docs/getting-started/INSTALL.md)
|
||||
2. **Node.js 20+**: For running the application
|
||||
3. **Git**: For version control
|
||||
|
||||
### Initial Setup
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone <repository-url>
|
||||
cd flyer-crawler.projectium.com
|
||||
|
||||
# Install dependencies
|
||||
npm install
|
||||
|
||||
# Start development containers
|
||||
podman start flyer-crawler-postgres flyer-crawler-redis
|
||||
|
||||
# Start development server
|
||||
npm run dev
|
||||
```
|
||||
|
||||
## Development Workflow
|
||||
|
||||
### Before Making Changes
|
||||
|
||||
1. **Read [CLAUDE.md](CLAUDE.md)** - Project guidelines and patterns
|
||||
2. **Review relevant ADRs** in [docs/adr/](docs/adr/) - Understand architectural decisions
|
||||
3. **Check existing issues** - Avoid duplicate work
|
||||
4. **Create a feature branch** - Use descriptive names
|
||||
|
||||
```bash
|
||||
git checkout -b feature/descriptive-name
|
||||
# or
|
||||
git checkout -b fix/issue-description
|
||||
```
|
||||
|
||||
### Making Changes
|
||||
|
||||
#### Code Changes
|
||||
|
||||
Follow the established patterns from [docs/development/CODE-PATTERNS.md](docs/development/CODE-PATTERNS.md):
|
||||
|
||||
1. **Routes** → **Services** → **Repositories** → **Database**
|
||||
2. Never access the database directly from routes
|
||||
3. Use Zod schemas for input validation
|
||||
4. Follow ADR-034 repository naming conventions:
|
||||
- `get*` - Throws NotFoundError if not found
|
||||
- `find*` - Returns null if not found
|
||||
- `list*` - Returns empty array if none found
|
||||
|
||||
#### Database Changes
|
||||
|
||||
When modifying the database schema:
|
||||
|
||||
1. Create migration: `sql/migrations/NNNN-description.sql`
|
||||
2. Update `sql/master_schema_rollup.sql` (complete schema)
|
||||
3. Update `sql/initial_schema.sql` (identical to rollup)
|
||||
4. Test with integration tests
|
||||
|
||||
**CRITICAL**: Schema files must stay synchronized. See [CLAUDE.md#database-schema-sync](CLAUDE.md#database-schema-sync).
|
||||
|
||||
#### NGINX Configuration Changes
|
||||
|
||||
When modifying NGINX configurations on the production or test servers:
|
||||
|
||||
1. Make changes on the server at `/etc/nginx/sites-available/`
|
||||
2. Test with `sudo nginx -t` and reload with `sudo systemctl reload nginx`
|
||||
3. Update the reference copies in the repository root:
|
||||
- `etc-nginx-sites-available-flyer-crawler.projectium.com` - Production
|
||||
- `etc-nginx-sites-available-flyer-crawler-test-projectium-com.txt` - Test
|
||||
4. Commit the updated reference files
|
||||
|
||||
These reference files serve as version-controlled documentation of the deployed configurations.
|
||||
|
||||
### Testing
|
||||
|
||||
**IMPORTANT**: All tests must run in the dev container.
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
podman exec -it flyer-crawler-dev npm test
|
||||
|
||||
# Run specific test file
|
||||
podman exec -it flyer-crawler-dev npm test -- --run src/path/to/file.test.ts
|
||||
|
||||
# Type check
|
||||
podman exec -it flyer-crawler-dev npm run type-check
|
||||
```
|
||||
|
||||
#### Before Committing
|
||||
|
||||
1. Write tests for new features
|
||||
2. Update existing tests if behavior changes
|
||||
3. Run full test suite
|
||||
4. Run type check
|
||||
5. Verify documentation is updated
|
||||
|
||||
See [docs/development/TESTING.md](docs/development/TESTING.md) for detailed testing guidelines.
|
||||
|
||||
## Code Standards
|
||||
|
||||
### TypeScript
|
||||
|
||||
- Use strict TypeScript mode
|
||||
- Define types in `src/types/*.ts` files
|
||||
- Avoid `any` - use `unknown` if type is truly unknown
|
||||
- Follow ADR-027 for naming conventions
|
||||
|
||||
### Error Handling
|
||||
|
||||
```typescript
|
||||
// Repository layer (ADR-001)
|
||||
import { handleDbError, NotFoundError } from '../services/db/errors.db';
|
||||
|
||||
try {
|
||||
const result = await client.query(query, values);
|
||||
if (result.rows.length === 0) {
|
||||
throw new NotFoundError('Flyer', id);
|
||||
}
|
||||
return result.rows[0];
|
||||
} catch (error) {
|
||||
throw handleDbError(error);
|
||||
}
|
||||
```
|
||||
|
||||
### API Responses
|
||||
|
||||
```typescript
|
||||
// Route handlers (ADR-028)
|
||||
import { sendSuccess, sendError } from '../utils/apiResponse';
|
||||
|
||||
// Success response
|
||||
return sendSuccess(res, flyer, 'Flyer retrieved successfully');
|
||||
|
||||
// Paginated response
|
||||
return sendPaginated(res, {
|
||||
items: flyers,
|
||||
total: count,
|
||||
page: 1,
|
||||
pageSize: 20,
|
||||
});
|
||||
```
|
||||
|
||||
### Transactions
|
||||
|
||||
```typescript
|
||||
// Multi-operation changes (ADR-002)
|
||||
import { withTransaction } from '../services/db/transaction.db';
|
||||
|
||||
await withTransaction(async (client) => {
|
||||
await flyerDb.createFlyer(flyerData, client);
|
||||
await flyerItemDb.createItems(items, client);
|
||||
// Automatically commits on success, rolls back on error
|
||||
});
|
||||
```
|
||||
|
||||
## Testing Requirements
|
||||
|
||||
### Test Coverage
|
||||
|
||||
- **Unit tests**: All service functions, utilities, and helpers
|
||||
- **Integration tests**: API endpoints, database operations
|
||||
- **E2E tests**: Critical user flows
|
||||
|
||||
### Test Patterns
|
||||
|
||||
See [docs/subagents/TESTER-GUIDE.md](docs/subagents/TESTER-GUIDE.md) for:
|
||||
|
||||
- Test helper functions
|
||||
- Mocking patterns
|
||||
- Known testing issues and solutions
|
||||
|
||||
### Test Naming
|
||||
|
||||
```typescript
|
||||
// Good test names
|
||||
describe('FlyerService.createFlyer', () => {
|
||||
it('should create flyer with valid data', async () => { ... });
|
||||
it('should throw ValidationError when store_id is missing', async () => { ... });
|
||||
it('should rollback transaction on item creation failure', async () => { ... });
|
||||
});
|
||||
```
|
||||
|
||||
## Pull Request Process
|
||||
|
||||
### 1. Prepare Your PR
|
||||
|
||||
- [ ] All tests pass in dev container
|
||||
- [ ] Type check passes
|
||||
- [ ] No console.log or debugging code
|
||||
- [ ] Documentation updated (if applicable)
|
||||
- [ ] ADR created (if architectural decision made)
|
||||
|
||||
### 2. Create Pull Request
|
||||
|
||||
**Title Format:**
|
||||
|
||||
- `feat: Add flyer bulk import endpoint`
|
||||
- `fix: Resolve cache invalidation bug`
|
||||
- `docs: Update testing guide`
|
||||
- `refactor: Simplify transaction handling`
|
||||
|
||||
**Description Template:**
|
||||
|
||||
```markdown
|
||||
## Summary
|
||||
|
||||
Brief description of changes
|
||||
|
||||
## Changes Made
|
||||
|
||||
- Added X
|
||||
- Modified Y
|
||||
- Fixed Z
|
||||
|
||||
## Related Issues
|
||||
|
||||
Fixes #123
|
||||
|
||||
## Testing
|
||||
|
||||
- [ ] Unit tests added/updated
|
||||
- [ ] Integration tests pass
|
||||
- [ ] Manual testing completed
|
||||
|
||||
## Documentation
|
||||
|
||||
- [ ] Code comments added where needed
|
||||
- [ ] ADR created/updated (if applicable)
|
||||
- [ ] User-facing docs updated
|
||||
```
|
||||
|
||||
### 3. Code Review
|
||||
|
||||
- Address all reviewer feedback
|
||||
- Keep discussions focused and constructive
|
||||
- Update PR based on feedback
|
||||
|
||||
### 4. Merge
|
||||
|
||||
- Squash commits if requested
|
||||
- Ensure CI passes
|
||||
- Maintainer will merge when approved
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
When making significant architectural decisions:
|
||||
|
||||
1. Create ADR in `docs/adr/`
|
||||
2. Use template from existing ADRs
|
||||
3. Number sequentially
|
||||
4. Update `docs/adr/index.md`
|
||||
|
||||
**Examples of ADR-worthy decisions:**
|
||||
|
||||
- New design patterns
|
||||
- Technology choices
|
||||
- API design changes
|
||||
- Database schema conventions
|
||||
|
||||
See [docs/adr/index.md](docs/adr/index.md) for existing decisions.
|
||||
|
||||
## Working with AI Agents
|
||||
|
||||
This project uses Claude Code with specialized subagents. See:
|
||||
|
||||
- [docs/subagents/OVERVIEW.md](docs/subagents/OVERVIEW.md) - Introduction
|
||||
- [CLAUDE.md](CLAUDE.md) - AI agent instructions
|
||||
|
||||
### When to Use Subagents
|
||||
|
||||
| Task | Subagent |
|
||||
| -------------------- | ------------------- |
|
||||
| Writing code | `coder` |
|
||||
| Creating tests | `testwriter` |
|
||||
| Database changes | `db-dev` |
|
||||
| Container/deployment | `devops` |
|
||||
| Security review | `security-engineer` |
|
||||
|
||||
### Example
|
||||
|
||||
```
|
||||
Use the coder subagent to implement the bulk flyer import endpoint with proper transaction handling and error responses.
|
||||
```
|
||||
|
||||
## Git Conventions
|
||||
|
||||
### Commit Messages
|
||||
|
||||
Follow conventional commits:
|
||||
|
||||
```
|
||||
feat: Add watchlist price alerts
|
||||
fix: Resolve duplicate flyer upload bug
|
||||
docs: Update deployment guide
|
||||
refactor: Simplify auth middleware
|
||||
test: Add integration tests for flyer API
|
||||
```
|
||||
|
||||
### Branch Naming
|
||||
|
||||
```
|
||||
feature/watchlist-alerts
|
||||
fix/duplicate-upload-bug
|
||||
docs/update-deployment-guide
|
||||
refactor/auth-middleware
|
||||
```
|
||||
|
||||
## Getting Help
|
||||
|
||||
- **Documentation**: Start with [docs/README.md](docs/README.md)
|
||||
- **Testing Issues**: See [docs/development/TESTING.md](docs/development/TESTING.md)
|
||||
- **Architecture Questions**: Review [docs/adr/index.md](docs/adr/index.md)
|
||||
- **Debugging**: Check [docs/development/DEBUGGING.md](docs/development/DEBUGGING.md)
|
||||
- **AI Agents**: Consult [docs/subagents/OVERVIEW.md](docs/subagents/OVERVIEW.md)
|
||||
|
||||
## Code of Conduct
|
||||
|
||||
- Be respectful and inclusive
|
||||
- Welcome newcomers
|
||||
- Focus on constructive feedback
|
||||
- Assume good intentions
|
||||
|
||||
## License
|
||||
|
||||
By contributing, you agree that your contributions will be licensed under the same license as the project.
|
||||
|
||||
---
|
||||
|
||||
Thank you for contributing to Flyer Crawler! 🎉
|
||||
196
Dockerfile.dev
196
Dockerfile.dev
@@ -29,6 +29,7 @@ ENV DEBIAN_FRONTEND=noninteractive
|
||||
# - nginx: for proxying Vite dev server with HTTPS
|
||||
# - libnss3-tools: required by mkcert for installing CA certificates
|
||||
# - wget: for downloading mkcert binary
|
||||
# - tzdata: timezone data required by Bugsink/Django (uses Europe/Amsterdam)
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
git \
|
||||
@@ -44,6 +45,7 @@ RUN apt-get update && apt-get install -y \
|
||||
nginx \
|
||||
libnss3-tools \
|
||||
wget \
|
||||
tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ============================================================================
|
||||
@@ -52,6 +54,13 @@ RUN apt-get update && apt-get install -y \
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
|
||||
&& apt-get install -y nodejs
|
||||
|
||||
# ============================================================================
|
||||
# Install PM2 Globally (ADR-014 Parity)
|
||||
# ============================================================================
|
||||
# Install PM2 to match production architecture. This allows dev container to
|
||||
# run the same process management as production (cluster mode, workers, etc.)
|
||||
RUN npm install -g pm2
|
||||
|
||||
# ============================================================================
|
||||
# Install mkcert and Generate Self-Signed Certificates
|
||||
# ============================================================================
|
||||
@@ -61,6 +70,27 @@ RUN wget -O /usr/local/bin/mkcert https://github.com/FiloSottile/mkcert/releases
|
||||
&& chmod +x /usr/local/bin/mkcert
|
||||
|
||||
# Create certificates directory and generate localhost certificates
|
||||
# ============================================================================
|
||||
# IMPORTANT: Certificate includes MULTIPLE hostnames (SANs)
|
||||
# ============================================================================
|
||||
# The certificate is generated for 'localhost', '127.0.0.1', AND '::1' because:
|
||||
#
|
||||
# 1. Users may access the site via https://localhost/ OR https://127.0.0.1/
|
||||
# 2. Database stores image URLs using one hostname (typically 127.0.0.1)
|
||||
# 3. The seed script uses https://127.0.0.1 for image URLs (database constraint)
|
||||
# 4. NGINX is configured to accept BOTH hostnames (see docker/nginx/dev.conf)
|
||||
#
|
||||
# Without all hostnames in the certificate's Subject Alternative Names (SANs),
|
||||
# browsers would show ERR_CERT_AUTHORITY_INVALID when loading images or other
|
||||
# resources that use a different hostname than the one in the address bar.
|
||||
#
|
||||
# The mkcert command below creates a certificate valid for all three:
|
||||
# - localhost (IPv4 hostname)
|
||||
# - 127.0.0.1 (IPv4 address)
|
||||
# - ::1 (IPv6 loopback)
|
||||
#
|
||||
# See also: docker/nginx/dev.conf, docs/FLYER-URL-CONFIGURATION.md
|
||||
# ============================================================================
|
||||
RUN mkdir -p /app/certs \
|
||||
&& cd /app/certs \
|
||||
&& mkcert -install \
|
||||
@@ -144,9 +174,27 @@ BUGSINK = {\n\
|
||||
}\n\
|
||||
\n\
|
||||
ALLOWED_HOSTS = deduce_allowed_hosts(BUGSINK["BASE_URL"])\n\
|
||||
# Also allow 127.0.0.1 access (both localhost and 127.0.0.1 should work)\n\
|
||||
if "127.0.0.1" not in ALLOWED_HOSTS:\n\
|
||||
ALLOWED_HOSTS.append("127.0.0.1")\n\
|
||||
if "localhost" not in ALLOWED_HOSTS:\n\
|
||||
ALLOWED_HOSTS.append("localhost")\n\
|
||||
\n\
|
||||
# CSRF Trusted Origins (Django 4.0+ requires full origin for HTTPS POST requests)\n\
|
||||
# This fixes "CSRF verification failed" errors when accessing Bugsink via HTTPS\n\
|
||||
# Both localhost and 127.0.0.1 must be trusted to support different access patterns\n\
|
||||
CSRF_TRUSTED_ORIGINS = [\n\
|
||||
"https://localhost:8443",\n\
|
||||
"https://127.0.0.1:8443",\n\
|
||||
"http://localhost:8000",\n\
|
||||
"http://127.0.0.1:8000",\n\
|
||||
]\n\
|
||||
\n\
|
||||
# Console email backend for dev\n\
|
||||
EMAIL_BACKEND = "bugsink.email_backends.QuietConsoleEmailBackend"\n\
|
||||
\n\
|
||||
# HTTPS proxy support (nginx reverse proxy on port 8443)\n\
|
||||
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")\n\
|
||||
' > /opt/bugsink/conf/bugsink_conf.py
|
||||
|
||||
# Create Bugsink startup script
|
||||
@@ -206,119 +254,25 @@ exec /opt/bugsink/bin/gunicorn \\\n\
|
||||
&& chmod +x /usr/local/bin/start-bugsink.sh
|
||||
|
||||
# ============================================================================
|
||||
# Create Logstash Pipeline Configuration
|
||||
# Copy Logstash Pipeline Configuration
|
||||
# ============================================================================
|
||||
# ADR-015: Pino and Redis logs → Bugsink
|
||||
# ADR-015 + ADR-050: Multi-source log aggregation to Bugsink
|
||||
# Configuration file includes:
|
||||
# - Pino application logs (Backend API errors)
|
||||
# - PostgreSQL logs (including fn_log() structured output)
|
||||
# - NGINX access and error logs
|
||||
# See docker/logstash/bugsink.conf for full configuration
|
||||
RUN mkdir -p /etc/logstash/conf.d /app/logs
|
||||
|
||||
RUN echo 'input {\n\
|
||||
# Pino application logs\n\
|
||||
file {\n\
|
||||
path => "/app/logs/*.log"\n\
|
||||
codec => json\n\
|
||||
type => "pino"\n\
|
||||
tags => ["app"]\n\
|
||||
start_position => "beginning"\n\
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pino"\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Redis logs\n\
|
||||
file {\n\
|
||||
path => "/var/log/redis/*.log"\n\
|
||||
type => "redis"\n\
|
||||
tags => ["redis"]\n\
|
||||
start_position => "beginning"\n\
|
||||
sincedb_path => "/var/lib/logstash/sincedb_redis"\n\
|
||||
}\n\
|
||||
\n\
|
||||
# PostgreSQL function logs (ADR-050)\n\
|
||||
file {\n\
|
||||
path => "/var/log/postgresql/*.log"\n\
|
||||
type => "postgres"\n\
|
||||
tags => ["postgres", "database"]\n\
|
||||
start_position => "beginning"\n\
|
||||
sincedb_path => "/var/lib/logstash/sincedb_postgres"\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
filter {\n\
|
||||
# Pino error detection (level 50 = error, 60 = fatal)\n\
|
||||
if [type] == "pino" and [level] >= 50 {\n\
|
||||
mutate { add_tag => ["error"] }\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Redis log parsing\n\
|
||||
if [type] == "redis" {\n\
|
||||
grok {\n\
|
||||
match => { "message" => "%%{POSINT:pid}:%%{WORD:role} %%{MONTHDAY} %%{MONTH} %%{TIME} %%{WORD:loglevel} %%{GREEDYDATA:redis_message}" }\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Tag errors (WARNING/ERROR) for Bugsink forwarding\n\
|
||||
if [loglevel] in ["WARNING", "ERROR"] {\n\
|
||||
mutate { add_tag => ["error"] }\n\
|
||||
}\n\
|
||||
# Tag INFO-level operational events (startup, config, persistence)\n\
|
||||
else if [loglevel] == "INFO" {\n\
|
||||
mutate { add_tag => ["redis_operational"] }\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
# PostgreSQL function log parsing (ADR-050)\n\
|
||||
if [type] == "postgres" {\n\
|
||||
# Extract timestamp and process ID from PostgreSQL log prefix\n\
|
||||
# Format: "2026-01-18 10:30:00 PST [12345] user@database "\n\
|
||||
grok {\n\
|
||||
match => { "message" => "%%{TIMESTAMP_ISO8601:pg_timestamp} \\\\[%%{POSINT:pg_pid}\\\\] %%{USERNAME:pg_user}@%%{WORD:pg_database} %%{GREEDYDATA:pg_message}" }\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Check if this is a structured JSON log from fn_log()\n\
|
||||
# fn_log() emits JSON like: {"timestamp":"...","level":"WARNING","source":"postgresql","function":"award_achievement",...}\n\
|
||||
if [pg_message] =~ /^\\{.*"source":"postgresql".*\\}$/ {\n\
|
||||
json {\n\
|
||||
source => "pg_message"\n\
|
||||
target => "fn_log"\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Mark as error if level is WARNING or ERROR\n\
|
||||
if [fn_log][level] in ["WARNING", "ERROR"] {\n\
|
||||
mutate { add_tag => ["error", "db_function"] }\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Also catch native PostgreSQL errors\n\
|
||||
if [pg_message] =~ /^ERROR:/ or [pg_message] =~ /^FATAL:/ {\n\
|
||||
mutate { add_tag => ["error", "postgres_native"] }\n\
|
||||
}\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
output {\n\
|
||||
# Forward errors to Bugsink\n\
|
||||
if "error" in [tags] {\n\
|
||||
http {\n\
|
||||
url => "http://localhost:8000/api/store/"\n\
|
||||
http_method => "post"\n\
|
||||
format => "json"\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Store Redis operational logs (INFO level) to file\n\
|
||||
if "redis_operational" in [tags] {\n\
|
||||
file {\n\
|
||||
path => "/var/log/logstash/redis-operational-%%{+YYYY-MM-dd}.log"\n\
|
||||
codec => json_lines\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Debug output (comment out in production)\n\
|
||||
stdout { codec => rubydebug }\n\
|
||||
}\n\
|
||||
' > /etc/logstash/conf.d/bugsink.conf
|
||||
COPY docker/logstash/bugsink.conf /etc/logstash/conf.d/bugsink.conf
|
||||
|
||||
# Create Logstash directories
|
||||
RUN mkdir -p /var/lib/logstash && chown -R logstash:logstash /var/lib/logstash
|
||||
RUN mkdir -p /var/log/logstash && chown -R logstash:logstash /var/log/logstash
|
||||
|
||||
# Create PM2 log directory (ADR-014 Parity)
|
||||
# Logs written here will be picked up by Logstash (ADR-050)
|
||||
RUN mkdir -p /var/log/pm2
|
||||
|
||||
# ============================================================================
|
||||
# Configure Nginx
|
||||
# ============================================================================
|
||||
@@ -340,8 +294,7 @@ WORKDIR /app
|
||||
COPY package*.json ./
|
||||
|
||||
# Install all dependencies (including devDependencies for development)
|
||||
# Use --legacy-peer-deps due to react-joyride peer dependency conflict with React 19
|
||||
RUN npm install --legacy-peer-deps
|
||||
RUN npm install
|
||||
|
||||
# ============================================================================
|
||||
# Environment Configuration
|
||||
@@ -371,6 +324,33 @@ ENV BUGSINK_ADMIN_PASSWORD=admin
|
||||
# 8000 - Bugsink error tracking
|
||||
EXPOSE 80 443 3001 8000
|
||||
|
||||
# ============================================================================
|
||||
# Copy Application Code and Scripts
|
||||
# ============================================================================
|
||||
# Copy the scripts directory which contains the entrypoint script
|
||||
COPY scripts/ /app/scripts/
|
||||
|
||||
# ============================================================================
|
||||
# Fix Line Endings for Windows Compatibility
|
||||
# ============================================================================
|
||||
# Convert ALL text files from CRLF to LF (Windows to Unix)
|
||||
# This ensures compatibility when building on Windows hosts
|
||||
# We process: shell scripts, JS/TS files, JSON, config files, etc.
|
||||
RUN find /app -type f \( \
|
||||
-name "*.sh" -o \
|
||||
-name "*.js" -o \
|
||||
-name "*.ts" -o \
|
||||
-name "*.tsx" -o \
|
||||
-name "*.jsx" -o \
|
||||
-name "*.json" -o \
|
||||
-name "*.conf" -o \
|
||||
-name "*.config" -o \
|
||||
-name "*.yml" -o \
|
||||
-name "*.yaml" \
|
||||
\) -exec sed -i 's/\r$//' {} \; && \
|
||||
find /etc/nginx -type f -name "*.conf" -exec sed -i 's/\r$//' {} \; && \
|
||||
chmod +x /app/scripts/*.sh
|
||||
|
||||
# ============================================================================
|
||||
# Default Command
|
||||
# ============================================================================
|
||||
|
||||
68
README.md
68
README.md
@@ -34,32 +34,80 @@ Flyer Crawler is a web application that uses Google Gemini AI to extract, analyz
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Development with Podman Containers
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
# 1. Start PostgreSQL and Redis containers
|
||||
podman start flyer-crawler-postgres flyer-crawler-redis
|
||||
|
||||
# 2. Install dependencies (first time only)
|
||||
npm install
|
||||
|
||||
# Run in development mode
|
||||
# 3. Run in development mode
|
||||
npm run dev
|
||||
```
|
||||
|
||||
See [INSTALL.md](INSTALL.md) for detailed setup instructions.
|
||||
The application will be available at:
|
||||
|
||||
- **Frontend**: http://localhost:5173
|
||||
- **Backend API**: http://localhost:3001
|
||||
|
||||
See [docs/getting-started/INSTALL.md](docs/getting-started/INSTALL.md) for detailed setup instructions including:
|
||||
|
||||
- Podman Desktop installation
|
||||
- Container configuration
|
||||
- Database initialization
|
||||
- Environment variables
|
||||
|
||||
### Testing
|
||||
|
||||
**IMPORTANT**: All tests must run inside the dev container for reliable results.
|
||||
|
||||
```bash
|
||||
# Run all tests in container
|
||||
podman exec -it flyer-crawler-dev npm test
|
||||
|
||||
# Run only unit tests
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
|
||||
# Run only integration tests
|
||||
podman exec -it flyer-crawler-dev npm run test:integration
|
||||
```
|
||||
|
||||
See [docs/development/TESTING.md](docs/development/TESTING.md) for testing guidelines.
|
||||
|
||||
---
|
||||
|
||||
## Documentation
|
||||
|
||||
| Document | Description |
|
||||
| -------------------------------------- | ---------------------------------------- |
|
||||
| [INSTALL.md](INSTALL.md) | Local development setup with Podman |
|
||||
| [DATABASE.md](DATABASE.md) | PostgreSQL setup, schema, and extensions |
|
||||
| [AUTHENTICATION.md](AUTHENTICATION.md) | OAuth configuration (Google, GitHub) |
|
||||
| [DEPLOYMENT.md](DEPLOYMENT.md) | Production server setup, NGINX, PM2 |
|
||||
### Core Documentation
|
||||
|
||||
| Document | Description |
|
||||
| --------------------------------------------------------- | --------------------------------------- |
|
||||
| [📚 Documentation Index](docs/README.md) | Navigate all documentation |
|
||||
| [⚙️ Installation Guide](docs/getting-started/INSTALL.md) | Local development setup with Podman |
|
||||
| [🏗️ Architecture Overview](docs/architecture/DATABASE.md) | System design, database, authentication |
|
||||
| [💻 Development Guide](docs/development/TESTING.md) | Testing, debugging, code patterns |
|
||||
| [🚀 Deployment Guide](docs/operations/DEPLOYMENT.md) | Production setup, NGINX, PM2 |
|
||||
| [🤖 AI Agent Guides](docs/subagents/OVERVIEW.md) | Working with Claude Code subagents |
|
||||
|
||||
### Quick References
|
||||
|
||||
| Document | Description |
|
||||
| -------------------------------------------------- | -------------------------------- |
|
||||
| [CLAUDE.md](CLAUDE.md) | AI agent project instructions |
|
||||
| [CONTRIBUTING.md](CONTRIBUTING.md) | Development workflow, PR process |
|
||||
| [Architecture Decision Records](docs/adr/index.md) | Design decisions and rationale |
|
||||
|
||||
---
|
||||
|
||||
## Environment Variables
|
||||
|
||||
This project uses environment variables for configuration (no `.env` files). Key variables:
|
||||
**Production/Test**: Uses Gitea CI/CD secrets injected during deployment (no local `.env` files)
|
||||
|
||||
**Dev Container**: Uses `.env.local` file which **overrides** the default DSNs in `compose.dev.yml`
|
||||
|
||||
Key variables:
|
||||
|
||||
| Variable | Description |
|
||||
| -------------------------------------------- | -------------------------------- |
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
using powershell on win10 use this command to run the integration tests only in the container
|
||||
|
||||
podman exec -i flyer-crawler-dev npm run test:integration 2>&1 | Tee-Object -FilePath test-output.txt
|
||||
303
READMEv2.md
303
READMEv2.md
@@ -1,303 +0,0 @@
|
||||
# Flyer Crawler - Development Environment Setup
|
||||
|
||||
Quick start guide for getting the development environment running with Podman containers.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **Windows with WSL 2**: Install WSL 2 by running `wsl --install` in an administrator PowerShell
|
||||
- **Podman Desktop**: Download and install [Podman Desktop for Windows](https://podman-desktop.io/)
|
||||
- **Node.js 20+**: Required for running the application
|
||||
|
||||
## Quick Start - Container Environment
|
||||
|
||||
### 1. Initialize Podman
|
||||
|
||||
```powershell
|
||||
# Start Podman machine (do this once after installing Podman Desktop)
|
||||
podman machine init
|
||||
podman machine start
|
||||
```
|
||||
|
||||
### 2. Start Required Services
|
||||
|
||||
Start PostgreSQL (with PostGIS) and Redis containers:
|
||||
|
||||
```powershell
|
||||
# Navigate to project directory
|
||||
cd D:\gitea\flyer-crawler.projectium.com\flyer-crawler.projectium.com
|
||||
|
||||
# Start PostgreSQL with PostGIS
|
||||
podman run -d \
|
||||
--name flyer-crawler-postgres \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=flyer_crawler_dev \
|
||||
-p 5432:5432 \
|
||||
docker.io/postgis/postgis:15-3.3
|
||||
|
||||
# Start Redis
|
||||
podman run -d \
|
||||
--name flyer-crawler-redis \
|
||||
-e REDIS_PASSWORD="" \
|
||||
-p 6379:6379 \
|
||||
docker.io/library/redis:alpine
|
||||
```
|
||||
|
||||
### 3. Wait for PostgreSQL to Initialize
|
||||
|
||||
```powershell
|
||||
# Wait a few seconds, then check if PostgreSQL is ready
|
||||
podman exec flyer-crawler-postgres pg_isready -U postgres
|
||||
# Should output: /var/run/postgresql:5432 - accepting connections
|
||||
```
|
||||
|
||||
### 4. Install Required PostgreSQL Extensions
|
||||
|
||||
```powershell
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS pg_trgm; CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";"
|
||||
```
|
||||
|
||||
### 5. Apply Database Schema
|
||||
|
||||
```powershell
|
||||
# Apply the complete schema with URL constraints enabled
|
||||
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/master_schema_rollup.sql
|
||||
```
|
||||
|
||||
### 6. Verify URL Constraints Are Enabled
|
||||
|
||||
```powershell
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "\d public.flyers" | grep -E "(image_url|icon_url|Check)"
|
||||
```
|
||||
|
||||
You should see:
|
||||
```
|
||||
image_url | text | | not null |
|
||||
icon_url | text | | not null |
|
||||
Check constraints:
|
||||
"flyers_icon_url_check" CHECK (icon_url ~* '^https?://.*'::text)
|
||||
"flyers_image_url_check" CHECK (image_url ~* '^https?://.*'::text)
|
||||
```
|
||||
|
||||
### 7. Set Environment Variables and Start Application
|
||||
|
||||
```powershell
|
||||
# Set required environment variables
|
||||
$env:NODE_ENV="development"
|
||||
$env:DB_HOST="localhost"
|
||||
$env:DB_USER="postgres"
|
||||
$env:DB_PASSWORD="postgres"
|
||||
$env:DB_NAME="flyer_crawler_dev"
|
||||
$env:REDIS_URL="redis://localhost:6379"
|
||||
$env:PORT="3001"
|
||||
$env:FRONTEND_URL="http://localhost:5173"
|
||||
|
||||
# Install dependencies (first time only)
|
||||
npm install
|
||||
|
||||
# Start the development server (runs both backend and frontend)
|
||||
npm run dev
|
||||
```
|
||||
|
||||
The application will be available at:
|
||||
- **Frontend**: http://localhost:5173
|
||||
- **Backend API**: http://localhost:3001
|
||||
|
||||
## Managing Containers
|
||||
|
||||
### View Running Containers
|
||||
```powershell
|
||||
podman ps
|
||||
```
|
||||
|
||||
### Stop Containers
|
||||
```powershell
|
||||
podman stop flyer-crawler-postgres flyer-crawler-redis
|
||||
```
|
||||
|
||||
### Start Containers (After They've Been Created)
|
||||
```powershell
|
||||
podman start flyer-crawler-postgres flyer-crawler-redis
|
||||
```
|
||||
|
||||
### Remove Containers (Clean Slate)
|
||||
```powershell
|
||||
podman stop flyer-crawler-postgres flyer-crawler-redis
|
||||
podman rm flyer-crawler-postgres flyer-crawler-redis
|
||||
```
|
||||
|
||||
### View Container Logs
|
||||
```powershell
|
||||
podman logs flyer-crawler-postgres
|
||||
podman logs flyer-crawler-redis
|
||||
```
|
||||
|
||||
## Database Management
|
||||
|
||||
### Connect to PostgreSQL
|
||||
```powershell
|
||||
podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev
|
||||
```
|
||||
|
||||
### Reset Database Schema
|
||||
```powershell
|
||||
# Drop all tables
|
||||
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/drop_tables.sql
|
||||
|
||||
# Reapply schema
|
||||
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/master_schema_rollup.sql
|
||||
```
|
||||
|
||||
### Seed Development Data
|
||||
```powershell
|
||||
npm run db:reset:dev
|
||||
```
|
||||
|
||||
## Running Tests
|
||||
|
||||
### Unit Tests
|
||||
```powershell
|
||||
npm run test:unit
|
||||
```
|
||||
|
||||
### Integration Tests
|
||||
|
||||
**IMPORTANT**: Integration tests require the PostgreSQL and Redis containers to be running.
|
||||
|
||||
```powershell
|
||||
# Make sure containers are running
|
||||
podman ps
|
||||
|
||||
# Run integration tests
|
||||
npm run test:integration
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Podman Machine Issues
|
||||
If you get "unable to connect to Podman socket" errors:
|
||||
```powershell
|
||||
podman machine start
|
||||
```
|
||||
|
||||
### PostgreSQL Connection Refused
|
||||
Make sure PostgreSQL is ready:
|
||||
```powershell
|
||||
podman exec flyer-crawler-postgres pg_isready -U postgres
|
||||
```
|
||||
|
||||
### Port Already in Use
|
||||
If ports 5432 or 6379 are already in use, you can either:
|
||||
1. Stop the conflicting service
|
||||
2. Change the port mapping when creating containers (e.g., `-p 5433:5432`)
|
||||
|
||||
### URL Validation Errors
|
||||
The database now enforces URL constraints. All `image_url` and `icon_url` fields must:
|
||||
- Start with `http://` or `https://`
|
||||
- Match the regex pattern: `^https?://.*`
|
||||
|
||||
Make sure the `FRONTEND_URL` environment variable is set correctly to avoid URL validation errors.
|
||||
|
||||
## ADR Implementation Status
|
||||
|
||||
This development environment implements:
|
||||
|
||||
- **ADR-0002**: Transaction Management ✅
|
||||
- All database operations use the `withTransaction` pattern
|
||||
- Automatic rollback on errors
|
||||
- No connection pool leaks
|
||||
|
||||
- **ADR-0003**: Input Validation ✅
|
||||
- Zod schemas for URL validation
|
||||
- Database constraints enabled
|
||||
- Validation at API boundaries
|
||||
|
||||
## Development Workflow
|
||||
|
||||
1. **Start Containers** (once per development session)
|
||||
```powershell
|
||||
podman start flyer-crawler-postgres flyer-crawler-redis
|
||||
```
|
||||
|
||||
2. **Start Application**
|
||||
```powershell
|
||||
npm run dev
|
||||
```
|
||||
|
||||
3. **Make Changes** to code (auto-reloads via `tsx watch`)
|
||||
|
||||
4. **Run Tests** before committing
|
||||
```powershell
|
||||
npm run test:unit
|
||||
npm run test:integration
|
||||
```
|
||||
|
||||
5. **Stop Application** (Ctrl+C)
|
||||
|
||||
6. **Stop Containers** (optional, or leave running)
|
||||
```powershell
|
||||
podman stop flyer-crawler-postgres flyer-crawler-redis
|
||||
```
|
||||
|
||||
## PM2 Worker Setup (Production-like)
|
||||
|
||||
To test with PM2 workers locally:
|
||||
|
||||
```powershell
|
||||
# Install PM2 globally (once)
|
||||
npm install -g pm2
|
||||
|
||||
# Start the worker
|
||||
pm2 start npm --name "flyer-crawler-worker" -- run worker:prod
|
||||
|
||||
# View logs
|
||||
pm2 logs flyer-crawler-worker
|
||||
|
||||
# Stop worker
|
||||
pm2 stop flyer-crawler-worker
|
||||
pm2 delete flyer-crawler-worker
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
After getting the environment running:
|
||||
|
||||
1. Review [docs/adr/](docs/adr/) for architectural decisions
|
||||
2. Check [sql/master_schema_rollup.sql](sql/master_schema_rollup.sql) for database schema
|
||||
3. Explore [src/routes/](src/routes/) for API endpoints
|
||||
4. Review [src/types.ts](src/types.ts) for TypeScript type definitions
|
||||
|
||||
## Common Environment Variables
|
||||
|
||||
Create these environment variables for development:
|
||||
|
||||
```powershell
|
||||
# Database
|
||||
$env:DB_HOST="localhost"
|
||||
$env:DB_USER="postgres"
|
||||
$env:DB_PASSWORD="postgres"
|
||||
$env:DB_NAME="flyer_crawler_dev"
|
||||
$env:DB_PORT="5432"
|
||||
|
||||
# Redis
|
||||
$env:REDIS_URL="redis://localhost:6379"
|
||||
|
||||
# Application
|
||||
$env:NODE_ENV="development"
|
||||
$env:PORT="3001"
|
||||
$env:FRONTEND_URL="http://localhost:5173"
|
||||
|
||||
# Authentication (generate your own secrets)
|
||||
$env:JWT_SECRET="your-dev-jwt-secret-change-this"
|
||||
$env:SESSION_SECRET="your-dev-session-secret-change-this"
|
||||
|
||||
# AI Services (get your own API keys)
|
||||
$env:VITE_GOOGLE_GENAI_API_KEY="your-google-genai-api-key"
|
||||
$env:GOOGLE_MAPS_API_KEY="your-google-maps-api-key"
|
||||
```
|
||||
|
||||
## Resources
|
||||
|
||||
- [Podman Desktop Documentation](https://podman-desktop.io/docs)
|
||||
- [PostGIS Documentation](https://postgis.net/documentation/)
|
||||
- [Original README.md](README.md) for production setup
|
||||
105
certs/README.md
Normal file
105
certs/README.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# Development SSL Certificates
|
||||
|
||||
This directory contains SSL certificates for the development container HTTPS setup.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose | Generated By |
|
||||
| --------------- | ---------------------------------------------------- | -------------------------- |
|
||||
| `localhost.crt` | SSL certificate for localhost and 127.0.0.1 | mkcert (in Dockerfile.dev) |
|
||||
| `localhost.key` | Private key for localhost.crt | mkcert (in Dockerfile.dev) |
|
||||
| `mkcert-ca.crt` | Root CA certificate for trusting mkcert certificates | mkcert |
|
||||
|
||||
## Certificate Details
|
||||
|
||||
The `localhost.crt` certificate includes the following Subject Alternative Names (SANs):
|
||||
|
||||
- `DNS:localhost`
|
||||
- `IP Address:127.0.0.1`
|
||||
- `IP Address:::1` (IPv6 localhost)
|
||||
|
||||
This allows the development server to be accessed via both `https://localhost/` and `https://127.0.0.1/` without SSL errors.
|
||||
|
||||
## Installing the CA Certificate (Recommended)
|
||||
|
||||
To avoid SSL certificate warnings in your browser, install the mkcert CA certificate on your system.
|
||||
|
||||
### Windows
|
||||
|
||||
1. Double-click `mkcert-ca.crt`
|
||||
2. Click **"Install Certificate..."**
|
||||
3. Select **"Local Machine"** > Next
|
||||
4. Select **"Place all certificates in the following store"**
|
||||
5. Click **Browse** > Select **"Trusted Root Certification Authorities"** > OK
|
||||
6. Click **Next** > **Finish**
|
||||
7. Restart your browser
|
||||
|
||||
### macOS
|
||||
|
||||
```bash
|
||||
sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain certs/mkcert-ca.crt
|
||||
```
|
||||
|
||||
### Linux
|
||||
|
||||
```bash
|
||||
# Ubuntu/Debian
|
||||
sudo cp certs/mkcert-ca.crt /usr/local/share/ca-certificates/mkcert-ca.crt
|
||||
sudo update-ca-certificates
|
||||
|
||||
# Fedora/RHEL
|
||||
sudo cp certs/mkcert-ca.crt /etc/pki/ca-trust/source/anchors/
|
||||
sudo update-ca-trust
|
||||
```
|
||||
|
||||
### Firefox (All Platforms)
|
||||
|
||||
Firefox uses its own certificate store:
|
||||
|
||||
1. Open Firefox Settings
|
||||
2. Search for "Certificates"
|
||||
3. Click **"View Certificates"**
|
||||
4. Go to **"Authorities"** tab
|
||||
5. Click **"Import..."**
|
||||
6. Select `certs/mkcert-ca.crt`
|
||||
7. Check **"Trust this CA to identify websites"**
|
||||
8. Click **OK**
|
||||
|
||||
## After Installation
|
||||
|
||||
Once the CA certificate is installed:
|
||||
|
||||
- Your browser will trust all mkcert certificates without warnings
|
||||
- Access `https://localhost/` with no security warnings
|
||||
- Images from `https://127.0.0.1/flyer-images/` will load without SSL errors
|
||||
|
||||
## Regenerating Certificates
|
||||
|
||||
If you need to regenerate the certificates (e.g., after rebuilding the container):
|
||||
|
||||
```bash
|
||||
# Inside the container
|
||||
cd /app/certs
|
||||
mkcert localhost 127.0.0.1 ::1
|
||||
mv localhost+2.pem localhost.crt
|
||||
mv localhost+2-key.pem localhost.key
|
||||
nginx -s reload
|
||||
|
||||
# Copy the new CA to the host
|
||||
podman cp flyer-crawler-dev:/app/certs/mkcert-ca.crt ./certs/mkcert-ca.crt
|
||||
```
|
||||
|
||||
Then reinstall the CA certificate as described above.
|
||||
|
||||
## Security Note
|
||||
|
||||
**DO NOT** commit the private key (`localhost.key`) to version control in production projects. For this development-only project, the certificates are checked in for convenience since they're only used locally with self-signed certificates.
|
||||
|
||||
The certificates in this directory are automatically generated by the Dockerfile.dev and should not be used in production.
|
||||
|
||||
## See Also
|
||||
|
||||
- [Dockerfile.dev](../Dockerfile.dev) - Certificate generation (line ~69)
|
||||
- [docker/nginx/dev.conf](../docker/nginx/dev.conf) - NGINX SSL configuration
|
||||
- [docs/FLYER-URL-CONFIGURATION.md](../docs/FLYER-URL-CONFIGURATION.md) - URL configuration details
|
||||
- [docs/development/DEBUGGING.md](../docs/development/DEBUGGING.md) - SSL troubleshooting
|
||||
@@ -1,19 +1,25 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDCTCCAfGgAwIBAgIUHhZUK1vmww2wCepWPuVcU6d27hMwDQYJKoZIhvcNAQEL
|
||||
BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDExODAyMzM0NFoXDTI3MDEx
|
||||
ODAyMzM0NFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
|
||||
AAOCAQ8AMIIBCgKCAQEAuUJGtSZzd+ZpLi+efjrkxJJNfVxVz2VLhknNM2WKeOYx
|
||||
JTK/VaTYq5hrczy6fEUnMhDAJCgEPUFlOK3vn1gFJKNMN8m7arkLVk6PYtrx8CTw
|
||||
w78Q06FLITr6hR0vlJNpN4MsmGxYwUoUpn1j5JdfZF7foxNAZRiwoopf7ZJxltDu
|
||||
PIuFjmVZqdzR8c6vmqIqdawx/V6sL9fizZr+CDH3oTsTUirn2qM+1ibBtPDiBvfX
|
||||
omUsr6MVOcTtvnMvAdy9NfV88qwF7MEWBGCjXkoT1bKCLD8hjn8l7GjRmPcmMFE2
|
||||
GqWEvfJiFkBK0CgSHYEUwzo0UtVNeQr0k0qkDRub6QIDAQABo1MwUTAdBgNVHQ4E
|
||||
FgQU5VeD67yFLV0QNYbHaJ6u9cM6UbkwHwYDVR0jBBgwFoAU5VeD67yFLV0QNYbH
|
||||
aJ6u9cM6UbkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEABueA
|
||||
8ujAD+yjeP5dTgqQH1G0hlriD5LmlJYnktaLarFU+y+EZlRFwjdORF/vLPwSG+y7
|
||||
CLty/xlmKKQop70QzQ5jtJcsWzUjww8w1sO3AevfZlIF3HNhJmt51ihfvtJ7DVCv
|
||||
CNyMeYO0pBqRKwOuhbG3EtJgyV7MF8J25UEtO4t+GzX3jcKKU4pWP+kyLBVfeDU3
|
||||
MQuigd2LBwBQQFxZdpYpcXVKnAJJlHZIt68ycO1oSBEJO9fIF0CiAlC6ITxjtYtz
|
||||
oCjd6cCLKMJiC6Zg7t1Q17vGl+FdGyQObSsiYsYO9N3CVaeDdpyGCH0Rfa0+oZzu
|
||||
a5U9/l1FHlvpX980bw==
|
||||
MIIEJDCCAoygAwIBAgIQfbdj1KSREvW82wxWZcDRsDANBgkqhkiG9w0BAQsFADBf
|
||||
MR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExGjAYBgNVBAsMEXJvb3RA
|
||||
OTIxZjA1YmRkNDk4MSEwHwYDVQQDDBhta2NlcnQgcm9vdEA5MjFmMDViZGQ0OTgw
|
||||
HhcNMjYwMTIyMjAwMzIwWhcNMjgwNDIyMjAwMzIwWjBFMScwJQYDVQQKEx5ta2Nl
|
||||
cnQgZGV2ZWxvcG1lbnQgY2VydGlmaWNhdGUxGjAYBgNVBAsMEXJvb3RANzk2OWU1
|
||||
ZjA2MGM4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2u+HpnTr+Ecj
|
||||
X6Ur4I8YHVKiEIgOFozJWWAeCPSZg0lr9/z3UDl7QPfRKaDk11OMN7DB0Y/ZAIHj
|
||||
kMFdYc+ODOUAKGv/MvlM41OeRUXhrt0Gr/TJoDe9RLzs9ffASpqTHUNNrmyj/fLP
|
||||
M5VZU+4Y2POFq8qji6Otkvqr6wp+2CTS/fkAtenFS2X+Z5u6BBq1/KBSWZhUFSuJ
|
||||
sAGN9u+l20Cj/Sxp2nD1npvMEvPehFKEK2tZGgFr+X426jJ4Znd19EyZoI/xetG6
|
||||
ybSzBdQk1KbhWFa3LPuOG814m9Qh21vaL0Hj2hpqC3KEQ2jiuCovjRS+RBGatltl
|
||||
5m47Cj0UrQIDAQABo3YwdDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYB
|
||||
BQUHAwEwHwYDVR0jBBgwFoAUjf0NOmUuoqSSD1Mbu/3G+wYxvjEwLAYDVR0RBCUw
|
||||
I4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3DQEB
|
||||
CwUAA4IBgQAorThUyu4FxHkOafMp/4CmvptfNvYeYP81DW0wpJxGL68Bz+idvXKv
|
||||
JopX/ZAr+dDhS/TLeSnzt83W7TaBBHY+usvsiBx9+pZOVZIpreXRamPu7utmuC46
|
||||
dictMNGlRNX9bwAApOJ24NCpOVSIIKtjHcjl4idwUHqLVGS+3wsmxIILYxigzkuT
|
||||
fcK5vs0ItZWeuunsBAGb/U/Iu9zZ71rtmBejxNPyEvd8+bZC0m2mtV8C0Lpn58jZ
|
||||
FiEf5OHiOdWG9O/uh3QeVWkuKLmaH6a8VdKRSIlOxEEZdkUYlwuVvzeWgFw4kjm8
|
||||
rNWz0aIPovmcgLXoUG1d1D8klveGd3plF7N2p3xWqKmS6R6FJHx7MIxH6XmBATii
|
||||
x/193Sgzqe8mwXQr14ulg2M/B3ZWleNdD6SeieADSgvRKjnlO7xwUmcFrffnEKEG
|
||||
WcSPoGIuZ8V2pkYLh7ipPN+tFUbhmrWnsao5kQ0sqLlfscyO9hYQeQRtTjtC3P8r
|
||||
FJYOdBMOCzE=
|
||||
-----END CERTIFICATE-----
|
||||
|
||||
@@ -1,28 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC5Qka1JnN35mku
|
||||
L55+OuTEkk19XFXPZUuGSc0zZYp45jElMr9VpNirmGtzPLp8RScyEMAkKAQ9QWU4
|
||||
re+fWAUko0w3ybtquQtWTo9i2vHwJPDDvxDToUshOvqFHS+Uk2k3gyyYbFjBShSm
|
||||
fWPkl19kXt+jE0BlGLCiil/tknGW0O48i4WOZVmp3NHxzq+aoip1rDH9Xqwv1+LN
|
||||
mv4IMfehOxNSKufaoz7WJsG08OIG99eiZSyvoxU5xO2+cy8B3L019XzyrAXswRYE
|
||||
YKNeShPVsoIsPyGOfyXsaNGY9yYwUTYapYS98mIWQErQKBIdgRTDOjRS1U15CvST
|
||||
SqQNG5vpAgMBAAECggEAAnv0Dw1Mv+rRy4ZyxtObEVPXPRzoxnDDXzHP4E16BTye
|
||||
Fc/4pSBUIAUn2bPvLz0/X8bMOa4dlDcIv7Eu9Pvns8AY70vMaUReA80fmtHVD2xX
|
||||
1PCT0X3InnxRAYKstSIUIGs+aHvV5Z+iJ8F82soOStN1MU56h+JLWElL5deCPHq3
|
||||
tLZT8wM9aOZlNG72kJ71+DlcViahynQj8+VrionOLNjTJ2Jv/ByjM3GMIuSdBrgd
|
||||
Sl4YAcdn6ontjJGoTgI+e+qkBAPwMZxHarNGQgbS0yNVIJe7Lq4zIKHErU/ZSmpD
|
||||
GzhdVNzhrjADNIDzS7G+pxtz+aUxGtmRvOyopy8GAQKBgQDEPp2mRM+uZVVT4e1j
|
||||
pkKO1c3O8j24I5mGKwFqhhNs3qGy051RXZa0+cQNx63GokXQan9DIXzc/Il7Y72E
|
||||
z9bCFbcSWnlP8dBIpWiJm+UmqLXRyY4N8ecNnzL5x+Tuxm5Ij+ixJwXgdz/TLNeO
|
||||
MBzu+Qy738/l/cAYxwcF7mR7AQKBgQDxq1F95HzCxBahRU9OGUO4s3naXqc8xKCC
|
||||
m3vbbI8V0Exse2cuiwtlPPQWzTPabLCJVvCGXNru98sdeOu9FO9yicwZX0knOABK
|
||||
QfPyDeITsh2u0C63+T9DNn6ixI/T68bTs7DHawEYbpS7bR50BnbHbQrrOAo6FSXF
|
||||
yC7+Te+o6QKBgQCXEWSmo/4D0Dn5Usg9l7VQ40GFd3EPmUgLwntal0/I1TFAyiom
|
||||
gpcLReIogXhCmpSHthO1h8fpDfZ/p+4ymRRHYBQH6uHMKugdpEdu9zVVpzYgArp5
|
||||
/afSEqVZJwoSzWoELdQA23toqiPV2oUtDdiYFdw5nDccY1RHPp8nb7amAQKBgQDj
|
||||
f4DhYDxKJMmg21xCiuoDb4DgHoaUYA0xpii8cL9pq4KmBK0nVWFO1kh5Robvsa2m
|
||||
PB+EfNjkaIPepLxWbOTUEAAASoDU2JT9UoTQcl1GaUAkFnpEWfBB14TyuNMkjinH
|
||||
lLpvn72SQFbm8VvfoU4jgfTrZP/LmajLPR1v6/IWMQKBgBh9qvOTax/GugBAWNj3
|
||||
ZvF99rHOx0rfotEdaPcRN66OOiSWILR9yfMsTvwt1V0VEj7OqO9juMRFuIyB57gd
|
||||
Hs/zgbkuggqjr1dW9r22P/UpzpodAEEN2d52RSX8nkMOkH61JXlH2MyRX65kdExA
|
||||
VkTDq6KwomuhrU3z0+r/MSOn
|
||||
MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDa74emdOv4RyNf
|
||||
pSvgjxgdUqIQiA4WjMlZYB4I9JmDSWv3/PdQOXtA99EpoOTXU4w3sMHRj9kAgeOQ
|
||||
wV1hz44M5QAoa/8y+UzjU55FReGu3Qav9MmgN71EvOz198BKmpMdQ02ubKP98s8z
|
||||
lVlT7hjY84WryqOLo62S+qvrCn7YJNL9+QC16cVLZf5nm7oEGrX8oFJZmFQVK4mw
|
||||
AY3276XbQKP9LGnacPWem8wS896EUoQra1kaAWv5fjbqMnhmd3X0TJmgj/F60brJ
|
||||
tLMF1CTUpuFYVrcs+44bzXib1CHbW9ovQePaGmoLcoRDaOK4Ki+NFL5EEZq2W2Xm
|
||||
bjsKPRStAgMBAAECggEBALFhpHwe+xh7OpPBlR0pkpYfXyMZuKBYjMIW9/61frM6
|
||||
B3oywIWFLPFkV1js/Lvg+xgb48zQSTb6BdBAelJHAYY8+7XEWk2IYt1D4FWr2r/8
|
||||
X/Cr2bgvsO9CSpK2mltXhZ4N66BIcU3NLkdS178Ch6svErwvP/ZhNL6Czktug3rG
|
||||
S2fxpKqoVQhqWiEBV0vBWpw2oskvcpP2Btev2eaJeDmP1IkCaKIU9jJzmSg8UKj3
|
||||
AxJJ8lJvlDi2Z8mfB0BVIFZSI1s44LqbuPdITsWvG49srdAhyLkifcbY2r7eH8+s
|
||||
rFNswCaqqNwmzZXHMFedfvgVJHtCTGX/U8G55dcG/YECgYEA6PvcX+axT4aSaG+/
|
||||
fQpyW8TmNmIwS/8HD8rA5dSBB2AE+RnqbxxB1SWWk4w47R/riYOJCy6C88XUZWLn
|
||||
05cYotR9lA6tZOYqIPUIxfCHl4vDDBnOGAB0ga3z5vF3X7HV6uspPFcrGVJ9k42S
|
||||
BK1gk7Kj8RxVQMqXSqkR/pXJ4l0CgYEA8JBkZ4MRi1D74SAU+TY1rnUCTs2EnP1U
|
||||
TKAI9qHyvNJOFZaObWPzegkZryZTm+yTblwvYkryMjvsiuCdv6ro/VKc667F3OBs
|
||||
dJ/8+ylO0lArP9a23QHViNSvQmyi3bsw2UpIGQRGq2C4LxDceo5cYoVOWIlLl8u3
|
||||
LUuL0IfxdpECgYEAyMd0DPljyGLyfSoAXaPJFajDtA4+DOAEl+lk/yt43oAzCPD6
|
||||
hTJW0XcJIrJuxHsDoohGa+pzU90iwxTPMBtAUeLJLfTQHOn1WF2SZ/J3B3ScbCs4
|
||||
3ppVzQO580YYV9GLxl1ONf/w1muuaKBSO9GmLuJ+QeTm22U7qE23gixXxMkCgYEA
|
||||
3R7cK4l+huBZpgUnQitiDInhJS4jx2nUItq3YnxZ8tYckBtjr4lAM9xJj4VbNOew
|
||||
XLC/nUnmdeY+9yif153xq2hUdQ6hMPXYuxqUHwlJOmgWWQez7lHRRYS51ASnb8iw
|
||||
jgqJWvVjQAQXSKvm/X/9y1FdQmRw54aJSUk3quZKPQECgYA5DqFXn/jYOUCGfCUD
|
||||
RENWe+3fNZSJCWmr4yvjEy9NUT5VcA+/hZHbbLUnVHfAHxJmdGhTzFe+ZGWFAGwi
|
||||
Wo62BDu1fqHHegCWaFFluDnz8iZTxXtHMEGSBZnXC4f5wylcxRtCPUbrLMRTPo4O
|
||||
t85qeBu1dMq1XQtU6ab3w3W0nw==
|
||||
-----END PRIVATE KEY-----
|
||||
|
||||
27
certs/mkcert-ca.crt
Normal file
27
certs/mkcert-ca.crt
Normal file
@@ -0,0 +1,27 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIEjjCCAvagAwIBAgIRAJtq2Z0+W981Ct2dMVPb3bQwDQYJKoZIhvcNAQELBQAw
|
||||
XzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMRowGAYDVQQLDBFyb290
|
||||
QDkyMWYwNWJkZDQ5ODEhMB8GA1UEAwwYbWtjZXJ0IHJvb3RAOTIxZjA1YmRkNDk4
|
||||
MB4XDTI2MDEyMjA5NDcxNVoXDTM2MDEyMjA5NDcxNVowXzEeMBwGA1UEChMVbWtj
|
||||
ZXJ0IGRldmVsb3BtZW50IENBMRowGAYDVQQLDBFyb290QDkyMWYwNWJkZDQ5ODEh
|
||||
MB8GA1UEAwwYbWtjZXJ0IHJvb3RAOTIxZjA1YmRkNDk4MIIBojANBgkqhkiG9w0B
|
||||
AQEFAAOCAY8AMIIBigKCAYEAxXR5gXDwv5cfQSud1eEhwDuaSaf5kf8NtPnucZXY
|
||||
AN+/QW1+OEKJFuawj/YrSbL/yIB8sUSJToEYNJ4LAgzIZ4+TPYVvOIPqQnimfj98
|
||||
2AKCt73U/AMvoEpts7U0s37f5wF8o+BHXfChxyN//z96+wsQ+2+Q9QBGjirvScF+
|
||||
8FRnupcygDeGZ8x3JQVaEfEV6iYyXFl/4tEDVr9QX4avyUlf0vp1Y90TG3L42JYQ
|
||||
xDU37Ct9dqsxPCPOPjmkQi9HV5TeqLTs/4NdrEYOSk7bOVMzL8EHs2prRL7sWzYJ
|
||||
gRT+VXFPpoSCkZs1gS3FNXukTGx5LNsstyJZRa99cGgDcqvNseig06KUzZrRnCig
|
||||
kARLF/n8VTpHETEuTdxdnXJO3i2N/99mG/2/lej9HNDMaqg45ur5EhaFhHarXMtc
|
||||
wy7nfGoThzscZvpFbVHorhgRxzsUqRHTMHa9mUOYtShMA0IwFccHcczY3CDxXLg9
|
||||
hC+24pdiCxtRmi23JB10Th+nAgMBAAGjRTBDMA4GA1UdDwEB/wQEAwICBDASBgNV
|
||||
HRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBSN/Q06ZS6ipJIPUxu7/cb7BjG+MTAN
|
||||
BgkqhkiG9w0BAQsFAAOCAYEAghQL80XpPs70EbCmETYri5ryjJPbsxK3Z3QgBYoy
|
||||
/P/021eK5woLx4EDeEUyrRayOtLlDWiIMdV7FmlY4GpEdonVbqFJY2gghkIuQpkw
|
||||
lWqbk08F+iXe8PSGD1qz4y6enioCRAx+RaZ6jnVJtaC8AR257FVhGIzDBiOA+LDM
|
||||
L1yq6Bxxij17q9s5HL9KuzxWRMuXACHmaGBXHpl/1n4dIxi2lXRgp+1xCR/VPNPt
|
||||
/ZRy29kncd8Fxx+VEtc0muoJvRo4ttVWhBvAVJrkAeukjYKpcDDfRU6Y22o54jD/
|
||||
mseDb+0UPwSxaKbnGJlCcRbbh6i14bA4KfPq93bZX+Tlq9VCp8LvQSl3oU+23RVc
|
||||
KjBB9EJnoBBNGIY7VmRI+QowrnlP2wtg2fTNaPqULtjqA9frbMTP0xTlumLzGB+6
|
||||
9Da7/+AE2in3Aa8Xyry4BiXbk2L6c1xz/Cd1ZpFrSXAOEi1Xt7so/Ck0yM3c2KWK
|
||||
5aSfCjjOzONMPJyY1oxodJ1p
|
||||
-----END CERTIFICATE-----
|
||||
@@ -46,12 +46,19 @@ services:
|
||||
- node_modules_data:/app/node_modules
|
||||
# Mount PostgreSQL logs for Logstash access (ADR-050)
|
||||
- postgres_logs:/var/log/postgresql:ro
|
||||
# Mount Redis logs for Logstash access (ADR-050)
|
||||
- redis_logs:/var/log/redis:ro
|
||||
# Mount PM2 logs for Logstash access (ADR-050, ADR-014)
|
||||
- pm2_logs:/var/log/pm2
|
||||
ports:
|
||||
- '80:80' # HTTP redirect to HTTPS (matches production)
|
||||
- '443:443' # Frontend HTTPS (nginx proxies Vite 5173 → 443)
|
||||
- '3001:3001' # Backend API
|
||||
- '8000:8000' # Bugsink error tracking (ADR-015)
|
||||
- '8000:8000' # Bugsink error tracking HTTP (ADR-015)
|
||||
- '8443:8443' # Bugsink error tracking HTTPS (ADR-015)
|
||||
environment:
|
||||
# Timezone: PST (America/Los_Angeles) for consistent log timestamps
|
||||
- TZ=America/Los_Angeles
|
||||
# Core settings
|
||||
- NODE_ENV=development
|
||||
# Database - use service name for Docker networking
|
||||
@@ -77,13 +84,16 @@ services:
|
||||
- BUGSINK_DB_USER=bugsink
|
||||
- BUGSINK_DB_PASSWORD=bugsink_dev_password
|
||||
- BUGSINK_PORT=8000
|
||||
- BUGSINK_BASE_URL=http://localhost:8000
|
||||
- BUGSINK_BASE_URL=https://localhost:8443
|
||||
- BUGSINK_ADMIN_EMAIL=admin@localhost
|
||||
- BUGSINK_ADMIN_PASSWORD=admin
|
||||
- BUGSINK_SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security
|
||||
# Sentry SDK configuration (points to local Bugsink)
|
||||
- SENTRY_DSN=http://59a58583-e869-7697-f94a-cfa0337676a8@localhost:8000/1
|
||||
- VITE_SENTRY_DSN=http://d5fc5221-4266-ff2f-9af8-5689696072f3@localhost:8000/2
|
||||
# Sentry SDK configuration (points to local Bugsink HTTP)
|
||||
# Note: Using HTTP with 127.0.0.1 instead of localhost because Sentry SDK
|
||||
# doesn't accept 'localhost' as a valid hostname in DSN validation
|
||||
# The browser accesses Bugsink at http://localhost:8000 (nginx proxies to HTTPS for the app)
|
||||
- SENTRY_DSN=http://cea01396-c562-46ad-b587-8fa5ee6b1d22@127.0.0.1:8000/1
|
||||
- VITE_SENTRY_DSN=http://d92663cb-73cf-4145-b677-b84029e4b762@127.0.0.1:8000/2
|
||||
- SENTRY_ENVIRONMENT=development
|
||||
- VITE_SENTRY_ENVIRONMENT=development
|
||||
- SENTRY_ENABLED=true
|
||||
@@ -114,6 +124,10 @@ services:
|
||||
ports:
|
||||
- '5432:5432'
|
||||
environment:
|
||||
# Timezone: PST (America/Los_Angeles) for consistent log timestamps
|
||||
TZ: America/Los_Angeles
|
||||
# PostgreSQL timezone setting (used by log_timezone and timezone parameters)
|
||||
PGTZ: America/Los_Angeles
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: flyer_crawler_dev
|
||||
@@ -134,6 +148,8 @@ services:
|
||||
postgres
|
||||
-c config_file=/var/lib/postgresql/data/postgresql.conf
|
||||
-c hba_file=/var/lib/postgresql/data/pg_hba.conf
|
||||
-c timezone=America/Los_Angeles
|
||||
-c log_timezone=America/Los_Angeles
|
||||
-c log_min_messages=notice
|
||||
-c client_min_messages=notice
|
||||
-c logging_collector=on
|
||||
@@ -162,10 +178,20 @@ services:
|
||||
redis:
|
||||
image: docker.io/library/redis:alpine
|
||||
container_name: flyer-crawler-redis
|
||||
# Run as root to allow entrypoint to set up log directory permissions
|
||||
# Redis will drop privileges after setup via gosu in entrypoint
|
||||
user: root
|
||||
ports:
|
||||
- '6379:6379'
|
||||
environment:
|
||||
# Timezone: PST (America/Los_Angeles) for consistent log timestamps
|
||||
TZ: America/Los_Angeles
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
# Create log volume for Logstash access (ADR-050)
|
||||
- redis_logs:/var/log/redis
|
||||
# Mount custom entrypoint for log directory setup (ADR-050)
|
||||
- ./docker/redis/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh:ro
|
||||
# Healthcheck ensures redis is ready before app starts
|
||||
healthcheck:
|
||||
test: ['CMD', 'redis-cli', 'ping']
|
||||
@@ -173,8 +199,17 @@ services:
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
# Enable persistence for development data
|
||||
command: redis-server --appendonly yes
|
||||
# Enable persistence and file logging for Logstash (ADR-050)
|
||||
# Redis log levels: debug, verbose, notice (default), warning
|
||||
# Custom entrypoint sets up log directory with correct permissions
|
||||
entrypoint: ['/usr/local/bin/docker-entrypoint.sh']
|
||||
command:
|
||||
- --appendonly
|
||||
- 'yes'
|
||||
- --logfile
|
||||
- /var/log/redis/redis-server.log
|
||||
- --loglevel
|
||||
- notice
|
||||
|
||||
# ===================
|
||||
# Named Volumes
|
||||
@@ -186,6 +221,10 @@ volumes:
|
||||
name: flyer-crawler-postgres-logs
|
||||
redis_data:
|
||||
name: flyer-crawler-redis-data
|
||||
redis_logs:
|
||||
name: flyer-crawler-redis-logs
|
||||
pm2_logs:
|
||||
name: flyer-crawler-pm2-logs
|
||||
node_modules_data:
|
||||
name: flyer-crawler-node-modules
|
||||
|
||||
|
||||
574
docker/logstash/bugsink.conf
Normal file
574
docker/logstash/bugsink.conf
Normal file
@@ -0,0 +1,574 @@
|
||||
# docker/logstash/bugsink.conf
|
||||
# ============================================================================
|
||||
# Logstash Pipeline Configuration for Bugsink Error Tracking
|
||||
# ============================================================================
|
||||
# This configuration aggregates logs from multiple sources and forwards errors
|
||||
# to Bugsink (Sentry-compatible error tracking) in the dev container.
|
||||
#
|
||||
# Sources:
|
||||
# - PM2 managed logs (/var/log/pm2/*.log) - API, Worker, Vite (ADR-014)
|
||||
# - Pino application logs (/app/logs/*.log) - JSON format (fallback)
|
||||
# - PostgreSQL logs (/var/log/postgresql/*.log) - Including fn_log() output
|
||||
# - NGINX logs (/var/log/nginx/*.log) - Access and error logs
|
||||
# - Redis logs (/var/log/redis/*.log) - Via shared volume (ADR-050)
|
||||
#
|
||||
# Bugsink Projects (3-project architecture):
|
||||
# - Project 1: Backend API (Dev) - Pino/PM2 app errors, PostgreSQL errors
|
||||
# DSN Key: cea01396c56246adb5878fa5ee6b1d22
|
||||
# - Project 2: Frontend (Dev) - Configured via Sentry SDK in browser
|
||||
# DSN Key: d92663cb73cf4145b677b84029e4b762
|
||||
# - Project 4: Infrastructure (Dev) - Redis, NGINX, PM2 operational logs
|
||||
# DSN Key: 14e8791da3d347fa98073261b596cab9
|
||||
#
|
||||
# Routing Logic:
|
||||
# - Backend logs (type: pm2_api, pm2_worker, pino, postgres) -> Project 1
|
||||
# - Infrastructure logs (type: redis, nginx_error, nginx_5xx) -> Project 4
|
||||
# - Vite errors (type: pm2_vite with errors) -> Project 4 (build tooling)
|
||||
#
|
||||
# Related Documentation:
|
||||
# - docs/adr/0050-postgresql-function-observability.md
|
||||
# - docs/adr/0015-application-performance-monitoring-and-error-tracking.md
|
||||
# - docs/operations/LOGSTASH-QUICK-REF.md
|
||||
# ============================================================================
|
||||
|
||||
input {
|
||||
# ============================================================================
|
||||
# PM2 Managed Process Logs (ADR-014, ADR-050)
|
||||
# ============================================================================
|
||||
# PM2 manages all Node.js processes in the dev container, matching production.
|
||||
# Logs are written to /var/log/pm2 for Logstash integration.
|
||||
#
|
||||
# Process logs:
|
||||
# - api-out.log / api-error.log: API server (Pino JSON format)
|
||||
# - worker-out.log / worker-error.log: Background worker (Pino JSON format)
|
||||
# - vite-out.log / vite-error.log: Vite dev server (plain text)
|
||||
# ============================================================================
|
||||
|
||||
# PM2 API Server Logs (Pino JSON format)
|
||||
file {
|
||||
path => "/var/log/pm2/api-out.log"
|
||||
codec => json_lines
|
||||
type => "pm2_api"
|
||||
tags => ["app", "backend", "pm2", "api"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_api_out"
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/pm2/api-error.log"
|
||||
codec => json_lines
|
||||
type => "pm2_api"
|
||||
tags => ["app", "backend", "pm2", "api", "stderr"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_api_error"
|
||||
}
|
||||
|
||||
# PM2 Worker Logs (Pino JSON format)
|
||||
file {
|
||||
path => "/var/log/pm2/worker-out.log"
|
||||
codec => json_lines
|
||||
type => "pm2_worker"
|
||||
tags => ["app", "worker", "pm2"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_worker_out"
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/pm2/worker-error.log"
|
||||
codec => json_lines
|
||||
type => "pm2_worker"
|
||||
tags => ["app", "worker", "pm2", "stderr"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_worker_error"
|
||||
}
|
||||
|
||||
# PM2 Vite Logs (plain text format)
|
||||
file {
|
||||
path => "/var/log/pm2/vite-out.log"
|
||||
codec => plain
|
||||
type => "pm2_vite"
|
||||
tags => ["app", "frontend", "pm2", "vite"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_vite_out"
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/pm2/vite-error.log"
|
||||
codec => plain
|
||||
type => "pm2_vite"
|
||||
tags => ["app", "frontend", "pm2", "vite", "stderr"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pm2_vite_error"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Pino Application Logs (Fallback Path)
|
||||
# ============================================================================
|
||||
# JSON-formatted logs from the Node.js application using Pino logger.
|
||||
# Note: Primary logs now go to /var/log/pm2. This is a fallback.
|
||||
# Log levels: 10=trace, 20=debug, 30=info, 40=warn, 50=error, 60=fatal
|
||||
file {
|
||||
path => "/app/logs/*.log"
|
||||
codec => json
|
||||
type => "pino"
|
||||
tags => ["app", "backend"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pino"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PostgreSQL Function Logs (ADR-050)
|
||||
# ============================================================================
|
||||
# Captures PostgreSQL log output including fn_log() structured JSON messages.
|
||||
# PostgreSQL is configured to write logs to /var/log/postgresql/ (shared volume).
|
||||
# Log format: "2026-01-22 14:30:00 PST [5724] postgres@flyer_crawler_dev LOG: message"
|
||||
# Note: Timestamps are in PST (America/Los_Angeles) timezone as configured in compose.dev.yml
|
||||
file {
|
||||
path => "/var/log/postgresql/*.log"
|
||||
type => "postgres"
|
||||
tags => ["postgres", "database"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_postgres"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# NGINX Logs
|
||||
# ============================================================================
|
||||
# Access logs for request monitoring and error logs for proxy/SSL issues.
|
||||
file {
|
||||
path => "/var/log/nginx/access.log"
|
||||
type => "nginx_access"
|
||||
tags => ["nginx", "access"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_nginx_access"
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/nginx/error.log"
|
||||
type => "nginx_error"
|
||||
tags => ["nginx", "error"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_nginx_error"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Redis Logs (ADR-050)
|
||||
# ============================================================================
|
||||
# Redis logs from the shared volume. Redis uses its own log format:
|
||||
# "<pid>:<role> <day> <month> <time> <loglevel> <message>"
|
||||
# Example: "1:M 22 Jan 2026 14:30:00.123 * Ready to accept connections"
|
||||
# Log levels: . (debug), - (verbose), * (notice), # (warning)
|
||||
file {
|
||||
path => "/var/log/redis/redis-server.log"
|
||||
type => "redis"
|
||||
tags => ["redis", "infra"]
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/var/lib/logstash/sincedb_redis"
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
# ============================================================================
|
||||
# PM2 API/Worker Log Processing (ADR-014)
|
||||
# ============================================================================
|
||||
# PM2 API and Worker logs are in Pino JSON format.
|
||||
# Process them the same way as direct Pino logs.
|
||||
if [type] in ["pm2_api", "pm2_worker"] {
|
||||
# Tag errors (level 50 = error, 60 = fatal)
|
||||
if [level] >= 50 {
|
||||
mutate { add_tag => ["error", "pm2_pino_error"] }
|
||||
|
||||
# Map Pino level to Sentry level and set error_message field
|
||||
if [level] == 60 {
|
||||
mutate { add_field => { "sentry_level" => "fatal" } }
|
||||
} else {
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
}
|
||||
|
||||
# Copy msg to error_message for consistent access in output
|
||||
mutate { add_field => { "error_message" => "%{msg}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PM2 Vite Log Processing (ADR-014)
|
||||
# ============================================================================
|
||||
# Vite logs are plain text. Look for error patterns.
|
||||
if [type] == "pm2_vite" {
|
||||
# Detect Vite build/compilation errors
|
||||
if [message] =~ /(?i)(error|failed|exception|cannot|enoent|eperm|EACCES|ECONNREFUSED)/ {
|
||||
mutate { add_tag => ["error", "vite_error"] }
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
mutate { add_field => { "error_message" => "Vite: %{message}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Pino Log Processing (Fallback)
|
||||
# ============================================================================
|
||||
if [type] == "pino" {
|
||||
# Tag errors (level 50 = error, 60 = fatal)
|
||||
if [level] >= 50 {
|
||||
mutate { add_tag => ["error", "pino_error"] }
|
||||
|
||||
# Map Pino level to Sentry level and set error_message field
|
||||
if [level] == 60 {
|
||||
mutate { add_field => { "sentry_level" => "fatal" } }
|
||||
} else {
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
}
|
||||
|
||||
# Copy msg to error_message for consistent access in output
|
||||
mutate { add_field => { "error_message" => "%{msg}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PostgreSQL Log Processing (ADR-050)
|
||||
# ============================================================================
|
||||
# PostgreSQL log format in dev container:
|
||||
# "2026-01-22 14:30:00 PST [5724] postgres@flyer_crawler_dev LOG: message"
|
||||
# "2026-01-22 15:06:03 PST [19851] postgres@flyer_crawler_dev ERROR: column "id" does not exist"
|
||||
# Note: Timestamps are in PST (America/Los_Angeles) timezone
|
||||
if [type] == "postgres" {
|
||||
# Parse PostgreSQL log prefix with timezone (PST in dev, may vary in prod)
|
||||
grok {
|
||||
match => { "message" => "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME} %{WORD:pg_timezone} \[%{POSINT:pg_pid}\] %{DATA:pg_user}@%{DATA:pg_database} %{WORD:pg_level}: ?%{GREEDYDATA:pg_message}" }
|
||||
tag_on_failure => ["_postgres_grok_failure"]
|
||||
}
|
||||
|
||||
# Check if this is a structured JSON log from fn_log()
|
||||
if [pg_message] =~ /^\{.*"source":"postgresql".*\}$/ {
|
||||
json {
|
||||
source => "pg_message"
|
||||
target => "fn_log"
|
||||
}
|
||||
|
||||
# Mark as error if level is WARNING or ERROR
|
||||
if [fn_log][level] in ["WARNING", "ERROR"] {
|
||||
mutate { add_tag => ["error", "db_function"] }
|
||||
mutate { add_field => { "sentry_level" => "warning" } }
|
||||
if [fn_log][level] == "ERROR" {
|
||||
mutate { replace => { "sentry_level" => "error" } }
|
||||
}
|
||||
# Use fn_log message for error_message
|
||||
mutate { add_field => { "error_message" => "%{[fn_log][message]}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# Catch native PostgreSQL errors (ERROR: or FATAL: prefix in pg_level)
|
||||
if [pg_level] == "ERROR" or [pg_level] == "FATAL" {
|
||||
mutate { add_tag => ["error", "postgres_native"] }
|
||||
|
||||
if [pg_level] == "FATAL" {
|
||||
mutate { add_field => { "sentry_level" => "fatal" } }
|
||||
} else {
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
}
|
||||
|
||||
# Use the full pg_message for error_message
|
||||
mutate { add_field => { "error_message" => "PostgreSQL %{pg_level}: %{pg_message}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# NGINX Access Log Processing
|
||||
# ============================================================================
|
||||
if [type] == "nginx_access" {
|
||||
grok {
|
||||
match => { "message" => '%{IPORHOST:client_ip} - %{DATA:user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:user_agent}"' }
|
||||
tag_on_failure => ["_nginx_access_grok_failure"]
|
||||
}
|
||||
|
||||
# Tag 5xx errors for Bugsink
|
||||
if [status] =~ /^5/ {
|
||||
mutate { add_tag => ["error", "nginx_5xx"] }
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
mutate { add_field => { "error_message" => "HTTP %{status} %{method} %{request}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# NGINX Error Log Processing
|
||||
# ============================================================================
|
||||
# NGINX error log format: "2026/01/22 17:55:01 [error] 16#16: *3 message..."
|
||||
if [type] == "nginx_error" {
|
||||
grok {
|
||||
match => { "message" => "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME} \[%{LOGLEVEL:nginx_level}\] %{POSINT:pid}#%{POSINT}: (\*%{POSINT:connection} )?%{GREEDYDATA:nginx_message}" }
|
||||
tag_on_failure => ["_nginx_error_grok_failure"]
|
||||
}
|
||||
|
||||
# Only process actual errors, not notices (like "signal process started")
|
||||
if [nginx_level] in ["error", "crit", "alert", "emerg"] {
|
||||
mutate { add_tag => ["error", "nginx_error"] }
|
||||
|
||||
# Map NGINX log level to Sentry level
|
||||
if [nginx_level] in ["crit", "alert", "emerg"] {
|
||||
mutate { add_field => { "sentry_level" => "fatal" } }
|
||||
} else {
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
}
|
||||
|
||||
mutate { add_field => { "error_message" => "NGINX [%{nginx_level}]: %{nginx_message}" } }
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Redis Log Processing (ADR-050)
|
||||
# ============================================================================
|
||||
# Redis log format: "<pid>:<role> <day> <month> <time>.<ms> <level> <message>"
|
||||
# Example: "1:M 22 Jan 14:30:00.123 * Ready to accept connections"
|
||||
# Roles: M=master, S=slave, C=sentinel, X=cluster
|
||||
# Levels: . (debug), - (verbose), * (notice), # (warning)
|
||||
if [type] == "redis" {
|
||||
# Parse Redis log format
|
||||
grok {
|
||||
match => { "message" => "%{POSINT:redis_pid}:%{WORD:redis_role} %{MONTHDAY} %{MONTH} %{YEAR}? ?%{TIME} %{DATA:redis_level_char} %{GREEDYDATA:redis_message}" }
|
||||
tag_on_failure => ["_redis_grok_failure"]
|
||||
}
|
||||
|
||||
# Map Redis level characters to human-readable levels
|
||||
# . = debug, - = verbose, * = notice, # = warning
|
||||
if [redis_level_char] == "#" {
|
||||
mutate {
|
||||
add_field => { "redis_level" => "warning" }
|
||||
add_tag => ["error", "redis_warning"]
|
||||
}
|
||||
mutate { add_field => { "sentry_level" => "warning" } }
|
||||
mutate { add_field => { "error_message" => "Redis WARNING: %{redis_message}" } }
|
||||
} else if [redis_level_char] == "*" {
|
||||
mutate { add_field => { "redis_level" => "notice" } }
|
||||
} else if [redis_level_char] == "-" {
|
||||
mutate { add_field => { "redis_level" => "verbose" } }
|
||||
} else if [redis_level_char] == "." {
|
||||
mutate { add_field => { "redis_level" => "debug" } }
|
||||
}
|
||||
|
||||
# Also detect error keywords in message content (e.g., ECONNREFUSED, OOM, etc.)
|
||||
if [redis_message] =~ /(?i)(error|failed|refused|denied|timeout|oom|crash|fatal|exception)/ {
|
||||
if "error" not in [tags] {
|
||||
mutate { add_tag => ["error", "redis_error"] }
|
||||
mutate { add_field => { "sentry_level" => "error" } }
|
||||
mutate { add_field => { "error_message" => "Redis ERROR: %{redis_message}" } }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Generate Sentry Event ID and Ensure Required Fields for all errors
|
||||
# ============================================================================
|
||||
# CRITICAL: sentry_level MUST be set for all errors before output.
|
||||
# Bugsink's PostgreSQL schema limits level to varchar(7), so valid values are:
|
||||
# fatal, error, warning, info, debug (all <= 7 chars)
|
||||
# If sentry_level is not set, the literal "%{sentry_level}" (16 chars) is sent,
|
||||
# causing PostgreSQL insertion failures.
|
||||
# ============================================================================
|
||||
if "error" in [tags] {
|
||||
# Use Ruby for robust field handling - handles all edge cases
|
||||
ruby {
|
||||
code => '
|
||||
require "securerandom"
|
||||
|
||||
# Generate unique event ID for Sentry
|
||||
event.set("sentry_event_id", SecureRandom.hex(16))
|
||||
|
||||
# =====================================================================
|
||||
# CRITICAL: Validate and set sentry_level
|
||||
# =====================================================================
|
||||
# Valid Sentry levels (max 7 chars for Bugsink PostgreSQL schema):
|
||||
# fatal, error, warning, info, debug
|
||||
# Default to "error" if missing, empty, or invalid.
|
||||
# =====================================================================
|
||||
valid_levels = ["fatal", "error", "warning", "info", "debug"]
|
||||
current_level = event.get("sentry_level")
|
||||
|
||||
if current_level.nil? || current_level.to_s.strip.empty? || !valid_levels.include?(current_level.to_s.downcase)
|
||||
event.set("sentry_level", "error")
|
||||
else
|
||||
# Normalize to lowercase
|
||||
event.set("sentry_level", current_level.to_s.downcase)
|
||||
end
|
||||
|
||||
# =====================================================================
|
||||
# Ensure error_message has a fallback value
|
||||
# =====================================================================
|
||||
error_msg = event.get("error_message")
|
||||
if error_msg.nil? || error_msg.to_s.strip.empty?
|
||||
fallback_msg = event.get("message") || event.get("msg") || "Unknown error"
|
||||
event.set("error_message", fallback_msg.to_s)
|
||||
end
|
||||
'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
# ============================================================================
|
||||
# Forward Errors to Bugsink (Project Routing)
|
||||
# ============================================================================
|
||||
# Bugsink uses Sentry-compatible API. Events must include:
|
||||
# - event_id: 32 hex characters (UUID without dashes)
|
||||
# - message: Human-readable error description
|
||||
# - level: fatal, error, warning, info, debug
|
||||
# - timestamp: Unix epoch in seconds
|
||||
# - platform: "node" for backend, "javascript" for frontend
|
||||
#
|
||||
# Authentication via X-Sentry-Auth header with project's public key.
|
||||
#
|
||||
# Project Routing:
|
||||
# - Project 1 (Backend): Pino app logs, PostgreSQL errors
|
||||
# - Project 4 (Infrastructure): Redis, NGINX, Vite build errors
|
||||
# ============================================================================
|
||||
|
||||
# ============================================================================
|
||||
# Infrastructure Errors -> Project 4
|
||||
# ============================================================================
|
||||
# Redis warnings/errors, NGINX errors, and Vite build errors go to
|
||||
# the Infrastructure project for separation from application code errors.
|
||||
if "error" in [tags] and ([type] == "redis" or [type] == "nginx_error" or [type] == "nginx_access" or [type] == "pm2_vite") {
|
||||
http {
|
||||
url => "http://localhost:8000/api/4/store/"
|
||||
http_method => "post"
|
||||
format => "json"
|
||||
headers => {
|
||||
"X-Sentry-Auth" => "Sentry sentry_key=14e8791da3d347fa98073261b596cab9, sentry_version=7"
|
||||
"Content-Type" => "application/json"
|
||||
}
|
||||
mapping => {
|
||||
"event_id" => "%{sentry_event_id}"
|
||||
"timestamp" => "%{@timestamp}"
|
||||
"level" => "%{sentry_level}"
|
||||
"platform" => "other"
|
||||
"logger" => "%{type}"
|
||||
"message" => "%{error_message}"
|
||||
"extra" => {
|
||||
"hostname" => "%{[host][name]}"
|
||||
"source_type" => "%{type}"
|
||||
"tags" => "%{tags}"
|
||||
"original_message" => "%{message}"
|
||||
"project" => "infrastructure"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Backend Application Errors -> Project 1
|
||||
# ============================================================================
|
||||
# Pino application logs (API, Worker), PostgreSQL function errors, and
|
||||
# native PostgreSQL errors go to the Backend API project.
|
||||
else if "error" in [tags] and ([type] in ["pm2_api", "pm2_worker", "pino", "postgres"]) {
|
||||
http {
|
||||
url => "http://localhost:8000/api/1/store/"
|
||||
http_method => "post"
|
||||
format => "json"
|
||||
headers => {
|
||||
"X-Sentry-Auth" => "Sentry sentry_key=cea01396c56246adb5878fa5ee6b1d22, sentry_version=7"
|
||||
"Content-Type" => "application/json"
|
||||
}
|
||||
mapping => {
|
||||
"event_id" => "%{sentry_event_id}"
|
||||
"timestamp" => "%{@timestamp}"
|
||||
"level" => "%{sentry_level}"
|
||||
"platform" => "node"
|
||||
"logger" => "%{type}"
|
||||
"message" => "%{error_message}"
|
||||
"extra" => {
|
||||
"hostname" => "%{[host][name]}"
|
||||
"source_type" => "%{type}"
|
||||
"tags" => "%{tags}"
|
||||
"original_message" => "%{message}"
|
||||
"project" => "backend"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Fallback: Any other errors -> Project 1
|
||||
# ============================================================================
|
||||
# Catch-all for any errors that don't match specific routing rules.
|
||||
else if "error" in [tags] {
|
||||
http {
|
||||
url => "http://localhost:8000/api/1/store/"
|
||||
http_method => "post"
|
||||
format => "json"
|
||||
headers => {
|
||||
"X-Sentry-Auth" => "Sentry sentry_key=cea01396c56246adb5878fa5ee6b1d22, sentry_version=7"
|
||||
"Content-Type" => "application/json"
|
||||
}
|
||||
mapping => {
|
||||
"event_id" => "%{sentry_event_id}"
|
||||
"timestamp" => "%{@timestamp}"
|
||||
"level" => "%{sentry_level}"
|
||||
"platform" => "node"
|
||||
"logger" => "%{type}"
|
||||
"message" => "%{error_message}"
|
||||
"extra" => {
|
||||
"hostname" => "%{[host][name]}"
|
||||
"source_type" => "%{type}"
|
||||
"tags" => "%{tags}"
|
||||
"original_message" => "%{message}"
|
||||
"project" => "backend-fallback"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Store Operational Logs to Files (for debugging/audit)
|
||||
# ============================================================================
|
||||
# NGINX access logs (all requests, not just errors)
|
||||
if [type] == "nginx_access" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/nginx-access-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# PostgreSQL operational logs (non-error)
|
||||
if [type] == "postgres" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/postgres-operational-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# Redis operational logs (non-error)
|
||||
if [type] == "redis" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/redis-operational-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# PM2 API operational logs (non-error) - ADR-014
|
||||
if [type] == "pm2_api" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/pm2-api-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# PM2 Worker operational logs (non-error) - ADR-014
|
||||
if [type] == "pm2_worker" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/pm2-worker-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# PM2 Vite operational logs (non-error) - ADR-014
|
||||
if [type] == "pm2_vite" and "error" not in [tags] {
|
||||
file {
|
||||
path => "/var/log/logstash/pm2-vite-%{+YYYY-MM-dd}.log"
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Debug Output (for development only)
|
||||
# ============================================================================
|
||||
# Uncomment to see all processed events in Logstash stdout
|
||||
# stdout { codec => rubydebug }
|
||||
}
|
||||
@@ -9,13 +9,39 @@
|
||||
# - Frontend accessible on https://localhost (port 443)
|
||||
# - Backend API on http://localhost:3001
|
||||
# - Port 80 redirects to HTTPS
|
||||
#
|
||||
# IMPORTANT: Dual Hostname Configuration (localhost AND 127.0.0.1)
|
||||
# ============================================================================
|
||||
# The server_name directive includes BOTH 'localhost' and '127.0.0.1' to
|
||||
# prevent SSL certificate errors when resources use different hostnames.
|
||||
#
|
||||
# Problem Scenario:
|
||||
# 1. User accesses site via https://localhost/
|
||||
# 2. Database stores image URLs as https://127.0.0.1/flyer-images/...
|
||||
# 3. Browser treats these as different origins, showing ERR_CERT_AUTHORITY_INVALID
|
||||
#
|
||||
# Solution:
|
||||
# - mkcert generates certificates valid for: localhost, 127.0.0.1, ::1
|
||||
# - NGINX accepts requests to BOTH hostnames using the same certificate
|
||||
# - Users can access via either hostname without SSL warnings
|
||||
#
|
||||
# The self-signed certificate is generated in Dockerfile.dev with:
|
||||
# mkcert localhost 127.0.0.1 ::1
|
||||
#
|
||||
# This creates a certificate with Subject Alternative Names (SANs) for all
|
||||
# three hostnames, allowing NGINX to serve valid HTTPS for each.
|
||||
#
|
||||
# See also:
|
||||
# - Dockerfile.dev (certificate generation, ~line 69)
|
||||
# - docs/FLYER-URL-CONFIGURATION.md (URL configuration details)
|
||||
# - docs/development/DEBUGGING.md (SSL troubleshooting)
|
||||
# ============================================================================
|
||||
|
||||
# HTTPS Server (main)
|
||||
server {
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
server_name localhost;
|
||||
server_name localhost 127.0.0.1;
|
||||
|
||||
# SSL Configuration (self-signed certificates from mkcert)
|
||||
ssl_certificate /app/certs/localhost.crt;
|
||||
@@ -24,7 +50,67 @@ server {
|
||||
# Allow large file uploads (matches production)
|
||||
client_max_body_size 100M;
|
||||
|
||||
# Proxy all requests to Vite dev server on port 5173
|
||||
# Proxy API requests to Express server on port 3001
|
||||
location /api/ {
|
||||
proxy_pass http://localhost:3001;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Bugsink Sentry API Proxy (for frontend error reporting)
|
||||
# ============================================================================
|
||||
# The frontend Sentry SDK cannot reach localhost:8000 directly from the browser
|
||||
# because port 8000 is only accessible within the container network.
|
||||
# This proxy allows the browser to send errors to https://localhost/bugsink-api/
|
||||
# which NGINX forwards to the Bugsink container on port 8000.
|
||||
#
|
||||
# Frontend DSN format: https://localhost/bugsink-api/<project_id>
|
||||
# Example: https://localhost/bugsink-api/2 for Frontend (Dev) project
|
||||
#
|
||||
# The Sentry SDK sends POST requests to /bugsink-api/<project>/store/
|
||||
# This proxy strips /bugsink-api and forwards to http://localhost:8000/api/
|
||||
# ============================================================================
|
||||
location /bugsink-api/ {
|
||||
proxy_pass http://localhost:8000/api/;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Allow large error payloads with stack traces
|
||||
client_max_body_size 10M;
|
||||
|
||||
# Timeouts for error reporting (should be fast)
|
||||
proxy_connect_timeout 10s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_read_timeout 30s;
|
||||
}
|
||||
|
||||
# Proxy WebSocket connections for real-time notifications
|
||||
location /ws {
|
||||
proxy_pass http://localhost:3001;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# Serve flyer images from static storage
|
||||
location /flyer-images/ {
|
||||
alias /app/public/flyer-images/;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# Proxy all other requests to Vite dev server on port 5173
|
||||
location / {
|
||||
proxy_pass http://localhost:5173;
|
||||
proxy_http_version 1.1;
|
||||
@@ -51,7 +137,7 @@ server {
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name localhost;
|
||||
server_name localhost 127.0.0.1;
|
||||
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
# This file is mounted into the PostgreSQL container to enable structured logging
|
||||
# from database functions via fn_log()
|
||||
|
||||
# Timezone: PST (America/Los_Angeles) for consistent log timestamps
|
||||
timezone = 'America/Los_Angeles'
|
||||
log_timezone = 'America/Los_Angeles'
|
||||
|
||||
# Enable logging to files for Logstash pickup
|
||||
logging_collector = on
|
||||
log_destination = 'stderr'
|
||||
|
||||
24
docker/redis/docker-entrypoint.sh
Normal file
24
docker/redis/docker-entrypoint.sh
Normal file
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
# docker/redis/docker-entrypoint.sh
|
||||
# ============================================================================
|
||||
# Redis Entrypoint Script for Dev Container
|
||||
# ============================================================================
|
||||
# This script ensures the Redis log directory exists and is writable before
|
||||
# starting Redis. This is needed because the redis_logs volume is mounted
|
||||
# at /var/log/redis but Redis Alpine runs as a non-root user.
|
||||
#
|
||||
# Related: ADR-050 (Log aggregation via Logstash)
|
||||
# ============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
# Create log directory if it doesn't exist
|
||||
mkdir -p /var/log/redis
|
||||
|
||||
# Ensure redis user can write to the log directory
|
||||
# Note: In Alpine Redis image, redis runs as uid 999
|
||||
chown -R redis:redis /var/log/redis
|
||||
chmod 755 /var/log/redis
|
||||
|
||||
# Start Redis with the provided arguments
|
||||
exec redis-server "$@"
|
||||
267
docs/BUGSINK-MCP-TROUBLESHOOTING.md
Normal file
267
docs/BUGSINK-MCP-TROUBLESHOOTING.md
Normal file
@@ -0,0 +1,267 @@
|
||||
# Bugsink MCP Troubleshooting Guide
|
||||
|
||||
This document tracks known issues and solutions for the Bugsink MCP server integration with Claude Code.
|
||||
|
||||
## Issue History
|
||||
|
||||
### 2026-01-22: Server Name Prefix Collision (LATEST)
|
||||
|
||||
**Problem:**
|
||||
|
||||
- `bugsink-dev` MCP server never starts (tools not available)
|
||||
- Production `bugsink` MCP works fine
|
||||
- Manual test works: `BUGSINK_URL=http://localhost:8000 BUGSINK_TOKEN=<token> node d:/gitea/bugsink-mcp/dist/index.js`
|
||||
- Configuration correct, environment variables correct, but server silently skipped
|
||||
|
||||
**Root Cause:**
|
||||
Claude Code silently skips MCP servers when server names share prefixes (e.g., `bugsink` and `bugsink-dev`).
|
||||
Debug logs showed `bugsink-dev` was NEVER started - no "Starting connection" message ever appeared.
|
||||
|
||||
**Discovery Method:**
|
||||
|
||||
- Analyzed Claude Code debug logs at `C:\Users\games3\.claude\debug\*.txt`
|
||||
- Found that MCP startup messages only showed: `memory`, `bugsink`, `redis`, `gitea-projectium`, etc.
|
||||
- `bugsink-dev` was completely absent from startup sequence
|
||||
- No error was logged - server was silently filtered out
|
||||
|
||||
**Solution Attempts:**
|
||||
|
||||
**Attempt 1:** Renamed `bugsink-dev` to `devbugsink`
|
||||
|
||||
- New MCP tool prefix: `mcp__devbugsink__*`
|
||||
- Changed URL from `http://localhost:8000` to `http://127.0.0.1:8000`
|
||||
- **Result:** Still failed after full VS Code restart - server never loaded
|
||||
|
||||
**Attempt 2:** Renamed `devbugsink` to `localerrors` (completely different name)
|
||||
|
||||
- New MCP tool prefix: `mcp__localerrors__*`
|
||||
- Uses completely unrelated name with no shared prefix
|
||||
- Based on infra-architect research showing name collision issues
|
||||
- **Result:** Still failed after full VS Code restart - server never loaded
|
||||
|
||||
**Attempt 3:** Created project-level `.mcp.json` file ✅ **SUCCESS**
|
||||
|
||||
- Location: `d:\gitea\flyer-crawler.projectium.com\flyer-crawler.projectium.com\.mcp.json`
|
||||
- Contains `localerrors` server configuration
|
||||
- Project-level config bypassed the global config loader issue
|
||||
- **Result:** ✅ Working! Server loads successfully, 2 projects found
|
||||
- Tool prefix: `mcp__localerrors__*`
|
||||
|
||||
**Alternative Solutions Researched:**
|
||||
|
||||
- Sentry MCP: Not compatible (different API endpoints `/api/canonical/0/` vs `/api/0/`)
|
||||
- MCP-Proxy: Could work but requires separate process
|
||||
- Project-level `.mcp.json`: Alternative if global config continues to fail
|
||||
|
||||
**Status:** ✅ RESOLVED - Project-level `.mcp.json` works successfully
|
||||
|
||||
**Root Cause Analysis:**
|
||||
Claude Code's global settings.json has an issue loading localhost stdio MCP servers, even with completely distinct names. The exact cause is unknown, but may be related to:
|
||||
|
||||
- Multiple servers using the same package (bugsink-mcp)
|
||||
- Localhost URL filtering in global config
|
||||
- Internal MCP loader bug specific to Windows/localhost combinations
|
||||
|
||||
**Working Solution:**
|
||||
Use **project-level `.mcp.json`** file instead of global `settings.json` for localhost MCP servers. This bypasses the global config loader issue entirely.
|
||||
|
||||
**Key Insights:**
|
||||
|
||||
1. Global config fails for localhost servers even with distinct names (`localerrors`)
|
||||
2. Project-level `.mcp.json` successfully loads the same configuration
|
||||
3. Production HTTPS servers work fine in global config
|
||||
4. Both configs can coexist: global for production, project-level for dev
|
||||
|
||||
---
|
||||
|
||||
### 2026-01-21: Environment Variable Typo (RESOLVED)
|
||||
|
||||
**Problem:**
|
||||
|
||||
- `bugsink-dev` MCP server fails to start (tools not available)
|
||||
- Production `bugsink` MCP works fine
|
||||
- User experienced repeated troubleshooting loops without resolution
|
||||
|
||||
**Root Cause:**
|
||||
Environment variable name mismatch:
|
||||
|
||||
- **Package expects:** `BUGSINK_TOKEN`
|
||||
- **Configuration had:** `BUGSINK_API_TOKEN`
|
||||
|
||||
**Discovery Method:**
|
||||
|
||||
- infra-architect subagent examined `d:\gitea\bugsink-mcp\README.md` line 47
|
||||
- Found correct variable name in package documentation
|
||||
- Production MCP continued working because it was started before config change
|
||||
|
||||
**Solution Applied:**
|
||||
|
||||
1. Updated `C:\Users\games3\.claude\settings.json`:
|
||||
- Changed `BUGSINK_API_TOKEN` to `BUGSINK_TOKEN` for both `bugsink` and `bugsink-dev`
|
||||
- Removed unused `BUGSINK_ORG_SLUG` environment variable
|
||||
|
||||
2. Updated documentation:
|
||||
- `CLAUDE.md` MCP Servers section (lines 307-327)
|
||||
- `docs/BUGSINK-SYNC.md` environment variables (line 66, 108)
|
||||
|
||||
3. Required action:
|
||||
- Restart Claude Code to reload MCP servers
|
||||
|
||||
**Status:** Fixed - but superseded by server name collision issue above
|
||||
|
||||
---
|
||||
|
||||
## Correct Configuration
|
||||
|
||||
### Required Environment Variables
|
||||
|
||||
The bugsink-mcp package requires exactly **TWO** environment variables:
|
||||
|
||||
```json
|
||||
{
|
||||
"bugsink": {
|
||||
"command": "node",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "https://bugsink.projectium.com",
|
||||
"BUGSINK_TOKEN": "<40-char-hex-token>"
|
||||
}
|
||||
},
|
||||
"localerrors": {
|
||||
"command": "node",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "http://127.0.0.1:8000",
|
||||
"BUGSINK_TOKEN": "<40-char-hex-token>"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Important:**
|
||||
|
||||
- Variable is `BUGSINK_TOKEN`, NOT `BUGSINK_API_TOKEN`
|
||||
- `BUGSINK_ORG_SLUG` is NOT used by the package
|
||||
- Works with both HTTPS (production) and HTTP (localhost)
|
||||
- Use completely distinct name like `localerrors` (not `bugsink-dev` or `devbugsink`) to avoid any name collision
|
||||
|
||||
---
|
||||
|
||||
## Common Issues
|
||||
|
||||
### MCP Server Tools Not Available
|
||||
|
||||
**Symptoms:**
|
||||
|
||||
- `mcp__localerrors__*` tools return "No such tool available"
|
||||
- Production `bugsink` MCP may work while `localerrors` fails
|
||||
|
||||
**Possible Causes:**
|
||||
|
||||
1. **Wrong environment variable name** (most common)
|
||||
- Check: Variable must be `BUGSINK_TOKEN`, not `BUGSINK_API_TOKEN`
|
||||
|
||||
2. **Invalid API token**
|
||||
- Check: Token must be 40-character lowercase hex
|
||||
- Verify: Token created via Django management command
|
||||
|
||||
3. **Bugsink instance not accessible**
|
||||
- Test: `curl -s -o /dev/null -w "%{http_code}" http://localhost:8000`
|
||||
- Expected: `302` (redirect) or `200`
|
||||
|
||||
4. **MCP server crashed on startup**
|
||||
- Check: Claude Code logs (if available)
|
||||
- Test manually: `BUGSINK_URL=http://localhost:8000 BUGSINK_TOKEN=<token> node d:/gitea/bugsink-mcp/dist/index.js`
|
||||
|
||||
**Solution:**
|
||||
|
||||
1. Verify correct variable names in settings.json
|
||||
2. Restart Claude Code
|
||||
3. Test connection: Use `mcp__bugsink__test_connection` tool
|
||||
|
||||
---
|
||||
|
||||
## Testing MCP Server
|
||||
|
||||
### Manual Test
|
||||
|
||||
Run the MCP server directly to see error output:
|
||||
|
||||
```bash
|
||||
# For localhost Bugsink
|
||||
cd d:\gitea\bugsink-mcp
|
||||
set BUGSINK_URL=http://localhost:8000
|
||||
set BUGSINK_TOKEN=a609c2886daa4e1e05f1517074d7779a5fb49056
|
||||
node dist/index.js
|
||||
```
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
Bugsink MCP server started
|
||||
Connected to: http://localhost:8000
|
||||
```
|
||||
|
||||
### Test in Claude Code
|
||||
|
||||
After restart, verify both MCP servers work:
|
||||
|
||||
```typescript
|
||||
// Production Bugsink
|
||||
mcp__bugsink__test_connection();
|
||||
// Expected: "Connection successful: Connected successfully. Found N project(s)."
|
||||
|
||||
// Dev Container Bugsink
|
||||
mcp__localerrors__test_connection();
|
||||
// Expected: "Connection successful: Connected successfully. Found N project(s)."
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Creating Bugsink API Tokens
|
||||
|
||||
Bugsink 2.0.11 does NOT have a "Settings > API Keys" menu in the UI. Tokens must be created via Django management command.
|
||||
|
||||
### For Dev Container (localhost:8000)
|
||||
|
||||
```bash
|
||||
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
|
||||
```
|
||||
|
||||
### For Production (bugsink.projectium.com via SSH)
|
||||
|
||||
```bash
|
||||
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
|
||||
```
|
||||
|
||||
Both commands output a 40-character hex token.
|
||||
|
||||
---
|
||||
|
||||
## Package Information
|
||||
|
||||
- **Repository:** https://github.com/j-shelfwood/bugsink-mcp.git
|
||||
- **Local Installation:** `d:\gitea\bugsink-mcp`
|
||||
- **Build Command:** `npm install && npm run build`
|
||||
- **Main File:** `dist/index.js`
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [CLAUDE.md MCP Servers Section](../CLAUDE.md#mcp-servers)
|
||||
- [DEV-CONTAINER-BUGSINK.md](./DEV-CONTAINER-BUGSINK.md)
|
||||
- [BUGSINK-SYNC.md](./BUGSINK-SYNC.md)
|
||||
|
||||
---
|
||||
|
||||
## Failed Solutions (Do Not Retry)
|
||||
|
||||
These approaches were tried and did NOT work:
|
||||
|
||||
1. ❌ Regenerating API token multiple times
|
||||
2. ❌ Restarting Claude Code without config changes
|
||||
3. ❌ Checking Bugsink instance accessibility (was already working)
|
||||
4. ❌ Adding `BUGSINK_ORG_SLUG` environment variable (not used by package)
|
||||
|
||||
**Lesson:** Always verify actual package requirements in source code/README before troubleshooting.
|
||||
@@ -63,7 +63,7 @@ Add these to **test environment only** (`deploy-to-test.yml`):
|
||||
```bash
|
||||
# Bugsink API
|
||||
BUGSINK_URL=https://bugsink.projectium.com
|
||||
BUGSINK_API_TOKEN=<from Bugsink Settings > API Keys>
|
||||
BUGSINK_TOKEN=<create via Django management command - see below>
|
||||
|
||||
# Gitea API
|
||||
GITEA_URL=https://gitea.projectium.com
|
||||
@@ -76,15 +76,38 @@ BUGSINK_SYNC_ENABLED=true # Only set true in test env
|
||||
BUGSINK_SYNC_INTERVAL=15 # Minutes between sync runs
|
||||
```
|
||||
|
||||
## Creating Bugsink API Token
|
||||
|
||||
Bugsink 2.0.11 does not have a "Settings > API Keys" UI. Create API tokens via Django management command:
|
||||
|
||||
**On Production Server:**
|
||||
|
||||
```bash
|
||||
sudo su - bugsink
|
||||
source venv/bin/activate
|
||||
cd ~
|
||||
bugsink-manage shell -c "
|
||||
from django.contrib.auth import get_user_model
|
||||
from rest_framework.authtoken.models import Token
|
||||
User = get_user_model()
|
||||
user = User.objects.get(email='admin@yourdomain.com') # Use your admin email
|
||||
token, created = Token.objects.get_or_create(user=user)
|
||||
print(f'Token: {token.key}')
|
||||
"
|
||||
exit
|
||||
```
|
||||
|
||||
This will output a 40-character lowercase hex token.
|
||||
|
||||
## Gitea Secrets to Add
|
||||
|
||||
Add these secrets in Gitea repository settings (Settings > Secrets):
|
||||
|
||||
| Secret Name | Value | Environment |
|
||||
| ---------------------- | ---------------------- | ----------- |
|
||||
| `BUGSINK_API_TOKEN` | API token from Bugsink | Test only |
|
||||
| `GITEA_SYNC_TOKEN` | Personal access token | Test only |
|
||||
| `BUGSINK_SYNC_ENABLED` | `true` | Test only |
|
||||
| Secret Name | Value | Environment |
|
||||
| ---------------------- | ------------------------ | ----------- |
|
||||
| `BUGSINK_TOKEN` | Token from command above | Test only |
|
||||
| `GITEA_SYNC_TOKEN` | Personal access token | Test only |
|
||||
| `BUGSINK_SYNC_ENABLED` | `true` | Test only |
|
||||
|
||||
## Redis Configuration
|
||||
|
||||
|
||||
100
docs/DEV-CONTAINER-BUGSINK.md
Normal file
100
docs/DEV-CONTAINER-BUGSINK.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# Dev Container Bugsink Setup
|
||||
|
||||
Local Bugsink instance for development - NOT connected to production.
|
||||
|
||||
## Quick Reference
|
||||
|
||||
| Item | Value |
|
||||
| ------------ | ----------------------------------------------------------- |
|
||||
| UI | `https://localhost:8443` (nginx proxy from 8000) |
|
||||
| Credentials | `admin@localhost` / `admin` |
|
||||
| Projects | Backend (Dev) = Project ID 1, Frontend (Dev) = Project ID 2 |
|
||||
| Backend DSN | `SENTRY_DSN=http://<key>@localhost:8000/1` |
|
||||
| Frontend DSN | `VITE_SENTRY_DSN=http://<key>@localhost:8000/2` |
|
||||
|
||||
## Configuration Files
|
||||
|
||||
| File | Purpose |
|
||||
| ----------------- | ----------------------------------------------------------------- |
|
||||
| `compose.dev.yml` | Initial DSNs using `127.0.0.1:8000` (container startup) |
|
||||
| `.env.local` | **OVERRIDES** compose.dev.yml with `localhost:8000` (app runtime) |
|
||||
|
||||
**CRITICAL**: `.env.local` takes precedence over `compose.dev.yml` environment variables.
|
||||
|
||||
## Why localhost vs 127.0.0.1?
|
||||
|
||||
The `.env.local` file uses `localhost` while `compose.dev.yml` uses `127.0.0.1`. Both work in practice - `localhost` was chosen when `.env.local` was created separately.
|
||||
|
||||
## HTTPS Setup
|
||||
|
||||
- Self-signed certificates auto-generated with mkcert on container startup
|
||||
- CSRF Protection: Django configured with `CSRF_TRUSTED_ORIGINS` for both `localhost` and `127.0.0.1` (see below)
|
||||
- HTTPS proxy: nginx on port 8443 proxies to Bugsink on port 8000
|
||||
- HTTPS is for UI access only - Sentry SDK uses HTTP directly
|
||||
|
||||
### CSRF Configuration
|
||||
|
||||
Django 4.0+ requires `CSRF_TRUSTED_ORIGINS` for HTTPS POST requests. The Bugsink configuration (`Dockerfile.dev`) includes:
|
||||
|
||||
```python
|
||||
CSRF_TRUSTED_ORIGINS = [
|
||||
"https://localhost:8443",
|
||||
"https://127.0.0.1:8443",
|
||||
"http://localhost:8000",
|
||||
"http://127.0.0.1:8000",
|
||||
]
|
||||
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
|
||||
```
|
||||
|
||||
**Both hostnames are required** because browsers treat `localhost` and `127.0.0.1` as different origins.
|
||||
|
||||
If you get "CSRF verification failed" errors, see [BUGSINK-SETUP.md](tools/BUGSINK-SETUP.md#csrf-verification-failed) for troubleshooting.
|
||||
|
||||
## Isolation Benefits
|
||||
|
||||
- Dev errors stay local, don't pollute production/test dashboards
|
||||
- No Gitea secrets needed - everything self-contained
|
||||
- Independent testing of error tracking without affecting metrics
|
||||
|
||||
## Accessing Errors
|
||||
|
||||
### Via Browser
|
||||
|
||||
1. Open `https://localhost:8443`
|
||||
2. Login with credentials above
|
||||
3. Navigate to Issues to view captured errors
|
||||
|
||||
### Via MCP (bugsink-dev)
|
||||
|
||||
Configure in `.claude/mcp.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"bugsink-dev": {
|
||||
"command": "node",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "http://localhost:8000",
|
||||
"BUGSINK_API_TOKEN": "<token-from-local-bugsink>",
|
||||
"BUGSINK_ORG_SLUG": "sentry"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Get auth token**:
|
||||
|
||||
API tokens must be created via Django management command (Bugsink 2.0.11 does not have a "Settings > API Keys" UI):
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && \
|
||||
DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink \
|
||||
SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security \
|
||||
DJANGO_SETTINGS_MODULE=bugsink_conf \
|
||||
PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages \
|
||||
/opt/bugsink/bin/python -m django create_auth_token'
|
||||
```
|
||||
|
||||
This will output a 40-character lowercase hex token. Copy it to your MCP configuration.
|
||||
|
||||
**MCP Tools**: Use `mcp__bugsink-dev__*` tools (not `mcp__bugsink__*` which connects to production).
|
||||
264
docs/FLYER-URL-CONFIGURATION.md
Normal file
264
docs/FLYER-URL-CONFIGURATION.md
Normal file
@@ -0,0 +1,264 @@
|
||||
# Flyer URL Configuration
|
||||
|
||||
## Overview
|
||||
|
||||
Flyer image and icon URLs are environment-specific to ensure they point to the correct server for each deployment. Images are served as static files by NGINX from the `/flyer-images/` path with 7-day browser caching enabled.
|
||||
|
||||
## Environment-Specific URLs
|
||||
|
||||
| Environment | Base URL | Example |
|
||||
| ------------- | ------------------------------------------- | -------------------------------------------------------------------------- |
|
||||
| Dev Container | `https://127.0.0.1` | `https://127.0.0.1/flyer-images/safeway-flyer.jpg` |
|
||||
| Test | `https://flyer-crawler-test.projectium.com` | `https://flyer-crawler-test.projectium.com/flyer-images/safeway-flyer.jpg` |
|
||||
| Production | `https://flyer-crawler.projectium.com` | `https://flyer-crawler.projectium.com/flyer-images/safeway-flyer.jpg` |
|
||||
|
||||
**Note:** The dev container accepts connections to **both** `https://localhost/` and `https://127.0.0.1/` thanks to the SSL certificate and NGINX configuration. See [SSL Certificate Configuration](#ssl-certificate-configuration-dev-container) below.
|
||||
|
||||
## SSL Certificate Configuration (Dev Container)
|
||||
|
||||
The dev container uses self-signed certificates generated by [mkcert](https://github.com/FiloSottile/mkcert) to enable HTTPS locally. This configuration solves a common mixed-origin SSL issue.
|
||||
|
||||
### The Problem
|
||||
|
||||
When users access the site via `https://localhost/` but image URLs in the database use `https://127.0.0.1/...`, browsers treat these as different origins. This causes `ERR_CERT_AUTHORITY_INVALID` errors when loading images, even though both hostnames point to the same server.
|
||||
|
||||
### The Solution
|
||||
|
||||
1. **Certificate Generation** (`Dockerfile.dev`):
|
||||
|
||||
```bash
|
||||
mkcert localhost 127.0.0.1 ::1
|
||||
```
|
||||
|
||||
This creates a certificate with Subject Alternative Names (SANs) for all three hostnames.
|
||||
|
||||
2. **NGINX Configuration** (`docker/nginx/dev.conf`):
|
||||
|
||||
```nginx
|
||||
server_name localhost 127.0.0.1;
|
||||
```
|
||||
|
||||
NGINX accepts requests to both hostnames using the same SSL certificate.
|
||||
|
||||
### How It Works
|
||||
|
||||
| Component | Configuration |
|
||||
| -------------------- | ---------------------------------------------------- |
|
||||
| SSL Certificate SANs | `localhost`, `127.0.0.1`, `::1` |
|
||||
| NGINX `server_name` | `localhost 127.0.0.1` |
|
||||
| Seed Script URLs | Uses `https://127.0.0.1` (works with DB constraints) |
|
||||
| User Access | Either `https://localhost/` or `https://127.0.0.1/` |
|
||||
|
||||
### Why This Matters
|
||||
|
||||
- **Database Constraints**: The `flyers` table has CHECK constraints requiring URLs to start with `http://` or `https://`. Relative URLs are not allowed.
|
||||
- **Consistent Behavior**: Users can access the site using either hostname without SSL warnings.
|
||||
- **Same Certificate**: Both hostnames use the same self-signed certificate, eliminating mixed-content errors.
|
||||
|
||||
### Verifying the Configuration
|
||||
|
||||
```bash
|
||||
# Check certificate SANs
|
||||
podman exec flyer-crawler-dev openssl x509 -in /app/certs/localhost.crt -text -noout | grep -A1 "Subject Alternative Name"
|
||||
|
||||
# Expected output:
|
||||
# X509v3 Subject Alternative Name:
|
||||
# DNS:localhost, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1
|
||||
|
||||
# Test both hostnames respond
|
||||
curl -k https://localhost/health
|
||||
curl -k https://127.0.0.1/health
|
||||
```
|
||||
|
||||
### Troubleshooting SSL Issues
|
||||
|
||||
If you encounter `ERR_CERT_AUTHORITY_INVALID`:
|
||||
|
||||
1. **Check NGINX is running**: `podman exec flyer-crawler-dev nginx -t`
|
||||
2. **Verify certificate exists**: `podman exec flyer-crawler-dev ls -la /app/certs/`
|
||||
3. **Ensure both hostnames are in server_name**: Check `/etc/nginx/sites-available/default`
|
||||
4. **Rebuild container if needed**: The certificate is generated at build time
|
||||
|
||||
### Permanent Fix: Install CA Certificate (Recommended)
|
||||
|
||||
To permanently eliminate SSL certificate warnings, install the mkcert CA certificate on your system. This is optional but provides a better development experience.
|
||||
|
||||
The CA certificate is located at `certs/mkcert-ca.crt` in the project root. See [`certs/README.md`](../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
|
||||
|
||||
After installation:
|
||||
|
||||
- Your browser will trust all mkcert certificates without warnings
|
||||
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
|
||||
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
|
||||
|
||||
See also: [Debugging Guide - SSL Issues](development/DEBUGGING.md#ssl-certificate-issues)
|
||||
|
||||
## NGINX Static File Serving
|
||||
|
||||
All environments serve flyer images as static files with browser caching:
|
||||
|
||||
```nginx
|
||||
# Serve flyer images from static storage (7-day cache)
|
||||
location /flyer-images/ {
|
||||
alias /path/to/flyer-images/;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
```
|
||||
|
||||
### Directory Paths by Environment
|
||||
|
||||
| Environment | NGINX Alias Path |
|
||||
| ------------- | ---------------------------------------------------------- |
|
||||
| Dev Container | `/app/public/flyer-images/` |
|
||||
| Test | `/var/www/flyer-crawler-test.projectium.com/flyer-images/` |
|
||||
| Production | `/var/www/flyer-crawler.projectium.com/flyer-images/` |
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variable
|
||||
|
||||
Set `FLYER_BASE_URL` in your environment configuration:
|
||||
|
||||
```bash
|
||||
# Dev container (.env)
|
||||
FLYER_BASE_URL=https://localhost
|
||||
|
||||
# Test environment
|
||||
FLYER_BASE_URL=https://flyer-crawler-test.projectium.com
|
||||
|
||||
# Production
|
||||
FLYER_BASE_URL=https://flyer-crawler.projectium.com
|
||||
```
|
||||
|
||||
### Seed Script
|
||||
|
||||
The seed script ([src/db/seed.ts](../src/db/seed.ts)) automatically uses the correct base URL based on:
|
||||
|
||||
1. `FLYER_BASE_URL` environment variable (if set)
|
||||
2. `NODE_ENV` value:
|
||||
- `production` → `https://flyer-crawler.projectium.com`
|
||||
- `test` → `https://flyer-crawler-test.projectium.com`
|
||||
- Default → `https://localhost`
|
||||
|
||||
The seed script also copies test images from `src/tests/assets/` to `public/flyer-images/`:
|
||||
|
||||
- `test-flyer-image.jpg` - Sample flyer image
|
||||
- `test-flyer-icon.png` - Sample 64x64 icon
|
||||
|
||||
## Updating Existing Data
|
||||
|
||||
If you need to update existing flyer URLs in the database, use the provided SQL script:
|
||||
|
||||
### Dev Container
|
||||
|
||||
```bash
|
||||
# Connect to dev database
|
||||
podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev
|
||||
|
||||
# Run the update (dev container uses HTTPS with self-signed certs)
|
||||
UPDATE flyers
|
||||
SET
|
||||
image_url = REPLACE(image_url, 'example.com', 'localhost'),
|
||||
icon_url = REPLACE(icon_url, 'example.com', 'localhost')
|
||||
WHERE
|
||||
image_url LIKE '%example.com%'
|
||||
OR icon_url LIKE '%example.com%';
|
||||
|
||||
# Verify
|
||||
SELECT flyer_id, image_url, icon_url FROM flyers;
|
||||
```
|
||||
|
||||
### Test Environment
|
||||
|
||||
```bash
|
||||
# Via SSH
|
||||
ssh root@projectium.com "psql -U flyer_crawler_test -d flyer-crawler-test -c \"
|
||||
UPDATE flyers
|
||||
SET
|
||||
image_url = REPLACE(image_url, 'example.com', 'flyer-crawler-test.projectium.com'),
|
||||
icon_url = REPLACE(icon_url, 'example.com', 'flyer-crawler-test.projectium.com')
|
||||
WHERE
|
||||
image_url LIKE '%example.com%'
|
||||
OR icon_url LIKE '%example.com%';
|
||||
\""
|
||||
```
|
||||
|
||||
### Production
|
||||
|
||||
```bash
|
||||
# Via SSH
|
||||
ssh root@projectium.com "psql -U flyer_crawler_prod -d flyer-crawler-prod -c \"
|
||||
UPDATE flyers
|
||||
SET
|
||||
image_url = REPLACE(image_url, 'example.com', 'flyer-crawler.projectium.com'),
|
||||
icon_url = REPLACE(icon_url, 'example.com', 'flyer-crawler.projectium.com')
|
||||
WHERE
|
||||
image_url LIKE '%example.com%'
|
||||
OR icon_url LIKE '%example.com%';
|
||||
\""
|
||||
```
|
||||
|
||||
## Test Data Updates
|
||||
|
||||
### Test Helper Function
|
||||
|
||||
A helper function `getFlyerBaseUrl()` is available in [src/tests/utils/testHelpers.ts](../src/tests/utils/testHelpers.ts) that automatically detects the correct base URL for tests:
|
||||
|
||||
```typescript
|
||||
export const getFlyerBaseUrl = (): string => {
|
||||
if (process.env.FLYER_BASE_URL) {
|
||||
return process.env.FLYER_BASE_URL;
|
||||
}
|
||||
|
||||
// Check if we're in dev container (DB_HOST=postgres is typical indicator)
|
||||
// Use 'localhost' instead of '127.0.0.1' to match the hostname users access
|
||||
// This avoids SSL certificate mixed-origin issues in browsers
|
||||
if (process.env.DB_HOST === 'postgres' || process.env.DB_HOST === '127.0.0.1') {
|
||||
return 'https://localhost';
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
return 'https://flyer-crawler.projectium.com';
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
return 'https://flyer-crawler-test.projectium.com';
|
||||
}
|
||||
|
||||
// Default for unit tests
|
||||
return 'https://example.com';
|
||||
};
|
||||
```
|
||||
|
||||
### Updated Test Files
|
||||
|
||||
The following test files now use `getFlyerBaseUrl()` for environment-aware URL generation:
|
||||
|
||||
- [src/db/seed.ts](../src/db/seed.ts) - Main seed script (uses `FLYER_BASE_URL`)
|
||||
- [src/tests/utils/testHelpers.ts](../src/tests/utils/testHelpers.ts) - `getFlyerBaseUrl()` helper function
|
||||
- [src/hooks/useDataExtraction.test.ts](../src/hooks/useDataExtraction.test.ts) - Mock flyer factory
|
||||
- [src/schemas/flyer.schemas.test.ts](../src/schemas/flyer.schemas.test.ts) - Schema validation tests
|
||||
- [src/services/flyerProcessingService.server.test.ts](../src/services/flyerProcessingService.server.test.ts) - Processing service tests
|
||||
- [src/tests/integration/flyer-processing.integration.test.ts](../src/tests/integration/flyer-processing.integration.test.ts) - Integration tests
|
||||
|
||||
This approach ensures tests work correctly in all environments (dev container, CI/CD, local development, test, production).
|
||||
|
||||
## Files Changed
|
||||
|
||||
| File | Change |
|
||||
| --------------------------- | ------------------------------------------------------------------------------------------------- |
|
||||
| `src/db/seed.ts` | Added `FLYER_BASE_URL` environment variable support, copies test images to `public/flyer-images/` |
|
||||
| `docker/nginx/dev.conf` | Added `/flyer-images/` location block for static file serving |
|
||||
| `.env.example` | Added `FLYER_BASE_URL` variable |
|
||||
| `sql/update_flyer_urls.sql` | SQL script for updating existing data |
|
||||
| Test files | Updated mock data to use `https://localhost` |
|
||||
|
||||
## Summary
|
||||
|
||||
- Seed script now uses environment-specific HTTPS URLs
|
||||
- Seed script copies test images from `src/tests/assets/` to `public/flyer-images/`
|
||||
- NGINX serves `/flyer-images/` as static files with 7-day cache
|
||||
- Test files updated with `https://localhost` (not `127.0.0.1` to avoid SSL mixed-origin issues)
|
||||
- SQL script provided for updating existing data
|
||||
- Documentation updated for each environment
|
||||
318
docs/POSTGRES-MCP-SETUP.md
Normal file
318
docs/POSTGRES-MCP-SETUP.md
Normal file
@@ -0,0 +1,318 @@
|
||||
# PostgreSQL MCP Server Setup
|
||||
|
||||
This document describes the configuration and troubleshooting for the PostgreSQL MCP server integration with Claude Code.
|
||||
|
||||
## Status
|
||||
|
||||
✅ **WORKING** - Successfully configured and tested on 2026-01-22
|
||||
|
||||
- **Server Name**: `devdb`
|
||||
- **Database**: `flyer_crawler_dev` (68 tables)
|
||||
- **Connection**: Verified working
|
||||
- **Tool Prefix**: `mcp__devdb__*`
|
||||
- **Configuration**: Project-level `.mcp.json`
|
||||
|
||||
## Overview
|
||||
|
||||
The PostgreSQL MCP server (`@modelcontextprotocol/server-postgres`) provides database query capabilities directly from Claude Code, enabling:
|
||||
|
||||
- Running SQL queries against the development database
|
||||
- Exploring database schema and tables
|
||||
- Testing queries before implementation
|
||||
- Debugging data issues
|
||||
|
||||
## Configuration
|
||||
|
||||
### Project-Level Configuration (Recommended)
|
||||
|
||||
The PostgreSQL MCP server is configured in the project-level `.mcp.json` file:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"devdb": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": [
|
||||
"-y",
|
||||
"@modelcontextprotocol/server-postgres",
|
||||
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Key Configuration Details:**
|
||||
|
||||
| Parameter | Value | Notes |
|
||||
| ----------- | --------------------------------------- | --------------------------------------- |
|
||||
| Server Name | `devdb` | Distinct name to avoid collision issues |
|
||||
| Package | `@modelcontextprotocol/server-postgres` | Official MCP PostgreSQL server |
|
||||
| Host | `127.0.0.1` | Use IP address, not `localhost` |
|
||||
| Port | `5432` | Default PostgreSQL port |
|
||||
| Database | `flyer_crawler_dev` | Development database name |
|
||||
| User | `postgres` | Default superuser for dev |
|
||||
| Password | `postgres` | Default password for dev |
|
||||
|
||||
### Why Project-Level Configuration?
|
||||
|
||||
Based on troubleshooting experience with other MCP servers (documented in `BUGSINK-MCP-TROUBLESHOOTING.md`), **localhost MCP servers work more reliably in project-level `.mcp.json`** than in global `settings.json`.
|
||||
|
||||
Issues observed with global configuration:
|
||||
|
||||
- MCP servers silently not loading
|
||||
- No error messages in logs
|
||||
- Tools not appearing in available tool list
|
||||
|
||||
Project-level configuration bypasses these issues entirely.
|
||||
|
||||
### Connection String Format
|
||||
|
||||
```
|
||||
postgresql://[user]:[password]@[host]:[port]/[database]
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
# Development (local container)
|
||||
postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev
|
||||
|
||||
# Test database (if needed)
|
||||
postgresql://flyer_crawler_test:password@127.0.0.1:5432/flyer_crawler_test
|
||||
```
|
||||
|
||||
## Available Tools
|
||||
|
||||
Once configured, the following tools become available (prefix `mcp__devdb__`):
|
||||
|
||||
| Tool | Description |
|
||||
| ------- | -------------------------------------- |
|
||||
| `query` | Execute SQL queries and return results |
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Query
|
||||
|
||||
```typescript
|
||||
// List all tables
|
||||
mcp__devdb__query("SELECT tablename FROM pg_tables WHERE schemaname = 'public'");
|
||||
|
||||
// Count records in a table
|
||||
mcp__devdb__query('SELECT COUNT(*) FROM flyers');
|
||||
|
||||
// Check table structure
|
||||
mcp__devdb__query(
|
||||
"SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'flyers'",
|
||||
);
|
||||
```
|
||||
|
||||
### Debugging Data Issues
|
||||
|
||||
```typescript
|
||||
// Find recent flyers
|
||||
mcp__devdb__query('SELECT id, name, created_at FROM flyers ORDER BY created_at DESC LIMIT 10');
|
||||
|
||||
// Check job queue status
|
||||
mcp__devdb__query('SELECT state, COUNT(*) FROM bullmq_jobs GROUP BY state');
|
||||
|
||||
// Verify user data
|
||||
mcp__devdb__query("SELECT id, email, created_at FROM users WHERE email LIKE '%test%'");
|
||||
```
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### 1. PostgreSQL Container Running
|
||||
|
||||
The PostgreSQL container must be running and healthy:
|
||||
|
||||
```bash
|
||||
# Check container status
|
||||
podman ps | grep flyer-crawler-postgres
|
||||
|
||||
# Expected output shows "healthy" status
|
||||
# flyer-crawler-postgres ... Up N hours (healthy) ...
|
||||
```
|
||||
|
||||
### 2. Port Accessible from Host
|
||||
|
||||
PostgreSQL port 5432 must be mapped to the host:
|
||||
|
||||
```bash
|
||||
# Verify port mapping
|
||||
podman port flyer-crawler-postgres
|
||||
# Expected: 5432/tcp -> 0.0.0.0:5432
|
||||
```
|
||||
|
||||
### 3. Database Exists
|
||||
|
||||
Verify the database exists:
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-postgres psql -U postgres -c "\l" | grep flyer_crawler_dev
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Tools Not Available
|
||||
|
||||
**Symptoms:**
|
||||
|
||||
- `mcp__devdb__*` tools not in available tool list
|
||||
- No error messages displayed
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. **Restart Claude Code** - MCP config changes require restart
|
||||
2. **Check container status** - Ensure PostgreSQL container is running
|
||||
3. **Verify port mapping** - Confirm port 5432 is accessible
|
||||
4. **Test connection manually**:
|
||||
```bash
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1"
|
||||
```
|
||||
|
||||
### Connection Refused
|
||||
|
||||
**Symptoms:**
|
||||
|
||||
- Connection error when using tools
|
||||
- "Connection refused" in error message
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. **Check container health**:
|
||||
|
||||
```bash
|
||||
podman ps | grep flyer-crawler-postgres
|
||||
```
|
||||
|
||||
2. **Restart the container**:
|
||||
|
||||
```bash
|
||||
podman restart flyer-crawler-postgres
|
||||
```
|
||||
|
||||
3. **Check for port conflicts**:
|
||||
```bash
|
||||
netstat -an | findstr 5432
|
||||
```
|
||||
|
||||
### Authentication Failed
|
||||
|
||||
**Symptoms:**
|
||||
|
||||
- "password authentication failed" error
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. **Verify credentials** in container environment:
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-dev env | grep DB_
|
||||
```
|
||||
|
||||
2. **Check PostgreSQL users**:
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-postgres psql -U postgres -c "\du"
|
||||
```
|
||||
|
||||
3. **Update connection string** in `.mcp.json` if credentials differ
|
||||
|
||||
### Database Does Not Exist
|
||||
|
||||
**Symptoms:**
|
||||
|
||||
- "database does not exist" error
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. **List available databases**:
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-postgres psql -U postgres -c "\l"
|
||||
```
|
||||
|
||||
2. **Create database if missing**:
|
||||
```bash
|
||||
podman exec flyer-crawler-postgres createdb -U postgres flyer_crawler_dev
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Development Only
|
||||
|
||||
The default credentials (`postgres:postgres`) are for **development only**. Never use these in production.
|
||||
|
||||
### Connection String in Config
|
||||
|
||||
The connection string includes the password in plain text. This is acceptable for:
|
||||
|
||||
- Local development
|
||||
- Container environments
|
||||
|
||||
For production MCP access (if ever needed):
|
||||
|
||||
- Use environment variables
|
||||
- Consider connection pooling
|
||||
- Implement proper access controls
|
||||
|
||||
### Query Permissions
|
||||
|
||||
The MCP server executes queries as the configured user (`postgres` in dev). Be aware that:
|
||||
|
||||
- `postgres` is a superuser with full access
|
||||
- For restricted access, create a dedicated MCP user with limited permissions:
|
||||
|
||||
```sql
|
||||
-- Example: Create read-only MCP user
|
||||
CREATE USER mcp_reader WITH PASSWORD 'secure_password';
|
||||
GRANT CONNECT ON DATABASE flyer_crawler_dev TO mcp_reader;
|
||||
GRANT USAGE ON SCHEMA public TO mcp_reader;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA public TO mcp_reader;
|
||||
```
|
||||
|
||||
## Database Information
|
||||
|
||||
### Development Environment
|
||||
|
||||
| Property | Value |
|
||||
| --------------------- | ------------------------- |
|
||||
| Container | `flyer-crawler-postgres` |
|
||||
| Image | `postgis/postgis:15-3.4` |
|
||||
| Host (from Windows) | `127.0.0.1` / `localhost` |
|
||||
| Host (from container) | `postgres` |
|
||||
| Port | `5432` |
|
||||
| Database | `flyer_crawler_dev` |
|
||||
| User | `postgres` |
|
||||
| Password | `postgres` |
|
||||
|
||||
### Schema Reference
|
||||
|
||||
The database uses PostGIS for geographic data. Key tables include:
|
||||
|
||||
- `users` - User accounts
|
||||
- `stores` - Store definitions
|
||||
- `store_locations` - Store geographic locations
|
||||
- `flyers` - Uploaded flyer metadata
|
||||
- `flyer_items` - Extracted deal items
|
||||
- `watchlists` - User watchlists
|
||||
- `shopping_lists` - User shopping lists
|
||||
- `recipes` - Recipe definitions
|
||||
|
||||
For complete schema, see `sql/master_schema_rollup.sql`.
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [CLAUDE.md - MCP Servers Section](../CLAUDE.md#mcp-servers)
|
||||
- [BUGSINK-MCP-TROUBLESHOOTING.md](./BUGSINK-MCP-TROUBLESHOOTING.md) - Similar MCP setup patterns
|
||||
- [sql/master_schema_rollup.sql](../sql/master_schema_rollup.sql) - Database schema
|
||||
|
||||
## Changelog
|
||||
|
||||
### 2026-01-21
|
||||
|
||||
- Initial configuration added to project-level `.mcp.json`
|
||||
- Server named `devdb` to avoid naming collisions
|
||||
- Using `127.0.0.1` instead of `localhost` based on Bugsink MCP experience
|
||||
- Documentation created
|
||||
130
docs/README.md
Normal file
130
docs/README.md
Normal file
@@ -0,0 +1,130 @@
|
||||
# Flyer Crawler Documentation
|
||||
|
||||
Welcome to the Flyer Crawler documentation. This guide will help you navigate the various documentation resources available.
|
||||
|
||||
## Quick Links
|
||||
|
||||
- [Main README](../README.md) - Project overview and quick start
|
||||
- [CLAUDE.md](../CLAUDE.md) - AI agent instructions and project guidelines
|
||||
- [CONTRIBUTING.md](../CONTRIBUTING.md) - Development workflow and contribution guide
|
||||
|
||||
## Documentation Structure
|
||||
|
||||
### 🚀 Getting Started
|
||||
|
||||
New to the project? Start here:
|
||||
|
||||
- [Installation Guide](getting-started/INSTALL.md) - Complete setup instructions
|
||||
- [Environment Configuration](getting-started/ENVIRONMENT.md) - Environment variables and secrets
|
||||
|
||||
### 🏗️ Architecture
|
||||
|
||||
Understand how the system works:
|
||||
|
||||
- [System Overview](architecture/OVERVIEW.md) - High-level architecture
|
||||
- [Database Schema](architecture/DATABASE.md) - Database design and entities
|
||||
- [Authentication](architecture/AUTHENTICATION.md) - OAuth and JWT authentication
|
||||
- [WebSocket Usage](architecture/WEBSOCKET_USAGE.md) - Real-time communication patterns
|
||||
|
||||
### 💻 Development
|
||||
|
||||
Day-to-day development guides:
|
||||
|
||||
- [Testing Guide](development/TESTING.md) - Unit, integration, and E2E testing
|
||||
- [Code Patterns](development/CODE-PATTERNS.md) - Common code patterns and ADR examples
|
||||
- [Design Tokens](development/DESIGN_TOKENS.md) - UI design system and Neo-Brutalism
|
||||
- [Debugging Guide](development/DEBUGGING.md) - Common debugging patterns
|
||||
|
||||
### 🔧 Operations
|
||||
|
||||
Production operations and deployment:
|
||||
|
||||
- [Deployment Guide](operations/DEPLOYMENT.md) - Deployment procedures
|
||||
- [Bare Metal Setup](operations/BARE-METAL-SETUP.md) - Server provisioning
|
||||
- [Logstash Quick Reference](operations/LOGSTASH-QUICK-REF.md) - Log aggregation
|
||||
- [Logstash Troubleshooting](operations/LOGSTASH-TROUBLESHOOTING.md) - Debugging logs
|
||||
- [Monitoring](operations/MONITORING.md) - Bugsink, health checks, observability
|
||||
|
||||
**NGINX Reference Configs** (in repository root):
|
||||
|
||||
- `etc-nginx-sites-available-flyer-crawler.projectium.com` - Production server config
|
||||
- `etc-nginx-sites-available-flyer-crawler-test-projectium-com.txt` - Test server config
|
||||
|
||||
### 🛠️ Tools
|
||||
|
||||
External tool configuration:
|
||||
|
||||
- [MCP Configuration](tools/MCP-CONFIGURATION.md) - Model Context Protocol servers
|
||||
- [Bugsink Setup](tools/BUGSINK-SETUP.md) - Error tracking configuration
|
||||
- [VS Code Setup](tools/VSCODE-SETUP.md) - Editor configuration
|
||||
|
||||
### 🤖 AI Agents
|
||||
|
||||
Working with Claude Code subagents:
|
||||
|
||||
- [Subagent Overview](subagents/OVERVIEW.md) - Introduction to specialized agents
|
||||
- [Coder Guide](subagents/CODER-GUIDE.md) - Code development patterns
|
||||
- [Tester Guide](subagents/TESTER-GUIDE.md) - Testing strategies
|
||||
- [Database Guide](subagents/DATABASE-GUIDE.md) - Database workflows
|
||||
- [DevOps Guide](subagents/DEVOPS-GUIDE.md) - Deployment and infrastructure
|
||||
- [AI Usage Guide](subagents/AI-USAGE-GUIDE.md) - Gemini integration
|
||||
- [Frontend Guide](subagents/FRONTEND-GUIDE.md) - UI/UX development
|
||||
- [Documentation Guide](subagents/DOCUMENTATION-GUIDE.md) - Writing docs
|
||||
- [Security & Debug Guide](subagents/SECURITY-DEBUG-GUIDE.md) - Security and debugging
|
||||
|
||||
**AI-Optimized References** (token-efficient quick refs):
|
||||
|
||||
- [Coder Reference](SUBAGENT-CODER-REFERENCE.md)
|
||||
- [Tester Reference](SUBAGENT-TESTER-REFERENCE.md)
|
||||
- [DB Reference](SUBAGENT-DB-REFERENCE.md)
|
||||
- [DevOps Reference](SUBAGENT-DEVOPS-REFERENCE.md)
|
||||
- [Integrations Reference](SUBAGENT-INTEGRATIONS-REFERENCE.md)
|
||||
|
||||
### 📐 Architecture Decision Records (ADRs)
|
||||
|
||||
Design decisions and rationale:
|
||||
|
||||
- [ADR Index](adr/index.md) - Complete list of all ADRs
|
||||
- 54+ ADRs covering patterns, conventions, and technical decisions
|
||||
|
||||
### 📦 Archive
|
||||
|
||||
Historical and completed documentation:
|
||||
|
||||
- [Session Notes](archive/sessions/) - Development session logs
|
||||
- [Planning Documents](archive/plans/) - Feature plans and implementation status
|
||||
- [Research Notes](archive/research/) - Investigation and research documents
|
||||
|
||||
## Documentation Conventions
|
||||
|
||||
- **File Names**: Use SCREAMING_SNAKE_CASE for human-readable docs (e.g., `INSTALL.md`)
|
||||
- **Links**: Use relative paths from the document's location
|
||||
- **Code Blocks**: Always specify language for syntax highlighting
|
||||
- **Tables**: Use markdown tables for structured data
|
||||
- **Cross-References**: Link to ADRs and other docs for detailed explanations
|
||||
|
||||
## Contributing to Documentation
|
||||
|
||||
See [CONTRIBUTING.md](../CONTRIBUTING.md) for guidelines on:
|
||||
|
||||
- Writing clear, concise documentation
|
||||
- Updating docs when code changes
|
||||
- Creating new ADRs for significant decisions
|
||||
- Documenting new features and APIs
|
||||
|
||||
## Need Help?
|
||||
|
||||
- Check the [Testing Guide](development/TESTING.md) for test-related issues
|
||||
- See [Debugging Guide](development/DEBUGGING.md) for troubleshooting
|
||||
- Review [ADRs](adr/index.md) for architectural context
|
||||
- Consult [Subagent Guides](subagents/OVERVIEW.md) for AI agent assistance
|
||||
|
||||
## Documentation Maintenance
|
||||
|
||||
This documentation is actively maintained. If you find:
|
||||
|
||||
- Broken links or outdated information
|
||||
- Missing documentation for features
|
||||
- Unclear or confusing sections
|
||||
|
||||
Please open an issue or submit a pull request with improvements.
|
||||
265
docs/SUBAGENT-CODER-REFERENCE.md
Normal file
265
docs/SUBAGENT-CODER-REFERENCE.md
Normal file
@@ -0,0 +1,265 @@
|
||||
# Coder Subagent Reference
|
||||
|
||||
## Quick Navigation
|
||||
|
||||
| Category | Key Files |
|
||||
| ------------ | ------------------------------------------------------------------ |
|
||||
| Routes | `src/routes/*.routes.ts` |
|
||||
| Services | `src/services/*.server.ts` (backend), `src/services/*.ts` (shared) |
|
||||
| Repositories | `src/services/db/*.db.ts` |
|
||||
| Types | `src/types.ts`, `src/types/*.ts` |
|
||||
| Schemas | `src/schemas/*.schemas.ts` |
|
||||
| Config | `src/config/env.ts` |
|
||||
| Utils | `src/utils/*.ts` |
|
||||
|
||||
---
|
||||
|
||||
## Architecture Patterns (ADR Summary)
|
||||
|
||||
### Layer Flow
|
||||
|
||||
```
|
||||
Route → validateRequest(schema) → Service → Repository → Database
|
||||
↓
|
||||
External APIs
|
||||
```
|
||||
|
||||
### Repository Naming Convention (ADR-034)
|
||||
|
||||
| Prefix | Behavior | Return |
|
||||
| --------- | --------------------------------- | -------------- |
|
||||
| `get*` | Throws `NotFoundError` if missing | Entity |
|
||||
| `find*` | Returns `null` if missing | Entity \| null |
|
||||
| `list*` | Returns empty array if none | Entity[] |
|
||||
| `create*` | Creates new record | Entity |
|
||||
| `update*` | Updates existing | Entity |
|
||||
| `delete*` | Removes record | void |
|
||||
|
||||
### Error Handling (ADR-001)
|
||||
|
||||
```typescript
|
||||
import { handleDbError, NotFoundError } from '../services/db/errors.db';
|
||||
import { logger } from '../services/logger.server';
|
||||
|
||||
// Repository pattern
|
||||
async function getById(id: string): Promise<Entity> {
|
||||
try {
|
||||
const result = await pool.query('SELECT * FROM table WHERE id = $1', [id]);
|
||||
if (result.rows.length === 0) throw new NotFoundError('Entity not found');
|
||||
return result.rows[0];
|
||||
} catch (error) {
|
||||
handleDbError(error, logger, 'Failed to get entity', { id });
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### API Response Helpers (ADR-028)
|
||||
|
||||
```typescript
|
||||
import {
|
||||
sendSuccess,
|
||||
sendPaginated,
|
||||
sendError,
|
||||
sendNoContent,
|
||||
sendMessage,
|
||||
ErrorCode,
|
||||
} from '../utils/apiResponse';
|
||||
|
||||
// Success with data
|
||||
sendSuccess(res, data); // 200
|
||||
sendSuccess(res, data, 201); // 201 Created
|
||||
|
||||
// Paginated
|
||||
sendPaginated(res, items, { page, limit, total });
|
||||
|
||||
// Error
|
||||
sendError(res, ErrorCode.NOT_FOUND, 'User not found', 404);
|
||||
sendError(res, ErrorCode.VALIDATION_ERROR, 'Invalid input', 400, validationErrors);
|
||||
|
||||
// No content / Message
|
||||
sendNoContent(res); // 204
|
||||
sendMessage(res, 'Password updated');
|
||||
```
|
||||
|
||||
### Transaction Pattern (ADR-002)
|
||||
|
||||
```typescript
|
||||
import { withTransaction } from '../services/db/connection.db';
|
||||
|
||||
const result = await withTransaction(async (client) => {
|
||||
await client.query('INSERT INTO a ...');
|
||||
await client.query('INSERT INTO b ...');
|
||||
return { success: true };
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Adding New Features
|
||||
|
||||
### New API Endpoint Checklist
|
||||
|
||||
1. **Schema** (`src/schemas/{domain}.schemas.ts`)
|
||||
|
||||
```typescript
|
||||
import { z } from 'zod';
|
||||
export const createEntitySchema = z.object({
|
||||
body: z.object({ name: z.string().min(1) }),
|
||||
});
|
||||
```
|
||||
|
||||
2. **Route** (`src/routes/{domain}.routes.ts`)
|
||||
|
||||
```typescript
|
||||
import { validateRequest } from '../middleware/validation.middleware';
|
||||
import { createEntitySchema } from '../schemas/{domain}.schemas';
|
||||
|
||||
router.post('/', validateRequest(createEntitySchema), async (req, res, next) => {
|
||||
try {
|
||||
const result = await entityService.create(req.body);
|
||||
sendSuccess(res, result, 201);
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
3. **Service** (`src/services/{domain}Service.server.ts`)
|
||||
|
||||
```typescript
|
||||
export async function create(data: CreateInput): Promise<Entity> {
|
||||
// Business logic here
|
||||
return repository.create(data);
|
||||
}
|
||||
```
|
||||
|
||||
4. **Repository** (`src/services/db/{domain}.db.ts`)
|
||||
```typescript
|
||||
export async function create(data: CreateInput, client?: PoolClient): Promise<Entity> {
|
||||
const pool = client || getPool();
|
||||
try {
|
||||
const result = await pool.query('INSERT INTO ...', [data.name]);
|
||||
return result.rows[0];
|
||||
} catch (error) {
|
||||
handleDbError(error, logger, 'Failed to create entity', { data });
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### New Background Job Checklist
|
||||
|
||||
1. **Queue** (`src/services/queues.server.ts`)
|
||||
|
||||
```typescript
|
||||
export const myQueue = new Queue('my-queue', { connection: redisConnection });
|
||||
```
|
||||
|
||||
2. **Worker** (`src/services/workers.server.ts`)
|
||||
|
||||
```typescript
|
||||
new Worker(
|
||||
'my-queue',
|
||||
async (job) => {
|
||||
// Process job
|
||||
},
|
||||
{ connection: redisConnection },
|
||||
);
|
||||
```
|
||||
|
||||
3. **Trigger** (in service)
|
||||
```typescript
|
||||
await myQueue.add('job-name', { data });
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Key Files Reference
|
||||
|
||||
### Database Repositories
|
||||
|
||||
| Repository | Purpose | Path |
|
||||
| -------------------- | ----------------------------- | ------------------------------------ |
|
||||
| `flyer.db.ts` | Flyer CRUD, processing status | `src/services/db/flyer.db.ts` |
|
||||
| `store.db.ts` | Store management | `src/services/db/store.db.ts` |
|
||||
| `user.db.ts` | User accounts | `src/services/db/user.db.ts` |
|
||||
| `shopping.db.ts` | Shopping lists, watchlists | `src/services/db/shopping.db.ts` |
|
||||
| `gamification.db.ts` | Achievements, points | `src/services/db/gamification.db.ts` |
|
||||
| `category.db.ts` | Item categories | `src/services/db/category.db.ts` |
|
||||
| `price.db.ts` | Price history, comparisons | `src/services/db/price.db.ts` |
|
||||
| `recipe.db.ts` | Recipe management | `src/services/db/recipe.db.ts` |
|
||||
|
||||
### Services
|
||||
|
||||
| Service | Purpose | Path |
|
||||
| ---------------------------------- | -------------------------------- | ----------------------------------------------- |
|
||||
| `flyerProcessingService.server.ts` | Orchestrates flyer AI extraction | `src/services/flyerProcessingService.server.ts` |
|
||||
| `flyerAiProcessor.server.ts` | Gemini AI integration | `src/services/flyerAiProcessor.server.ts` |
|
||||
| `cacheService.server.ts` | Redis caching | `src/services/cacheService.server.ts` |
|
||||
| `queues.server.ts` | BullMQ queue definitions | `src/services/queues.server.ts` |
|
||||
| `workers.server.ts` | BullMQ workers | `src/services/workers.server.ts` |
|
||||
| `emailService.server.ts` | Nodemailer integration | `src/services/emailService.server.ts` |
|
||||
| `geocodingService.server.ts` | Address geocoding | `src/services/geocodingService.server.ts` |
|
||||
|
||||
### Routes
|
||||
|
||||
| Route | Base Path | Auth Required |
|
||||
| ------------------ | ------------- | ------------- |
|
||||
| `flyer.routes.ts` | `/api/flyers` | Mixed |
|
||||
| `store.routes.ts` | `/api/stores` | Mixed |
|
||||
| `user.routes.ts` | `/api/users` | Yes |
|
||||
| `auth.routes.ts` | `/api/auth` | No |
|
||||
| `admin.routes.ts` | `/api/admin` | Admin only |
|
||||
| `deals.routes.ts` | `/api/deals` | No |
|
||||
| `health.routes.ts` | `/api/health` | No |
|
||||
|
||||
---
|
||||
|
||||
## Error Types
|
||||
|
||||
| Error Class | HTTP Status | Use Case |
|
||||
| --------------------------- | ----------- | ------------------------- |
|
||||
| `NotFoundError` | 404 | Resource not found |
|
||||
| `ForbiddenError` | 403 | Access denied |
|
||||
| `ValidationError` | 400 | Input validation failed |
|
||||
| `UniqueConstraintError` | 409 | Duplicate record |
|
||||
| `ForeignKeyConstraintError` | 400 | Referenced record missing |
|
||||
| `NotNullConstraintError` | 400 | Required field null |
|
||||
|
||||
Import: `import { NotFoundError, ... } from '../services/db/errors.db'`
|
||||
|
||||
---
|
||||
|
||||
## Middleware
|
||||
|
||||
| Middleware | Purpose | Usage |
|
||||
| ------------------------- | -------------------- | ------------------------------------------------------------ |
|
||||
| `validateRequest(schema)` | Zod validation | `router.post('/', validateRequest(schema), handler)` |
|
||||
| `requireAuth` | JWT authentication | `router.get('/', requireAuth, handler)` |
|
||||
| `requireAdmin` | Admin role check | `router.delete('/', requireAuth, requireAdmin, handler)` |
|
||||
| `fileUpload` | Multer file handling | `router.post('/upload', fileUpload.single('file'), handler)` |
|
||||
|
||||
---
|
||||
|
||||
## Type Definitions
|
||||
|
||||
| File | Contains |
|
||||
| --------------------------- | ----------------------------------------------- |
|
||||
| `src/types.ts` | Main types: User, Flyer, FlyerItem, Store, etc. |
|
||||
| `src/types/api.ts` | API response envelopes, pagination |
|
||||
| `src/types/auth.ts` | Auth-related types |
|
||||
| `src/types/gamification.ts` | Achievement types |
|
||||
|
||||
---
|
||||
|
||||
## Naming Conventions (ADR-027)
|
||||
|
||||
| Context | Convention | Example |
|
||||
| ----------------- | ---------------- | ----------------------------------- |
|
||||
| AI output types | `Ai*` prefix | `AiFlyerItem`, `AiExtractionResult` |
|
||||
| Database types | `Db*` prefix | `DbFlyer`, `DbUser` |
|
||||
| API types | No prefix | `Flyer`, `User` |
|
||||
| Schema validation | `*Schema` suffix | `createFlyerSchema` |
|
||||
| Routes | `*.routes.ts` | `flyer.routes.ts` |
|
||||
| Repositories | `*.db.ts` | `flyer.db.ts` |
|
||||
| Server services | `*.server.ts` | `aiService.server.ts` |
|
||||
| Client services | `*.client.ts` | `logger.client.ts` |
|
||||
377
docs/SUBAGENT-DB-REFERENCE.md
Normal file
377
docs/SUBAGENT-DB-REFERENCE.md
Normal file
@@ -0,0 +1,377 @@
|
||||
# Database Subagent Reference
|
||||
|
||||
## Quick Navigation
|
||||
|
||||
| Resource | Path |
|
||||
| ------------------ | ---------------------------------------- |
|
||||
| Master Schema | `sql/master_schema_rollup.sql` |
|
||||
| Initial Schema | `sql/initial_schema.sql` |
|
||||
| Migrations | `sql/migrations/*.sql` |
|
||||
| Triggers/Functions | `sql/Initial_triggers_and_functions.sql` |
|
||||
| Initial Data | `sql/initial_data.sql` |
|
||||
| Drop Script | `sql/drop_tables.sql` |
|
||||
| Repositories | `src/services/db/*.db.ts` |
|
||||
| Connection | `src/services/db/connection.db.ts` |
|
||||
| Errors | `src/services/db/errors.db.ts` |
|
||||
|
||||
---
|
||||
|
||||
## Database Credentials
|
||||
|
||||
### Environments
|
||||
|
||||
| Environment | User | Database | Host |
|
||||
| ------------- | -------------------- | -------------------- | --------------------------- |
|
||||
| Production | `flyer_crawler_prod` | `flyer-crawler-prod` | `DB_HOST` secret |
|
||||
| Test | `flyer_crawler_test` | `flyer-crawler-test` | `DB_HOST` secret |
|
||||
| Dev Container | `postgres` | `flyer_crawler_dev` | `postgres` (container name) |
|
||||
|
||||
### Connection (Dev Container)
|
||||
|
||||
```bash
|
||||
# Inside container
|
||||
psql -U postgres -d flyer_crawler_dev
|
||||
|
||||
# From Windows via Podman
|
||||
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev
|
||||
```
|
||||
|
||||
### Connection (Production/Test via SSH)
|
||||
|
||||
```bash
|
||||
# SSH to server, then:
|
||||
PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -U $DB_USER -d $DB_NAME
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Schema Tables (Core)
|
||||
|
||||
| Table | Purpose | Key Columns |
|
||||
| --------------------- | -------------------- | --------------------------------------------------------------- |
|
||||
| `users` | Authentication | `user_id` (UUID PK), `email`, `password_hash` |
|
||||
| `profiles` | User data | `user_id` (FK), `full_name`, `role`, `points` |
|
||||
| `addresses` | Normalized addresses | `address_id`, `address_line_1`, `city`, `latitude`, `longitude` |
|
||||
| `stores` | Store chains | `store_id`, `name`, `logo_url` |
|
||||
| `store_locations` | Physical locations | `store_location_id`, `store_id` (FK), `address_id` (FK) |
|
||||
| `flyers` | Uploaded flyers | `flyer_id`, `store_id` (FK), `image_url`, `status` |
|
||||
| `flyer_items` | Extracted deals | `flyer_item_id`, `flyer_id` (FK), `name`, `price` |
|
||||
| `categories` | Item categories | `category_id`, `name` |
|
||||
| `master_items` | Canonical items | `master_item_id`, `name`, `category_id` (FK) |
|
||||
| `shopping_lists` | User lists | `shopping_list_id`, `user_id` (FK), `name` |
|
||||
| `shopping_list_items` | List items | `shopping_list_item_id`, `shopping_list_id` (FK) |
|
||||
| `watchlist` | Price alerts | `watchlist_id`, `user_id` (FK), `search_term` |
|
||||
| `activity_log` | Audit trail | `activity_log_id`, `user_id`, `action`, `details` |
|
||||
|
||||
---
|
||||
|
||||
## Schema Sync Rule (CRITICAL)
|
||||
|
||||
**Both files MUST stay synchronized:**
|
||||
|
||||
- `sql/master_schema_rollup.sql` - Used by test DB setup
|
||||
- `sql/initial_schema.sql` - Used for fresh installs
|
||||
|
||||
**When adding columns:**
|
||||
|
||||
1. Add migration in `sql/migrations/NNN_description.sql`
|
||||
2. Add column to `master_schema_rollup.sql`
|
||||
3. Add column to `initial_schema.sql`
|
||||
4. Test DB uses `master_schema_rollup.sql` - out-of-sync = test failures
|
||||
|
||||
---
|
||||
|
||||
## Migration Pattern
|
||||
|
||||
### Creating a Migration
|
||||
|
||||
```sql
|
||||
-- sql/migrations/NNN_descriptive_name.sql
|
||||
|
||||
-- Add column with default
|
||||
ALTER TABLE public.flyers
|
||||
ADD COLUMN IF NOT EXISTS new_column TEXT DEFAULT 'value';
|
||||
|
||||
-- Add index
|
||||
CREATE INDEX IF NOT EXISTS idx_flyers_new_column
|
||||
ON public.flyers(new_column);
|
||||
|
||||
-- Update schema_info
|
||||
UPDATE public.schema_info
|
||||
SET schema_hash = 'new_hash', updated_at = now()
|
||||
WHERE environment = 'production';
|
||||
```
|
||||
|
||||
### Running Migrations
|
||||
|
||||
```bash
|
||||
# Via psql
|
||||
PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -U $DB_USER -d $DB_NAME -f sql/migrations/NNN_description.sql
|
||||
|
||||
# In CI/CD - migrations are checked via schema hash
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Repository Pattern (ADR-034)
|
||||
|
||||
### Method Naming Convention
|
||||
|
||||
| Prefix | Behavior | Return Type |
|
||||
| --------- | --------------------------------- | ---------------- |
|
||||
| `get*` | Throws `NotFoundError` if missing | `Entity` |
|
||||
| `find*` | Returns `null` if missing | `Entity \| null` |
|
||||
| `list*` | Returns empty array if none | `Entity[]` |
|
||||
| `create*` | Creates new record | `Entity` |
|
||||
| `update*` | Updates existing record | `Entity` |
|
||||
| `delete*` | Removes record | `void` |
|
||||
| `count*` | Returns count | `number` |
|
||||
|
||||
### Repository Template
|
||||
|
||||
```typescript
|
||||
// src/services/db/entity.db.ts
|
||||
import { getPool } from './connection.db';
|
||||
import { handleDbError, NotFoundError } from './errors.db';
|
||||
import type { PoolClient } from 'pg';
|
||||
import type { Logger } from 'pino';
|
||||
|
||||
export async function getEntityById(
|
||||
id: string,
|
||||
logger: Logger,
|
||||
client?: PoolClient,
|
||||
): Promise<Entity> {
|
||||
const pool = client || getPool();
|
||||
try {
|
||||
const result = await pool.query('SELECT * FROM public.entities WHERE entity_id = $1', [id]);
|
||||
if (result.rows.length === 0) {
|
||||
throw new NotFoundError('Entity not found');
|
||||
}
|
||||
return result.rows[0];
|
||||
} catch (error) {
|
||||
handleDbError(error, logger, 'Failed to get entity', { id });
|
||||
}
|
||||
}
|
||||
|
||||
export async function findEntityByName(
|
||||
name: string,
|
||||
logger: Logger,
|
||||
client?: PoolClient,
|
||||
): Promise<Entity | null> {
|
||||
const pool = client || getPool();
|
||||
try {
|
||||
const result = await pool.query('SELECT * FROM public.entities WHERE name = $1', [name]);
|
||||
return result.rows[0] || null;
|
||||
} catch (error) {
|
||||
handleDbError(error, logger, 'Failed to find entity', { name });
|
||||
}
|
||||
}
|
||||
|
||||
export async function listEntities(logger: Logger, client?: PoolClient): Promise<Entity[]> {
|
||||
const pool = client || getPool();
|
||||
try {
|
||||
const result = await pool.query('SELECT * FROM public.entities ORDER BY name');
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
handleDbError(error, logger, 'Failed to list entities', {});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Transaction Pattern (ADR-002)
|
||||
|
||||
```typescript
|
||||
import { withTransaction } from './connection.db';
|
||||
|
||||
const result = await withTransaction(async (client) => {
|
||||
// All queries use same client
|
||||
const user = await userRepo.createUser(data, logger, client);
|
||||
const profile = await profileRepo.createProfile(user.user_id, profileData, logger, client);
|
||||
await activityRepo.logActivity('user_created', user.user_id, logger, client);
|
||||
return { user, profile };
|
||||
});
|
||||
// Commits on success, rolls back on any error
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Error Handling (ADR-001)
|
||||
|
||||
### Error Types
|
||||
|
||||
| Error | PostgreSQL Code | HTTP Status | Use Case |
|
||||
| -------------------------------- | --------------- | ----------- | ------------------------- |
|
||||
| `UniqueConstraintError` | `23505` | 409 | Duplicate record |
|
||||
| `ForeignKeyConstraintError` | `23503` | 400 | Referenced record missing |
|
||||
| `NotNullConstraintError` | `23502` | 400 | Required field null |
|
||||
| `CheckConstraintError` | `23514` | 400 | Check constraint violated |
|
||||
| `InvalidTextRepresentationError` | `22P02` | 400 | Invalid type format |
|
||||
| `NumericValueOutOfRangeError` | `22003` | 400 | Number out of range |
|
||||
| `NotFoundError` | - | 404 | Record not found |
|
||||
| `ForbiddenError` | - | 403 | Access denied |
|
||||
|
||||
### Using handleDbError
|
||||
|
||||
```typescript
|
||||
import { handleDbError, NotFoundError } from './errors.db';
|
||||
|
||||
try {
|
||||
const result = await pool.query('INSERT INTO ...', [data]);
|
||||
if (result.rows.length === 0) throw new NotFoundError('Entity not found');
|
||||
return result.rows[0];
|
||||
} catch (error) {
|
||||
handleDbError(
|
||||
error,
|
||||
logger,
|
||||
'Failed to create entity',
|
||||
{ data },
|
||||
{
|
||||
uniqueMessage: 'Entity with this name already exists',
|
||||
fkMessage: 'Referenced category does not exist',
|
||||
defaultMessage: 'Failed to create entity',
|
||||
},
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Connection Pool
|
||||
|
||||
```typescript
|
||||
import { getPool, getPoolStatus } from './connection.db';
|
||||
|
||||
// Get pool (singleton)
|
||||
const pool = getPool();
|
||||
|
||||
// Check pool status
|
||||
const status = getPoolStatus();
|
||||
// { totalCount: 20, idleCount: 15, waitingCount: 0 }
|
||||
```
|
||||
|
||||
### Pool Configuration
|
||||
|
||||
| Setting | Value | Purpose |
|
||||
| ------------------------- | ----- | ------------------- |
|
||||
| `max` | 20 | Max clients in pool |
|
||||
| `idleTimeoutMillis` | 30000 | Idle client timeout |
|
||||
| `connectionTimeoutMillis` | 2000 | Connection timeout |
|
||||
|
||||
---
|
||||
|
||||
## Common Queries
|
||||
|
||||
### Paginated List
|
||||
|
||||
```typescript
|
||||
const result = await pool.query(
|
||||
`SELECT * FROM public.flyers
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $1 OFFSET $2`,
|
||||
[limit, (page - 1) * limit],
|
||||
);
|
||||
|
||||
const countResult = await pool.query('SELECT COUNT(*) FROM public.flyers');
|
||||
const total = parseInt(countResult.rows[0].count, 10);
|
||||
```
|
||||
|
||||
### Spatial Query (Find Nearby)
|
||||
|
||||
```typescript
|
||||
const result = await pool.query(
|
||||
`SELECT sl.*, a.*,
|
||||
ST_Distance(a.location, ST_SetSRID(ST_MakePoint($1, $2), 4326)::geography) as distance
|
||||
FROM public.store_locations sl
|
||||
JOIN public.addresses a ON sl.address_id = a.address_id
|
||||
WHERE ST_DWithin(a.location, ST_SetSRID(ST_MakePoint($1, $2), 4326)::geography, $3)
|
||||
ORDER BY distance`,
|
||||
[longitude, latitude, radiusMeters],
|
||||
);
|
||||
```
|
||||
|
||||
### Upsert Pattern
|
||||
|
||||
```typescript
|
||||
const result = await pool.query(
|
||||
`INSERT INTO public.stores (name, logo_url)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT (name) DO UPDATE SET
|
||||
logo_url = EXCLUDED.logo_url,
|
||||
updated_at = now()
|
||||
RETURNING *`,
|
||||
[name, logoUrl],
|
||||
);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Database Reset Commands
|
||||
|
||||
### Dev Container
|
||||
|
||||
```bash
|
||||
# Reset dev database (runs seed script)
|
||||
podman exec -it flyer-crawler-dev npm run db:reset:dev
|
||||
|
||||
# Reset test database
|
||||
podman exec -it flyer-crawler-dev npm run db:reset:test
|
||||
```
|
||||
|
||||
### Manual SQL
|
||||
|
||||
```bash
|
||||
# Drop all tables
|
||||
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -f /app/sql/drop_tables.sql
|
||||
|
||||
# Recreate schema
|
||||
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -f /app/sql/master_schema_rollup.sql
|
||||
|
||||
# Load initial data
|
||||
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -f /app/sql/initial_data.sql
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Database Users Setup
|
||||
|
||||
```sql
|
||||
-- Create database and user (as postgres superuser)
|
||||
CREATE DATABASE "flyer-crawler-test";
|
||||
CREATE USER flyer_crawler_test WITH PASSWORD 'password';
|
||||
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
|
||||
|
||||
-- Grant permissions
|
||||
\c "flyer-crawler-test"
|
||||
ALTER SCHEMA public OWNER TO flyer_crawler_test;
|
||||
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
|
||||
|
||||
-- Required extensions
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "postgis";
|
||||
|
||||
-- Verify permissions
|
||||
\dn+ public
|
||||
-- Should show 'UC' for the user
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Repository Files
|
||||
|
||||
| Repository | Domain | Path |
|
||||
| --------------------- | -------------------------- | ------------------------------------- |
|
||||
| `user.db.ts` | Users, profiles | `src/services/db/user.db.ts` |
|
||||
| `flyer.db.ts` | Flyers, processing | `src/services/db/flyer.db.ts` |
|
||||
| `store.db.ts` | Stores | `src/services/db/store.db.ts` |
|
||||
| `storeLocation.db.ts` | Store locations | `src/services/db/storeLocation.db.ts` |
|
||||
| `address.db.ts` | Addresses | `src/services/db/address.db.ts` |
|
||||
| `category.db.ts` | Categories | `src/services/db/category.db.ts` |
|
||||
| `shopping.db.ts` | Shopping lists, watchlists | `src/services/db/shopping.db.ts` |
|
||||
| `price.db.ts` | Price history | `src/services/db/price.db.ts` |
|
||||
| `gamification.db.ts` | Achievements, points | `src/services/db/gamification.db.ts` |
|
||||
| `notification.db.ts` | Notifications | `src/services/db/notification.db.ts` |
|
||||
| `recipe.db.ts` | Recipes | `src/services/db/recipe.db.ts` |
|
||||
| `receipt.db.ts` | Receipts | `src/services/db/receipt.db.ts` |
|
||||
| `admin.db.ts` | Admin operations | `src/services/db/admin.db.ts` |
|
||||
357
docs/SUBAGENT-DEVOPS-REFERENCE.md
Normal file
357
docs/SUBAGENT-DEVOPS-REFERENCE.md
Normal file
@@ -0,0 +1,357 @@
|
||||
# DevOps Subagent Reference
|
||||
|
||||
## Critical Rule: Git Bash Path Conversion
|
||||
|
||||
Git Bash on Windows auto-converts Unix paths, breaking container commands.
|
||||
|
||||
| Solution | Example |
|
||||
| ---------------------------- | -------------------------------------------------------- |
|
||||
| `sh -c` with single quotes | `podman exec container sh -c '/usr/local/bin/script.sh'` |
|
||||
| Double slashes | `podman exec container //usr//local//bin//script.sh` |
|
||||
| MSYS_NO_PATHCONV=1 | `MSYS_NO_PATHCONV=1 podman exec ...` |
|
||||
| Windows paths for host files | `podman cp "d:/path/file" container:/tmp/file` |
|
||||
|
||||
---
|
||||
|
||||
## Container Commands (Podman)
|
||||
|
||||
### Dev Container Operations
|
||||
|
||||
```bash
|
||||
# List running containers
|
||||
podman ps
|
||||
|
||||
# Container logs
|
||||
podman logs flyer-crawler-dev
|
||||
podman logs -f flyer-crawler-dev # Follow
|
||||
|
||||
# Execute in container
|
||||
podman exec -it flyer-crawler-dev bash
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
|
||||
# Restart container
|
||||
podman restart flyer-crawler-dev
|
||||
|
||||
# Container resource usage
|
||||
podman stats flyer-crawler-dev
|
||||
```
|
||||
|
||||
### Test Execution (from Windows)
|
||||
|
||||
```bash
|
||||
# Unit tests - pipe output for AI processing
|
||||
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
|
||||
|
||||
# Integration tests
|
||||
podman exec -it flyer-crawler-dev npm run test:integration
|
||||
|
||||
# Type check (CRITICAL before commit)
|
||||
podman exec -it flyer-crawler-dev npm run type-check
|
||||
|
||||
# Specific test file
|
||||
podman exec -it flyer-crawler-dev npm test -- --run src/hooks/useAuth.test.tsx
|
||||
```
|
||||
|
||||
### Database Operations (from Windows)
|
||||
|
||||
```bash
|
||||
# Reset dev database
|
||||
podman exec -it flyer-crawler-dev npm run db:reset:dev
|
||||
|
||||
# Access PostgreSQL
|
||||
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev
|
||||
|
||||
# Run SQL file (use MSYS_NO_PATHCONV to avoid path conversion)
|
||||
MSYS_NO_PATHCONV=1 podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -f /app/sql/master_schema_rollup.sql
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## PM2 Commands
|
||||
|
||||
### Production Server (via SSH)
|
||||
|
||||
```bash
|
||||
# SSH to server
|
||||
ssh root@projectium.com
|
||||
|
||||
# List all apps
|
||||
pm2 list
|
||||
|
||||
# App status
|
||||
pm2 show flyer-crawler-api
|
||||
|
||||
# Logs
|
||||
pm2 logs flyer-crawler-api
|
||||
pm2 logs --lines 100
|
||||
|
||||
# Restart apps
|
||||
pm2 restart flyer-crawler-api
|
||||
pm2 reload flyer-crawler-api # Zero-downtime
|
||||
|
||||
# Stop/Start
|
||||
pm2 stop flyer-crawler-api
|
||||
pm2 start flyer-crawler-api
|
||||
|
||||
# Delete and reload from config
|
||||
pm2 delete all
|
||||
pm2 start ecosystem.config.cjs
|
||||
```
|
||||
|
||||
### PM2 Config: `ecosystem.config.cjs`
|
||||
|
||||
| App | Purpose | Memory | Mode |
|
||||
| -------------------------------- | ---------------- | ------ | ------- |
|
||||
| `flyer-crawler-api` | Express server | 500M | Cluster |
|
||||
| `flyer-crawler-worker` | BullMQ worker | 1G | Fork |
|
||||
| `flyer-crawler-analytics-worker` | Analytics worker | 1G | Fork |
|
||||
|
||||
Test variants: `*-test` suffix
|
||||
|
||||
---
|
||||
|
||||
## CI/CD Workflows
|
||||
|
||||
### Location: `.gitea/workflows/`
|
||||
|
||||
| Workflow | Trigger | Purpose |
|
||||
| ----------------------------- | ------------ | ----------------------- |
|
||||
| `deploy-to-test.yml` | Push to main | Auto-deploy to test env |
|
||||
| `deploy-to-prod.yml` | Manual | Deploy to production |
|
||||
| `manual-db-backup.yml` | Manual | Database backup |
|
||||
| `manual-db-reset-test.yml` | Manual | Reset test database |
|
||||
| `manual-db-reset-prod.yml` | Manual | Reset prod database |
|
||||
| `manual-db-restore.yml` | Manual | Restore database |
|
||||
| `manual-deploy-major.yml` | Manual | Major version release |
|
||||
| `manual-redis-flush-prod.yml` | Manual | Flush Redis cache |
|
||||
|
||||
### Deploy to Test Pipeline Steps
|
||||
|
||||
1. Checkout code
|
||||
2. Setup Node.js 20
|
||||
3. `npm ci`
|
||||
4. Bump patch version (creates git tag)
|
||||
5. `npm run type-check`
|
||||
6. `npm run test:unit`
|
||||
7. Check schema hash against deployed DB
|
||||
8. `npm run build`
|
||||
9. Copy files to `/var/www/flyer-crawler-test.projectium.com/`
|
||||
10. `pm2 reload ecosystem-test.config.cjs`
|
||||
|
||||
### Deploy to Production Pipeline Steps
|
||||
|
||||
1. Verify confirmation phrase ("deploy-to-prod")
|
||||
2. Checkout `main` branch
|
||||
3. `npm ci`
|
||||
4. Bump minor version (creates git tag)
|
||||
5. Check schema hash against prod DB
|
||||
6. `npm run build` (with Sentry source maps)
|
||||
7. Copy files to `/var/www/flyer-crawler.projectium.com/`
|
||||
8. `pm2 reload ecosystem.config.cjs`
|
||||
|
||||
---
|
||||
|
||||
## Deployment Paths
|
||||
|
||||
| Environment | Path | Domain |
|
||||
| ------------- | --------------------------------------------- | ----------------------------------- |
|
||||
| Production | `/var/www/flyer-crawler.projectium.com/` | `flyer-crawler.projectium.com` |
|
||||
| Test | `/var/www/flyer-crawler-test.projectium.com/` | `flyer-crawler-test.projectium.com` |
|
||||
| Dev Container | `/app/` | `localhost:3000` |
|
||||
|
||||
---
|
||||
|
||||
## Environment Configuration
|
||||
|
||||
### Files
|
||||
|
||||
| File | Purpose |
|
||||
| --------------------------- | ----------------------- |
|
||||
| `ecosystem.config.cjs` | PM2 production config |
|
||||
| `ecosystem-test.config.cjs` | PM2 test config |
|
||||
| `src/config/env.ts` | Zod schema for env vars |
|
||||
| `.env.example` | Template for env vars |
|
||||
|
||||
### Required Secrets (Gitea CI/CD)
|
||||
|
||||
| Category | Secrets |
|
||||
| ---------- | -------------------------------------------------------------------------------------------- |
|
||||
| Database | `DB_HOST`, `DB_USER_PROD`, `DB_PASSWORD_PROD`, `DB_DATABASE_PROD` |
|
||||
| Test DB | `DB_USER_TEST`, `DB_PASSWORD_TEST`, `DB_DATABASE_TEST` |
|
||||
| Redis | `REDIS_PASSWORD_PROD`, `REDIS_PASSWORD_TEST` |
|
||||
| Auth | `JWT_SECRET`, `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, `GH_CLIENT_ID`, `GH_CLIENT_SECRET` |
|
||||
| AI | `VITE_GOOGLE_GENAI_API_KEY`, `VITE_GOOGLE_GENAI_API_KEY_TEST` |
|
||||
| Monitoring | `SENTRY_DSN`, `SENTRY_DSN_TEST`, `SENTRY_AUTH_TOKEN` |
|
||||
| Maps | `GOOGLE_MAPS_API_KEY` |
|
||||
|
||||
### Adding New Secret
|
||||
|
||||
1. Add to Gitea Settings > Secrets
|
||||
2. Update workflow YAML: `SENTRY_DSN: ${{ secrets.SENTRY_DSN }}`
|
||||
3. Update `ecosystem.config.cjs`
|
||||
4. Update `src/config/env.ts` Zod schema
|
||||
5. Update `.env.example`
|
||||
|
||||
---
|
||||
|
||||
## Redis Commands
|
||||
|
||||
### Dev Container
|
||||
|
||||
```bash
|
||||
# Access Redis CLI
|
||||
podman exec -it flyer-crawler-dev redis-cli
|
||||
|
||||
# Common commands
|
||||
KEYS *
|
||||
FLUSHALL
|
||||
INFO
|
||||
```
|
||||
|
||||
### Production
|
||||
|
||||
```bash
|
||||
# Via SSH
|
||||
ssh root@projectium.com
|
||||
redis-cli -a $REDIS_PASSWORD
|
||||
|
||||
# Flush cache (use with caution)
|
||||
# Or use manual-redis-flush-prod.yml workflow
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Health Checks
|
||||
|
||||
### Endpoints
|
||||
|
||||
| Endpoint | Purpose |
|
||||
| -------------------------- | -------------------------------------- |
|
||||
| `GET /api/health` | Basic health check |
|
||||
| `GET /api/health/detailed` | Full system status (DB, Redis, queues) |
|
||||
|
||||
### Manual Health Check
|
||||
|
||||
```bash
|
||||
# From Windows
|
||||
curl http://localhost:3000/api/health
|
||||
|
||||
# Or via Podman
|
||||
podman exec -it flyer-crawler-dev curl http://localhost:3000/api/health
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Log Locations
|
||||
|
||||
### Production Server
|
||||
|
||||
```bash
|
||||
# PM2 logs
|
||||
~/.pm2/logs/
|
||||
|
||||
# NGINX logs
|
||||
/var/log/nginx/access.log
|
||||
/var/log/nginx/error.log
|
||||
|
||||
# Application logs (via PM2)
|
||||
pm2 logs flyer-crawler-api --lines 200
|
||||
```
|
||||
|
||||
### Dev Container
|
||||
|
||||
```bash
|
||||
# View container logs
|
||||
podman logs flyer-crawler-dev
|
||||
|
||||
# Follow logs
|
||||
podman logs -f flyer-crawler-dev
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Backup/Restore
|
||||
|
||||
### Database Backup (Manual Workflow)
|
||||
|
||||
Trigger `manual-db-backup.yml` from Gitea Actions UI.
|
||||
|
||||
### Manual Backup
|
||||
|
||||
```bash
|
||||
# SSH to server
|
||||
ssh root@projectium.com
|
||||
|
||||
# Backup
|
||||
PGPASSWORD=$DB_PASSWORD pg_dump -h $DB_HOST -U $DB_USER $DB_NAME > backup_$(date +%Y%m%d).sql
|
||||
|
||||
# Restore
|
||||
PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -U $DB_USER $DB_NAME < backup.sql
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Bugsink (Error Tracking)
|
||||
|
||||
### Dev Container Token Generation
|
||||
|
||||
```bash
|
||||
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
|
||||
```
|
||||
|
||||
### Production Token Generation
|
||||
|
||||
```bash
|
||||
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Common Troubleshooting
|
||||
|
||||
### Container Won't Start
|
||||
|
||||
```bash
|
||||
# Check logs
|
||||
podman logs flyer-crawler-dev
|
||||
|
||||
# Inspect container
|
||||
podman inspect flyer-crawler-dev
|
||||
|
||||
# Remove and recreate
|
||||
podman rm -f flyer-crawler-dev
|
||||
# Then recreate with docker-compose or podman run
|
||||
```
|
||||
|
||||
### Database Connection Issues
|
||||
|
||||
```bash
|
||||
# Test connection inside container
|
||||
podman exec -it flyer-crawler-dev psql -U postgres -d flyer_crawler_dev -c "SELECT 1"
|
||||
|
||||
# Check if PostgreSQL is running
|
||||
podman exec -it flyer-crawler-dev pg_isready
|
||||
```
|
||||
|
||||
### PM2 App Keeps Restarting
|
||||
|
||||
```bash
|
||||
# Check logs
|
||||
pm2 logs flyer-crawler-api --err --lines 100
|
||||
|
||||
# Check for memory issues
|
||||
pm2 monit
|
||||
|
||||
# View app details
|
||||
pm2 show flyer-crawler-api
|
||||
```
|
||||
|
||||
### Redis Connection Issues
|
||||
|
||||
```bash
|
||||
# Test Redis inside container
|
||||
podman exec -it flyer-crawler-dev redis-cli ping
|
||||
|
||||
# Check Redis logs
|
||||
podman logs flyer-crawler-redis
|
||||
```
|
||||
410
docs/SUBAGENT-INTEGRATIONS-REFERENCE.md
Normal file
410
docs/SUBAGENT-INTEGRATIONS-REFERENCE.md
Normal file
@@ -0,0 +1,410 @@
|
||||
# Integrations Subagent Reference
|
||||
|
||||
## MCP Servers Overview
|
||||
|
||||
| Server | Purpose | URL | Tools Prefix |
|
||||
| ------------------ | ------------------------- | ----------------------------------------------------------------- | -------------------------- |
|
||||
| `bugsink` | Production error tracking | `https://bugsink.projectium.com` | `mcp__bugsink__*` |
|
||||
| `localerrors` | Dev container errors | `http://127.0.0.1:8000` | `mcp__localerrors__*` |
|
||||
| `devdb` | Dev PostgreSQL | `postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev` | `mcp__devdb__*` |
|
||||
| `gitea-projectium` | Gitea API | `gitea.projectium.com` | `mcp__gitea-projectium__*` |
|
||||
| `gitea-torbonium` | Gitea API | `gitea.torbonium.com` | `mcp__gitea-torbonium__*` |
|
||||
| `podman` | Container management | - | `mcp__podman__*` |
|
||||
| `filesystem` | File system access | - | `mcp__filesystem__*` |
|
||||
| `memory` | Knowledge graph | - | `mcp__memory__*` |
|
||||
| `redis` | Cache management | `localhost:6379` | `mcp__redis__*` |
|
||||
|
||||
---
|
||||
|
||||
## MCP Server Configuration
|
||||
|
||||
### Global Config: `~/.claude/settings.json`
|
||||
|
||||
Used for production/remote servers (HTTPS works fine).
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"bugsink": {
|
||||
"command": "node",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "https://bugsink.projectium.com",
|
||||
"BUGSINK_TOKEN": "<40-char-hex-token>"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Project Config: `.mcp.json`
|
||||
|
||||
**CRITICAL:** Use project-level `.mcp.json` for localhost servers. Global config has issues loading localhost stdio MCP servers.
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"localerrors": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "http://127.0.0.1:8000",
|
||||
"BUGSINK_TOKEN": "<40-char-hex-token>"
|
||||
}
|
||||
},
|
||||
"devdb": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": [
|
||||
"-y",
|
||||
"@modelcontextprotocol/server-postgres",
|
||||
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Bugsink Integration
|
||||
|
||||
### API Token Generation
|
||||
|
||||
**Bugsink 2.0.11 has NO UI for API tokens.** Use Django management command.
|
||||
|
||||
#### Dev Container
|
||||
|
||||
```bash
|
||||
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
|
||||
```
|
||||
|
||||
#### Production
|
||||
|
||||
```bash
|
||||
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
|
||||
```
|
||||
|
||||
### Bugsink MCP Tools
|
||||
|
||||
| Tool | Purpose |
|
||||
| ----------------- | ---------------------- |
|
||||
| `test_connection` | Verify connection |
|
||||
| `list_projects` | List all projects |
|
||||
| `list_issues` | List issues by project |
|
||||
| `get_issue` | Issue details |
|
||||
| `list_events` | Events for an issue |
|
||||
| `get_event` | Event details |
|
||||
| `get_stacktrace` | Formatted stacktrace |
|
||||
|
||||
### Usage Example
|
||||
|
||||
```typescript
|
||||
// Test connection
|
||||
mcp__bugsink__test_connection();
|
||||
|
||||
// List production issues
|
||||
mcp__bugsink__list_issues({ project_id: 1, status: 'unresolved', limit: 10 });
|
||||
|
||||
// Get stacktrace
|
||||
mcp__bugsink__get_stacktrace({ event_id: 'uuid-here' });
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## PostgreSQL MCP Integration
|
||||
|
||||
### Setup
|
||||
|
||||
```bash
|
||||
# Uses @modelcontextprotocol/server-postgres package
|
||||
# Connection string in .mcp.json
|
||||
```
|
||||
|
||||
### Tools
|
||||
|
||||
| Tool | Purpose |
|
||||
| ------- | --------------------- |
|
||||
| `query` | Execute read-only SQL |
|
||||
|
||||
### Usage Example
|
||||
|
||||
```typescript
|
||||
mcp__devdb__query({ sql: 'SELECT * FROM public.users LIMIT 5' });
|
||||
mcp__devdb__query({ sql: 'SELECT COUNT(*) FROM public.flyers' });
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Gitea MCP Integration
|
||||
|
||||
### Common Tools
|
||||
|
||||
| Tool | Purpose |
|
||||
| ------------------------- | ----------------- |
|
||||
| `get_my_user_info` | Current user info |
|
||||
| `list_my_repos` | List repositories |
|
||||
| `get_issue_by_index` | Issue details |
|
||||
| `list_repo_issues` | Repository issues |
|
||||
| `create_issue` | Create new issue |
|
||||
| `create_pull_request` | Create PR |
|
||||
| `list_repo_pull_requests` | List PRs |
|
||||
| `get_file_content` | Read file |
|
||||
| `list_repo_commits` | Commit history |
|
||||
|
||||
### Usage Example
|
||||
|
||||
```typescript
|
||||
// List issues
|
||||
mcp__gitea -
|
||||
projectium__list_repo_issues({
|
||||
owner: 'james',
|
||||
repo: 'flyer-crawler',
|
||||
state: 'open',
|
||||
});
|
||||
|
||||
// Create issue
|
||||
mcp__gitea -
|
||||
projectium__create_issue({
|
||||
owner: 'james',
|
||||
repo: 'flyer-crawler',
|
||||
title: 'Bug: Description',
|
||||
body: 'Details here',
|
||||
});
|
||||
|
||||
// Get file content
|
||||
mcp__gitea -
|
||||
projectium__get_file_content({
|
||||
owner: 'james',
|
||||
repo: 'flyer-crawler',
|
||||
ref: 'main',
|
||||
filePath: 'CLAUDE.md',
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Redis MCP Integration
|
||||
|
||||
### Tools
|
||||
|
||||
| Tool | Purpose |
|
||||
| -------- | -------------------- |
|
||||
| `get` | Get key value |
|
||||
| `set` | Set key value |
|
||||
| `delete` | Delete key(s) |
|
||||
| `list` | List keys by pattern |
|
||||
|
||||
### Usage Example
|
||||
|
||||
```typescript
|
||||
// List cache keys
|
||||
mcp__redis__list({ pattern: 'flyer:*' });
|
||||
|
||||
// Get cached value
|
||||
mcp__redis__get({ key: 'flyer:123' });
|
||||
|
||||
// Set with expiration
|
||||
mcp__redis__set({ key: 'test:key', value: 'data', expireSeconds: 3600 });
|
||||
|
||||
// Delete key
|
||||
mcp__redis__delete({ key: 'test:key' });
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Podman MCP Integration
|
||||
|
||||
### Tools
|
||||
|
||||
| Tool | Purpose |
|
||||
| ------------------- | ----------------- |
|
||||
| `container_list` | List containers |
|
||||
| `container_logs` | View logs |
|
||||
| `container_inspect` | Container details |
|
||||
| `container_stop` | Stop container |
|
||||
| `container_remove` | Remove container |
|
||||
| `container_run` | Run container |
|
||||
| `image_list` | List images |
|
||||
| `image_pull` | Pull image |
|
||||
|
||||
### Usage Example
|
||||
|
||||
```typescript
|
||||
// List running containers
|
||||
mcp__podman__container_list();
|
||||
|
||||
// View container logs
|
||||
mcp__podman__container_logs({ name: 'flyer-crawler-dev' });
|
||||
|
||||
// Inspect container
|
||||
mcp__podman__container_inspect({ name: 'flyer-crawler-dev' });
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Memory MCP (Knowledge Graph)
|
||||
|
||||
### Tools
|
||||
|
||||
| Tool | Purpose |
|
||||
| ------------------ | -------------------- |
|
||||
| `read_graph` | Read entire graph |
|
||||
| `search_nodes` | Search by query |
|
||||
| `open_nodes` | Get specific nodes |
|
||||
| `create_entities` | Create entities |
|
||||
| `create_relations` | Create relationships |
|
||||
| `add_observations` | Add observations |
|
||||
| `delete_entities` | Delete entities |
|
||||
|
||||
### Usage Example
|
||||
|
||||
```typescript
|
||||
// Search for context
|
||||
mcp__memory__search_nodes({ query: 'flyer-crawler' });
|
||||
|
||||
// Read full graph
|
||||
mcp__memory__read_graph();
|
||||
|
||||
// Create entity
|
||||
mcp__memory__create_entities({
|
||||
entities: [
|
||||
{
|
||||
name: 'FlyCrawler',
|
||||
entityType: 'Project',
|
||||
observations: ['Uses PostgreSQL', 'Express backend'],
|
||||
},
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Filesystem MCP
|
||||
|
||||
### Tools
|
||||
|
||||
| Tool | Purpose |
|
||||
| ---------------- | --------------------- |
|
||||
| `read_text_file` | Read file contents |
|
||||
| `write_file` | Write file |
|
||||
| `edit_file` | Edit file |
|
||||
| `list_directory` | List directory |
|
||||
| `directory_tree` | Tree view |
|
||||
| `search_files` | Find files by pattern |
|
||||
|
||||
### Usage Example
|
||||
|
||||
```typescript
|
||||
// Read file
|
||||
mcp__filesystem__read_text_file({ path: 'd:\\gitea\\project\\README.md' });
|
||||
|
||||
// List directory
|
||||
mcp__filesystem__list_directory({ path: 'd:\\gitea\\project\\src' });
|
||||
|
||||
// Search for files
|
||||
mcp__filesystem__search_files({
|
||||
path: 'd:\\gitea\\project',
|
||||
pattern: '**/*.test.ts',
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting MCP Servers
|
||||
|
||||
### Server Not Loading
|
||||
|
||||
1. **Check server name** - Avoid shared prefixes (e.g., `bugsink` and `bugsink-dev`)
|
||||
2. **Use project-level `.mcp.json`** for localhost servers
|
||||
3. **Restart Claude Code** after config changes
|
||||
|
||||
### Test Connection Manually
|
||||
|
||||
```bash
|
||||
# Bugsink
|
||||
set BUGSINK_URL=http://localhost:8000
|
||||
set BUGSINK_TOKEN=<token>
|
||||
node d:\gitea\bugsink-mcp\dist\index.js
|
||||
|
||||
# PostgreSQL
|
||||
npx -y @modelcontextprotocol/server-postgres "postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
|
||||
```
|
||||
|
||||
### Check Claude Debug Logs
|
||||
|
||||
```
|
||||
C:\Users\<username>\.claude\debug\*.txt
|
||||
```
|
||||
|
||||
Look for "Starting connection" messages - missing server = never started.
|
||||
|
||||
---
|
||||
|
||||
## External API Integrations
|
||||
|
||||
### Gemini AI (Flyer Extraction)
|
||||
|
||||
| Config | Location |
|
||||
| ------- | ---------------------------------------------- |
|
||||
| API Key | `VITE_GOOGLE_GENAI_API_KEY` / `GEMINI_API_KEY` |
|
||||
| Service | `src/services/flyerAiProcessor.server.ts` |
|
||||
| Client | `@google/genai` package |
|
||||
|
||||
### Google OAuth
|
||||
|
||||
| Config | Location |
|
||||
| ------------- | ------------------------ |
|
||||
| Client ID | `GOOGLE_CLIENT_ID` |
|
||||
| Client Secret | `GOOGLE_CLIENT_SECRET` |
|
||||
| Service | `src/config/passport.ts` |
|
||||
|
||||
### GitHub OAuth
|
||||
|
||||
| Config | Location |
|
||||
| ------------- | ------------------------------------------- |
|
||||
| Client ID | `GH_CLIENT_ID` / `GITHUB_CLIENT_ID` |
|
||||
| Client Secret | `GH_CLIENT_SECRET` / `GITHUB_CLIENT_SECRET` |
|
||||
| Service | `src/config/passport.ts` |
|
||||
|
||||
### Google Maps (Geocoding)
|
||||
|
||||
| Config | Location |
|
||||
| ------- | ----------------------------------------------- |
|
||||
| API Key | `GOOGLE_MAPS_API_KEY` |
|
||||
| Service | `src/services/googleGeocodingService.server.ts` |
|
||||
|
||||
### Nominatim (Fallback Geocoding)
|
||||
|
||||
| Config | Location |
|
||||
| ------- | -------------------------------------------------- |
|
||||
| URL | `https://nominatim.openstreetmap.org` |
|
||||
| Service | `src/services/nominatimGeocodingService.server.ts` |
|
||||
|
||||
### Sentry (Error Tracking)
|
||||
|
||||
| Config | Location |
|
||||
| -------------- | ------------------------------------------------- |
|
||||
| DSN | `SENTRY_DSN` (server), `VITE_SENTRY_DSN` (client) |
|
||||
| Auth Token | `SENTRY_AUTH_TOKEN` (source map upload) |
|
||||
| Server Service | `src/services/sentry.server.ts` |
|
||||
| Client Service | `src/services/sentry.client.ts` |
|
||||
|
||||
### SMTP (Email)
|
||||
|
||||
| Config | Location |
|
||||
| ----------- | ------------------------------------- |
|
||||
| Host | `SMTP_HOST` |
|
||||
| Port | `SMTP_PORT` |
|
||||
| Credentials | `SMTP_USER`, `SMTP_PASS` |
|
||||
| Service | `src/services/emailService.server.ts` |
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
| Document | Purpose |
|
||||
| -------------------------------- | ----------------------- |
|
||||
| `BUGSINK-MCP-TROUBLESHOOTING.md` | MCP server issues |
|
||||
| `POSTGRES-MCP-SETUP.md` | PostgreSQL MCP setup |
|
||||
| `DEV-CONTAINER-BUGSINK.md` | Local Bugsink setup |
|
||||
| `BUGSINK-SYNC.md` | Bugsink synchronization |
|
||||
358
docs/SUBAGENT-TESTER-REFERENCE.md
Normal file
358
docs/SUBAGENT-TESTER-REFERENCE.md
Normal file
@@ -0,0 +1,358 @@
|
||||
# Tester Subagent Reference
|
||||
|
||||
## Critical Rule: Linux Only (ADR-014)
|
||||
|
||||
**ALL tests MUST run in the dev container.** Windows test results are unreliable.
|
||||
|
||||
| Result | Interpretation |
|
||||
| ------------------------- | -------------------- |
|
||||
| Pass Windows / Fail Linux | BROKEN - must fix |
|
||||
| Fail Windows / Pass Linux | PASSING - acceptable |
|
||||
|
||||
---
|
||||
|
||||
## Test Commands
|
||||
|
||||
### From Windows Host (via Podman)
|
||||
|
||||
```bash
|
||||
# Unit tests (~2900 tests) - pipe to file for AI processing
|
||||
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
|
||||
|
||||
# Integration tests (requires DB/Redis)
|
||||
podman exec -it flyer-crawler-dev npm run test:integration
|
||||
|
||||
# E2E tests (requires all services)
|
||||
podman exec -it flyer-crawler-dev npm run test:e2e
|
||||
|
||||
# Specific test file
|
||||
podman exec -it flyer-crawler-dev npm test -- --run src/hooks/useAuth.test.tsx
|
||||
|
||||
# Type checking (CRITICAL before commit)
|
||||
podman exec -it flyer-crawler-dev npm run type-check
|
||||
|
||||
# Coverage report
|
||||
podman exec -it flyer-crawler-dev npm run test:coverage
|
||||
```
|
||||
|
||||
### Inside Dev Container
|
||||
|
||||
```bash
|
||||
npm test # All tests
|
||||
npm run test:unit # Unit tests only
|
||||
npm run test:integration # Integration tests
|
||||
npm run test:e2e # E2E tests
|
||||
npm run type-check # TypeScript check
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test File Locations
|
||||
|
||||
| Test Type | Location | Config |
|
||||
| ----------- | --------------------------------------------- | ------------------------------ |
|
||||
| Unit | `src/**/*.test.ts`, `src/**/*.test.tsx` | `vite.config.ts` |
|
||||
| Integration | `src/tests/integration/*.integration.test.ts` | `vitest.config.integration.ts` |
|
||||
| E2E | `src/tests/e2e/*.e2e.test.ts` | `vitest.config.e2e.ts` |
|
||||
| Setup | `src/tests/setup/*.ts` | - |
|
||||
| Helpers | `src/tests/utils/*.ts` | - |
|
||||
|
||||
---
|
||||
|
||||
## Test Helpers
|
||||
|
||||
### Location: `src/tests/utils/`
|
||||
|
||||
| Helper | Purpose | Import |
|
||||
| ----------------------- | --------------------------------------------------------------- | ----------------------------------- |
|
||||
| `testHelpers.ts` | `createAndLoginUser()`, `getTestBaseUrl()`, `getFlyerBaseUrl()` | `../tests/utils/testHelpers` |
|
||||
| `cleanup.ts` | `cleanupDb({ userIds, flyerIds })` | `../tests/utils/cleanup` |
|
||||
| `mockFactories.ts` | `createMockStore()`, `createMockAddress()`, `createMockFlyer()` | `../tests/utils/mockFactories` |
|
||||
| `storeHelpers.ts` | `createStoreWithLocation()`, `cleanupStoreLocations()` | `../tests/utils/storeHelpers` |
|
||||
| `poll.ts` | `poll(fn, predicate, options)` - wait for async conditions | `../tests/utils/poll` |
|
||||
| `mockLogger.ts` | Mock pino logger for tests | `../tests/utils/mockLogger` |
|
||||
| `createTestApp.ts` | Create Express app instance for route tests | `../tests/utils/createTestApp` |
|
||||
| `createMockRequest.ts` | Create mock Express request objects | `../tests/utils/createMockRequest` |
|
||||
| `cleanupFiles.ts` | Clean up test file uploads | `../tests/utils/cleanupFiles` |
|
||||
| `websocketTestUtils.ts` | WebSocket testing utilities | `../tests/utils/websocketTestUtils` |
|
||||
|
||||
### Usage Examples
|
||||
|
||||
```typescript
|
||||
// Create authenticated user for tests
|
||||
import { createAndLoginUser, TEST_PASSWORD } from '../tests/utils/testHelpers';
|
||||
const { user, token } = await createAndLoginUser({
|
||||
email: `test-${Date.now()}@example.com`,
|
||||
request: request(app), // For integration tests
|
||||
role: 'admin', // Optional: make admin
|
||||
});
|
||||
|
||||
// Cleanup after tests
|
||||
import { cleanupDb } from '../tests/utils/cleanup';
|
||||
afterEach(async () => {
|
||||
await cleanupDb({ userIds: [user.user.user_id] });
|
||||
});
|
||||
|
||||
// Wait for async operation
|
||||
import { poll } from '../tests/utils/poll';
|
||||
await poll(
|
||||
() => db.userRepo.findUserByEmail(email, logger),
|
||||
(user) => !!user,
|
||||
{ timeout: 5000, interval: 500, description: 'user to be findable' },
|
||||
);
|
||||
|
||||
// Create mock data
|
||||
import { createMockStore, createMockFlyer } from '../tests/utils/mockFactories';
|
||||
const mockStore = createMockStore({ name: 'Test Store' });
|
||||
const mockFlyer = createMockFlyer({ store_id: mockStore.store_id });
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test Setup Files
|
||||
|
||||
| File | Purpose |
|
||||
| --------------------------------------------- | ---------------------------------------- |
|
||||
| `src/tests/setup/tests-setup-unit.ts` | Unit test setup (mocks, DOM environment) |
|
||||
| `src/tests/setup/tests-setup-integration.ts` | Integration test setup (DB connections) |
|
||||
| `src/tests/setup/global-setup.ts` | Global setup for unit tests |
|
||||
| `src/tests/setup/integration-global-setup.ts` | Global setup for integration tests |
|
||||
| `src/tests/setup/e2e-global-setup.ts` | Global setup for E2E tests |
|
||||
| `src/tests/setup/mockHooks.ts` | React hook mocking utilities |
|
||||
| `src/tests/setup/mockUI.ts` | UI component mocking |
|
||||
| `src/tests/setup/globalApiMock.ts` | API mocking setup |
|
||||
|
||||
---
|
||||
|
||||
## Known Integration Test Issues
|
||||
|
||||
### 1. Vitest globalSetup Context Isolation
|
||||
|
||||
**Problem**: globalSetup runs in separate Node.js context. Singletons/mocks don't share instances.
|
||||
|
||||
**Affected**: BullMQ worker mocks
|
||||
|
||||
**Solution**: Use `.todo()`, test-only API endpoints, or Redis-based flags.
|
||||
|
||||
### 2. Cleanup Queue Race Condition
|
||||
|
||||
**Problem**: Cleanup worker processes jobs before test verification.
|
||||
|
||||
**Solution**:
|
||||
|
||||
```typescript
|
||||
const { cleanupQueue } = await import('../../services/queues.server');
|
||||
await cleanupQueue.drain();
|
||||
await cleanupQueue.pause();
|
||||
// ... run test ...
|
||||
await cleanupQueue.resume();
|
||||
```
|
||||
|
||||
### 3. Cache Stale After Direct SQL
|
||||
|
||||
**Problem**: Direct `pool.query()` bypasses cache invalidation.
|
||||
|
||||
**Solution**:
|
||||
|
||||
```typescript
|
||||
await pool.query('INSERT INTO flyers ...');
|
||||
await cacheService.invalidateFlyers(); // Add this
|
||||
```
|
||||
|
||||
### 4. File Upload Filename Collisions
|
||||
|
||||
**Problem**: Multer predictable filenames cause race conditions.
|
||||
|
||||
**Solution**:
|
||||
|
||||
```typescript
|
||||
const filename = `test-${Date.now()}-${Math.round(Math.random() * 1e9)}.jpg`;
|
||||
```
|
||||
|
||||
### 5. Response Format Mismatches
|
||||
|
||||
**Problem**: API response structure changes (`data.jobId` vs `data.job.id`).
|
||||
|
||||
**Solution**: Log response bodies, update assertions to match actual format.
|
||||
|
||||
---
|
||||
|
||||
## Test Patterns
|
||||
|
||||
### Unit Test Pattern
|
||||
|
||||
```typescript
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
describe('MyService', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should do something', () => {
|
||||
// Arrange
|
||||
const input = { data: 'test' };
|
||||
|
||||
// Act
|
||||
const result = myService.process(input);
|
||||
|
||||
// Assert
|
||||
expect(result).toBe('expected');
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Integration Test Pattern
|
||||
|
||||
```typescript
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import { createAndLoginUser } from '../utils/testHelpers';
|
||||
import { cleanupDb } from '../utils/cleanup';
|
||||
|
||||
describe('API Integration', () => {
|
||||
let user, token;
|
||||
|
||||
beforeEach(async () => {
|
||||
const result = await createAndLoginUser({ request: request(app) });
|
||||
user = result.user;
|
||||
token = result.token;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await cleanupDb({ userIds: [user.user.user_id] });
|
||||
});
|
||||
|
||||
it('GET /api/resource returns data', async () => {
|
||||
const res = await request(app).get('/api/resource').set('Authorization', `Bearer ${token}`);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(true);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Route Test Pattern
|
||||
|
||||
```typescript
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import { createTestApp } from '../tests/utils/createTestApp';
|
||||
|
||||
vi.mock('../services/db/flyer.db', () => ({
|
||||
getFlyerById: vi.fn(),
|
||||
}));
|
||||
|
||||
describe('Flyer Routes', () => {
|
||||
let app;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
app = createTestApp();
|
||||
});
|
||||
|
||||
it('GET /api/flyers/:id returns flyer', async () => {
|
||||
const mockFlyer = { id: '123', name: 'Test' };
|
||||
vi.mocked(flyerRepo.getFlyerById).mockResolvedValue(mockFlyer);
|
||||
|
||||
const res = await request(app).get('/api/flyers/123');
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.data).toEqual(mockFlyer);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Mocking Patterns
|
||||
|
||||
### Mock Modules
|
||||
|
||||
```typescript
|
||||
// At top of test file
|
||||
vi.mock('../services/db/flyer.db', () => ({
|
||||
getFlyerById: vi.fn(),
|
||||
listFlyers: vi.fn(),
|
||||
}));
|
||||
|
||||
// In test
|
||||
import * as flyerDb from '../services/db/flyer.db';
|
||||
vi.mocked(flyerDb.getFlyerById).mockResolvedValue(mockFlyer);
|
||||
```
|
||||
|
||||
### Mock React Query
|
||||
|
||||
```typescript
|
||||
vi.mock('@tanstack/react-query', async () => {
|
||||
const actual = await vi.importActual('@tanstack/react-query');
|
||||
return {
|
||||
...actual,
|
||||
useQuery: vi.fn().mockReturnValue({
|
||||
data: mockData,
|
||||
isLoading: false,
|
||||
error: null,
|
||||
}),
|
||||
};
|
||||
});
|
||||
```
|
||||
|
||||
### Mock Pino Logger
|
||||
|
||||
```typescript
|
||||
import { createMockLogger } from '../tests/utils/mockLogger';
|
||||
const mockLogger = createMockLogger();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Testing React Components
|
||||
|
||||
```typescript
|
||||
import { render, screen, fireEvent, waitFor } from '@testing-library/react';
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||
import { BrowserRouter } from 'react-router-dom';
|
||||
|
||||
const createWrapper = () => {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: { queries: { retry: false } },
|
||||
});
|
||||
return ({ children }) => (
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<BrowserRouter>{children}</BrowserRouter>
|
||||
</QueryClientProvider>
|
||||
);
|
||||
};
|
||||
|
||||
it('renders component', () => {
|
||||
render(<MyComponent />, { wrapper: createWrapper() });
|
||||
expect(screen.getByText('Expected Text')).toBeInTheDocument();
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test Coverage
|
||||
|
||||
```bash
|
||||
# Generate coverage report
|
||||
podman exec -it flyer-crawler-dev npm run test:coverage
|
||||
|
||||
# View HTML report
|
||||
# Coverage reports generated in coverage/ directory
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Debugging Tests
|
||||
|
||||
```bash
|
||||
# Verbose output
|
||||
npm test -- --reporter=verbose
|
||||
|
||||
# Run single test with debugging
|
||||
DEBUG=* npm test -- --run src/path/to/test.test.ts
|
||||
|
||||
# Vitest UI (interactive)
|
||||
npm run test:ui
|
||||
```
|
||||
@@ -76,16 +76,18 @@ This provides a secondary error capture path for:
|
||||
- Database function errors and slow queries
|
||||
- Historical error analysis from log files
|
||||
|
||||
### 5. MCP Server Integration: sentry-selfhosted-mcp
|
||||
### 5. MCP Server Integration: bugsink-mcp
|
||||
|
||||
For AI tool integration (Claude Code, Cursor, etc.), we use the open-source [sentry-selfhosted-mcp](https://github.com/ddfourtwo/sentry-selfhosted-mcp) server:
|
||||
For AI tool integration (Claude Code, Cursor, etc.), we use the open-source [bugsink-mcp](https://github.com/j-shelfwood/bugsink-mcp) server:
|
||||
|
||||
- **No code changes required**: Configurable via environment variables
|
||||
- **Capabilities**: List projects, get issues, view events, update status, add comments
|
||||
- **Capabilities**: List projects, get issues, view events, get stacktraces, manage releases
|
||||
- **Configuration**:
|
||||
- `SENTRY_URL`: Points to Bugsink instance
|
||||
- `SENTRY_AUTH_TOKEN`: API token from Bugsink
|
||||
- `SENTRY_ORG_SLUG`: Organization identifier
|
||||
- `BUGSINK_URL`: Points to Bugsink instance (`http://localhost:8000` for dev, `https://bugsink.projectium.com` for prod)
|
||||
- `BUGSINK_API_TOKEN`: API token from Bugsink (created via Django management command)
|
||||
- `BUGSINK_ORG_SLUG`: Organization identifier (usually "sentry")
|
||||
|
||||
**Note:** Despite the name `sentry-selfhosted-mcp` mentioned in earlier drafts of this ADR, the actual MCP server used is `bugsink-mcp` which is specifically designed for Bugsink's API structure.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -144,12 +146,12 @@ External (Developer Machine):
|
||||
┌──────────────────────────────────────┐
|
||||
│ Claude Code / Cursor / VS Code │
|
||||
│ ┌────────────────────────────────┐ │
|
||||
│ │ sentry-selfhosted-mcp │ │
|
||||
│ │ bugsink-mcp │ │
|
||||
│ │ (MCP Server) │ │
|
||||
│ │ │ │
|
||||
│ │ SENTRY_URL=http://localhost:8000
|
||||
│ │ SENTRY_AUTH_TOKEN=... │ │
|
||||
│ │ SENTRY_ORG_SLUG=... │ │
|
||||
│ │ BUGSINK_URL=http://localhost:8000
|
||||
│ │ BUGSINK_API_TOKEN=... │ │
|
||||
│ │ BUGSINK_ORG_SLUG=... │ │
|
||||
│ └────────────────────────────────┘ │
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
@@ -279,7 +281,7 @@ output {
|
||||
- Configure Redis log monitoring (connection errors, slow commands)
|
||||
|
||||
7. **MCP server documentation**:
|
||||
- Document `sentry-selfhosted-mcp` setup in CLAUDE.md
|
||||
- Document `bugsink-mcp` setup in CLAUDE.md
|
||||
|
||||
8. **PostgreSQL function logging** (future):
|
||||
- Configure PostgreSQL to log function execution errors
|
||||
@@ -318,5 +320,5 @@ output {
|
||||
- [Bugsink Docker Install](https://www.bugsink.com/docs/docker-install/)
|
||||
- [@sentry/node Documentation](https://docs.sentry.io/platforms/javascript/guides/node/)
|
||||
- [@sentry/react Documentation](https://docs.sentry.io/platforms/javascript/guides/react/)
|
||||
- [sentry-selfhosted-mcp](https://github.com/ddfourtwo/sentry-selfhosted-mcp)
|
||||
- [bugsink-mcp](https://github.com/j-shelfwood/bugsink-mcp)
|
||||
- [Logstash Reference](https://www.elastic.co/guide/en/logstash/current/index.html)
|
||||
|
||||
@@ -384,21 +384,21 @@ const AuthCallback = () => {
|
||||
|
||||
### Mitigation
|
||||
|
||||
- Document OAuth enablement steps clearly (see AUTHENTICATION.md).
|
||||
- Document OAuth enablement steps clearly (see [../architecture/AUTHENTICATION.md](../architecture/AUTHENTICATION.md)).
|
||||
- Consider adding OAuth provider ID columns for future account linking.
|
||||
- Use URL fragment (`#token=`) instead of query parameter for callback.
|
||||
|
||||
## Key Files
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------- | ------------------------------------------------ |
|
||||
| `src/routes/passport.routes.ts` | Passport strategies (local, JWT, OAuth) |
|
||||
| `src/routes/auth.routes.ts` | Auth endpoints (login, register, refresh, OAuth) |
|
||||
| `src/services/authService.ts` | Auth business logic |
|
||||
| `src/services/db/user.db.ts` | User database operations |
|
||||
| `src/config/env.ts` | Environment variable validation |
|
||||
| `AUTHENTICATION.md` | OAuth setup guide |
|
||||
| `.env.example` | Environment variable template |
|
||||
| File | Purpose |
|
||||
| ------------------------------------------------------ | ------------------------------------------------ |
|
||||
| `src/routes/passport.routes.ts` | Passport strategies (local, JWT, OAuth) |
|
||||
| `src/routes/auth.routes.ts` | Auth endpoints (login, register, refresh, OAuth) |
|
||||
| `src/services/authService.ts` | Auth business logic |
|
||||
| `src/services/db/user.db.ts` | User database operations |
|
||||
| `src/config/env.ts` | Environment variable validation |
|
||||
| [AUTHENTICATION.md](../architecture/AUTHENTICATION.md) | OAuth setup guide |
|
||||
| `.env.example` | Environment variable template |
|
||||
|
||||
## Related ADRs
|
||||
|
||||
|
||||
@@ -149,7 +149,7 @@ All synced issues also receive the `source:bugsink` label.
|
||||
```bash
|
||||
# Bugsink Configuration
|
||||
BUGSINK_URL=https://bugsink.projectium.com
|
||||
BUGSINK_API_TOKEN=77deaa5e... # From Bugsink Settings > API Keys
|
||||
BUGSINK_API_TOKEN=77deaa5e... # Created via Django management command (see BUGSINK-SYNC.md)
|
||||
|
||||
# Gitea Configuration
|
||||
GITEA_URL=https://gitea.projectium.com
|
||||
|
||||
877
docs/architecture/OVERVIEW.md
Normal file
877
docs/architecture/OVERVIEW.md
Normal file
@@ -0,0 +1,877 @@
|
||||
# Flyer Crawler - System Architecture Overview
|
||||
|
||||
**Version**: 0.12.5
|
||||
**Last Updated**: 2026-01-22
|
||||
**Platform**: Linux (Production and Development)
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Executive Summary](#executive-summary)
|
||||
2. [System Architecture Diagram](#system-architecture-diagram)
|
||||
3. [Technology Stack](#technology-stack)
|
||||
4. [System Components](#system-components)
|
||||
5. [Data Flow](#data-flow)
|
||||
6. [Architecture Layers](#architecture-layers)
|
||||
7. [Key Entities](#key-entities)
|
||||
8. [Authentication Flow](#authentication-flow)
|
||||
9. [Background Processing](#background-processing)
|
||||
10. [Deployment Architecture](#deployment-architecture)
|
||||
11. [Design Principles and ADRs](#design-principles-and-adrs)
|
||||
12. [Key Files Reference](#key-files-reference)
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
|
||||
**Flyer Crawler** is a grocery deal extraction and analysis platform that uses AI-powered processing to extract deals from grocery store flyer images and PDFs. The system provides users with features including watchlists, price history tracking, shopping lists, deal alerts, and recipe management.
|
||||
|
||||
### Core Capabilities
|
||||
|
||||
| Domain | Description |
|
||||
| ------------------------- | --------------------------------------------------------------------------------------- |
|
||||
| **Deal Extraction** | AI-powered extraction of deals from grocery store flyer images/PDFs using Google Gemini |
|
||||
| **Price Tracking** | Historical price data, trend analysis, and price alerts |
|
||||
| **User Features** | Watchlists, shopping lists, recipes, pantry management, achievements |
|
||||
| **Real-time Updates** | WebSocket-based notifications for price alerts and processing status |
|
||||
| **Background Processing** | Asynchronous job queues for flyer processing, emails, and analytics |
|
||||
|
||||
---
|
||||
|
||||
## System Architecture Diagram
|
||||
|
||||
```
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| CLIENT LAYER |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| |
|
||||
| +-------------------+ +-------------------+ +-------------------+ |
|
||||
| | Web Browser | | Mobile PWA | | API Clients | |
|
||||
| | (React SPA) | | (React SPA) | | (REST/JSON) | |
|
||||
| +--------+----------+ +--------+----------+ +--------+----------+ |
|
||||
| | | | |
|
||||
+-------------|-------------------------|-------------------------|------------------+
|
||||
| | |
|
||||
v v v
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| NGINX REVERSE PROXY |
|
||||
| - SSL/TLS Termination - Rate Limiting - Static Asset Serving |
|
||||
| - Load Balancing - Compression - WebSocket Proxying |
|
||||
| - Flyer Images (/flyer-images/) with 7-day cache |
|
||||
+----------------------------------+------------------------------------------------+
|
||||
|
|
||||
v
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| APPLICATION LAYER |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| |
|
||||
| +-----------------------------------------------------------------------------+ |
|
||||
| | EXPRESS.JS SERVER (Node.js) | |
|
||||
| | | |
|
||||
| | +-------------------------+ +-------------------------+ | |
|
||||
| | | Routes Layer | | Middleware Chain | | |
|
||||
| | | - API Endpoints | | - Authentication | | |
|
||||
| | | - Request Validation | | - Rate Limiting | | |
|
||||
| | | - Response Formatting | | - Logging | | |
|
||||
| | +------------+------------+ | - Error Handling | | |
|
||||
| | | +-------------------------+ | |
|
||||
| | v | |
|
||||
| | +-------------------------+ +-------------------------+ | |
|
||||
| | | Services Layer | | External Services | | |
|
||||
| | | - Business Logic | | - Google Gemini AI | | |
|
||||
| | | - Transaction Coord. | | - Google Maps API | | |
|
||||
| | | - Event Publishing | | - OAuth Providers | | |
|
||||
| | +------------+------------+ | - Email (SMTP) | | |
|
||||
| | | +-------------------------+ | |
|
||||
| | v | |
|
||||
| | +-------------------------+ | |
|
||||
| | | Repository Layer | | |
|
||||
| | | - Database Access | | |
|
||||
| | | - Query Construction | | |
|
||||
| | | - Entity Mapping | | |
|
||||
| | +------------+------------+ | |
|
||||
| | | | |
|
||||
| +---------------|-------------------------------------------------------------+ |
|
||||
| | |
|
||||
+------------------|----------------------------------------------------------------+
|
||||
|
|
||||
v
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| DATA LAYER |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| |
|
||||
| +---------------------------+ +---------------------------+ |
|
||||
| | PostgreSQL 16 | | Redis 7 | |
|
||||
| | (with PostGIS) | | | |
|
||||
| | | | - Session Cache | |
|
||||
| | - Primary Data Store | | - Query Cache | |
|
||||
| | - Geographic Queries | | - Job Queue Backing | |
|
||||
| | - Full-Text Search | | - Rate Limit Counters | |
|
||||
| | - Stored Functions | | - Real-time Pub/Sub | |
|
||||
| +---------------------------+ +---------------------------+ |
|
||||
| |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| BACKGROUND PROCESSING LAYER |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| |
|
||||
| +---------------------------+ +---------------------------+ |
|
||||
| | PM2 Process | | BullMQ Workers | |
|
||||
| | Manager | | | |
|
||||
| | | | - Flyer Processing | |
|
||||
| | - Process Clustering | | - Receipt Processing | |
|
||||
| | - Auto-restart | | - Email Sending | |
|
||||
| | - Log Management | | - Analytics Reports | |
|
||||
| | - Health Monitoring | | - File Cleanup | |
|
||||
| +---------------------------+ | - Token Cleanup | |
|
||||
| | - Expiry Alerts | |
|
||||
| | - Barcode Detection | |
|
||||
| +---------------------------+ |
|
||||
| |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| OBSERVABILITY LAYER |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| |
|
||||
| +------------------+ +------------------+ +------------------+ |
|
||||
| | Bugsink/Sentry | | Pino Logger | | Logstash | |
|
||||
| | (Error Track) | | (Structured) | | (Aggregation) | |
|
||||
| +------------------+ +------------------+ +------------------+ |
|
||||
| |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Technology Stack
|
||||
|
||||
### Core Technologies
|
||||
|
||||
| Component | Technology | Version | Purpose |
|
||||
| ---------------------- | ---------- | -------- | -------------------------------- |
|
||||
| **Runtime** | Node.js | 22.x LTS | Server-side JavaScript runtime |
|
||||
| **Language** | TypeScript | 5.9.x | Type-safe JavaScript superset |
|
||||
| **Web Framework** | Express.js | 5.1.x | HTTP server and routing |
|
||||
| **Frontend Framework** | React | 19.2.x | UI component library |
|
||||
| **Build Tool** | Vite | 7.2.x | Frontend bundling and dev server |
|
||||
|
||||
### Data Storage
|
||||
|
||||
| Component | Technology | Version | Purpose |
|
||||
| --------------------- | ---------------- | ------- | ---------------------------------------------- |
|
||||
| **Primary Database** | PostgreSQL | 16.x | Relational data storage |
|
||||
| **Spatial Extension** | PostGIS | 3.x | Geographic queries and store location features |
|
||||
| **Cache & Queues** | Redis | 7.x | Caching, session storage, job queue backing |
|
||||
| **File Storage** | Local Filesystem | - | Uploaded flyers and processed images |
|
||||
|
||||
### AI and External Services
|
||||
|
||||
| Component | Technology | Purpose |
|
||||
| --------------- | --------------------------- | --------------------------------------- |
|
||||
| **AI Provider** | Google Gemini | Flyer data extraction, image analysis |
|
||||
| **Geocoding** | Google Maps API / Nominatim | Address geocoding and location services |
|
||||
| **OAuth** | Google, GitHub | Social authentication |
|
||||
| **Email** | Nodemailer (SMTP) | Transactional emails |
|
||||
|
||||
### Background Processing
|
||||
|
||||
| Component | Technology | Version | Purpose |
|
||||
| ------------------- | ---------- | ------- | --------------------------------- |
|
||||
| **Job Queues** | BullMQ | 5.65.x | Reliable async job processing |
|
||||
| **Process Manager** | PM2 | Latest | Process management and clustering |
|
||||
| **Scheduler** | node-cron | 4.2.x | Scheduled tasks |
|
||||
|
||||
### Frontend Stack
|
||||
|
||||
| Component | Technology | Version | Purpose |
|
||||
| -------------------- | -------------- | ------- | ---------------------------------------- |
|
||||
| **State Management** | TanStack Query | 5.90.x | Server state caching and synchronization |
|
||||
| **Routing** | React Router | 7.9.x | Client-side routing |
|
||||
| **Styling** | Tailwind CSS | 4.1.x | Utility-first CSS framework |
|
||||
| **Icons** | Lucide React | 0.555.x | Icon components |
|
||||
| **Charts** | Recharts | 3.4.x | Data visualization |
|
||||
|
||||
### Observability and Quality
|
||||
|
||||
| Component | Technology | Purpose |
|
||||
| --------------------- | ---------------- | ----------------------------- |
|
||||
| **Error Tracking** | Sentry / Bugsink | Error monitoring and alerting |
|
||||
| **Logging** | Pino | Structured JSON logging |
|
||||
| **Log Aggregation** | Logstash | Centralized log collection |
|
||||
| **Testing** | Vitest | Unit and integration testing |
|
||||
| **API Documentation** | Swagger/OpenAPI | Interactive API documentation |
|
||||
|
||||
---
|
||||
|
||||
## System Components
|
||||
|
||||
### Frontend (React/Vite)
|
||||
|
||||
The frontend is a single-page application (SPA) built with React 19 and Vite.
|
||||
|
||||
**Key Characteristics**:
|
||||
|
||||
- Server state management via TanStack Query
|
||||
- Neo-Brutalism design system (ADR-012)
|
||||
- Responsive design for mobile and desktop
|
||||
- PWA-capable for offline access
|
||||
|
||||
**Directory Structure**:
|
||||
|
||||
```
|
||||
src/
|
||||
+-- components/ # Reusable UI components
|
||||
+-- contexts/ # React context providers
|
||||
+-- features/ # Feature-specific modules (ADR-047)
|
||||
+-- hooks/ # Custom React hooks
|
||||
+-- layouts/ # Page layout components
|
||||
+-- pages/ # Route page components
|
||||
+-- services/ # API client services
|
||||
```
|
||||
|
||||
### Backend (Express/Node.js)
|
||||
|
||||
The backend is a RESTful API server built with Express.js 5.
|
||||
|
||||
**Key Characteristics**:
|
||||
|
||||
- Layered architecture (Routes -> Services -> Repositories)
|
||||
- JWT-based authentication with OAuth support
|
||||
- Request validation via Zod schemas
|
||||
- Structured logging with Pino
|
||||
- Standardized error handling (ADR-001)
|
||||
|
||||
**API Route Modules**:
|
||||
| Route | Purpose |
|
||||
|-------|---------|
|
||||
| `/api/auth` | Authentication (login, register, OAuth) |
|
||||
| `/api/users` | User profile management |
|
||||
| `/api/flyers` | Flyer CRUD and processing |
|
||||
| `/api/recipes` | Recipe management |
|
||||
| `/api/deals` | Best prices and deal discovery |
|
||||
| `/api/stores` | Store management |
|
||||
| `/api/admin` | Administrative functions |
|
||||
| `/api/health` | Health checks and monitoring |
|
||||
|
||||
### Database (PostgreSQL/PostGIS)
|
||||
|
||||
PostgreSQL serves as the primary data store with PostGIS extension for geographic queries.
|
||||
|
||||
**Key Features**:
|
||||
|
||||
- UUID primary keys for user data
|
||||
- BIGINT IDENTITY for auto-incrementing IDs
|
||||
- PostGIS geography types for store locations
|
||||
- Stored functions for complex business logic
|
||||
- Triggers for automated updates (e.g., `item_count` maintenance)
|
||||
|
||||
### Cache (Redis)
|
||||
|
||||
Redis provides caching and backing for the job queue system.
|
||||
|
||||
**Usage Patterns**:
|
||||
|
||||
- Query result caching (flyers, prices, stats)
|
||||
- Rate limiting counters
|
||||
- BullMQ job queue storage
|
||||
- Session token storage
|
||||
|
||||
### AI (Google Gemini)
|
||||
|
||||
Google Gemini powers the AI extraction capabilities.
|
||||
|
||||
**Capabilities**:
|
||||
|
||||
- Flyer image analysis and data extraction
|
||||
- Store name and logo detection
|
||||
- Deal item parsing (name, price, quantity)
|
||||
- Date range extraction
|
||||
- Category classification
|
||||
|
||||
### Background Workers (BullMQ/PM2)
|
||||
|
||||
BullMQ workers handle asynchronous processing tasks. PM2 manages both the API server and worker processes in production and development environments (ADR-014).
|
||||
|
||||
**Dev Container PM2 Processes**:
|
||||
|
||||
| Process | Script | Purpose |
|
||||
| -------------------------- | ---------------------------------- | -------------------------- |
|
||||
| `flyer-crawler-api-dev` | `tsx watch server.ts` | API server with hot reload |
|
||||
| `flyer-crawler-worker-dev` | `tsx watch src/services/worker.ts` | Background job processing |
|
||||
| `flyer-crawler-vite-dev` | `vite --host` | Frontend dev server |
|
||||
|
||||
**Production PM2 Processes**:
|
||||
|
||||
| Process | Script | Purpose |
|
||||
| -------------------------------- | ------------------ | --------------------------- |
|
||||
| `flyer-crawler-api` | `tsx server.ts` | API server (cluster mode) |
|
||||
| `flyer-crawler-worker` | `tsx worker.ts` | Background job processing |
|
||||
| `flyer-crawler-analytics-worker` | `tsx analytics.ts` | Analytics report generation |
|
||||
|
||||
**Job Queues**:
|
||||
|
||||
| Queue | Purpose | Retry Strategy |
|
||||
| ---------------------------- | -------------------------------- | ------------------------------------- |
|
||||
| `flyer-processing` | Process uploaded flyers with AI | 3 attempts, exponential backoff (5s) |
|
||||
| `receipt-processing` | OCR and parse receipts | 3 attempts, exponential backoff (10s) |
|
||||
| `email-sending` | Send transactional emails | 5 attempts, exponential backoff (10s) |
|
||||
| `analytics-reporting` | Generate daily analytics | 2 attempts, exponential backoff (60s) |
|
||||
| `weekly-analytics-reporting` | Generate weekly reports | 2 attempts, exponential backoff (1h) |
|
||||
| `file-cleanup` | Remove temporary files | 3 attempts, exponential backoff (30s) |
|
||||
| `token-cleanup` | Expire old refresh tokens | 2 attempts, exponential backoff (1h) |
|
||||
| `expiry-alerts` | Send pantry expiry notifications | 2 attempts, exponential backoff (5m) |
|
||||
| `barcode-detection` | Process barcode scans | 2 attempts, exponential backoff (5s) |
|
||||
|
||||
---
|
||||
|
||||
## Data Flow
|
||||
|
||||
### Flyer Processing Pipeline
|
||||
|
||||
```
|
||||
+-------------+ +----------------+ +------------------+ +---------------+
|
||||
| User | | Express | | BullMQ | | PostgreSQL |
|
||||
| Upload +---->+ Route +---->+ Queue +---->+ Storage |
|
||||
+-------------+ +-------+--------+ +--------+---------+ +-------+-------+
|
||||
| | |
|
||||
v v v
|
||||
+-------+--------+ +--------+---------+ +-------+-------+
|
||||
| Validate | | Worker | | Cache |
|
||||
| & Store | | Process | | Invalidate |
|
||||
| Temp File | | | | |
|
||||
+----------------+ +--------+---------+ +---------------+
|
||||
|
|
||||
v
|
||||
+--------+---------+
|
||||
| Google |
|
||||
| Gemini AI |
|
||||
| Extraction |
|
||||
+--------+---------+
|
||||
|
|
||||
v
|
||||
+--------+---------+
|
||||
| Transform |
|
||||
| & Validate |
|
||||
| Data |
|
||||
+--------+---------+
|
||||
|
|
||||
v
|
||||
+--------+---------+
|
||||
| Persist to |
|
||||
| Database |
|
||||
| (Transaction) |
|
||||
+--------+---------+
|
||||
|
|
||||
v
|
||||
+--------+---------+
|
||||
| WebSocket |
|
||||
| Notification |
|
||||
+------------------+
|
||||
```
|
||||
|
||||
### Detailed Processing Steps
|
||||
|
||||
1. **Upload**: User uploads flyer image via `/api/flyers/upload`
|
||||
2. **Validation**: Server validates file type, size, and generates checksum
|
||||
3. **Queueing**: Job added to `flyer-processing` queue with file path
|
||||
4. **Worker Pickup**: BullMQ worker picks up job for processing
|
||||
5. **AI Extraction**: Google Gemini analyzes image and extracts:
|
||||
- Store name
|
||||
- Valid date range
|
||||
- Store address (if present)
|
||||
- Deal items (name, price, quantity, category)
|
||||
6. **Data Transformation**: Raw AI output transformed to database schema
|
||||
7. **Persistence**: Transactional insert of flyer + items + store
|
||||
8. **Cache Invalidation**: Redis cache cleared for affected queries
|
||||
9. **Notification**: WebSocket message sent to user with results
|
||||
10. **Cleanup**: Temporary files scheduled for deletion
|
||||
|
||||
---
|
||||
|
||||
## Architecture Layers
|
||||
|
||||
The application follows a strict layered architecture as defined in ADR-035.
|
||||
|
||||
```
|
||||
+-----------------------------------------------------------------------+
|
||||
| ROUTES LAYER |
|
||||
| Responsibilities: |
|
||||
| - HTTP request/response handling |
|
||||
| - Input validation (via middleware) |
|
||||
| - Authentication/authorization checks |
|
||||
| - Rate limiting |
|
||||
| - Response formatting (sendSuccess, sendPaginated, sendError) |
|
||||
+----------------------------------+------------------------------------+
|
||||
|
|
||||
v
|
||||
+-----------------------------------------------------------------------+
|
||||
| SERVICES LAYER |
|
||||
| Responsibilities: |
|
||||
| - Business logic orchestration |
|
||||
| - Transaction coordination (withTransaction) |
|
||||
| - External API integration |
|
||||
| - Cross-repository operations |
|
||||
| - Event publishing |
|
||||
+----------------------------------+------------------------------------+
|
||||
|
|
||||
v
|
||||
+-----------------------------------------------------------------------+
|
||||
| REPOSITORY LAYER |
|
||||
| Responsibilities: |
|
||||
| - Direct database access |
|
||||
| - Query construction |
|
||||
| - Entity mapping |
|
||||
| - Error translation (handleDbError) |
|
||||
+-----------------------------------------------------------------------+
|
||||
```
|
||||
|
||||
### Layer Communication Rules
|
||||
|
||||
1. **Routes MUST NOT** directly access repositories (except simple CRUD)
|
||||
2. **Repositories MUST NOT** call other repositories (use services)
|
||||
3. **Services MAY** call other services
|
||||
4. **Infrastructure services MAY** be called from any layer
|
||||
|
||||
### Service Types and Naming Conventions
|
||||
|
||||
| Type | Suffix | Example | Location |
|
||||
| ------------------- | ------------- | --------------------- | ------------------ |
|
||||
| Business Service | `*Service.ts` | `authService.ts` | `src/services/` |
|
||||
| Server-Only Service | `*.server.ts` | `aiService.server.ts` | `src/services/` |
|
||||
| Database Repository | `*.db.ts` | `user.db.ts` | `src/services/db/` |
|
||||
| Infrastructure | Descriptive | `logger.server.ts` | `src/services/` |
|
||||
|
||||
### Repository Method Naming (ADR-034)
|
||||
|
||||
| Prefix | Behavior | Return Type |
|
||||
| ------- | ----------------------------------- | -------------- |
|
||||
| `get*` | Throws `NotFoundError` if not found | Entity |
|
||||
| `find*` | Returns `null` if not found | Entity or null |
|
||||
| `list*` | Returns empty array if none found | Entity[] |
|
||||
|
||||
---
|
||||
|
||||
## Key Entities
|
||||
|
||||
### Entity Relationship Overview
|
||||
|
||||
```
|
||||
+------------------+ +------------------+ +------------------+
|
||||
| users | | profiles | | addresses |
|
||||
|------------------| |------------------| |------------------|
|
||||
| user_id (PK) |<-------->| user_id (PK,FK) |--------->| address_id (PK) |
|
||||
| email | | full_name | | address_line_1 |
|
||||
| password_hash | | avatar_url | | city |
|
||||
| refresh_token | | points | | province_state |
|
||||
+--------+---------+ | role | | latitude |
|
||||
| +------------------+ | longitude |
|
||||
| | location (GIS) |
|
||||
| +--------+---------+
|
||||
| ^
|
||||
v |
|
||||
+--------+---------+ +------------------+ +--------+---------+
|
||||
| stores |--------->| store_locations |--------->| |
|
||||
|------------------| |------------------| | |
|
||||
| store_id (PK) | | store_location_id| | |
|
||||
| name | | store_id (FK) | | |
|
||||
| logo_url | | address_id (FK) | | |
|
||||
+--------+---------+ +------------------+ +------------------+
|
||||
|
|
||||
v
|
||||
+--------+---------+ +------------------+ +------------------+
|
||||
| flyers |--------->| flyer_items |--------->| master_grocery_ |
|
||||
|------------------| |------------------| | items |
|
||||
| flyer_id (PK) | | flyer_item_id | |------------------|
|
||||
| store_id (FK) | | flyer_id (FK) | | master_grocery_ |
|
||||
| file_name | | item | | item_id (PK) |
|
||||
| image_url | | price_display | | name |
|
||||
| valid_from | | price_in_cents | | category_id (FK) |
|
||||
| valid_to | | quantity | | is_allergen |
|
||||
| status | | master_item_id | +------------------+
|
||||
| item_count | | category_id (FK) |
|
||||
+------------------+ +------------------+
|
||||
```
|
||||
|
||||
### Core Entities
|
||||
|
||||
| Entity | Table | Purpose |
|
||||
| --------------------- | ---------------------- | --------------------------------------------- |
|
||||
| **User** | `users` | Authentication credentials and login tracking |
|
||||
| **Profile** | `profiles` | Public user data, preferences, points |
|
||||
| **Store** | `stores` | Grocery store chains (Safeway, Kroger, etc.) |
|
||||
| **StoreLocation** | `store_locations` | Physical store locations with addresses |
|
||||
| **Address** | `addresses` | Normalized address storage with geocoding |
|
||||
| **Flyer** | `flyers` | Uploaded flyer metadata and status |
|
||||
| **FlyerItem** | `flyer_items` | Individual deals extracted from flyers |
|
||||
| **MasterGroceryItem** | `master_grocery_items` | Canonical grocery item dictionary |
|
||||
| **Category** | `categories` | Item categorization (Produce, Dairy, etc.) |
|
||||
|
||||
### User Feature Entities
|
||||
|
||||
| Entity | Table | Purpose |
|
||||
| -------------------- | --------------------- | ------------------------------------ |
|
||||
| **UserWatchedItem** | `user_watched_items` | Items user wants to track prices for |
|
||||
| **UserAlert** | `user_alerts` | Price alert thresholds |
|
||||
| **ShoppingList** | `shopping_lists` | User shopping lists |
|
||||
| **ShoppingListItem** | `shopping_list_items` | Items on shopping lists |
|
||||
| **Recipe** | `recipes` | User recipes with ingredients |
|
||||
| **RecipeIngredient** | `recipe_ingredients` | Recipe ingredient list |
|
||||
| **PantryItem** | `pantry_items` | User pantry inventory |
|
||||
| **Receipt** | `receipts` | Scanned receipt data |
|
||||
| **ReceiptItem** | `receipt_items` | Items parsed from receipts |
|
||||
|
||||
### Gamification Entities
|
||||
|
||||
| Entity | Table | Purpose |
|
||||
| ------------------- | ------------------- | ------------------------------------- |
|
||||
| **Achievement** | `achievements` | Defined achievements |
|
||||
| **UserAchievement** | `user_achievements` | Achievements earned by users |
|
||||
| **ActivityLog** | `activity_log` | User activity for feeds and analytics |
|
||||
|
||||
---
|
||||
|
||||
## Authentication Flow
|
||||
|
||||
### JWT Token Architecture
|
||||
|
||||
```
|
||||
+-------------------+ +-------------------+ +-------------------+
|
||||
| Login Request | | Server | | Database |
|
||||
| (email/pass) +---->+ Validates +---->+ Verify User |
|
||||
+-------------------+ +--------+----------+ +-------------------+
|
||||
|
|
||||
v
|
||||
+--------+----------+
|
||||
| Generate |
|
||||
| JWT Tokens |
|
||||
| - Access (15m) |
|
||||
| - Refresh (7d) |
|
||||
+--------+----------+
|
||||
|
|
||||
v
|
||||
+-------------------+ +--------+----------+
|
||||
| Client Storage |<----+ Return Tokens |
|
||||
| - Access: Memory| | - Access: Body |
|
||||
| - Refresh: HTTP | | - Refresh: Cookie|
|
||||
| Only Cookie | +-------------------+
|
||||
+-------------------+
|
||||
```
|
||||
|
||||
### Authentication Methods
|
||||
|
||||
1. **Local Authentication**: Email/password with bcrypt hashing
|
||||
2. **Google OAuth 2.0**: Social login via Google account
|
||||
3. **GitHub OAuth 2.0**: Social login via GitHub account
|
||||
|
||||
### Security Features (ADR-016, ADR-048)
|
||||
|
||||
- **Rate Limiting**: Login attempts rate-limited per IP
|
||||
- **Account Lockout**: 15-minute lockout after 5 failed attempts
|
||||
- **Password Requirements**: Strength validation via zxcvbn
|
||||
- **JWT Rotation**: Access tokens are short-lived, refresh tokens are rotated
|
||||
- **HTTPS Only**: All production traffic encrypted
|
||||
|
||||
### Protected Route Flow
|
||||
|
||||
```
|
||||
+-------------------+ +-------------------+ +-------------------+
|
||||
| API Request | | requireAuth | | JWT Strategy |
|
||||
| + Bearer Token +---->+ Middleware +---->+ Validate |
|
||||
+-------------------+ +--------+----------+ +--------+----------+
|
||||
| |
|
||||
| +-------------------+
|
||||
| |
|
||||
v v
|
||||
+--------+-----+----+
|
||||
| req.user |
|
||||
| populated |
|
||||
+--------+----------+
|
||||
|
|
||||
v
|
||||
+--------+----------+
|
||||
| Route Handler |
|
||||
| Executes |
|
||||
+-------------------+
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Background Processing
|
||||
|
||||
### Worker Architecture
|
||||
|
||||
```
|
||||
+-------------------+ +-------------------+ +-------------------+
|
||||
| API Server | | Redis | | Worker Process |
|
||||
| (Queue Producer)| | (Job Storage) | | (Consumer) |
|
||||
+--------+----------+ +--------+----------+ +--------+----------+
|
||||
| ^ |
|
||||
| Add Job | Poll/Process |
|
||||
+------------------------>+<------------------------+
|
||||
|
|
||||
|
|
||||
+-------------------------+-------------------------+
|
||||
| | |
|
||||
v v v
|
||||
+--------+----------+ +--------+----------+ +--------+----------+
|
||||
| Flyer Worker | | Email Worker | | Analytics |
|
||||
| Concurrency: 1 | | Concurrency: 10 | | Worker |
|
||||
+-------------------+ +-------------------+ | Concurrency: 1 |
|
||||
+-------------------+
|
||||
```
|
||||
|
||||
### Job Lifecycle
|
||||
|
||||
1. **Queued**: Job added to queue with data payload
|
||||
2. **Active**: Worker picks up job and begins processing
|
||||
3. **Completed**: Job finishes successfully
|
||||
4. **Failed**: Job encounters error, may retry
|
||||
5. **Delayed**: Job waiting for retry backoff
|
||||
|
||||
### Retry Strategy
|
||||
|
||||
Jobs use exponential backoff for retries:
|
||||
|
||||
```
|
||||
Attempt 1: Immediate
|
||||
Attempt 2: Initial delay (e.g., 5 seconds)
|
||||
Attempt 3: 2x delay (e.g., 10 seconds)
|
||||
Attempt 4: 4x delay (e.g., 20 seconds)
|
||||
...
|
||||
```
|
||||
|
||||
### Scheduled Jobs (ADR-037)
|
||||
|
||||
| Schedule | Job | Purpose |
|
||||
| --------------------- | ---------------- | ------------------------------------------ |
|
||||
| Daily 2:00 AM | Analytics Report | Generate daily usage statistics |
|
||||
| Weekly Sunday 3:00 AM | Weekly Analytics | Generate weekly summary reports |
|
||||
| Every 6 hours | Token Cleanup | Remove expired refresh tokens |
|
||||
| Every hour | Expiry Alerts | Check and send pantry expiry notifications |
|
||||
|
||||
---
|
||||
|
||||
## Deployment Architecture
|
||||
|
||||
### Environment Overview
|
||||
|
||||
```
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| DEVELOPMENT |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| |
|
||||
| +-----------------------------------+ +-----------------------------------+ |
|
||||
| | Windows Host Machine | | Linux Dev Container | |
|
||||
| | - VS Code | | (flyer-crawler-dev) | |
|
||||
| | - Podman Desktop +---->+ - Node.js 22 | |
|
||||
| | - Git | | - PM2 (process manager) | |
|
||||
| +-----------------------------------+ | - PostgreSQL 16 | |
|
||||
| | - Redis 7 | |
|
||||
| | - Bugsink (local) | |
|
||||
| | - Logstash (log aggregation) | |
|
||||
| +-----------------------------------+ |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| TEST SERVER |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| |
|
||||
| +-----------------------------------+ +-----------------------------------+ |
|
||||
| | NGINX Reverse Proxy | | Application Server | |
|
||||
| | flyer-crawler-test.projectium.com | - PM2 Process Manager | |
|
||||
| | - SSL/TLS (Let's Encrypt) +---->+ - Node.js 22 | |
|
||||
| | - Rate Limiting | | - PostgreSQL 16 | |
|
||||
| +-----------------------------------+ | - Redis 7 | |
|
||||
| +-----------------------------------+ |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| PRODUCTION |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
| |
|
||||
| +-----------------------------------+ +-----------------------------------+ |
|
||||
| | NGINX Reverse Proxy | | Application Server | |
|
||||
| | flyer-crawler.projectium.com | - PM2 Process Manager | |
|
||||
| | - SSL/TLS (Let's Encrypt) +---->+ - Node.js 22 (Clustered) | |
|
||||
| | - Rate Limiting | | - PostgreSQL 16 | |
|
||||
| | - Gzip Compression | | - Redis 7 | |
|
||||
| +-----------------------------------+ +-----------------------------------+ |
|
||||
| |
|
||||
| +-----------------------------------+ |
|
||||
| | Monitoring | |
|
||||
| | - Bugsink (Error Tracking) | |
|
||||
| | - Logstash (Log Aggregation) | |
|
||||
| +-----------------------------------+ |
|
||||
+-----------------------------------------------------------------------------------+
|
||||
```
|
||||
|
||||
### Deployment Pipeline (ADR-017)
|
||||
|
||||
```
|
||||
+------------+ +------------+ +------------+ +------------+
|
||||
| Push to | | Gitea | | Build & | | Deploy |
|
||||
| main +---->+ Actions +---->+ Test +---->+ to Prod |
|
||||
+------------+ +------------+ +------------+ +------------+
|
||||
|
|
||||
v
|
||||
+------+------+
|
||||
| Type |
|
||||
| Check |
|
||||
+------+------+
|
||||
|
|
||||
v
|
||||
+------+------+
|
||||
| Unit |
|
||||
| Tests |
|
||||
+------+------+
|
||||
|
|
||||
v
|
||||
+------+------+
|
||||
| Build |
|
||||
| Assets |
|
||||
+-------------+
|
||||
```
|
||||
|
||||
### Server Paths
|
||||
|
||||
| Environment | Web Root | Data Storage | Flyer Images |
|
||||
| ----------- | --------------------------------------------- | ----------------------------------------------------- | ---------------------------------------------------------- |
|
||||
| Production | `/var/www/flyer-crawler.projectium.com/` | `/var/www/flyer-crawler.projectium.com/uploads/` | `/var/www/flyer-crawler.projectium.com/flyer-images/` |
|
||||
| Test | `/var/www/flyer-crawler-test.projectium.com/` | `/var/www/flyer-crawler-test.projectium.com/uploads/` | `/var/www/flyer-crawler-test.projectium.com/flyer-images/` |
|
||||
| Development | Container-local | Container-local | `/app/public/flyer-images/` |
|
||||
|
||||
Flyer images are served by NGINX as static files at `/flyer-images/` with 7-day browser caching.
|
||||
|
||||
---
|
||||
|
||||
## Design Principles and ADRs
|
||||
|
||||
The system architecture is governed by Architecture Decision Records (ADRs). Key decisions include:
|
||||
|
||||
### Core Infrastructure
|
||||
|
||||
| ADR | Title | Status |
|
||||
| ------- | ------------------------------------ | -------- |
|
||||
| ADR-001 | Standardized Error Handling | Accepted |
|
||||
| ADR-002 | Standardized Transaction Management | Accepted |
|
||||
| ADR-007 | Configuration and Secrets Management | Accepted |
|
||||
| ADR-020 | Health Checks and Probes | Accepted |
|
||||
|
||||
### API and Integration
|
||||
|
||||
| ADR | Title | Status |
|
||||
| ------- | ----------------------------- | ----------- |
|
||||
| ADR-003 | Standardized Input Validation | Accepted |
|
||||
| ADR-022 | Real-time Notification System | Proposed |
|
||||
| ADR-028 | API Response Standardization | Implemented |
|
||||
|
||||
### Security
|
||||
|
||||
| ADR | Title | Status |
|
||||
| ------- | ----------------------- | --------------------- |
|
||||
| ADR-016 | API Security Hardening | Accepted |
|
||||
| ADR-032 | Rate Limiting Strategy | Accepted |
|
||||
| ADR-048 | Authentication Strategy | Partially Implemented |
|
||||
|
||||
### Architecture Patterns
|
||||
|
||||
| ADR | Title | Status |
|
||||
| ------- | ---------------------------------- | -------- |
|
||||
| ADR-034 | Repository Pattern Standards | Accepted |
|
||||
| ADR-035 | Service Layer Architecture | Accepted |
|
||||
| ADR-036 | Event Bus and Pub/Sub Pattern | Accepted |
|
||||
| ADR-041 | AI/Gemini Integration Architecture | Accepted |
|
||||
|
||||
### Operations
|
||||
|
||||
| ADR | Title | Status |
|
||||
| ------- | ------------------------------- | --------------------- |
|
||||
| ADR-006 | Background Job Processing | Accepted |
|
||||
| ADR-014 | Containerization and Deployment | Partially Implemented |
|
||||
| ADR-037 | Scheduled Jobs and Cron Pattern | Accepted |
|
||||
| ADR-038 | Graceful Shutdown Pattern | Accepted |
|
||||
|
||||
### Observability
|
||||
|
||||
| ADR | Title | Status |
|
||||
| ------- | --------------------------------- | -------- |
|
||||
| ADR-004 | Structured Logging | Accepted |
|
||||
| ADR-015 | APM and Error Tracking | Proposed |
|
||||
| ADR-050 | PostgreSQL Function Observability | Accepted |
|
||||
|
||||
**Full ADR Index**: [docs/adr/index.md](../adr/index.md)
|
||||
|
||||
---
|
||||
|
||||
## Key Files Reference
|
||||
|
||||
### Configuration Files
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------ | ------------------------------------------------------ |
|
||||
| `server.ts` | Express application setup and middleware configuration |
|
||||
| `src/config/env.ts` | Environment variable validation (Zod schema) |
|
||||
| `src/config/passport.ts` | Authentication strategies (Local, JWT, OAuth) |
|
||||
| `ecosystem.config.cjs` | PM2 process manager configuration |
|
||||
| `vite.config.ts` | Vite build and dev server configuration |
|
||||
|
||||
### Route Files
|
||||
|
||||
| File | API Prefix |
|
||||
| ----------------------------- | -------------- |
|
||||
| `src/routes/auth.routes.ts` | `/api/auth` |
|
||||
| `src/routes/user.routes.ts` | `/api/users` |
|
||||
| `src/routes/flyer.routes.ts` | `/api/flyers` |
|
||||
| `src/routes/recipe.routes.ts` | `/api/recipes` |
|
||||
| `src/routes/deals.routes.ts` | `/api/deals` |
|
||||
| `src/routes/store.routes.ts` | `/api/stores` |
|
||||
| `src/routes/admin.routes.ts` | `/api/admin` |
|
||||
| `src/routes/health.routes.ts` | `/api/health` |
|
||||
|
||||
### Service Files
|
||||
|
||||
| File | Purpose |
|
||||
| ----------------------------------------------- | --------------------------------------- |
|
||||
| `src/services/flyerProcessingService.server.ts` | Flyer processing pipeline orchestration |
|
||||
| `src/services/aiService.server.ts` | Google Gemini AI integration |
|
||||
| `src/services/cacheService.server.ts` | Redis caching abstraction |
|
||||
| `src/services/emailService.server.ts` | Email sending |
|
||||
| `src/services/queues.server.ts` | BullMQ queue definitions |
|
||||
| `src/services/workers.server.ts` | BullMQ worker definitions |
|
||||
|
||||
### Database Files
|
||||
|
||||
| File | Purpose |
|
||||
| ---------------------------------- | -------------------------------------------- |
|
||||
| `src/services/db/connection.db.ts` | Database pool and transaction management |
|
||||
| `src/services/db/errors.db.ts` | Database error types |
|
||||
| `src/services/db/user.db.ts` | User repository |
|
||||
| `src/services/db/flyer.db.ts` | Flyer repository |
|
||||
| `sql/master_schema_rollup.sql` | Complete database schema (for test DB setup) |
|
||||
| `sql/initial_schema.sql` | Fresh installation schema |
|
||||
|
||||
### Type Definitions
|
||||
|
||||
| File | Purpose |
|
||||
| ----------------------- | ---------------------------- |
|
||||
| `src/types.ts` | Core entity type definitions |
|
||||
| `src/types/job-data.ts` | BullMQ job payload types |
|
||||
|
||||
---
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- **API Documentation**: Available at `/docs/api-docs` in development environments
|
||||
- **Testing Guide**: [docs/tests/](../tests/)
|
||||
- **Getting Started**: [docs/getting-started/](../getting-started/)
|
||||
- **Operations Guide**: [docs/operations/](../operations/)
|
||||
- **Authentication Details**: [docs/architecture/AUTHENTICATION.md](./AUTHENTICATION.md)
|
||||
- **Database Schema**: [docs/architecture/DATABASE.md](./DATABASE.md)
|
||||
- **WebSocket Usage**: [docs/architecture/WEBSOCKET_USAGE.md](./WEBSOCKET_USAGE.md)
|
||||
|
||||
---
|
||||
|
||||
_This document is maintained as part of the Flyer Crawler project documentation. For updates, contact the development team or submit a pull request._
|
||||
534
docs/archive/sessions/TESTING_SESSION_2026-01-21.md
Normal file
534
docs/archive/sessions/TESTING_SESSION_2026-01-21.md
Normal file
@@ -0,0 +1,534 @@
|
||||
# Testing Session - UI/UX Improvements
|
||||
|
||||
**Date**: 2026-01-21
|
||||
**Tester**: [Your Name]
|
||||
**Session Start**: [Time]
|
||||
**Environment**: Dev Container
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Session Objective
|
||||
|
||||
Test all 4 critical UI/UX improvements:
|
||||
|
||||
1. Brand Colors (visual verification)
|
||||
2. Button Component (functional testing)
|
||||
3. Onboarding Tour (flow testing)
|
||||
4. Mobile Navigation (responsive testing)
|
||||
|
||||
---
|
||||
|
||||
## ✅ Pre-Test Setup Checklist
|
||||
|
||||
### 1. Dev Server Status
|
||||
|
||||
- [ ] Dev server running at `http://localhost:5173`
|
||||
- [ ] Browser open (Chrome/Edge recommended)
|
||||
- [ ] DevTools open (F12)
|
||||
|
||||
**Command to start**:
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev npm run dev:container
|
||||
```
|
||||
|
||||
**Server Status**: [ ] Running [ ] Not Running
|
||||
|
||||
---
|
||||
|
||||
### 2. Browser Setup
|
||||
|
||||
- [ ] Clear cache (Ctrl+Shift+Delete)
|
||||
- [ ] Clear localStorage for localhost
|
||||
- [ ] Enable responsive design mode (Ctrl+Shift+M)
|
||||
|
||||
**Browser Version**: **\*\*\*\***\_**\*\*\*\***
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Test Execution
|
||||
|
||||
### TEST 1: Onboarding Tour ⭐ CRITICAL
|
||||
|
||||
**Priority**: 🔴 Must Pass
|
||||
**Time**: 5 minutes
|
||||
|
||||
#### Steps:
|
||||
|
||||
1. Open DevTools → Application → Local Storage
|
||||
2. Delete key: `flyer_crawler_onboarding_completed`
|
||||
3. Refresh page (F5)
|
||||
4. Observe if tour appears
|
||||
|
||||
#### Expected:
|
||||
|
||||
- ✅ Tour modal appears within 2 seconds
|
||||
- ✅ Shows "Step 1 of 6"
|
||||
- ✅ Points to Flyer Uploader section
|
||||
- ✅ Skip button visible
|
||||
- ✅ Next button visible
|
||||
|
||||
#### Actual Result:
|
||||
|
||||
```
|
||||
[Record what you see here]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
```
|
||||
|
||||
**Status**: [ ] ✅ PASS [ ] ❌ FAIL [ ] ⚠️ PARTIAL
|
||||
|
||||
**Screenshots**: [Attach if needed]
|
||||
|
||||
---
|
||||
|
||||
### TEST 2: Tour Navigation
|
||||
|
||||
**Time**: 5 minutes
|
||||
|
||||
#### Steps:
|
||||
|
||||
Click "Next" button 6 times, observe each step
|
||||
|
||||
#### Verification Table:
|
||||
|
||||
| Step | Target | Visible? | Correct Text? | Notes |
|
||||
| ---- | -------------- | -------- | ------------- | ----- |
|
||||
| 1 | Flyer Uploader | [ ] | [ ] | |
|
||||
| 2 | Data Table | [ ] | [ ] | |
|
||||
| 3 | Watch Button | [ ] | [ ] | |
|
||||
| 4 | Watchlist | [ ] | [ ] | |
|
||||
| 5 | Price Chart | [ ] | [ ] | |
|
||||
| 6 | Shopping List | [ ] | [ ] | |
|
||||
|
||||
#### Additional Checks:
|
||||
|
||||
- [ ] Progress indicator updates (1/6 → 6/6)
|
||||
- [ ] Can click "Previous" button
|
||||
- [ ] Tour closes after step 6
|
||||
- [ ] localStorage key saved
|
||||
|
||||
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
|
||||
|
||||
---
|
||||
|
||||
### TEST 3: Mobile Tab Bar ⭐ CRITICAL
|
||||
|
||||
**Priority**: 🔴 Must Pass
|
||||
**Time**: 8 minutes
|
||||
|
||||
#### Part A: Mobile View (375px)
|
||||
|
||||
**Setup**: Toggle device toolbar → iPhone SE
|
||||
|
||||
#### Checks:
|
||||
|
||||
- [ ] Bottom tab bar visible
|
||||
- [ ] 4 tabs present: Home, Deals, Lists, Profile
|
||||
- [ ] Left sidebar (flyer list) HIDDEN
|
||||
- [ ] Right sidebar (widgets) HIDDEN
|
||||
- [ ] Main content uses full width
|
||||
|
||||
**Visual Check**:
|
||||
|
||||
```
|
||||
Tab Bar Position: [ ] Bottom [ ] Other: _______
|
||||
Number of Tabs: _______
|
||||
Tab Bar Height: ~64px? [ ] Yes [ ] No
|
||||
```
|
||||
|
||||
#### Part B: Tab Navigation
|
||||
|
||||
Click each tab and verify:
|
||||
|
||||
| Tab | URL | Page Loads? | Highlights? | Content Correct? |
|
||||
| ------- | ---------- | ----------- | ----------- | ---------------- |
|
||||
| Home | `/` | [ ] | [ ] | [ ] |
|
||||
| Deals | `/deals` | [ ] | [ ] | [ ] |
|
||||
| Lists | `/lists` | [ ] | [ ] | [ ] |
|
||||
| Profile | `/profile` | [ ] | [ ] | [ ] |
|
||||
|
||||
#### Part C: Desktop View (1440px)
|
||||
|
||||
**Setup**: Exit device mode, maximize window
|
||||
|
||||
#### Checks:
|
||||
|
||||
- [ ] Tab bar HIDDEN (not visible)
|
||||
- [ ] Left sidebar VISIBLE
|
||||
- [ ] Right sidebar VISIBLE
|
||||
- [ ] 3-column layout intact
|
||||
- [ ] No layout regressions
|
||||
|
||||
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
|
||||
|
||||
---
|
||||
|
||||
### TEST 4: Dark Mode ⭐ CRITICAL
|
||||
|
||||
**Priority**: 🔴 Must Pass
|
||||
**Time**: 5 minutes
|
||||
|
||||
#### Steps:
|
||||
|
||||
1. Click dark mode toggle in header
|
||||
2. Navigate: Home → Deals → Lists → Profile
|
||||
3. Observe colors and contrast
|
||||
|
||||
#### Visual Verification:
|
||||
|
||||
**Mobile Tab Bar**:
|
||||
|
||||
- [ ] Dark background (#111827 or similar)
|
||||
- [ ] Dark border color
|
||||
- [ ] Active tab: teal (#14b8a6)
|
||||
- [ ] Inactive tabs: gray
|
||||
|
||||
**New Pages**:
|
||||
|
||||
- [ ] DealsPage: dark background, light text
|
||||
- [ ] ShoppingListsPage: dark cards
|
||||
- [ ] FlyersPage: dark theme
|
||||
- [ ] No white boxes visible
|
||||
|
||||
**Button Component**:
|
||||
|
||||
- [ ] Primary buttons: teal background
|
||||
- [ ] Secondary buttons: gray background
|
||||
- [ ] Danger buttons: red background
|
||||
- [ ] All text readable
|
||||
|
||||
#### Toggle Back:
|
||||
|
||||
- [ ] Light mode restores correctly
|
||||
- [ ] No stuck dark elements
|
||||
|
||||
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
|
||||
|
||||
---
|
||||
|
||||
### TEST 5: Brand Colors Visual Check
|
||||
|
||||
**Time**: 3 minutes
|
||||
|
||||
#### Verification:
|
||||
|
||||
Navigate through app and check teal color consistency:
|
||||
|
||||
- [ ] Active tab: teal
|
||||
- [ ] Primary buttons: teal
|
||||
- [ ] Links on hover: teal
|
||||
- [ ] Focus rings: teal
|
||||
- [ ] All teal shades match (#14b8a6)
|
||||
|
||||
**Color Picker Check** (optional):
|
||||
Use DevTools color picker on active tab:
|
||||
|
||||
- Expected: `#14b8a6` or `rgb(20, 184, 166)`
|
||||
- Actual: **\*\*\*\***\_\_\_**\*\*\*\***
|
||||
|
||||
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
|
||||
|
||||
---
|
||||
|
||||
### TEST 6: Button Component
|
||||
|
||||
**Time**: 5 minutes
|
||||
|
||||
#### Find and Test Buttons:
|
||||
|
||||
**FlyerUploader Page**:
|
||||
|
||||
- [ ] "Upload Another Flyer" button (primary, teal)
|
||||
- [ ] Button clickable
|
||||
- [ ] Hover effect works
|
||||
- [ ] Loading state (if applicable)
|
||||
|
||||
**ShoppingList Page** (navigate to /lists):
|
||||
|
||||
- [ ] "New List" button (secondary, gray)
|
||||
- [ ] "Delete List" button (danger, red)
|
||||
- [ ] Buttons functional
|
||||
- [ ] Hover states work
|
||||
|
||||
**In Dark Mode**:
|
||||
|
||||
- [ ] All button variants visible
|
||||
- [ ] Good contrast
|
||||
- [ ] No white backgrounds
|
||||
|
||||
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
|
||||
|
||||
---
|
||||
|
||||
### TEST 7: Responsive Breakpoints
|
||||
|
||||
**Time**: 5 minutes
|
||||
|
||||
#### Test at each width:
|
||||
|
||||
**375px (Mobile)**:
|
||||
|
||||
```
|
||||
Tab bar: [ ] Visible [ ] Hidden
|
||||
Sidebars: [ ] Visible [ ] Hidden
|
||||
Layout: [ ] Single column [ ] Multi-column
|
||||
```
|
||||
|
||||
**768px (Tablet)**:
|
||||
|
||||
```
|
||||
Tab bar: [ ] Visible [ ] Hidden
|
||||
Sidebars: [ ] Visible [ ] Hidden
|
||||
Layout: [ ] Single column [ ] Multi-column
|
||||
```
|
||||
|
||||
**1024px (Desktop)**:
|
||||
|
||||
```
|
||||
Tab bar: [ ] Visible [ ] Hidden
|
||||
Sidebars: [ ] Visible [ ] Hidden
|
||||
Layout: [ ] Single column [ ] Multi-column
|
||||
```
|
||||
|
||||
**1440px (Large Desktop)**:
|
||||
|
||||
```
|
||||
Layout: [ ] Unchanged [ ] Broken
|
||||
All elements: [ ] Visible [ ] Hidden/Cut off
|
||||
```
|
||||
|
||||
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
|
||||
|
||||
---
|
||||
|
||||
### TEST 8: Admin Routes (If Admin User)
|
||||
|
||||
**Time**: 3 minutes
|
||||
**Skip if**: [ ] Not admin user
|
||||
|
||||
#### Steps:
|
||||
|
||||
1. Log in as admin
|
||||
2. Navigate to `/admin`
|
||||
3. Check for tab bar
|
||||
|
||||
#### Checks:
|
||||
|
||||
- [ ] Admin dashboard loads
|
||||
- [ ] Tab bar NOT visible
|
||||
- [ ] Layout looks correct
|
||||
- [ ] Can navigate to subpages
|
||||
- [ ] Subpages work in mobile view
|
||||
|
||||
**Status**: [ ] ✅ PASS [ ] ❌ FAIL [ ] ⏭️ SKIPPED
|
||||
|
||||
---
|
||||
|
||||
### TEST 9: Console Errors
|
||||
|
||||
**Time**: 2 minutes
|
||||
|
||||
#### Steps:
|
||||
|
||||
1. Open Console tab in DevTools
|
||||
2. Clear console
|
||||
3. Navigate through app: Home → Deals → Lists → Profile → Home
|
||||
4. Check for red error messages
|
||||
|
||||
#### Results:
|
||||
|
||||
```
|
||||
Errors Found: [ ] None [ ] Some (list below)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
```
|
||||
|
||||
**React 19 warnings are OK** (peer dependencies)
|
||||
|
||||
**Status**: [ ] ✅ PASS (no errors) [ ] ❌ FAIL (errors present)
|
||||
|
||||
---
|
||||
|
||||
### TEST 10: Integration Flow
|
||||
|
||||
**Time**: 5 minutes
|
||||
|
||||
#### User Journey:
|
||||
|
||||
1. Start on Home page (mobile view)
|
||||
2. Navigate to Deals tab
|
||||
3. Navigate to Lists tab
|
||||
4. Navigate to Profile tab
|
||||
5. Navigate back to Home
|
||||
6. Toggle dark mode
|
||||
7. Navigate through tabs again
|
||||
|
||||
#### Checks:
|
||||
|
||||
- [ ] All navigation smooth
|
||||
- [ ] No data loss
|
||||
- [ ] Active tab always correct
|
||||
- [ ] Browser back button works
|
||||
- [ ] Dark mode persists across routes
|
||||
- [ ] No JavaScript errors
|
||||
- [ ] No layout shifting
|
||||
|
||||
**Status**: [ ] ✅ PASS [ ] ❌ FAIL
|
||||
|
||||
---
|
||||
|
||||
## 📊 Test Results Summary
|
||||
|
||||
### Critical Tests Status
|
||||
|
||||
| Test | Status | Priority | Notes |
|
||||
| ------------------- | ------ | ----------- | ----- |
|
||||
| 1. Onboarding Tour | [ ] | 🔴 Critical | |
|
||||
| 2. Tour Navigation | [ ] | 🟡 High | |
|
||||
| 3. Mobile Tab Bar | [ ] | 🔴 Critical | |
|
||||
| 4. Dark Mode | [ ] | 🔴 Critical | |
|
||||
| 5. Brand Colors | [ ] | 🟡 High | |
|
||||
| 6. Button Component | [ ] | 🟢 Medium | |
|
||||
| 7. Responsive | [ ] | 🔴 Critical | |
|
||||
| 8. Admin Routes | [ ] | 🟢 Medium | |
|
||||
| 9. Console Errors | [ ] | 🔴 Critical | |
|
||||
| 10. Integration | [ ] | 🟡 High | |
|
||||
|
||||
**Pass Rate**: **\_** / 10 tests passed
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Issues Found
|
||||
|
||||
### Critical Issues (Blockers)
|
||||
|
||||
1. ***
|
||||
2. ***
|
||||
3. ***
|
||||
|
||||
### High Priority Issues
|
||||
|
||||
1. ***
|
||||
2. ***
|
||||
3. ***
|
||||
|
||||
### Medium/Low Priority Issues
|
||||
|
||||
1. ***
|
||||
2. ***
|
||||
3. ***
|
||||
|
||||
---
|
||||
|
||||
## 📸 Screenshots
|
||||
|
||||
Attach screenshots for:
|
||||
|
||||
- [ ] Onboarding tour (step 1)
|
||||
- [ ] Mobile tab bar (375px)
|
||||
- [ ] Desktop layout (1440px)
|
||||
- [ ] Dark mode (tab bar)
|
||||
- [ ] Any bugs/issues found
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Final Decision
|
||||
|
||||
### Must-Pass Criteria
|
||||
|
||||
**Critical tests** (all must pass):
|
||||
|
||||
- [ ] Test 1: Onboarding Tour
|
||||
- [ ] Test 3: Mobile Tab Bar
|
||||
- [ ] Test 4: Dark Mode
|
||||
- [ ] Test 7: Responsive
|
||||
- [ ] Test 9: No Console Errors
|
||||
|
||||
**Result**: [ ] ALL CRITICAL PASS [ ] SOME FAIL
|
||||
|
||||
---
|
||||
|
||||
### Production Readiness
|
||||
|
||||
**Overall Assessment**:
|
||||
[ ] ✅ READY FOR PRODUCTION
|
||||
[ ] ⚠️ READY WITH MINOR ISSUES
|
||||
[ ] ❌ NOT READY (critical issues)
|
||||
|
||||
**Blocking Issues** (must fix before deploy):
|
||||
|
||||
1. ***
|
||||
2. ***
|
||||
3. ***
|
||||
|
||||
**Recommended Fixes** (can deploy, fix later):
|
||||
|
||||
1. ***
|
||||
2. ***
|
||||
3. ***
|
||||
|
||||
---
|
||||
|
||||
## 🔐 Sign-Off
|
||||
|
||||
**Tester Name**: ******\*\*\*\*******\_\_\_******\*\*\*\*******
|
||||
|
||||
**Date/Time Completed**: ****\*\*\*\*****\_\_\_****\*\*\*\*****
|
||||
|
||||
**Total Testing Time**: **\_\_** minutes
|
||||
|
||||
**Recommended Action**:
|
||||
[ ] Deploy to production
|
||||
[ ] Deploy to staging first
|
||||
[ ] Fix issues, re-test
|
||||
[ ] Hold deployment
|
||||
|
||||
**Additional Notes**:
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
## 📋 Next Steps
|
||||
|
||||
**If PASS**:
|
||||
|
||||
1. [ ] Create commit with test results
|
||||
2. [ ] Update CHANGELOG.md
|
||||
3. [ ] Tag release (v0.12.4)
|
||||
4. [ ] Deploy to staging
|
||||
5. [ ] Monitor for 24 hours
|
||||
6. [ ] Deploy to production
|
||||
|
||||
**If FAIL**:
|
||||
|
||||
1. [ ] Log issues in GitHub/Gitea
|
||||
2. [ ] Assign to developer
|
||||
3. [ ] Schedule re-test
|
||||
4. [ ] Update test plan if needed
|
||||
|
||||
---
|
||||
|
||||
**Session End**: [Time]
|
||||
**Session Duration**: **\_\_** minutes
|
||||
@@ -135,7 +135,7 @@ New users saw "Welcome to Flyer Crawler!" with no explanation of features or how
|
||||
|
||||
### Solution
|
||||
|
||||
Implemented interactive guided tour using `react-joyride`:
|
||||
Implemented interactive guided tour using `driver.js` (framework-agnostic, React 19 compatible):
|
||||
|
||||
**Tour Steps** (6 total):
|
||||
|
||||
@@ -148,24 +148,27 @@ Implemented interactive guided tour using `react-joyride`:
|
||||
|
||||
**Features**:
|
||||
|
||||
- Auto-starts for first-time users
|
||||
- Auto-starts for first-time users (500ms delay for DOM readiness)
|
||||
- Persists completion in localStorage (`flyer_crawler_onboarding_completed`)
|
||||
- Skip button for experienced users
|
||||
- Progress indicator showing current step
|
||||
- Styled with brand colors (#14b8a6)
|
||||
- Custom styled with pastel colors, sharp borders (design system)
|
||||
- Dark mode compatible
|
||||
- Zero React peer dependencies (compatible with React 19)
|
||||
|
||||
### Deliverables
|
||||
|
||||
- **Created**: `src/hooks/useOnboardingTour.ts` (custom hook)
|
||||
- **Created**: `src/hooks/useOnboardingTour.ts` (custom hook with Driver.js)
|
||||
- **Modified**: Added `data-tour` attributes to 6 components:
|
||||
- `src/features/flyer/FlyerUploader.tsx`
|
||||
- `src/features/flyer/ExtractedDataTable.tsx`
|
||||
- `src/features/shopping/WatchedItemsList.tsx`
|
||||
- `src/features/charts/PriceChart.tsx`
|
||||
- `src/features/shopping/ShoppingList.tsx`
|
||||
- **Modified**: `src/layouts/MainLayout.tsx` - Integrated Joyride component
|
||||
- **Installed**: `react-joyride@2.9.3`, `@types/react-joyride@2.0.2`
|
||||
- **Modified**: `src/layouts/MainLayout.tsx` - Integrated tour via hook
|
||||
- **Installed**: `driver.js@^1.3.1`
|
||||
|
||||
**Migration Note (2026-01-21)**: Originally implemented with `react-joyride@2.9.3`, but migrated to `driver.js` for React 19 compatibility.
|
||||
|
||||
### User Flow
|
||||
|
||||
@@ -360,13 +363,13 @@ npm test -- --run src/components/Button.test.tsx
|
||||
|
||||
1. `tailwind.config.js` - Brand colors
|
||||
2. `src/App.tsx` - New routes, MobileTabBar
|
||||
3. `src/layouts/MainLayout.tsx` - Joyride, responsive layout
|
||||
3. `src/layouts/MainLayout.tsx` - Tour integration, responsive layout
|
||||
4. `src/features/flyer/FlyerUploader.tsx` - Button, data-tour
|
||||
5. `src/features/flyer/ExtractedDataTable.tsx` - data-tour
|
||||
6. `src/features/shopping/WatchedItemsList.tsx` - Button, data-tour
|
||||
7. `src/features/shopping/ShoppingList.tsx` - Button, data-tour
|
||||
8. `src/features/charts/PriceChart.tsx` - data-tour
|
||||
9. `package.json` - Dependencies (react-joyride)
|
||||
9. `package.json` - Dependencies (driver.js)
|
||||
10. `package-lock.json` - Dependency lock
|
||||
|
||||
### Statistics
|
||||
@@ -383,7 +386,7 @@ npm test -- --run src/components/Button.test.tsx
|
||||
|
||||
### Bundle Size Impact
|
||||
|
||||
- `react-joyride`: ~30KB gzipped
|
||||
- `driver.js`: ~10KB gzipped (lightweight, zero dependencies)
|
||||
- `Button` component: <5KB (reduces duplication)
|
||||
- Brand colors: 0KB (CSS utilities, tree-shaken)
|
||||
- **Total increase**: ~25KB gzipped
|
||||
@@ -461,8 +464,9 @@ If issues arise:
|
||||
|
||||
1. Revert commit containing `src/components/MobileTabBar.tsx`
|
||||
2. Remove new routes from `src/App.tsx`
|
||||
3. Restore previous `MainLayout.tsx` (remove Joyride)
|
||||
3. Restore previous `MainLayout.tsx` (remove tour integration)
|
||||
4. Keep Button component and brand colors (safe changes)
|
||||
5. Remove `driver.js` and restore localStorage keys if needed
|
||||
|
||||
---
|
||||
|
||||
478
docs/development/CODE-PATTERNS.md
Normal file
478
docs/development/CODE-PATTERNS.md
Normal file
@@ -0,0 +1,478 @@
|
||||
# Code Patterns
|
||||
|
||||
Common code patterns extracted from Architecture Decision Records (ADRs). Use these as templates when writing new code.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Error Handling](#error-handling)
|
||||
- [Repository Patterns](#repository-patterns)
|
||||
- [API Response Patterns](#api-response-patterns)
|
||||
- [Transaction Management](#transaction-management)
|
||||
- [Input Validation](#input-validation)
|
||||
- [Authentication](#authentication)
|
||||
- [Caching](#caching)
|
||||
- [Background Jobs](#background-jobs)
|
||||
|
||||
---
|
||||
|
||||
## Error Handling
|
||||
|
||||
**ADR**: [ADR-001](../adr/0001-standardized-error-handling-for-database-operations.md)
|
||||
|
||||
### Repository Layer Error Handling
|
||||
|
||||
```typescript
|
||||
import { handleDbError, NotFoundError } from '../services/db/errors.db';
|
||||
import { PoolClient } from 'pg';
|
||||
|
||||
export async function getFlyerById(id: number, client?: PoolClient): Promise<Flyer> {
|
||||
const db = client || pool;
|
||||
|
||||
try {
|
||||
const result = await db.query('SELECT * FROM flyers WHERE id = $1', [id]);
|
||||
|
||||
if (result.rows.length === 0) {
|
||||
throw new NotFoundError('Flyer', id);
|
||||
}
|
||||
|
||||
return result.rows[0];
|
||||
} catch (error) {
|
||||
throw handleDbError(error);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Route Layer Error Handling
|
||||
|
||||
```typescript
|
||||
import { sendError } from '../utils/apiResponse';
|
||||
|
||||
app.get('/api/flyers/:id', async (req, res) => {
|
||||
try {
|
||||
const flyer = await flyerDb.getFlyerById(parseInt(req.params.id));
|
||||
return sendSuccess(res, flyer);
|
||||
} catch (error) {
|
||||
return sendError(res, error);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### Custom Error Types
|
||||
|
||||
```typescript
|
||||
// NotFoundError - Entity not found
|
||||
throw new NotFoundError('Flyer', id);
|
||||
|
||||
// ValidationError - Invalid input
|
||||
throw new ValidationError('Invalid email format');
|
||||
|
||||
// DatabaseError - Database operation failed
|
||||
throw new DatabaseError('Failed to insert flyer', originalError);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Repository Patterns
|
||||
|
||||
**ADR**: [ADR-034](../adr/0034-repository-layer-method-naming-conventions.md)
|
||||
|
||||
### Method Naming Conventions
|
||||
|
||||
| Prefix | Returns | Not Found Behavior | Use Case |
|
||||
| ------- | -------------- | -------------------- | ------------------------- |
|
||||
| `get*` | Entity | Throws NotFoundError | When entity must exist |
|
||||
| `find*` | Entity \| null | Returns null | When entity may not exist |
|
||||
| `list*` | Array | Returns [] | When returning multiple |
|
||||
|
||||
### Get Method (Must Exist)
|
||||
|
||||
```typescript
|
||||
/**
|
||||
* Get a flyer by ID. Throws NotFoundError if not found.
|
||||
*/
|
||||
export async function getFlyerById(id: number, client?: PoolClient): Promise<Flyer> {
|
||||
const db = client || pool;
|
||||
|
||||
try {
|
||||
const result = await db.query('SELECT * FROM flyers WHERE id = $1', [id]);
|
||||
|
||||
if (result.rows.length === 0) {
|
||||
throw new NotFoundError('Flyer', id);
|
||||
}
|
||||
|
||||
return result.rows[0];
|
||||
} catch (error) {
|
||||
throw handleDbError(error);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Find Method (May Not Exist)
|
||||
|
||||
```typescript
|
||||
/**
|
||||
* Find a flyer by ID. Returns null if not found.
|
||||
*/
|
||||
export async function findFlyerById(id: number, client?: PoolClient): Promise<Flyer | null> {
|
||||
const db = client || pool;
|
||||
|
||||
try {
|
||||
const result = await db.query('SELECT * FROM flyers WHERE id = $1', [id]);
|
||||
|
||||
return result.rows[0] || null;
|
||||
} catch (error) {
|
||||
throw handleDbError(error);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### List Method (Multiple Results)
|
||||
|
||||
```typescript
|
||||
/**
|
||||
* List all active flyers. Returns empty array if none found.
|
||||
*/
|
||||
export async function listActiveFlyers(client?: PoolClient): Promise<Flyer[]> {
|
||||
const db = client || pool;
|
||||
|
||||
try {
|
||||
const result = await db.query(
|
||||
'SELECT * FROM flyers WHERE end_date >= CURRENT_DATE ORDER BY start_date DESC',
|
||||
);
|
||||
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
throw handleDbError(error);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## API Response Patterns
|
||||
|
||||
**ADR**: [ADR-028](../adr/0028-consistent-api-response-format.md)
|
||||
|
||||
### Success Response
|
||||
|
||||
```typescript
|
||||
import { sendSuccess } from '../utils/apiResponse';
|
||||
|
||||
app.post('/api/flyers', async (req, res) => {
|
||||
const flyer = await flyerService.createFlyer(req.body);
|
||||
return sendSuccess(res, flyer, 'Flyer created successfully', 201);
|
||||
});
|
||||
```
|
||||
|
||||
### Paginated Response
|
||||
|
||||
```typescript
|
||||
import { sendPaginated } from '../utils/apiResponse';
|
||||
|
||||
app.get('/api/flyers', async (req, res) => {
|
||||
const { page = 1, pageSize = 20 } = req.query;
|
||||
const { items, total } = await flyerService.listFlyers(page, pageSize);
|
||||
|
||||
return sendPaginated(res, {
|
||||
items,
|
||||
total,
|
||||
page: parseInt(page),
|
||||
pageSize: parseInt(pageSize),
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Error Response
|
||||
|
||||
```typescript
|
||||
import { sendError } from '../utils/apiResponse';
|
||||
|
||||
app.get('/api/flyers/:id', async (req, res) => {
|
||||
try {
|
||||
const flyer = await flyerDb.getFlyerById(parseInt(req.params.id));
|
||||
return sendSuccess(res, flyer);
|
||||
} catch (error) {
|
||||
return sendError(res, error); // Automatically maps error to correct status
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Transaction Management
|
||||
|
||||
**ADR**: [ADR-002](../adr/0002-transaction-management-pattern.md)
|
||||
|
||||
### Basic Transaction
|
||||
|
||||
```typescript
|
||||
import { withTransaction } from '../services/db/transaction.db';
|
||||
|
||||
export async function createFlyerWithItems(
|
||||
flyerData: FlyerInput,
|
||||
items: FlyerItemInput[],
|
||||
): Promise<Flyer> {
|
||||
return withTransaction(async (client) => {
|
||||
// Create flyer
|
||||
const flyer = await flyerDb.createFlyer(flyerData, client);
|
||||
|
||||
// Create items
|
||||
const createdItems = await flyerItemDb.createItems(
|
||||
items.map((item) => ({ ...item, flyer_id: flyer.id })),
|
||||
client,
|
||||
);
|
||||
|
||||
// Automatically commits on success, rolls back on error
|
||||
return { ...flyer, items: createdItems };
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### Nested Transactions
|
||||
|
||||
```typescript
|
||||
export async function bulkImportFlyers(flyersData: FlyerInput[]): Promise<ImportResult> {
|
||||
return withTransaction(async (client) => {
|
||||
const results = [];
|
||||
|
||||
for (const flyerData of flyersData) {
|
||||
try {
|
||||
// Each flyer import is atomic
|
||||
const flyer = await createFlyerWithItems(
|
||||
flyerData,
|
||||
flyerData.items,
|
||||
client, // Pass transaction client
|
||||
);
|
||||
results.push({ success: true, flyer });
|
||||
} catch (error) {
|
||||
results.push({ success: false, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Input Validation
|
||||
|
||||
**ADR**: [ADR-003](../adr/0003-input-validation-framework.md)
|
||||
|
||||
### Zod Schema Definition
|
||||
|
||||
```typescript
|
||||
// src/schemas/flyer.schemas.ts
|
||||
import { z } from 'zod';
|
||||
|
||||
export const createFlyerSchema = z.object({
|
||||
store_id: z.number().int().positive(),
|
||||
image_url: z
|
||||
.string()
|
||||
.url()
|
||||
.regex(/^https?:\/\/.*/),
|
||||
start_date: z.string().datetime(),
|
||||
end_date: z.string().datetime(),
|
||||
items: z
|
||||
.array(
|
||||
z.object({
|
||||
name: z.string().min(1).max(255),
|
||||
price: z.number().positive(),
|
||||
quantity: z.string().optional(),
|
||||
}),
|
||||
)
|
||||
.min(1),
|
||||
});
|
||||
|
||||
export type CreateFlyerInput = z.infer<typeof createFlyerSchema>;
|
||||
```
|
||||
|
||||
### Route Validation Middleware
|
||||
|
||||
```typescript
|
||||
import { validateRequest } from '../middleware/validation';
|
||||
import { createFlyerSchema } from '../schemas/flyer.schemas';
|
||||
|
||||
app.post('/api/flyers', validateRequest(createFlyerSchema), async (req, res) => {
|
||||
// req.body is now type-safe and validated
|
||||
const flyer = await flyerService.createFlyer(req.body);
|
||||
return sendSuccess(res, flyer, 'Flyer created successfully', 201);
|
||||
});
|
||||
```
|
||||
|
||||
### Manual Validation
|
||||
|
||||
```typescript
|
||||
import { createFlyerSchema } from '../schemas/flyer.schemas';
|
||||
|
||||
export async function processFlyer(data: unknown): Promise<Flyer> {
|
||||
// Validate and parse input
|
||||
const validated = createFlyerSchema.parse(data);
|
||||
|
||||
// Type-safe from here on
|
||||
return flyerDb.createFlyer(validated);
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Authentication
|
||||
|
||||
**ADR**: [ADR-048](../adr/0048-authentication-strategy.md)
|
||||
|
||||
### Protected Route with JWT
|
||||
|
||||
```typescript
|
||||
import { authenticateJWT } from '../middleware/auth';
|
||||
|
||||
app.get(
|
||||
'/api/profile',
|
||||
authenticateJWT, // Middleware adds req.user
|
||||
async (req, res) => {
|
||||
// req.user is guaranteed to exist
|
||||
const user = await userDb.getUserById(req.user.id);
|
||||
return sendSuccess(res, user);
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
### Optional Authentication
|
||||
|
||||
```typescript
|
||||
import { optionalAuth } from '../middleware/auth';
|
||||
|
||||
app.get(
|
||||
'/api/flyers',
|
||||
optionalAuth, // req.user may or may not exist
|
||||
async (req, res) => {
|
||||
const flyers = req.user
|
||||
? await flyerDb.listFlyersForUser(req.user.id)
|
||||
: await flyerDb.listPublicFlyers();
|
||||
|
||||
return sendSuccess(res, flyers);
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
### Generate JWT Token
|
||||
|
||||
```typescript
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { env } from '../config/env';
|
||||
|
||||
export function generateToken(user: User): string {
|
||||
return jwt.sign({ id: user.id, email: user.email }, env.JWT_SECRET, { expiresIn: '7d' });
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Caching
|
||||
|
||||
**ADR**: [ADR-029](../adr/0029-redis-caching-strategy.md)
|
||||
|
||||
### Cache Pattern
|
||||
|
||||
```typescript
|
||||
import { cacheService } from '../services/cache.server';
|
||||
|
||||
export async function getFlyer(id: number): Promise<Flyer> {
|
||||
// Try cache first
|
||||
const cached = await cacheService.get<Flyer>(`flyer:${id}`);
|
||||
if (cached) return cached;
|
||||
|
||||
// Cache miss - fetch from database
|
||||
const flyer = await flyerDb.getFlyerById(id);
|
||||
|
||||
// Store in cache (1 hour TTL)
|
||||
await cacheService.set(`flyer:${id}`, flyer, 3600);
|
||||
|
||||
return flyer;
|
||||
}
|
||||
```
|
||||
|
||||
### Cache Invalidation
|
||||
|
||||
```typescript
|
||||
export async function updateFlyer(id: number, data: UpdateFlyerInput): Promise<Flyer> {
|
||||
const flyer = await flyerDb.updateFlyer(id, data);
|
||||
|
||||
// Invalidate cache
|
||||
await cacheService.delete(`flyer:${id}`);
|
||||
await cacheService.invalidatePattern('flyers:list:*');
|
||||
|
||||
return flyer;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Background Jobs
|
||||
|
||||
**ADR**: [ADR-036](../adr/0036-background-job-processing-architecture.md)
|
||||
|
||||
### Queue Job
|
||||
|
||||
```typescript
|
||||
import { flyerProcessingQueue } from '../services/queues.server';
|
||||
|
||||
export async function enqueueFlyerProcessing(flyerId: number): Promise<void> {
|
||||
await flyerProcessingQueue.add(
|
||||
'process-flyer',
|
||||
{
|
||||
flyerId,
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
{
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 2000,
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
### Process Job
|
||||
|
||||
```typescript
|
||||
// src/services/workers.server.ts
|
||||
import { Worker } from 'bullmq';
|
||||
|
||||
const flyerWorker = new Worker(
|
||||
'flyer-processing',
|
||||
async (job) => {
|
||||
const { flyerId } = job.data;
|
||||
|
||||
try {
|
||||
// Process flyer
|
||||
const result = await aiService.extractFlyerData(flyerId);
|
||||
await flyerDb.updateFlyerWithData(flyerId, result);
|
||||
|
||||
// Update progress
|
||||
await job.updateProgress(100);
|
||||
|
||||
return { success: true, itemCount: result.items.length };
|
||||
} catch (error) {
|
||||
logger.error('Flyer processing failed', { flyerId, error });
|
||||
throw error; // Will retry automatically
|
||||
}
|
||||
},
|
||||
{
|
||||
connection: redisConnection,
|
||||
concurrency: 5,
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [ADR Index](../adr/index.md) - All architecture decision records
|
||||
- [TESTING.md](TESTING.md) - Testing patterns
|
||||
- [DEBUGGING.md](DEBUGGING.md) - Debugging strategies
|
||||
- [Database Guide](../subagents/DATABASE-GUIDE.md) - Database patterns
|
||||
- [Coder Reference](../SUBAGENT-CODER-REFERENCE.md) - Quick reference for AI agents
|
||||
844
docs/development/DEBUGGING.md
Normal file
844
docs/development/DEBUGGING.md
Normal file
@@ -0,0 +1,844 @@
|
||||
# Debugging Guide
|
||||
|
||||
Common debugging strategies and troubleshooting patterns for Flyer Crawler.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Quick Debugging Checklist](#quick-debugging-checklist)
|
||||
- [Container Issues](#container-issues)
|
||||
- [Database Issues](#database-issues)
|
||||
- [Test Failures](#test-failures)
|
||||
- [API Errors](#api-errors)
|
||||
- [Authentication Problems](#authentication-problems)
|
||||
- [Background Job Issues](#background-job-issues)
|
||||
- [SSL Certificate Issues](#ssl-certificate-issues)
|
||||
- [Frontend Issues](#frontend-issues)
|
||||
- [Performance Problems](#performance-problems)
|
||||
- [Debugging Tools](#debugging-tools)
|
||||
|
||||
---
|
||||
|
||||
## Quick Debugging Checklist
|
||||
|
||||
When something breaks, check these first:
|
||||
|
||||
1. **Are containers running?**
|
||||
|
||||
```bash
|
||||
podman ps
|
||||
```
|
||||
|
||||
2. **Is the database accessible?**
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-postgres pg_isready -U postgres
|
||||
```
|
||||
|
||||
3. **Are environment variables set?**
|
||||
|
||||
```bash
|
||||
# Check .env.local exists
|
||||
cat .env.local
|
||||
```
|
||||
|
||||
4. **Are there recent errors in logs?**
|
||||
|
||||
```bash
|
||||
# PM2 logs (dev container)
|
||||
podman exec -it flyer-crawler-dev pm2 logs
|
||||
|
||||
# Container logs
|
||||
podman logs -f flyer-crawler-dev
|
||||
|
||||
# PM2 logs (production)
|
||||
pm2 logs flyer-crawler-api
|
||||
```
|
||||
|
||||
5. **Are PM2 processes running?**
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
```
|
||||
|
||||
6. **Is Redis accessible?**
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-redis redis-cli ping
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Container Issues
|
||||
|
||||
### Container Won't Start
|
||||
|
||||
**Symptom**: `podman start` fails or container exits immediately
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check container status
|
||||
podman ps -a
|
||||
|
||||
# View container logs
|
||||
podman logs flyer-crawler-postgres
|
||||
podman logs flyer-crawler-redis
|
||||
podman logs flyer-crawler-dev
|
||||
|
||||
# Inspect container
|
||||
podman inspect flyer-crawler-dev
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Port already in use
|
||||
- Insufficient resources
|
||||
- Configuration error
|
||||
|
||||
**Solutions**:
|
||||
|
||||
```bash
|
||||
# Check port usage
|
||||
netstat -an | findstr "5432"
|
||||
netstat -an | findstr "6379"
|
||||
|
||||
# Remove and recreate container
|
||||
podman stop flyer-crawler-postgres
|
||||
podman rm flyer-crawler-postgres
|
||||
# ... recreate with podman run ...
|
||||
```
|
||||
|
||||
### "Unable to connect to Podman socket"
|
||||
|
||||
**Symptom**: `Error: unable to connect to Podman socket`
|
||||
|
||||
**Solution**:
|
||||
|
||||
```bash
|
||||
# Start Podman machine
|
||||
podman machine start
|
||||
|
||||
# Verify it's running
|
||||
podman machine list
|
||||
```
|
||||
|
||||
### Port Already in Use
|
||||
|
||||
**Symptom**: `Error: port 5432 is already allocated`
|
||||
|
||||
**Solutions**:
|
||||
|
||||
**Option 1**: Stop conflicting service
|
||||
|
||||
```bash
|
||||
# Find process using port
|
||||
netstat -ano | findstr "5432"
|
||||
|
||||
# Stop the service or kill process
|
||||
```
|
||||
|
||||
**Option 2**: Use different port
|
||||
|
||||
```bash
|
||||
# Run container on different host port
|
||||
podman run -d --name flyer-crawler-postgres -p 5433:5432 ...
|
||||
|
||||
# Update .env.local
|
||||
DB_PORT=5433
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Database Issues
|
||||
|
||||
### Connection Refused
|
||||
|
||||
**Symptom**: `Error: connect ECONNREFUSED 127.0.0.1:5432`
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# 1. Check if PostgreSQL container is running
|
||||
podman ps | grep postgres
|
||||
|
||||
# 2. Check if PostgreSQL is ready
|
||||
podman exec flyer-crawler-postgres pg_isready -U postgres
|
||||
|
||||
# 3. Test connection
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1;"
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Container not running
|
||||
- PostgreSQL still initializing
|
||||
- Wrong credentials in `.env.local`
|
||||
|
||||
**Solutions**:
|
||||
|
||||
```bash
|
||||
# Start container
|
||||
podman start flyer-crawler-postgres
|
||||
|
||||
# Wait for initialization (check logs)
|
||||
podman logs -f flyer-crawler-postgres
|
||||
|
||||
# Verify credentials match .env.local
|
||||
cat .env.local | grep DB_
|
||||
```
|
||||
|
||||
### Schema Out of Sync
|
||||
|
||||
**Symptom**: Tests fail with missing column or table errors
|
||||
|
||||
**Cause**: `master_schema_rollup.sql` not in sync with migrations
|
||||
|
||||
**Solution**:
|
||||
|
||||
```bash
|
||||
# Reset database with current schema
|
||||
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/drop_tables.sql
|
||||
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/master_schema_rollup.sql
|
||||
|
||||
# Verify schema
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "\dt"
|
||||
```
|
||||
|
||||
### Query Performance Issues
|
||||
|
||||
**Debug**:
|
||||
|
||||
```sql
|
||||
-- Enable query logging
|
||||
ALTER DATABASE flyer_crawler_dev SET log_statement = 'all';
|
||||
|
||||
-- Check slow queries
|
||||
SELECT query, mean_exec_time, calls
|
||||
FROM pg_stat_statements
|
||||
WHERE mean_exec_time > 100
|
||||
ORDER BY mean_exec_time DESC
|
||||
LIMIT 10;
|
||||
|
||||
-- Analyze query plan
|
||||
EXPLAIN ANALYZE
|
||||
SELECT * FROM flyers WHERE store_id = 1;
|
||||
```
|
||||
|
||||
**Solutions**:
|
||||
|
||||
- Add missing indexes
|
||||
- Optimize WHERE clauses
|
||||
- Use connection pooling
|
||||
- See [ADR-034](../adr/0034-repository-layer-method-naming-conventions.md)
|
||||
|
||||
---
|
||||
|
||||
## Test Failures
|
||||
|
||||
### Tests Pass on Windows, Fail in Container
|
||||
|
||||
**Cause**: Platform-specific behavior (ADR-014)
|
||||
|
||||
**Rule**: Container results are authoritative. Windows results are unreliable.
|
||||
|
||||
**Solution**:
|
||||
|
||||
```bash
|
||||
# Always run tests in container
|
||||
podman exec -it flyer-crawler-dev npm test
|
||||
|
||||
# For specific test
|
||||
podman exec -it flyer-crawler-dev npm test -- --run src/path/to/test.test.ts
|
||||
```
|
||||
|
||||
### Integration Tests Fail
|
||||
|
||||
**Common Issues**:
|
||||
|
||||
**1. Vitest globalSetup Context Isolation**
|
||||
|
||||
**Symptom**: Mocks or spies don't work in integration tests
|
||||
|
||||
**Cause**: `globalSetup` runs in separate Node.js context
|
||||
|
||||
**Solutions**:
|
||||
|
||||
- Mark test as `.todo()` and document limitation
|
||||
- Create test-only API endpoints
|
||||
- Use Redis-based mock flags
|
||||
|
||||
See [CLAUDE.md#integration-test-issues](../../CLAUDE.md#integration-test-issues) for details.
|
||||
|
||||
**2. Cache Stale After Direct SQL**
|
||||
|
||||
**Symptom**: Test reads stale data after direct database insert
|
||||
|
||||
**Cause**: Cache not invalidated
|
||||
|
||||
**Solution**:
|
||||
|
||||
```typescript
|
||||
// After direct SQL insert
|
||||
await cacheService.invalidateFlyers();
|
||||
```
|
||||
|
||||
**3. Queue Interference**
|
||||
|
||||
**Symptom**: Cleanup worker processes test data before assertions
|
||||
|
||||
**Solution**:
|
||||
|
||||
```typescript
|
||||
const { cleanupQueue } = await import('../../services/queues.server');
|
||||
await cleanupQueue.drain();
|
||||
await cleanupQueue.pause();
|
||||
// ... test ...
|
||||
await cleanupQueue.resume();
|
||||
```
|
||||
|
||||
### Type Check Failures
|
||||
|
||||
**Symptom**: `npm run type-check` fails
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Run type check in container
|
||||
podman exec -it flyer-crawler-dev npm run type-check
|
||||
|
||||
# Check specific file
|
||||
podman exec -it flyer-crawler-dev npx tsc --noEmit src/path/to/file.ts
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Missing type definitions
|
||||
- Incorrect imports
|
||||
- Type mismatch in function calls
|
||||
|
||||
---
|
||||
|
||||
## API Errors
|
||||
|
||||
### 404 Not Found
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check route registration
|
||||
grep -r "router.get" src/routes/
|
||||
|
||||
# Check route path matches request
|
||||
# Verify middleware order
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Route not registered in `server.ts`
|
||||
- Typo in route path
|
||||
- Middleware blocking request
|
||||
|
||||
### 500 Internal Server Error
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check application logs
|
||||
podman logs -f flyer-crawler-dev
|
||||
|
||||
# Check Bugsink for errors
|
||||
# Visit: http://localhost:8443 (dev) or https://bugsink.projectium.com (prod)
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Unhandled exception
|
||||
- Database error
|
||||
- Missing environment variable
|
||||
|
||||
**Solution Pattern**:
|
||||
|
||||
```typescript
|
||||
// Always wrap route handlers
|
||||
app.get('/api/endpoint', async (req, res) => {
|
||||
try {
|
||||
const result = await service.doSomething();
|
||||
return sendSuccess(res, result);
|
||||
} catch (error) {
|
||||
return sendError(res, error); // Handles error types automatically
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### 401 Unauthorized
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check JWT token in request
|
||||
# Verify token is valid and not expired
|
||||
|
||||
# Test token decoding
|
||||
node -e "console.log(require('jsonwebtoken').decode('YOUR_TOKEN_HERE'))"
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Token expired
|
||||
- Invalid token format
|
||||
- Missing Authorization header
|
||||
- Wrong JWT_SECRET
|
||||
|
||||
---
|
||||
|
||||
## Authentication Problems
|
||||
|
||||
### OAuth Not Working
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# 1. Verify OAuth credentials
|
||||
cat .env.local | grep GOOGLE_CLIENT
|
||||
|
||||
# 2. Check OAuth routes are registered
|
||||
grep -r "passport.authenticate" src/routes/
|
||||
|
||||
# 3. Verify redirect URI matches Google Console
|
||||
# Should be: http://localhost:3001/api/auth/google/callback
|
||||
```
|
||||
|
||||
**Common Issues**:
|
||||
|
||||
- Redirect URI mismatch in Google Console
|
||||
- OAuth not enabled (commented out in config)
|
||||
- Wrong client ID/secret
|
||||
|
||||
See [AUTHENTICATION.md](../architecture/AUTHENTICATION.md) for setup.
|
||||
|
||||
### JWT Token Invalid
|
||||
|
||||
**Debug**:
|
||||
|
||||
```typescript
|
||||
// Decode token to inspect
|
||||
import jwt from 'jsonwebtoken';
|
||||
|
||||
const decoded = jwt.decode(token);
|
||||
console.log('Token payload:', decoded);
|
||||
console.log('Expired:', decoded.exp < Date.now() / 1000);
|
||||
```
|
||||
|
||||
**Solutions**:
|
||||
|
||||
- Regenerate token
|
||||
- Check JWT_SECRET matches between environments
|
||||
- Verify token hasn't expired
|
||||
|
||||
---
|
||||
|
||||
## PM2 Process Issues (Dev Container)
|
||||
|
||||
### PM2 Process Not Starting
|
||||
|
||||
**Symptom**: `pm2 status` shows process as "errored" or "stopped"
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check process status
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
|
||||
# View process logs
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev --lines 50
|
||||
|
||||
# Show detailed process info
|
||||
podman exec -it flyer-crawler-dev pm2 show flyer-crawler-api-dev
|
||||
|
||||
# Check if port is in use
|
||||
podman exec -it flyer-crawler-dev netstat -tlnp | grep 3001
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Port already in use by another process
|
||||
- Missing environment variables
|
||||
- Syntax error in code
|
||||
- Database connection failure
|
||||
|
||||
**Solutions**:
|
||||
|
||||
```bash
|
||||
# Restart specific process
|
||||
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
|
||||
|
||||
# Restart all processes
|
||||
podman exec -it flyer-crawler-dev pm2 restart all
|
||||
|
||||
# Delete and recreate processes
|
||||
podman exec -it flyer-crawler-dev pm2 delete all
|
||||
podman exec -it flyer-crawler-dev pm2 start /app/ecosystem.dev.config.cjs
|
||||
```
|
||||
|
||||
### PM2 Log Access
|
||||
|
||||
**View logs in real-time**:
|
||||
|
||||
```bash
|
||||
# All processes
|
||||
podman exec -it flyer-crawler-dev pm2 logs
|
||||
|
||||
# Specific process
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
|
||||
|
||||
# Last 100 lines
|
||||
podman exec -it flyer-crawler-dev pm2 logs --lines 100
|
||||
```
|
||||
|
||||
**View log files directly**:
|
||||
|
||||
```bash
|
||||
# API logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-out.log
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-error.log
|
||||
|
||||
# Worker logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/worker-out.log
|
||||
|
||||
# Vite logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/vite-out.log
|
||||
```
|
||||
|
||||
### Redis Log Access
|
||||
|
||||
**View Redis logs**:
|
||||
|
||||
```bash
|
||||
# View Redis log file
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
|
||||
|
||||
# View from dev container (shared volume)
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/redis/redis-server.log
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Background Job Issues
|
||||
|
||||
### Jobs Not Processing
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check if worker is running (dev container)
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
|
||||
# Check worker logs (dev container)
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-worker-dev
|
||||
|
||||
# Check if worker is running (production)
|
||||
pm2 list
|
||||
|
||||
# Check worker logs (production)
|
||||
pm2 logs flyer-crawler-worker
|
||||
|
||||
# Check Redis connection
|
||||
podman exec flyer-crawler-redis redis-cli ping
|
||||
|
||||
# Check queue status
|
||||
node -e "
|
||||
const { flyerProcessingQueue } = require('./dist/services/queues.server.js');
|
||||
flyerProcessingQueue.getJobCounts().then(console.log);
|
||||
"
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Worker not running
|
||||
- Redis connection lost
|
||||
- Queue paused
|
||||
- Job stuck in failed state
|
||||
|
||||
**Solutions**:
|
||||
|
||||
```bash
|
||||
# Restart worker
|
||||
pm2 restart flyer-crawler-worker
|
||||
|
||||
# Clear failed jobs
|
||||
node -e "
|
||||
const { flyerProcessingQueue } = require('./dist/services/queues.server.js');
|
||||
flyerProcessingQueue.clean(0, 1000, 'failed');
|
||||
"
|
||||
```
|
||||
|
||||
### Jobs Failing
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check failed jobs
|
||||
node -e "
|
||||
const { flyerProcessingQueue } = require('./dist/services/queues.server.js');
|
||||
flyerProcessingQueue.getFailed().then(jobs => {
|
||||
jobs.forEach(job => console.log(job.failedReason));
|
||||
});
|
||||
"
|
||||
|
||||
# Check worker logs for stack traces
|
||||
pm2 logs flyer-crawler-worker --lines 100
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Gemini API errors
|
||||
- Database errors
|
||||
- Invalid job data
|
||||
|
||||
---
|
||||
|
||||
## SSL Certificate Issues
|
||||
|
||||
### Images Not Loading (ERR_CERT_AUTHORITY_INVALID)
|
||||
|
||||
**Symptom**: Flyer images fail to load with `ERR_CERT_AUTHORITY_INVALID` in browser console
|
||||
|
||||
**Cause**: Mixed hostname origins - user accesses via `localhost` but images use `127.0.0.1` (or vice versa)
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check which hostname images are using
|
||||
podman exec flyer-crawler-dev psql -U postgres -d flyer_crawler_dev \
|
||||
-c "SELECT image_url FROM flyers LIMIT 1;"
|
||||
|
||||
# Verify certificate includes both hostnames
|
||||
podman exec flyer-crawler-dev openssl x509 -in /app/certs/localhost.crt -text -noout | grep -A1 "Subject Alternative Name"
|
||||
|
||||
# Check NGINX accepts both hostnames
|
||||
podman exec flyer-crawler-dev grep "server_name" /etc/nginx/sites-available/default
|
||||
```
|
||||
|
||||
**Solution**: The dev container is configured to handle both hostnames:
|
||||
|
||||
1. Certificate includes SANs for `localhost`, `127.0.0.1`, and `::1`
|
||||
2. NGINX `server_name` directive includes both `localhost` and `127.0.0.1`
|
||||
|
||||
If you still see errors:
|
||||
|
||||
```bash
|
||||
# Rebuild container to regenerate certificate
|
||||
podman-compose down
|
||||
podman-compose build --no-cache flyer-crawler-dev
|
||||
podman-compose up -d
|
||||
```
|
||||
|
||||
See [FLYER-URL-CONFIGURATION.md](../FLYER-URL-CONFIGURATION.md#ssl-certificate-configuration-dev-container) for full details.
|
||||
|
||||
### Self-Signed Certificate Not Trusted
|
||||
|
||||
**Symptom**: Browser shows security warning for `https://localhost`
|
||||
|
||||
**Temporary Workaround**: Accept the warning by clicking "Advanced" > "Proceed to localhost"
|
||||
|
||||
**Permanent Fix (Recommended)**: Install the mkcert CA certificate to eliminate all SSL warnings.
|
||||
|
||||
The CA certificate is located at `certs/mkcert-ca.crt` in the project root. See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
|
||||
|
||||
After installation:
|
||||
|
||||
- Your browser will trust all mkcert certificates without warnings
|
||||
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
|
||||
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
|
||||
|
||||
### NGINX SSL Configuration Test
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Test NGINX configuration
|
||||
podman exec flyer-crawler-dev nginx -t
|
||||
|
||||
# Check if NGINX is listening on 443
|
||||
podman exec flyer-crawler-dev netstat -tlnp | grep 443
|
||||
|
||||
# Verify certificate files exist
|
||||
podman exec flyer-crawler-dev ls -la /app/certs/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Frontend Issues
|
||||
|
||||
### Hot Reload Not Working
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check Vite is running
|
||||
curl http://localhost:5173
|
||||
|
||||
# Check for port conflicts
|
||||
netstat -an | findstr "5173"
|
||||
```
|
||||
|
||||
**Solution**:
|
||||
|
||||
```bash
|
||||
# Restart dev server
|
||||
npm run dev
|
||||
```
|
||||
|
||||
### API Calls Failing (CORS)
|
||||
|
||||
**Symptom**: `CORS policy: No 'Access-Control-Allow-Origin' header`
|
||||
|
||||
**Debug**:
|
||||
|
||||
```typescript
|
||||
// Check CORS configuration in server.ts
|
||||
import cors from 'cors';
|
||||
|
||||
app.use(
|
||||
cors({
|
||||
origin: env.FRONTEND_URL, // Should match http://localhost:5173 in dev
|
||||
credentials: true,
|
||||
}),
|
||||
);
|
||||
```
|
||||
|
||||
**Solution**: Verify `FRONTEND_URL` in `.env.local` matches the frontend URL
|
||||
|
||||
---
|
||||
|
||||
## Performance Problems
|
||||
|
||||
### Slow API Responses
|
||||
|
||||
**Debug**:
|
||||
|
||||
```typescript
|
||||
// Add timing logs
|
||||
const start = Date.now();
|
||||
const result = await slowOperation();
|
||||
console.log(`Operation took ${Date.now() - start}ms`);
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- N+1 query problem
|
||||
- Missing database indexes
|
||||
- Large payload size
|
||||
- No caching
|
||||
|
||||
**Solutions**:
|
||||
|
||||
- Use JOINs instead of multiple queries
|
||||
- Add indexes: `CREATE INDEX idx_name ON table(column);`
|
||||
- Implement pagination
|
||||
- Add Redis caching
|
||||
|
||||
### High Memory Usage
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check PM2 memory usage
|
||||
pm2 monit
|
||||
|
||||
# Check container memory
|
||||
podman stats flyer-crawler-dev
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Memory leak
|
||||
- Large in-memory cache
|
||||
- Unbounded array growth
|
||||
|
||||
---
|
||||
|
||||
## Debugging Tools
|
||||
|
||||
### VS Code Debugger
|
||||
|
||||
**Launch Configuration** (`.vscode/launch.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
"name": "Debug Tests",
|
||||
"program": "${workspaceFolder}/node_modules/vitest/vitest.mjs",
|
||||
"args": ["--run", "${file}"],
|
||||
"console": "integratedTerminal",
|
||||
"internalConsoleOptions": "neverOpen"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Logging
|
||||
|
||||
```typescript
|
||||
import { logger } from './utils/logger';
|
||||
|
||||
// Structured logging
|
||||
logger.info('Processing flyer', { flyerId, userId });
|
||||
logger.error('Failed to process', { error, context });
|
||||
logger.debug('Cache hit', { key, ttl });
|
||||
```
|
||||
|
||||
### Database Query Logging
|
||||
|
||||
```typescript
|
||||
// In development, log all queries
|
||||
if (env.NODE_ENV === 'development') {
|
||||
pool.on('connect', () => {
|
||||
console.log('Database connected');
|
||||
});
|
||||
|
||||
// Log slow queries
|
||||
const originalQuery = pool.query.bind(pool);
|
||||
pool.query = async (...args) => {
|
||||
const start = Date.now();
|
||||
const result = await originalQuery(...args);
|
||||
const duration = Date.now() - start;
|
||||
if (duration > 100) {
|
||||
console.log(`Slow query (${duration}ms):`, args[0]);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### Redis Debugging
|
||||
|
||||
```bash
|
||||
# Monitor Redis commands
|
||||
podman exec -it flyer-crawler-redis redis-cli monitor
|
||||
|
||||
# Check keys
|
||||
podman exec flyer-crawler-redis redis-cli keys "*"
|
||||
|
||||
# Get key value
|
||||
podman exec flyer-crawler-redis redis-cli get "flyer:123"
|
||||
|
||||
# Check cache stats
|
||||
podman exec flyer-crawler-redis redis-cli info stats
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## See Also
|
||||
|
||||
- [DEV-CONTAINER.md](DEV-CONTAINER.md) - Dev container guide (PM2, Logstash)
|
||||
- [TESTING.md](TESTING.md) - Testing strategies
|
||||
- [CODE-PATTERNS.md](CODE-PATTERNS.md) - Common patterns
|
||||
- [MONITORING.md](../operations/MONITORING.md) - Production monitoring
|
||||
- [LOGSTASH-QUICK-REF.md](../operations/LOGSTASH-QUICK-REF.md) - Log aggregation
|
||||
- [Bugsink Setup](../tools/BUGSINK-SETUP.md) - Error tracking
|
||||
- [DevOps Guide](../subagents/DEVOPS-GUIDE.md) - Container debugging
|
||||
408
docs/development/DEV-CONTAINER.md
Normal file
408
docs/development/DEV-CONTAINER.md
Normal file
@@ -0,0 +1,408 @@
|
||||
# Dev Container Guide
|
||||
|
||||
Comprehensive documentation for the Flyer Crawler development container.
|
||||
|
||||
**Last Updated**: 2026-01-22
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Overview](#overview)
|
||||
2. [Architecture](#architecture)
|
||||
3. [PM2 Process Management](#pm2-process-management)
|
||||
4. [Log Aggregation](#log-aggregation)
|
||||
5. [Quick Reference](#quick-reference)
|
||||
6. [Troubleshooting](#troubleshooting)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
The dev container provides a production-like environment for local development. Key features:
|
||||
|
||||
- **PM2 Process Management** - Matches production architecture (ADR-014)
|
||||
- **Logstash Integration** - All logs forwarded to Bugsink (ADR-050)
|
||||
- **HTTPS by Default** - Self-signed certificates via mkcert
|
||||
- **Hot Reloading** - tsx watch mode for API and worker processes
|
||||
|
||||
### Container Services
|
||||
|
||||
| Service | Container Name | Purpose |
|
||||
| ----------- | ------------------------ | ------------------------------ |
|
||||
| Application | `flyer-crawler-dev` | Node.js app, Bugsink, Logstash |
|
||||
| PostgreSQL | `flyer-crawler-postgres` | Primary database with PostGIS |
|
||||
| Redis | `flyer-crawler-redis` | Cache and job queue backing |
|
||||
|
||||
### Access Points
|
||||
|
||||
| Service | URL | Notes |
|
||||
| ----------- | ------------------------ | ------------------------- |
|
||||
| Frontend | `https://localhost` | NGINX proxies Vite (5173) |
|
||||
| Backend API | `http://localhost:3001` | Express server |
|
||||
| Bugsink | `https://localhost:8443` | Error tracking UI |
|
||||
| PostgreSQL | `localhost:5432` | Direct database access |
|
||||
| Redis | `localhost:6379` | Direct Redis access |
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
### Production vs Development Parity
|
||||
|
||||
The dev container was updated to match production architecture (ADR-014):
|
||||
|
||||
| Component | Production | Dev Container (OLD) | Dev Container (NEW) |
|
||||
| ------------ | ---------------------- | ---------------------- | ----------------------- |
|
||||
| API Server | PM2 cluster mode | `npm run dev` (inline) | PM2 fork + tsx watch |
|
||||
| Worker | PM2 process | Inline with API | PM2 process + tsx watch |
|
||||
| Frontend | Static files via NGINX | Vite standalone | PM2 + Vite dev server |
|
||||
| Logs | PM2 logs -> Logstash | Console only | PM2 logs -> Logstash |
|
||||
| Process Mgmt | PM2 | None | PM2 |
|
||||
|
||||
### Container Startup Flow
|
||||
|
||||
When the container starts (`scripts/dev-entrypoint.sh`):
|
||||
|
||||
1. **NGINX** starts (HTTPS proxy for Vite and Bugsink)
|
||||
2. **Bugsink** starts (error tracking on port 8000)
|
||||
3. **Logstash** starts (log aggregation)
|
||||
4. **PM2** starts with `ecosystem.dev.config.cjs`:
|
||||
- `flyer-crawler-api-dev` - API server
|
||||
- `flyer-crawler-worker-dev` - Background worker
|
||||
- `flyer-crawler-vite-dev` - Vite dev server
|
||||
5. Container tails PM2 logs to stay alive
|
||||
|
||||
### Key Configuration Files
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------ | ---------------------------------- |
|
||||
| `ecosystem.dev.config.cjs` | PM2 development configuration |
|
||||
| `ecosystem.config.cjs` | PM2 production configuration |
|
||||
| `scripts/dev-entrypoint.sh` | Container startup script |
|
||||
| `docker/logstash/bugsink.conf` | Logstash pipeline configuration |
|
||||
| `docker/nginx/dev.conf` | NGINX development configuration |
|
||||
| `compose.dev.yml` | Docker Compose service definitions |
|
||||
| `Dockerfile.dev` | Container image definition |
|
||||
|
||||
---
|
||||
|
||||
## PM2 Process Management
|
||||
|
||||
### Process Overview
|
||||
|
||||
PM2 manages three processes in the dev container:
|
||||
|
||||
```
|
||||
+--------------------+ +------------------------+ +--------------------+
|
||||
| flyer-crawler- | | flyer-crawler- | | flyer-crawler- |
|
||||
| api-dev | | worker-dev | | vite-dev |
|
||||
+--------------------+ +------------------------+ +--------------------+
|
||||
| tsx watch | | tsx watch | | vite --host |
|
||||
| server.ts | | src/services/worker.ts | | |
|
||||
| Port: 3001 | | No port | | Port: 5173 |
|
||||
+--------------------+ +------------------------+ +--------------------+
|
||||
| | |
|
||||
v v v
|
||||
+------------------------------------------------------------------------+
|
||||
| /var/log/pm2/*.log |
|
||||
| (Logstash picks up for Bugsink) |
|
||||
+------------------------------------------------------------------------+
|
||||
```
|
||||
|
||||
### PM2 Commands
|
||||
|
||||
All commands should be run inside the container:
|
||||
|
||||
```bash
|
||||
# View process status
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
|
||||
# View all logs (tail -f style)
|
||||
podman exec -it flyer-crawler-dev pm2 logs
|
||||
|
||||
# View specific process logs
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
|
||||
|
||||
# Restart all processes
|
||||
podman exec -it flyer-crawler-dev pm2 restart all
|
||||
|
||||
# Restart specific process
|
||||
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
|
||||
|
||||
# Stop all processes
|
||||
podman exec -it flyer-crawler-dev pm2 delete all
|
||||
|
||||
# Show detailed process info
|
||||
podman exec -it flyer-crawler-dev pm2 show flyer-crawler-api-dev
|
||||
|
||||
# Monitor processes in real-time
|
||||
podman exec -it flyer-crawler-dev pm2 monit
|
||||
```
|
||||
|
||||
### PM2 Log Locations
|
||||
|
||||
| Process | stdout Log | stderr Log |
|
||||
| -------------------------- | ----------------------------- | ------------------------------- |
|
||||
| `flyer-crawler-api-dev` | `/var/log/pm2/api-out.log` | `/var/log/pm2/api-error.log` |
|
||||
| `flyer-crawler-worker-dev` | `/var/log/pm2/worker-out.log` | `/var/log/pm2/worker-error.log` |
|
||||
| `flyer-crawler-vite-dev` | `/var/log/pm2/vite-out.log` | `/var/log/pm2/vite-error.log` |
|
||||
|
||||
### NPM Scripts for PM2
|
||||
|
||||
The following npm scripts are available for PM2 management:
|
||||
|
||||
```bash
|
||||
# Start PM2 with dev config (inside container)
|
||||
npm run dev:pm2
|
||||
|
||||
# Restart all PM2 processes
|
||||
npm run dev:pm2:restart
|
||||
|
||||
# Stop all PM2 processes
|
||||
npm run dev:pm2:stop
|
||||
|
||||
# View PM2 status
|
||||
npm run dev:pm2:status
|
||||
|
||||
# View PM2 logs
|
||||
npm run dev:pm2:logs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Log Aggregation
|
||||
|
||||
### Log Flow Architecture (ADR-050)
|
||||
|
||||
All application logs flow through Logstash to Bugsink using a 3-project architecture:
|
||||
|
||||
```text
|
||||
+------------------+ +------------------+ +------------------+
|
||||
| PM2 Logs | | PostgreSQL | | Redis/NGINX |
|
||||
| /var/log/pm2/ | | /var/log/ | | /var/log/redis/ |
|
||||
| (API + Worker) | | postgresql/ | | /var/log/nginx/ |
|
||||
+--------+---------+ +--------+---------+ +--------+---------+
|
||||
| | |
|
||||
v v v
|
||||
+------------------------------------------------------------------------+
|
||||
| LOGSTASH |
|
||||
| /etc/logstash/conf.d/bugsink.conf |
|
||||
| (Routes by log type) |
|
||||
+------------------------------------------------------------------------+
|
||||
| | |
|
||||
v v v
|
||||
+------------------+ +------------------+ +------------------+
|
||||
| Backend API | | Frontend (Dev) | | Infrastructure |
|
||||
| (Project 1) | | (Project 2) | | (Project 4) |
|
||||
| - Pino errors | | - Browser SDK | | - Redis warnings |
|
||||
| - PostgreSQL | | (not Logstash) | | - NGINX errors |
|
||||
+------------------+ +------------------+ | - Vite errors |
|
||||
+------------------+
|
||||
```
|
||||
|
||||
### Log Sources
|
||||
|
||||
| Source | Log Path | Format | Errors To Bugsink |
|
||||
| ------------ | --------------------------------- | ---------- | ----------------- |
|
||||
| API Server | `/var/log/pm2/api-*.log` | Pino JSON | Yes |
|
||||
| Worker | `/var/log/pm2/worker-*.log` | Pino JSON | Yes |
|
||||
| Vite | `/var/log/pm2/vite-*.log` | Plain text | Yes (if error) |
|
||||
| PostgreSQL | `/var/log/postgresql/*.log` | PostgreSQL | Yes (ERROR/FATAL) |
|
||||
| Redis | `/var/log/redis/redis-server.log` | Redis | Yes (warnings) |
|
||||
| NGINX Access | `/var/log/nginx/access.log` | Combined | Yes (5xx only) |
|
||||
| NGINX Error | `/var/log/nginx/error.log` | NGINX | Yes |
|
||||
|
||||
### Viewing Logs
|
||||
|
||||
```bash
|
||||
# View Logstash processed logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-api-$(date +%Y-%m-%d).log
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
|
||||
|
||||
# View raw Redis logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
|
||||
|
||||
# Check Logstash status
|
||||
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
|
||||
```
|
||||
|
||||
### Bugsink Access
|
||||
|
||||
- **URL**: `https://localhost:8443`
|
||||
- **Login**: `admin@localhost` / `admin`
|
||||
- **Projects**:
|
||||
- Project 1: Backend API (Dev) - Pino app errors, PostgreSQL errors
|
||||
- Project 2: Frontend (Dev) - Browser errors via Sentry SDK
|
||||
- Project 4: Infrastructure (Dev) - Redis warnings, NGINX errors, Vite build errors
|
||||
|
||||
**Note**: Frontend DSN uses nginx proxy (`/bugsink-api/`) because browsers cannot reach `localhost:8000` directly. See [BUGSINK-SETUP.md](../tools/BUGSINK-SETUP.md#frontend-nginx-proxy) for details.
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference
|
||||
|
||||
### Starting the Dev Container
|
||||
|
||||
```bash
|
||||
# Start all services
|
||||
podman-compose -f compose.dev.yml up -d
|
||||
|
||||
# View container logs
|
||||
podman-compose -f compose.dev.yml logs -f
|
||||
|
||||
# Stop all services
|
||||
podman-compose -f compose.dev.yml down
|
||||
```
|
||||
|
||||
### Common Tasks
|
||||
|
||||
| Task | Command |
|
||||
| -------------------- | ------------------------------------------------------------------------------ |
|
||||
| Run tests | `podman exec -it flyer-crawler-dev npm test` |
|
||||
| Run type check | `podman exec -it flyer-crawler-dev npm run type-check` |
|
||||
| View PM2 status | `podman exec -it flyer-crawler-dev pm2 status` |
|
||||
| View PM2 logs | `podman exec -it flyer-crawler-dev pm2 logs` |
|
||||
| Restart API | `podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev` |
|
||||
| Access PostgreSQL | `podman exec -it flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev` |
|
||||
| Access Redis CLI | `podman exec -it flyer-crawler-redis redis-cli` |
|
||||
| Shell into container | `podman exec -it flyer-crawler-dev bash` |
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Key environment variables are set in `compose.dev.yml`:
|
||||
|
||||
| Variable | Value | Purpose |
|
||||
| ----------------- | ----------------------------- | --------------------------- |
|
||||
| `TZ` | `America/Los_Angeles` | Timezone (PST) for all logs |
|
||||
| `NODE_ENV` | `development` | Environment mode |
|
||||
| `DB_HOST` | `postgres` | PostgreSQL hostname |
|
||||
| `REDIS_URL` | `redis://redis:6379` | Redis connection URL |
|
||||
| `FRONTEND_URL` | `https://localhost` | CORS origin |
|
||||
| `SENTRY_DSN` | `http://...@127.0.0.1:8000/1` | Backend Bugsink DSN |
|
||||
| `VITE_SENTRY_DSN` | `http://...@127.0.0.1:8000/2` | Frontend Bugsink DSN |
|
||||
|
||||
### Timezone Configuration
|
||||
|
||||
All dev container services are configured to use PST (America/Los_Angeles) timezone for consistent log timestamps:
|
||||
|
||||
| Service | Configuration | Notes |
|
||||
| ---------- | ------------------------------------------------ | ------------------------------ |
|
||||
| App | `TZ=America/Los_Angeles` in compose.dev.yml | Also set via dev-entrypoint.sh |
|
||||
| PostgreSQL | `timezone` and `log_timezone` in postgres config | Logs timestamps in PST |
|
||||
| Redis | `TZ=America/Los_Angeles` in compose.dev.yml | Alpine uses TZ env var |
|
||||
| PM2 | `TZ` in ecosystem.dev.config.cjs | Pino timestamps use local time |
|
||||
|
||||
**Verifying Timezone**:
|
||||
|
||||
```bash
|
||||
# Check container timezone
|
||||
podman exec flyer-crawler-dev date
|
||||
|
||||
# Check PostgreSQL timezone
|
||||
podman exec flyer-crawler-postgres psql -U postgres -c "SHOW timezone;"
|
||||
|
||||
# Check Redis log timestamps
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log | head -5
|
||||
```
|
||||
|
||||
**Note**: If you need UTC timestamps for production compatibility, change `TZ=UTC` in compose.dev.yml and restart containers.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### PM2 Process Not Starting
|
||||
|
||||
**Symptom**: `pm2 status` shows process as "errored" or "stopped"
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check process logs
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev --lines 50
|
||||
|
||||
# Check if port is in use
|
||||
podman exec -it flyer-crawler-dev netstat -tlnp | grep 3001
|
||||
|
||||
# Try manual start to see errors
|
||||
podman exec -it flyer-crawler-dev tsx server.ts
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Port already in use
|
||||
- Missing environment variables
|
||||
- Syntax error in code
|
||||
|
||||
### Logs Not Appearing in Bugsink
|
||||
|
||||
**Symptom**: Errors in application but nothing in Bugsink UI
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check Logstash is running
|
||||
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
|
||||
|
||||
# Check Logstash logs for errors
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log
|
||||
|
||||
# Verify PM2 logs exist
|
||||
podman exec flyer-crawler-dev ls -la /var/log/pm2/
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- Logstash not started
|
||||
- Log file permissions
|
||||
- Bugsink not running
|
||||
|
||||
### Redis Logs Not Captured
|
||||
|
||||
**Symptom**: Redis warnings not appearing in Bugsink
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Verify Redis logs exist
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
|
||||
|
||||
# Verify shared volume is mounted
|
||||
podman exec flyer-crawler-dev ls -la /var/log/redis/
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- `redis_logs` volume not mounted
|
||||
- Redis not configured to write to file
|
||||
|
||||
### Hot Reload Not Working
|
||||
|
||||
**Symptom**: Code changes not reflected in running application
|
||||
|
||||
**Debug**:
|
||||
|
||||
```bash
|
||||
# Check tsx watch is running
|
||||
podman exec -it flyer-crawler-dev pm2 logs flyer-crawler-api-dev
|
||||
|
||||
# Manually restart process
|
||||
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
|
||||
```
|
||||
|
||||
**Common Causes**:
|
||||
|
||||
- File watcher limit reached
|
||||
- Volume mount issues on Windows
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [QUICKSTART.md](../getting-started/QUICKSTART.md) - Getting started guide
|
||||
- [DEBUGGING.md](DEBUGGING.md) - Debugging strategies
|
||||
- [LOGSTASH-QUICK-REF.md](../operations/LOGSTASH-QUICK-REF.md) - Logstash quick reference
|
||||
- [DEV-CONTAINER-BUGSINK.md](../DEV-CONTAINER-BUGSINK.md) - Bugsink setup in dev container
|
||||
- [ADR-014](../adr/0014-linux-only-platform.md) - Linux-only platform decision
|
||||
- [ADR-050](../adr/0050-postgresql-function-observability.md) - PostgreSQL function observability
|
||||
@@ -187,6 +187,17 @@ const mockStoreWithLocations = createMockStoreWithLocations({
|
||||
});
|
||||
```
|
||||
|
||||
### Test Assets
|
||||
|
||||
Test images and other assets are located in `src/tests/assets/`:
|
||||
|
||||
| File | Purpose |
|
||||
| ---------------------- | ---------------------------------------------- |
|
||||
| `test-flyer-image.jpg` | Sample flyer image for upload/processing tests |
|
||||
| `test-flyer-icon.png` | Sample flyer icon (64x64) for thumbnail tests |
|
||||
|
||||
These images are copied to `public/flyer-images/` by the seed script (`npm run seed`) and served via NGINX at `/flyer-images/`.
|
||||
|
||||
## Known Integration Test Issues
|
||||
|
||||
See `CLAUDE.md` for documentation of common integration test issues and their solutions, including:
|
||||
271
docs/getting-started/ENVIRONMENT.md
Normal file
271
docs/getting-started/ENVIRONMENT.md
Normal file
@@ -0,0 +1,271 @@
|
||||
# Environment Variables Reference
|
||||
|
||||
Complete guide to environment variables used in Flyer Crawler.
|
||||
|
||||
## Configuration by Environment
|
||||
|
||||
### Production
|
||||
|
||||
**Location**: Gitea CI/CD secrets injected during deployment
|
||||
**Path**: `/var/www/flyer-crawler.projectium.com/`
|
||||
**Note**: No `.env` file exists - all variables come from CI/CD
|
||||
|
||||
### Test
|
||||
|
||||
**Location**: Gitea CI/CD secrets + `.env.test` file
|
||||
**Path**: `/var/www/flyer-crawler-test.projectium.com/`
|
||||
**Note**: `.env.test` overrides for test-specific values
|
||||
|
||||
### Development Container
|
||||
|
||||
**Location**: `.env.local` file in project root
|
||||
**Note**: Overrides default DSNs in `compose.dev.yml`
|
||||
|
||||
## Required Variables
|
||||
|
||||
### Database
|
||||
|
||||
| Variable | Description | Example |
|
||||
| ------------------ | ---------------------------- | ------------------------------------------ |
|
||||
| `DB_HOST` | PostgreSQL host | `localhost` (dev), `projectium.com` (prod) |
|
||||
| `DB_PORT` | PostgreSQL port | `5432` |
|
||||
| `DB_USER_PROD` | Production database user | `flyer_crawler_prod` |
|
||||
| `DB_PASSWORD_PROD` | Production database password | (secret) |
|
||||
| `DB_DATABASE_PROD` | Production database name | `flyer-crawler-prod` |
|
||||
| `DB_USER_TEST` | Test database user | `flyer_crawler_test` |
|
||||
| `DB_PASSWORD_TEST` | Test database password | (secret) |
|
||||
| `DB_DATABASE_TEST` | Test database name | `flyer-crawler-test` |
|
||||
| `DB_USER` | Dev database user | `postgres` |
|
||||
| `DB_PASSWORD` | Dev database password | `postgres` |
|
||||
| `DB_NAME` | Dev database name | `flyer_crawler_dev` |
|
||||
|
||||
**Note**: Production and test use separate `_PROD` and `_TEST` suffixed variables. Development uses unsuffixed variables.
|
||||
|
||||
### Redis
|
||||
|
||||
| Variable | Description | Example |
|
||||
| --------------------- | ------------------------- | ------------------------------ |
|
||||
| `REDIS_URL` | Redis connection URL | `redis://localhost:6379` (dev) |
|
||||
| `REDIS_PASSWORD_PROD` | Production Redis password | (secret) |
|
||||
| `REDIS_PASSWORD_TEST` | Test Redis password | (secret) |
|
||||
|
||||
### Authentication
|
||||
|
||||
| Variable | Description | Example |
|
||||
| ---------------------- | -------------------------- | -------------------------------- |
|
||||
| `JWT_SECRET` | JWT token signing key | (minimum 32 characters) |
|
||||
| `SESSION_SECRET` | Session encryption key | (minimum 32 characters) |
|
||||
| `GOOGLE_CLIENT_ID` | Google OAuth client ID | `xxx.apps.googleusercontent.com` |
|
||||
| `GOOGLE_CLIENT_SECRET` | Google OAuth client secret | (secret) |
|
||||
| `GH_CLIENT_ID` | GitHub OAuth client ID | `xxx` |
|
||||
| `GH_CLIENT_SECRET` | GitHub OAuth client secret | (secret) |
|
||||
|
||||
### AI Services
|
||||
|
||||
| Variable | Description | Example |
|
||||
| -------------------------------- | ---------------------------- | ----------- |
|
||||
| `VITE_GOOGLE_GENAI_API_KEY` | Google Gemini API key (prod) | `AIzaSy...` |
|
||||
| `VITE_GOOGLE_GENAI_API_KEY_TEST` | Google Gemini API key (test) | `AIzaSy...` |
|
||||
| `GOOGLE_MAPS_API_KEY` | Google Maps Geocoding API | `AIzaSy...` |
|
||||
|
||||
### Application
|
||||
|
||||
| Variable | Description | Example |
|
||||
| -------------- | ------------------------ | ----------------------------------- |
|
||||
| `NODE_ENV` | Environment mode | `development`, `test`, `production` |
|
||||
| `PORT` | Backend server port | `3001` |
|
||||
| `FRONTEND_URL` | Frontend application URL | `http://localhost:5173` (dev) |
|
||||
|
||||
### Error Tracking
|
||||
|
||||
| Variable | Description | Example |
|
||||
| ---------------------- | -------------------------------- | --------------------------- |
|
||||
| `SENTRY_DSN` | Sentry DSN (production) | `https://xxx@sentry.io/xxx` |
|
||||
| `VITE_SENTRY_DSN` | Frontend Sentry DSN (production) | `https://xxx@sentry.io/xxx` |
|
||||
| `SENTRY_DSN_TEST` | Sentry DSN (test) | `https://xxx@sentry.io/xxx` |
|
||||
| `VITE_SENTRY_DSN_TEST` | Frontend Sentry DSN (test) | `https://xxx@sentry.io/xxx` |
|
||||
| `SENTRY_AUTH_TOKEN` | Sentry API token for releases | (secret) |
|
||||
|
||||
## Optional Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
| ------------------- | ----------------------- | ----------------- |
|
||||
| `LOG_LEVEL` | Logging verbosity | `info` |
|
||||
| `REDIS_TTL` | Cache TTL in seconds | `3600` |
|
||||
| `MAX_UPLOAD_SIZE` | Max file upload size | `10mb` |
|
||||
| `RATE_LIMIT_WINDOW` | Rate limit window (ms) | `900000` (15 min) |
|
||||
| `RATE_LIMIT_MAX` | Max requests per window | `100` |
|
||||
|
||||
## Configuration Files
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------------- | ------------------------------------------- |
|
||||
| `src/config/env.ts` | Zod schema validation - **source of truth** |
|
||||
| `ecosystem.config.cjs` | PM2 process manager config |
|
||||
| `.gitea/workflows/deploy-to-prod.yml` | Production deployment workflow |
|
||||
| `.gitea/workflows/deploy-to-test.yml` | Test deployment workflow |
|
||||
| `.env.example` | Template with all variables |
|
||||
| `.env.local` | Dev container overrides (not in git) |
|
||||
| `.env.test` | Test environment overrides (not in git) |
|
||||
|
||||
## Adding New Variables
|
||||
|
||||
### 1. Update Zod Schema
|
||||
|
||||
Edit `src/config/env.ts`:
|
||||
|
||||
```typescript
|
||||
const envSchema = z.object({
|
||||
// ... existing variables ...
|
||||
NEW_VARIABLE: z.string().min(1),
|
||||
});
|
||||
```
|
||||
|
||||
### 2. Add to Gitea Secrets
|
||||
|
||||
For prod/test environments:
|
||||
|
||||
1. Go to Gitea repository Settings > Secrets
|
||||
2. Add `NEW_VARIABLE` with value
|
||||
3. Add `NEW_VARIABLE_TEST` if test needs different value
|
||||
|
||||
### 3. Update Deployment Workflows
|
||||
|
||||
Edit `.gitea/workflows/deploy-to-prod.yml`:
|
||||
|
||||
```yaml
|
||||
env:
|
||||
NEW_VARIABLE: ${{ secrets.NEW_VARIABLE }}
|
||||
```
|
||||
|
||||
Edit `.gitea/workflows/deploy-to-test.yml`:
|
||||
|
||||
```yaml
|
||||
env:
|
||||
NEW_VARIABLE: ${{ secrets.NEW_VARIABLE_TEST }}
|
||||
```
|
||||
|
||||
### 4. Update PM2 Config
|
||||
|
||||
Edit `ecosystem.config.cjs`:
|
||||
|
||||
```javascript
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
env: {
|
||||
NEW_VARIABLE: process.env.NEW_VARIABLE,
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
```
|
||||
|
||||
### 5. Update Documentation
|
||||
|
||||
- Add to `.env.example`
|
||||
- Update this document
|
||||
- Document in relevant feature docs
|
||||
|
||||
## Security Best Practices
|
||||
|
||||
### Secrets Management
|
||||
|
||||
- **NEVER** commit secrets to git
|
||||
- Use Gitea Secrets for prod/test
|
||||
- Use `.env.local` for dev (gitignored)
|
||||
- Rotate secrets regularly
|
||||
|
||||
### Secret Generation
|
||||
|
||||
```bash
|
||||
# Generate secure random secrets
|
||||
node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
|
||||
```
|
||||
|
||||
### Database Users
|
||||
|
||||
Each environment has its own PostgreSQL user:
|
||||
|
||||
| Environment | User | Database |
|
||||
| ----------- | -------------------- | -------------------- |
|
||||
| Production | `flyer_crawler_prod` | `flyer-crawler-prod` |
|
||||
| Test | `flyer_crawler_test` | `flyer-crawler-test` |
|
||||
| Development | `postgres` | `flyer_crawler_dev` |
|
||||
|
||||
**Setup Commands** (as postgres superuser):
|
||||
|
||||
```sql
|
||||
-- Production
|
||||
CREATE DATABASE "flyer-crawler-prod";
|
||||
CREATE USER flyer_crawler_prod WITH PASSWORD 'secure-password';
|
||||
ALTER DATABASE "flyer-crawler-prod" OWNER TO flyer_crawler_prod;
|
||||
\c "flyer-crawler-prod"
|
||||
ALTER SCHEMA public OWNER TO flyer_crawler_prod;
|
||||
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_prod;
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS postgis;
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
|
||||
-- Test (similar commands with _test suffix)
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
Environment variables are validated at startup via `src/config/env.ts`. If validation fails:
|
||||
|
||||
1. Check the error message for missing/invalid variables
|
||||
2. Verify `.env.local` (dev) or Gitea Secrets (prod/test)
|
||||
3. Ensure values match schema requirements (min length, format, etc.)
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Variable Not Found
|
||||
|
||||
```
|
||||
Error: Missing required environment variable: JWT_SECRET
|
||||
```
|
||||
|
||||
**Solution**: Add the variable to your environment configuration.
|
||||
|
||||
### Invalid Value
|
||||
|
||||
```
|
||||
Error: JWT_SECRET must be at least 32 characters
|
||||
```
|
||||
|
||||
**Solution**: Generate a longer secret value.
|
||||
|
||||
### Wrong Environment Selected
|
||||
|
||||
Check `NODE_ENV` is set correctly:
|
||||
|
||||
- `development` - Local dev container
|
||||
- `test` - CI/CD test server
|
||||
- `production` - Production server
|
||||
|
||||
### Database Connection Issues
|
||||
|
||||
Verify database credentials:
|
||||
|
||||
```bash
|
||||
# Development
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1;"
|
||||
|
||||
# Production (via SSH)
|
||||
ssh root@projectium.com "psql -U flyer_crawler_prod -d flyer-crawler-prod -c 'SELECT 1;'"
|
||||
```
|
||||
|
||||
## Reference
|
||||
|
||||
- **Validation Schema**: [src/config/env.ts](../../src/config/env.ts)
|
||||
- **Template**: [.env.example](../../.env.example)
|
||||
- **Deployment Workflows**: [.gitea/workflows/](../../.gitea/workflows/)
|
||||
- **PM2 Config**: [ecosystem.config.cjs](../../ecosystem.config.cjs)
|
||||
|
||||
## See Also
|
||||
|
||||
- [QUICKSTART.md](QUICKSTART.md) - Quick setup guide
|
||||
- [INSTALL.md](INSTALL.md) - Detailed installation
|
||||
- [DEPLOYMENT.md](../operations/DEPLOYMENT.md) - Production deployment
|
||||
- [AUTHENTICATION.md](../architecture/AUTHENTICATION.md) - OAuth setup
|
||||
@@ -119,6 +119,31 @@ npm run dev
|
||||
- **Frontend**: http://localhost:5173
|
||||
- **Backend API**: http://localhost:3001
|
||||
|
||||
### Dev Container with HTTPS (Full Stack)
|
||||
|
||||
When using the full dev container stack with NGINX (via `compose.dev.yml`), access the application over HTTPS:
|
||||
|
||||
- **Frontend**: https://localhost or https://127.0.0.1
|
||||
- **Backend API**: http://localhost:3001
|
||||
|
||||
**SSL Certificate Notes:**
|
||||
|
||||
- The dev container uses self-signed certificates generated by mkcert
|
||||
- Both `localhost` and `127.0.0.1` are valid hostnames (certificate includes both as SANs)
|
||||
- If images fail to load with SSL errors, see [FLYER-URL-CONFIGURATION.md](../FLYER-URL-CONFIGURATION.md#ssl-certificate-configuration-dev-container)
|
||||
|
||||
**Eliminate SSL Warnings (Recommended):**
|
||||
|
||||
To avoid browser security warnings for self-signed certificates, install the mkcert CA certificate on your system. The CA certificate is located at `certs/mkcert-ca.crt` in the project root.
|
||||
|
||||
See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions (Windows, macOS, Linux, Firefox).
|
||||
|
||||
After installation:
|
||||
|
||||
- Your browser will trust all mkcert certificates without warnings
|
||||
- Both `https://localhost/` and `https://127.0.0.1/` will work without SSL errors
|
||||
- Flyer images will load without `ERR_CERT_AUTHORITY_INVALID` errors
|
||||
|
||||
### Managing the Container
|
||||
|
||||
| Action | Command |
|
||||
@@ -149,14 +174,24 @@ For local development, you can export these in your shell or use your IDE's envi
|
||||
|
||||
---
|
||||
|
||||
## Seeding Development Users
|
||||
## Seeding Development Data
|
||||
|
||||
To create initial test accounts (`admin@example.com` and `user@example.com`):
|
||||
To create initial test accounts (`admin@example.com` and `user@example.com`) and sample data:
|
||||
|
||||
```bash
|
||||
npm run seed
|
||||
```
|
||||
|
||||
The seed script performs the following actions:
|
||||
|
||||
1. Rebuilds the database schema from `sql/master_schema_rollup.sql`
|
||||
2. Creates test user accounts (admin and regular user)
|
||||
3. Copies test flyer images from `src/tests/assets/` to `public/flyer-images/`
|
||||
4. Creates a sample flyer with items linked to the test images
|
||||
5. Seeds watched items and a shopping list for the test user
|
||||
|
||||
**Test Images**: The seed script copies `test-flyer-image.jpg` and `test-flyer-icon.png` to the `public/flyer-images/` directory, which is served by NGINX at `/flyer-images/`.
|
||||
|
||||
After running, you may need to restart your IDE's TypeScript server to pick up any generated types.
|
||||
|
||||
---
|
||||
186
docs/getting-started/QUICKSTART.md
Normal file
186
docs/getting-started/QUICKSTART.md
Normal file
@@ -0,0 +1,186 @@
|
||||
# Quick Start Guide
|
||||
|
||||
Get Flyer Crawler running in 5 minutes.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **Windows 10/11** with WSL 2
|
||||
- **Podman Desktop** installed
|
||||
- **Node.js 20+** installed
|
||||
|
||||
## 1. Start Containers (1 minute)
|
||||
|
||||
```bash
|
||||
# Start PostgreSQL and Redis
|
||||
podman start flyer-crawler-postgres flyer-crawler-redis
|
||||
|
||||
# If containers don't exist yet, create them:
|
||||
podman run -d --name flyer-crawler-postgres \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=flyer_crawler_dev \
|
||||
-p 5432:5432 \
|
||||
docker.io/postgis/postgis:15-3.3
|
||||
|
||||
podman run -d --name flyer-crawler-redis \
|
||||
-p 6379:6379 \
|
||||
docker.io/library/redis:alpine
|
||||
```
|
||||
|
||||
## 2. Initialize Database (2 minutes)
|
||||
|
||||
```bash
|
||||
# Wait for PostgreSQL to be ready
|
||||
podman exec flyer-crawler-postgres pg_isready -U postgres
|
||||
|
||||
# Install extensions
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev \
|
||||
-c "CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS pg_trgm; CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";"
|
||||
|
||||
# Apply schema
|
||||
podman exec -i flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev < sql/master_schema_rollup.sql
|
||||
```
|
||||
|
||||
## 3. Configure Environment (1 minute)
|
||||
|
||||
Create `.env.local` in the project root:
|
||||
|
||||
```bash
|
||||
# Database
|
||||
DB_HOST=localhost
|
||||
DB_USER=postgres
|
||||
DB_PASSWORD=postgres
|
||||
DB_NAME=flyer_crawler_dev
|
||||
DB_PORT=5432
|
||||
|
||||
# Redis
|
||||
REDIS_URL=redis://localhost:6379
|
||||
|
||||
# Application
|
||||
NODE_ENV=development
|
||||
PORT=3001
|
||||
FRONTEND_URL=http://localhost:5173
|
||||
|
||||
# Secrets (generate your own)
|
||||
JWT_SECRET=your-dev-jwt-secret-at-least-32-chars-long
|
||||
SESSION_SECRET=your-dev-session-secret-at-least-32-chars-long
|
||||
|
||||
# AI Services (get your own keys)
|
||||
VITE_GOOGLE_GENAI_API_KEY=your-google-genai-api-key
|
||||
GOOGLE_MAPS_API_KEY=your-google-maps-api-key
|
||||
```
|
||||
|
||||
## 4. Install & Run (1 minute)
|
||||
|
||||
```bash
|
||||
# Install dependencies (first time only)
|
||||
npm install
|
||||
|
||||
# Start development server
|
||||
npm run dev
|
||||
```
|
||||
|
||||
## 5. Access Application
|
||||
|
||||
- **Frontend**: http://localhost:5173
|
||||
- **Backend API**: http://localhost:3001
|
||||
- **Health Check**: http://localhost:3001/health
|
||||
|
||||
### Dev Container (HTTPS)
|
||||
|
||||
When using the full dev container with NGINX, access via HTTPS:
|
||||
|
||||
- **Frontend**: https://localhost or https://127.0.0.1
|
||||
- **Backend API**: http://localhost:3001
|
||||
- **Bugsink**: `https://localhost:8443` (error tracking)
|
||||
|
||||
**Note:** The dev container accepts both `localhost` and `127.0.0.1` for HTTPS connections. The self-signed certificate is valid for both hostnames.
|
||||
|
||||
**SSL Certificate Warnings:** To eliminate browser security warnings for self-signed certificates, install the mkcert CA certificate. See [`certs/README.md`](../../certs/README.md) for platform-specific installation instructions. This is optional but recommended for a better development experience.
|
||||
|
||||
### Dev Container Architecture
|
||||
|
||||
The dev container uses PM2 for process management, matching production (ADR-014):
|
||||
|
||||
| Process | Description | Port |
|
||||
| -------------------------- | ------------------------ | ---- |
|
||||
| `flyer-crawler-api-dev` | API server (tsx watch) | 3001 |
|
||||
| `flyer-crawler-worker-dev` | Background job worker | - |
|
||||
| `flyer-crawler-vite-dev` | Vite frontend dev server | 5173 |
|
||||
|
||||
**PM2 Commands** (run inside container):
|
||||
|
||||
```bash
|
||||
# View process status
|
||||
podman exec -it flyer-crawler-dev pm2 status
|
||||
|
||||
# View logs (all processes)
|
||||
podman exec -it flyer-crawler-dev pm2 logs
|
||||
|
||||
# Restart all processes
|
||||
podman exec -it flyer-crawler-dev pm2 restart all
|
||||
|
||||
# Restart specific process
|
||||
podman exec -it flyer-crawler-dev pm2 restart flyer-crawler-api-dev
|
||||
```
|
||||
|
||||
## Verify Installation
|
||||
|
||||
```bash
|
||||
# Check containers are running
|
||||
podman ps
|
||||
|
||||
# Test database connection
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT version();"
|
||||
|
||||
# Run tests (in dev container)
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
|
||||
### "Unable to connect to Podman socket"
|
||||
|
||||
```bash
|
||||
podman machine start
|
||||
```
|
||||
|
||||
### "Connection refused" to PostgreSQL
|
||||
|
||||
Wait a few seconds for PostgreSQL to initialize:
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-postgres pg_isready -U postgres
|
||||
```
|
||||
|
||||
### Port 5432 or 6379 already in use
|
||||
|
||||
Stop conflicting services or change port mappings:
|
||||
|
||||
```bash
|
||||
# Use different host port
|
||||
podman run -d --name flyer-crawler-postgres -p 5433:5432 ...
|
||||
```
|
||||
|
||||
Then update `DB_PORT=5433` in `.env.local`.
|
||||
|
||||
## Next Steps
|
||||
|
||||
- **Read the docs**: [docs/README.md](../README.md)
|
||||
- **Understand the architecture**: [docs/architecture/DATABASE.md](../architecture/DATABASE.md)
|
||||
- **Learn testing**: [docs/development/TESTING.md](../development/TESTING.md)
|
||||
- **Explore ADRs**: [docs/adr/index.md](../adr/index.md)
|
||||
- **Contributing**: [CONTRIBUTING.md](../../CONTRIBUTING.md)
|
||||
|
||||
## Development Workflow
|
||||
|
||||
```bash
|
||||
# Daily workflow
|
||||
podman start flyer-crawler-postgres flyer-crawler-redis
|
||||
npm run dev
|
||||
# ... make changes ...
|
||||
npm test
|
||||
git commit
|
||||
```
|
||||
|
||||
For detailed setup instructions, see [INSTALL.md](INSTALL.md).
|
||||
@@ -369,6 +369,17 @@ pm2 delete flyer-crawler-api-test flyer-crawler-worker-test flyer-crawler-analyt
|
||||
sudo apt install -y nginx
|
||||
```
|
||||
|
||||
### Reference Configuration Files
|
||||
|
||||
The repository contains reference copies of the actual production NGINX configurations at the project root:
|
||||
|
||||
- `etc-nginx-sites-available-flyer-crawler.projectium.com` - Production config
|
||||
- `etc-nginx-sites-available-flyer-crawler-test-projectium-com.txt` - Test config
|
||||
|
||||
These reference files document the exact configuration deployed on the server, including SSL settings managed by Certbot. Use them as a reference when setting up new servers or troubleshooting configuration issues.
|
||||
|
||||
**Note:** The simplified example below shows the basic structure. For the complete production configuration with SSL, security headers, and all location blocks, refer to the reference files in the repository root.
|
||||
|
||||
### Create Site Configuration
|
||||
|
||||
Create `/etc/nginx/sites-available/flyer-crawler.projectium.com`:
|
||||
@@ -408,6 +419,13 @@ server {
|
||||
client_max_body_size 50M;
|
||||
}
|
||||
|
||||
# Serve flyer images from static storage (7-day cache)
|
||||
location /flyer-images/ {
|
||||
alias /var/www/flyer-crawler.projectium.com/flyer-images/;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# MIME type fix for .mjs files
|
||||
types {
|
||||
application/javascript js mjs;
|
||||
@@ -415,6 +433,26 @@ server {
|
||||
}
|
||||
```
|
||||
|
||||
### Static Flyer Images Directory
|
||||
|
||||
Create the directory for storing flyer images:
|
||||
|
||||
```bash
|
||||
# Production
|
||||
sudo mkdir -p /var/www/flyer-crawler.projectium.com/flyer-images
|
||||
sudo chown www-data:www-data /var/www/flyer-crawler.projectium.com/flyer-images
|
||||
|
||||
# Test environment
|
||||
sudo mkdir -p /var/www/flyer-crawler-test.projectium.com/flyer-images
|
||||
sudo chown www-data:www-data /var/www/flyer-crawler-test.projectium.com/flyer-images
|
||||
```
|
||||
|
||||
The `/flyer-images/` location serves static images with:
|
||||
|
||||
- **7-day browser cache** (`expires 7d`)
|
||||
- **Immutable cache header** for optimal CDN/browser caching
|
||||
- Direct file serving (no proxy overhead)
|
||||
|
||||
### Enable the Site
|
||||
|
||||
```bash
|
||||
@@ -11,6 +11,22 @@ This guide covers deploying Flyer Crawler to a production server.
|
||||
- NGINX (reverse proxy)
|
||||
- PM2 (process manager)
|
||||
|
||||
## Dev Container Parity (ADR-014)
|
||||
|
||||
The development container now matches production architecture:
|
||||
|
||||
| Component | Production | Dev Container |
|
||||
| ------------ | ---------------- | -------------------------- |
|
||||
| Process Mgmt | PM2 | PM2 |
|
||||
| API Server | PM2 cluster mode | PM2 fork + tsx watch |
|
||||
| Worker | PM2 process | PM2 process + tsx watch |
|
||||
| Logs | PM2 -> Logstash | PM2 -> Logstash -> Bugsink |
|
||||
| HTTPS | Let's Encrypt | Self-signed (mkcert) |
|
||||
|
||||
This ensures issues caught in dev will behave the same in production.
|
||||
|
||||
**Dev Container Guide**: See [docs/development/DEV-CONTAINER.md](../development/DEV-CONTAINER.md)
|
||||
|
||||
---
|
||||
|
||||
## Server Setup
|
||||
@@ -80,6 +96,22 @@ For deployments using Gitea CI/CD workflows, configure these as **repository sec
|
||||
|
||||
## NGINX Configuration
|
||||
|
||||
### Reference Configuration Files
|
||||
|
||||
The repository contains reference copies of the production NGINX configurations for documentation and version control purposes:
|
||||
|
||||
| File | Server Config |
|
||||
| ----------------------------------------------------------------- | ------------------------- |
|
||||
| `etc-nginx-sites-available-flyer-crawler.projectium.com` | Production NGINX config |
|
||||
| `etc-nginx-sites-available-flyer-crawler-test-projectium-com.txt` | Test/staging NGINX config |
|
||||
|
||||
**Important Notes:**
|
||||
|
||||
- These are **reference copies only** - they are not used directly by NGINX
|
||||
- The actual live configurations reside on the server at `/etc/nginx/sites-available/`
|
||||
- When modifying server NGINX configs, update these reference files to keep them in sync
|
||||
- Use the dev container config at `docker/nginx/dev.conf` for local development
|
||||
|
||||
### Reverse Proxy Setup
|
||||
|
||||
Create a site configuration at `/etc/nginx/sites-available/flyer-crawler.projectium.com`:
|
||||
@@ -106,9 +138,35 @@ server {
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
}
|
||||
|
||||
# Serve flyer images from static storage (7-day cache)
|
||||
location /flyer-images/ {
|
||||
alias /var/www/flyer-crawler.projectium.com/flyer-images/;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Static Flyer Images
|
||||
|
||||
Flyer images are served as static files from the `/flyer-images/` path with browser caching enabled:
|
||||
|
||||
| Environment | Directory | URL Pattern |
|
||||
| ------------- | ---------------------------------------------------------- | --------------------------------------------------------- |
|
||||
| Production | `/var/www/flyer-crawler.projectium.com/flyer-images/` | `https://flyer-crawler.projectium.com/flyer-images/` |
|
||||
| Test | `/var/www/flyer-crawler-test.projectium.com/flyer-images/` | `https://flyer-crawler-test.projectium.com/flyer-images/` |
|
||||
| Dev Container | `/app/public/flyer-images/` | `https://localhost/flyer-images/` |
|
||||
|
||||
**Cache Settings**: Files are served with `expires 7d` and `Cache-Control: public, immutable` headers for optimal browser caching.
|
||||
|
||||
Create the flyer images directory if it does not exist:
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /var/www/flyer-crawler.projectium.com/flyer-images
|
||||
sudo chown www-data:www-data /var/www/flyer-crawler.projectium.com/flyer-images
|
||||
```
|
||||
|
||||
Enable the site:
|
||||
|
||||
```bash
|
||||
111
docs/operations/LOGSTASH-QUICK-REF.md
Normal file
111
docs/operations/LOGSTASH-QUICK-REF.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# Logstash Quick Reference (ADR-050)
|
||||
|
||||
Aggregates logs from PostgreSQL, PM2, Redis, NGINX; forwards errors to Bugsink.
|
||||
|
||||
## Configuration
|
||||
|
||||
**Primary config**: `/etc/logstash/conf.d/bugsink.conf`
|
||||
|
||||
### Related Files
|
||||
|
||||
| Path | Purpose |
|
||||
| --------------------------------------------------- | ------------------------- |
|
||||
| `/etc/postgresql/14/main/conf.d/observability.conf` | PostgreSQL logging config |
|
||||
| `/var/log/postgresql/*.log` | PostgreSQL logs |
|
||||
| `/home/gitea-runner/.pm2/logs/*.log` | PM2 worker logs |
|
||||
| `/var/log/redis/redis-server.log` | Redis logs |
|
||||
| `/var/log/nginx/access.log` | NGINX access logs |
|
||||
| `/var/log/nginx/error.log` | NGINX error logs |
|
||||
| `/var/log/logstash/*.log` | Logstash file outputs |
|
||||
| `/var/lib/logstash/sincedb_*` | Position tracking files |
|
||||
|
||||
## Features
|
||||
|
||||
- **Multi-source aggregation**: PostgreSQL, PM2 workers, Redis, NGINX
|
||||
- **Environment routing**: Auto-detects prod/test, routes to correct Bugsink project
|
||||
- **JSON parsing**: Extracts `fn_log()` from PostgreSQL, Pino JSON from PM2
|
||||
- **Sentry format**: Transforms to `event_id`, `timestamp`, `level`, `message`, `extra`
|
||||
- **Error filtering**: Only forwards WARNING/ERROR to Bugsink
|
||||
- **Operational storage**: Non-error logs saved to `/var/log/logstash/`
|
||||
- **Request monitoring**: NGINX requests categorized by status, slow request detection
|
||||
|
||||
## Commands
|
||||
|
||||
### Production (Bare Metal)
|
||||
|
||||
```bash
|
||||
# Status and control
|
||||
systemctl status logstash
|
||||
systemctl restart logstash
|
||||
|
||||
# Test configuration
|
||||
/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf
|
||||
|
||||
# View logs
|
||||
journalctl -u logstash -f
|
||||
|
||||
# Check stats (events processed, failures)
|
||||
curl -XGET 'localhost:9600/_node/stats/pipelines?pretty' | jq '.pipelines.main.plugins.filters'
|
||||
|
||||
# Monitor sources
|
||||
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log
|
||||
tail -f /var/log/logstash/pm2-workers-$(date +%Y-%m-%d).log
|
||||
tail -f /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
|
||||
tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
|
||||
|
||||
# Check disk usage
|
||||
du -sh /var/log/logstash/
|
||||
```
|
||||
|
||||
### Dev Container
|
||||
|
||||
```bash
|
||||
# View Logstash logs (inside container)
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log
|
||||
|
||||
# Check PM2 API logs are being processed (ADR-014)
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-api-$(date +%Y-%m-%d).log
|
||||
|
||||
# Check PM2 Worker logs are being processed
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/pm2-worker-$(date +%Y-%m-%d).log
|
||||
|
||||
# Check Redis logs are being processed
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
|
||||
|
||||
# View raw PM2 logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/api-out.log
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev cat /var/log/pm2/worker-out.log
|
||||
|
||||
# View raw Redis logs
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-redis cat /var/log/redis/redis-server.log
|
||||
|
||||
# Check Logstash stats
|
||||
podman exec flyer-crawler-dev curl -s localhost:9600/_node/stats/pipelines?pretty
|
||||
|
||||
# Verify shared volume mounts
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev ls -la /var/log/pm2/
|
||||
MSYS_NO_PATHCONV=1 podman exec flyer-crawler-dev ls -la /var/log/redis/
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Issue | Check | Solution |
|
||||
| --------------------- | ---------------- | ---------------------------------------------------------------------------------------------- |
|
||||
| No Bugsink errors | Logstash running | `systemctl status logstash` |
|
||||
| Config syntax error | Test config | `/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf` |
|
||||
| Grok pattern failures | Stats endpoint | `curl localhost:9600/_node/stats/pipelines?pretty \| jq '.pipelines.main.plugins.filters'` |
|
||||
| Wrong Bugsink project | Env detection | Check tags in logs match expected environment |
|
||||
| Permission denied | Logstash groups | `groups logstash` should include `postgres`, `adm` |
|
||||
| PM2 not captured | File paths | Dev: `ls /var/log/pm2/`; Prod: `ls /home/gitea-runner/.pm2/logs/` |
|
||||
| PM2 logs empty | PM2 running | Dev: `podman exec flyer-crawler-dev pm2 status`; Prod: `pm2 status` |
|
||||
| NGINX logs missing | Output directory | `ls -lh /var/log/logstash/nginx-access-*.log` |
|
||||
| Redis logs missing | Shared volume | Dev: Check `redis_logs` volume mounted; Prod: Check `/var/log/redis/redis-server.log` exists |
|
||||
| High disk usage | Log rotation | Verify `/etc/logrotate.d/logstash` configured |
|
||||
| varchar(7) error | Level validation | Add Ruby filter to validate/normalize `sentry_level` before output |
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- **Dev Container Guide**: [DEV-CONTAINER.md](../development/DEV-CONTAINER.md) - PM2 and log aggregation in dev
|
||||
- **Full setup**: [BARE-METAL-SETUP.md](BARE-METAL-SETUP.md) - PostgreSQL Function Observability section
|
||||
- **Architecture**: [adr/0050-postgresql-function-observability.md](../adr/0050-postgresql-function-observability.md)
|
||||
- **Troubleshooting details**: [LOGSTASH-TROUBLESHOOTING.md](LOGSTASH-TROUBLESHOOTING.md)
|
||||
@@ -11,6 +11,7 @@ This runbook provides step-by-step diagnostics and solutions for common Logstash
|
||||
| Wrong Bugsink project | Environment detection failed | Verify `pg_database` field extraction |
|
||||
| 403 authentication error | Missing/wrong DSN key | Check `X-Sentry-Auth` header |
|
||||
| 500 error from Bugsink | Invalid event format | Verify `event_id` and required fields |
|
||||
| varchar(7) constraint | Unresolved `%{sentry_level}` | Add Ruby filter for level validation |
|
||||
|
||||
---
|
||||
|
||||
@@ -385,7 +386,88 @@ systemctl status logstash
|
||||
|
||||
---
|
||||
|
||||
### Issue 7: Log File Rotation Issues
|
||||
### Issue 7: Level Field Constraint Violation (varchar(7))
|
||||
|
||||
**Symptoms:**
|
||||
|
||||
- Bugsink returns HTTP 500 errors
|
||||
- PostgreSQL errors: `value too long for type character varying(7)`
|
||||
- Events fail to insert with literal `%{sentry_level}` string (16 characters)
|
||||
|
||||
**Root Cause:**
|
||||
|
||||
When Logstash cannot determine the log level (no error patterns matched), the `sentry_level` field remains as the unresolved placeholder `%{sentry_level}`. Bugsink's PostgreSQL schema has a `varchar(7)` constraint on the level field.
|
||||
|
||||
Valid Sentry levels (all <= 7 characters): `fatal`, `error`, `warning`, `info`, `debug`
|
||||
|
||||
**Diagnosis:**
|
||||
|
||||
```bash
|
||||
# Check for HTTP 500 responses in Logstash logs
|
||||
podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log | grep "500"
|
||||
|
||||
# Check Bugsink for constraint violation errors
|
||||
# Via MCP:
|
||||
mcp__localerrors__list_issues({ project_id: 1, status: 'unresolved' })
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
|
||||
Add a Ruby filter block in `docker/logstash/bugsink.conf` to validate and normalize the `sentry_level` field before sending to Bugsink:
|
||||
|
||||
```ruby
|
||||
# Add this AFTER all mutate filters that set sentry_level
|
||||
# and BEFORE the output section
|
||||
|
||||
ruby {
|
||||
code => '
|
||||
level = event.get("sentry_level")
|
||||
# Check if level is invalid (nil, empty, contains placeholder, or too long)
|
||||
if level.nil? || level.to_s.empty? || level.to_s.include?("%{") || level.to_s.length > 7
|
||||
# Default to "error" for error-tagged events, "info" otherwise
|
||||
if event.get("tags")&.include?("error")
|
||||
event.set("sentry_level", "error")
|
||||
else
|
||||
event.set("sentry_level", "info")
|
||||
end
|
||||
else
|
||||
# Normalize to lowercase and validate
|
||||
normalized = level.to_s.downcase
|
||||
valid_levels = ["fatal", "error", "warning", "info", "debug"]
|
||||
unless valid_levels.include?(normalized)
|
||||
normalized = "error"
|
||||
end
|
||||
event.set("sentry_level", normalized)
|
||||
end
|
||||
'
|
||||
}
|
||||
```
|
||||
|
||||
**Key validations performed:**
|
||||
|
||||
1. Checks for nil or empty values
|
||||
2. Detects unresolved placeholders (`%{...}`)
|
||||
3. Enforces 7-character maximum length
|
||||
4. Normalizes to lowercase
|
||||
5. Validates against allowed Sentry levels
|
||||
6. Defaults to "error" for error-tagged events, "info" otherwise
|
||||
|
||||
**Verification:**
|
||||
|
||||
```bash
|
||||
# Restart Logstash
|
||||
podman exec flyer-crawler-dev systemctl restart logstash
|
||||
|
||||
# Generate a test log that triggers the filter
|
||||
podman exec flyer-crawler-dev pm2 restart flyer-crawler-api-dev
|
||||
|
||||
# Check no new HTTP 500 errors
|
||||
podman exec flyer-crawler-dev cat /var/log/logstash/logstash.log | tail -50 | grep -E "(500|error)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Issue 8: Log File Rotation Issues
|
||||
|
||||
**Symptoms:**
|
||||
|
||||
896
docs/operations/MONITORING.md
Normal file
896
docs/operations/MONITORING.md
Normal file
@@ -0,0 +1,896 @@
|
||||
# Monitoring Guide
|
||||
|
||||
This guide covers all aspects of monitoring the Flyer Crawler application across development, test, and production environments.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Health Checks](#health-checks)
|
||||
2. [Bugsink Error Tracking](#bugsink-error-tracking)
|
||||
3. [Logstash Log Aggregation](#logstash-log-aggregation)
|
||||
4. [PM2 Process Monitoring](#pm2-process-monitoring)
|
||||
5. [Database Monitoring](#database-monitoring)
|
||||
6. [Redis Monitoring](#redis-monitoring)
|
||||
7. [Production Alerts and On-Call](#production-alerts-and-on-call)
|
||||
|
||||
---
|
||||
|
||||
## Health Checks
|
||||
|
||||
The application exposes health check endpoints at `/api/health/*` implementing ADR-020.
|
||||
|
||||
### Endpoint Reference
|
||||
|
||||
| Endpoint | Purpose | Use Case |
|
||||
| ----------------------- | ---------------------- | --------------------------------------- |
|
||||
| `/api/health/ping` | Simple connectivity | Quick "is it running?" check |
|
||||
| `/api/health/live` | Liveness probe | Container orchestration restart trigger |
|
||||
| `/api/health/ready` | Readiness probe | Load balancer traffic routing |
|
||||
| `/api/health/startup` | Startup probe | Initial container readiness |
|
||||
| `/api/health/db-schema` | Schema verification | Deployment validation |
|
||||
| `/api/health/db-pool` | Connection pool status | Performance diagnostics |
|
||||
| `/api/health/redis` | Redis connectivity | Cache/queue health |
|
||||
| `/api/health/storage` | File storage access | Upload capability |
|
||||
| `/api/health/time` | Server time sync | Time-sensitive operations |
|
||||
|
||||
### Liveness Probe (`/api/health/live`)
|
||||
|
||||
Returns 200 OK if the Node.js process is running. No external dependencies.
|
||||
|
||||
```bash
|
||||
# Check liveness
|
||||
curl -s https://flyer-crawler.projectium.com/api/health/live | jq .
|
||||
|
||||
# Expected response
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"status": "ok",
|
||||
"timestamp": "2026-01-22T10:00:00.000Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Usage**: If this endpoint fails, restart the application immediately.
|
||||
|
||||
### Readiness Probe (`/api/health/ready`)
|
||||
|
||||
Comprehensive check of all critical dependencies: database, Redis, and storage.
|
||||
|
||||
```bash
|
||||
# Check readiness
|
||||
curl -s https://flyer-crawler.projectium.com/api/health/ready | jq .
|
||||
|
||||
# Expected healthy response (200)
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"status": "healthy",
|
||||
"timestamp": "2026-01-22T10:00:00.000Z",
|
||||
"uptime": 3600.5,
|
||||
"services": {
|
||||
"database": {
|
||||
"status": "healthy",
|
||||
"latency": 5,
|
||||
"details": {
|
||||
"totalConnections": 10,
|
||||
"idleConnections": 8,
|
||||
"waitingConnections": 0
|
||||
}
|
||||
},
|
||||
"redis": {
|
||||
"status": "healthy",
|
||||
"latency": 2
|
||||
},
|
||||
"storage": {
|
||||
"status": "healthy",
|
||||
"latency": 1,
|
||||
"details": {
|
||||
"path": "/var/www/flyer-crawler.projectium.com/flyer-images"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Status Values**:
|
||||
|
||||
| Status | Meaning | Action |
|
||||
| ----------- | ------------------------------------------------ | ------------------------- |
|
||||
| `healthy` | All critical services operational | None required |
|
||||
| `degraded` | Non-critical issues (e.g., high connection wait) | Monitor closely |
|
||||
| `unhealthy` | Critical service unavailable (returns 503) | Remove from load balancer |
|
||||
|
||||
### Database Health Thresholds
|
||||
|
||||
| Metric | Healthy | Degraded | Unhealthy |
|
||||
| ------------------- | ------------------- | -------- | ---------------- |
|
||||
| Query response | `SELECT 1` succeeds | N/A | Connection fails |
|
||||
| Waiting connections | 0-3 | 4+ | N/A |
|
||||
|
||||
### Verifying Services from CLI
|
||||
|
||||
**Production**:
|
||||
|
||||
```bash
|
||||
# Quick health check
|
||||
curl -s https://flyer-crawler.projectium.com/api/health/ready | jq '.data.status'
|
||||
|
||||
# Database pool status
|
||||
curl -s https://flyer-crawler.projectium.com/api/health/db-pool | jq .
|
||||
|
||||
# Redis health
|
||||
curl -s https://flyer-crawler.projectium.com/api/health/redis | jq .
|
||||
```
|
||||
|
||||
**Test Environment**:
|
||||
|
||||
```bash
|
||||
# Test environment runs on port 3002
|
||||
curl -s https://flyer-crawler-test.projectium.com/api/health/ready | jq .
|
||||
```
|
||||
|
||||
**Dev Container**:
|
||||
|
||||
```bash
|
||||
# From inside the container
|
||||
curl -s http://localhost:3001/api/health/ready | jq .
|
||||
|
||||
# From Windows host (via port mapping)
|
||||
curl -s http://localhost:3001/api/health/ready | jq .
|
||||
```
|
||||
|
||||
### Admin System Check UI
|
||||
|
||||
The admin dashboard at `/admin` includes a **System Check** component that runs all health checks with a visual interface:
|
||||
|
||||
1. Navigate to `https://flyer-crawler.projectium.com/admin`
|
||||
2. Login with admin credentials
|
||||
3. View the "System Check" section
|
||||
4. Click "Re-run Checks" to verify all services
|
||||
|
||||
Checks include:
|
||||
|
||||
- Backend Server Connection
|
||||
- PM2 Process Status
|
||||
- Database Connection Pool
|
||||
- Redis Connection
|
||||
- Database Schema
|
||||
- Default Admin User
|
||||
- Assets Storage Directory
|
||||
- Gemini API Key
|
||||
|
||||
---
|
||||
|
||||
## Bugsink Error Tracking
|
||||
|
||||
Bugsink is our self-hosted, Sentry-compatible error tracking system (ADR-015).
|
||||
|
||||
### Access Points
|
||||
|
||||
| Environment | URL | Purpose |
|
||||
| ----------------- | -------------------------------- | -------------------------- |
|
||||
| **Production** | `https://bugsink.projectium.com` | Production and test errors |
|
||||
| **Dev Container** | `https://localhost:8443` | Local development errors |
|
||||
|
||||
### Credentials
|
||||
|
||||
**Production Bugsink**:
|
||||
|
||||
- Credentials stored in password manager
|
||||
- Admin account created during initial deployment
|
||||
|
||||
**Dev Container Bugsink**:
|
||||
|
||||
- Email: `admin@localhost`
|
||||
- Password: `admin`
|
||||
|
||||
### Projects
|
||||
|
||||
| Project ID | Name | Environment | Error Source |
|
||||
| ---------- | --------------------------------- | ----------- | ------------------------------- |
|
||||
| 1 | flyer-crawler-backend | Production | Backend Node.js errors |
|
||||
| 2 | flyer-crawler-frontend | Production | Frontend JavaScript errors |
|
||||
| 3 | flyer-crawler-backend-test | Test | Test environment backend |
|
||||
| 4 | flyer-crawler-frontend-test | Test | Test environment frontend |
|
||||
| 5 | flyer-crawler-infrastructure | Production | PostgreSQL, Redis, NGINX errors |
|
||||
| 6 | flyer-crawler-test-infrastructure | Test | Test infra errors |
|
||||
|
||||
**Dev Container Projects** (localhost:8000):
|
||||
|
||||
- Project 1: Backend (Dev)
|
||||
- Project 2: Frontend (Dev)
|
||||
|
||||
### Accessing Errors via Web UI
|
||||
|
||||
1. Navigate to the Bugsink URL
|
||||
2. Login with credentials
|
||||
3. Select project from the sidebar
|
||||
4. Click on an issue to view details
|
||||
|
||||
**Issue Details Include**:
|
||||
|
||||
- Exception type and message
|
||||
- Full stack trace
|
||||
- Request context (URL, method, headers)
|
||||
- User context (if authenticated)
|
||||
- Occurrence statistics (first seen, last seen, count)
|
||||
- Release/version information
|
||||
|
||||
### Accessing Errors via MCP
|
||||
|
||||
Claude Code and other AI tools can access Bugsink via MCP servers.
|
||||
|
||||
**Available MCP Tools**:
|
||||
|
||||
```bash
|
||||
# List all projects
|
||||
mcp__bugsink__list_projects
|
||||
|
||||
# List unresolved issues for a project
|
||||
mcp__bugsink__list_issues --project_id 1 --status unresolved
|
||||
|
||||
# Get issue details
|
||||
mcp__bugsink__get_issue --issue_id <uuid>
|
||||
|
||||
# Get stacktrace (pre-rendered Markdown)
|
||||
mcp__bugsink__get_stacktrace --event_id <uuid>
|
||||
|
||||
# List events for an issue
|
||||
mcp__bugsink__list_events --issue_id <uuid>
|
||||
```
|
||||
|
||||
**MCP Server Configuration**:
|
||||
|
||||
Production (in `~/.claude/settings.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"bugsink": {
|
||||
"command": "node",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "https://bugsink.projectium.com",
|
||||
"BUGSINK_TOKEN": "<token>"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Dev Container (in `.mcp.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"localerrors": {
|
||||
"command": "node",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "http://127.0.0.1:8000",
|
||||
"BUGSINK_TOKEN": "<token>"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Creating API Tokens
|
||||
|
||||
Bugsink 2.0.11 does not have a UI for API tokens. Create via Django management command.
|
||||
|
||||
**Production**:
|
||||
|
||||
```bash
|
||||
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
|
||||
```
|
||||
|
||||
**Dev Container**:
|
||||
|
||||
```bash
|
||||
MSYS_NO_PATHCONV=1 podman exec -e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink -e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && DJANGO_SETTINGS_MODULE=bugsink_conf PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages /opt/bugsink/bin/python -m django create_auth_token'
|
||||
```
|
||||
|
||||
The command outputs a 40-character hex token.
|
||||
|
||||
### Interpreting Errors
|
||||
|
||||
**Error Anatomy**:
|
||||
|
||||
```
|
||||
TypeError: Cannot read properties of undefined (reading 'map')
|
||||
├── Exception Type: TypeError
|
||||
├── Message: Cannot read properties of undefined (reading 'map')
|
||||
├── Where: FlyerItemsList.tsx:45:23
|
||||
├── When: 2026-01-22T10:30:00.000Z
|
||||
├── Count: 12 occurrences
|
||||
└── Context:
|
||||
├── URL: GET /api/flyers/123/items
|
||||
├── User: user@example.com
|
||||
└── Release: v0.12.5
|
||||
```
|
||||
|
||||
**Common Error Patterns**:
|
||||
|
||||
| Pattern | Likely Cause | Investigation |
|
||||
| ----------------------------------- | ------------------------------------------------- | -------------------------------------------------- |
|
||||
| `TypeError: ... undefined` | Missing null check, API returned unexpected shape | Check API response, add defensive coding |
|
||||
| `DatabaseError: Connection timeout` | Pool exhaustion, slow queries | Check `/api/health/db-pool`, review slow query log |
|
||||
| `RedisConnectionError` | Redis unavailable | Check Redis service, network connectivity |
|
||||
| `ValidationError: ...` | Invalid input, schema mismatch | Review request payload, update validation |
|
||||
| `NotFoundError: ...` | Missing resource | Verify resource exists, check ID format |
|
||||
|
||||
### Error Triage Workflow
|
||||
|
||||
1. **Review new issues daily** in Bugsink
|
||||
2. **Categorize by severity**:
|
||||
- **Critical**: Data corruption, security, payment failures
|
||||
- **High**: Core feature broken for many users
|
||||
- **Medium**: Feature degraded, workaround available
|
||||
- **Low**: Minor UX issues, cosmetic bugs
|
||||
3. **Check occurrence count** - frequent errors need urgent attention
|
||||
4. **Review stack trace** - identify root cause
|
||||
5. **Check recent deployments** - did a release introduce this?
|
||||
6. **Create Gitea issue** if not auto-synced
|
||||
|
||||
### Bugsink-to-Gitea Sync
|
||||
|
||||
The test environment automatically syncs Bugsink issues to Gitea (see `docs/BUGSINK-SYNC.md`).
|
||||
|
||||
**Sync Workflow**:
|
||||
|
||||
1. Runs every 15 minutes on test server
|
||||
2. Fetches unresolved issues from all Bugsink projects
|
||||
3. Creates Gitea issues with appropriate labels
|
||||
4. Marks synced issues as resolved in Bugsink
|
||||
|
||||
**Manual Sync**:
|
||||
|
||||
```bash
|
||||
# Trigger sync via API (test environment only)
|
||||
curl -X POST https://flyer-crawler-test.projectium.com/api/admin/bugsink/sync \
|
||||
-H "Authorization: Bearer <admin_jwt>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Logstash Log Aggregation
|
||||
|
||||
Logstash aggregates logs from multiple sources and forwards errors to Bugsink (ADR-050).
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
Log Sources Logstash Outputs
|
||||
┌──────────────┐ ┌─────────────┐ ┌─────────────┐
|
||||
│ PostgreSQL │──────────────│ │───────────│ Bugsink │
|
||||
│ PM2 Workers │──────────────│ Filter │───────────│ (errors) │
|
||||
│ Redis │──────────────│ & Route │───────────│ │
|
||||
│ NGINX │──────────────│ │───────────│ File Logs │
|
||||
└──────────────┘ └─────────────┘ │ (all logs) │
|
||||
└─────────────┘
|
||||
```
|
||||
|
||||
### Configuration Files
|
||||
|
||||
| Path | Purpose |
|
||||
| --------------------------------------------------- | --------------------------- |
|
||||
| `/etc/logstash/conf.d/bugsink.conf` | Main pipeline configuration |
|
||||
| `/etc/postgresql/14/main/conf.d/observability.conf` | PostgreSQL logging settings |
|
||||
| `/var/log/logstash/` | Logstash file outputs |
|
||||
| `/var/lib/logstash/sincedb_*` | File position tracking |
|
||||
|
||||
### Log Sources
|
||||
|
||||
| Source | Path | Contents |
|
||||
| ----------- | -------------------------------------------------- | ----------------------------------- |
|
||||
| PostgreSQL | `/var/log/postgresql/*.log` | Function logs, slow queries, errors |
|
||||
| PM2 Workers | `/home/gitea-runner/.pm2/logs/flyer-crawler-*.log` | Worker stdout/stderr |
|
||||
| Redis | `/var/log/redis/redis-server.log` | Connection errors, memory warnings |
|
||||
| NGINX | `/var/log/nginx/access.log`, `error.log` | HTTP requests, upstream errors |
|
||||
|
||||
### Pipeline Status
|
||||
|
||||
**Check Logstash Service**:
|
||||
|
||||
```bash
|
||||
ssh root@projectium.com
|
||||
|
||||
# Service status
|
||||
systemctl status logstash
|
||||
|
||||
# Recent logs
|
||||
journalctl -u logstash -n 50 --no-pager
|
||||
|
||||
# Pipeline statistics
|
||||
curl -s http://localhost:9600/_node/stats/pipelines?pretty | jq '.pipelines.main.events'
|
||||
|
||||
# Events processed today
|
||||
curl -s http://localhost:9600/_node/stats/pipelines?pretty | jq '{
|
||||
in: .pipelines.main.events.in,
|
||||
out: .pipelines.main.events.out,
|
||||
filtered: .pipelines.main.events.filtered
|
||||
}'
|
||||
```
|
||||
|
||||
**Check Filter Performance**:
|
||||
|
||||
```bash
|
||||
# Grok pattern success/failure rates
|
||||
curl -s http://localhost:9600/_node/stats/pipelines?pretty | \
|
||||
jq '.pipelines.main.plugins.filters[] | select(.name == "grok") | {name, events_in: .events.in, events_out: .events.out, failures}'
|
||||
```
|
||||
|
||||
### Viewing Aggregated Logs
|
||||
|
||||
```bash
|
||||
# PM2 worker logs (all workers combined)
|
||||
tail -f /var/log/logstash/pm2-workers-$(date +%Y-%m-%d).log
|
||||
|
||||
# Redis operational logs
|
||||
tail -f /var/log/logstash/redis-operational-$(date +%Y-%m-%d).log
|
||||
|
||||
# NGINX access logs (parsed)
|
||||
tail -f /var/log/logstash/nginx-access-$(date +%Y-%m-%d).log
|
||||
|
||||
# PostgreSQL function logs
|
||||
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log
|
||||
```
|
||||
|
||||
### Troubleshooting Logstash
|
||||
|
||||
| Issue | Diagnostic | Solution |
|
||||
| --------------------- | --------------------------- | ------------------------------- |
|
||||
| No events processed | `systemctl status logstash` | Start/restart service |
|
||||
| Config syntax error | Test config command | Fix config file |
|
||||
| Grok failures | Check stats endpoint | Update grok patterns |
|
||||
| Wrong Bugsink project | Check environment tags | Verify tag routing |
|
||||
| Permission denied | `groups logstash` | Add to `postgres`, `adm` groups |
|
||||
| PM2 logs not captured | Check file paths | Verify log file existence |
|
||||
| High disk usage | Check log rotation | Configure logrotate |
|
||||
|
||||
**Test Configuration**:
|
||||
|
||||
```bash
|
||||
/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/conf.d/bugsink.conf
|
||||
```
|
||||
|
||||
**Restart After Config Change**:
|
||||
|
||||
```bash
|
||||
systemctl restart logstash
|
||||
journalctl -u logstash -f # Watch for startup errors
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## PM2 Process Monitoring
|
||||
|
||||
PM2 manages the Node.js application processes in production.
|
||||
|
||||
### Process Overview
|
||||
|
||||
**Production Processes** (`ecosystem.config.cjs`):
|
||||
|
||||
| Process Name | Script | Purpose | Instances |
|
||||
| -------------------------------- | ----------- | -------------------- | ------------------ |
|
||||
| `flyer-crawler-api` | `server.ts` | Express API server | Cluster (max CPUs) |
|
||||
| `flyer-crawler-worker` | `worker.ts` | BullMQ job processor | 1 |
|
||||
| `flyer-crawler-analytics-worker` | `worker.ts` | Analytics jobs | 1 |
|
||||
|
||||
**Test Processes** (`ecosystem-test.config.cjs`):
|
||||
|
||||
| Process Name | Script | Port | Instances |
|
||||
| ------------------------------------- | ----------- | ---- | ------------- |
|
||||
| `flyer-crawler-api-test` | `server.ts` | 3002 | 1 (fork mode) |
|
||||
| `flyer-crawler-worker-test` | `worker.ts` | N/A | 1 |
|
||||
| `flyer-crawler-analytics-worker-test` | `worker.ts` | N/A | 1 |
|
||||
|
||||
### Basic Commands
|
||||
|
||||
```bash
|
||||
ssh root@projectium.com
|
||||
su - gitea-runner # PM2 runs under this user
|
||||
|
||||
# List all processes
|
||||
pm2 list
|
||||
|
||||
# Process details
|
||||
pm2 show flyer-crawler-api
|
||||
|
||||
# Monitor in real-time
|
||||
pm2 monit
|
||||
|
||||
# View logs
|
||||
pm2 logs flyer-crawler-api
|
||||
pm2 logs flyer-crawler-worker --lines 100
|
||||
|
||||
# View all logs
|
||||
pm2 logs
|
||||
|
||||
# Restart processes
|
||||
pm2 restart flyer-crawler-api
|
||||
pm2 restart all
|
||||
|
||||
# Reload without downtime (cluster mode only)
|
||||
pm2 reload flyer-crawler-api
|
||||
|
||||
# Stop processes
|
||||
pm2 stop flyer-crawler-api
|
||||
```
|
||||
|
||||
### Health Indicators
|
||||
|
||||
**Healthy Process**:
|
||||
|
||||
```
|
||||
┌─────────────────────┬────┬─────────┬─────────┬───────┬────────┬─────────┬──────────┐
|
||||
│ Name │ id │ mode │ status │ cpu │ mem │ uptime │ restarts │
|
||||
├─────────────────────┼────┼─────────┼─────────┼───────┼────────┼─────────┼──────────┤
|
||||
│ flyer-crawler-api │ 0 │ cluster │ online │ 0.5% │ 150MB │ 5d │ 0 │
|
||||
│ flyer-crawler-api │ 1 │ cluster │ online │ 0.3% │ 145MB │ 5d │ 0 │
|
||||
│ flyer-crawler-worker│ 2 │ fork │ online │ 0.1% │ 200MB │ 5d │ 0 │
|
||||
└─────────────────────┴────┴─────────┴─────────┴───────┴────────┴─────────┴──────────┘
|
||||
```
|
||||
|
||||
**Warning Signs**:
|
||||
|
||||
- `status: errored` - Process crashed
|
||||
- High `restarts` count - Instability
|
||||
- High `mem` (>500MB for API, >1GB for workers) - Memory leak
|
||||
- Low `uptime` with high restarts - Repeated crashes
|
||||
|
||||
### Log File Locations
|
||||
|
||||
| Process | stdout | stderr |
|
||||
| ---------------------- | ----------------------------------------------------------- | --------------- |
|
||||
| `flyer-crawler-api` | `/home/gitea-runner/.pm2/logs/flyer-crawler-api-out.log` | `...-error.log` |
|
||||
| `flyer-crawler-worker` | `/home/gitea-runner/.pm2/logs/flyer-crawler-worker-out.log` | `...-error.log` |
|
||||
|
||||
### Memory Management
|
||||
|
||||
PM2 is configured to restart processes when they exceed memory limits:
|
||||
|
||||
| Process | Memory Limit | Action |
|
||||
| ---------------- | ------------ | ------------ |
|
||||
| API | 500MB | Auto-restart |
|
||||
| Worker | 1GB | Auto-restart |
|
||||
| Analytics Worker | 1GB | Auto-restart |
|
||||
|
||||
**Check Memory Usage**:
|
||||
|
||||
```bash
|
||||
pm2 show flyer-crawler-api | grep memory
|
||||
pm2 show flyer-crawler-worker | grep memory
|
||||
```
|
||||
|
||||
### Restart Strategies
|
||||
|
||||
PM2 uses exponential backoff for restarts:
|
||||
|
||||
```javascript
|
||||
{
|
||||
max_restarts: 40,
|
||||
exp_backoff_restart_delay: 100, // Start at 100ms, exponentially increase
|
||||
min_uptime: '10s', // Must run 10s to be considered "started"
|
||||
}
|
||||
```
|
||||
|
||||
**Force Restart After Repeated Failures**:
|
||||
|
||||
```bash
|
||||
pm2 delete flyer-crawler-api
|
||||
pm2 start ecosystem.config.cjs --only flyer-crawler-api
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Database Monitoring
|
||||
|
||||
### Connection Pool Status
|
||||
|
||||
The application uses a PostgreSQL connection pool with these defaults:
|
||||
|
||||
| Setting | Value | Purpose |
|
||||
| ------------------------- | ----- | -------------------------------- |
|
||||
| `max` | 20 | Maximum concurrent connections |
|
||||
| `idleTimeoutMillis` | 30000 | Close idle connections after 30s |
|
||||
| `connectionTimeoutMillis` | 2000 | Fail if connection takes >2s |
|
||||
|
||||
**Check Pool Status via API**:
|
||||
|
||||
```bash
|
||||
curl -s https://flyer-crawler.projectium.com/api/health/db-pool | jq .
|
||||
|
||||
# Response
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"message": "Pool Status: 10 total, 8 idle, 0 waiting.",
|
||||
"totalCount": 10,
|
||||
"idleCount": 8,
|
||||
"waitingCount": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Pool Health Thresholds**:
|
||||
|
||||
| Metric | Healthy | Warning | Critical |
|
||||
| ------------------- | ------- | ------- | ---------- |
|
||||
| Waiting Connections | 0-2 | 3-4 | 5+ |
|
||||
| Total Connections | 1-15 | 16-19 | 20 (maxed) |
|
||||
|
||||
### Slow Query Logging
|
||||
|
||||
PostgreSQL is configured to log slow queries:
|
||||
|
||||
```ini
|
||||
# /etc/postgresql/14/main/conf.d/observability.conf
|
||||
log_min_duration_statement = 1000 # Log queries over 1 second
|
||||
```
|
||||
|
||||
**View Slow Queries**:
|
||||
|
||||
```bash
|
||||
ssh root@projectium.com
|
||||
grep "duration:" /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log | tail -20
|
||||
```
|
||||
|
||||
### Database Size Monitoring
|
||||
|
||||
```bash
|
||||
# Connect to production database
|
||||
psql -h localhost -U flyer_crawler_prod -d flyer-crawler-prod
|
||||
|
||||
# Database size
|
||||
SELECT pg_size_pretty(pg_database_size('flyer-crawler-prod'));
|
||||
|
||||
# Table sizes
|
||||
SELECT
|
||||
relname AS table,
|
||||
pg_size_pretty(pg_total_relation_size(relid)) AS total_size,
|
||||
pg_size_pretty(pg_relation_size(relid)) AS data_size,
|
||||
pg_size_pretty(pg_indexes_size(relid)) AS index_size
|
||||
FROM pg_catalog.pg_statio_user_tables
|
||||
ORDER BY pg_total_relation_size(relid) DESC
|
||||
LIMIT 10;
|
||||
|
||||
# Check for bloat
|
||||
SELECT schemaname, relname, n_dead_tup, n_live_tup,
|
||||
round(n_dead_tup * 100.0 / nullif(n_live_tup + n_dead_tup, 0), 2) as dead_pct
|
||||
FROM pg_stat_user_tables
|
||||
WHERE n_dead_tup > 1000
|
||||
ORDER BY n_dead_tup DESC;
|
||||
```
|
||||
|
||||
### Disk Space Monitoring
|
||||
|
||||
```bash
|
||||
# Check PostgreSQL data directory
|
||||
du -sh /var/lib/postgresql/14/main/
|
||||
|
||||
# Check available disk space
|
||||
df -h /var/lib/postgresql/
|
||||
|
||||
# Estimate growth rate
|
||||
psql -c "SELECT date_trunc('day', created_at) as day, count(*)
|
||||
FROM flyer_items
|
||||
WHERE created_at > now() - interval '7 days'
|
||||
GROUP BY 1 ORDER BY 1;"
|
||||
```
|
||||
|
||||
### Database Health via MCP
|
||||
|
||||
```bash
|
||||
# Query database directly
|
||||
mcp__devdb__query --sql "SELECT count(*) FROM flyers WHERE created_at > now() - interval '1 day'"
|
||||
|
||||
# Check connection count
|
||||
mcp__devdb__query --sql "SELECT count(*) FROM pg_stat_activity WHERE datname = 'flyer_crawler_dev'"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Redis Monitoring
|
||||
|
||||
### Basic Health Check
|
||||
|
||||
```bash
|
||||
# Via API endpoint
|
||||
curl -s https://flyer-crawler.projectium.com/api/health/redis | jq .
|
||||
|
||||
# Direct Redis check (on server)
|
||||
redis-cli ping # Should return PONG
|
||||
```
|
||||
|
||||
### Memory Usage
|
||||
|
||||
```bash
|
||||
redis-cli info memory | grep -E "used_memory_human|maxmemory_human|mem_fragmentation_ratio"
|
||||
|
||||
# Expected output
|
||||
used_memory_human:50.00M
|
||||
maxmemory_human:256.00M
|
||||
mem_fragmentation_ratio:1.05
|
||||
```
|
||||
|
||||
**Memory Thresholds**:
|
||||
|
||||
| Metric | Healthy | Warning | Critical |
|
||||
| ------------------- | ----------- | ------- | -------- |
|
||||
| Used Memory | <70% of max | 70-85% | >85% |
|
||||
| Fragmentation Ratio | 1.0-1.5 | 1.5-2.0 | >2.0 |
|
||||
|
||||
### Cache Statistics
|
||||
|
||||
```bash
|
||||
redis-cli info stats | grep -E "keyspace_hits|keyspace_misses|evicted_keys"
|
||||
|
||||
# Calculate hit rate
|
||||
# Hit Rate = keyspace_hits / (keyspace_hits + keyspace_misses) * 100
|
||||
```
|
||||
|
||||
**Cache Hit Rate Targets**:
|
||||
|
||||
- Excellent: >95%
|
||||
- Good: 85-95%
|
||||
- Needs attention: <85%
|
||||
|
||||
### Queue Monitoring
|
||||
|
||||
BullMQ queues are stored in Redis:
|
||||
|
||||
```bash
|
||||
# List all queues
|
||||
redis-cli keys "bull:*:id"
|
||||
|
||||
# Check queue depths
|
||||
redis-cli llen "bull:flyer-processing:wait"
|
||||
redis-cli llen "bull:email-sending:wait"
|
||||
redis-cli llen "bull:analytics-reporting:wait"
|
||||
|
||||
# Check failed jobs
|
||||
redis-cli llen "bull:flyer-processing:failed"
|
||||
```
|
||||
|
||||
**Queue Depth Thresholds**:
|
||||
|
||||
| Queue | Normal | Warning | Critical |
|
||||
| ------------------- | ------ | ------- | -------- |
|
||||
| flyer-processing | 0-10 | 11-50 | >50 |
|
||||
| email-sending | 0-100 | 101-500 | >500 |
|
||||
| analytics-reporting | 0-5 | 6-20 | >20 |
|
||||
|
||||
### Bull Board UI
|
||||
|
||||
Access the job queue dashboard:
|
||||
|
||||
- **Production**: `https://flyer-crawler.projectium.com/api/admin/jobs` (requires admin auth)
|
||||
- **Test**: `https://flyer-crawler-test.projectium.com/api/admin/jobs`
|
||||
- **Dev**: `http://localhost:3001/api/admin/jobs`
|
||||
|
||||
Features:
|
||||
|
||||
- View all queues and job counts
|
||||
- Inspect job data and errors
|
||||
- Retry failed jobs
|
||||
- Clean completed jobs
|
||||
|
||||
### Redis Database Allocation
|
||||
|
||||
| Database | Purpose |
|
||||
| -------- | ------------------------ |
|
||||
| 0 | BullMQ production queues |
|
||||
| 1 | BullMQ test queues |
|
||||
| 15 | Bugsink sync state |
|
||||
|
||||
---
|
||||
|
||||
## Production Alerts and On-Call
|
||||
|
||||
### Critical Monitoring Targets
|
||||
|
||||
| Service | Check | Interval | Alert Threshold |
|
||||
| ---------- | ------------------- | -------- | ---------------------- |
|
||||
| API Server | `/api/health/ready` | 1 min | 2 consecutive failures |
|
||||
| Database | Pool waiting count | 1 min | >5 waiting |
|
||||
| Redis | Memory usage | 5 min | >85% of maxmemory |
|
||||
| Disk Space | `/var/log` | 15 min | <10GB free |
|
||||
| Worker | Queue depth | 5 min | >50 jobs waiting |
|
||||
| Error Rate | Bugsink issue count | 15 min | >10 new issues/hour |
|
||||
|
||||
### Alert Channels
|
||||
|
||||
Configure alerts in your monitoring tool (UptimeRobot, Datadog, etc.):
|
||||
|
||||
1. **Slack channel**: `#flyer-crawler-alerts`
|
||||
2. **Email**: On-call rotation email
|
||||
3. **PagerDuty**: Critical issues only
|
||||
|
||||
### On-Call Response Procedures
|
||||
|
||||
**P1 - Critical (Site Down)**:
|
||||
|
||||
1. Acknowledge alert within 5 minutes
|
||||
2. Check `/api/health/ready` - identify failing service
|
||||
3. Check PM2 status: `pm2 list`
|
||||
4. Check recent deploys: `git log -5 --oneline`
|
||||
5. If database: check pool, restart if needed
|
||||
6. If Redis: check memory, flush if critical
|
||||
7. If application: restart PM2 processes
|
||||
8. Document in incident channel
|
||||
|
||||
**P2 - High (Degraded Service)**:
|
||||
|
||||
1. Acknowledge within 15 minutes
|
||||
2. Review Bugsink for error patterns
|
||||
3. Check system resources (CPU, memory, disk)
|
||||
4. Identify root cause
|
||||
5. Plan remediation
|
||||
6. Create Gitea issue if not auto-created
|
||||
|
||||
**P3 - Medium (Non-Critical)**:
|
||||
|
||||
1. Acknowledge within 1 hour
|
||||
2. Review during business hours
|
||||
3. Create Gitea issue for tracking
|
||||
|
||||
### Quick Diagnostic Commands
|
||||
|
||||
```bash
|
||||
# Full system health check
|
||||
ssh root@projectium.com << 'EOF'
|
||||
echo "=== Service Status ==="
|
||||
systemctl status pm2-gitea-runner --no-pager
|
||||
systemctl status logstash --no-pager
|
||||
systemctl status redis --no-pager
|
||||
systemctl status postgresql --no-pager
|
||||
|
||||
echo "=== PM2 Processes ==="
|
||||
su - gitea-runner -c "pm2 list"
|
||||
|
||||
echo "=== Disk Space ==="
|
||||
df -h / /var
|
||||
|
||||
echo "=== Memory ==="
|
||||
free -h
|
||||
|
||||
echo "=== Recent Errors ==="
|
||||
journalctl -p err -n 20 --no-pager
|
||||
EOF
|
||||
```
|
||||
|
||||
### Runbook Quick Reference
|
||||
|
||||
| Symptom | First Action | If That Fails |
|
||||
| --------------- | ---------------- | --------------------- |
|
||||
| 503 errors | Restart PM2 | Check database, Redis |
|
||||
| Slow responses | Check DB pool | Review slow query log |
|
||||
| High error rate | Check Bugsink | Review recent deploys |
|
||||
| Queue backlog | Restart worker | Scale workers |
|
||||
| Out of memory | Restart process | Increase PM2 limit |
|
||||
| Disk full | Clean old logs | Expand volume |
|
||||
| Redis OOM | Flush cache keys | Increase maxmemory |
|
||||
|
||||
### Post-Incident Review
|
||||
|
||||
After any P1/P2 incident:
|
||||
|
||||
1. Write incident report within 24 hours
|
||||
2. Identify root cause
|
||||
3. Document timeline of events
|
||||
4. List action items to prevent recurrence
|
||||
5. Schedule review meeting if needed
|
||||
6. Update runbooks if new procedures discovered
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [ADR-015: Application Performance Monitoring](../adr/0015-application-performance-monitoring-and-error-tracking.md)
|
||||
- [ADR-020: Health Checks](../adr/0020-health-checks-and-liveness-readiness-probes.md)
|
||||
- [ADR-050: PostgreSQL Function Observability](../adr/0050-postgresql-function-observability.md)
|
||||
- [ADR-053: Worker Health Checks](../adr/0053-worker-health-checks.md)
|
||||
- [DEV-CONTAINER-BUGSINK.md](../DEV-CONTAINER-BUGSINK.md)
|
||||
- [BUGSINK-SYNC.md](../BUGSINK-SYNC.md)
|
||||
- [LOGSTASH-QUICK-REF.md](LOGSTASH-QUICK-REF.md)
|
||||
- [LOGSTASH-TROUBLESHOOTING.md](LOGSTASH-TROUBLESHOOTING.md)
|
||||
- [LOGSTASH_DEPLOYMENT_CHECKLIST.md](../LOGSTASH_DEPLOYMENT_CHECKLIST.md)
|
||||
300
docs/subagents/AI-USAGE-GUIDE.md
Normal file
300
docs/subagents/AI-USAGE-GUIDE.md
Normal file
@@ -0,0 +1,300 @@
|
||||
# AI Usage Subagent Guide
|
||||
|
||||
The **ai-usage** subagent specializes in LLM APIs (Gemini, Claude), prompt engineering, and AI-powered features in the Flyer Crawler project.
|
||||
|
||||
## When to Use
|
||||
|
||||
Use the **ai-usage** subagent when you need to:
|
||||
|
||||
- Integrate with the Gemini API for flyer extraction
|
||||
- Debug AI extraction failures
|
||||
- Optimize prompts for better accuracy
|
||||
- Handle rate limiting and API errors
|
||||
- Implement new AI-powered features
|
||||
- Fine-tune extraction schemas
|
||||
|
||||
## What ai-usage Knows
|
||||
|
||||
The ai-usage subagent understands:
|
||||
|
||||
- Google Generative AI (Gemini) API
|
||||
- Flyer extraction prompts and schemas
|
||||
- Error handling for AI services
|
||||
- Rate limiting strategies
|
||||
- Token optimization
|
||||
- AI service architecture (ADR-041)
|
||||
|
||||
## AI Architecture Overview
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Flyer Upload │───►│ AI Service │───►│ Gemini API │
|
||||
│ │ │ │ │ │
|
||||
└─────────────────┘ │ - Preprocessing │ │ - Vision model │
|
||||
│ - Prompt build │ │ - JSON output │
|
||||
│ - Response parse│ │ │
|
||||
└─────────────────┘ └─────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Validation & │
|
||||
│ Normalization │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
## Key Files
|
||||
|
||||
| File | Purpose |
|
||||
| ----------------------------------------------- | ------------------------------------ |
|
||||
| `src/services/aiService.server.ts` | Gemini API integration |
|
||||
| `src/services/flyerProcessingService.server.ts` | Flyer extraction pipeline |
|
||||
| `src/schemas/flyer.schemas.ts` | Zod schemas for AI output validation |
|
||||
| `src/types/ai.types.ts` | TypeScript types for AI responses |
|
||||
|
||||
## Example Requests
|
||||
|
||||
### Debugging Extraction Failures
|
||||
|
||||
```
|
||||
"Use ai-usage to debug why flyer extractions are failing for
|
||||
multi-page PDFs. The error logs show 'Invalid JSON response'
|
||||
but only for certain stores."
|
||||
```
|
||||
|
||||
### Optimizing Prompts
|
||||
|
||||
```
|
||||
"Use ai-usage to improve the item extraction prompt. Currently
|
||||
it's missing unit prices when items show 'X for $Y' pricing
|
||||
(e.g., '3 for $5')."
|
||||
```
|
||||
|
||||
### Handling Rate Limits
|
||||
|
||||
```
|
||||
"Use ai-usage to implement exponential backoff for Gemini API
|
||||
rate limits. We're seeing 429 errors during high-volume uploads."
|
||||
```
|
||||
|
||||
### Adding New AI Features
|
||||
|
||||
```
|
||||
"Use ai-usage to add a feature that uses Gemini to categorize
|
||||
extracted items into grocery categories (produce, dairy, meat, etc.)."
|
||||
```
|
||||
|
||||
## Extraction Pipeline
|
||||
|
||||
### 1. Image Preprocessing
|
||||
|
||||
```typescript
|
||||
// Convert PDF to images, resize large images
|
||||
const processedImages = await imageProcessor.prepareForAI(uploadedFile);
|
||||
```
|
||||
|
||||
### 2. Prompt Construction
|
||||
|
||||
The extraction prompt includes:
|
||||
|
||||
- System instructions for the AI model
|
||||
- Expected output schema (JSON)
|
||||
- Examples of correct extraction
|
||||
- Handling instructions for edge cases
|
||||
|
||||
### 3. API Call
|
||||
|
||||
```typescript
|
||||
const response = await aiService.extractFlyerData(processedImages, storeContext, extractionOptions);
|
||||
```
|
||||
|
||||
### 4. Response Validation
|
||||
|
||||
```typescript
|
||||
// Validate against Zod schema
|
||||
const validatedItems = flyerItemsSchema.parse(response.items);
|
||||
```
|
||||
|
||||
### 5. Normalization
|
||||
|
||||
```typescript
|
||||
// Normalize prices, units, quantities
|
||||
const normalizedItems = normalizeExtractedItems(validatedItems);
|
||||
```
|
||||
|
||||
## Common Issues and Solutions
|
||||
|
||||
### Issue: Inconsistent Price Extraction
|
||||
|
||||
**Symptoms**: Same item priced differently on different extractions.
|
||||
|
||||
**Solution**: Improve prompt with explicit price format examples:
|
||||
|
||||
```
|
||||
"Price formats to recognize:
|
||||
- $X.XX (regular price)
|
||||
- X for $Y.YY (multi-buy)
|
||||
- $X.XX/lb or $X.XX/kg (unit price)
|
||||
- $X.XX each (per item)
|
||||
- SAVE $X.XX (discount amount, not item price)"
|
||||
```
|
||||
|
||||
### Issue: Missing Items from Dense Flyers
|
||||
|
||||
**Symptoms**: Flyers with many items on one page have missing extractions.
|
||||
|
||||
**Solution**:
|
||||
|
||||
1. Split page into quadrants for separate extraction
|
||||
2. Increase token limit for response
|
||||
3. Use structured grid-based prompting
|
||||
|
||||
### Issue: Rate Limit Errors (429)
|
||||
|
||||
**Symptoms**: `429 Too Many Requests` errors during bulk uploads.
|
||||
|
||||
**Solution**: Implement request queuing:
|
||||
|
||||
```typescript
|
||||
// Add to job queue instead of direct call
|
||||
await flyerQueue.add(
|
||||
'extract',
|
||||
{
|
||||
flyerId,
|
||||
images,
|
||||
},
|
||||
{
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 2000,
|
||||
},
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
### Issue: Hallucinated Items
|
||||
|
||||
**Symptoms**: Items extracted that don't exist in the flyer.
|
||||
|
||||
**Solution**:
|
||||
|
||||
1. Add confidence scoring to extraction
|
||||
2. Request bounding box coordinates for verification
|
||||
3. Add post-extraction validation against image
|
||||
|
||||
## Prompt Engineering Best Practices
|
||||
|
||||
### 1. Be Specific About Output Format
|
||||
|
||||
```
|
||||
Output MUST be valid JSON matching this schema:
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"name": "string (product name as shown)",
|
||||
"brand": "string or null",
|
||||
"price": number (in dollars),
|
||||
"unit": "string (each, lb, kg, etc.)",
|
||||
"quantity": number (default 1)
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Provide Examples
|
||||
|
||||
```
|
||||
Example extractions:
|
||||
- "Chicken Breast $4.99/lb" -> {"name": "Chicken Breast", "price": 4.99, "unit": "lb"}
|
||||
- "Coca-Cola 12pk $5.99" -> {"name": "Coca-Cola", "quantity": 12, "price": 5.99, "unit": "each"}
|
||||
```
|
||||
|
||||
### 3. Handle Edge Cases Explicitly
|
||||
|
||||
```
|
||||
Special cases:
|
||||
- If "LIMIT X" shown, add to notes, don't affect price
|
||||
- If "SAVE $X" shown without base price, mark price as null
|
||||
- If item is "FREE with purchase", set price to 0
|
||||
```
|
||||
|
||||
### 4. Request Structured Thinking
|
||||
|
||||
```
|
||||
For each item:
|
||||
1. Identify the product name and brand
|
||||
2. Find the associated price
|
||||
3. Determine if price is per-unit or total
|
||||
4. Extract any quantity information
|
||||
```
|
||||
|
||||
## Monitoring AI Performance
|
||||
|
||||
### Metrics to Track
|
||||
|
||||
| Metric | Description | Target |
|
||||
| ----------------------- | --------------------------------------- | --------------- |
|
||||
| Extraction success rate | % of flyers processed without error | >95% |
|
||||
| Items per flyer | Average items extracted | Varies by store |
|
||||
| Price accuracy | Match rate vs manual verification | >98% |
|
||||
| Response time | Time from upload to extraction complete | <30s |
|
||||
|
||||
### Logging
|
||||
|
||||
```typescript
|
||||
log.info(
|
||||
{
|
||||
flyerId,
|
||||
itemCount: extractedItems.length,
|
||||
processingTime: duration,
|
||||
modelVersion: response.model,
|
||||
tokenUsage: response.usage,
|
||||
},
|
||||
'Flyer extraction completed',
|
||||
);
|
||||
```
|
||||
|
||||
## Environment Configuration
|
||||
|
||||
| Variable | Purpose |
|
||||
| -------------------------------- | --------------------------- |
|
||||
| `VITE_GOOGLE_GENAI_API_KEY` | Gemini API key (production) |
|
||||
| `VITE_GOOGLE_GENAI_API_KEY_TEST` | Gemini API key (test) |
|
||||
|
||||
**Note**: Use separate API keys for production and test to avoid rate limit conflicts and enable separate billing tracking.
|
||||
|
||||
## Testing AI Features
|
||||
|
||||
### Unit Tests
|
||||
|
||||
Mock the Gemini API response:
|
||||
|
||||
```typescript
|
||||
vi.mock('@google/generative-ai', () => ({
|
||||
GoogleGenerativeAI: vi.fn().mockImplementation(() => ({
|
||||
getGenerativeModel: () => ({
|
||||
generateContent: vi.fn().mockResolvedValue({
|
||||
response: {
|
||||
text: () => JSON.stringify({ items: mockItems }),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
})),
|
||||
}));
|
||||
```
|
||||
|
||||
### Integration Tests
|
||||
|
||||
Use recorded responses for deterministic testing:
|
||||
|
||||
```typescript
|
||||
// Save real API responses to fixtures
|
||||
const fixtureResponse = await fs.readFile('fixtures/gemini-response.json');
|
||||
```
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
|
||||
- [../adr/0041-ai-gemini-integration-architecture.md](../adr/0041-ai-gemini-integration-architecture.md) - AI integration ADR
|
||||
- [../adr/0046-image-processing-pipeline.md](../adr/0046-image-processing-pipeline.md) - Image processing
|
||||
- [CODER-GUIDE.md](./CODER-GUIDE.md) - For implementing AI features
|
||||
312
docs/subagents/CODER-GUIDE.md
Normal file
312
docs/subagents/CODER-GUIDE.md
Normal file
@@ -0,0 +1,312 @@
|
||||
# Coder Subagent Guide
|
||||
|
||||
The **coder** subagent is your primary tool for writing and modifying production Node.js/TypeScript code in the Flyer Crawler project. This guide explains how to work effectively with the coder subagent.
|
||||
|
||||
## When to Use the Coder Subagent
|
||||
|
||||
Use the coder subagent when you need to:
|
||||
|
||||
- Implement new features or functionality
|
||||
- Fix bugs in existing code
|
||||
- Refactor existing code
|
||||
- Add new API endpoints
|
||||
- Create new React components
|
||||
- Write service layer logic
|
||||
- Implement business rules
|
||||
|
||||
## What the Coder Subagent Knows
|
||||
|
||||
The coder subagent has deep knowledge of:
|
||||
|
||||
### Project Architecture
|
||||
|
||||
```
|
||||
Routes -> Services -> Repositories -> Database
|
||||
|
|
||||
External APIs (*.server.ts)
|
||||
```
|
||||
|
||||
- **Routes Layer**: Request/response handling, validation, authentication
|
||||
- **Services Layer**: Business logic, transaction coordination, external APIs
|
||||
- **Repositories Layer**: Database access, query construction, error translation
|
||||
|
||||
### Key Patterns
|
||||
|
||||
| Pattern | ADR | Implementation |
|
||||
| ------------------ | ------- | ---------------------------------------------------------- |
|
||||
| Error Handling | ADR-001 | `handleDbError()`, throw `NotFoundError` |
|
||||
| Repository Methods | ADR-034 | `get*` throws, `find*` returns null, `list*` returns array |
|
||||
| API Responses | ADR-028 | `sendSuccess()`, `sendPaginated()`, `sendError()` |
|
||||
| Transactions | ADR-002 | `withTransaction(async (client) => {...})` |
|
||||
|
||||
### File Naming Conventions
|
||||
|
||||
| Pattern | Location | Purpose |
|
||||
| ------------- | ------------------ | -------------------------------- |
|
||||
| `*.db.ts` | `src/services/db/` | Database repositories |
|
||||
| `*.server.ts` | `src/services/` | Server-only code (external APIs) |
|
||||
| `*.routes.ts` | `src/routes/` | Express route handlers |
|
||||
| `*.test.ts` | Colocated | Unit tests |
|
||||
|
||||
## How to Request Code Changes
|
||||
|
||||
### Good Request Examples
|
||||
|
||||
**Specific and contextual:**
|
||||
|
||||
```
|
||||
"Use the coder subagent to add a new endpoint GET /api/stores/:id/locations
|
||||
that returns all locations for a store, following the existing patterns
|
||||
in stores.routes.ts"
|
||||
```
|
||||
|
||||
**With acceptance criteria:**
|
||||
|
||||
```
|
||||
"Use the coder subagent to implement the shopping list sharing feature:
|
||||
- Add a share_token column to shopping_lists table
|
||||
- Create POST /api/shopping-lists/:id/share endpoint
|
||||
- Return a shareable link with the token
|
||||
- Allow anonymous users to view shared lists"
|
||||
```
|
||||
|
||||
**Bug fix with reproduction steps:**
|
||||
|
||||
```
|
||||
"Use the coder subagent to fix the issue where flyer items are not
|
||||
sorted by price on the deals page. The expected behavior is lowest
|
||||
price first, but currently they appear in insertion order."
|
||||
```
|
||||
|
||||
### Less Effective Request Examples
|
||||
|
||||
**Too vague:**
|
||||
|
||||
```
|
||||
"Make the code better"
|
||||
```
|
||||
|
||||
**Missing context:**
|
||||
|
||||
```
|
||||
"Add a feature to search things"
|
||||
```
|
||||
|
||||
**Multiple unrelated tasks:**
|
||||
|
||||
```
|
||||
"Fix the login bug, add a new table, and update the homepage"
|
||||
```
|
||||
|
||||
## Common Workflows
|
||||
|
||||
### Adding a New API Endpoint
|
||||
|
||||
The coder subagent will follow this workflow:
|
||||
|
||||
1. **Add route** in `src/routes/{domain}.routes.ts`
|
||||
2. **Use `validateRequest(schema)`** middleware for input validation
|
||||
3. **Call service layer** (never access DB directly from routes)
|
||||
4. **Return via** `sendSuccess()` or `sendPaginated()`
|
||||
5. **Add tests** in `*.routes.test.ts`
|
||||
|
||||
**Example Code Pattern:**
|
||||
|
||||
```typescript
|
||||
// src/routes/stores.routes.ts
|
||||
router.get('/:id/locations', validateRequest(getStoreLocationsSchema), async (req, res, next) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const locations = await storeService.getLocationsForStore(parseInt(id, 10), req.log);
|
||||
sendSuccess(res, { locations });
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### Adding a New Database Operation
|
||||
|
||||
The coder subagent will:
|
||||
|
||||
1. **Add method** to `src/services/db/{domain}.db.ts`
|
||||
2. **Follow naming**: `get*` (throws), `find*` (returns null), `list*` (array)
|
||||
3. **Use `handleDbError()`** for error handling
|
||||
4. **Accept optional `PoolClient`** for transaction support
|
||||
5. **Add unit test**
|
||||
|
||||
**Example Code Pattern:**
|
||||
|
||||
```typescript
|
||||
// src/services/db/store.db.ts
|
||||
export async function listLocationsByStoreId(
|
||||
storeId: number,
|
||||
client?: PoolClient,
|
||||
): Promise<StoreLocation[]> {
|
||||
const queryable = client || getPool();
|
||||
try {
|
||||
const result = await queryable.query<StoreLocation>(
|
||||
`SELECT * FROM store_locations WHERE store_id = $1 ORDER BY created_at`,
|
||||
[storeId],
|
||||
);
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
handleDbError(
|
||||
error,
|
||||
log,
|
||||
'Database error in listLocationsByStoreId',
|
||||
{ storeId },
|
||||
{
|
||||
entityName: 'StoreLocation',
|
||||
defaultMessage: 'Failed to list store locations.',
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Adding a New React Component
|
||||
|
||||
The coder subagent will:
|
||||
|
||||
1. **Create component** in `src/components/` or feature-specific folder
|
||||
2. **Follow Neo-Brutalism design** patterns (ADR-012)
|
||||
3. **Use existing design tokens** from `src/styles/`
|
||||
4. **Add unit tests** using Testing Library
|
||||
|
||||
**Example Code Pattern:**
|
||||
|
||||
```typescript
|
||||
// src/components/StoreCard.tsx
|
||||
import { Store } from '@/types';
|
||||
|
||||
interface StoreCardProps {
|
||||
store: Store;
|
||||
onSelect?: (store: Store) => void;
|
||||
}
|
||||
|
||||
export function StoreCard({ store, onSelect }: StoreCardProps) {
|
||||
return (
|
||||
<div
|
||||
className="brutal-card p-4 cursor-pointer hover:translate-x-1 hover:-translate-y-1 transition-transform"
|
||||
onClick={() => onSelect?.(store)}
|
||||
>
|
||||
<h3 className="text-lg font-bold">{store.name}</h3>
|
||||
<p className="text-sm text-gray-600">{store.location_count} locations</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
## Code Quality Standards
|
||||
|
||||
The coder subagent adheres to these standards:
|
||||
|
||||
### TypeScript
|
||||
|
||||
- Strict TypeScript mode enabled
|
||||
- No `any` types unless absolutely necessary
|
||||
- Explicit return types for functions
|
||||
- Proper interface/type definitions
|
||||
|
||||
### Error Handling
|
||||
|
||||
- Use custom error classes from `src/services/db/errors.db.ts`
|
||||
- Never swallow errors silently
|
||||
- Log errors with appropriate context
|
||||
- Return meaningful error messages to API consumers
|
||||
|
||||
### Logging
|
||||
|
||||
- Use Pino logger (`src/services/logger.server.ts`)
|
||||
- Include module context in log child
|
||||
- Log at appropriate levels (info, warn, error)
|
||||
- Include relevant data in structured format
|
||||
|
||||
### Testing
|
||||
|
||||
- All new code should have corresponding tests
|
||||
- Follow testing patterns in ADR-010
|
||||
- Use mock factories from `src/tests/utils/mockFactories.ts`
|
||||
- Run tests in the dev container
|
||||
|
||||
## Platform Considerations
|
||||
|
||||
### Linux-Only Development
|
||||
|
||||
The coder subagent knows that this application runs exclusively on Linux:
|
||||
|
||||
- Uses POSIX-style paths (`/`)
|
||||
- Assumes Linux shell commands
|
||||
- References dev container environment
|
||||
|
||||
**Important**: Any code changes should be tested in the dev container:
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
```
|
||||
|
||||
### Database Schema Synchronization
|
||||
|
||||
When the coder subagent modifies database-related code, it will remind you:
|
||||
|
||||
> **Schema files must stay synchronized:**
|
||||
>
|
||||
> - `sql/master_schema_rollup.sql` - Test DB setup
|
||||
> - `sql/initial_schema.sql` - Fresh install schema
|
||||
> - `sql/migrations/*.sql` - Production changes
|
||||
|
||||
## Working with the Coder Subagent
|
||||
|
||||
### Before Starting
|
||||
|
||||
1. **Identify the scope** - What exactly needs to change?
|
||||
2. **Check existing patterns** - Is there similar code to follow?
|
||||
3. **Consider tests** - Will you need the testwriter subagent too?
|
||||
|
||||
### During Development
|
||||
|
||||
1. **Review changes incrementally** - Don't wait until the end
|
||||
2. **Ask for explanations** - Understand why certain approaches are chosen
|
||||
3. **Provide feedback** - Tell the coder if something doesn't look right
|
||||
|
||||
### After Completion
|
||||
|
||||
1. **Run tests** in the dev container
|
||||
2. **Run type-check**: `npm run type-check`
|
||||
3. **Review the changes** before committing
|
||||
4. **Consider code review** with the code-reviewer subagent
|
||||
|
||||
## Common Issues and Solutions
|
||||
|
||||
### Issue: Code Doesn't Follow Project Patterns
|
||||
|
||||
**Solution**: Provide examples of existing code that follows the desired pattern. The coder will align with it.
|
||||
|
||||
### Issue: Missing Error Handling
|
||||
|
||||
**Solution**: Explicitly request comprehensive error handling:
|
||||
|
||||
```
|
||||
"Include proper error handling using handleDbError and the project's
|
||||
error classes for all database operations"
|
||||
```
|
||||
|
||||
### Issue: Tests Not Included
|
||||
|
||||
**Solution**: Either:
|
||||
|
||||
1. Ask the coder to include tests: "Include unit tests for all new code"
|
||||
2. Use the testwriter subagent separately for comprehensive test coverage
|
||||
|
||||
### Issue: Code Works on Windows but Fails on Linux
|
||||
|
||||
**Solution**: Always test in the dev container. The coder subagent writes Linux-compatible code, but IDE tooling might behave differently.
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
|
||||
- [TESTER-GUIDE.md](./TESTER-GUIDE.md) - Testing strategies
|
||||
- [../adr/0034-repository-pattern-standards.md](../adr/0034-repository-pattern-standards.md) - Repository patterns
|
||||
- [../adr/0035-service-layer-architecture.md](../adr/0035-service-layer-architecture.md) - Service layer architecture
|
||||
- [../adr/0028-api-response-standardization.md](../adr/0028-api-response-standardization.md) - API response patterns
|
||||
419
docs/subagents/DATABASE-GUIDE.md
Normal file
419
docs/subagents/DATABASE-GUIDE.md
Normal file
@@ -0,0 +1,419 @@
|
||||
# Database Subagent Guide
|
||||
|
||||
This guide covers two database-focused subagents:
|
||||
|
||||
- **db-dev**: Database development - schemas, queries, migrations, optimization
|
||||
- **db-admin**: Database administration - PostgreSQL/Redis admin, security, backups
|
||||
|
||||
## Understanding the Difference
|
||||
|
||||
| Aspect | db-dev | db-admin |
|
||||
| --------------- | ----------------------------------- | ------------------------------------- |
|
||||
| **Focus** | Application database code | Infrastructure and operations |
|
||||
| **Tasks** | Queries, migrations, repositories | Performance tuning, backups, security |
|
||||
| **Output** | SQL migrations, repository methods | Configuration, monitoring scripts |
|
||||
| **When to Use** | Adding features, optimizing queries | Production issues, capacity planning |
|
||||
|
||||
## The db-dev Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **db-dev** subagent when you need to:
|
||||
|
||||
- Design new database tables or modify existing ones
|
||||
- Write SQL queries or optimize existing ones
|
||||
- Create database migrations
|
||||
- Implement repository pattern methods
|
||||
- Fix N+1 query problems
|
||||
- Add indexes for performance
|
||||
- Work with PostGIS spatial queries
|
||||
|
||||
### What db-dev Knows
|
||||
|
||||
The db-dev subagent has deep knowledge of:
|
||||
|
||||
- Project database schema (`sql/master_schema_rollup.sql`)
|
||||
- Repository pattern standards (ADR-034)
|
||||
- Transaction management (ADR-002)
|
||||
- PostgreSQL-specific features (PostGIS, pg_trgm, etc.)
|
||||
- Schema synchronization requirements
|
||||
|
||||
### Schema Synchronization (Critical)
|
||||
|
||||
> **Schema files MUST stay synchronized:**
|
||||
>
|
||||
> | File | Purpose |
|
||||
> | ------------------------------ | --------------------------------- |
|
||||
> | `sql/master_schema_rollup.sql` | Test DB setup, complete reference |
|
||||
> | `sql/initial_schema.sql` | Fresh install schema |
|
||||
> | `sql/migrations/*.sql` | Production incremental changes |
|
||||
|
||||
When db-dev creates a migration, it will also update the schema files.
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Adding a new table:**
|
||||
|
||||
```
|
||||
"Use db-dev to design a table for storing user recipe reviews.
|
||||
Include fields for rating (1-5), review text, and relationships
|
||||
to users and recipes. Create the migration and update schema files."
|
||||
```
|
||||
|
||||
**Optimizing a slow query:**
|
||||
|
||||
```
|
||||
"Use db-dev to optimize the query that lists flyers with their
|
||||
item counts. It's currently doing N+1 queries and takes too long
|
||||
with many flyers."
|
||||
```
|
||||
|
||||
**Adding spatial search:**
|
||||
|
||||
```
|
||||
"Use db-dev to add the ability to search stores within a radius
|
||||
of a given location using PostGIS. Include the migration for
|
||||
adding the geography column."
|
||||
```
|
||||
|
||||
### Repository Pattern Standards
|
||||
|
||||
The db-dev subagent follows these naming conventions:
|
||||
|
||||
| Prefix | Returns | Behavior on Not Found |
|
||||
| --------- | ------------------- | ------------------------------------ |
|
||||
| `get*` | Single entity | Throws `NotFoundError` |
|
||||
| `find*` | Entity or `null` | Returns `null` |
|
||||
| `list*` | Array | Returns `[]` |
|
||||
| `create*` | Created entity | Throws on constraint violation |
|
||||
| `update*` | Updated entity | Throws `NotFoundError` if not exists |
|
||||
| `delete*` | `void` or `boolean` | Throws `NotFoundError` if not exists |
|
||||
|
||||
### Example Migration
|
||||
|
||||
```sql
|
||||
-- sql/migrations/20260121_add_recipe_reviews.sql
|
||||
|
||||
-- Create recipe_reviews table
|
||||
CREATE TABLE IF NOT EXISTS recipe_reviews (
|
||||
review_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
recipe_id UUID NOT NULL REFERENCES recipes(recipe_id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||
rating INTEGER NOT NULL CHECK (rating >= 1 AND rating <= 5),
|
||||
review_text TEXT,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE (recipe_id, user_id)
|
||||
);
|
||||
|
||||
-- Add indexes
|
||||
CREATE INDEX idx_recipe_reviews_recipe_id ON recipe_reviews(recipe_id);
|
||||
CREATE INDEX idx_recipe_reviews_user_id ON recipe_reviews(user_id);
|
||||
CREATE INDEX idx_recipe_reviews_rating ON recipe_reviews(rating);
|
||||
|
||||
-- Add trigger for updated_at
|
||||
CREATE TRIGGER update_recipe_reviews_updated_at
|
||||
BEFORE UPDATE ON recipe_reviews
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
```
|
||||
|
||||
### Example Repository Method
|
||||
|
||||
```typescript
|
||||
// src/services/db/recipeReview.db.ts
|
||||
import { handleDbError, NotFoundError } from './errors.db';
|
||||
|
||||
export async function getReviewById(reviewId: string, client?: PoolClient): Promise<RecipeReview> {
|
||||
const queryable = client || getPool();
|
||||
try {
|
||||
const result = await queryable.query<RecipeReview>(
|
||||
`SELECT * FROM recipe_reviews WHERE review_id = $1`,
|
||||
[reviewId],
|
||||
);
|
||||
if (result.rows.length === 0) {
|
||||
throw new NotFoundError(`Review with ID ${reviewId} not found.`);
|
||||
}
|
||||
return result.rows[0];
|
||||
} catch (error) {
|
||||
handleDbError(
|
||||
error,
|
||||
log,
|
||||
'Database error in getReviewById',
|
||||
{ reviewId },
|
||||
{
|
||||
entityName: 'RecipeReview',
|
||||
defaultMessage: 'Failed to fetch review.',
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function listReviewsByRecipeId(
|
||||
recipeId: string,
|
||||
options: { limit?: number; offset?: number } = {},
|
||||
client?: PoolClient,
|
||||
): Promise<RecipeReview[]> {
|
||||
const queryable = client || getPool();
|
||||
const { limit = 50, offset = 0 } = options;
|
||||
try {
|
||||
const result = await queryable.query<RecipeReview>(
|
||||
`SELECT * FROM recipe_reviews
|
||||
WHERE recipe_id = $1
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $2 OFFSET $3`,
|
||||
[recipeId, limit, offset],
|
||||
);
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
handleDbError(
|
||||
error,
|
||||
log,
|
||||
'Database error in listReviewsByRecipeId',
|
||||
{ recipeId, limit, offset },
|
||||
{
|
||||
entityName: 'RecipeReview',
|
||||
defaultMessage: 'Failed to list reviews.',
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## The db-admin Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **db-admin** subagent when you need to:
|
||||
|
||||
- Debug production database issues
|
||||
- Configure PostgreSQL settings
|
||||
- Set up database backups
|
||||
- Analyze slow query logs
|
||||
- Configure Redis for production
|
||||
- Plan database capacity
|
||||
- Manage database users and permissions
|
||||
- Handle replication or failover
|
||||
|
||||
### What db-admin Knows
|
||||
|
||||
The db-admin subagent understands:
|
||||
|
||||
- PostgreSQL configuration and tuning
|
||||
- Redis configuration for BullMQ queues
|
||||
- Backup and recovery strategies (ADR-019)
|
||||
- Connection pooling settings
|
||||
- Production deployment setup
|
||||
- Bugsink PostgreSQL observability (ADR-050)
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Performance tuning:**
|
||||
|
||||
```
|
||||
"Use db-admin to analyze why the database is running slow.
|
||||
Check connection pool settings, identify slow queries, and
|
||||
recommend PostgreSQL configuration changes."
|
||||
```
|
||||
|
||||
**Backup configuration:**
|
||||
|
||||
```
|
||||
"Use db-admin to set up daily automated backups for the
|
||||
production database with 30-day retention."
|
||||
```
|
||||
|
||||
**User management:**
|
||||
|
||||
```
|
||||
"Use db-admin to create a read-only database user for
|
||||
reporting purposes that can only SELECT from specific tables."
|
||||
```
|
||||
|
||||
### Database Users
|
||||
|
||||
| User | Database | Purpose |
|
||||
| -------------------- | -------------------- | ---------------------- |
|
||||
| `flyer_crawler_prod` | `flyer-crawler-prod` | Production |
|
||||
| `flyer_crawler_test` | `flyer-crawler-test` | Testing |
|
||||
| `postgres` | All | Superuser (admin only) |
|
||||
|
||||
### Creating Database Users
|
||||
|
||||
```sql
|
||||
-- As postgres superuser
|
||||
CREATE DATABASE "flyer-crawler-test";
|
||||
CREATE USER flyer_crawler_test WITH PASSWORD 'secure_password';
|
||||
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
|
||||
\c "flyer-crawler-test"
|
||||
ALTER SCHEMA public OWNER TO flyer_crawler_test;
|
||||
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
```
|
||||
|
||||
### PostgreSQL Configuration Guidance
|
||||
|
||||
For production, db-admin may recommend settings like:
|
||||
|
||||
```ini
|
||||
# /etc/postgresql/14/main/conf.d/performance.conf
|
||||
|
||||
# Connection settings
|
||||
max_connections = 100
|
||||
shared_buffers = 256MB
|
||||
|
||||
# Query optimization
|
||||
effective_cache_size = 768MB
|
||||
random_page_cost = 1.1
|
||||
|
||||
# Write performance
|
||||
wal_buffers = 16MB
|
||||
checkpoint_completion_target = 0.9
|
||||
|
||||
# Logging
|
||||
log_min_duration_statement = 1000 # Log queries over 1 second
|
||||
```
|
||||
|
||||
### Redis Configuration Guidance
|
||||
|
||||
For BullMQ queues:
|
||||
|
||||
```ini
|
||||
# /etc/redis/redis.conf
|
||||
|
||||
# Memory management
|
||||
maxmemory 256mb
|
||||
maxmemory-policy noeviction # BullMQ requires this
|
||||
|
||||
# Persistence
|
||||
appendonly yes
|
||||
appendfsync everysec
|
||||
|
||||
# Security
|
||||
requirepass your_redis_password
|
||||
```
|
||||
|
||||
## Common Database Tasks
|
||||
|
||||
### Running Migrations in Production
|
||||
|
||||
```bash
|
||||
# SSH to production server
|
||||
ssh root@projectium.com
|
||||
|
||||
# Run migration
|
||||
cd /var/www/flyer-crawler.projectium.com
|
||||
npm run db:migrate
|
||||
```
|
||||
|
||||
### Checking Database Health
|
||||
|
||||
```bash
|
||||
# Connection count
|
||||
psql -c "SELECT count(*) FROM pg_stat_activity WHERE datname = 'flyer-crawler-prod';"
|
||||
|
||||
# Table sizes
|
||||
psql -d "flyer-crawler-prod" -c "
|
||||
SELECT
|
||||
tablename,
|
||||
pg_size_pretty(pg_total_relation_size(schemaname || '.' || tablename)) as size
|
||||
FROM pg_tables
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY pg_total_relation_size(schemaname || '.' || tablename) DESC
|
||||
LIMIT 10;"
|
||||
|
||||
# Slow queries
|
||||
psql -d "flyer-crawler-prod" -c "
|
||||
SELECT
|
||||
calls,
|
||||
mean_exec_time::numeric(10,2) as avg_ms,
|
||||
query
|
||||
FROM pg_stat_statements
|
||||
ORDER BY mean_exec_time DESC
|
||||
LIMIT 5;"
|
||||
```
|
||||
|
||||
### Database Backup Commands
|
||||
|
||||
```bash
|
||||
# Manual backup
|
||||
pg_dump -U flyer_crawler_prod -h localhost "flyer-crawler-prod" > backup_$(date +%Y%m%d).sql
|
||||
|
||||
# Restore from backup
|
||||
psql -U flyer_crawler_prod -h localhost "flyer-crawler-prod" < backup_20260121.sql
|
||||
```
|
||||
|
||||
## N+1 Query Detection
|
||||
|
||||
The db-dev subagent is particularly skilled at identifying N+1 query problems:
|
||||
|
||||
**Problematic Pattern:**
|
||||
|
||||
```typescript
|
||||
// BAD: N+1 queries
|
||||
const flyers = await listFlyers();
|
||||
for (const flyer of flyers) {
|
||||
flyer.items = await listItemsByFlyerId(flyer.flyer_id); // N queries!
|
||||
}
|
||||
```
|
||||
|
||||
**Optimized Pattern:**
|
||||
|
||||
```typescript
|
||||
// GOOD: Single query with JOIN or separate batch query
|
||||
const flyersWithItems = await listFlyersWithItems(); // 1 query
|
||||
|
||||
// Or with batching:
|
||||
const flyers = await listFlyers();
|
||||
const flyerIds = flyers.map((f) => f.flyer_id);
|
||||
const allItems = await listItemsByFlyerIds(flyerIds); // 1 query
|
||||
// Group items by flyer_id in application code
|
||||
```
|
||||
|
||||
## Working with PostGIS
|
||||
|
||||
The project uses PostGIS for spatial queries. Example:
|
||||
|
||||
```sql
|
||||
-- Find stores within 10km of a location
|
||||
SELECT
|
||||
s.store_id,
|
||||
s.name,
|
||||
ST_Distance(
|
||||
sl.location::geography,
|
||||
ST_MakePoint(-79.3832, 43.6532)::geography
|
||||
) / 1000 as distance_km
|
||||
FROM stores s
|
||||
JOIN store_locations sl ON s.store_id = sl.store_id
|
||||
WHERE ST_DWithin(
|
||||
sl.location::geography,
|
||||
ST_MakePoint(-79.3832, 43.6532)::geography,
|
||||
10000 -- 10km in meters
|
||||
)
|
||||
ORDER BY distance_km;
|
||||
```
|
||||
|
||||
## MCP Database Access
|
||||
|
||||
For direct database queries during development, use the MCP server:
|
||||
|
||||
```
|
||||
// Query the dev database
|
||||
mcp__devdb__query("SELECT * FROM flyers LIMIT 5")
|
||||
```
|
||||
|
||||
This is useful for:
|
||||
|
||||
- Verifying data during debugging
|
||||
- Checking schema state
|
||||
- Testing queries before implementing
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
|
||||
- [CODER-GUIDE.md](./CODER-GUIDE.md) - Working with the coder subagent
|
||||
- [../adr/0034-repository-pattern-standards.md](../adr/0034-repository-pattern-standards.md) - Repository patterns
|
||||
- [../adr/0002-standardized-transaction-management.md](../adr/0002-standardized-transaction-management.md) - Transaction management
|
||||
- [../adr/0019-data-backup-and-recovery-strategy.md](../adr/0019-data-backup-and-recovery-strategy.md) - Backup strategy
|
||||
- [../adr/0050-postgresql-function-observability.md](../adr/0050-postgresql-function-observability.md) - Database observability
|
||||
- [../BARE-METAL-SETUP.md](../BARE-METAL-SETUP.md) - Production database setup
|
||||
475
docs/subagents/DEVOPS-GUIDE.md
Normal file
475
docs/subagents/DEVOPS-GUIDE.md
Normal file
@@ -0,0 +1,475 @@
|
||||
# DevOps Subagent Guide
|
||||
|
||||
This guide covers DevOps-related subagents for deployment, infrastructure, and operations:
|
||||
|
||||
- **devops**: Containers, services, CI/CD pipelines, deployments
|
||||
- **infra-architect**: Resource optimization, capacity planning
|
||||
- **bg-worker**: Background jobs, PM2 workers, BullMQ queues
|
||||
|
||||
## The devops Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **devops** subagent when you need to:
|
||||
|
||||
- Debug container issues in development
|
||||
- Modify CI/CD pipelines
|
||||
- Configure PM2 for production
|
||||
- Update deployment workflows
|
||||
- Troubleshoot service startup issues
|
||||
- Configure NGINX or reverse proxy
|
||||
- Set up SSL/TLS certificates
|
||||
|
||||
### What devops Knows
|
||||
|
||||
The devops subagent understands:
|
||||
|
||||
- Podman/Docker container management
|
||||
- Dev container configuration (`.devcontainer/`)
|
||||
- Compose files (`compose.dev.yml`)
|
||||
- PM2 ecosystem configuration
|
||||
- Gitea Actions CI/CD workflows
|
||||
- NGINX configuration
|
||||
- Systemd service management
|
||||
|
||||
### Development Environment
|
||||
|
||||
**Container Architecture:**
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Development Environment │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
|
||||
│ │ app │ │ postgres │ │ redis │ │
|
||||
│ │ (Node.js) │───►│ (PostGIS) │ │ (Cache) │ │
|
||||
│ │ │───►│ │ │ │ │
|
||||
│ └─────────────┘ └─────────────┘ └─────────────┘ │
|
||||
│ :3000/:3001 :5432 :6379 │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Container Services:**
|
||||
|
||||
| Service | Image | Purpose | Port |
|
||||
| ---------- | ----------------------- | ---------------------- | ---------- |
|
||||
| `app` | Custom (Dockerfile.dev) | Node.js application | 3000, 3001 |
|
||||
| `postgres` | postgis/postgis:15-3.4 | Database with PostGIS | 5432 |
|
||||
| `redis` | redis:alpine | Caching and job queues | 6379 |
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Container debugging:**
|
||||
|
||||
```
|
||||
"Use devops to debug why the dev container fails to start.
|
||||
The postgres service shows as unhealthy and the app can't connect."
|
||||
```
|
||||
|
||||
**CI/CD pipeline update:**
|
||||
|
||||
```
|
||||
"Use devops to add a step to the deploy-to-test.yml workflow
|
||||
that runs database migrations before restarting the app."
|
||||
```
|
||||
|
||||
**PM2 configuration:**
|
||||
|
||||
```
|
||||
"Use devops to update the PM2 ecosystem config to use cluster
|
||||
mode with 4 instances instead of max for the API server."
|
||||
```
|
||||
|
||||
### Container Commands Reference
|
||||
|
||||
```bash
|
||||
# Start development environment
|
||||
podman-compose -f compose.dev.yml up -d
|
||||
|
||||
# View container logs
|
||||
podman-compose -f compose.dev.yml logs -f app
|
||||
|
||||
# Restart specific service
|
||||
podman-compose -f compose.dev.yml restart app
|
||||
|
||||
# Rebuild container (after Dockerfile changes)
|
||||
podman-compose -f compose.dev.yml build app
|
||||
|
||||
# Reset everything
|
||||
podman-compose -f compose.dev.yml down -v
|
||||
podman-compose -f compose.dev.yml up -d --build
|
||||
|
||||
# Enter container shell
|
||||
podman exec -it flyer-crawler-dev bash
|
||||
|
||||
# Run tests in container (from Windows)
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
```
|
||||
|
||||
### Git Bash Path Conversion (Windows)
|
||||
|
||||
When running commands from Git Bash on Windows, paths may be incorrectly converted:
|
||||
|
||||
| Solution | Example |
|
||||
| -------------------------- | -------------------------------------------------------- |
|
||||
| `sh -c` with single quotes | `podman exec container sh -c '/usr/local/bin/script.sh'` |
|
||||
| Double slashes | `podman exec container //usr//local//bin//script.sh` |
|
||||
| MSYS_NO_PATHCONV=1 | `MSYS_NO_PATHCONV=1 podman exec ...` |
|
||||
|
||||
### PM2 Production Configuration
|
||||
|
||||
**ecosystem.config.cjs Structure:**
|
||||
|
||||
```javascript
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
name: 'flyer-crawler-api',
|
||||
script: './node_modules/.bin/tsx',
|
||||
args: 'server.ts',
|
||||
instances: 'max', // Use all CPU cores
|
||||
exec_mode: 'cluster', // Enable cluster mode
|
||||
max_memory_restart: '500M',
|
||||
kill_timeout: 5000, // Graceful shutdown
|
||||
|
||||
env_production: {
|
||||
NODE_ENV: 'production',
|
||||
cwd: '/var/www/flyer-crawler.projectium.com',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'flyer-crawler-worker',
|
||||
script: './node_modules/.bin/tsx',
|
||||
args: 'src/services/worker.ts',
|
||||
instances: 1, // Single instance for workers
|
||||
max_memory_restart: '1G',
|
||||
kill_timeout: 10000, // Workers need more time
|
||||
},
|
||||
],
|
||||
};
|
||||
```
|
||||
|
||||
**PM2 Commands:**
|
||||
|
||||
```bash
|
||||
# Start/reload with environment
|
||||
pm2 startOrReload ecosystem.config.cjs --env production --update-env
|
||||
|
||||
# Save process list
|
||||
pm2 save
|
||||
|
||||
# View logs
|
||||
pm2 logs flyer-crawler-api --lines 50
|
||||
|
||||
# Monitor processes
|
||||
pm2 monit
|
||||
|
||||
# Describe process
|
||||
pm2 describe flyer-crawler-api
|
||||
```
|
||||
|
||||
### CI/CD Workflow Files
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------------- | --------------------------- |
|
||||
| `.gitea/workflows/deploy-to-prod.yml` | Production deployment |
|
||||
| `.gitea/workflows/deploy-to-test.yml` | Test environment deployment |
|
||||
|
||||
**Deployment Flow:**
|
||||
|
||||
1. Push to `main` branch
|
||||
2. Gitea Actions triggered
|
||||
3. SSH to production server
|
||||
4. Pull latest code
|
||||
5. Install dependencies
|
||||
6. Run build
|
||||
7. Run migrations
|
||||
8. Restart PM2 processes
|
||||
|
||||
### Directory Structure (Production)
|
||||
|
||||
```
|
||||
/var/www/
|
||||
├── flyer-crawler.projectium.com/ # Production
|
||||
│ ├── server.ts
|
||||
│ ├── ecosystem.config.cjs
|
||||
│ ├── package.json
|
||||
│ ├── flyer-images/
|
||||
│ │ ├── icons/
|
||||
│ │ └── archive/
|
||||
│ └── logs/
|
||||
│ └── app.log
|
||||
└── flyer-crawler-test.projectium.com/ # Test environment
|
||||
└── ... (same structure)
|
||||
```
|
||||
|
||||
## The infra-architect Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **infra-architect** subagent when you need to:
|
||||
|
||||
- Analyze resource usage and optimize
|
||||
- Plan for scaling
|
||||
- Reduce infrastructure costs
|
||||
- Configure memory limits
|
||||
- Analyze disk usage
|
||||
- Plan capacity for growth
|
||||
|
||||
### What infra-architect Knows
|
||||
|
||||
The infra-architect subagent understands:
|
||||
|
||||
- Node.js memory management
|
||||
- PostgreSQL resource tuning
|
||||
- Redis memory configuration
|
||||
- Container resource limits
|
||||
- PM2 process monitoring
|
||||
- Disk and storage management
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Memory optimization:**
|
||||
|
||||
```
|
||||
"Use infra-architect to analyze memory usage of the worker
|
||||
processes. They're frequently hitting the 1GB limit and restarting."
|
||||
```
|
||||
|
||||
**Capacity planning:**
|
||||
|
||||
```
|
||||
"Use infra-architect to estimate resource requirements for
|
||||
handling 10x current traffic. Include database, Redis, and
|
||||
application server recommendations."
|
||||
```
|
||||
|
||||
**Cost optimization:**
|
||||
|
||||
```
|
||||
"Use infra-architect to identify opportunities to reduce
|
||||
infrastructure costs without impacting performance."
|
||||
```
|
||||
|
||||
### Resource Limits Reference
|
||||
|
||||
| Process | Memory Limit | Notes |
|
||||
| ---------------- | ------------ | --------------------- |
|
||||
| API Server | 500MB | Per cluster instance |
|
||||
| Worker | 1GB | Single instance |
|
||||
| Analytics Worker | 1GB | Single instance |
|
||||
| PostgreSQL | System RAM | Tune `shared_buffers` |
|
||||
| Redis | 256MB | `maxmemory` setting |
|
||||
|
||||
## The bg-worker Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **bg-worker** subagent when you need to:
|
||||
|
||||
- Debug BullMQ queue issues
|
||||
- Add new background job types
|
||||
- Configure job retry logic
|
||||
- Analyze job processing failures
|
||||
- Optimize worker performance
|
||||
- Handle job timeouts
|
||||
|
||||
### What bg-worker Knows
|
||||
|
||||
The bg-worker subagent understands:
|
||||
|
||||
- BullMQ queue patterns
|
||||
- PM2 worker configuration
|
||||
- Job retry and backoff strategies
|
||||
- Queue monitoring and debugging
|
||||
- Redis connection for queues
|
||||
- Worker health checks (ADR-053)
|
||||
|
||||
### Queue Architecture
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ API Server │───►│ Redis (BullMQ) │◄───│ Worker │
|
||||
│ │ │ │ │ │
|
||||
│ queue.add() │ │ flyerQueue │ │ process jobs │
|
||||
│ │ │ cleanupQueue │ │ │
|
||||
└─────────────────┘ │ analyticsQueue │ └─────────────────┘
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Debugging stuck jobs:**
|
||||
|
||||
```
|
||||
"Use bg-worker to debug why jobs are stuck in the flyer processing
|
||||
queue. Check for failed jobs, worker status, and Redis connectivity."
|
||||
```
|
||||
|
||||
**Adding retry logic:**
|
||||
|
||||
```
|
||||
"Use bg-worker to add exponential backoff retry logic to the
|
||||
AI extraction job. It should retry up to 3 times with increasing
|
||||
delays for rate limit errors."
|
||||
```
|
||||
|
||||
**Queue monitoring:**
|
||||
|
||||
```
|
||||
"Use bg-worker to add health check endpoints for monitoring
|
||||
queue depth and worker status."
|
||||
```
|
||||
|
||||
### Queue Configuration
|
||||
|
||||
```typescript
|
||||
// src/services/queues.server.ts
|
||||
export const flyerQueue = new Queue('flyer-processing', {
|
||||
connection: redisConnection,
|
||||
defaultJobOptions: {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 1000,
|
||||
},
|
||||
removeOnComplete: { count: 100 },
|
||||
removeOnFail: { count: 1000 },
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Worker Configuration
|
||||
|
||||
```typescript
|
||||
// src/services/workers.server.ts
|
||||
export const flyerWorker = new Worker(
|
||||
'flyer-processing',
|
||||
async (job) => {
|
||||
// Process job
|
||||
},
|
||||
{
|
||||
connection: redisConnection,
|
||||
concurrency: 5,
|
||||
limiter: {
|
||||
max: 10,
|
||||
duration: 1000,
|
||||
},
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
### Monitoring Queues
|
||||
|
||||
```bash
|
||||
# Check queue status via Redis
|
||||
redis-cli -a $REDIS_PASSWORD
|
||||
|
||||
> KEYS bull:*
|
||||
> LLEN bull:flyer-processing:wait
|
||||
> ZRANGE bull:flyer-processing:failed 0 -1
|
||||
```
|
||||
|
||||
## Service Management Commands
|
||||
|
||||
### PM2 Commands
|
||||
|
||||
```bash
|
||||
# Start/reload
|
||||
pm2 startOrReload ecosystem.config.cjs --env production --update-env && pm2 save
|
||||
|
||||
# View status
|
||||
pm2 list
|
||||
pm2 status
|
||||
|
||||
# View logs
|
||||
pm2 logs
|
||||
pm2 logs flyer-crawler-api --lines 100
|
||||
|
||||
# Restart specific process
|
||||
pm2 restart flyer-crawler-api
|
||||
pm2 restart flyer-crawler-worker
|
||||
|
||||
# Stop all
|
||||
pm2 stop all
|
||||
|
||||
# Delete all
|
||||
pm2 delete all
|
||||
```
|
||||
|
||||
### Systemd Services (Production)
|
||||
|
||||
| Service | Command |
|
||||
| ---------- | ---------------------- | ---- | ------------------------- |
|
||||
| PostgreSQL | `sudo systemctl {start | stop | status} postgresql` |
|
||||
| Redis | `sudo systemctl {start | stop | status} redis-server` |
|
||||
| NGINX | `sudo systemctl {start | stop | status} nginx` |
|
||||
| Bugsink | `sudo systemctl {start | stop | status} gunicorn-bugsink` |
|
||||
| Logstash | `sudo systemctl {start | stop | status} logstash` |
|
||||
|
||||
### Health Checks
|
||||
|
||||
```bash
|
||||
# API health check
|
||||
curl http://localhost:3001/api/health
|
||||
|
||||
# PM2 health
|
||||
pm2 list
|
||||
|
||||
# PostgreSQL health
|
||||
pg_isready -h localhost -p 5432
|
||||
|
||||
# Redis health
|
||||
redis-cli -a $REDIS_PASSWORD ping
|
||||
```
|
||||
|
||||
## Troubleshooting Guide
|
||||
|
||||
### Container Won't Start
|
||||
|
||||
1. Check container logs: `podman-compose logs app`
|
||||
2. Verify services are healthy: `podman-compose ps`
|
||||
3. Check environment variables in `compose.dev.yml`
|
||||
4. Try rebuilding: `podman-compose build --no-cache app`
|
||||
|
||||
### Tests Fail in Container but Pass Locally
|
||||
|
||||
Tests must run in the Linux container environment:
|
||||
|
||||
```bash
|
||||
# Wrong (Windows)
|
||||
npm test
|
||||
|
||||
# Correct (in container)
|
||||
podman exec -it flyer-crawler-dev npm test
|
||||
```
|
||||
|
||||
### PM2 Process Keeps Restarting
|
||||
|
||||
1. Check logs: `pm2 logs <process-name>`
|
||||
2. Check memory usage: `pm2 monit`
|
||||
3. Verify environment variables: `pm2 env <process-id>`
|
||||
4. Check for unhandled errors in application code
|
||||
|
||||
### Database Connection Refused
|
||||
|
||||
1. Verify PostgreSQL is running
|
||||
2. Check connection string in environment
|
||||
3. Verify database user has permissions
|
||||
4. Check `pg_hba.conf` for allowed connections
|
||||
|
||||
### Redis Connection Issues
|
||||
|
||||
1. Verify Redis is running: `redis-cli ping`
|
||||
2. Check password in environment variables
|
||||
3. Verify Redis is listening on expected port
|
||||
4. Check `maxmemory` setting if queue operations fail
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
|
||||
- [../BARE-METAL-SETUP.md](../BARE-METAL-SETUP.md) - Production setup guide
|
||||
- [../adr/0014-containerization-and-deployment-strategy.md](../adr/0014-containerization-and-deployment-strategy.md) - Containerization ADR
|
||||
- [../adr/0006-background-job-processing-and-task-queues.md](../adr/0006-background-job-processing-and-task-queues.md) - Background jobs ADR
|
||||
- [../adr/0017-ci-cd-and-branching-strategy.md](../adr/0017-ci-cd-and-branching-strategy.md) - CI/CD strategy
|
||||
- [../adr/0053-worker-health-checks.md](../adr/0053-worker-health-checks.md) - Worker health checks
|
||||
442
docs/subagents/DOCUMENTATION-GUIDE.md
Normal file
442
docs/subagents/DOCUMENTATION-GUIDE.md
Normal file
@@ -0,0 +1,442 @@
|
||||
# Documentation Subagent Guide
|
||||
|
||||
This guide covers documentation-focused subagents:
|
||||
|
||||
- **documenter**: User docs, API specs, feature documentation
|
||||
- **describer-for-ai**: Technical docs for AI, ADRs, system overviews
|
||||
- **planner**: Feature breakdown, roadmaps, scope management
|
||||
- **product-owner**: Requirements, user stories, backlog prioritization
|
||||
|
||||
## The documenter Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **documenter** subagent when you need to:
|
||||
|
||||
- Write user-facing documentation
|
||||
- Create API endpoint documentation
|
||||
- Document feature usage guides
|
||||
- Write setup or installation guides
|
||||
- Create troubleshooting guides
|
||||
|
||||
### What documenter Knows
|
||||
|
||||
The documenter subagent understands:
|
||||
|
||||
- Markdown formatting and best practices
|
||||
- API documentation standards
|
||||
- User documentation patterns
|
||||
- Project-specific terminology
|
||||
- Existing documentation structure
|
||||
|
||||
### Example Requests
|
||||
|
||||
**API Documentation:**
|
||||
|
||||
```
|
||||
"Use documenter to create API documentation for the shopping
|
||||
list endpoints. Include request/response schemas, authentication
|
||||
requirements, and example curl commands."
|
||||
```
|
||||
|
||||
**Feature Guide:**
|
||||
|
||||
```
|
||||
"Use documenter to write a user guide for the price watchlist
|
||||
feature. Explain how to add items, set price alerts, and view
|
||||
price history."
|
||||
```
|
||||
|
||||
**Troubleshooting Guide:**
|
||||
|
||||
```
|
||||
"Use documenter to create a troubleshooting guide for common
|
||||
flyer upload issues, including file format errors, size limits,
|
||||
and processing failures."
|
||||
```
|
||||
|
||||
### Documentation Standards
|
||||
|
||||
#### API Documentation Format
|
||||
|
||||
````markdown
|
||||
### [METHOD] /api/endpoint
|
||||
|
||||
**Description**: Brief purpose of the endpoint
|
||||
|
||||
**Authentication**: Required (Bearer token)
|
||||
|
||||
**Request**:
|
||||
|
||||
- Headers: `Content-Type: application/json`, `Authorization: Bearer {token}`
|
||||
- Body:
|
||||
```json
|
||||
{
|
||||
"field": "string (required) - Description",
|
||||
"optional_field": "number (optional) - Description"
|
||||
}
|
||||
```
|
||||
````
|
||||
|
||||
**Response**:
|
||||
|
||||
- Success (200):
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": { ... }
|
||||
}
|
||||
```
|
||||
- Error (400):
|
||||
```json
|
||||
{
|
||||
"success": false,
|
||||
"error": {
|
||||
"code": "VALIDATION_ERROR",
|
||||
"message": "Description of error"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Example**:
|
||||
|
||||
```bash
|
||||
curl -X POST https://api.example.com/api/endpoint \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"field": "value"}'
|
||||
```
|
||||
|
||||
````
|
||||
|
||||
## The describer-for-ai Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **describer-for-ai** subagent when you need to:
|
||||
|
||||
- Write Architecture Decision Records (ADRs)
|
||||
- Create technical specifications for AI consumption
|
||||
- Document system architecture for context
|
||||
- Write CLAUDE.md updates
|
||||
- Create technical overviews
|
||||
|
||||
### What describer-for-ai Knows
|
||||
|
||||
The describer-for-ai subagent understands:
|
||||
|
||||
- ADR format and conventions
|
||||
- Technical documentation for AI assistants
|
||||
- System architecture patterns
|
||||
- Project conventions and patterns
|
||||
- How to provide context efficiently for AI
|
||||
|
||||
### ADR Format
|
||||
|
||||
```markdown
|
||||
# ADR-NNN: Title of Decision
|
||||
|
||||
**Date**: YYYY-MM-DD
|
||||
|
||||
**Status**: Proposed | Accepted | Implemented | Superseded
|
||||
|
||||
## Context
|
||||
|
||||
Describe the problem space and constraints that led to this decision.
|
||||
|
||||
## Decision
|
||||
|
||||
The chosen solution and its rationale.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Benefits of this decision
|
||||
|
||||
### Negative
|
||||
- Trade-offs or limitations
|
||||
|
||||
### Neutral
|
||||
- Other notable effects
|
||||
|
||||
## Implementation Details
|
||||
|
||||
Technical details, code examples, configuration.
|
||||
|
||||
## Key Files
|
||||
|
||||
- `path/to/file.ts` - Description
|
||||
- `path/to/other.ts` - Description
|
||||
|
||||
## Related ADRs
|
||||
|
||||
- [ADR-XXX](./XXXX-title.md) - Related decision
|
||||
````
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Creating an ADR:**
|
||||
|
||||
```
|
||||
"Use describer-for-ai to create an ADR for adding websocket
|
||||
support for real-time price alerts. Include the technical
|
||||
approach, alternatives considered, and implementation details."
|
||||
```
|
||||
|
||||
**CLAUDE.md Update:**
|
||||
|
||||
```
|
||||
"Use describer-for-ai to update CLAUDE.md with the new
|
||||
authentication flow and any new patterns developers should
|
||||
be aware of."
|
||||
```
|
||||
|
||||
**Technical Overview:**
|
||||
|
||||
```
|
||||
"Use describer-for-ai to create a technical overview of the
|
||||
caching layer for future AI context, including how Redis is
|
||||
used, cache invalidation patterns, and key prefixes."
|
||||
```
|
||||
|
||||
## The planner Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **planner** subagent when you need to:
|
||||
|
||||
- Break down a feature into tasks
|
||||
- Create implementation roadmaps
|
||||
- Scope work for sprints
|
||||
- Identify dependencies
|
||||
- Estimate effort
|
||||
|
||||
### What planner Knows
|
||||
|
||||
The planner subagent understands:
|
||||
|
||||
- Project architecture and conventions
|
||||
- Existing codebase structure
|
||||
- Common implementation patterns
|
||||
- Task estimation heuristics
|
||||
- Dependency identification
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Feature Breakdown:**
|
||||
|
||||
```
|
||||
"Use planner to break down the 'store comparison' feature
|
||||
into implementable tasks. Include frontend, backend, and
|
||||
database work. Identify dependencies between tasks."
|
||||
```
|
||||
|
||||
**Roadmap Planning:**
|
||||
|
||||
```
|
||||
"Use planner to create a roadmap for the Q2 features:
|
||||
recipe integration, mobile app preparation, and store
|
||||
notifications. Identify what can be parallelized."
|
||||
```
|
||||
|
||||
**Scope Assessment:**
|
||||
|
||||
```
|
||||
"Use planner to assess the scope of adding multi-language
|
||||
support. What systems would need to change? What's the
|
||||
minimum viable implementation?"
|
||||
```
|
||||
|
||||
### Planning Output Format
|
||||
|
||||
```markdown
|
||||
# Feature: [Feature Name]
|
||||
|
||||
## Overview
|
||||
|
||||
Brief description of the feature and its value.
|
||||
|
||||
## Tasks
|
||||
|
||||
### Phase 1: Foundation
|
||||
|
||||
1. **[Task Name]** (S/M/L)
|
||||
- Description
|
||||
- Files: `path/to/file.ts`
|
||||
- Dependencies: None
|
||||
- Acceptance: What "done" looks like
|
||||
|
||||
2. **[Task Name]** (S/M/L)
|
||||
- Description
|
||||
- Files: `path/to/file.ts`
|
||||
- Dependencies: Task 1
|
||||
- Acceptance: What "done" looks like
|
||||
|
||||
### Phase 2: Core Implementation
|
||||
|
||||
...
|
||||
|
||||
### Phase 3: Polish & Testing
|
||||
|
||||
...
|
||||
|
||||
## Dependencies
|
||||
|
||||
- External: Third-party services, APIs
|
||||
- Internal: Other features that must be complete first
|
||||
|
||||
## Risks
|
||||
|
||||
- Risk 1: Mitigation strategy
|
||||
- Risk 2: Mitigation strategy
|
||||
|
||||
## Estimates
|
||||
|
||||
- Phase 1: X days
|
||||
- Phase 2: Y days
|
||||
- Phase 3: Z days
|
||||
- Total: X+Y+Z days
|
||||
```
|
||||
|
||||
## The product-owner Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **product-owner** subagent when you need to:
|
||||
|
||||
- Write user stories
|
||||
- Define acceptance criteria
|
||||
- Prioritize backlog items
|
||||
- Validate requirements
|
||||
- Clarify feature scope
|
||||
|
||||
### What product-owner Knows
|
||||
|
||||
The product-owner subagent understands:
|
||||
|
||||
- User story format
|
||||
- Acceptance criteria patterns
|
||||
- Feature prioritization frameworks
|
||||
- User research interpretation
|
||||
- Business value assessment
|
||||
|
||||
### Example Requests
|
||||
|
||||
**User Story Writing:**
|
||||
|
||||
```
|
||||
"Use product-owner to write user stories for the meal planning
|
||||
feature. Consider different user personas: budget shoppers,
|
||||
health-conscious users, and busy families."
|
||||
```
|
||||
|
||||
**Acceptance Criteria:**
|
||||
|
||||
```
|
||||
"Use product-owner to define acceptance criteria for the price
|
||||
alert feature. What conditions must be met for this feature
|
||||
to be considered complete?"
|
||||
```
|
||||
|
||||
**Prioritization:**
|
||||
|
||||
```
|
||||
"Use product-owner to prioritize these feature requests based
|
||||
on user value and development effort:
|
||||
1. Dark mode
|
||||
2. Recipe suggestions based on deals
|
||||
3. Store location search
|
||||
4. Price history graphs"
|
||||
```
|
||||
|
||||
### User Story Format
|
||||
|
||||
```markdown
|
||||
## User Story: [Short Title]
|
||||
|
||||
**As a** [type of user]
|
||||
**I want to** [goal/desire]
|
||||
**So that** [benefit/value]
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
**Given** [context/starting state]
|
||||
**When** [action taken]
|
||||
**Then** [expected outcome]
|
||||
|
||||
### Additional Notes
|
||||
|
||||
- Edge cases to consider
|
||||
- Related features
|
||||
- Out of scope items
|
||||
|
||||
### Technical Notes
|
||||
|
||||
- API endpoints needed
|
||||
- Database changes
|
||||
- Third-party integrations
|
||||
```
|
||||
|
||||
## Documentation Organization
|
||||
|
||||
The project organizes documentation as follows:
|
||||
|
||||
```
|
||||
docs/
|
||||
├── adr/ # Architecture Decision Records
|
||||
│ ├── index.md # ADR index
|
||||
│ └── NNNN-title.md # Individual ADRs
|
||||
├── subagents/ # Subagent guides (this directory)
|
||||
├── plans/ # Implementation plans
|
||||
├── tests/ # Test documentation
|
||||
├── TESTING.md # Testing guide
|
||||
├── BARE-METAL-SETUP.md # Production setup
|
||||
├── DESIGN_TOKENS.md # Design system tokens
|
||||
└── ... # Other documentation
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Keep Documentation Current
|
||||
|
||||
Documentation should be updated alongside code changes. The `describer-for-ai` subagent can help identify what documentation needs updating after code changes.
|
||||
|
||||
### 2. Use Consistent Terminology
|
||||
|
||||
Refer to entities and concepts consistently:
|
||||
|
||||
- "Flyer" not "Ad" or "Circular"
|
||||
- "Store" not "Retailer" or "Shop"
|
||||
- "Deal" not "Offer" or "Sale"
|
||||
|
||||
### 3. Include Examples
|
||||
|
||||
All documentation should include concrete examples:
|
||||
|
||||
- API docs: Include curl commands and JSON payloads
|
||||
- User guides: Include screenshots or step-by-step instructions
|
||||
- Technical docs: Include code snippets
|
||||
|
||||
### 4. Cross-Reference Related Documentation
|
||||
|
||||
Use relative links to connect related documentation:
|
||||
|
||||
```markdown
|
||||
See [Testing Guide](../TESTING.md) for test execution details.
|
||||
```
|
||||
|
||||
### 5. Date and Version Documentation
|
||||
|
||||
Include dates on documentation that may become stale:
|
||||
|
||||
```markdown
|
||||
**Last Updated**: 2026-01-21
|
||||
**Applies to**: v0.12.x
|
||||
```
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
|
||||
- [../adr/index.md](../adr/index.md) - ADR index
|
||||
- [../TESTING.md](../TESTING.md) - Testing guide
|
||||
- [../../CLAUDE.md](../../CLAUDE.md) - AI instructions
|
||||
412
docs/subagents/FRONTEND-GUIDE.md
Normal file
412
docs/subagents/FRONTEND-GUIDE.md
Normal file
@@ -0,0 +1,412 @@
|
||||
# Frontend Subagent Guide
|
||||
|
||||
This guide covers frontend-focused subagents:
|
||||
|
||||
- **frontend-specialist**: UI components, Neo-Brutalism, Core Web Vitals, accessibility
|
||||
- **uiux-designer**: UI/UX decisions, component design, user experience
|
||||
|
||||
## The frontend-specialist Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **frontend-specialist** subagent when you need to:
|
||||
|
||||
- Build new React components
|
||||
- Fix CSS/styling issues
|
||||
- Improve Core Web Vitals performance
|
||||
- Implement accessibility features
|
||||
- Debug React rendering issues
|
||||
- Optimize bundle size
|
||||
|
||||
### What frontend-specialist Knows
|
||||
|
||||
The frontend-specialist subagent understands:
|
||||
|
||||
- React 18+ patterns and hooks
|
||||
- TanStack Query for server state
|
||||
- Zustand for client state
|
||||
- Tailwind CSS with custom design tokens
|
||||
- Neo-Brutalism design system
|
||||
- Accessibility standards (WCAG)
|
||||
- Performance optimization
|
||||
|
||||
## Design System: Neo-Brutalism
|
||||
|
||||
The project uses a Neo-Brutalism design aesthetic characterized by:
|
||||
|
||||
- Bold, black borders
|
||||
- High contrast colors
|
||||
- Shadow offsets for depth
|
||||
- Raw, honest UI elements
|
||||
- Playful but functional
|
||||
|
||||
### Design Tokens
|
||||
|
||||
Located in `src/styles/` and documented in `docs/DESIGN_TOKENS.md`:
|
||||
|
||||
```css
|
||||
/* Core colors */
|
||||
--color-primary: #ff6b35;
|
||||
--color-secondary: #004e89;
|
||||
--color-accent: #f7c548;
|
||||
--color-background: #fffdf7;
|
||||
--color-text: #1a1a1a;
|
||||
|
||||
/* Borders */
|
||||
--border-width: 3px;
|
||||
--border-color: #1a1a1a;
|
||||
|
||||
/* Shadows (offset style) */
|
||||
--shadow-sm: 2px 2px 0 0 #1a1a1a;
|
||||
--shadow-md: 4px 4px 0 0 #1a1a1a;
|
||||
--shadow-lg: 6px 6px 0 0 #1a1a1a;
|
||||
```
|
||||
|
||||
### Component Patterns
|
||||
|
||||
**Brutal Card:**
|
||||
|
||||
```tsx
|
||||
<div className="border-3 border-black bg-white p-4 shadow-[4px_4px_0_0_#1A1A1A] hover:shadow-[6px_6px_0_0_#1A1A1A] hover:translate-x-[-2px] hover:translate-y-[-2px] transition-all">
|
||||
{children}
|
||||
</div>
|
||||
```
|
||||
|
||||
**Brutal Button:**
|
||||
|
||||
```tsx
|
||||
<button className="border-3 border-black bg-primary px-4 py-2 font-bold shadow-[4px_4px_0_0_#1A1A1A] hover:shadow-[2px_2px_0_0_#1A1A1A] hover:translate-x-[2px] hover:translate-y-[2px] active:shadow-none active:translate-x-[4px] active:translate-y-[4px] transition-all">
|
||||
Click Me
|
||||
</button>
|
||||
```
|
||||
|
||||
## Example Requests
|
||||
|
||||
### Building New Components
|
||||
|
||||
```
|
||||
"Use frontend-specialist to create a PriceTag component that
|
||||
displays the current price and original price (if discounted)
|
||||
in the Neo-Brutalism style with a 'SALE' badge when applicable."
|
||||
```
|
||||
|
||||
### Performance Optimization
|
||||
|
||||
```
|
||||
"Use frontend-specialist to optimize the deals list page.
|
||||
It's showing poor Largest Contentful Paint scores and the
|
||||
initial load feels sluggish."
|
||||
```
|
||||
|
||||
### Accessibility Fix
|
||||
|
||||
```
|
||||
"Use frontend-specialist to audit and fix accessibility issues
|
||||
on the shopping list page. Screen reader users report that
|
||||
the checkbox states aren't being announced correctly."
|
||||
```
|
||||
|
||||
### Responsive Design
|
||||
|
||||
```
|
||||
"Use frontend-specialist to make the store search component
|
||||
work better on mobile. The dropdown menu is getting cut off
|
||||
on smaller screens."
|
||||
```
|
||||
|
||||
## State Management
|
||||
|
||||
### Server State (TanStack Query)
|
||||
|
||||
```tsx
|
||||
// Fetching data with caching
|
||||
const {
|
||||
data: deals,
|
||||
isLoading,
|
||||
error,
|
||||
} = useQuery({
|
||||
queryKey: ['deals', storeId],
|
||||
queryFn: () => dealsApi.getByStore(storeId),
|
||||
staleTime: 5 * 60 * 1000, // 5 minutes
|
||||
});
|
||||
|
||||
// Mutations with optimistic updates
|
||||
const mutation = useMutation({
|
||||
mutationFn: dealsApi.favorite,
|
||||
onMutate: async (dealId) => {
|
||||
await queryClient.cancelQueries(['deals']);
|
||||
const previous = queryClient.getQueryData(['deals']);
|
||||
queryClient.setQueryData(['deals'], (old) =>
|
||||
old.map((d) => (d.id === dealId ? { ...d, isFavorite: true } : d)),
|
||||
);
|
||||
return { previous };
|
||||
},
|
||||
onError: (err, dealId, context) => {
|
||||
queryClient.setQueryData(['deals'], context.previous);
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Client State (Zustand)
|
||||
|
||||
```tsx
|
||||
// Simple client-only state
|
||||
const useUIStore = create((set) => ({
|
||||
sidebarOpen: false,
|
||||
toggleSidebar: () => set((s) => ({ sidebarOpen: !s.sidebarOpen })),
|
||||
}));
|
||||
```
|
||||
|
||||
## The uiux-designer Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **uiux-designer** subagent when you need to:
|
||||
|
||||
- Make design decisions for new features
|
||||
- Improve user flows
|
||||
- Design component layouts
|
||||
- Choose appropriate UI patterns
|
||||
- Plan information architecture
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Design new feature:**
|
||||
|
||||
```
|
||||
"Use uiux-designer to design the user flow for adding items
|
||||
to a shopping list from the deals page. Consider both desktop
|
||||
and mobile experiences."
|
||||
```
|
||||
|
||||
**Improve existing UX:**
|
||||
|
||||
```
|
||||
"Use uiux-designer to improve the flyer upload experience.
|
||||
Users are confused about which file types are supported and
|
||||
don't understand the processing status."
|
||||
```
|
||||
|
||||
**Component design:**
|
||||
|
||||
```
|
||||
"Use uiux-designer to design a price comparison component
|
||||
that shows the same item across multiple stores."
|
||||
```
|
||||
|
||||
## Component Structure
|
||||
|
||||
### Feature-Based Organization
|
||||
|
||||
```
|
||||
src/
|
||||
├── components/ # Shared UI components
|
||||
│ ├── ui/ # Basic UI primitives
|
||||
│ │ ├── Button.tsx
|
||||
│ │ ├── Card.tsx
|
||||
│ │ └── Input.tsx
|
||||
│ ├── layout/ # Layout components
|
||||
│ │ ├── Header.tsx
|
||||
│ │ └── Sidebar.tsx
|
||||
│ └── shared/ # Complex shared components
|
||||
│ └── PriceDisplay.tsx
|
||||
├── features/ # Feature-specific components
|
||||
│ ├── deals/
|
||||
│ │ ├── components/
|
||||
│ │ ├── hooks/
|
||||
│ │ └── api/
|
||||
│ └── shopping-list/
|
||||
│ ├── components/
|
||||
│ ├── hooks/
|
||||
│ └── api/
|
||||
└── pages/ # Route page components
|
||||
├── DealsPage.tsx
|
||||
└── ShoppingListPage.tsx
|
||||
```
|
||||
|
||||
### Component Pattern
|
||||
|
||||
```tsx
|
||||
// src/components/PriceTag.tsx
|
||||
import { cn } from '@/utils/cn';
|
||||
|
||||
interface PriceTagProps {
|
||||
currentPrice: number;
|
||||
originalPrice?: number;
|
||||
currency?: string;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
export function PriceTag({
|
||||
currentPrice,
|
||||
originalPrice,
|
||||
currency = '$',
|
||||
className,
|
||||
}: PriceTagProps) {
|
||||
const isOnSale = originalPrice && originalPrice > currentPrice;
|
||||
const discount = isOnSale ? Math.round((1 - currentPrice / originalPrice) * 100) : 0;
|
||||
|
||||
return (
|
||||
<div className={cn('flex items-baseline gap-2', className)}>
|
||||
<span className="text-2xl font-bold text-primary">
|
||||
{currency}
|
||||
{currentPrice.toFixed(2)}
|
||||
</span>
|
||||
{isOnSale && (
|
||||
<>
|
||||
<span className="text-sm text-gray-500 line-through">
|
||||
{currency}
|
||||
{originalPrice.toFixed(2)}
|
||||
</span>
|
||||
<span className="border-2 border-black bg-accent px-1 text-xs font-bold">
|
||||
-{discount}%
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
## Testing React Components
|
||||
|
||||
### Component Test Pattern
|
||||
|
||||
```tsx
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { renderWithProviders, screen } from '@/tests/utils/renderWithProviders';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { PriceTag } from './PriceTag';
|
||||
|
||||
describe('PriceTag', () => {
|
||||
it('displays current price', () => {
|
||||
renderWithProviders(<PriceTag currentPrice={9.99} />);
|
||||
expect(screen.getByText('$9.99')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows discount when original price is higher', () => {
|
||||
renderWithProviders(<PriceTag currentPrice={7.99} originalPrice={9.99} />);
|
||||
expect(screen.getByText('$7.99')).toBeInTheDocument();
|
||||
expect(screen.getByText('$9.99')).toBeInTheDocument();
|
||||
expect(screen.getByText('-20%')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Hook Test Pattern
|
||||
|
||||
```tsx
|
||||
import { renderHook, waitFor } from '@testing-library/react';
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||
import { useDeals } from './useDeals';
|
||||
|
||||
describe('useDeals', () => {
|
||||
it('fetches deals for store', async () => {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: { queries: { retry: false } },
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useDeals('store-123'), {
|
||||
wrapper: ({ children }) => (
|
||||
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
|
||||
),
|
||||
});
|
||||
|
||||
await waitFor(() => expect(result.current.isSuccess).toBe(true));
|
||||
expect(result.current.data).toHaveLength(10);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## Accessibility Guidelines
|
||||
|
||||
### ARIA Patterns
|
||||
|
||||
```tsx
|
||||
// Proper button with loading state
|
||||
<button
|
||||
aria-busy={isLoading}
|
||||
aria-label={isLoading ? 'Loading...' : 'Add to cart'}
|
||||
disabled={isLoading}
|
||||
>
|
||||
{isLoading ? <Spinner /> : 'Add to Cart'}
|
||||
</button>
|
||||
|
||||
// Proper form field
|
||||
<label htmlFor="email">Email Address</label>
|
||||
<input
|
||||
id="email"
|
||||
type="email"
|
||||
aria-describedby="email-error"
|
||||
aria-invalid={!!errors.email}
|
||||
/>
|
||||
{errors.email && (
|
||||
<span id="email-error" role="alert">
|
||||
{errors.email}
|
||||
</span>
|
||||
)}
|
||||
```
|
||||
|
||||
### Keyboard Navigation
|
||||
|
||||
- All interactive elements must be focusable
|
||||
- Focus order should be logical
|
||||
- Focus traps for modals
|
||||
- Skip links for main content
|
||||
|
||||
### Color Contrast
|
||||
|
||||
- Normal text: minimum 4.5:1 contrast ratio
|
||||
- Large text: minimum 3:1 contrast ratio
|
||||
- Use the Neo-Brutalism palette which is designed for high contrast
|
||||
|
||||
## Performance Optimization
|
||||
|
||||
### Code Splitting
|
||||
|
||||
```tsx
|
||||
// Lazy load heavy components
|
||||
const PdfViewer = lazy(() => import('./PdfViewer'));
|
||||
|
||||
function FlyerPage() {
|
||||
return (
|
||||
<Suspense fallback={<LoadingSpinner />}>
|
||||
<PdfViewer />
|
||||
</Suspense>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
### Image Optimization
|
||||
|
||||
```tsx
|
||||
// Use appropriate sizes and formats
|
||||
<img
|
||||
src={imageUrl}
|
||||
srcSet={`${imageUrl}?w=400 400w, ${imageUrl}?w=800 800w`}
|
||||
sizes="(max-width: 600px) 400px, 800px"
|
||||
loading="lazy"
|
||||
alt={itemName}
|
||||
/>
|
||||
```
|
||||
|
||||
### Memoization
|
||||
|
||||
```tsx
|
||||
// Memoize expensive computations
|
||||
const sortedDeals = useMemo(() => deals.slice().sort((a, b) => a.price - b.price), [deals]);
|
||||
|
||||
// Memoize callbacks passed to children
|
||||
const handleSelect = useCallback((id: string) => {
|
||||
setSelectedId(id);
|
||||
}, []);
|
||||
```
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
|
||||
- [CODER-GUIDE.md](./CODER-GUIDE.md) - For implementing features
|
||||
- [../DESIGN_TOKENS.md](../DESIGN_TOKENS.md) - Design token reference
|
||||
- [../adr/0012-frontend-component-library-and-design-system.md](../adr/0012-frontend-component-library-and-design-system.md) - Design system ADR
|
||||
- [../adr/0005-frontend-state-management-and-server-cache-strategy.md](../adr/0005-frontend-state-management-and-server-cache-strategy.md) - State management ADR
|
||||
- [../adr/0044-frontend-feature-organization.md](../adr/0044-frontend-feature-organization.md) - Feature organization
|
||||
217
docs/subagents/OVERVIEW.md
Normal file
217
docs/subagents/OVERVIEW.md
Normal file
@@ -0,0 +1,217 @@
|
||||
# Claude Code Subagent System Overview
|
||||
|
||||
This document provides a comprehensive guide to the subagent system used in the Flyer Crawler project. Subagents are specialized AI assistants that focus on specific domains, allowing for more targeted and effective development workflows.
|
||||
|
||||
## What Are Subagents?
|
||||
|
||||
Subagents are task-specific Claude instances that can be launched using the `Task` tool in Claude Code. Each subagent has specialized knowledge and instructions tailored to a particular domain, such as coding, testing, database work, or DevOps.
|
||||
|
||||
**Why Use Subagents?**
|
||||
|
||||
- **Focused Expertise**: Each subagent has domain-specific knowledge and instructions
|
||||
- **Better Context Management**: Subagents can work on isolated tasks without polluting the main conversation
|
||||
- **Parallel Work**: Multiple subagents can work on independent tasks simultaneously
|
||||
- **Consistency**: Subagents follow project-specific patterns and conventions automatically
|
||||
|
||||
## Available Subagents
|
||||
|
||||
The following subagents are available for use in this project:
|
||||
|
||||
### Core Development
|
||||
|
||||
| Subagent | Purpose | When to Use |
|
||||
| --------- | --------------------------------------------------------------- | ---------------------------------------------------------------- |
|
||||
| **plan** | Design implementation plans, identify files, analyze trade-offs | Starting new features, major refactoring, architecture decisions |
|
||||
| **coder** | Write and modify production Node.js/TypeScript code | Implementing features, fixing bugs, writing new modules |
|
||||
|
||||
### Testing and Quality
|
||||
|
||||
| Subagent | Purpose | When to Use |
|
||||
| ----------------- | ----------------------------------------------------------------- | ---------------------------------------------------- |
|
||||
| **tester** | Adversarial testing: edge cases, race conditions, vulnerabilities | Finding bugs, security testing, stress testing |
|
||||
| **testwriter** | Create comprehensive tests for features and fixes | Writing unit tests, integration tests, test coverage |
|
||||
| **code-reviewer** | Review code quality, security, best practices | Code review, PR reviews, architecture review |
|
||||
|
||||
### Database and Infrastructure
|
||||
|
||||
| Subagent | Purpose | When to Use |
|
||||
| ------------------- | --------------------------------------------------------- | -------------------------------------------------------------- |
|
||||
| **db-dev** | Schemas, queries, migrations, optimization, N+1 problems | Database development, query optimization, schema changes |
|
||||
| **db-admin** | PostgreSQL/Redis admin, security, backups, infrastructure | Database administration, performance tuning, backup strategies |
|
||||
| **devops** | Containers, services, CI/CD pipelines, deployments | Deployment issues, container configuration, CI/CD pipelines |
|
||||
| **infra-architect** | Resource optimization: RAM, CPU, disk, storage | Capacity planning, performance optimization, cost reduction |
|
||||
|
||||
### Specialized Technical
|
||||
|
||||
| Subagent | Purpose | When to Use |
|
||||
| --------------------------- | ---------------------------------------------------------- | --------------------------------------------------------- |
|
||||
| **bg-worker** | Background jobs: PM2 workers, BullMQ queues, async tasks | Queue management, worker debugging, job scheduling |
|
||||
| **ai-usage** | LLM APIs (Gemini, Claude), prompt engineering, AI features | AI integration, prompt optimization, Gemini API issues |
|
||||
| **security-engineer** | Security audits, vulnerability scanning, OWASP, pentesting | Security reviews, vulnerability assessments, compliance |
|
||||
| **log-debug** | Production errors, observability, Bugsink/Sentry analysis | Debugging production issues, log analysis, error tracking |
|
||||
| **integrations-specialist** | Third-party services, webhooks, external APIs | External API integration, webhook implementation |
|
||||
|
||||
### Frontend and Design
|
||||
|
||||
| Subagent | Purpose | When to Use |
|
||||
| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------- |
|
||||
| **frontend-specialist** | UI components, Neo-Brutalism, Core Web Vitals, accessibility | Frontend development, performance optimization, accessibility |
|
||||
| **uiux-designer** | UI/UX decisions, component design, Neo-Brutalism compliance | Design decisions, user experience improvements |
|
||||
|
||||
### Documentation and Planning
|
||||
|
||||
| Subagent | Purpose | When to Use |
|
||||
| -------------------- | ----------------------------------------------------------- | ---------------------------------------------------------- |
|
||||
| **documenter** | User docs, API specs, feature documentation | Writing documentation, API specs, user guides |
|
||||
| **describer-for-ai** | Technical docs for AI: ADRs, system overviews, context docs | Writing ADRs, technical specifications, context documents |
|
||||
| **planner** | Break down features, roadmaps, scope management | Project planning, feature breakdown, roadmap development |
|
||||
| **product-owner** | Feature requirements, user stories, validation, backlog | Requirements gathering, user story writing, prioritization |
|
||||
|
||||
### Support
|
||||
|
||||
| Subagent | Purpose | When to Use |
|
||||
| -------------------------------- | ---------------------------------------- | ---------------------------------------------------- |
|
||||
| **tools-integration-specialist** | Bugsink, Gitea, OAuth, operational tools | Tool configuration, OAuth setup, operational tooling |
|
||||
|
||||
## How to Launch a Subagent
|
||||
|
||||
Subagents are launched using the `Task` tool in Claude Code. Simply ask Claude to use a specific subagent for a task:
|
||||
|
||||
```
|
||||
"Use the coder subagent to implement the new store search feature"
|
||||
```
|
||||
|
||||
Or:
|
||||
|
||||
```
|
||||
"Launch the db-dev subagent to optimize the flyer items query"
|
||||
```
|
||||
|
||||
Claude will automatically invoke the appropriate subagent with the relevant context.
|
||||
|
||||
## Subagent Selection Guide
|
||||
|
||||
### Which Subagent Should I Use?
|
||||
|
||||
**For Writing Code:**
|
||||
|
||||
- New features or modules: `coder`
|
||||
- Complex architectural changes: `plan` first, then `coder`
|
||||
- Database-related code: `db-dev`
|
||||
- Frontend components: `frontend-specialist`
|
||||
- Background job code: `bg-worker`
|
||||
|
||||
**For Testing:**
|
||||
|
||||
- Writing new tests: `testwriter`
|
||||
- Finding edge cases and bugs: `tester`
|
||||
- Reviewing test coverage: `code-reviewer`
|
||||
|
||||
**For Infrastructure:**
|
||||
|
||||
- Container issues: `devops`
|
||||
- CI/CD pipelines: `devops`
|
||||
- Database administration: `db-admin`
|
||||
- Performance optimization: `infra-architect`
|
||||
|
||||
**For Debugging:**
|
||||
|
||||
- Production errors: `log-debug`
|
||||
- Database issues: `db-admin` or `db-dev`
|
||||
- AI/Gemini issues: `ai-usage`
|
||||
|
||||
**For Documentation:**
|
||||
|
||||
- API documentation: `documenter`
|
||||
- Architecture decisions: `describer-for-ai`
|
||||
- Planning and requirements: `planner` or `product-owner`
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Start with Planning
|
||||
|
||||
For complex features, always start with the `plan` subagent to:
|
||||
|
||||
- Identify affected files
|
||||
- Understand architectural implications
|
||||
- Break down the work into manageable tasks
|
||||
|
||||
### 2. Use Specialized Subagents for Specialized Work
|
||||
|
||||
Avoid using `coder` for database migrations. Use `db-dev` instead - it understands:
|
||||
|
||||
- The project's migration patterns
|
||||
- Schema synchronization requirements
|
||||
- PostgreSQL-specific optimizations
|
||||
|
||||
### 3. Let Subagents Follow Project Conventions
|
||||
|
||||
All subagents are pre-configured with knowledge of project conventions:
|
||||
|
||||
- ADR patterns (see [docs/adr/index.md](../adr/index.md))
|
||||
- Repository pattern standards (ADR-034)
|
||||
- Service layer architecture (ADR-035)
|
||||
- Testing standards (ADR-010)
|
||||
|
||||
### 4. Combine Subagents for Complex Tasks
|
||||
|
||||
Some tasks benefit from multiple subagents:
|
||||
|
||||
1. **New API endpoint**: `plan` -> `coder` -> `testwriter` -> `code-reviewer`
|
||||
2. **Database optimization**: `db-dev` -> `tester` -> `infra-architect`
|
||||
3. **Security fix**: `security-engineer` -> `coder` -> `testwriter`
|
||||
|
||||
### 5. Always Run Tests in the Dev Container
|
||||
|
||||
Regardless of which subagent you use, remember:
|
||||
|
||||
> **ALL tests MUST be run in the dev container (Linux environment)**
|
||||
|
||||
The subagents know this, but as a developer, ensure you verify test results in the correct environment:
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
```
|
||||
|
||||
## Subagent Communication
|
||||
|
||||
Subagents can pass information back to the main conversation and to each other through:
|
||||
|
||||
1. **Direct Output**: Results and recommendations returned to the conversation
|
||||
2. **File Changes**: Code, documentation, and configuration changes
|
||||
3. **Todo Lists**: Task tracking and progress updates
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [CODER-GUIDE.md](./CODER-GUIDE.md) - Working with the coder subagent
|
||||
- [TESTER-GUIDE.md](./TESTER-GUIDE.md) - Testing strategies and patterns
|
||||
- [DATABASE-GUIDE.md](./DATABASE-GUIDE.md) - Database development workflows
|
||||
- [DEVOPS-GUIDE.md](./DEVOPS-GUIDE.md) - DevOps and deployment workflows
|
||||
- [../adr/index.md](../adr/index.md) - Architecture Decision Records
|
||||
- [../TESTING.md](../TESTING.md) - Testing guide
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Subagent Not Available
|
||||
|
||||
If a subagent fails to launch, it may be due to:
|
||||
|
||||
- Incorrect subagent name (check the list above)
|
||||
- Network or API issues
|
||||
- Context length limitations
|
||||
|
||||
### Subagent Gives Incorrect Advice
|
||||
|
||||
All subagents follow the CLAUDE.md instructions. If advice seems wrong:
|
||||
|
||||
1. Verify the project context is correct
|
||||
2. Check if the advice conflicts with an ADR
|
||||
3. Provide additional context to the subagent
|
||||
|
||||
### Subagent Takes Too Long
|
||||
|
||||
For complex tasks, subagents may take time. Consider:
|
||||
|
||||
- Breaking the task into smaller pieces
|
||||
- Using the `plan` subagent first to scope the work
|
||||
- Running simpler queries first to verify understanding
|
||||
439
docs/subagents/SECURITY-DEBUG-GUIDE.md
Normal file
439
docs/subagents/SECURITY-DEBUG-GUIDE.md
Normal file
@@ -0,0 +1,439 @@
|
||||
# Security and Debugging Subagent Guide
|
||||
|
||||
This guide covers security and debugging-focused subagents:
|
||||
|
||||
- **security-engineer**: Security audits, vulnerability scanning, OWASP, pentesting
|
||||
- **log-debug**: Production errors, observability, Bugsink/Sentry analysis
|
||||
- **code-reviewer**: Code quality, security review, best practices
|
||||
|
||||
## The security-engineer Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **security-engineer** subagent when you need to:
|
||||
|
||||
- Conduct security audits of code or features
|
||||
- Review authentication/authorization flows
|
||||
- Identify vulnerabilities (OWASP Top 10)
|
||||
- Review API security
|
||||
- Assess data protection measures
|
||||
- Plan security improvements
|
||||
|
||||
### What security-engineer Knows
|
||||
|
||||
The security-engineer subagent understands:
|
||||
|
||||
- OWASP Top 10 vulnerabilities
|
||||
- Node.js/Express security best practices
|
||||
- JWT authentication security
|
||||
- SQL injection prevention
|
||||
- XSS and CSRF protection
|
||||
- Rate limiting strategies (ADR-032)
|
||||
- API security hardening (ADR-016)
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Security Audit:**
|
||||
|
||||
```
|
||||
"Use security-engineer to audit the user registration and
|
||||
login flow for security vulnerabilities. Check for common
|
||||
issues like credential stuffing, brute force, and session
|
||||
management problems."
|
||||
```
|
||||
|
||||
**API Security Review:**
|
||||
|
||||
```
|
||||
"Use security-engineer to review the flyer upload endpoint
|
||||
for security issues. Consider file type validation, size
|
||||
limits, malicious file handling, and authorization."
|
||||
```
|
||||
|
||||
**Vulnerability Assessment:**
|
||||
|
||||
```
|
||||
"Use security-engineer to assess our exposure to the OWASP
|
||||
Top 10 vulnerabilities. Identify any gaps in our current
|
||||
security measures."
|
||||
```
|
||||
|
||||
### Security Checklist
|
||||
|
||||
The security-engineer subagent uses this checklist:
|
||||
|
||||
#### Authentication & Authorization
|
||||
|
||||
- [ ] Password hashing with bcrypt (cost factor >= 10)
|
||||
- [ ] JWT tokens with appropriate expiration
|
||||
- [ ] Refresh token rotation
|
||||
- [ ] Session invalidation on password change
|
||||
- [ ] Role-based access control (RBAC)
|
||||
|
||||
#### Input Validation
|
||||
|
||||
- [ ] All user input validated with Zod schemas
|
||||
- [ ] SQL queries use parameterized statements
|
||||
- [ ] File uploads validated for type and size
|
||||
- [ ] Path traversal prevention
|
||||
|
||||
#### Data Protection
|
||||
|
||||
- [ ] Sensitive data encrypted at rest
|
||||
- [ ] HTTPS enforced in production
|
||||
- [ ] No secrets in source code
|
||||
- [ ] Proper error messages (no stack traces to users)
|
||||
|
||||
#### Rate Limiting
|
||||
|
||||
- [ ] Login attempts limited
|
||||
- [ ] API endpoints rate limited
|
||||
- [ ] File upload rate limited
|
||||
|
||||
#### Headers & CORS
|
||||
|
||||
- [ ] Security headers set (Helmet.js)
|
||||
- [ ] CORS configured appropriately
|
||||
- [ ] Content-Security-Policy defined
|
||||
|
||||
### Security Patterns in This Project
|
||||
|
||||
**Rate Limiting (ADR-032):**
|
||||
|
||||
```typescript
|
||||
// src/config/rateLimiters.ts
|
||||
export const loginLimiter = rateLimit({
|
||||
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||
max: 5, // 5 attempts per window
|
||||
message: 'Too many login attempts',
|
||||
});
|
||||
```
|
||||
|
||||
**Input Validation (ADR-003):**
|
||||
|
||||
```typescript
|
||||
// src/middleware/validation.middleware.ts
|
||||
router.post(
|
||||
'/register',
|
||||
validateRequest(registerSchema),
|
||||
async (req, res, next) => { ... }
|
||||
);
|
||||
```
|
||||
|
||||
**Authentication (ADR-048):**
|
||||
|
||||
```typescript
|
||||
// JWT with refresh tokens
|
||||
const accessToken = jwt.sign(payload, secret, { expiresIn: '15m' });
|
||||
const refreshToken = jwt.sign({ userId }, refreshSecret, { expiresIn: '7d' });
|
||||
```
|
||||
|
||||
## The log-debug Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **log-debug** subagent when you need to:
|
||||
|
||||
- Debug production errors
|
||||
- Analyze Bugsink/Sentry error reports
|
||||
- Investigate performance issues
|
||||
- Trace request flows through logs
|
||||
- Identify patterns in error occurrences
|
||||
|
||||
### What log-debug Knows
|
||||
|
||||
The log-debug subagent understands:
|
||||
|
||||
- Pino structured logging
|
||||
- Bugsink/Sentry error tracking
|
||||
- Log aggregation with Logstash
|
||||
- PostgreSQL function observability (ADR-050)
|
||||
- Request tracing patterns
|
||||
- Error correlation
|
||||
|
||||
### MCP Tools for Debugging
|
||||
|
||||
The log-debug subagent can use MCP tools to access error tracking:
|
||||
|
||||
```
|
||||
// Check Bugsink for production errors
|
||||
mcp__bugsink__list_projects()
|
||||
mcp__bugsink__list_issues({ project_id: 1 })
|
||||
mcp__bugsink__get_event({ event_id: "..." })
|
||||
mcp__bugsink__get_stacktrace({ event_id: "..." })
|
||||
|
||||
// Check local dev errors
|
||||
mcp__localerrors__list_issues({ project_id: 1 })
|
||||
```
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Production Error Investigation:**
|
||||
|
||||
```
|
||||
"Use log-debug to investigate the spike in 500 errors on the
|
||||
flyer processing endpoint yesterday. Check Bugsink for error
|
||||
patterns and identify the root cause."
|
||||
```
|
||||
|
||||
**Performance Analysis:**
|
||||
|
||||
```
|
||||
"Use log-debug to analyze the slow response times on the deals
|
||||
page. Check logs for database query timing and identify any
|
||||
bottlenecks."
|
||||
```
|
||||
|
||||
**Error Pattern Analysis:**
|
||||
|
||||
```
|
||||
"Use log-debug to identify patterns in the authentication
|
||||
failures over the past week. Are they coming from specific
|
||||
IPs or affecting specific users?"
|
||||
```
|
||||
|
||||
### Log Analysis Patterns
|
||||
|
||||
**Structured Log Format (Pino):**
|
||||
|
||||
```json
|
||||
{
|
||||
"level": 50,
|
||||
"time": 1704067200000,
|
||||
"pid": 1234,
|
||||
"hostname": "server1",
|
||||
"module": "flyerService",
|
||||
"requestId": "abc-123",
|
||||
"userId": "user-456",
|
||||
"msg": "Flyer processing failed",
|
||||
"err": {
|
||||
"type": "AIExtractionError",
|
||||
"message": "Rate limit exceeded",
|
||||
"stack": "..."
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Request Tracing:**
|
||||
|
||||
```typescript
|
||||
// Each request gets a unique ID for tracing
|
||||
app.use((req, res, next) => {
|
||||
req.requestId = crypto.randomUUID();
|
||||
req.log = logger.child({ requestId: req.requestId });
|
||||
next();
|
||||
});
|
||||
```
|
||||
|
||||
**Error Correlation:**
|
||||
|
||||
- Same `requestId` across all logs for a request
|
||||
- Same `userId` for user-related errors
|
||||
- Same `flyerId` for flyer processing errors
|
||||
|
||||
### Bugsink Error Tracking
|
||||
|
||||
**Production Bugsink Projects:**
|
||||
|
||||
| Project | ID | Purpose |
|
||||
| ---------------------------- | --- | --------------- |
|
||||
| flyer-crawler-backend | 1 | Backend errors |
|
||||
| flyer-crawler-frontend | 2 | Frontend errors |
|
||||
| flyer-crawler-backend-test | 3 | Test backend |
|
||||
| flyer-crawler-frontend-test | 4 | Test frontend |
|
||||
| flyer-crawler-infrastructure | 5 | Infra errors |
|
||||
|
||||
**Accessing Bugsink:**
|
||||
|
||||
- Production: https://bugsink.projectium.com
|
||||
- Dev Container: http://localhost:8000
|
||||
|
||||
### Log File Locations
|
||||
|
||||
| Environment | Log Path |
|
||||
| ------------- | --------------------------------------------------------- |
|
||||
| Production | `/var/www/flyer-crawler.projectium.com/logs/app.log` |
|
||||
| Test | `/var/www/flyer-crawler-test.projectium.com/logs/app.log` |
|
||||
| Dev Container | `/app/logs/app.log` |
|
||||
|
||||
## The code-reviewer Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **code-reviewer** subagent when you need to:
|
||||
|
||||
- Review code quality before merging
|
||||
- Identify potential issues in implementations
|
||||
- Check adherence to project patterns
|
||||
- Review security implications
|
||||
- Assess test coverage
|
||||
|
||||
### What code-reviewer Knows
|
||||
|
||||
The code-reviewer subagent understands:
|
||||
|
||||
- Project architecture patterns (ADRs)
|
||||
- Repository pattern standards (ADR-034)
|
||||
- Service layer architecture (ADR-035)
|
||||
- Testing standards (ADR-010)
|
||||
- TypeScript best practices
|
||||
- Security considerations
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Code Review:**
|
||||
|
||||
```
|
||||
"Use code-reviewer to review the changes in the shopping list
|
||||
feature branch. Check for adherence to project patterns,
|
||||
potential bugs, and security issues."
|
||||
```
|
||||
|
||||
**Architecture Review:**
|
||||
|
||||
```
|
||||
"Use code-reviewer to review the proposed changes to the
|
||||
caching layer. Does it follow our patterns? Are there
|
||||
potential issues with cache invalidation?"
|
||||
```
|
||||
|
||||
**Security-Focused Review:**
|
||||
|
||||
```
|
||||
"Use code-reviewer to review the new file upload handling
|
||||
code with a focus on security. Check for path traversal,
|
||||
file type validation, and size limits."
|
||||
```
|
||||
|
||||
### Code Review Checklist
|
||||
|
||||
The code-reviewer subagent checks:
|
||||
|
||||
#### Code Quality
|
||||
|
||||
- [ ] Follows TypeScript strict mode
|
||||
- [ ] No `any` types without justification
|
||||
- [ ] Proper error handling
|
||||
- [ ] Meaningful variable names
|
||||
- [ ] Appropriate comments
|
||||
|
||||
#### Architecture
|
||||
|
||||
- [ ] Follows layer separation (Routes -> Services -> Repositories)
|
||||
- [ ] Uses correct file naming conventions
|
||||
- [ ] Repository methods follow naming patterns
|
||||
- [ ] Transactions used for multi-operation changes
|
||||
|
||||
#### Testing
|
||||
|
||||
- [ ] New code has corresponding tests
|
||||
- [ ] Tests follow project patterns
|
||||
- [ ] Edge cases covered
|
||||
- [ ] Mocks used appropriately
|
||||
|
||||
#### Security
|
||||
|
||||
- [ ] Input validation present
|
||||
- [ ] Authorization checks in place
|
||||
- [ ] No secrets in code
|
||||
- [ ] Error messages don't leak information
|
||||
|
||||
#### Performance
|
||||
|
||||
- [ ] No obvious N+1 queries
|
||||
- [ ] Appropriate use of caching
|
||||
- [ ] Large data sets paginated
|
||||
- [ ] Expensive operations async/queued
|
||||
|
||||
### Review Output Format
|
||||
|
||||
```markdown
|
||||
## Code Review: [Feature/PR Name]
|
||||
|
||||
### Summary
|
||||
|
||||
Brief overview of the changes reviewed.
|
||||
|
||||
### Issues Found
|
||||
|
||||
#### Critical
|
||||
|
||||
- **[File:Line]** Description of critical issue
|
||||
- Impact: What could go wrong
|
||||
- Suggestion: How to fix
|
||||
|
||||
#### High Priority
|
||||
|
||||
- **[File:Line]** Description
|
||||
|
||||
#### Medium Priority
|
||||
|
||||
- **[File:Line]** Description
|
||||
|
||||
#### Low Priority / Suggestions
|
||||
|
||||
- **[File:Line]** Description
|
||||
|
||||
### Positive Observations
|
||||
|
||||
- Good patterns followed
|
||||
- Well-tested areas
|
||||
- Clean implementations
|
||||
|
||||
### Recommendations
|
||||
|
||||
1. Priority items to address before merge
|
||||
2. Items for follow-up tickets
|
||||
```
|
||||
|
||||
## Debugging Workflow
|
||||
|
||||
### 1. Error Investigation
|
||||
|
||||
```
|
||||
1. Identify the error in Bugsink
|
||||
mcp__bugsink__list_issues({ project_id: 1, status: "unresolved" })
|
||||
|
||||
2. Get error details
|
||||
mcp__bugsink__get_issue({ issue_id: "..." })
|
||||
|
||||
3. Get full stacktrace
|
||||
mcp__bugsink__get_stacktrace({ event_id: "..." })
|
||||
|
||||
4. Check for patterns across events
|
||||
mcp__bugsink__list_events({ issue_id: "..." })
|
||||
```
|
||||
|
||||
### 2. Log Correlation
|
||||
|
||||
```bash
|
||||
# Find related logs by request ID
|
||||
grep "requestId\":\"abc-123\"" /var/www/flyer-crawler.projectium.com/logs/app.log
|
||||
|
||||
# Find all errors in a time range
|
||||
jq 'select(.level >= 50 and .time >= 1704067200000)' app.log
|
||||
```
|
||||
|
||||
### 3. Database Query Analysis
|
||||
|
||||
```bash
|
||||
# Check slow query log
|
||||
tail -f /var/log/postgresql/postgresql-$(date +%Y-%m-%d).log | grep "duration:"
|
||||
```
|
||||
|
||||
### 4. Root Cause Analysis
|
||||
|
||||
- Correlate error timing with deployments
|
||||
- Check for resource constraints (memory, connections)
|
||||
- Review recent code changes
|
||||
- Check external service status
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
|
||||
- [DEVOPS-GUIDE.md](./DEVOPS-GUIDE.md) - Infrastructure debugging
|
||||
- [../adr/0016-api-security-hardening.md](../adr/0016-api-security-hardening.md) - Security ADR
|
||||
- [../adr/0032-rate-limiting-strategy.md](../adr/0032-rate-limiting-strategy.md) - Rate limiting
|
||||
- [../adr/0015-application-performance-monitoring-and-error-tracking.md](../adr/0015-application-performance-monitoring-and-error-tracking.md) - Monitoring ADR
|
||||
- [../adr/0050-postgresql-function-observability.md](../adr/0050-postgresql-function-observability.md) - Database observability
|
||||
- [../BARE-METAL-SETUP.md](../BARE-METAL-SETUP.md) - Production setup
|
||||
404
docs/subagents/TESTER-GUIDE.md
Normal file
404
docs/subagents/TESTER-GUIDE.md
Normal file
@@ -0,0 +1,404 @@
|
||||
# Tester and Testwriter Subagent Guide
|
||||
|
||||
This guide covers two related but distinct subagents for testing in the Flyer Crawler project:
|
||||
|
||||
- **tester**: Adversarial testing to find edge cases, race conditions, and vulnerabilities
|
||||
- **testwriter**: Creating comprehensive test suites for features and fixes
|
||||
|
||||
## Understanding the Difference
|
||||
|
||||
| Aspect | tester | testwriter |
|
||||
| --------------- | ------------------------------- | ------------------------------- |
|
||||
| **Purpose** | Find bugs and weaknesses | Create test coverage |
|
||||
| **Approach** | Adversarial, exploratory | Systematic, comprehensive |
|
||||
| **Output** | Bug reports, security findings | Test files, test utilities |
|
||||
| **When to Use** | Before release, security review | During development, refactoring |
|
||||
|
||||
## The tester Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **tester** subagent when you need to:
|
||||
|
||||
- Find edge cases that might cause failures
|
||||
- Identify race conditions in async code
|
||||
- Test security vulnerabilities
|
||||
- Stress test APIs or database queries
|
||||
- Validate error handling paths
|
||||
- Find memory leaks or performance issues
|
||||
|
||||
### What the tester Knows
|
||||
|
||||
The tester subagent understands:
|
||||
|
||||
- Common vulnerability patterns (SQL injection, XSS, CSRF)
|
||||
- Race condition scenarios in Node.js
|
||||
- Edge cases in data validation
|
||||
- Authentication and authorization bypasses
|
||||
- BullMQ queue edge cases
|
||||
- Database transaction isolation issues
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Finding edge cases:**
|
||||
|
||||
```
|
||||
"Use the tester subagent to find edge cases in the flyer upload
|
||||
endpoint. Consider file types, sizes, concurrent uploads, and
|
||||
invalid data scenarios."
|
||||
```
|
||||
|
||||
**Security testing:**
|
||||
|
||||
```
|
||||
"Use the tester subagent to review the authentication flow for
|
||||
security vulnerabilities, including JWT handling, session management,
|
||||
and OAuth integration."
|
||||
```
|
||||
|
||||
**Race condition analysis:**
|
||||
|
||||
```
|
||||
"Use the tester subagent to identify potential race conditions in
|
||||
the shopping list sharing feature where multiple users might modify
|
||||
the same list simultaneously."
|
||||
```
|
||||
|
||||
### Sample Output from tester
|
||||
|
||||
The tester subagent typically produces:
|
||||
|
||||
1. **Vulnerability Reports**
|
||||
- Issue description
|
||||
- Reproduction steps
|
||||
- Severity assessment
|
||||
- Recommended fix
|
||||
|
||||
2. **Edge Case Catalog**
|
||||
- Input combinations to test
|
||||
- Expected vs actual behavior
|
||||
- Priority for fixing
|
||||
|
||||
3. **Test Scenarios**
|
||||
- Detailed test cases for the testwriter
|
||||
- Setup and teardown requirements
|
||||
- Assertions to verify
|
||||
|
||||
## The testwriter Subagent
|
||||
|
||||
### When to Use
|
||||
|
||||
Use the **testwriter** subagent when you need to:
|
||||
|
||||
- Write unit tests for new features
|
||||
- Add integration tests for API endpoints
|
||||
- Create end-to-end test scenarios
|
||||
- Improve test coverage for existing code
|
||||
- Write regression tests for bug fixes
|
||||
- Create test utilities and factories
|
||||
|
||||
### What the testwriter Knows
|
||||
|
||||
The testwriter subagent understands:
|
||||
|
||||
- Project testing stack (Vitest, Testing Library, Supertest)
|
||||
- Mock factory patterns (`src/tests/utils/mockFactories.ts`)
|
||||
- Test helper utilities (`src/tests/utils/testHelpers.ts`)
|
||||
- Database cleanup patterns
|
||||
- Integration test setup with globalSetup
|
||||
- Known testing issues documented in CLAUDE.md
|
||||
|
||||
### Testing Framework Stack
|
||||
|
||||
| Tool | Version | Purpose |
|
||||
| ------------------------- | ------- | ----------------- |
|
||||
| Vitest | 4.0.15 | Test runner |
|
||||
| @testing-library/react | 16.3.0 | Component testing |
|
||||
| @testing-library/jest-dom | 6.9.1 | DOM assertions |
|
||||
| supertest | 7.1.4 | API testing |
|
||||
| msw | 2.12.3 | Network mocking |
|
||||
|
||||
### Test File Organization
|
||||
|
||||
```
|
||||
src/
|
||||
├── components/
|
||||
│ └── *.test.tsx # Component tests (colocated)
|
||||
├── hooks/
|
||||
│ └── *.test.ts # Hook tests (colocated)
|
||||
├── services/
|
||||
│ └── *.test.ts # Service tests (colocated)
|
||||
├── routes/
|
||||
│ └── *.test.ts # Route handler tests (colocated)
|
||||
└── tests/
|
||||
├── integration/ # Integration tests
|
||||
└── e2e/ # End-to-end tests
|
||||
```
|
||||
|
||||
### Example Requests
|
||||
|
||||
**Unit tests for a new feature:**
|
||||
|
||||
```
|
||||
"Use the testwriter subagent to create comprehensive unit tests
|
||||
for the new StoreSearchService in src/services/storeSearchService.ts.
|
||||
Include edge cases for empty results, partial matches, and pagination."
|
||||
```
|
||||
|
||||
**Integration tests for API:**
|
||||
|
||||
```
|
||||
"Use the testwriter subagent to add integration tests for the
|
||||
POST /api/flyers endpoint, covering successful uploads, validation
|
||||
errors, authentication requirements, and file size limits."
|
||||
```
|
||||
|
||||
**Regression test for bug fix:**
|
||||
|
||||
```
|
||||
"Use the testwriter subagent to create a regression test that
|
||||
verifies the fix for issue #123 where duplicate flyer items were
|
||||
created when uploading certain PDFs."
|
||||
```
|
||||
|
||||
### Test Patterns the testwriter Uses
|
||||
|
||||
#### Unit Test Pattern
|
||||
|
||||
```typescript
|
||||
// src/services/storeSearchService.test.ts
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
||||
import { createMockStore, resetMockIds } from '@/tests/utils/mockFactories';
|
||||
|
||||
describe('StoreSearchService', () => {
|
||||
beforeEach(() => {
|
||||
resetMockIds(); // Ensure deterministic IDs
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('searchByName', () => {
|
||||
it('returns matching stores when query matches', async () => {
|
||||
const mockStore = createMockStore({ name: 'Test Mart' });
|
||||
// ... test implementation
|
||||
});
|
||||
|
||||
it('returns empty array when no matches found', async () => {
|
||||
// ... test implementation
|
||||
});
|
||||
|
||||
it('handles special characters in search query', async () => {
|
||||
// ... test implementation
|
||||
});
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
#### Integration Test Pattern
|
||||
|
||||
```typescript
|
||||
// src/tests/integration/stores.integration.test.ts
|
||||
import supertest from 'supertest';
|
||||
import { createAndLoginUser, cleanupDb } from '@/tests/utils/testHelpers';
|
||||
|
||||
describe('Stores API', () => {
|
||||
let request: ReturnType<typeof supertest>;
|
||||
let authToken: string;
|
||||
let testUserId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
const app = (await import('../../../server')).default;
|
||||
request = supertest(app);
|
||||
const { token, userId } = await createAndLoginUser(request);
|
||||
authToken = token;
|
||||
testUserId = userId;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await cleanupDb({ users: [testUserId] });
|
||||
});
|
||||
|
||||
describe('GET /api/stores', () => {
|
||||
it('returns list of stores', async () => {
|
||||
const response = await request.get('/api/stores').set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data.stores).toBeInstanceOf(Array);
|
||||
});
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
#### Component Test Pattern
|
||||
|
||||
```typescript
|
||||
// src/components/StoreCard.test.tsx
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { renderWithProviders, screen } from '@/tests/utils/renderWithProviders';
|
||||
import { createMockStore } from '@/tests/utils/mockFactories';
|
||||
import { StoreCard } from './StoreCard';
|
||||
|
||||
describe('StoreCard', () => {
|
||||
it('renders store name and location count', () => {
|
||||
const store = createMockStore({
|
||||
name: 'Test Store',
|
||||
location_count: 5
|
||||
});
|
||||
|
||||
renderWithProviders(<StoreCard store={store} />);
|
||||
|
||||
expect(screen.getByText('Test Store')).toBeInTheDocument();
|
||||
expect(screen.getByText('5 locations')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('calls onSelect when clicked', async () => {
|
||||
const store = createMockStore();
|
||||
const handleSelect = vi.fn();
|
||||
|
||||
renderWithProviders(<StoreCard store={store} onSelect={handleSelect} />);
|
||||
|
||||
await userEvent.click(screen.getByText(store.name));
|
||||
|
||||
expect(handleSelect).toHaveBeenCalledWith(store);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## Test Execution Environment
|
||||
|
||||
### Critical Requirement
|
||||
|
||||
> **ALL tests MUST be executed inside the dev container (Linux environment)**
|
||||
|
||||
Tests that pass on Windows but fail on Linux are considered **broken tests**.
|
||||
|
||||
### Running Tests
|
||||
|
||||
```bash
|
||||
# From Windows host - run in container
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
podman exec -it flyer-crawler-dev npm run test:integration
|
||||
|
||||
# Inside dev container
|
||||
npm run test:unit
|
||||
npm run test:integration
|
||||
|
||||
# Run specific test file
|
||||
npm test -- --run src/services/storeService.test.ts
|
||||
```
|
||||
|
||||
### Test Commands Reference
|
||||
|
||||
| Command | Description |
|
||||
| -------------------------- | ------------------------------------- |
|
||||
| `npm test` | All unit tests |
|
||||
| `npm run test:unit` | Unit tests only |
|
||||
| `npm run test:integration` | Integration tests (requires DB/Redis) |
|
||||
| `npm run test:coverage` | Tests with coverage report |
|
||||
|
||||
## Known Testing Issues
|
||||
|
||||
The testwriter subagent is aware of these documented issues:
|
||||
|
||||
### 1. Vitest globalSetup Context Isolation
|
||||
|
||||
Vitest's `globalSetup` runs in a separate Node.js context. Mocks and spies do NOT share instances with test files.
|
||||
|
||||
**Impact**: BullMQ worker service mocks don't work in integration tests.
|
||||
|
||||
**Solution**: Use `.todo()` for affected tests or create test-only API endpoints.
|
||||
|
||||
### 2. Cleanup Queue Timing
|
||||
|
||||
The cleanup worker may process jobs before tests can verify them.
|
||||
|
||||
**Solution**:
|
||||
|
||||
```typescript
|
||||
const { cleanupQueue } = await import('../../services/queues.server');
|
||||
await cleanupQueue.drain();
|
||||
await cleanupQueue.pause();
|
||||
// ... run test ...
|
||||
await cleanupQueue.resume();
|
||||
```
|
||||
|
||||
### 3. Cache Stale After Direct SQL
|
||||
|
||||
Direct database inserts bypass cache invalidation.
|
||||
|
||||
**Solution**:
|
||||
|
||||
```typescript
|
||||
await cacheService.invalidateFlyers();
|
||||
```
|
||||
|
||||
### 4. Unique Filenames Required
|
||||
|
||||
File upload tests need unique filenames to avoid collisions.
|
||||
|
||||
**Solution**:
|
||||
|
||||
```typescript
|
||||
const filename = `test-${Date.now()}-${Math.round(Math.random() * 1e9)}.jpg`;
|
||||
```
|
||||
|
||||
## Test Coverage Guidelines
|
||||
|
||||
### When Writing Tests
|
||||
|
||||
1. **Unit Tests** (required for all new code):
|
||||
- Pure functions and utilities
|
||||
- React components
|
||||
- Custom hooks
|
||||
- Service methods
|
||||
- Repository methods
|
||||
|
||||
2. **Integration Tests** (required for API changes):
|
||||
- New API endpoints
|
||||
- Authentication flows
|
||||
- Middleware behavior
|
||||
|
||||
3. **E2E Tests** (for critical paths):
|
||||
- User registration/login
|
||||
- Flyer upload workflow
|
||||
- Admin operations
|
||||
|
||||
### Test Isolation
|
||||
|
||||
1. Reset mock IDs in `beforeEach()`
|
||||
2. Use unique test data (timestamps, UUIDs)
|
||||
3. Clean up after tests with `cleanupDb()`
|
||||
4. Don't share state between tests
|
||||
|
||||
## Combining tester and testwriter
|
||||
|
||||
A typical workflow for thorough testing:
|
||||
|
||||
1. **Development**: Write code with basic tests using `testwriter`
|
||||
2. **Edge Cases**: Use `tester` to identify edge cases and vulnerabilities
|
||||
3. **Coverage**: Use `testwriter` to add tests for identified edge cases
|
||||
4. **Review**: Use `code-reviewer` to verify test quality
|
||||
|
||||
### Example Combined Workflow
|
||||
|
||||
```
|
||||
1. "Use testwriter to create initial tests for the new discount
|
||||
calculation feature"
|
||||
|
||||
2. "Use tester to find edge cases in the discount calculation -
|
||||
consider rounding errors, negative values, percentage limits,
|
||||
and currency precision"
|
||||
|
||||
3. "Use testwriter to add tests for the edge cases identified:
|
||||
- Rounding to 2 decimal places
|
||||
- Negative discount values
|
||||
- Discounts over 100%
|
||||
- Very small amounts (under $0.01)"
|
||||
```
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [OVERVIEW.md](./OVERVIEW.md) - Subagent system overview
|
||||
- [CODER-GUIDE.md](./CODER-GUIDE.md) - Working with the coder subagent
|
||||
- [../TESTING.md](../TESTING.md) - Testing guide
|
||||
- [../adr/0010-testing-strategy-and-standards.md](../adr/0010-testing-strategy-and-standards.md) - Testing ADR
|
||||
- [../adr/0040-testing-economics-and-priorities.md](../adr/0040-testing-economics-and-priorities.md) - Testing priorities
|
||||
1037
docs/tools/BUGSINK-SETUP.md
Normal file
1037
docs/tools/BUGSINK-SETUP.md
Normal file
File diff suppressed because it is too large
Load Diff
892
docs/tools/MCP-CONFIGURATION.md
Normal file
892
docs/tools/MCP-CONFIGURATION.md
Normal file
@@ -0,0 +1,892 @@
|
||||
# MCP Configuration Guide
|
||||
|
||||
This document provides comprehensive guidance for configuring Model Context Protocol (MCP) servers with Claude Code for the Flyer Crawler project.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [What is MCP](#what-is-mcp)
|
||||
2. [Server Overview](#server-overview)
|
||||
3. [Configuration Locations](#configuration-locations)
|
||||
4. [Global Settings Configuration](#global-settings-configuration)
|
||||
5. [Project-Level Configuration](#project-level-configuration)
|
||||
6. [Server Setup Instructions](#server-setup-instructions)
|
||||
7. [Bugsink MCP](#bugsink-mcp)
|
||||
8. [PostgreSQL MCP](#postgresql-mcp)
|
||||
9. [Gitea MCP](#gitea-mcp)
|
||||
10. [Other MCP Servers](#other-mcp-servers)
|
||||
11. [Troubleshooting](#troubleshooting)
|
||||
12. [Best Practices](#best-practices)
|
||||
|
||||
---
|
||||
|
||||
## What is MCP
|
||||
|
||||
Model Context Protocol (MCP) is a standardized protocol that allows AI assistants like Claude to interact with external tools and services. MCP servers expose capabilities (tools) that Claude can invoke to:
|
||||
|
||||
- Query databases
|
||||
- Manage containers
|
||||
- Access file systems
|
||||
- Interact with APIs (Gitea, Bugsink, etc.)
|
||||
- Store and retrieve knowledge graph data
|
||||
- Inspect caches and key-value stores
|
||||
|
||||
**Why We Use MCP:**
|
||||
|
||||
| Benefit | Description |
|
||||
| ------------------ | ------------------------------------------------------------------------ |
|
||||
| Direct Integration | Claude can directly query databases, inspect containers, and access APIs |
|
||||
| Context Awareness | Tools provide real-time information without manual copy-paste |
|
||||
| Automation | Complex workflows can be executed through tool chains |
|
||||
| Consistency | Standardized interface across different services |
|
||||
|
||||
---
|
||||
|
||||
## Server Overview
|
||||
|
||||
The Flyer Crawler project uses the following MCP servers:
|
||||
|
||||
| Server | Tool Prefix | Purpose | Config Location |
|
||||
| ------------------ | -------------------------- | -------------------------------------------------- | --------------- |
|
||||
| `gitea-projectium` | `mcp__gitea-projectium__*` | Gitea API at gitea.projectium.com | Global |
|
||||
| `gitea-torbonium` | `mcp__gitea-torbonium__*` | Gitea API at gitea.torbonium.com | Global |
|
||||
| `podman` | `mcp__podman__*` | Container management | Global |
|
||||
| `filesystem` | `mcp__filesystem__*` | File system access | Global |
|
||||
| `memory` | `mcp__memory__*` | Knowledge graph persistence | Global |
|
||||
| `redis` | `mcp__redis__*` | Redis cache inspection | Global |
|
||||
| `bugsink` | `mcp__bugsink__*` | Production error tracking (bugsink.projectium.com) | Global |
|
||||
| `localerrors` | `mcp__localerrors__*` | Dev container error tracking (localhost:8000) | Project |
|
||||
| `devdb` | `mcp__devdb__*` | Development PostgreSQL database | Project |
|
||||
|
||||
---
|
||||
|
||||
## Configuration Locations
|
||||
|
||||
Claude Code uses **two separate configuration systems** for MCP servers:
|
||||
|
||||
### Global Configuration
|
||||
|
||||
**Location (Windows):**
|
||||
|
||||
```text
|
||||
C:\Users\<username>\.claude\settings.json
|
||||
```
|
||||
|
||||
**Used For:**
|
||||
|
||||
- Production services (HTTPS endpoints)
|
||||
- Servers shared across all projects
|
||||
- Container management (Podman)
|
||||
- Knowledge graph (Memory)
|
||||
|
||||
### Project-Level Configuration
|
||||
|
||||
**Location:**
|
||||
|
||||
```text
|
||||
<project-root>/.mcp.json
|
||||
```
|
||||
|
||||
**Used For:**
|
||||
|
||||
- Localhost services (HTTP endpoints)
|
||||
- Development databases
|
||||
- Project-specific tools
|
||||
|
||||
### When to Use Each
|
||||
|
||||
| Scenario | Configuration |
|
||||
| --------------------------------- | ---------------------- |
|
||||
| Production APIs (HTTPS) | Global `settings.json` |
|
||||
| Shared tools (memory, filesystem) | Global `settings.json` |
|
||||
| Localhost services (HTTP) | Project `.mcp.json` |
|
||||
| Development databases | Project `.mcp.json` |
|
||||
| Per-project customization | Project `.mcp.json` |
|
||||
|
||||
**Important:** Localhost MCP servers work more reliably in project-level `.mcp.json` than in global `settings.json`. See [Troubleshooting](#localhost-servers-not-loading) for details.
|
||||
|
||||
---
|
||||
|
||||
## Global Settings Configuration
|
||||
|
||||
### File Format
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"server-name": {
|
||||
"command": "path/to/executable",
|
||||
"args": ["arg1", "arg2"],
|
||||
"env": {
|
||||
"ENV_VAR": "value"
|
||||
},
|
||||
"disabled": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Configuration Options:**
|
||||
|
||||
| Field | Required | Description |
|
||||
| ---------- | -------- | ----------------------------------------- |
|
||||
| `command` | Yes | Path to executable or command |
|
||||
| `args` | No | Array of command-line arguments |
|
||||
| `env` | No | Environment variables for the server |
|
||||
| `disabled` | No | Set to `true` to disable without removing |
|
||||
|
||||
### Example Global Configuration
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"memory": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@modelcontextprotocol/server-memory"]
|
||||
},
|
||||
"filesystem": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": [
|
||||
"c:\\Users\\<user>\\AppData\\Roaming\\npm\\node_modules\\@modelcontextprotocol\\server-filesystem\\dist\\index.js",
|
||||
"d:\\gitea"
|
||||
]
|
||||
},
|
||||
"podman": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "podman-mcp-server@latest"],
|
||||
"env": {
|
||||
"DOCKER_HOST": "npipe:////./pipe/podman-machine-default"
|
||||
}
|
||||
},
|
||||
"redis": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@modelcontextprotocol/server-redis", "redis://localhost:6379"]
|
||||
},
|
||||
"bugsink": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "https://bugsink.projectium.com",
|
||||
"BUGSINK_TOKEN": "<40-char-hex-token>"
|
||||
}
|
||||
},
|
||||
"gitea-projectium": {
|
||||
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
|
||||
"args": ["run", "-t", "stdio"],
|
||||
"env": {
|
||||
"GITEA_HOST": "https://gitea.projectium.com",
|
||||
"GITEA_ACCESS_TOKEN": "<your-token>"
|
||||
}
|
||||
},
|
||||
"gitea-torbonium": {
|
||||
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
|
||||
"args": ["run", "-t", "stdio"],
|
||||
"env": {
|
||||
"GITEA_HOST": "https://gitea.torbonium.com",
|
||||
"GITEA_ACCESS_TOKEN": "<your-token>"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Project-Level Configuration
|
||||
|
||||
### File Location
|
||||
|
||||
Create `.mcp.json` in the project root:
|
||||
|
||||
```text
|
||||
d:\gitea\flyer-crawler.projectium.com\flyer-crawler.projectium.com\.mcp.json
|
||||
```
|
||||
|
||||
### File Format
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"server-name": {
|
||||
"command": "path/to/executable",
|
||||
"args": ["arg1", "arg2"],
|
||||
"env": {
|
||||
"ENV_VAR": "value"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Current Project Configuration
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"localerrors": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "http://127.0.0.1:8000",
|
||||
"BUGSINK_TOKEN": "<40-char-hex-token>"
|
||||
}
|
||||
},
|
||||
"devdb": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": [
|
||||
"-y",
|
||||
"@modelcontextprotocol/server-postgres",
|
||||
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Server Setup Instructions
|
||||
|
||||
### Memory (Knowledge Graph)
|
||||
|
||||
**Package:** `@modelcontextprotocol/server-memory`
|
||||
|
||||
**Purpose:** Persists knowledge across sessions - project context, credentials, known issues.
|
||||
|
||||
**Configuration:**
|
||||
|
||||
```json
|
||||
"memory": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@modelcontextprotocol/server-memory"]
|
||||
}
|
||||
```
|
||||
|
||||
**Key Tools:**
|
||||
|
||||
- `mcp__memory__read_graph` - Read entire knowledge graph
|
||||
- `mcp__memory__search_nodes` - Search for specific entities
|
||||
- `mcp__memory__create_entities` - Add new knowledge
|
||||
|
||||
### Filesystem
|
||||
|
||||
**Package:** `@modelcontextprotocol/server-filesystem`
|
||||
|
||||
**Purpose:** Provides file system access to specified directories.
|
||||
|
||||
**Configuration:**
|
||||
|
||||
```json
|
||||
"filesystem": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": [
|
||||
"c:\\Users\\<user>\\AppData\\Roaming\\npm\\node_modules\\@modelcontextprotocol\\server-filesystem\\dist\\index.js",
|
||||
"d:\\gitea"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Note:** The last argument(s) specify allowed directories.
|
||||
|
||||
### Podman/Docker
|
||||
|
||||
**Package:** `podman-mcp-server`
|
||||
|
||||
**Purpose:** Container management - list, start, stop, inspect containers.
|
||||
|
||||
**Configuration:**
|
||||
|
||||
```json
|
||||
"podman": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "podman-mcp-server@latest"],
|
||||
"env": {
|
||||
"DOCKER_HOST": "npipe:////./pipe/podman-machine-default"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Key Tools:**
|
||||
|
||||
- `mcp__podman__container_list` - List running containers
|
||||
- `mcp__podman__container_logs` - View container logs
|
||||
- `mcp__podman__container_inspect` - Detailed container info
|
||||
- `mcp__podman__image_list` - List images
|
||||
|
||||
### Redis
|
||||
|
||||
**Package:** `@modelcontextprotocol/server-redis`
|
||||
|
||||
**Purpose:** Inspect Redis cache, set/get values, list keys.
|
||||
|
||||
**Configuration:**
|
||||
|
||||
```json
|
||||
"redis": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@modelcontextprotocol/server-redis", "redis://localhost:6379"]
|
||||
}
|
||||
```
|
||||
|
||||
**Key Tools:**
|
||||
|
||||
- `mcp__redis__get` - Get value by key
|
||||
- `mcp__redis__set` - Set key-value pair
|
||||
- `mcp__redis__list` - List keys matching pattern
|
||||
- `mcp__redis__delete` - Delete key(s)
|
||||
|
||||
---
|
||||
|
||||
## Bugsink MCP
|
||||
|
||||
Bugsink is a self-hosted error tracking service. We run two instances:
|
||||
|
||||
| Instance | URL | MCP Server | Purpose |
|
||||
| ----------- | -------------------------------- | ------------- | ---------------------------- |
|
||||
| Production | `https://bugsink.projectium.com` | `bugsink` | Production error tracking |
|
||||
| Development | `http://localhost:8000` | `localerrors` | Dev container error tracking |
|
||||
|
||||
### Installation
|
||||
|
||||
The `bugsink-mcp` package is **NOT published to npm**. Clone and build from source:
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/j-shelfwood/bugsink-mcp.git d:\gitea\bugsink-mcp
|
||||
|
||||
# Install and build
|
||||
cd d:\gitea\bugsink-mcp
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
**Repository:** https://github.com/j-shelfwood/bugsink-mcp
|
||||
|
||||
### Configuration
|
||||
|
||||
**Production (Global `settings.json`):**
|
||||
|
||||
```json
|
||||
"bugsink": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "https://bugsink.projectium.com",
|
||||
"BUGSINK_TOKEN": "<40-char-hex-token>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Development (Project `.mcp.json`):**
|
||||
|
||||
```json
|
||||
"localerrors": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "http://127.0.0.1:8000",
|
||||
"BUGSINK_TOKEN": "<40-char-hex-token>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Required Environment Variables:**
|
||||
|
||||
| Variable | Description |
|
||||
| --------------- | -------------------------------------------- |
|
||||
| `BUGSINK_URL` | Full URL to Bugsink instance (with protocol) |
|
||||
| `BUGSINK_TOKEN` | 40-character hex API token |
|
||||
|
||||
**Important:**
|
||||
|
||||
- Variable is `BUGSINK_TOKEN`, NOT `BUGSINK_API_TOKEN`
|
||||
- Do NOT use `npx` - the package is not on npm
|
||||
- Use `http://127.0.0.1:8000` not `http://localhost:8000` for localhost
|
||||
|
||||
### Creating API Tokens
|
||||
|
||||
Bugsink 2.0.11 does NOT have a "Settings > API Keys" menu in the UI. Tokens must be created via Django management command.
|
||||
|
||||
**For Dev Container (localhost:8000):**
|
||||
|
||||
```bash
|
||||
MSYS_NO_PATHCONV=1 podman exec \
|
||||
-e DATABASE_URL=postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink \
|
||||
-e SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security \
|
||||
flyer-crawler-dev sh -c 'cd /opt/bugsink/conf && \
|
||||
DJANGO_SETTINGS_MODULE=bugsink_conf \
|
||||
PYTHONPATH=/opt/bugsink/conf:/opt/bugsink/lib/python3.10/site-packages \
|
||||
/opt/bugsink/bin/python -m django create_auth_token'
|
||||
```
|
||||
|
||||
**For Production (via SSH):**
|
||||
|
||||
```bash
|
||||
ssh root@projectium.com "cd /opt/bugsink && bugsink-manage create_auth_token"
|
||||
```
|
||||
|
||||
Both commands output a 40-character lowercase hex token (e.g., `a609c2886daa4e1e05f1517074d7779a5fb49056`).
|
||||
|
||||
### Key Tools
|
||||
|
||||
- `mcp__bugsink__test_connection` / `mcp__localerrors__test_connection` - Verify connection
|
||||
- `mcp__bugsink__list_projects` - List all projects
|
||||
- `mcp__bugsink__list_issues` - List issues for a project
|
||||
- `mcp__bugsink__get_issue` - Get issue details
|
||||
- `mcp__bugsink__get_stacktrace` - Get event stacktrace as Markdown
|
||||
|
||||
### Testing Connection
|
||||
|
||||
```typescript
|
||||
// Production
|
||||
mcp__bugsink__test_connection();
|
||||
// Expected: "Connection successful: Connected successfully. Found N project(s)."
|
||||
|
||||
// Development
|
||||
mcp__localerrors__test_connection();
|
||||
// Expected: "Connection successful: Connected successfully. Found N project(s)."
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## PostgreSQL MCP
|
||||
|
||||
**Package:** `@modelcontextprotocol/server-postgres`
|
||||
|
||||
**Purpose:** Execute SQL queries against the development database.
|
||||
|
||||
### Configuration
|
||||
|
||||
Add to project-level `.mcp.json`:
|
||||
|
||||
```json
|
||||
"devdb": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": [
|
||||
"-y",
|
||||
"@modelcontextprotocol/server-postgres",
|
||||
"postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Connection String Format
|
||||
|
||||
```text
|
||||
postgresql://[user]:[password]@[host]:[port]/[database]
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
|
||||
```text
|
||||
# Development (local container)
|
||||
postgresql://postgres:postgres@127.0.0.1:5432/flyer_crawler_dev
|
||||
|
||||
# Test database
|
||||
postgresql://flyer_crawler_test:password@127.0.0.1:5432/flyer_crawler_test
|
||||
```
|
||||
|
||||
### Database Information
|
||||
|
||||
| Property | Value |
|
||||
| --------------------- | ------------------------ |
|
||||
| Container | `flyer-crawler-postgres` |
|
||||
| Image | `postgis/postgis:15-3.4` |
|
||||
| Host (from Windows) | `127.0.0.1` |
|
||||
| Host (from container) | `postgres` |
|
||||
| Port | `5432` |
|
||||
| Database | `flyer_crawler_dev` |
|
||||
| User | `postgres` |
|
||||
| Password | `postgres` |
|
||||
|
||||
### Usage Examples
|
||||
|
||||
```typescript
|
||||
// List all tables
|
||||
mcp__devdb__query({ sql: "SELECT tablename FROM pg_tables WHERE schemaname = 'public'" });
|
||||
|
||||
// Count records
|
||||
mcp__devdb__query({ sql: 'SELECT COUNT(*) FROM flyers' });
|
||||
|
||||
// Check table structure
|
||||
mcp__devdb__query({
|
||||
sql: "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'flyers'",
|
||||
});
|
||||
|
||||
// Find recent records
|
||||
mcp__devdb__query({
|
||||
sql: 'SELECT id, name, created_at FROM flyers ORDER BY created_at DESC LIMIT 10',
|
||||
});
|
||||
```
|
||||
|
||||
### Prerequisites
|
||||
|
||||
1. **PostgreSQL container must be running:**
|
||||
|
||||
```bash
|
||||
podman ps | grep flyer-crawler-postgres
|
||||
```
|
||||
|
||||
2. **Port 5432 must be mapped:**
|
||||
|
||||
```bash
|
||||
podman port flyer-crawler-postgres
|
||||
# Expected: 5432/tcp -> 0.0.0.0:5432
|
||||
```
|
||||
|
||||
3. **Database must exist:**
|
||||
```bash
|
||||
podman exec flyer-crawler-postgres psql -U postgres -c "\l" | grep flyer_crawler_dev
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Gitea MCP
|
||||
|
||||
**Binary:** `gitea-mcp` (compiled Go binary)
|
||||
|
||||
**Purpose:** Interact with Gitea repositories, issues, pull requests.
|
||||
|
||||
### Configuration
|
||||
|
||||
```json
|
||||
"gitea-projectium": {
|
||||
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
|
||||
"args": ["run", "-t", "stdio"],
|
||||
"env": {
|
||||
"GITEA_HOST": "https://gitea.projectium.com",
|
||||
"GITEA_ACCESS_TOKEN": "<your-token>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Getting Access Token
|
||||
|
||||
1. Log in to Gitea web interface
|
||||
2. Go to **Settings > Applications**
|
||||
3. Under **Generate New Token**, enter a name
|
||||
4. Select required scopes (typically `read:user`, `write:repository`, `write:issue`)
|
||||
5. Click **Generate Token**
|
||||
6. Copy the token immediately (shown only once)
|
||||
|
||||
### Key Tools
|
||||
|
||||
- `mcp__gitea-projectium__list_my_repos` - List accessible repositories
|
||||
- `mcp__gitea-projectium__list_repo_issues` - List issues in a repo
|
||||
- `mcp__gitea-projectium__get_issue_by_index` - Get issue details
|
||||
- `mcp__gitea-projectium__create_issue` - Create new issue
|
||||
- `mcp__gitea-projectium__create_pull_request` - Create PR
|
||||
- `mcp__gitea-projectium__get_file_content` - Read file from repo
|
||||
- `mcp__gitea-projectium__list_branches` - List branches
|
||||
|
||||
### Example Operations
|
||||
|
||||
```typescript
|
||||
// List repositories
|
||||
mcp__gitea - projectium__list_my_repos({ page: 1, pageSize: 20 });
|
||||
|
||||
// Get issue
|
||||
mcp__gitea -
|
||||
projectium__get_issue_by_index({
|
||||
owner: 'username',
|
||||
repo: 'repository-name',
|
||||
index: 42,
|
||||
});
|
||||
|
||||
// Create issue
|
||||
mcp__gitea -
|
||||
projectium__create_issue({
|
||||
owner: 'username',
|
||||
repo: 'repository-name',
|
||||
title: 'Bug: Something is broken',
|
||||
body: '## Description\n\nSteps to reproduce...',
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Other MCP Servers
|
||||
|
||||
### Sequential Thinking
|
||||
|
||||
**Package:** `@modelcontextprotocol/server-sequential-thinking`
|
||||
|
||||
**Purpose:** Structured step-by-step reasoning for complex problems.
|
||||
|
||||
```json
|
||||
"sequential-thinking": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@modelcontextprotocol/server-sequential-thinking"]
|
||||
}
|
||||
```
|
||||
|
||||
### Playwright (Browser Automation)
|
||||
|
||||
**Package:** `@anthropics/mcp-server-playwright`
|
||||
|
||||
**Purpose:** Browser automation for testing and scraping.
|
||||
|
||||
```json
|
||||
"playwright": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@anthropics/mcp-server-playwright"]
|
||||
}
|
||||
```
|
||||
|
||||
### Sentry (Cloud Error Tracking)
|
||||
|
||||
**Package:** `@sentry/mcp-server`
|
||||
|
||||
**Purpose:** Error tracking for Sentry instances (NOT Bugsink).
|
||||
|
||||
```json
|
||||
"sentry": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@sentry/mcp-server"],
|
||||
"env": {
|
||||
"SENTRY_AUTH_TOKEN": "<your-sentry-token>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Note:** Bugsink has a different API than Sentry. Use `bugsink-mcp` for Bugsink instances.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Localhost Servers Not Loading
|
||||
|
||||
**Symptoms:**
|
||||
|
||||
- `mcp__localerrors__*` or `mcp__devdb__*` tools not available
|
||||
- No error messages in logs
|
||||
- Server silently skipped during startup
|
||||
|
||||
**Root Cause:**
|
||||
Claude Code's global `settings.json` has issues loading localhost stdio MCP servers on Windows. The exact cause may be related to:
|
||||
|
||||
- Multiple servers using the same underlying package
|
||||
- Localhost URL filtering
|
||||
- Windows-specific MCP loader bugs
|
||||
|
||||
**Solution:**
|
||||
Use **project-level `.mcp.json`** for all localhost MCP servers. This bypasses the global config loader entirely.
|
||||
|
||||
**Working Pattern:**
|
||||
|
||||
- Global `settings.json`: Production HTTPS servers
|
||||
- Project `.mcp.json`: Localhost HTTP servers
|
||||
|
||||
### Server Name Collision
|
||||
|
||||
**Symptoms:**
|
||||
|
||||
- Second server with similar name never starts
|
||||
- No error logged - server silently filtered out
|
||||
|
||||
**Root Cause:**
|
||||
Claude Code may skip MCP servers when names share prefixes (e.g., `bugsink` and `bugsink-dev`).
|
||||
|
||||
**Solution:**
|
||||
Use completely distinct names:
|
||||
|
||||
- `bugsink` for production
|
||||
- `localerrors` for development (NOT `bugsink-dev` or `devbugsink`)
|
||||
|
||||
### Connection Timed Out
|
||||
|
||||
**Error:** `Connection timed out after 30000ms`
|
||||
|
||||
**Causes:**
|
||||
|
||||
- Server takes too long to start
|
||||
- npx download is slow
|
||||
- Server crashes during initialization
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. Move important servers earlier in config
|
||||
2. Use pre-installed packages instead of npx:
|
||||
```json
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["path/to/installed/package/dist/index.js"]
|
||||
```
|
||||
3. Check server can start manually
|
||||
|
||||
### Environment Variable Issues
|
||||
|
||||
**Common Mistakes:**
|
||||
|
||||
| Wrong | Correct |
|
||||
| ----------------------- | ----------------------- |
|
||||
| `BUGSINK_API_TOKEN` | `BUGSINK_TOKEN` |
|
||||
| `http://localhost:8000` | `http://127.0.0.1:8000` |
|
||||
|
||||
**Verification:**
|
||||
Test server manually with environment variables:
|
||||
|
||||
```bash
|
||||
cd d:\gitea\bugsink-mcp
|
||||
set BUGSINK_URL=http://127.0.0.1:8000
|
||||
set BUGSINK_TOKEN=<your-token>
|
||||
node dist/index.js
|
||||
```
|
||||
|
||||
Expected output:
|
||||
|
||||
```
|
||||
Bugsink MCP server started
|
||||
Connected to: http://127.0.0.1:8000
|
||||
```
|
||||
|
||||
### PostgreSQL Connection Refused
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. Check container is running:
|
||||
|
||||
```bash
|
||||
podman ps | grep flyer-crawler-postgres
|
||||
```
|
||||
|
||||
2. Verify port mapping:
|
||||
|
||||
```bash
|
||||
podman port flyer-crawler-postgres
|
||||
```
|
||||
|
||||
3. Test connection:
|
||||
|
||||
```bash
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT 1"
|
||||
```
|
||||
|
||||
4. Check for port conflicts:
|
||||
```bash
|
||||
netstat -an | findstr 5432
|
||||
```
|
||||
|
||||
### Verifying Configuration
|
||||
|
||||
**List loaded servers:**
|
||||
|
||||
```bash
|
||||
claude mcp list
|
||||
```
|
||||
|
||||
**Check debug logs (Windows):**
|
||||
|
||||
```text
|
||||
C:\Users\<username>\.claude\debug\*.txt
|
||||
```
|
||||
|
||||
Look for MCP server startup messages. Missing servers indicate configuration problems.
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Keep Configs Organized
|
||||
|
||||
- **Global config:** Shared/production servers
|
||||
- **Project config:** Local development servers
|
||||
- **Never duplicate** the same server in both
|
||||
|
||||
### 2. Order Servers by Importance
|
||||
|
||||
Place essential servers first in configuration:
|
||||
|
||||
1. `memory` - Knowledge persistence
|
||||
2. `filesystem` - File access
|
||||
3. `podman` - Container management
|
||||
4. Other servers...
|
||||
|
||||
### 3. Use Direct Node Execution
|
||||
|
||||
For faster startup, avoid npx and use direct node execution:
|
||||
|
||||
```json
|
||||
// Slow (npx downloads on each start)
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "package-name"]
|
||||
|
||||
// Fast (pre-installed)
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["path/to/installed/dist/index.js"]
|
||||
```
|
||||
|
||||
### 4. Disable Instead of Delete
|
||||
|
||||
Use `"disabled": true` to troubleshoot without losing configuration:
|
||||
|
||||
```json
|
||||
"problem-server": {
|
||||
"command": "...",
|
||||
"disabled": true
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Test Manually First
|
||||
|
||||
Before adding to config, verify server works:
|
||||
|
||||
```bash
|
||||
cd /path/to/mcp-server
|
||||
set ENV_VAR=value
|
||||
node dist/index.js
|
||||
```
|
||||
|
||||
### 6. Store Secrets Securely
|
||||
|
||||
- Use the memory MCP to store API tokens for future sessions
|
||||
- Never commit `.mcp.json` with real tokens (add to `.gitignore`)
|
||||
- Use environment variables where possible
|
||||
|
||||
### 7. Restart After Changes
|
||||
|
||||
MCP configuration changes require a full VS Code restart (not just window reload).
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference
|
||||
|
||||
### Available MCP Packages
|
||||
|
||||
| Server | Package/Source | npm? |
|
||||
| ------------------- | -------------------------------------------------- | ---------------------- |
|
||||
| memory | `@modelcontextprotocol/server-memory` | Yes |
|
||||
| filesystem | `@modelcontextprotocol/server-filesystem` | Yes |
|
||||
| redis | `@modelcontextprotocol/server-redis` | Yes |
|
||||
| postgres | `@modelcontextprotocol/server-postgres` | Yes |
|
||||
| sequential-thinking | `@modelcontextprotocol/server-sequential-thinking` | Yes |
|
||||
| podman | `podman-mcp-server` | Yes |
|
||||
| gitea | `gitea-mcp` (binary) | No |
|
||||
| bugsink | `j-shelfwood/bugsink-mcp` | No (build from source) |
|
||||
| sentry | `@sentry/mcp-server` | Yes |
|
||||
| playwright | `@anthropics/mcp-server-playwright` | Yes |
|
||||
|
||||
### Common Tool Prefixes
|
||||
|
||||
| Server | Tool Prefix | Example |
|
||||
| -------------- | ------------------------- | -------------------------------------- |
|
||||
| Memory | `mcp__memory__` | `mcp__memory__read_graph` |
|
||||
| Filesystem | `mcp__filesystem__` | `mcp__filesystem__read_file` |
|
||||
| Podman | `mcp__podman__` | `mcp__podman__container_list` |
|
||||
| Redis | `mcp__redis__` | `mcp__redis__get` |
|
||||
| Bugsink (prod) | `mcp__bugsink__` | `mcp__bugsink__list_issues` |
|
||||
| Bugsink (dev) | `mcp__localerrors__` | `mcp__localerrors__list_issues` |
|
||||
| PostgreSQL | `mcp__devdb__` | `mcp__devdb__query` |
|
||||
| Gitea | `mcp__gitea-projectium__` | `mcp__gitea-projectium__list_my_repos` |
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [CLAUDE.md - MCP Servers Section](../../CLAUDE.md#mcp-servers)
|
||||
- [DEV-CONTAINER-BUGSINK.md](../DEV-CONTAINER-BUGSINK.md)
|
||||
- [BUGSINK-SYNC.md](../BUGSINK-SYNC.md)
|
||||
- [sql/master_schema_rollup.sql](../../sql/master_schema_rollup.sql)
|
||||
|
||||
---
|
||||
|
||||
_Last updated: January 2026_
|
||||
@@ -22,6 +22,7 @@ MCP (Model Context Protocol) allows AI assistants to interact with external tool
|
||||
Access to multiple Gitea instances for repository management, code search, issue tracking, and CI/CD workflows.
|
||||
|
||||
#### Gitea Projectium (Primary)
|
||||
|
||||
- **Host**: `https://gitea.projectium.com`
|
||||
- **Purpose**: Main production Gitea server
|
||||
- **Capabilities**:
|
||||
@@ -31,11 +32,13 @@ Access to multiple Gitea instances for repository management, code search, issue
|
||||
- Repository cloning and management
|
||||
|
||||
#### Gitea Torbonium
|
||||
|
||||
- **Host**: `https://gitea.torbonium.com`
|
||||
- **Purpose**: Development/testing Gitea instance
|
||||
- **Capabilities**: Same as Gitea Projectium
|
||||
|
||||
#### Gitea LAN
|
||||
|
||||
- **Host**: `https://gitea.torbolan.com`
|
||||
- **Purpose**: Local network Gitea instance
|
||||
- **Status**: Disabled (requires token configuration)
|
||||
@@ -43,6 +46,7 @@ Access to multiple Gitea instances for repository management, code search, issue
|
||||
**Executable Location**: `d:\gitea-mcp\gitea-mcp.exe`
|
||||
|
||||
**Configuration Example** (Gemini Code - mcp.json):
|
||||
|
||||
```json
|
||||
{
|
||||
"servers": {
|
||||
@@ -59,6 +63,7 @@ Access to multiple Gitea instances for repository management, code search, issue
|
||||
```
|
||||
|
||||
**Configuration Example** (Claude Code - settings.json):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
@@ -87,10 +92,12 @@ Manages local containers via Podman Desktop (using Docker-compatible API).
|
||||
- Inspect container status and configuration
|
||||
|
||||
**Current Containers** (for this project):
|
||||
|
||||
- `flyer-crawler-postgres` - PostgreSQL 15 + PostGIS on port 5432
|
||||
- `flyer-crawler-redis` - Redis on port 6379
|
||||
|
||||
**Configuration** (Gemini Code - mcp.json):
|
||||
|
||||
```json
|
||||
{
|
||||
"servers": {
|
||||
@@ -106,6 +113,7 @@ Manages local containers via Podman Desktop (using Docker-compatible API).
|
||||
```
|
||||
|
||||
**Configuration** (Claude Code):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
@@ -133,6 +141,7 @@ Direct file system access to the project directory.
|
||||
- Search files
|
||||
|
||||
**Configuration** (Gemini Code - mcp.json):
|
||||
|
||||
```json
|
||||
{
|
||||
"servers": {
|
||||
@@ -149,6 +158,7 @@ Direct file system access to the project directory.
|
||||
```
|
||||
|
||||
**Configuration** (Claude Code):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
@@ -175,6 +185,7 @@ Web request capabilities for documentation lookups and API testing.
|
||||
- Test endpoints
|
||||
|
||||
**Configuration** (Gemini Code - mcp.json):
|
||||
|
||||
```json
|
||||
{
|
||||
"servers": {
|
||||
@@ -187,6 +198,7 @@ Web request capabilities for documentation lookups and API testing.
|
||||
```
|
||||
|
||||
**Configuration** (Claude Code):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
@@ -211,6 +223,7 @@ Browser automation and debugging capabilities.
|
||||
- Network monitoring
|
||||
|
||||
**Configuration** (when enabled):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
@@ -218,9 +231,12 @@ Browser automation and debugging capabilities.
|
||||
"command": "npx",
|
||||
"args": [
|
||||
"chrome-devtools-mcp@latest",
|
||||
"--headless", "false",
|
||||
"--isolated", "false",
|
||||
"--channel", "stable"
|
||||
"--headless",
|
||||
"false",
|
||||
"--isolated",
|
||||
"false",
|
||||
"--channel",
|
||||
"stable"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -240,6 +256,7 @@ Document conversion capabilities.
|
||||
- Convert other document formats
|
||||
|
||||
**Configuration** (when enabled):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
@@ -254,6 +271,7 @@ Document conversion capabilities.
|
||||
## Prerequisites
|
||||
|
||||
### For Podman MCP
|
||||
|
||||
1. **Podman Desktop** installed and running
|
||||
2. Podman machine initialized and started:
|
||||
```powershell
|
||||
@@ -262,6 +280,7 @@ Document conversion capabilities.
|
||||
```
|
||||
|
||||
### For Gitea MCP
|
||||
|
||||
1. **Gitea MCP executable** at `d:\gitea-mcp\gitea-mcp.exe`
|
||||
2. **Gitea Access Tokens** with appropriate permissions:
|
||||
- `repo` - Full repository access
|
||||
@@ -269,10 +288,12 @@ Document conversion capabilities.
|
||||
- `read:organization` - Organization access
|
||||
|
||||
### For Chrome DevTools MCP
|
||||
|
||||
1. **Chrome browser** installed (stable channel)
|
||||
2. **Node.js 18+** for npx execution
|
||||
|
||||
### For Markitdown MCP
|
||||
|
||||
1. **Python 3.8+** installed
|
||||
2. **uvx** (universal virtualenv executor):
|
||||
```powershell
|
||||
@@ -282,39 +303,160 @@ Document conversion capabilities.
|
||||
## Testing MCP Servers
|
||||
|
||||
### Test Podman Connection
|
||||
|
||||
```powershell
|
||||
podman ps
|
||||
# Should list running containers
|
||||
```
|
||||
|
||||
### Test Gitea API Access
|
||||
|
||||
```powershell
|
||||
curl -H "Authorization: token YOUR_TOKEN" https://gitea.projectium.com/api/v1/user
|
||||
# Should return your user information
|
||||
```
|
||||
|
||||
### Test Database Container
|
||||
|
||||
```powershell
|
||||
podman exec flyer-crawler-postgres psql -U postgres -d flyer_crawler_dev -c "SELECT version();"
|
||||
# Should return PostgreSQL version
|
||||
### Claude Code Extension Auto-Update Issues
|
||||
|
||||
#### Problem: Version 2.1.15 Crashes on CPUs Without AVX Support
|
||||
|
||||
Claude Code version 2.1.15 introduced a regression that causes crashes on CPUs that do not support AVX (Advanced Vector Extensions) instructions. The error manifests as:
|
||||
|
||||
```
|
||||
|
||||
Illegal instruction (core dumped)
|
||||
|
||||
```
|
||||
|
||||
or similar AVX-related illegal instruction errors when the extension tries to start.
|
||||
|
||||
**Affected**: CPUs without AVX support (typically older processors or certain VMs)
|
||||
**Working Version**: 2.1.11 (and earlier)
|
||||
**Broken Version**: 2.1.15+
|
||||
|
||||
#### Solution: Disable Auto-Updates for Claude Code Extension
|
||||
|
||||
The VS Code right-click menu option "Disable Auto Update" for extensions may be greyed out and non-functional. Use the settings.json workaround instead.
|
||||
|
||||
**Step 1: Open VS Code Settings JSON**
|
||||
|
||||
Press `Ctrl+Shift+P` and type "Preferences: Open User Settings (JSON)" or manually edit:
|
||||
|
||||
```
|
||||
|
||||
C:\Users\<username>\AppData\Roaming\Code\User\settings.json
|
||||
|
||||
````
|
||||
|
||||
**Step 2: Add the Extension to Ignore List**
|
||||
|
||||
Add the following setting to your `settings.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"extensions.ignoreAutoUpdate": ["anthropic.claude-code"]
|
||||
}
|
||||
````
|
||||
|
||||
If you already have other settings, add it within the existing JSON object:
|
||||
|
||||
```json
|
||||
{
|
||||
"editor.fontSize": 14,
|
||||
"extensions.ignoreAutoUpdate": ["anthropic.claude-code"],
|
||||
"other.settings": "value"
|
||||
}
|
||||
```
|
||||
|
||||
**Step 3: Downgrade to Working Version**
|
||||
|
||||
If you're already on 2.1.15, you need to downgrade:
|
||||
|
||||
1. Open the Extensions view (`Ctrl+Shift+X`)
|
||||
2. Find "Claude Code" in the list
|
||||
3. Click the gear icon next to the extension
|
||||
4. Select "Install Another Version..."
|
||||
5. Choose version **2.1.11** from the list
|
||||
6. Wait for installation to complete
|
||||
7. Reload VS Code when prompted
|
||||
|
||||
**Step 4: Verify Configuration**
|
||||
|
||||
To verify the setting is working:
|
||||
|
||||
1. Open VS Code Settings JSON and confirm `extensions.ignoreAutoUpdate` includes `anthropic.claude-code`
|
||||
2. Check the Extensions view - Claude Code should show version 2.1.11
|
||||
3. VS Code should no longer prompt to update Claude Code automatically
|
||||
|
||||
#### Updating Later When Bug is Fixed
|
||||
|
||||
Once Anthropic releases a fixed version:
|
||||
|
||||
1. **Remove the ignore setting** from `settings.json`:
|
||||
|
||||
```json
|
||||
// Remove or comment out:
|
||||
// "extensions.ignoreAutoUpdate": ["anthropic.claude-code"]
|
||||
```
|
||||
|
||||
2. **Manually update** the extension:
|
||||
- Open Extensions view (`Ctrl+Shift+X`)
|
||||
- Find Claude Code
|
||||
- Click "Update" or use the gear menu to install a specific version
|
||||
|
||||
3. **Or re-enable auto-updates** by removing the extension from the ignore list, then:
|
||||
- Reload VS Code
|
||||
- The extension will update automatically
|
||||
|
||||
#### Alternative: Pin to Specific Version
|
||||
|
||||
If you prefer to pin to a specific version rather than just disabling auto-updates:
|
||||
|
||||
```json
|
||||
{
|
||||
"extensions.autoUpdate": "onlyEnabledExtensions",
|
||||
"extensions.ignoreAutoUpdate": ["anthropic.claude-code"]
|
||||
}
|
||||
```
|
||||
|
||||
This allows other extensions to update automatically while keeping Claude Code locked.
|
||||
|
||||
#### Checking Current Extension Version
|
||||
|
||||
To verify which version is installed:
|
||||
|
||||
1. Open Extensions view (`Ctrl+Shift+X`)
|
||||
2. Find "Claude Code" by Anthropic
|
||||
3. The version number appears below the extension name
|
||||
4. Or click on the extension to see full details including version history
|
||||
|
||||
````
|
||||
|
||||
## Security Notes
|
||||
|
||||
### Token Management
|
||||
|
||||
- **Never commit tokens** to version control
|
||||
- Store tokens in environment variables or secure password managers
|
||||
- Rotate tokens periodically
|
||||
- Use minimal required permissions
|
||||
|
||||
### Access Tokens in Configuration Files
|
||||
|
||||
The configuration files (`mcp.json` and `settings.json`) contain sensitive access tokens. These files should:
|
||||
|
||||
- Be added to `.gitignore`
|
||||
- Have restricted file permissions
|
||||
- Be backed up securely
|
||||
- Be updated when tokens are rotated
|
||||
|
||||
### Current Security Setup
|
||||
|
||||
- `%APPDATA%\Code\User\mcp.json` - Gitea tokens embedded
|
||||
- `%USERPROFILE%\.claude\settings.json` - Gitea tokens embedded
|
||||
- Both files are in user-specific directories with appropriate Windows ACLs
|
||||
@@ -322,10 +464,12 @@ The configuration files (`mcp.json` and `settings.json`) contain sensitive acces
|
||||
## Troubleshooting
|
||||
|
||||
### Podman MCP Not Working
|
||||
|
||||
1. Check Podman machine status:
|
||||
```powershell
|
||||
podman machine list
|
||||
```
|
||||
````
|
||||
|
||||
2. Ensure Podman Desktop is running
|
||||
3. Verify Docker socket is accessible:
|
||||
```powershell
|
||||
@@ -333,6 +477,7 @@ The configuration files (`mcp.json` and `settings.json`) contain sensitive acces
|
||||
```
|
||||
|
||||
### Gitea MCP Connection Issues
|
||||
|
||||
1. Verify token has correct permissions
|
||||
2. Check network connectivity to Gitea server:
|
||||
```powershell
|
||||
@@ -341,11 +486,13 @@ The configuration files (`mcp.json` and `settings.json`) contain sensitive acces
|
||||
3. Ensure `gitea-mcp.exe` is not blocked by antivirus/firewall
|
||||
|
||||
### VS Code Extension Issues
|
||||
|
||||
1. **Reload Window**: Press `Ctrl+Shift+P` → "Developer: Reload Window"
|
||||
2. **Check Extension Logs**: View → Output → Select extension from dropdown
|
||||
3. **Verify JSON Syntax**: Ensure both config files have valid JSON
|
||||
|
||||
### MCP Server Not Loading
|
||||
|
||||
1. Check config file syntax with JSON validator
|
||||
2. Verify executable paths are correct (use forward slashes or escaped backslashes)
|
||||
3. Ensure required dependencies are installed (Node.js, Python, etc.)
|
||||
@@ -356,11 +503,13 @@ The configuration files (`mcp.json` and `settings.json`) contain sensitive acces
|
||||
To add a new MCP server to both Gemini Code and Claude Code:
|
||||
|
||||
1. **Install the MCP server** (if it's an npm package):
|
||||
|
||||
```powershell
|
||||
npm install -g @modelcontextprotocol/server-YOUR-SERVER
|
||||
```
|
||||
|
||||
2. **Add to Gemini Code** (`mcp.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"servers": {
|
||||
@@ -375,6 +524,7 @@ To add a new MCP server to both Gemini Code and Claude Code:
|
||||
```
|
||||
|
||||
3. **Add to Claude Code** (`settings.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
@@ -392,10 +542,12 @@ To add a new MCP server to both Gemini Code and Claude Code:
|
||||
## Current Project Integration
|
||||
|
||||
### ADR Implementation Status
|
||||
|
||||
- **ADR-0002**: Transaction Management ✅ Enforced
|
||||
- **ADR-0003**: Input Validation ✅ Enforced with URL validation
|
||||
|
||||
### Database Setup
|
||||
|
||||
- PostgreSQL 15 + PostGIS running in container
|
||||
- 63 tables created
|
||||
- URL constraints active:
|
||||
@@ -403,6 +555,7 @@ To add a new MCP server to both Gemini Code and Claude Code:
|
||||
- `flyers_icon_url_check` enforces `^https?://.*`
|
||||
|
||||
### Development Workflow
|
||||
|
||||
1. Start containers: `podman start flyer-crawler-postgres flyer-crawler-redis`
|
||||
2. Use MCP servers to manage development environment
|
||||
3. AI assistants can:
|
||||
@@ -421,6 +574,7 @@ To add a new MCP server to both Gemini Code and Claude Code:
|
||||
## Maintenance
|
||||
|
||||
### Regular Tasks
|
||||
|
||||
- **Monthly**: Rotate Gitea access tokens
|
||||
- **Weekly**: Update MCP server packages:
|
||||
```powershell
|
||||
@@ -429,7 +583,9 @@ To add a new MCP server to both Gemini Code and Claude Code:
|
||||
- **As Needed**: Update Gitea MCP executable when new version is released
|
||||
|
||||
### Backup Configuration
|
||||
|
||||
Recommended to backup these files regularly:
|
||||
|
||||
- `%APPDATA%\Code\User\mcp.json`
|
||||
- `%USERPROFILE%\.claude\settings.json`
|
||||
|
||||
@@ -442,6 +598,7 @@ This project uses Gitea Actions for continuous integration and deployment. The w
|
||||
#### Automated Workflows
|
||||
|
||||
**deploy-to-test.yml** - Automated deployment to test environment
|
||||
|
||||
- **Trigger**: Automatically on every push to `main` branch
|
||||
- **Runner**: `projectium.com` (self-hosted)
|
||||
- **Process**:
|
||||
@@ -459,6 +616,7 @@ This project uses Gitea Actions for continuous integration and deployment. The w
|
||||
#### Manual Workflows
|
||||
|
||||
**deploy-to-prod.yml** - Manual deployment to production
|
||||
|
||||
- **Trigger**: Manual via workflow_dispatch
|
||||
- **Confirmation Required**: Must type "deploy-to-prod"
|
||||
- **Process**:
|
||||
@@ -471,28 +629,34 @@ This project uses Gitea Actions for continuous integration and deployment. The w
|
||||
- **Optional**: Force PM2 reload even if version matches
|
||||
|
||||
**manual-db-backup.yml** - Database backup workflow
|
||||
|
||||
- Creates timestamped backup of production database
|
||||
- Stored in `/var/backups/postgres/`
|
||||
|
||||
**manual-db-restore.yml** - Database restore workflow
|
||||
|
||||
- Restores production database from backup file
|
||||
- Requires confirmation and backup filename
|
||||
|
||||
**manual-db-reset-test.yml** - Reset test database
|
||||
|
||||
- Drops and recreates test database schema
|
||||
- Used for testing schema migrations
|
||||
|
||||
**manual-db-reset-prod.yml** - Reset production database
|
||||
|
||||
- **DANGER**: Drops and recreates production database
|
||||
- Requires multiple confirmations
|
||||
|
||||
**manual-deploy-major.yml** - Major version deployment
|
||||
|
||||
- Similar to deploy-to-prod but bumps major version
|
||||
- For breaking changes or major releases
|
||||
|
||||
### Accessing Workflows via Gitea MCP
|
||||
|
||||
With the Gitea MCP server configured, AI assistants can:
|
||||
|
||||
- View workflow files
|
||||
- Monitor workflow runs
|
||||
- Check deployment status
|
||||
@@ -500,6 +664,7 @@ With the Gitea MCP server configured, AI assistants can:
|
||||
- Trigger manual workflows (via API)
|
||||
|
||||
**Example MCP Operations**:
|
||||
|
||||
```bash
|
||||
# Via Gitea MCP, you can:
|
||||
# - List recent workflow runs
|
||||
@@ -514,6 +679,7 @@ With the Gitea MCP server configured, AI assistants can:
|
||||
The workflows use these Gitea repository secrets:
|
||||
|
||||
**Database**:
|
||||
|
||||
- `DB_HOST` - PostgreSQL host
|
||||
- `DB_USER` - Database user
|
||||
- `DB_PASSWORD` - Database password
|
||||
@@ -521,15 +687,18 @@ The workflows use these Gitea repository secrets:
|
||||
- `DB_DATABASE_TEST` - Test database name
|
||||
|
||||
**Redis**:
|
||||
|
||||
- `REDIS_PASSWORD_PROD` - Production Redis password
|
||||
- `REDIS_PASSWORD_TEST` - Test Redis password
|
||||
|
||||
**API Keys**:
|
||||
|
||||
- `VITE_GOOGLE_GENAI_API_KEY` - Production Gemini API key
|
||||
- `VITE_GOOGLE_GENAI_API_KEY_TEST` - Test Gemini API key
|
||||
- `GOOGLE_MAPS_API_KEY` - Google Maps Geocoding API key
|
||||
|
||||
**Authentication**:
|
||||
|
||||
- `JWT_SECRET` - JWT signing secret
|
||||
|
||||
### Schema Migration Process
|
||||
@@ -542,6 +711,7 @@ The workflows use a schema hash comparison system:
|
||||
4. **Protection**: Deployment fails if schemas don't match
|
||||
|
||||
**Manual Migration Steps** (when schema changes):
|
||||
|
||||
1. Update `sql/master_schema_rollup.sql`
|
||||
2. Run manual migration workflow or:
|
||||
```bash
|
||||
@@ -554,16 +724,23 @@ The workflows use a schema hash comparison system:
|
||||
The workflows manage three PM2 processes per environment:
|
||||
|
||||
**Production** (`ecosystem.config.cjs --env production`):
|
||||
|
||||
- `flyer-crawler-api` - Express API server
|
||||
- `flyer-crawler-worker` - Background job worker
|
||||
- `flyer-crawler-analytics-worker` - Analytics processor
|
||||
|
||||
**Test** (`ecosystem.config.cjs --env test`):
|
||||
|
||||
- `flyer-crawler-api-test` - Test Express API server
|
||||
- `flyer-crawler-worker-test` - Test background worker
|
||||
- `flyer-crawler-analytics-worker-test` - Test analytics worker
|
||||
|
||||
**Process Cleanup**:
|
||||
|
||||
- **2026-01-22**: Added Claude Code extension auto-update troubleshooting
|
||||
- Documented AVX CPU crash bug in version 2.1.15
|
||||
- Added workaround using `extensions.ignoreAutoUpdate` setting
|
||||
- Included instructions for downgrading and re-enabling updates
|
||||
- Workflows automatically delete errored/stopped processes
|
||||
- Version comparison prevents unnecessary reloads
|
||||
- Force reload option available for production
|
||||
@@ -605,17 +782,20 @@ Using Gitea MCP, you can monitor deployments in real-time:
|
||||
With the configured MCP servers, you can:
|
||||
|
||||
**Via Gitea MCP**:
|
||||
|
||||
- Trigger manual workflows
|
||||
- View deployment history
|
||||
- Monitor test results
|
||||
- Access workflow logs
|
||||
|
||||
**Via Podman MCP**:
|
||||
|
||||
- Inspect container logs (for local testing)
|
||||
- Manage local database containers
|
||||
- Test migrations locally
|
||||
|
||||
**Via Filesystem MCP**:
|
||||
|
||||
- Review workflow files
|
||||
- Edit deployment scripts
|
||||
- Update ecosystem config
|
||||
176
ecosystem.dev.config.cjs
Normal file
176
ecosystem.dev.config.cjs
Normal file
@@ -0,0 +1,176 @@
|
||||
// ecosystem.dev.config.cjs
|
||||
// ============================================================================
|
||||
// DEVELOPMENT PM2 CONFIGURATION
|
||||
// ============================================================================
|
||||
// This file mirrors the production ecosystem.config.cjs but with settings
|
||||
// appropriate for the development environment inside the dev container.
|
||||
//
|
||||
// Key differences from production:
|
||||
// - Single instance for API (not cluster mode) to allow debugger attachment
|
||||
// - tsx watch mode for hot reloading
|
||||
// - Development-specific environment variables
|
||||
// - Logs written to /var/log/pm2 for Logstash integration (ADR-050)
|
||||
//
|
||||
// Usage:
|
||||
// pm2 start ecosystem.dev.config.cjs
|
||||
// pm2 logs # View all logs
|
||||
// pm2 logs flyer-crawler-api-dev # View API logs only
|
||||
// pm2 restart all # Restart all processes
|
||||
// pm2 delete all # Stop all processes
|
||||
//
|
||||
// Related:
|
||||
// - ecosystem.config.cjs (production configuration)
|
||||
// - docs/adr/0014-linux-only-platform.md
|
||||
// - docs/adr/0050-postgresql-function-observability.md
|
||||
// ============================================================================
|
||||
|
||||
// --- Environment Variable Validation ---
|
||||
// In dev, we warn but don't fail - variables come from compose.dev.yml
|
||||
const requiredVars = ['DB_HOST', 'JWT_SECRET'];
|
||||
const missingVars = requiredVars.filter((key) => !process.env[key]);
|
||||
|
||||
if (missingVars.length > 0) {
|
||||
console.warn(
|
||||
'\n[ecosystem.dev.config.cjs] WARNING: The following environment variables are MISSING:',
|
||||
);
|
||||
missingVars.forEach((key) => console.warn(` - ${key}`));
|
||||
console.warn(
|
||||
'[ecosystem.dev.config.cjs] These should be set in compose.dev.yml or .env.local\n',
|
||||
);
|
||||
} else {
|
||||
console.log('[ecosystem.dev.config.cjs] Required environment variables are present.');
|
||||
}
|
||||
|
||||
// --- Shared Environment Variables ---
|
||||
// These come from compose.dev.yml environment section
|
||||
const sharedEnv = {
|
||||
// Timezone: PST (America/Los_Angeles) for consistent log timestamps
|
||||
TZ: process.env.TZ || 'America/Los_Angeles',
|
||||
NODE_ENV: 'development',
|
||||
DB_HOST: process.env.DB_HOST || 'postgres',
|
||||
DB_PORT: process.env.DB_PORT || '5432',
|
||||
DB_USER: process.env.DB_USER || 'postgres',
|
||||
DB_PASSWORD: process.env.DB_PASSWORD || 'postgres',
|
||||
DB_NAME: process.env.DB_NAME || 'flyer_crawler_dev',
|
||||
REDIS_URL: process.env.REDIS_URL || 'redis://redis:6379',
|
||||
REDIS_HOST: process.env.REDIS_HOST || 'redis',
|
||||
REDIS_PORT: process.env.REDIS_PORT || '6379',
|
||||
FRONTEND_URL: process.env.FRONTEND_URL || 'https://localhost',
|
||||
JWT_SECRET: process.env.JWT_SECRET || 'dev-jwt-secret-change-in-production',
|
||||
WORKER_LOCK_DURATION: process.env.WORKER_LOCK_DURATION || '120000',
|
||||
// Sentry/Bugsink (ADR-015)
|
||||
SENTRY_DSN: process.env.SENTRY_DSN,
|
||||
SENTRY_ENVIRONMENT: process.env.SENTRY_ENVIRONMENT || 'development',
|
||||
SENTRY_ENABLED: process.env.SENTRY_ENABLED || 'true',
|
||||
SENTRY_DEBUG: process.env.SENTRY_DEBUG || 'true',
|
||||
// Optional API keys (may not be set in dev)
|
||||
GEMINI_API_KEY: process.env.GEMINI_API_KEY,
|
||||
GOOGLE_MAPS_API_KEY: process.env.GOOGLE_MAPS_API_KEY,
|
||||
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
|
||||
GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
apps: [
|
||||
// =========================================================================
|
||||
// API SERVER (Development)
|
||||
// =========================================================================
|
||||
// Single instance with tsx watch for hot reloading.
|
||||
// Unlike production, we don't use cluster mode to allow:
|
||||
// - Debugger attachment
|
||||
// - Simpler log output
|
||||
// - Hot reloading via tsx watch
|
||||
{
|
||||
name: 'flyer-crawler-api-dev',
|
||||
script: './node_modules/.bin/tsx',
|
||||
args: 'watch server.ts',
|
||||
cwd: '/app',
|
||||
watch: false, // tsx watch handles this
|
||||
instances: 1, // Single instance for debugging
|
||||
exec_mode: 'fork', // Fork mode (not cluster) for debugging
|
||||
max_memory_restart: '1G', // More generous in dev
|
||||
kill_timeout: 5000,
|
||||
// PM2 log configuration (ADR-050 - Logstash integration)
|
||||
output: '/var/log/pm2/api-out.log',
|
||||
error: '/var/log/pm2/api-error.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
// Restart configuration
|
||||
max_restarts: 10,
|
||||
exp_backoff_restart_delay: 500,
|
||||
min_uptime: '5s',
|
||||
// Environment
|
||||
env: {
|
||||
...sharedEnv,
|
||||
PORT: '3001',
|
||||
},
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// BACKGROUND WORKER (Development)
|
||||
// =========================================================================
|
||||
// Processes background jobs (flyer processing, cleanup, etc.)
|
||||
// Runs as a separate process like in production.
|
||||
{
|
||||
name: 'flyer-crawler-worker-dev',
|
||||
script: './node_modules/.bin/tsx',
|
||||
args: 'watch src/services/worker.ts',
|
||||
cwd: '/app',
|
||||
watch: false, // tsx watch handles this
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
max_memory_restart: '1G',
|
||||
kill_timeout: 10000, // Workers need more time to finish jobs
|
||||
// PM2 log configuration (ADR-050)
|
||||
output: '/var/log/pm2/worker-out.log',
|
||||
error: '/var/log/pm2/worker-error.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
// Restart configuration
|
||||
max_restarts: 10,
|
||||
exp_backoff_restart_delay: 500,
|
||||
min_uptime: '5s',
|
||||
// Environment
|
||||
env: {
|
||||
...sharedEnv,
|
||||
},
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// VITE FRONTEND DEV SERVER (Development)
|
||||
// =========================================================================
|
||||
// Vite dev server for frontend hot module replacement.
|
||||
// Listens on 5173 and is proxied by nginx to 443.
|
||||
{
|
||||
name: 'flyer-crawler-vite-dev',
|
||||
script: './node_modules/.bin/vite',
|
||||
args: '--host',
|
||||
cwd: '/app',
|
||||
watch: false, // Vite handles its own watching
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
max_memory_restart: '2G', // Vite can use significant memory
|
||||
kill_timeout: 5000,
|
||||
// PM2 log configuration (ADR-050)
|
||||
output: '/var/log/pm2/vite-out.log',
|
||||
error: '/var/log/pm2/vite-error.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
// Restart configuration
|
||||
max_restarts: 10,
|
||||
exp_backoff_restart_delay: 500,
|
||||
min_uptime: '5s',
|
||||
// Environment
|
||||
env: {
|
||||
// Timezone: PST (America/Los_Angeles) for consistent log timestamps
|
||||
TZ: process.env.TZ || 'America/Los_Angeles',
|
||||
NODE_ENV: 'development',
|
||||
// Vite-specific env vars (VITE_ prefix)
|
||||
VITE_SENTRY_DSN: process.env.VITE_SENTRY_DSN,
|
||||
VITE_SENTRY_ENVIRONMENT: process.env.VITE_SENTRY_ENVIRONMENT || 'development',
|
||||
VITE_SENTRY_ENABLED: process.env.VITE_SENTRY_ENABLED || 'true',
|
||||
VITE_SENTRY_DEBUG: process.env.VITE_SENTRY_DEBUG || 'true',
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -47,6 +47,13 @@ server {
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
}
|
||||
|
||||
# Serve flyer images from static storage
|
||||
location /flyer-images/ {
|
||||
alias /var/www/flyer-crawler-test.projectium.com/flyer-images/;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# Correct MIME type for .mjs files
|
||||
location ~ \.mjs$ {
|
||||
include /etc/nginx/mime.types;
|
||||
|
||||
@@ -51,6 +51,13 @@ server {
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
}
|
||||
|
||||
# Serve flyer images from static storage
|
||||
location /flyer-images/ {
|
||||
alias /var/www/flyer-crawler.projectium.com/flyer-images/;
|
||||
expires 7d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# This block specifically targets requests for .mjs files.
|
||||
location ~ \.mjs$ {
|
||||
# It ensures that these files are served with the correct JavaScript MIME type.
|
||||
@@ -65,7 +72,7 @@ server {
|
||||
add_header X-XSS-Protection "1; mode=block" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
|
||||
listen [::]:443 ssl ipv6only=on; # managed by Certbot
|
||||
listen [::]:443 ssl; # managed by Certbot
|
||||
listen 443 ssl; # managed by Certbot
|
||||
ssl_certificate /etc/letsencrypt/live/flyer-crawler.projectium.com/fullchain.pem; # managed by Certbot
|
||||
ssl_certificate_key /etc/letsencrypt/live/flyer-crawler.projectium.com/privkey.pem; # managed by Certbot
|
||||
|
||||
186
package-lock.json
generated
186
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "flyer-crawler",
|
||||
"version": "0.12.1",
|
||||
"version": "0.12.12",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "flyer-crawler",
|
||||
"version": "0.12.1",
|
||||
"version": "0.12.12",
|
||||
"dependencies": {
|
||||
"@bull-board/api": "^6.14.2",
|
||||
"@bull-board/express": "^6.14.2",
|
||||
@@ -15,12 +15,12 @@
|
||||
"@sentry/react": "^10.32.1",
|
||||
"@tanstack/react-query": "^5.90.12",
|
||||
"@types/connect-timeout": "^1.9.0",
|
||||
"@types/react-joyride": "^2.0.2",
|
||||
"bcrypt": "^5.1.1",
|
||||
"bullmq": "^5.65.1",
|
||||
"connect-timeout": "^1.9.1",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"date-fns": "^4.1.0",
|
||||
"driver.js": "^1.3.1",
|
||||
"exif-parser": "^0.1.12",
|
||||
"express": "^5.1.0",
|
||||
"express-list-endpoints": "^7.1.1",
|
||||
@@ -45,7 +45,6 @@
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
"react-hot-toast": "^2.6.0",
|
||||
"react-joyride": "^2.9.3",
|
||||
"react-router-dom": "^7.9.6",
|
||||
"recharts": "^3.4.1",
|
||||
"sharp": "^0.34.5",
|
||||
@@ -2144,12 +2143,6 @@
|
||||
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@gilbarbara/deep-equal": {
|
||||
"version": "0.3.1",
|
||||
"resolved": "https://registry.npmjs.org/@gilbarbara/deep-equal/-/deep-equal-0.3.1.tgz",
|
||||
"integrity": "sha512-I7xWjLs2YSVMc5gGx1Z3ZG1lgFpITPndpi8Ku55GeEIKpACCPQNS/OTqQbxgTCfq0Ncvcc+CrFov96itVh6Qvw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@google/genai": {
|
||||
"version": "1.34.0",
|
||||
"resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.34.0.tgz",
|
||||
@@ -6603,6 +6596,7 @@
|
||||
"version": "19.2.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.7.tgz",
|
||||
"integrity": "sha512-MWtvHrGZLFttgeEj28VXHxpmwYbor/ATPYbBfSFZEIRK0ecCFLl2Qo55z52Hss+UV9CRN7trSeq1zbgx7YDWWg==",
|
||||
"devOptional": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"csstype": "^3.2.2"
|
||||
@@ -6618,15 +6612,6 @@
|
||||
"@types/react": "^19.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/react-joyride": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/react-joyride/-/react-joyride-2.0.2.tgz",
|
||||
"integrity": "sha512-RbixI8KE4K4B4bVzigT765oiQMCbWqlb9vj5qz1pFvkOvynkiAGurGVVf+nGszGGa89WrQhUnAwd0t1tqxeoDw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/react": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/send": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz",
|
||||
@@ -9354,13 +9339,6 @@
|
||||
"integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/deep-diff": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/deep-diff/-/deep-diff-1.0.2.tgz",
|
||||
"integrity": "sha512-aWS3UIVH+NPGCD1kki+DCU9Dua032iSsO43LqQpcs4R3+dVv7tX0qBGjiVHJHjplsoUM2XRO/KB92glqc68awg==",
|
||||
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/deep-is": {
|
||||
"version": "0.1.4",
|
||||
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
|
||||
@@ -9368,15 +9346,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/deepmerge": {
|
||||
"version": "4.3.1",
|
||||
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
|
||||
"integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/default-require-extensions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-3.0.1.tgz",
|
||||
@@ -9625,6 +9594,12 @@
|
||||
"url": "https://dotenvx.com"
|
||||
}
|
||||
},
|
||||
"node_modules/driver.js": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.4.0.tgz",
|
||||
"integrity": "sha512-Gm64jm6PmcU+si21sQhBrTAM1JvUrR0QhNmjkprNLxohOBzul9+pNHXgQaT9lW84gwg9GMLB3NZGuGolsz5uew==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
@@ -12186,12 +12161,6 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-lite": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/is-lite/-/is-lite-1.2.1.tgz",
|
||||
"integrity": "sha512-pgF+L5bxC+10hLBgf6R2P4ZZUBOQIIacbdo8YvuCP8/JvsWxG7aZ9p10DYuLtifFci4l3VITphhMlMV4Y+urPw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/is-map": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.3.tgz",
|
||||
@@ -12731,6 +12700,7 @@
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/js-yaml": {
|
||||
@@ -13629,6 +13599,7 @@
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
|
||||
"integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"js-tokens": "^3.0.0 || ^4.0.0"
|
||||
@@ -14792,6 +14763,13 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/openapi-types": {
|
||||
"version": "12.1.3",
|
||||
"resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz",
|
||||
"integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==",
|
||||
"license": "MIT",
|
||||
"peer": true
|
||||
},
|
||||
"node_modules/optionator": {
|
||||
"version": "0.9.4",
|
||||
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
|
||||
@@ -15420,17 +15398,6 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/popper.js": {
|
||||
"version": "1.16.1",
|
||||
"resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz",
|
||||
"integrity": "sha512-Wb4p1J4zyFTbM+u6WuO4XstYx4Ky9Cewe4DWrel7B0w6VVICvPwdOpotjzcf6eD8TsckVnIMNONQyPIUFOUbCQ==",
|
||||
"deprecated": "You can find the new Popper v2 at @popperjs/core, this package is dedicated to the legacy v1",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/popperjs"
|
||||
}
|
||||
},
|
||||
"node_modules/possible-typed-array-names": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz",
|
||||
@@ -15630,6 +15597,7 @@
|
||||
"version": "15.8.1",
|
||||
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
|
||||
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"loose-envify": "^1.4.0",
|
||||
@@ -15641,6 +15609,7 @@
|
||||
"version": "16.13.1",
|
||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
|
||||
"integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/proper-lockfile": {
|
||||
@@ -15856,45 +15825,6 @@
|
||||
"react": "^19.2.3"
|
||||
}
|
||||
},
|
||||
"node_modules/react-floater": {
|
||||
"version": "0.7.9",
|
||||
"resolved": "https://registry.npmjs.org/react-floater/-/react-floater-0.7.9.tgz",
|
||||
"integrity": "sha512-NXqyp9o8FAXOATOEo0ZpyaQ2KPb4cmPMXGWkx377QtJkIXHlHRAGer7ai0r0C1kG5gf+KJ6Gy+gdNIiosvSicg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"deepmerge": "^4.3.1",
|
||||
"is-lite": "^0.8.2",
|
||||
"popper.js": "^1.16.0",
|
||||
"prop-types": "^15.8.1",
|
||||
"tree-changes": "^0.9.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "15 - 18",
|
||||
"react-dom": "15 - 18"
|
||||
}
|
||||
},
|
||||
"node_modules/react-floater/node_modules/@gilbarbara/deep-equal": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@gilbarbara/deep-equal/-/deep-equal-0.1.2.tgz",
|
||||
"integrity": "sha512-jk+qzItoEb0D0xSSmrKDDzf9sheQj/BAPxlgNxgmOaA3mxpUa6ndJLYGZKsJnIVEQSD8zcTbyILz7I0HcnBCRA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/react-floater/node_modules/is-lite": {
|
||||
"version": "0.8.2",
|
||||
"resolved": "https://registry.npmjs.org/is-lite/-/is-lite-0.8.2.tgz",
|
||||
"integrity": "sha512-JZfH47qTsslwaAsqbMI3Q6HNNjUuq6Cmzzww50TdP5Esb6e1y2sK2UAaZZuzfAzpoI2AkxoPQapZdlDuP6Vlsw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/react-floater/node_modules/tree-changes": {
|
||||
"version": "0.9.3",
|
||||
"resolved": "https://registry.npmjs.org/tree-changes/-/tree-changes-0.9.3.tgz",
|
||||
"integrity": "sha512-vvvS+O6kEeGRzMglTKbc19ltLWNtmNt1cpBoSYLj/iEcPVvpJasemKOlxBrmZaCtDJoF+4bwv3m01UKYi8mukQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@gilbarbara/deep-equal": "^0.1.1",
|
||||
"is-lite": "^0.8.2"
|
||||
}
|
||||
},
|
||||
"node_modules/react-hot-toast": {
|
||||
"version": "2.6.0",
|
||||
"resolved": "https://registry.npmjs.org/react-hot-toast/-/react-hot-toast-2.6.0.tgz",
|
||||
@@ -15912,64 +15842,12 @@
|
||||
"react-dom": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/react-innertext": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/react-innertext/-/react-innertext-1.1.5.tgz",
|
||||
"integrity": "sha512-PWAqdqhxhHIv80dT9znP2KvS+hfkbRovFp4zFYHFFlOoQLRiawIic81gKb3U1wEyJZgMwgs3JoLtwryASRWP3Q==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"@types/react": ">=0.0.0 <=99",
|
||||
"react": ">=0.0.0 <=99"
|
||||
}
|
||||
},
|
||||
"node_modules/react-is": {
|
||||
"version": "17.0.2",
|
||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",
|
||||
"integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/react-joyride": {
|
||||
"version": "2.9.3",
|
||||
"resolved": "https://registry.npmjs.org/react-joyride/-/react-joyride-2.9.3.tgz",
|
||||
"integrity": "sha512-1+Mg34XK5zaqJ63eeBhqdbk7dlGCFp36FXwsEvgpjqrtyywX2C6h9vr3jgxP0bGHCw8Ilsp/nRDzNVq6HJ3rNw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@gilbarbara/deep-equal": "^0.3.1",
|
||||
"deep-diff": "^1.0.2",
|
||||
"deepmerge": "^4.3.1",
|
||||
"is-lite": "^1.2.1",
|
||||
"react-floater": "^0.7.9",
|
||||
"react-innertext": "^1.1.5",
|
||||
"react-is": "^16.13.1",
|
||||
"scroll": "^3.0.1",
|
||||
"scrollparent": "^2.1.0",
|
||||
"tree-changes": "^0.11.2",
|
||||
"type-fest": "^4.27.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "15 - 18",
|
||||
"react-dom": "15 - 18"
|
||||
}
|
||||
},
|
||||
"node_modules/react-joyride/node_modules/react-is": {
|
||||
"version": "16.13.1",
|
||||
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
|
||||
"integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/react-joyride/node_modules/type-fest": {
|
||||
"version": "4.41.0",
|
||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz",
|
||||
"integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==",
|
||||
"license": "(MIT OR CC0-1.0)",
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/react-redux": {
|
||||
"version": "9.2.0",
|
||||
"resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz",
|
||||
@@ -16603,18 +16481,6 @@
|
||||
"integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/scroll": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/scroll/-/scroll-3.0.1.tgz",
|
||||
"integrity": "sha512-pz7y517OVls1maEzlirKO5nPYle9AXsFzTMNJrRGmT951mzpIBy7sNHOg5o/0MQd/NqliCiWnAi0kZneMPFLcg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/scrollparent": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/scrollparent/-/scrollparent-2.1.0.tgz",
|
||||
"integrity": "sha512-bnnvJL28/Rtz/kz2+4wpBjHzWoEzXhVg/TE8BeVGJHUqE8THNIRnDxDWMktwM+qahvlRdvlLdsQfYe+cuqfZeA==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/secure-json-parse": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz",
|
||||
@@ -18065,16 +17931,6 @@
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-changes": {
|
||||
"version": "0.11.3",
|
||||
"resolved": "https://registry.npmjs.org/tree-changes/-/tree-changes-0.11.3.tgz",
|
||||
"integrity": "sha512-r14mvDZ6tqz8PRQmlFKjhUVngu4VZ9d92ON3tp0EGpFBE6PAHOq8Bx8m8ahbNoGE3uI/npjYcJiqVydyOiYXag==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@gilbarbara/deep-equal": "^0.3.1",
|
||||
"is-lite": "^1.2.1"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-kill": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
|
||||
|
||||
20
package.json
20
package.json
@@ -1,20 +1,25 @@
|
||||
{
|
||||
"name": "flyer-crawler",
|
||||
"private": true,
|
||||
"version": "0.12.1",
|
||||
"version": "0.12.12",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "concurrently \"npm:start:dev\" \"vite\"",
|
||||
"dev:container": "concurrently \"npm:start:dev\" \"vite --host\"",
|
||||
"dev:pm2": "pm2 start ecosystem.dev.config.cjs && pm2 logs",
|
||||
"dev:pm2:restart": "pm2 restart ecosystem.dev.config.cjs",
|
||||
"dev:pm2:stop": "pm2 delete all",
|
||||
"dev:pm2:status": "pm2 status",
|
||||
"dev:pm2:logs": "pm2 logs",
|
||||
"start": "npm run start:prod",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"test": "node scripts/check-linux.js && cross-env NODE_ENV=test tsx ./node_modules/vitest/vitest.mjs run",
|
||||
"test-wsl": "cross-env NODE_ENV=test vitest run",
|
||||
"test": "node scripts/check-linux.js && cross-env NODE_ENV=test TZ= tsx ./node_modules/vitest/vitest.mjs run",
|
||||
"test-wsl": "cross-env NODE_ENV=test TZ= vitest run",
|
||||
"test:coverage": "npm run clean && npm run test:unit -- --coverage && npm run test:integration -- --coverage",
|
||||
"test:unit": "node scripts/check-linux.js && cross-env NODE_ENV=test tsx --max-old-space-size=8192 ./node_modules/vitest/vitest.mjs run --project unit -c vite.config.ts",
|
||||
"test:integration": "node scripts/check-linux.js && cross-env NODE_ENV=test tsx --max-old-space-size=8192 ./node_modules/vitest/vitest.mjs run --project integration -c vitest.config.integration.ts",
|
||||
"test:e2e": "node scripts/check-linux.js && cross-env NODE_ENV=test tsx --max-old-space-size=8192 ./node_modules/vitest/vitest.mjs run --config vitest.config.e2e.ts",
|
||||
"test:unit": "node scripts/check-linux.js && cross-env NODE_ENV=test TZ= tsx --max-old-space-size=8192 ./node_modules/vitest/vitest.mjs run --project unit -c vite.config.ts",
|
||||
"test:integration": "node scripts/check-linux.js && cross-env NODE_ENV=test TZ= tsx --max-old-space-size=8192 ./node_modules/vitest/vitest.mjs run --project integration -c vitest.config.integration.ts",
|
||||
"test:e2e": "node scripts/check-linux.js && cross-env NODE_ENV=test TZ= tsx --max-old-space-size=8192 ./node_modules/vitest/vitest.mjs run --config vitest.config.e2e.ts",
|
||||
"format": "prettier --write .",
|
||||
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
|
||||
"type-check": "tsc --noEmit",
|
||||
@@ -36,7 +41,6 @@
|
||||
"@sentry/react": "^10.32.1",
|
||||
"@tanstack/react-query": "^5.90.12",
|
||||
"@types/connect-timeout": "^1.9.0",
|
||||
"@types/react-joyride": "^2.0.2",
|
||||
"bcrypt": "^5.1.1",
|
||||
"bullmq": "^5.65.1",
|
||||
"connect-timeout": "^1.9.1",
|
||||
@@ -66,7 +70,7 @@
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0",
|
||||
"react-hot-toast": "^2.6.0",
|
||||
"react-joyride": "^2.9.3",
|
||||
"driver.js": "^1.3.1",
|
||||
"react-router-dom": "^7.9.6",
|
||||
"recharts": "^3.4.1",
|
||||
"sharp": "^0.34.5",
|
||||
|
||||
26
scripts/create-test-icon.js
Normal file
26
scripts/create-test-icon.js
Normal file
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Creates a 64x64 icon from test-flyer-image.png
|
||||
* Run from container: node scripts/create-test-icon.js
|
||||
*/
|
||||
|
||||
import sharp from 'sharp';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const inputPath = path.join(__dirname, '../src/tests/assets/test-flyer-image.png');
|
||||
const outputPath = path.join(__dirname, '../src/tests/assets/test-flyer-icon.png');
|
||||
|
||||
sharp(inputPath)
|
||||
.resize(64, 64, { fit: 'cover' })
|
||||
.toFile(outputPath)
|
||||
.then(() => {
|
||||
console.log(`✓ Created icon: ${outputPath}`);
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error('Error creating icon:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -3,47 +3,168 @@
|
||||
# ============================================================================
|
||||
# Development Container Entrypoint
|
||||
# ============================================================================
|
||||
# This script starts the development server automatically when the container
|
||||
# starts, both with VS Code Dev Containers and with plain podman-compose.
|
||||
# This script starts all development services when the container starts,
|
||||
# both with VS Code Dev Containers and with plain podman-compose.
|
||||
#
|
||||
# Services started:
|
||||
# - Nginx (proxies Vite 5173 → 3000)
|
||||
# - Nginx (HTTPS proxy: Vite 5173 -> 443, Bugsink 8000 -> 8443, API 3001)
|
||||
# - Bugsink (error tracking) on port 8000
|
||||
# - Logstash (log aggregation)
|
||||
# - Node.js dev server (API + Frontend) on ports 3001 and 5173
|
||||
# - PM2 (process manager) running:
|
||||
# - flyer-crawler-api-dev: API server (port 3001)
|
||||
# - flyer-crawler-worker-dev: Background job worker
|
||||
# - flyer-crawler-vite-dev: Vite frontend dev server (port 5173)
|
||||
#
|
||||
# ADR-014: This architecture matches production (PM2 managing processes)
|
||||
# ADR-050: Logs are written to /var/log/pm2 for Logstash integration
|
||||
# ============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
echo "🚀 Starting Flyer Crawler Dev Container..."
|
||||
echo "Starting Flyer Crawler Dev Container..."
|
||||
|
||||
# ============================================================================
|
||||
# Timezone Configuration
|
||||
# ============================================================================
|
||||
# Ensure TZ is set for consistent log timestamps across all services.
|
||||
# TZ should be set via compose.dev.yml environment (default: America/Los_Angeles)
|
||||
# ============================================================================
|
||||
if [ -n "$TZ" ]; then
|
||||
echo "Timezone configured: $TZ"
|
||||
# Link timezone data if available (for date command and other tools)
|
||||
if [ -f "/usr/share/zoneinfo/$TZ" ]; then
|
||||
ln -sf "/usr/share/zoneinfo/$TZ" /etc/localtime
|
||||
echo "$TZ" > /etc/timezone
|
||||
echo "System timezone set to: $(date +%Z) ($(date))"
|
||||
else
|
||||
echo "Warning: Timezone data not found for $TZ, using TZ environment variable only"
|
||||
fi
|
||||
else
|
||||
echo "Warning: TZ environment variable not set, using container default timezone"
|
||||
fi
|
||||
|
||||
# Configure Bugsink HTTPS (ADR-015)
|
||||
echo "Configuring Bugsink HTTPS..."
|
||||
mkdir -p /etc/bugsink/ssl
|
||||
if [ ! -f "/etc/bugsink/ssl/localhost+2.pem" ]; then
|
||||
cd /etc/bugsink/ssl && mkcert localhost 127.0.0.1 ::1 > /dev/null 2>&1
|
||||
fi
|
||||
|
||||
# Create nginx config for Bugsink HTTPS
|
||||
cat > /etc/nginx/sites-available/bugsink <<'NGINX_EOF'
|
||||
server {
|
||||
listen 8443 ssl http2;
|
||||
listen [::]:8443 ssl http2;
|
||||
server_name localhost;
|
||||
|
||||
ssl_certificate /etc/bugsink/ssl/localhost+2.pem;
|
||||
ssl_certificate_key /etc/bugsink/ssl/localhost+2-key.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_redirect off;
|
||||
proxy_buffering off;
|
||||
client_max_body_size 20M;
|
||||
}
|
||||
}
|
||||
NGINX_EOF
|
||||
|
||||
ln -sf /etc/nginx/sites-available/bugsink /etc/nginx/sites-enabled/bugsink
|
||||
|
||||
# Copy the dev nginx config from mounted volume to nginx sites-available
|
||||
echo "Copying nginx dev config..."
|
||||
cp /app/docker/nginx/dev.conf /etc/nginx/sites-available/default
|
||||
|
||||
# Ensure PM2 log directory exists with correct permissions
|
||||
echo "Setting up PM2 log directory..."
|
||||
mkdir -p /var/log/pm2
|
||||
chmod 755 /var/log/pm2
|
||||
|
||||
# Start nginx in background (if installed)
|
||||
if command -v nginx &> /dev/null; then
|
||||
echo "🌐 Starting nginx (HTTPS proxy: Vite 5173 → port 443)..."
|
||||
echo "Starting nginx (HTTPS: Vite 5173 -> 443, Bugsink 8000 -> 8443, API 3001 -> /api/)..."
|
||||
nginx &
|
||||
fi
|
||||
|
||||
# Start Bugsink in background
|
||||
echo "📊 Starting Bugsink error tracking..."
|
||||
echo "Starting Bugsink error tracking..."
|
||||
/usr/local/bin/start-bugsink.sh > /var/log/bugsink/server.log 2>&1 &
|
||||
|
||||
# Wait for Bugsink to initialize, then run snappea migrations
|
||||
echo "Waiting for Bugsink to initialize..."
|
||||
sleep 5
|
||||
echo "Running Bugsink snappea database migrations..."
|
||||
cd /opt/bugsink/conf && \
|
||||
export DATABASE_URL="postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink" && \
|
||||
export SECRET_KEY="dev-bugsink-secret-key-minimum-50-characters-for-security" && \
|
||||
/opt/bugsink/bin/bugsink-manage migrate --database=snappea > /dev/null 2>&1
|
||||
|
||||
# Start Snappea task worker
|
||||
echo "Starting Snappea task worker..."
|
||||
cd /opt/bugsink/conf && \
|
||||
export DATABASE_URL="postgresql://bugsink:bugsink_dev_password@postgres:5432/bugsink" && \
|
||||
export SECRET_KEY="dev-bugsink-secret-key-minimum-50-characters-for-security" && \
|
||||
/opt/bugsink/bin/bugsink-manage runsnappea > /var/log/bugsink/snappea.log 2>&1 &
|
||||
|
||||
# Start Logstash in background
|
||||
echo "📝 Starting Logstash..."
|
||||
echo "Starting Logstash..."
|
||||
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/bugsink.conf > /var/log/logstash/logstash.log 2>&1 &
|
||||
|
||||
# Wait a few seconds for services to initialize
|
||||
# Wait for services to initialize
|
||||
sleep 3
|
||||
|
||||
# Change to app directory
|
||||
cd /app
|
||||
|
||||
# Start development server
|
||||
echo "💻 Starting development server..."
|
||||
echo " - Frontend: https://localhost (nginx HTTPS → Vite on 5173)"
|
||||
echo " - Backend API: http://localhost:3001"
|
||||
echo " - Bugsink: http://localhost:8000"
|
||||
echo " - Note: Accept the self-signed certificate warning in your browser"
|
||||
# ============================================================================
|
||||
# Start PM2 (ADR-014: Production-like process management)
|
||||
# ============================================================================
|
||||
# PM2 manages all Node.js processes:
|
||||
# - API server (tsx watch server.ts)
|
||||
# - Background worker (tsx watch src/services/worker.ts)
|
||||
# - Vite dev server (vite --host)
|
||||
#
|
||||
# Logs are written to /var/log/pm2 for Logstash integration (ADR-050)
|
||||
# ============================================================================
|
||||
|
||||
echo "Starting PM2 with development configuration..."
|
||||
echo " - API Server: http://localhost:3001 (proxied via nginx)"
|
||||
echo " - Frontend: https://localhost (nginx HTTPS -> Vite on 5173)"
|
||||
echo " - Bugsink: https://localhost:8443 (nginx HTTPS -> Bugsink on 8000)"
|
||||
echo " - PM2 Logs: /var/log/pm2/*.log"
|
||||
echo ""
|
||||
echo "Note: Accept the self-signed certificate warnings in your browser"
|
||||
echo ""
|
||||
|
||||
# Run npm dev server (this will block and keep container alive)
|
||||
exec npm run dev:container
|
||||
# Delete any existing PM2 processes (clean start)
|
||||
pm2 delete all 2>/dev/null || true
|
||||
|
||||
# Start PM2 with the dev ecosystem config
|
||||
pm2 start /app/ecosystem.dev.config.cjs
|
||||
|
||||
# Show PM2 status
|
||||
echo ""
|
||||
echo "--- PM2 Process Status ---"
|
||||
pm2 status
|
||||
echo "-------------------------"
|
||||
echo ""
|
||||
echo "PM2 Commands:"
|
||||
echo " pm2 logs # View all logs (tail -f style)"
|
||||
echo " pm2 logs api # View API logs only"
|
||||
echo " pm2 restart all # Restart all processes"
|
||||
echo " pm2 status # Show process status"
|
||||
echo ""
|
||||
|
||||
# Keep the container running by tailing PM2 logs
|
||||
# This ensures:
|
||||
# 1. Container stays alive
|
||||
# 2. Logs are visible in docker logs / podman logs
|
||||
# 3. PM2 processes continue running
|
||||
exec pm2 logs --raw
|
||||
|
||||
53
sql/update_flyer_urls.sql
Normal file
53
sql/update_flyer_urls.sql
Normal file
@@ -0,0 +1,53 @@
|
||||
-- Update flyer URLs from example.com to environment-specific URLs
|
||||
--
|
||||
-- This script should be run after determining the correct base URL for the environment:
|
||||
-- - Dev container: https://localhost (NOT 127.0.0.1 - avoids SSL mixed-origin issues)
|
||||
-- - Test environment: https://flyer-crawler-test.projectium.com
|
||||
-- - Production: https://flyer-crawler.projectium.com
|
||||
|
||||
-- For dev container (run in dev database):
|
||||
-- Uses 'localhost' instead of '127.0.0.1' to match how users access the site.
|
||||
-- This avoids ERR_CERT_AUTHORITY_INVALID errors when images are loaded from a
|
||||
-- different origin than the page.
|
||||
UPDATE flyers
|
||||
SET
|
||||
image_url = REPLACE(image_url, 'example.com', 'localhost'),
|
||||
icon_url = REPLACE(icon_url, 'example.com', 'localhost')
|
||||
WHERE
|
||||
image_url LIKE '%example.com%'
|
||||
OR icon_url LIKE '%example.com%';
|
||||
|
||||
-- Also fix any existing 127.0.0.1 URLs to use localhost:
|
||||
UPDATE flyers
|
||||
SET
|
||||
image_url = REPLACE(image_url, '127.0.0.1', 'localhost'),
|
||||
icon_url = REPLACE(icon_url, '127.0.0.1', 'localhost')
|
||||
WHERE
|
||||
image_url LIKE '%127.0.0.1%'
|
||||
OR icon_url LIKE '%127.0.0.1%';
|
||||
|
||||
-- For test environment (run in test database):
|
||||
-- UPDATE flyers
|
||||
-- SET
|
||||
-- image_url = REPLACE(image_url, 'example.com', 'flyer-crawler-test.projectium.com'),
|
||||
-- icon_url = REPLACE(icon_url, 'example.com', 'flyer-crawler-test.projectium.com')
|
||||
-- WHERE
|
||||
-- image_url LIKE '%example.com%'
|
||||
-- OR icon_url LIKE '%example.com%';
|
||||
|
||||
-- For production (run in production database):
|
||||
-- UPDATE flyers
|
||||
-- SET
|
||||
-- image_url = REPLACE(image_url, 'example.com', 'flyer-crawler.projectium.com'),
|
||||
-- icon_url = REPLACE(icon_url, 'example.com', 'flyer-crawler.projectium.com')
|
||||
-- WHERE
|
||||
-- image_url LIKE '%example.com%'
|
||||
-- OR icon_url LIKE '%example.com%';
|
||||
|
||||
-- Verify the changes:
|
||||
SELECT flyer_id, image_url, icon_url
|
||||
FROM flyers
|
||||
WHERE image_url LIKE '%localhost%'
|
||||
OR icon_url LIKE '%localhost%'
|
||||
OR image_url LIKE '%flyer-crawler%'
|
||||
OR icon_url LIKE '%flyer-crawler%';
|
||||
@@ -15,7 +15,7 @@ export const Dashboard: React.FC = () => {
|
||||
<RecipeSuggester />
|
||||
|
||||
{/* Other Dashboard Widgets */}
|
||||
<div className="bg-white dark:bg-gray-800 shadow rounded-lg p-6">
|
||||
<div className="bg-white dark:bg-gray-800 shadow rounded-lg p-6 transition-colors hover:bg-gray-50 dark:hover:bg-gray-800/80">
|
||||
<h2 className="text-lg font-medium text-gray-900 dark:text-white mb-4">Your Flyers</h2>
|
||||
<FlyerCountDisplay />
|
||||
</div>
|
||||
|
||||
@@ -31,7 +31,7 @@ export const Header: React.FC<HeaderProps> = ({
|
||||
// The state and handlers for the old AuthModal and SignUpModal have been removed.
|
||||
return (
|
||||
<>
|
||||
<header className="bg-white dark:bg-gray-900 shadow-md sticky top-0 z-20">
|
||||
<header className="bg-white dark:bg-gray-900 shadow-md sticky top-0 z-20 border-b-2 border-brand-primary dark:border-brand-secondary">
|
||||
<div className="max-w-screen-2xl mx-auto px-4 sm:px-6 lg:px-8">
|
||||
<div className="flex items-center justify-between h-16">
|
||||
<div className="flex items-center">
|
||||
|
||||
@@ -43,7 +43,7 @@ export const Leaderboard: React.FC = () => {
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="bg-white dark:bg-gray-800 shadow-lg rounded-lg p-6">
|
||||
<div className="bg-white dark:bg-gray-800 shadow-lg rounded-lg p-6 transition-colors hover:bg-gray-50 dark:hover:bg-gray-800/80">
|
||||
<h2 className="text-2xl font-bold text-gray-900 dark:text-white mb-4 flex items-center">
|
||||
<Award className="w-6 h-6 mr-2 text-blue-500" />
|
||||
Top Users
|
||||
@@ -57,7 +57,7 @@ export const Leaderboard: React.FC = () => {
|
||||
{leaderboard.map((user) => (
|
||||
<li
|
||||
key={user.user_id}
|
||||
className="flex items-center space-x-4 p-3 bg-gray-50 dark:bg-gray-700 rounded-lg transition hover:bg-gray-100 dark:hover:bg-gray-600"
|
||||
className="flex items-center space-x-4 p-3 bg-gray-50 dark:bg-gray-700 rounded-lg transition-colors hover:bg-brand-light/30 dark:hover:bg-brand-dark/20"
|
||||
>
|
||||
<div className="shrink-0 w-8 text-center">{getRankIcon(user.rank)}</div>
|
||||
<img
|
||||
|
||||
@@ -48,7 +48,7 @@ export const RecipeSuggester: React.FC = () => {
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="bg-white dark:bg-gray-800 shadow rounded-lg p-6">
|
||||
<div className="bg-white dark:bg-gray-800 shadow rounded-lg p-6 transition-colors hover:bg-gray-50 dark:hover:bg-gray-800/80">
|
||||
<h2 className="text-xl font-semibold text-gray-900 dark:text-white mb-2">
|
||||
Get a Recipe Suggestion
|
||||
</h2>
|
||||
|
||||
@@ -12,6 +12,17 @@ import path from 'node:path';
|
||||
import bcrypt from 'bcrypt';
|
||||
import { logger } from '../services/logger.server';
|
||||
|
||||
// Determine base URL for flyer images based on environment
|
||||
// Dev container: https://127.0.0.1 (NGINX now accepts both localhost and 127.0.0.1)
|
||||
// Test: https://flyer-crawler-test.projectium.com
|
||||
// Production: https://flyer-crawler.projectium.com
|
||||
const BASE_URL =
|
||||
process.env.FLYER_BASE_URL || process.env.NODE_ENV === 'production'
|
||||
? 'https://flyer-crawler.projectium.com'
|
||||
: process.env.NODE_ENV === 'test'
|
||||
? 'https://flyer-crawler-test.projectium.com'
|
||||
: 'https://127.0.0.1';
|
||||
|
||||
const pool = new Pool({
|
||||
user: process.env.DB_USER,
|
||||
host: process.env.DB_HOST,
|
||||
@@ -101,7 +112,21 @@ async function main() {
|
||||
const userId = userRes.rows[0].user_id;
|
||||
logger.info('Seeded regular user (user@example.com / userpass)');
|
||||
|
||||
// 4. Seed a Flyer
|
||||
// 4. Copy test images to flyer-images directory
|
||||
logger.info('--- Copying test flyer images... ---');
|
||||
const flyerImagesDir = path.resolve(process.cwd(), 'public/flyer-images');
|
||||
await fs.mkdir(flyerImagesDir, { recursive: true });
|
||||
|
||||
const testImageSource = path.resolve(process.cwd(), 'src/tests/assets/test-flyer-image.jpg');
|
||||
const testIconSource = path.resolve(process.cwd(), 'src/tests/assets/test-flyer-icon.png');
|
||||
const testImageDest = path.join(flyerImagesDir, 'test-flyer-image.jpg');
|
||||
const testIconDest = path.join(flyerImagesDir, 'test-flyer-icon.png');
|
||||
|
||||
await fs.copyFile(testImageSource, testImageDest);
|
||||
await fs.copyFile(testIconSource, testIconDest);
|
||||
logger.info(`Copied test images to ${flyerImagesDir}`);
|
||||
|
||||
// 5. Seed a Flyer
|
||||
logger.info('--- Seeding a Sample Flyer... ---');
|
||||
const today = new Date();
|
||||
const validFrom = new Date(today);
|
||||
@@ -111,7 +136,7 @@ async function main() {
|
||||
|
||||
const flyerQuery = `
|
||||
INSERT INTO public.flyers (file_name, image_url, icon_url, checksum, store_id, valid_from, valid_to)
|
||||
VALUES ('safeway-flyer.jpg', 'https://example.com/flyer-images/safeway-flyer.jpg', 'https://example.com/flyer-images/icons/safeway-flyer.jpg', 'a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0', ${storeMap.get('Safeway')}, $1, $2)
|
||||
VALUES ('test-flyer-image.jpg', '${BASE_URL}/flyer-images/test-flyer-image.jpg', '${BASE_URL}/flyer-images/test-flyer-icon.png', 'a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0', ${storeMap.get('Safeway')}, $1, $2)
|
||||
RETURNING flyer_id;
|
||||
`;
|
||||
const flyerRes = await client.query<{ flyer_id: number }>(flyerQuery, [
|
||||
@@ -121,7 +146,7 @@ async function main() {
|
||||
const flyerId = flyerRes.rows[0].flyer_id;
|
||||
logger.info(`Seeded flyer for Safeway (ID: ${flyerId}).`);
|
||||
|
||||
// 5. Seed Flyer Items
|
||||
// 6. Seed Flyer Items
|
||||
logger.info('--- Seeding Flyer Items... ---');
|
||||
const flyerItems = [
|
||||
{
|
||||
@@ -169,7 +194,7 @@ async function main() {
|
||||
}
|
||||
logger.info(`Seeded ${flyerItems.length} items for the Safeway flyer.`);
|
||||
|
||||
// 6. Seed Watched Items for the user
|
||||
// 7. Seed Watched Items for the user
|
||||
logger.info('--- Seeding Watched Items... ---');
|
||||
const watchedItemIds = [
|
||||
masterItemMap.get('chicken breast'),
|
||||
@@ -186,7 +211,7 @@ async function main() {
|
||||
}
|
||||
logger.info(`Seeded ${watchedItemIds.length} watched items for Test User.`);
|
||||
|
||||
// 7. Seed a Shopping List
|
||||
// 8. Seed a Shopping List
|
||||
logger.info('--- Seeding a Shopping List... ---');
|
||||
const listRes = await client.query<{ shopping_list_id: number }>(
|
||||
'INSERT INTO public.shopping_lists (user_id, name) VALUES ($1, $2) RETURNING shopping_list_id',
|
||||
|
||||
@@ -3,6 +3,7 @@ import { useMutation } from '@tanstack/react-query';
|
||||
import * as apiClient from '../../services/apiClient';
|
||||
import { notifyError } from '../../services/notificationService';
|
||||
import type { UserProfile } from '../../types';
|
||||
import { logger } from '../../services/logger.client';
|
||||
|
||||
interface AuthResponse {
|
||||
userprofile: UserProfile;
|
||||
@@ -29,7 +30,9 @@ export const useLoginMutation = () => {
|
||||
password: string;
|
||||
rememberMe: boolean;
|
||||
}): Promise<AuthResponse> => {
|
||||
logger.info('[useLoginMutation] MUTATION STARTED', { email, rememberMe: rememberMe });
|
||||
const response = await apiClient.loginUser(email, password, rememberMe);
|
||||
logger.info('[useLoginMutation] Got response', { status: response.status });
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({
|
||||
@@ -38,7 +41,18 @@ export const useLoginMutation = () => {
|
||||
throw new Error(error.message || 'Failed to login');
|
||||
}
|
||||
|
||||
return response.json();
|
||||
const result = await response.json();
|
||||
// DEBUG: Log the actual response structure
|
||||
logger.debug('[useLoginMutation] Raw API response', { result });
|
||||
logger.debug('[useLoginMutation] result.data', { data: result.data });
|
||||
logger.debug('[useLoginMutation] result.data.token', { token: result.data?.token });
|
||||
// The API returns {success, data: {userprofile, token}}, so extract the data
|
||||
const extracted = result.data;
|
||||
logger.info('[useLoginMutation] Returning extracted data', {
|
||||
hasUserprofile: !!extracted?.userprofile,
|
||||
hasToken: !!extracted?.token,
|
||||
});
|
||||
return extracted;
|
||||
},
|
||||
onError: (error: Error) => {
|
||||
notifyError(error.message || 'Failed to login');
|
||||
@@ -75,7 +89,9 @@ export const useRegisterMutation = () => {
|
||||
throw new Error(error.message || 'Failed to register');
|
||||
}
|
||||
|
||||
return response.json();
|
||||
const result = await response.json();
|
||||
// The API returns {success, data: {userprofile, token}}, so extract the data
|
||||
return result.data;
|
||||
},
|
||||
onError: (error: Error) => {
|
||||
notifyError(error.message || 'Failed to register');
|
||||
|
||||
@@ -3,6 +3,9 @@ import { renderHook, act } from '@testing-library/react';
|
||||
import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest';
|
||||
import { useDataExtraction } from './useDataExtraction';
|
||||
import type { Flyer } from '../types';
|
||||
import { getFlyerBaseUrl } from '../tests/utils/testHelpers';
|
||||
|
||||
const FLYER_BASE_URL = getFlyerBaseUrl();
|
||||
|
||||
// Create a mock flyer for testing
|
||||
const createMockFlyer = (id: number, storeName: string = `Store ${id}`): Flyer => ({
|
||||
@@ -14,8 +17,8 @@ const createMockFlyer = (id: number, storeName: string = `Store ${id}`): Flyer =
|
||||
updated_at: '2024-01-01T00:00:00Z',
|
||||
},
|
||||
file_name: `flyer${id}.jpg`,
|
||||
image_url: `https://example.com/flyer${id}.jpg`,
|
||||
icon_url: `https://example.com/flyer${id}_icon.jpg`,
|
||||
image_url: `${FLYER_BASE_URL}/flyer${id}.jpg`,
|
||||
icon_url: `${FLYER_BASE_URL}/flyer${id}_icon.jpg`,
|
||||
status: 'processed',
|
||||
item_count: 0,
|
||||
created_at: '2024-01-01T00:00:00Z',
|
||||
@@ -76,7 +79,7 @@ describe('useDataExtraction Hook', () => {
|
||||
expect(updatedFlyer.store?.name).toBe('New Store Name');
|
||||
// Ensure other properties are preserved
|
||||
expect(updatedFlyer.flyer_id).toBe(1);
|
||||
expect(updatedFlyer.image_url).toBe('https://example.com/flyer1.jpg');
|
||||
expect(updatedFlyer.image_url).toBe(`${FLYER_BASE_URL}/flyer1.jpg`);
|
||||
});
|
||||
|
||||
it('should preserve store_id when updating store name', () => {
|
||||
|
||||
@@ -1,87 +1,286 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import type { Step, CallBackProps } from 'react-joyride';
|
||||
import { useEffect, useCallback, useRef } from 'react';
|
||||
import { driver, Driver, DriveStep } from 'driver.js';
|
||||
import 'driver.js/dist/driver.css';
|
||||
|
||||
const ONBOARDING_STORAGE_KEY = 'flyer_crawler_onboarding_completed';
|
||||
|
||||
export const useOnboardingTour = () => {
|
||||
const [runTour, setRunTour] = useState(false);
|
||||
const [stepIndex, setStepIndex] = useState(0);
|
||||
// Custom CSS to match design system: pastel colors, sharp borders, minimalist
|
||||
const DRIVER_CSS = `
|
||||
.driver-popover {
|
||||
background-color: #f0fdfa !important;
|
||||
border: 2px solid #0d9488 !important;
|
||||
border-radius: 0 !important;
|
||||
box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06) !important;
|
||||
max-width: 320px !important;
|
||||
}
|
||||
|
||||
.driver-popover-title {
|
||||
color: #134e4a !important;
|
||||
font-size: 1rem !important;
|
||||
font-weight: 600 !important;
|
||||
margin-bottom: 0.5rem !important;
|
||||
}
|
||||
|
||||
.driver-popover-description {
|
||||
color: #1f2937 !important;
|
||||
font-size: 0.875rem !important;
|
||||
line-height: 1.5 !important;
|
||||
}
|
||||
|
||||
.driver-popover-progress-text {
|
||||
color: #0d9488 !important;
|
||||
font-size: 0.75rem !important;
|
||||
font-weight: 500 !important;
|
||||
}
|
||||
|
||||
.driver-popover-navigation-btns {
|
||||
gap: 0.5rem !important;
|
||||
}
|
||||
|
||||
.driver-popover-prev-btn,
|
||||
.driver-popover-next-btn {
|
||||
background-color: #14b8a6 !important;
|
||||
color: white !important;
|
||||
border: 1px solid #0d9488 !important;
|
||||
border-radius: 0 !important;
|
||||
padding: 0.5rem 1rem !important;
|
||||
font-size: 0.875rem !important;
|
||||
font-weight: 500 !important;
|
||||
transition: background-color 0.15s ease !important;
|
||||
}
|
||||
|
||||
.driver-popover-prev-btn:hover,
|
||||
.driver-popover-next-btn:hover {
|
||||
background-color: #115e59 !important;
|
||||
}
|
||||
|
||||
.driver-popover-prev-btn {
|
||||
background-color: #ccfbf1 !important;
|
||||
color: #134e4a !important;
|
||||
}
|
||||
|
||||
.driver-popover-prev-btn:hover {
|
||||
background-color: #99f6e4 !important;
|
||||
}
|
||||
|
||||
.driver-popover-close-btn {
|
||||
color: #0d9488 !important;
|
||||
font-size: 1.25rem !important;
|
||||
}
|
||||
|
||||
.driver-popover-close-btn:hover {
|
||||
color: #115e59 !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-left,
|
||||
.driver-popover-arrow-side-right,
|
||||
.driver-popover-arrow-side-top,
|
||||
.driver-popover-arrow-side-bottom {
|
||||
border-color: #0d9488 !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-left::after,
|
||||
.driver-popover-arrow-side-right::after,
|
||||
.driver-popover-arrow-side-top::after,
|
||||
.driver-popover-arrow-side-bottom::after {
|
||||
border-color: transparent !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-left::before {
|
||||
border-right-color: #f0fdfa !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-right::before {
|
||||
border-left-color: #f0fdfa !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-top::before {
|
||||
border-bottom-color: #f0fdfa !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-bottom::before {
|
||||
border-top-color: #f0fdfa !important;
|
||||
}
|
||||
|
||||
.driver-overlay {
|
||||
background-color: rgba(0, 0, 0, 0.5) !important;
|
||||
}
|
||||
|
||||
.driver-active-element {
|
||||
box-shadow: 0 0 0 4px #14b8a6 !important;
|
||||
}
|
||||
|
||||
/* Dark mode support */
|
||||
@media (prefers-color-scheme: dark) {
|
||||
.driver-popover {
|
||||
background-color: #1f2937 !important;
|
||||
border-color: #14b8a6 !important;
|
||||
}
|
||||
|
||||
.driver-popover-title {
|
||||
color: #ccfbf1 !important;
|
||||
}
|
||||
|
||||
.driver-popover-description {
|
||||
color: #e5e7eb !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-left::before {
|
||||
border-right-color: #1f2937 !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-right::before {
|
||||
border-left-color: #1f2937 !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-top::before {
|
||||
border-bottom-color: #1f2937 !important;
|
||||
}
|
||||
|
||||
.driver-popover-arrow-side-bottom::before {
|
||||
border-top-color: #1f2937 !important;
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const tourSteps: DriveStep[] = [
|
||||
{
|
||||
element: '[data-tour="flyer-uploader"]',
|
||||
popover: {
|
||||
title: 'Upload Flyers',
|
||||
description:
|
||||
'Upload a grocery flyer here by clicking or dragging a PDF/image file. Our AI will extract prices and items automatically.',
|
||||
side: 'bottom',
|
||||
align: 'start',
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="extracted-data-table"]',
|
||||
popover: {
|
||||
title: 'Extracted Items',
|
||||
description:
|
||||
'View all extracted items from your flyers here. You can watch items to track price changes and deals.',
|
||||
side: 'top',
|
||||
align: 'start',
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="watch-button"]',
|
||||
popover: {
|
||||
title: 'Watch Items',
|
||||
description:
|
||||
'Click the eye icon to watch items and get notified when prices drop or deals appear.',
|
||||
side: 'left',
|
||||
align: 'start',
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="watched-items"]',
|
||||
popover: {
|
||||
title: 'Watched Items',
|
||||
description:
|
||||
'Your watched items appear here. Track prices across different stores and get deal alerts.',
|
||||
side: 'left',
|
||||
align: 'start',
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="price-chart"]',
|
||||
popover: {
|
||||
title: 'Active Deals',
|
||||
description:
|
||||
'Active deals show here with price comparisons. See which store has the best price!',
|
||||
side: 'left',
|
||||
align: 'start',
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="shopping-list"]',
|
||||
popover: {
|
||||
title: 'Shopping Lists',
|
||||
description:
|
||||
'Create shopping lists from your watched items and get the best prices automatically.',
|
||||
side: 'left',
|
||||
align: 'start',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// Inject custom styles into the document head
|
||||
const injectStyles = () => {
|
||||
const styleId = 'driver-js-custom-styles';
|
||||
if (!document.getElementById(styleId)) {
|
||||
const style = document.createElement('style');
|
||||
style.id = styleId;
|
||||
style.textContent = DRIVER_CSS;
|
||||
document.head.appendChild(style);
|
||||
}
|
||||
};
|
||||
|
||||
export const useOnboardingTour = () => {
|
||||
const driverRef = useRef<Driver | null>(null);
|
||||
|
||||
const markTourComplete = useCallback(() => {
|
||||
localStorage.setItem(ONBOARDING_STORAGE_KEY, 'true');
|
||||
}, []);
|
||||
|
||||
const startTour = useCallback(() => {
|
||||
injectStyles();
|
||||
|
||||
if (driverRef.current) {
|
||||
driverRef.current.destroy();
|
||||
}
|
||||
|
||||
driverRef.current = driver({
|
||||
showProgress: true,
|
||||
steps: tourSteps,
|
||||
nextBtnText: 'Next',
|
||||
prevBtnText: 'Previous',
|
||||
doneBtnText: 'Done',
|
||||
progressText: 'Step {{current}} of {{total}}',
|
||||
onDestroyed: () => {
|
||||
markTourComplete();
|
||||
},
|
||||
});
|
||||
|
||||
driverRef.current.drive();
|
||||
}, [markTourComplete]);
|
||||
|
||||
const skipTour = useCallback(() => {
|
||||
if (driverRef.current) {
|
||||
driverRef.current.destroy();
|
||||
}
|
||||
markTourComplete();
|
||||
}, [markTourComplete]);
|
||||
|
||||
const replayTour = useCallback(() => {
|
||||
startTour();
|
||||
}, [startTour]);
|
||||
|
||||
// Auto-start tour on mount if not completed
|
||||
useEffect(() => {
|
||||
const hasCompletedOnboarding = localStorage.getItem(ONBOARDING_STORAGE_KEY);
|
||||
if (!hasCompletedOnboarding) {
|
||||
setRunTour(true);
|
||||
// Small delay to ensure DOM elements are mounted
|
||||
const timer = setTimeout(() => {
|
||||
startTour();
|
||||
}, 500);
|
||||
return () => clearTimeout(timer);
|
||||
}
|
||||
}, []);
|
||||
}, [startTour]);
|
||||
|
||||
const steps: Step[] = [
|
||||
{
|
||||
target: '[data-tour="flyer-uploader"]',
|
||||
content:
|
||||
'Upload a grocery flyer here by clicking or dragging a PDF/image file. Our AI will extract prices and items automatically.',
|
||||
disableBeacon: true,
|
||||
placement: 'bottom',
|
||||
},
|
||||
{
|
||||
target: '[data-tour="extracted-data-table"]',
|
||||
content:
|
||||
'View all extracted items from your flyers here. You can watch items to track price changes and deals.',
|
||||
placement: 'top',
|
||||
},
|
||||
{
|
||||
target: '[data-tour="watch-button"]',
|
||||
content:
|
||||
'Click the eye icon to watch items and get notified when prices drop or deals appear.',
|
||||
placement: 'left',
|
||||
},
|
||||
{
|
||||
target: '[data-tour="watched-items"]',
|
||||
content:
|
||||
'Your watched items appear here. Track prices across different stores and get deal alerts.',
|
||||
placement: 'left',
|
||||
},
|
||||
{
|
||||
target: '[data-tour="price-chart"]',
|
||||
content: 'Active deals show here with price comparisons. See which store has the best price!',
|
||||
placement: 'left',
|
||||
},
|
||||
{
|
||||
target: '[data-tour="shopping-list"]',
|
||||
content:
|
||||
'Create shopping lists from your watched items and get the best prices automatically.',
|
||||
placement: 'left',
|
||||
},
|
||||
];
|
||||
|
||||
const handleJoyrideCallback = useCallback((data: CallBackProps) => {
|
||||
const { status, index } = data;
|
||||
|
||||
if (status === 'finished' || status === 'skipped') {
|
||||
localStorage.setItem(ONBOARDING_STORAGE_KEY, 'true');
|
||||
setRunTour(false);
|
||||
setStepIndex(0);
|
||||
} else if (data.action === 'next' || data.action === 'prev') {
|
||||
setStepIndex(index + (data.action === 'next' ? 1 : 0));
|
||||
}
|
||||
}, []);
|
||||
|
||||
const skipTour = useCallback(() => {
|
||||
localStorage.setItem(ONBOARDING_STORAGE_KEY, 'true');
|
||||
setRunTour(false);
|
||||
setStepIndex(0);
|
||||
}, []);
|
||||
|
||||
const replayTour = useCallback(() => {
|
||||
setStepIndex(0);
|
||||
setRunTour(true);
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (driverRef.current) {
|
||||
driverRef.current.destroy();
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
return {
|
||||
runTour,
|
||||
steps,
|
||||
stepIndex,
|
||||
handleJoyrideCallback,
|
||||
skipTour,
|
||||
replayTour,
|
||||
startTour,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -195,7 +195,7 @@ describe('MainLayout Component', () => {
|
||||
onOpenProfile: mockOnOpenProfile,
|
||||
};
|
||||
|
||||
it('renders all main sections and the Outlet content', () => {
|
||||
it('renders all main sections and the Outlet content for unauthenticated users', () => {
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
|
||||
expect(screen.getByTestId('flyer-list')).toBeInTheDocument();
|
||||
@@ -203,9 +203,10 @@ describe('MainLayout Component', () => {
|
||||
expect(screen.getByTestId('shopping-list')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('watched-items-list')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('price-chart')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('price-history-chart')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('leaderboard')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('activity-log')).toBeInTheDocument();
|
||||
// Auth-gated components should NOT be present for unauthenticated users
|
||||
expect(screen.queryByTestId('price-history-chart')).not.toBeInTheDocument();
|
||||
expect(screen.queryByTestId('leaderboard')).not.toBeInTheDocument();
|
||||
expect(screen.queryByTestId('activity-log')).not.toBeInTheDocument();
|
||||
expect(screen.getByText('Outlet Content')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
@@ -216,7 +217,7 @@ describe('MainLayout Component', () => {
|
||||
});
|
||||
|
||||
it('does not show the AnonymousUserBanner if there are no flyers', () => {
|
||||
mockedUseFlyers.mockReturnValueOnce({ ...defaultUseFlyersReturn, flyers: [] });
|
||||
mockedUseFlyers.mockReturnValue({ ...defaultUseFlyersReturn, flyers: [] });
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.queryByTestId('anonymous-banner')).not.toBeInTheDocument();
|
||||
});
|
||||
@@ -235,58 +236,16 @@ describe('MainLayout Component', () => {
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.queryByTestId('anonymous-banner')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
it('displays an error message if useData has an error', () => {
|
||||
mockedUseFlyers.mockReturnValueOnce({
|
||||
...defaultUseFlyersReturn,
|
||||
flyersError: new Error('Data Fetch Failed'),
|
||||
});
|
||||
it('renders auth-gated components (PriceHistoryChart, Leaderboard, ActivityLog)', () => {
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Data Fetch Failed');
|
||||
expect(screen.getByTestId('price-history-chart')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('leaderboard')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('activity-log')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('displays an error message if useShoppingLists has an error', () => {
|
||||
mockedUseShoppingLists.mockReturnValueOnce({
|
||||
...defaultUseShoppingListsReturn,
|
||||
error: 'Shopping List Failed',
|
||||
});
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Shopping List Failed');
|
||||
});
|
||||
|
||||
it('displays an error message if useMasterItems has an error', () => {
|
||||
mockedUseMasterItems.mockReturnValueOnce({
|
||||
...defaultUseMasterItemsReturn,
|
||||
error: 'Master Items Failed',
|
||||
});
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Master Items Failed');
|
||||
});
|
||||
|
||||
it('displays an error message if useWatchedItems has an error', () => {
|
||||
mockedUseWatchedItems.mockReturnValueOnce({
|
||||
...defaultUseWatchedItemsReturn,
|
||||
error: 'Watched Items Failed',
|
||||
});
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Watched Items Failed');
|
||||
});
|
||||
|
||||
it('displays an error message if useActiveDeals has an error', () => {
|
||||
mockedUseActiveDeals.mockReturnValueOnce({
|
||||
...defaultUseActiveDealsReturn,
|
||||
error: 'Active Deals Failed',
|
||||
});
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Active Deals Failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Event Handlers', () => {
|
||||
it('calls setActiveListId when a list is shared via ActivityLog and the list exists', () => {
|
||||
mockedUseShoppingLists.mockReturnValueOnce({
|
||||
mockedUseShoppingLists.mockReturnValue({
|
||||
...defaultUseShoppingListsReturn,
|
||||
shoppingLists: [
|
||||
createMockShoppingList({ shopping_list_id: 1, name: 'My List', user_id: 'user-123' }),
|
||||
@@ -315,10 +274,59 @@ describe('MainLayout Component', () => {
|
||||
|
||||
expect(mockSetActiveListId).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
it('displays an error message if useData has an error', () => {
|
||||
mockedUseFlyers.mockReturnValue({
|
||||
...defaultUseFlyersReturn,
|
||||
flyersError: new Error('Data Fetch Failed'),
|
||||
});
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Data Fetch Failed');
|
||||
});
|
||||
|
||||
it('displays an error message if useShoppingLists has an error', () => {
|
||||
mockedUseShoppingLists.mockReturnValue({
|
||||
...defaultUseShoppingListsReturn,
|
||||
error: 'Shopping List Failed',
|
||||
});
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Shopping List Failed');
|
||||
});
|
||||
|
||||
it('displays an error message if useMasterItems has an error', () => {
|
||||
mockedUseMasterItems.mockReturnValue({
|
||||
...defaultUseMasterItemsReturn,
|
||||
error: 'Master Items Failed',
|
||||
});
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Master Items Failed');
|
||||
});
|
||||
|
||||
it('displays an error message if useWatchedItems has an error', () => {
|
||||
mockedUseWatchedItems.mockReturnValue({
|
||||
...defaultUseWatchedItemsReturn,
|
||||
error: 'Watched Items Failed',
|
||||
});
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Watched Items Failed');
|
||||
});
|
||||
|
||||
it('displays an error message if useActiveDeals has an error', () => {
|
||||
mockedUseActiveDeals.mockReturnValue({
|
||||
...defaultUseActiveDealsReturn,
|
||||
error: 'Active Deals Failed',
|
||||
});
|
||||
renderWithRouter(<MainLayout {...defaultProps} />);
|
||||
expect(screen.getByTestId('error-display')).toHaveTextContent('Active Deals Failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Event Handlers', () => {
|
||||
it('calls addItemToList when an item is added from ShoppingListComponent and a list is active', () => {
|
||||
const mockAddItemToList = vi.fn();
|
||||
mockedUseShoppingLists.mockReturnValueOnce({
|
||||
mockedUseShoppingLists.mockReturnValue({
|
||||
...defaultUseShoppingListsReturn,
|
||||
activeListId: 1,
|
||||
addItemToList: mockAddItemToList,
|
||||
@@ -332,7 +340,7 @@ describe('MainLayout Component', () => {
|
||||
|
||||
it('does not call addItemToList from ShoppingListComponent if no list is active', () => {
|
||||
const mockAddItemToList = vi.fn();
|
||||
mockedUseShoppingLists.mockReturnValueOnce({
|
||||
mockedUseShoppingLists.mockReturnValue({
|
||||
...defaultUseShoppingListsReturn,
|
||||
activeListId: null,
|
||||
addItemToList: mockAddItemToList,
|
||||
@@ -346,7 +354,7 @@ describe('MainLayout Component', () => {
|
||||
|
||||
it('calls addItemToList when an item is added from WatchedItemsList and a list is active', () => {
|
||||
const mockAddItemToList = vi.fn();
|
||||
mockedUseShoppingLists.mockReturnValueOnce({
|
||||
mockedUseShoppingLists.mockReturnValue({
|
||||
...defaultUseShoppingListsReturn,
|
||||
activeListId: 5,
|
||||
addItemToList: mockAddItemToList,
|
||||
@@ -360,7 +368,7 @@ describe('MainLayout Component', () => {
|
||||
|
||||
it('does not call addItemToList from WatchedItemsList if no list is active', () => {
|
||||
const mockAddItemToList = vi.fn();
|
||||
mockedUseShoppingLists.mockReturnValueOnce({
|
||||
mockedUseShoppingLists.mockReturnValue({
|
||||
...defaultUseShoppingListsReturn,
|
||||
activeListId: null,
|
||||
addItemToList: mockAddItemToList,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// src/layouts/MainLayout.tsx
|
||||
import React, { useCallback } from 'react';
|
||||
import { Outlet } from 'react-router-dom';
|
||||
import Joyride from 'react-joyride';
|
||||
import { useAuth } from '../hooks/useAuth';
|
||||
import { useOnboardingTour } from '../hooks/useOnboardingTour';
|
||||
import { useFlyers } from '../hooks/useFlyers';
|
||||
@@ -34,7 +33,8 @@ export const MainLayout: React.FC<MainLayoutProps> = ({
|
||||
}) => {
|
||||
const { userProfile, authStatus } = useAuth();
|
||||
const user = userProfile?.user ?? null;
|
||||
const { runTour, steps, stepIndex, handleJoyrideCallback } = useOnboardingTour();
|
||||
// Driver.js tour is initialized and managed imperatively inside the hook
|
||||
useOnboardingTour();
|
||||
const { flyers, refetchFlyers, flyersError } = useFlyers();
|
||||
const { masterItems, error: masterItemsError } = useMasterItems();
|
||||
const {
|
||||
@@ -94,28 +94,13 @@ export const MainLayout: React.FC<MainLayoutProps> = ({
|
||||
watchedItemsError ||
|
||||
activeDealsError;
|
||||
|
||||
// Only show banner for unauthenticated users when there are flyers to view
|
||||
const shouldShowBanner = authStatus === 'SIGNED_OUT' && flyers.length > 0;
|
||||
|
||||
return (
|
||||
<main className="max-w-screen-2xl mx-auto py-4 px-2.5 sm:py-6 lg:py-8">
|
||||
<Joyride
|
||||
steps={steps}
|
||||
run={runTour}
|
||||
stepIndex={stepIndex}
|
||||
callback={handleJoyrideCallback}
|
||||
continuous
|
||||
showProgress
|
||||
showSkipButton
|
||||
styles={{
|
||||
options: {
|
||||
primaryColor: '#14b8a6',
|
||||
textColor: '#1f2937',
|
||||
zIndex: 10000,
|
||||
},
|
||||
}}
|
||||
/>
|
||||
{authStatus === 'SIGNED_OUT' && flyers.length > 0 && (
|
||||
{shouldShowBanner && (
|
||||
<div className="max-w-5xl mx-auto mb-6 px-4 lg:px-0">
|
||||
{' '}
|
||||
{/* This div was missing a closing tag in the original code, but it's outside the diff scope. */}
|
||||
<AnonymousUserBanner onOpenProfile={onOpenProfile} />
|
||||
</div>
|
||||
)}
|
||||
@@ -170,9 +155,15 @@ export const MainLayout: React.FC<MainLayoutProps> = ({
|
||||
unitSystem={'imperial'} // This can be passed down or sourced from a context
|
||||
user={user}
|
||||
/>
|
||||
<PriceHistoryChart />
|
||||
<Leaderboard />
|
||||
<ActivityLog userProfile={userProfile} onLogClick={handleActivityLogClick} />
|
||||
{user && (
|
||||
<>
|
||||
<PriceHistoryChart />
|
||||
<Leaderboard />
|
||||
{userProfile?.role === 'admin' && (
|
||||
<ActivityLog userProfile={userProfile} onLogClick={handleActivityLogClick} />
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -71,6 +71,8 @@ export const AuthProvider: React.FC<{ children: ReactNode }> = ({ children }) =>
|
||||
const login = useCallback(
|
||||
async (token: string, profileData?: UserProfile) => {
|
||||
logger.info(`[AuthProvider-Login] Attempting login.`);
|
||||
console.log('[AuthProvider-Login] Received token:', token);
|
||||
console.log('[AuthProvider-Login] Received profileData:', profileData);
|
||||
setToken(token);
|
||||
|
||||
if (profileData) {
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
// src/schemas/flyer.schemas.test.ts
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { flyerInsertSchema, flyerDbInsertSchema } from './flyer.schemas';
|
||||
import { getFlyerBaseUrl } from '../tests/utils/testHelpers';
|
||||
|
||||
const FLYER_BASE_URL = getFlyerBaseUrl();
|
||||
|
||||
describe('flyerInsertSchema', () => {
|
||||
const validFlyer = {
|
||||
file_name: 'flyer.jpg',
|
||||
image_url: 'https://example.com/flyer.jpg',
|
||||
icon_url: 'https://example.com/icon.jpg',
|
||||
image_url: `${FLYER_BASE_URL}/flyer.jpg`,
|
||||
icon_url: `${FLYER_BASE_URL}/icon.jpg`,
|
||||
checksum: 'a'.repeat(64),
|
||||
store_name: 'Test Store',
|
||||
valid_from: '2023-01-01T00:00:00Z',
|
||||
@@ -128,8 +131,8 @@ describe('flyerInsertSchema', () => {
|
||||
describe('flyerDbInsertSchema', () => {
|
||||
const validDbFlyer = {
|
||||
file_name: 'flyer.jpg',
|
||||
image_url: 'https://example.com/flyer.jpg',
|
||||
icon_url: 'https://example.com/icon.jpg',
|
||||
image_url: `${FLYER_BASE_URL}/flyer.jpg`,
|
||||
icon_url: `${FLYER_BASE_URL}/icon.jpg`,
|
||||
checksum: 'a'.repeat(64),
|
||||
store_id: 1,
|
||||
valid_from: '2023-01-01T00:00:00Z',
|
||||
|
||||
@@ -62,16 +62,21 @@ const _performTokenRefresh = async (): Promise<string> => {
|
||||
// This endpoint relies on the HttpOnly cookie, so no body is needed.
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
const data = await response.json();
|
||||
const result = await response.json();
|
||||
if (!response.ok) {
|
||||
throw new Error(data.message || 'Failed to refresh token.');
|
||||
throw new Error(result.error?.message || result.message || 'Failed to refresh token.');
|
||||
}
|
||||
// The API returns {success, data: {token}}, so extract the token
|
||||
const token = result.data?.token;
|
||||
if (!token) {
|
||||
throw new Error('No token received from refresh endpoint.');
|
||||
}
|
||||
// On successful refresh, store the new access token.
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem('authToken', data.token);
|
||||
localStorage.setItem('authToken', token);
|
||||
}
|
||||
logger.info('Successfully refreshed access token.');
|
||||
return data.token;
|
||||
return token;
|
||||
} catch (error) {
|
||||
logger.error({ error }, 'Failed to refresh token. User session has expired.');
|
||||
// Only perform browser-specific actions if in the browser environment.
|
||||
|
||||
@@ -4,6 +4,9 @@ import { Job, UnrecoverableError } from 'bullmq';
|
||||
import path from 'node:path';
|
||||
import type { FlyerInsert } from '../types';
|
||||
import type { CleanupJobData, FlyerJobData } from '../types/job-data';
|
||||
import { getFlyerBaseUrl } from '../tests/utils/testHelpers';
|
||||
|
||||
const FLYER_BASE_URL = getFlyerBaseUrl();
|
||||
|
||||
// 1. Create hoisted mocks FIRST
|
||||
const mocks = vi.hoisted(() => ({
|
||||
@@ -840,8 +843,8 @@ describe('FlyerProcessingService', () => {
|
||||
|
||||
const job = createMockCleanupJob({ flyerId: 1, paths: [] }); // Empty paths
|
||||
const mockFlyer = createMockFlyer({
|
||||
image_url: 'https://example.com/flyer-images/flyer-abc.jpg',
|
||||
icon_url: 'https://example.com/flyer-images/icons/icon-flyer-abc.webp',
|
||||
image_url: `${FLYER_BASE_URL}/flyer-images/flyer-abc.jpg`,
|
||||
icon_url: `${FLYER_BASE_URL}/flyer-images/icons/icon-flyer-abc.webp`,
|
||||
});
|
||||
// Mock DB call to return a flyer
|
||||
vi.mocked(mockedDb.flyerRepo.getFlyerById).mockResolvedValue(mockFlyer);
|
||||
|
||||
@@ -21,6 +21,8 @@ const isProduction = process.env.NODE_ENV === 'production';
|
||||
const isTest = process.env.NODE_ENV === 'test';
|
||||
const isStaging = process.env.NODE_ENV === 'staging';
|
||||
const isDevelopment = !isProduction && !isTest && !isStaging;
|
||||
// PM2_HOME is set when running under PM2 process manager
|
||||
const isRunningUnderPM2 = !!process.env.PM2_HOME;
|
||||
|
||||
// Determine log directory based on environment
|
||||
// In production/test, use the application directory's logs folder
|
||||
@@ -73,8 +75,9 @@ const redactConfig = {
|
||||
const createLogger = (): pino.Logger => {
|
||||
const logDir = ensureLogDirectory();
|
||||
|
||||
// Development: Use pino-pretty for human-readable output
|
||||
if (isDevelopment) {
|
||||
// Development WITHOUT PM2: Use pino-pretty for human-readable console output
|
||||
// This is for local development outside of containers (rare case now)
|
||||
if (isDevelopment && !isRunningUnderPM2) {
|
||||
return pino({
|
||||
level: 'debug',
|
||||
transport: {
|
||||
@@ -89,6 +92,15 @@ const createLogger = (): pino.Logger => {
|
||||
});
|
||||
}
|
||||
|
||||
// Development WITH PM2 (ADR-014): Output JSON for Logstash integration
|
||||
// PM2 captures stdout to /var/log/pm2/*.log files which Logstash parses
|
||||
if (isDevelopment && isRunningUnderPM2) {
|
||||
return pino({
|
||||
level: 'debug',
|
||||
redact: redactConfig,
|
||||
});
|
||||
}
|
||||
|
||||
// Production/Test: Write to both stdout and file
|
||||
if (logDir) {
|
||||
const logFilePath = path.join(logDir, 'app.log');
|
||||
|
||||
BIN
src/tests/assets/test-flyer-icon.png
Normal file
BIN
src/tests/assets/test-flyer-icon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
@@ -1,14 +1,18 @@
|
||||
// src/tests/e2e/admin-authorization.e2e.test.ts
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import * as apiClient from '../../services/apiClient';
|
||||
import supertest from 'supertest';
|
||||
import { cleanupDb } from '../utils/cleanup';
|
||||
import { createAndLoginUser } from '../utils/testHelpers';
|
||||
import type { UserProfile } from '../../types';
|
||||
import { getServerUrl } from '../setup/e2e-global-setup';
|
||||
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
describe('Admin Route Authorization', () => {
|
||||
// Create a getter function that returns supertest instance with the app
|
||||
const getRequest = () => supertest(getServerUrl());
|
||||
|
||||
let regularUser: UserProfile;
|
||||
let regularUserAuthToken: string;
|
||||
|
||||
@@ -17,6 +21,7 @@ describe('Admin Route Authorization', () => {
|
||||
const { user, token } = await createAndLoginUser({
|
||||
email: `e2e-authz-user-${Date.now()}@example.com`,
|
||||
fullName: 'E2E AuthZ User',
|
||||
request: getRequest(),
|
||||
});
|
||||
regularUser = user;
|
||||
regularUserAuthToken = token;
|
||||
@@ -33,48 +38,42 @@ describe('Admin Route Authorization', () => {
|
||||
const adminEndpoints = [
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/admin/stats',
|
||||
action: (token: string) => apiClient.getApplicationStats(token),
|
||||
path: '/api/admin/stats',
|
||||
},
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/admin/users',
|
||||
action: (token: string) => apiClient.authedGet('/admin/users', { tokenOverride: token }),
|
||||
path: '/api/admin/users',
|
||||
},
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/admin/corrections',
|
||||
action: (token: string) => apiClient.getSuggestedCorrections(token),
|
||||
path: '/api/admin/corrections',
|
||||
},
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/admin/corrections/1/approve',
|
||||
action: (token: string) => apiClient.approveCorrection(1, token),
|
||||
path: '/api/admin/corrections/1/approve',
|
||||
},
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/admin/trigger/daily-deal-check',
|
||||
action: (token: string) =>
|
||||
apiClient.authedPostEmpty('/admin/trigger/daily-deal-check', { tokenOverride: token }),
|
||||
path: '/api/admin/trigger/daily-deal-check',
|
||||
},
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/admin/queues/status',
|
||||
action: (token: string) =>
|
||||
apiClient.authedGet('/admin/queues/status', { tokenOverride: token }),
|
||||
path: '/api/admin/queues/status',
|
||||
},
|
||||
];
|
||||
|
||||
it.each(adminEndpoints)(
|
||||
'should return 403 Forbidden for a regular user trying to access $method $path',
|
||||
async ({ action }) => {
|
||||
async ({ method, path }) => {
|
||||
// Act: Attempt to access the admin endpoint with the regular user's token
|
||||
const response = await action(regularUserAuthToken);
|
||||
const requestBuilder = method === 'GET' ? getRequest().get(path) : getRequest().post(path);
|
||||
const response = await requestBuilder
|
||||
.set('Authorization', `Bearer ${regularUserAuthToken}`)
|
||||
.send();
|
||||
|
||||
// Assert: The request should be forbidden
|
||||
expect(response.status).toBe(403);
|
||||
const responseBody = await response.json();
|
||||
expect(responseBody.error.message).toBe('Forbidden: Administrator access required.');
|
||||
expect(response.body.error.message).toBe('Forbidden: Administrator access required.');
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
// src/tests/e2e/admin-dashboard.e2e.test.ts
|
||||
import { describe, it, expect, afterAll } from 'vitest';
|
||||
import * as apiClient from '../../services/apiClient';
|
||||
import supertest from 'supertest';
|
||||
import { getPool } from '../../services/db/connection.db';
|
||||
import { cleanupDb } from '../utils/cleanup';
|
||||
import { getServerUrl } from '../setup/e2e-global-setup';
|
||||
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
|
||||
describe('E2E Admin Dashboard Flow', () => {
|
||||
// Create a getter function that returns supertest instance with the app
|
||||
const getRequest = () => supertest(getServerUrl());
|
||||
|
||||
// Use a unique email for every run to avoid collisions
|
||||
const uniqueId = Date.now();
|
||||
const adminEmail = `e2e-admin-${uniqueId}@example.com`;
|
||||
@@ -26,15 +30,12 @@ describe('E2E Admin Dashboard Flow', () => {
|
||||
|
||||
it('should allow an admin to log in and access dashboard features', async () => {
|
||||
// 1. Register a new user (initially a regular user)
|
||||
const registerResponse = await apiClient.registerUser(
|
||||
adminEmail,
|
||||
adminPassword,
|
||||
'E2E Admin User',
|
||||
);
|
||||
const registerResponse = await getRequest()
|
||||
.post('/api/auth/register')
|
||||
.send({ email: adminEmail, password: adminPassword, full_name: 'E2E Admin User' });
|
||||
|
||||
expect(registerResponse.status).toBe(201);
|
||||
const registerResponseBody = await registerResponse.json();
|
||||
const registeredUser = registerResponseBody.data.userprofile.user;
|
||||
const registeredUser = registerResponse.body.data.userprofile.user;
|
||||
adminUserId = registeredUser.user_id;
|
||||
expect(adminUserId).toBeDefined();
|
||||
|
||||
@@ -49,49 +50,47 @@ describe('E2E Admin Dashboard Flow', () => {
|
||||
// and to provide a buffer for any rate limits from previous tests.
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
|
||||
const loginResponse = await apiClient.loginUser(adminEmail, adminPassword, false);
|
||||
if (!loginResponse.ok) {
|
||||
const errorText = await loginResponse.text();
|
||||
throw new Error(`Failed to log in as admin: ${loginResponse.status} ${errorText}`);
|
||||
}
|
||||
const loginResponseBody = await loginResponse.json();
|
||||
const loginResponse = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: adminEmail, password: adminPassword, rememberMe: false });
|
||||
|
||||
expect(loginResponse.status).toBe(200);
|
||||
authToken = loginResponseBody.data.token;
|
||||
authToken = loginResponse.body.data.token;
|
||||
expect(authToken).toBeDefined();
|
||||
// Verify the role returned in the login response is now 'admin'
|
||||
expect(loginResponseBody.data.userprofile.role).toBe('admin');
|
||||
expect(loginResponse.body.data.userprofile.role).toBe('admin');
|
||||
|
||||
// 4. Fetch System Stats (Protected Admin Route)
|
||||
const statsResponse = await apiClient.getApplicationStats(authToken);
|
||||
const statsResponse = await getRequest()
|
||||
.get('/api/admin/stats')
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(statsResponse.status).toBe(200);
|
||||
const statsResponseBody = await statsResponse.json();
|
||||
expect(statsResponseBody.data).toHaveProperty('userCount');
|
||||
expect(statsResponseBody.data).toHaveProperty('flyerCount');
|
||||
expect(statsResponse.body.data).toHaveProperty('userCount');
|
||||
expect(statsResponse.body.data).toHaveProperty('flyerCount');
|
||||
|
||||
// 5. Fetch User List (Protected Admin Route)
|
||||
const usersResponse = await apiClient.authedGet('/admin/users', { tokenOverride: authToken });
|
||||
const usersResponse = await getRequest()
|
||||
.get('/api/admin/users')
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(usersResponse.status).toBe(200);
|
||||
const usersResponseBody = await usersResponse.json();
|
||||
expect(usersResponseBody.data).toHaveProperty('users');
|
||||
expect(usersResponseBody.data).toHaveProperty('total');
|
||||
expect(Array.isArray(usersResponseBody.data.users)).toBe(true);
|
||||
expect(usersResponse.body.data).toHaveProperty('users');
|
||||
expect(usersResponse.body.data).toHaveProperty('total');
|
||||
expect(Array.isArray(usersResponse.body.data.users)).toBe(true);
|
||||
// The list should contain the admin user we just created
|
||||
const self = usersResponseBody.data.users.find((u: any) => u.user_id === adminUserId);
|
||||
const self = usersResponse.body.data.users.find((u: any) => u.user_id === adminUserId);
|
||||
expect(self).toBeDefined();
|
||||
|
||||
// 6. Check Queue Status (Protected Admin Route)
|
||||
const queueResponse = await apiClient.authedGet('/admin/queues/status', {
|
||||
tokenOverride: authToken,
|
||||
});
|
||||
const queueResponse = await getRequest()
|
||||
.get('/api/admin/queues/status')
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(queueResponse.status).toBe(200);
|
||||
const queueResponseBody = await queueResponse.json();
|
||||
expect(Array.isArray(queueResponseBody.data)).toBe(true);
|
||||
expect(Array.isArray(queueResponse.body.data)).toBe(true);
|
||||
// Verify that the 'flyer-processing' queue is present in the status report
|
||||
const flyerQueue = queueResponseBody.data.find((q: any) => q.name === 'flyer-processing');
|
||||
const flyerQueue = queueResponse.body.data.find((q: any) => q.name === 'flyer-processing');
|
||||
expect(flyerQueue).toBeDefined();
|
||||
expect(flyerQueue.counts).toBeDefined();
|
||||
});
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
// src/tests/e2e/auth.e2e.test.ts
|
||||
import { describe, it, expect, afterAll, beforeAll } from 'vitest';
|
||||
import * as apiClient from '../../services/apiClient';
|
||||
import supertest from 'supertest';
|
||||
import { cleanupDb } from '../utils/cleanup';
|
||||
import { poll } from '../utils/poll';
|
||||
import { createAndLoginUser, TEST_PASSWORD } from '../utils/testHelpers';
|
||||
import type { UserProfile } from '../../types';
|
||||
import { getServerUrl } from '../setup/e2e-global-setup';
|
||||
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
|
||||
describe('Authentication E2E Flow', () => {
|
||||
// Create a getter function that returns supertest instance with the app
|
||||
const getRequest = () => supertest(getServerUrl());
|
||||
|
||||
let testUser: UserProfile;
|
||||
let testUserAuthToken: string;
|
||||
const createdUserIds: string[] = [];
|
||||
@@ -21,6 +24,7 @@ describe('Authentication E2E Flow', () => {
|
||||
const { user, token } = await createAndLoginUser({
|
||||
email: `e2e-login-user-${Date.now()}@example.com`,
|
||||
fullName: 'E2E Login User',
|
||||
request: getRequest(),
|
||||
});
|
||||
testUserAuthToken = token;
|
||||
testUser = user;
|
||||
@@ -43,18 +47,19 @@ describe('Authentication E2E Flow', () => {
|
||||
const fullName = 'E2E Register User';
|
||||
|
||||
// Act
|
||||
const response = await apiClient.registerUser(email, TEST_PASSWORD, fullName);
|
||||
const responseBody = await response.json();
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/register')
|
||||
.send({ email, password: TEST_PASSWORD, full_name: fullName });
|
||||
|
||||
// Assert
|
||||
expect(response.status).toBe(201);
|
||||
expect(responseBody.data.message).toBe('User registered successfully!');
|
||||
expect(responseBody.data.userprofile).toBeDefined();
|
||||
expect(responseBody.data.userprofile.user.email).toBe(email);
|
||||
expect(responseBody.data.token).toBeTypeOf('string');
|
||||
expect(response.body.data.message).toBe('User registered successfully!');
|
||||
expect(response.body.data.userprofile).toBeDefined();
|
||||
expect(response.body.data.userprofile.user.email).toBe(email);
|
||||
expect(response.body.data.token).toBeTypeOf('string');
|
||||
|
||||
// Add to cleanup
|
||||
createdUserIds.push(responseBody.data.userprofile.user.user_id);
|
||||
createdUserIds.push(response.body.data.userprofile.user.user_id);
|
||||
});
|
||||
|
||||
it('should fail to register a user with a weak password', async () => {
|
||||
@@ -62,12 +67,13 @@ describe('Authentication E2E Flow', () => {
|
||||
const weakPassword = '123';
|
||||
|
||||
// Act
|
||||
const response = await apiClient.registerUser(email, weakPassword, 'Weak Pass User');
|
||||
const responseBody = await response.json();
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/register')
|
||||
.send({ email, password: weakPassword, full_name: 'Weak Pass User' });
|
||||
|
||||
// Assert
|
||||
expect(response.status).toBe(400);
|
||||
expect(responseBody.error.details[0].message).toContain(
|
||||
expect(response.body.error.details[0].message).toContain(
|
||||
'Password must be at least 8 characters long.',
|
||||
);
|
||||
});
|
||||
@@ -76,18 +82,20 @@ describe('Authentication E2E Flow', () => {
|
||||
const email = `e2e-register-duplicate-${Date.now()}@example.com`;
|
||||
|
||||
// Act 1: Register the user successfully
|
||||
const firstResponse = await apiClient.registerUser(email, TEST_PASSWORD, 'Duplicate User');
|
||||
const firstResponseBody = await firstResponse.json();
|
||||
const firstResponse = await getRequest()
|
||||
.post('/api/auth/register')
|
||||
.send({ email, password: TEST_PASSWORD, full_name: 'Duplicate User' });
|
||||
expect(firstResponse.status).toBe(201);
|
||||
createdUserIds.push(firstResponseBody.data.userprofile.user.user_id);
|
||||
createdUserIds.push(firstResponse.body.data.userprofile.user.user_id);
|
||||
|
||||
// Act 2: Attempt to register the same user again
|
||||
const secondResponse = await apiClient.registerUser(email, TEST_PASSWORD, 'Duplicate User');
|
||||
const secondResponseBody = await secondResponse.json();
|
||||
const secondResponse = await getRequest()
|
||||
.post('/api/auth/register')
|
||||
.send({ email, password: TEST_PASSWORD, full_name: 'Duplicate User' });
|
||||
|
||||
// Assert
|
||||
expect(secondResponse.status).toBe(409); // Conflict
|
||||
expect(secondResponseBody.error.message).toContain(
|
||||
expect(secondResponse.body.error.message).toContain(
|
||||
'A user with this email address already exists.',
|
||||
);
|
||||
});
|
||||
@@ -96,32 +104,35 @@ describe('Authentication E2E Flow', () => {
|
||||
describe('Login Flow', () => {
|
||||
it('should successfully log in a registered user', async () => {
|
||||
// Act: Attempt to log in with the user created in beforeAll
|
||||
const response = await apiClient.loginUser(testUser.user.email, TEST_PASSWORD, false);
|
||||
const responseBody = await response.json();
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: testUser.user.email, password: TEST_PASSWORD, rememberMe: false });
|
||||
|
||||
// Assert
|
||||
expect(response.status).toBe(200);
|
||||
expect(responseBody.data.userprofile).toBeDefined();
|
||||
expect(responseBody.data.userprofile.user.email).toBe(testUser.user.email);
|
||||
expect(responseBody.data.token).toBeTypeOf('string');
|
||||
expect(response.body.data.userprofile).toBeDefined();
|
||||
expect(response.body.data.userprofile.user.email).toBe(testUser.user.email);
|
||||
expect(response.body.data.token).toBeTypeOf('string');
|
||||
});
|
||||
|
||||
it('should fail to log in with an incorrect password', async () => {
|
||||
// Act: Attempt to log in with the wrong password
|
||||
const response = await apiClient.loginUser(testUser.user.email, 'wrong-password', false);
|
||||
const responseBody = await response.json();
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: testUser.user.email, password: 'wrong-password', rememberMe: false });
|
||||
|
||||
// Assert
|
||||
expect(response.status).toBe(401);
|
||||
expect(responseBody.error.message).toBe('Incorrect email or password.');
|
||||
expect(response.body.error.message).toBe('Incorrect email or password.');
|
||||
});
|
||||
|
||||
it('should fail to log in with a non-existent email', async () => {
|
||||
const response = await apiClient.loginUser('no-one-here@example.com', TEST_PASSWORD, false);
|
||||
const responseBody = await response.json();
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: 'no-one-here@example.com', password: TEST_PASSWORD, rememberMe: false });
|
||||
|
||||
expect(response.status).toBe(401);
|
||||
expect(responseBody.error.message).toBe('Incorrect email or password.');
|
||||
expect(response.body.error.message).toBe('Incorrect email or password.');
|
||||
});
|
||||
|
||||
it('should be able to access a protected route after logging in', async () => {
|
||||
@@ -130,15 +141,16 @@ describe('Authentication E2E Flow', () => {
|
||||
expect(token).toBeDefined();
|
||||
|
||||
// Act: Use the token to access a protected route
|
||||
const profileResponse = await apiClient.getAuthenticatedUserProfile({ tokenOverride: token });
|
||||
const responseBody = await profileResponse.json();
|
||||
const response = await getRequest()
|
||||
.get('/api/users/profile')
|
||||
.set('Authorization', `Bearer ${token}`);
|
||||
|
||||
// Assert
|
||||
expect(profileResponse.status).toBe(200);
|
||||
expect(responseBody.data).toBeDefined();
|
||||
expect(responseBody.data.user.user_id).toBe(testUser.user.user_id);
|
||||
expect(responseBody.data.user.email).toBe(testUser.user.email);
|
||||
expect(responseBody.data.role).toBe('user');
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.data).toBeDefined();
|
||||
expect(response.body.data.user.user_id).toBe(testUser.user.user_id);
|
||||
expect(response.body.data.user.email).toBe(testUser.user.email);
|
||||
expect(response.body.data.role).toBe('user');
|
||||
});
|
||||
|
||||
it('should allow an authenticated user to update their profile', async () => {
|
||||
@@ -152,23 +164,24 @@ describe('Authentication E2E Flow', () => {
|
||||
};
|
||||
|
||||
// Act: Call the update endpoint
|
||||
const updateResponse = await apiClient.updateUserProfile(profileUpdates, {
|
||||
tokenOverride: token,
|
||||
});
|
||||
const updateResponseBody = await updateResponse.json();
|
||||
const updateResponse = await getRequest()
|
||||
.put('/api/users/profile')
|
||||
.set('Authorization', `Bearer ${token}`)
|
||||
.send(profileUpdates);
|
||||
|
||||
// Assert: Check the response from the update call
|
||||
expect(updateResponse.status).toBe(200);
|
||||
expect(updateResponseBody.data.full_name).toBe(profileUpdates.full_name);
|
||||
expect(updateResponseBody.data.avatar_url).toBe(profileUpdates.avatar_url);
|
||||
expect(updateResponse.body.data.full_name).toBe(profileUpdates.full_name);
|
||||
expect(updateResponse.body.data.avatar_url).toBe(profileUpdates.avatar_url);
|
||||
|
||||
// Act 2: Fetch the profile again to verify persistence
|
||||
const verifyResponse = await apiClient.getAuthenticatedUserProfile({ tokenOverride: token });
|
||||
const verifyResponseBody = await verifyResponse.json();
|
||||
const verifyResponse = await getRequest()
|
||||
.get('/api/users/profile')
|
||||
.set('Authorization', `Bearer ${token}`);
|
||||
|
||||
// Assert 2: Check the fetched data
|
||||
expect(verifyResponseBody.data.full_name).toBe(profileUpdates.full_name);
|
||||
expect(verifyResponseBody.data.avatar_url).toBe(profileUpdates.avatar_url);
|
||||
expect(verifyResponse.body.data.full_name).toBe(profileUpdates.full_name);
|
||||
expect(verifyResponse.body.data.avatar_url).toBe(profileUpdates.avatar_url);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -176,27 +189,29 @@ describe('Authentication E2E Flow', () => {
|
||||
it('should allow a user to reset their password and log in with the new one', async () => {
|
||||
// Arrange: Create a user to reset the password for
|
||||
const email = `e2e-reset-pass-${Date.now()}@example.com`;
|
||||
const registerResponse = await apiClient.registerUser(
|
||||
email,
|
||||
TEST_PASSWORD,
|
||||
'Reset Pass User',
|
||||
);
|
||||
const registerResponseBody = await registerResponse.json();
|
||||
const registerResponse = await getRequest()
|
||||
.post('/api/auth/register')
|
||||
.send({ email, password: TEST_PASSWORD, full_name: 'Reset Pass User' });
|
||||
expect(registerResponse.status).toBe(201);
|
||||
createdUserIds.push(registerResponseBody.data.userprofile.user.user_id);
|
||||
createdUserIds.push(registerResponse.body.data.userprofile.user.user_id);
|
||||
|
||||
// Poll until the user can log in, confirming the record has propagated.
|
||||
await poll(
|
||||
() => apiClient.loginUser(email, TEST_PASSWORD, false),
|
||||
(response) => response.ok,
|
||||
{ timeout: 10000, interval: 1000, description: 'user login after registration' },
|
||||
);
|
||||
// Verify user can log in (confirming registration completed)
|
||||
let loginAttempts = 0;
|
||||
let loginResponse;
|
||||
while (loginAttempts < 10) {
|
||||
loginResponse = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email, password: TEST_PASSWORD, rememberMe: false });
|
||||
if (loginResponse.status === 200) break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
loginAttempts++;
|
||||
}
|
||||
expect(loginResponse?.status).toBe(200);
|
||||
|
||||
// Request password reset (do not poll, as this endpoint is rate-limited)
|
||||
const forgotResponse = await apiClient.requestPasswordReset(email);
|
||||
const forgotResponse = await getRequest().post('/api/auth/forgot-password').send({ email });
|
||||
expect(forgotResponse.status).toBe(200);
|
||||
const forgotResponseBody = await forgotResponse.json();
|
||||
const resetToken = forgotResponseBody.data.token;
|
||||
const resetToken = forgotResponse.body.data.token;
|
||||
|
||||
// Assert 1: Check that we received a token.
|
||||
expect(
|
||||
@@ -207,20 +222,22 @@ describe('Authentication E2E Flow', () => {
|
||||
|
||||
// Act 2: Use the token to set a new password.
|
||||
const newPassword = 'my-new-e2e-password-!@#$';
|
||||
const resetResponse = await apiClient.resetPassword(resetToken, newPassword);
|
||||
const resetResponseBody = await resetResponse.json();
|
||||
const resetResponse = await getRequest()
|
||||
.post('/api/auth/reset-password')
|
||||
.send({ token: resetToken, newPassword });
|
||||
|
||||
// Assert 2: Check for a successful password reset message.
|
||||
expect(resetResponse.status).toBe(200);
|
||||
expect(resetResponseBody.data.message).toBe('Password has been reset successfully.');
|
||||
expect(resetResponse.body.data.message).toBe('Password has been reset successfully.');
|
||||
|
||||
// Act 3: Log in with the NEW password
|
||||
const loginResponse = await apiClient.loginUser(email, newPassword, false);
|
||||
const loginResponseBody = await loginResponse.json();
|
||||
const newLoginResponse = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email, password: newPassword, rememberMe: false });
|
||||
|
||||
expect(loginResponse.status).toBe(200);
|
||||
expect(loginResponseBody.data.userprofile).toBeDefined();
|
||||
expect(loginResponseBody.data.userprofile.user.email).toBe(email);
|
||||
expect(newLoginResponse.status).toBe(200);
|
||||
expect(newLoginResponse.body.data.userprofile).toBeDefined();
|
||||
expect(newLoginResponse.body.data.userprofile.user.email).toBe(email);
|
||||
});
|
||||
|
||||
it('should return a generic success message for a non-existent email to prevent enumeration', async () => {
|
||||
@@ -228,73 +245,71 @@ describe('Authentication E2E Flow', () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
|
||||
const nonExistentEmail = `non-existent-e2e-${Date.now()}@example.com`;
|
||||
const response = await apiClient.requestPasswordReset(nonExistentEmail);
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/forgot-password')
|
||||
.send({ email: nonExistentEmail });
|
||||
|
||||
// Check for rate limiting or other errors before parsing JSON to avoid SyntaxError
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(`Request failed with status ${response.status}: ${text}`);
|
||||
}
|
||||
|
||||
const responseBody = await response.json();
|
||||
expect(response.status).toBe(200);
|
||||
expect(responseBody.data.message).toBe(
|
||||
expect(response.body.data.message).toBe(
|
||||
'If an account with that email exists, a password reset link has been sent.',
|
||||
);
|
||||
expect(responseBody.data.token).toBeUndefined();
|
||||
expect(response.body.data.token).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Token Refresh Flow', () => {
|
||||
it('should allow an authenticated user to refresh their access token and use it', async () => {
|
||||
// 1. Log in to get the refresh token cookie and an initial access token.
|
||||
const loginResponse = await apiClient.loginUser(testUser.user.email, TEST_PASSWORD, false);
|
||||
const loginResponse = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: testUser.user.email, password: TEST_PASSWORD, rememberMe: false });
|
||||
expect(loginResponse.status).toBe(200);
|
||||
const loginResponseBody = await loginResponse.json();
|
||||
const initialAccessToken = loginResponseBody.data.token;
|
||||
const initialAccessToken = loginResponse.body.data.token;
|
||||
|
||||
// 2. Extract the refresh token from the 'set-cookie' header.
|
||||
const setCookieHeader = loginResponse.headers.get('set-cookie');
|
||||
const setCookieHeaders = loginResponse.headers['set-cookie'];
|
||||
expect(
|
||||
setCookieHeader,
|
||||
setCookieHeaders,
|
||||
'Set-Cookie header should be present in login response',
|
||||
).toBeDefined();
|
||||
// A typical Set-Cookie header might be 'refreshToken=...; Path=/; HttpOnly; Max-Age=...'. We just need the 'refreshToken=...' part.
|
||||
const refreshTokenCookie = setCookieHeader!.split(';')[0];
|
||||
// Find the refreshToken cookie
|
||||
const refreshTokenCookie = Array.isArray(setCookieHeaders)
|
||||
? setCookieHeaders.find((cookie: string) => cookie.startsWith('refreshToken='))
|
||||
: setCookieHeaders;
|
||||
expect(refreshTokenCookie).toBeDefined();
|
||||
|
||||
// Wait for >1 second to ensure the 'iat' (Issued At) claim in the new JWT changes.
|
||||
// JWT timestamps have second-level precision.
|
||||
await new Promise((resolve) => setTimeout(resolve, 1100));
|
||||
|
||||
// 3. Call the refresh token endpoint, passing the cookie.
|
||||
// This assumes a new method in apiClient to handle this specific request.
|
||||
const refreshResponse = await apiClient.refreshToken(refreshTokenCookie);
|
||||
const refreshResponse = await getRequest()
|
||||
.post('/api/auth/refresh-token')
|
||||
.set('Cookie', refreshTokenCookie!);
|
||||
|
||||
// 4. Assert the refresh was successful and we got a new token.
|
||||
expect(refreshResponse.status).toBe(200);
|
||||
const refreshResponseBody = await refreshResponse.json();
|
||||
const newAccessToken = refreshResponseBody.data.token;
|
||||
const newAccessToken = refreshResponse.body.data.token;
|
||||
expect(newAccessToken).toBeDefined();
|
||||
expect(newAccessToken).not.toBe(initialAccessToken);
|
||||
|
||||
// 5. Use the new access token to access a protected route.
|
||||
const profileResponse = await apiClient.getAuthenticatedUserProfile({
|
||||
tokenOverride: newAccessToken,
|
||||
});
|
||||
const profileResponse = await getRequest()
|
||||
.get('/api/users/profile')
|
||||
.set('Authorization', `Bearer ${newAccessToken}`);
|
||||
expect(profileResponse.status).toBe(200);
|
||||
const profileResponseBody = await profileResponse.json();
|
||||
expect(profileResponseBody.data.user.user_id).toBe(testUser.user.user_id);
|
||||
expect(profileResponse.body.data.user.user_id).toBe(testUser.user.user_id);
|
||||
});
|
||||
|
||||
it('should fail to refresh with an invalid or missing token', async () => {
|
||||
// Case 1: No cookie provided. This assumes refreshToken can handle an empty string.
|
||||
const noCookieResponse = await apiClient.refreshToken('');
|
||||
// Case 1: No cookie provided
|
||||
const noCookieResponse = await getRequest().post('/api/auth/refresh-token');
|
||||
expect(noCookieResponse.status).toBe(401);
|
||||
|
||||
// Case 2: Invalid cookie provided
|
||||
const invalidCookieResponse = await apiClient.refreshToken(
|
||||
'refreshToken=invalid-garbage-token',
|
||||
);
|
||||
const invalidCookieResponse = await getRequest()
|
||||
.post('/api/auth/refresh-token')
|
||||
.set('Cookie', 'refreshToken=invalid-garbage-token');
|
||||
expect(invalidCookieResponse.status).toBe(403);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* Tests the complete flow from user registration to creating budgets, tracking spending, and managing finances.
|
||||
*/
|
||||
import { describe, it, expect, afterAll } from 'vitest';
|
||||
import * as apiClient from '../../services/apiClient';
|
||||
import supertest from 'supertest';
|
||||
import { cleanupDb } from '../utils/cleanup';
|
||||
import { poll } from '../utils/poll';
|
||||
import { getPool } from '../../services/db/connection.db';
|
||||
@@ -13,35 +13,16 @@ import {
|
||||
cleanupStoreLocations,
|
||||
type CreatedStoreLocation,
|
||||
} from '../utils/storeHelpers';
|
||||
import { getServerUrl } from '../setup/e2e-global-setup';
|
||||
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
|
||||
const API_BASE_URL = process.env.VITE_API_BASE_URL || 'http://localhost:3000/api';
|
||||
|
||||
// Helper to make authenticated API calls
|
||||
const authedFetch = async (
|
||||
path: string,
|
||||
options: RequestInit & { token?: string } = {},
|
||||
): Promise<Response> => {
|
||||
const { token, ...fetchOptions } = options;
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
...(fetchOptions.headers as Record<string, string>),
|
||||
};
|
||||
|
||||
if (token) {
|
||||
headers['Authorization'] = `Bearer ${token}`;
|
||||
}
|
||||
|
||||
return fetch(`${API_BASE_URL}${path}`, {
|
||||
...fetchOptions,
|
||||
headers,
|
||||
});
|
||||
};
|
||||
|
||||
describe('E2E Budget Management Journey', () => {
|
||||
// Create a getter function that returns supertest instance with the app
|
||||
const getRequest = () => supertest(getServerUrl());
|
||||
|
||||
const uniqueId = Date.now();
|
||||
const userEmail = `budget-e2e-${uniqueId}@example.com`;
|
||||
const userPassword = 'StrongBudgetPassword123!';
|
||||
@@ -83,21 +64,23 @@ describe('E2E Budget Management Journey', () => {
|
||||
|
||||
it('should complete budget journey: Register -> Create Budget -> Track Spending -> Update -> Delete', async () => {
|
||||
// Step 1: Register a new user
|
||||
const registerResponse = await apiClient.registerUser(
|
||||
userEmail,
|
||||
userPassword,
|
||||
'Budget E2E User',
|
||||
);
|
||||
const registerResponse = await getRequest().post('/api/auth/register').send({
|
||||
email: userEmail,
|
||||
password: userPassword,
|
||||
full_name: 'Budget E2E User',
|
||||
});
|
||||
expect(registerResponse.status).toBe(201);
|
||||
|
||||
// Step 2: Login to get auth token
|
||||
const { response: loginResponse, responseBody: loginResponseBody } = await poll(
|
||||
async () => {
|
||||
const response = await apiClient.loginUser(userEmail, userPassword, false);
|
||||
const responseBody = response.ok ? await response.clone().json() : {};
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: userEmail, password: userPassword, rememberMe: false });
|
||||
const responseBody = response.status === 200 ? response.body : {};
|
||||
return { response, responseBody };
|
||||
},
|
||||
(result) => result.response.ok,
|
||||
(result) => result.response.status === 200,
|
||||
{ timeout: 10000, interval: 1000, description: 'user login after registration' },
|
||||
);
|
||||
|
||||
@@ -111,73 +94,65 @@ describe('E2E Budget Management Journey', () => {
|
||||
const startOfMonth = new Date(today.getFullYear(), today.getMonth(), 1);
|
||||
const formatDate = (d: Date) => d.toISOString().split('T')[0];
|
||||
|
||||
const createBudgetResponse = await authedFetch('/budgets', {
|
||||
method: 'POST',
|
||||
token: authToken,
|
||||
body: JSON.stringify({
|
||||
const createBudgetResponse = await getRequest()
|
||||
.post('/api/budgets')
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send({
|
||||
name: 'Monthly Groceries',
|
||||
amount_cents: 50000, // $500.00
|
||||
period: 'monthly',
|
||||
start_date: formatDate(startOfMonth),
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
expect(createBudgetResponse.status).toBe(201);
|
||||
const createBudgetData = await createBudgetResponse.json();
|
||||
expect(createBudgetData.data.name).toBe('Monthly Groceries');
|
||||
expect(createBudgetData.data.amount_cents).toBe(50000);
|
||||
expect(createBudgetData.data.period).toBe('monthly');
|
||||
const budgetId = createBudgetData.data.budget_id;
|
||||
expect(createBudgetResponse.body.data.name).toBe('Monthly Groceries');
|
||||
expect(createBudgetResponse.body.data.amount_cents).toBe(50000);
|
||||
expect(createBudgetResponse.body.data.period).toBe('monthly');
|
||||
const budgetId = createBudgetResponse.body.data.budget_id;
|
||||
createdBudgetIds.push(budgetId);
|
||||
|
||||
// Step 4: Create a weekly budget
|
||||
const weeklyBudgetResponse = await authedFetch('/budgets', {
|
||||
method: 'POST',
|
||||
token: authToken,
|
||||
body: JSON.stringify({
|
||||
const weeklyBudgetResponse = await getRequest()
|
||||
.post('/api/budgets')
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send({
|
||||
name: 'Weekly Dining Out',
|
||||
amount_cents: 10000, // $100.00
|
||||
period: 'weekly',
|
||||
start_date: formatDate(today),
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
expect(weeklyBudgetResponse.status).toBe(201);
|
||||
const weeklyBudgetData = await weeklyBudgetResponse.json();
|
||||
expect(weeklyBudgetData.data.period).toBe('weekly');
|
||||
createdBudgetIds.push(weeklyBudgetData.data.budget_id);
|
||||
expect(weeklyBudgetResponse.body.data.period).toBe('weekly');
|
||||
createdBudgetIds.push(weeklyBudgetResponse.body.data.budget_id);
|
||||
|
||||
// Step 5: View all budgets
|
||||
const listBudgetsResponse = await authedFetch('/budgets', {
|
||||
method: 'GET',
|
||||
token: authToken,
|
||||
});
|
||||
const listBudgetsResponse = await getRequest()
|
||||
.get('/api/budgets')
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(listBudgetsResponse.status).toBe(200);
|
||||
const listBudgetsData = await listBudgetsResponse.json();
|
||||
expect(listBudgetsData.data.length).toBe(2);
|
||||
expect(listBudgetsResponse.body.data.length).toBe(2);
|
||||
|
||||
// Find our budgets
|
||||
const monthlyBudget = listBudgetsData.data.find(
|
||||
const monthlyBudget = listBudgetsResponse.body.data.find(
|
||||
(b: { name: string }) => b.name === 'Monthly Groceries',
|
||||
);
|
||||
expect(monthlyBudget).toBeDefined();
|
||||
expect(monthlyBudget.amount_cents).toBe(50000);
|
||||
|
||||
// Step 6: Update a budget
|
||||
const updateBudgetResponse = await authedFetch(`/budgets/${budgetId}`, {
|
||||
method: 'PUT',
|
||||
token: authToken,
|
||||
body: JSON.stringify({
|
||||
const updateBudgetResponse = await getRequest()
|
||||
.put(`/api/budgets/${budgetId}`)
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send({
|
||||
amount_cents: 55000, // Increase to $550.00
|
||||
name: 'Monthly Groceries (Updated)',
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
expect(updateBudgetResponse.status).toBe(200);
|
||||
const updateBudgetData = await updateBudgetResponse.json();
|
||||
expect(updateBudgetData.data.amount_cents).toBe(55000);
|
||||
expect(updateBudgetData.data.name).toBe('Monthly Groceries (Updated)');
|
||||
expect(updateBudgetResponse.body.data.amount_cents).toBe(55000);
|
||||
expect(updateBudgetResponse.body.data.name).toBe('Monthly Groceries (Updated)');
|
||||
|
||||
// Step 7: Create test spending data (receipts) to track against budget
|
||||
const pool = getPool();
|
||||
@@ -212,69 +187,67 @@ describe('E2E Budget Management Journey', () => {
|
||||
|
||||
// Step 8: Check spending analysis
|
||||
const endOfMonth = new Date(today.getFullYear(), today.getMonth() + 1, 0);
|
||||
const spendingResponse = await authedFetch(
|
||||
`/budgets/spending-analysis?startDate=${formatDate(startOfMonth)}&endDate=${formatDate(endOfMonth)}`,
|
||||
{
|
||||
method: 'GET',
|
||||
token: authToken,
|
||||
},
|
||||
);
|
||||
const spendingResponse = await getRequest()
|
||||
.get(
|
||||
`/api/budgets/spending-analysis?startDate=${formatDate(startOfMonth)}&endDate=${formatDate(endOfMonth)}`,
|
||||
)
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(spendingResponse.status).toBe(200);
|
||||
const spendingData = await spendingResponse.json();
|
||||
expect(spendingData.success).toBe(true);
|
||||
expect(Array.isArray(spendingData.data)).toBe(true);
|
||||
expect(spendingResponse.body.success).toBe(true);
|
||||
expect(Array.isArray(spendingResponse.body.data)).toBe(true);
|
||||
|
||||
// Verify we have spending data
|
||||
// Note: The spending might be $0 or have data depending on how the backend calculates spending
|
||||
// The test is mainly verifying the endpoint works
|
||||
|
||||
// Step 9: Test budget validation - try to create invalid budget
|
||||
const invalidBudgetResponse = await authedFetch('/budgets', {
|
||||
method: 'POST',
|
||||
token: authToken,
|
||||
body: JSON.stringify({
|
||||
const invalidBudgetResponse = await getRequest()
|
||||
.post('/api/budgets')
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send({
|
||||
name: 'Invalid Budget',
|
||||
amount_cents: -100, // Negative amount should be rejected
|
||||
period: 'monthly',
|
||||
start_date: formatDate(today),
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
expect(invalidBudgetResponse.status).toBe(400);
|
||||
|
||||
// Step 10: Test budget validation - missing required fields
|
||||
const missingFieldsResponse = await authedFetch('/budgets', {
|
||||
method: 'POST',
|
||||
token: authToken,
|
||||
body: JSON.stringify({
|
||||
const missingFieldsResponse = await getRequest()
|
||||
.post('/api/budgets')
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send({
|
||||
name: 'Incomplete Budget',
|
||||
// Missing amount_cents, period, start_date
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
expect(missingFieldsResponse.status).toBe(400);
|
||||
|
||||
// Step 11: Test update validation - empty update
|
||||
const emptyUpdateResponse = await authedFetch(`/budgets/${budgetId}`, {
|
||||
method: 'PUT',
|
||||
token: authToken,
|
||||
body: JSON.stringify({}), // No fields to update
|
||||
});
|
||||
const emptyUpdateResponse = await getRequest()
|
||||
.put(`/api/budgets/${budgetId}`)
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send({}); // No fields to update
|
||||
|
||||
expect(emptyUpdateResponse.status).toBe(400);
|
||||
|
||||
// Step 12: Verify another user cannot access our budgets
|
||||
const otherUserEmail = `other-budget-e2e-${uniqueId}@example.com`;
|
||||
await apiClient.registerUser(otherUserEmail, userPassword, 'Other Budget User');
|
||||
await getRequest()
|
||||
.post('/api/auth/register')
|
||||
.send({ email: otherUserEmail, password: userPassword, full_name: 'Other Budget User' });
|
||||
|
||||
const { responseBody: otherLoginData } = await poll(
|
||||
async () => {
|
||||
const response = await apiClient.loginUser(otherUserEmail, userPassword, false);
|
||||
const responseBody = response.ok ? await response.clone().json() : {};
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: otherUserEmail, password: userPassword, rememberMe: false });
|
||||
const responseBody = response.status === 200 ? response.body : {};
|
||||
return { response, responseBody };
|
||||
},
|
||||
(result) => result.response.ok,
|
||||
(result) => result.response.status === 200,
|
||||
{ timeout: 10000, interval: 1000, description: 'other user login' },
|
||||
);
|
||||
|
||||
@@ -282,31 +255,27 @@ describe('E2E Budget Management Journey', () => {
|
||||
const otherUserId = otherLoginData.data.userprofile.user.user_id;
|
||||
|
||||
// Other user should not see our budgets
|
||||
const otherBudgetsResponse = await authedFetch('/budgets', {
|
||||
method: 'GET',
|
||||
token: otherToken,
|
||||
});
|
||||
const otherBudgetsResponse = await getRequest()
|
||||
.get('/api/budgets')
|
||||
.set('Authorization', `Bearer ${otherToken}`);
|
||||
|
||||
expect(otherBudgetsResponse.status).toBe(200);
|
||||
const otherBudgetsData = await otherBudgetsResponse.json();
|
||||
expect(otherBudgetsData.data.length).toBe(0);
|
||||
expect(otherBudgetsResponse.body.data.length).toBe(0);
|
||||
|
||||
// Other user should not be able to update our budget
|
||||
const otherUpdateResponse = await authedFetch(`/budgets/${budgetId}`, {
|
||||
method: 'PUT',
|
||||
token: otherToken,
|
||||
body: JSON.stringify({
|
||||
const otherUpdateResponse = await getRequest()
|
||||
.put(`/api/budgets/${budgetId}`)
|
||||
.set('Authorization', `Bearer ${otherToken}`)
|
||||
.send({
|
||||
amount_cents: 99999,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
expect(otherUpdateResponse.status).toBe(404); // Should not find the budget
|
||||
|
||||
// Other user should not be able to delete our budget
|
||||
const otherDeleteAttemptResponse = await authedFetch(`/budgets/${budgetId}`, {
|
||||
method: 'DELETE',
|
||||
token: otherToken,
|
||||
});
|
||||
const otherDeleteAttemptResponse = await getRequest()
|
||||
.delete(`/api/budgets/${budgetId}`)
|
||||
.set('Authorization', `Bearer ${otherToken}`);
|
||||
|
||||
expect(otherDeleteAttemptResponse.status).toBe(404);
|
||||
|
||||
@@ -314,38 +283,36 @@ describe('E2E Budget Management Journey', () => {
|
||||
await cleanupDb({ userIds: [otherUserId] });
|
||||
|
||||
// Step 13: Delete the weekly budget
|
||||
const deleteBudgetResponse = await authedFetch(`/budgets/${weeklyBudgetData.data.budget_id}`, {
|
||||
method: 'DELETE',
|
||||
token: authToken,
|
||||
});
|
||||
const deleteBudgetResponse = await getRequest()
|
||||
.delete(`/api/budgets/${weeklyBudgetResponse.body.data.budget_id}`)
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(deleteBudgetResponse.status).toBe(204);
|
||||
|
||||
// Remove from cleanup list
|
||||
const deleteIndex = createdBudgetIds.indexOf(weeklyBudgetData.data.budget_id);
|
||||
const deleteIndex = createdBudgetIds.indexOf(weeklyBudgetResponse.body.data.budget_id);
|
||||
if (deleteIndex > -1) {
|
||||
createdBudgetIds.splice(deleteIndex, 1);
|
||||
}
|
||||
|
||||
// Step 14: Verify deletion
|
||||
const verifyDeleteResponse = await authedFetch('/budgets', {
|
||||
method: 'GET',
|
||||
token: authToken,
|
||||
});
|
||||
const verifyDeleteResponse = await getRequest()
|
||||
.get('/api/budgets')
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(verifyDeleteResponse.status).toBe(200);
|
||||
const verifyDeleteData = await verifyDeleteResponse.json();
|
||||
expect(verifyDeleteData.data.length).toBe(1); // Only monthly budget remains
|
||||
expect(verifyDeleteResponse.body.data.length).toBe(1); // Only monthly budget remains
|
||||
|
||||
const deletedBudget = verifyDeleteData.data.find(
|
||||
(b: { budget_id: number }) => b.budget_id === weeklyBudgetData.data.budget_id,
|
||||
const deletedBudget = verifyDeleteResponse.body.data.find(
|
||||
(b: { budget_id: number }) => b.budget_id === weeklyBudgetResponse.body.data.budget_id,
|
||||
);
|
||||
expect(deletedBudget).toBeUndefined();
|
||||
|
||||
// Step 15: Delete account
|
||||
const deleteAccountResponse = await apiClient.deleteUserAccount(userPassword, {
|
||||
tokenOverride: authToken,
|
||||
});
|
||||
const deleteAccountResponse = await getRequest()
|
||||
.delete('/api/users/account')
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send({ password: userPassword });
|
||||
|
||||
expect(deleteAccountResponse.status).toBe(200);
|
||||
userId = null;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* Tests the complete flow from user registration to watching items and viewing best prices.
|
||||
*/
|
||||
import { describe, it, expect, afterAll } from 'vitest';
|
||||
import * as apiClient from '../../services/apiClient';
|
||||
import supertest from 'supertest';
|
||||
import { cleanupDb } from '../utils/cleanup';
|
||||
import { poll } from '../utils/poll';
|
||||
import { getPool } from '../../services/db/connection.db';
|
||||
@@ -13,35 +13,16 @@ import {
|
||||
cleanupStoreLocations,
|
||||
type CreatedStoreLocation,
|
||||
} from '../utils/storeHelpers';
|
||||
import { getServerUrl } from '../setup/e2e-global-setup';
|
||||
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
|
||||
const API_BASE_URL = process.env.VITE_API_BASE_URL || 'http://localhost:3000/api';
|
||||
|
||||
// Helper to make authenticated API calls
|
||||
const authedFetch = async (
|
||||
path: string,
|
||||
options: RequestInit & { token?: string } = {},
|
||||
): Promise<Response> => {
|
||||
const { token, ...fetchOptions } = options;
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
...(fetchOptions.headers as Record<string, string>),
|
||||
};
|
||||
|
||||
if (token) {
|
||||
headers['Authorization'] = `Bearer ${token}`;
|
||||
}
|
||||
|
||||
return fetch(`${API_BASE_URL}${path}`, {
|
||||
...fetchOptions,
|
||||
headers,
|
||||
});
|
||||
};
|
||||
|
||||
describe('E2E Deals and Price Tracking Journey', () => {
|
||||
// Create a getter function that returns supertest instance with the app
|
||||
const getRequest = () => supertest(getServerUrl());
|
||||
|
||||
const uniqueId = Date.now();
|
||||
const userEmail = `deals-e2e-${uniqueId}@example.com`;
|
||||
const userPassword = 'StrongDealsPassword123!';
|
||||
@@ -98,87 +79,70 @@ describe('E2E Deals and Price Tracking Journey', () => {
|
||||
// will support both category names and IDs in the watched items API.
|
||||
|
||||
// Get all available categories
|
||||
const categoriesResponse = await authedFetch('/categories', {
|
||||
method: 'GET',
|
||||
});
|
||||
const categoriesResponse = await getRequest().get('/api/categories');
|
||||
expect(categoriesResponse.status).toBe(200);
|
||||
const categoriesData = await categoriesResponse.json();
|
||||
expect(categoriesData.success).toBe(true);
|
||||
expect(categoriesData.data.length).toBeGreaterThan(0);
|
||||
expect(categoriesResponse.body.success).toBe(true);
|
||||
expect(categoriesResponse.body.data.length).toBeGreaterThan(0);
|
||||
|
||||
// Find "Dairy & Eggs" category by name using the lookup endpoint
|
||||
const categoryLookupResponse = await authedFetch(
|
||||
'/categories/lookup?name=' + encodeURIComponent('Dairy & Eggs'),
|
||||
{
|
||||
method: 'GET',
|
||||
},
|
||||
const categoryLookupResponse = await getRequest().get(
|
||||
'/api/categories/lookup?name=' + encodeURIComponent('Dairy & Eggs'),
|
||||
);
|
||||
expect(categoryLookupResponse.status).toBe(200);
|
||||
const categoryLookupData = await categoryLookupResponse.json();
|
||||
expect(categoryLookupData.success).toBe(true);
|
||||
expect(categoryLookupData.data.name).toBe('Dairy & Eggs');
|
||||
expect(categoryLookupResponse.body.success).toBe(true);
|
||||
expect(categoryLookupResponse.body.data.name).toBe('Dairy & Eggs');
|
||||
|
||||
const dairyEggsCategoryId = categoryLookupData.data.category_id;
|
||||
const dairyEggsCategoryId = categoryLookupResponse.body.data.category_id;
|
||||
expect(dairyEggsCategoryId).toBeGreaterThan(0);
|
||||
|
||||
// Verify we can retrieve the category by ID
|
||||
const categoryByIdResponse = await authedFetch(`/categories/${dairyEggsCategoryId}`, {
|
||||
method: 'GET',
|
||||
});
|
||||
const categoryByIdResponse = await getRequest().get(`/api/categories/${dairyEggsCategoryId}`);
|
||||
expect(categoryByIdResponse.status).toBe(200);
|
||||
const categoryByIdData = await categoryByIdResponse.json();
|
||||
expect(categoryByIdData.success).toBe(true);
|
||||
expect(categoryByIdData.data.category_id).toBe(dairyEggsCategoryId);
|
||||
expect(categoryByIdData.data.name).toBe('Dairy & Eggs');
|
||||
expect(categoryByIdResponse.body.success).toBe(true);
|
||||
expect(categoryByIdResponse.body.data.category_id).toBe(dairyEggsCategoryId);
|
||||
expect(categoryByIdResponse.body.data.name).toBe('Dairy & Eggs');
|
||||
|
||||
// Look up other category IDs we'll need
|
||||
const bakeryResponse = await authedFetch(
|
||||
'/categories/lookup?name=' + encodeURIComponent('Bakery & Bread'),
|
||||
{ method: 'GET' },
|
||||
const bakeryResponse = await getRequest().get(
|
||||
'/api/categories/lookup?name=' + encodeURIComponent('Bakery & Bread'),
|
||||
);
|
||||
const bakeryData = await bakeryResponse.json();
|
||||
const bakeryCategoryId = bakeryData.data.category_id;
|
||||
const bakeryCategoryId = bakeryResponse.body.data.category_id;
|
||||
|
||||
const beveragesResponse = await authedFetch('/categories/lookup?name=Beverages', {
|
||||
method: 'GET',
|
||||
});
|
||||
const beveragesData = await beveragesResponse.json();
|
||||
const beveragesCategoryId = beveragesData.data.category_id;
|
||||
const beveragesResponse = await getRequest().get('/api/categories/lookup?name=Beverages');
|
||||
const beveragesCategoryId = beveragesResponse.body.data.category_id;
|
||||
|
||||
const produceResponse = await authedFetch(
|
||||
'/categories/lookup?name=' + encodeURIComponent('Fruits & Vegetables'),
|
||||
{ method: 'GET' },
|
||||
const produceResponse = await getRequest().get(
|
||||
'/api/categories/lookup?name=' + encodeURIComponent('Fruits & Vegetables'),
|
||||
);
|
||||
const produceData = await produceResponse.json();
|
||||
const produceCategoryId = produceData.data.category_id;
|
||||
const produceCategoryId = produceResponse.body.data.category_id;
|
||||
|
||||
const meatResponse = await authedFetch(
|
||||
'/categories/lookup?name=' + encodeURIComponent('Meat & Seafood'),
|
||||
{ method: 'GET' },
|
||||
const meatResponse = await getRequest().get(
|
||||
'/api/categories/lookup?name=' + encodeURIComponent('Meat & Seafood'),
|
||||
);
|
||||
const meatData = await meatResponse.json();
|
||||
const meatCategoryId = meatData.data.category_id;
|
||||
const meatCategoryId = meatResponse.body.data.category_id;
|
||||
|
||||
// NOTE: The watched items API now uses category_id (number) as of Phase 3.
|
||||
// Category names are no longer accepted. Use the category discovery endpoints
|
||||
// to look up category IDs before creating watched items.
|
||||
|
||||
// Step 1: Register a new user
|
||||
const registerResponse = await apiClient.registerUser(
|
||||
userEmail,
|
||||
userPassword,
|
||||
'Deals E2E User',
|
||||
);
|
||||
const registerResponse = await getRequest().post('/api/auth/register').send({
|
||||
email: userEmail,
|
||||
password: userPassword,
|
||||
full_name: 'Deals E2E User',
|
||||
});
|
||||
expect(registerResponse.status).toBe(201);
|
||||
|
||||
// Step 2: Login to get auth token
|
||||
const { response: loginResponse, responseBody: loginResponseBody } = await poll(
|
||||
async () => {
|
||||
const response = await apiClient.loginUser(userEmail, userPassword, false);
|
||||
const responseBody = response.ok ? await response.clone().json() : {};
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: userEmail, password: userPassword, rememberMe: false });
|
||||
const responseBody = response.status === 200 ? response.body : {};
|
||||
return { response, responseBody };
|
||||
},
|
||||
(result) => result.response.ok,
|
||||
(result) => result.response.status === 200,
|
||||
{ timeout: 10000, interval: 1000, description: 'user login after registration' },
|
||||
);
|
||||
|
||||
@@ -280,18 +244,16 @@ describe('E2E Deals and Price Tracking Journey', () => {
|
||||
);
|
||||
|
||||
// Step 4: Add items to watch list (using category_id from lookups above)
|
||||
const watchItem1Response = await authedFetch('/users/watched-items', {
|
||||
method: 'POST',
|
||||
token: authToken,
|
||||
body: JSON.stringify({
|
||||
const watchItem1Response = await getRequest()
|
||||
.post('/api/users/watched-items')
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send({
|
||||
itemName: 'E2E Milk 2%',
|
||||
category_id: dairyEggsCategoryId,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
expect(watchItem1Response.status).toBe(201);
|
||||
const watchItem1Data = await watchItem1Response.json();
|
||||
expect(watchItem1Data.data.name).toBe('E2E Milk 2%');
|
||||
expect(watchItem1Response.body.data.name).toBe('E2E Milk 2%');
|
||||
|
||||
// Add more items to watch list
|
||||
const itemsToWatch = [
|
||||
@@ -300,47 +262,42 @@ describe('E2E Deals and Price Tracking Journey', () => {
|
||||
];
|
||||
|
||||
for (const item of itemsToWatch) {
|
||||
const response = await authedFetch('/users/watched-items', {
|
||||
method: 'POST',
|
||||
token: authToken,
|
||||
body: JSON.stringify(item),
|
||||
});
|
||||
const response = await getRequest()
|
||||
.post('/api/users/watched-items')
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send(item);
|
||||
expect(response.status).toBe(201);
|
||||
}
|
||||
|
||||
// Step 5: View all watched items
|
||||
const watchedListResponse = await authedFetch('/users/watched-items', {
|
||||
method: 'GET',
|
||||
token: authToken,
|
||||
});
|
||||
const watchedListResponse = await getRequest()
|
||||
.get('/api/users/watched-items')
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(watchedListResponse.status).toBe(200);
|
||||
const watchedListData = await watchedListResponse.json();
|
||||
expect(watchedListData.data.length).toBeGreaterThanOrEqual(3);
|
||||
expect(watchedListResponse.body.data.length).toBeGreaterThanOrEqual(3);
|
||||
|
||||
// Find our watched items
|
||||
const watchedMilk = watchedListData.data.find(
|
||||
const watchedMilk = watchedListResponse.body.data.find(
|
||||
(item: { name: string }) => item.name === 'E2E Milk 2%',
|
||||
);
|
||||
expect(watchedMilk).toBeDefined();
|
||||
expect(watchedMilk.category_id).toBe(dairyEggsCategoryId);
|
||||
|
||||
// Step 6: Get best prices for watched items
|
||||
const bestPricesResponse = await authedFetch('/deals/best-watched-prices', {
|
||||
method: 'GET',
|
||||
token: authToken,
|
||||
});
|
||||
const bestPricesResponse = await getRequest()
|
||||
.get('/api/deals/best-watched-prices')
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(bestPricesResponse.status).toBe(200);
|
||||
const bestPricesData = await bestPricesResponse.json();
|
||||
expect(bestPricesData.success).toBe(true);
|
||||
expect(bestPricesResponse.body.success).toBe(true);
|
||||
|
||||
// Verify we got deals for our watched items
|
||||
expect(Array.isArray(bestPricesData.data)).toBe(true);
|
||||
expect(Array.isArray(bestPricesResponse.body.data)).toBe(true);
|
||||
|
||||
// Find the milk deal and verify it's the best price (Store 2 at $4.99)
|
||||
if (bestPricesData.data.length > 0) {
|
||||
const milkDeal = bestPricesData.data.find(
|
||||
if (bestPricesResponse.body.data.length > 0) {
|
||||
const milkDeal = bestPricesResponse.body.data.find(
|
||||
(deal: { item_name: string }) => deal.item_name === 'E2E Milk 2%',
|
||||
);
|
||||
|
||||
@@ -356,38 +313,39 @@ describe('E2E Deals and Price Tracking Journey', () => {
|
||||
|
||||
// Step 8: Remove an item from watch list
|
||||
const milkMasterItemId = createdMasterItemIds[0];
|
||||
const removeResponse = await authedFetch(`/users/watched-items/${milkMasterItemId}`, {
|
||||
method: 'DELETE',
|
||||
token: authToken,
|
||||
});
|
||||
const removeResponse = await getRequest()
|
||||
.delete(`/api/users/watched-items/${milkMasterItemId}`)
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(removeResponse.status).toBe(204);
|
||||
|
||||
// Step 9: Verify item was removed
|
||||
const updatedWatchedListResponse = await authedFetch('/users/watched-items', {
|
||||
method: 'GET',
|
||||
token: authToken,
|
||||
});
|
||||
const updatedWatchedListResponse = await getRequest()
|
||||
.get('/api/users/watched-items')
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
|
||||
expect(updatedWatchedListResponse.status).toBe(200);
|
||||
const updatedWatchedListData = await updatedWatchedListResponse.json();
|
||||
|
||||
const milkStillWatched = updatedWatchedListData.data.find(
|
||||
const milkStillWatched = updatedWatchedListResponse.body.data.find(
|
||||
(item: { item_name: string }) => item.item_name === 'E2E Milk 2%',
|
||||
);
|
||||
expect(milkStillWatched).toBeUndefined();
|
||||
|
||||
// Step 10: Verify another user cannot see our watched items
|
||||
const otherUserEmail = `other-deals-e2e-${uniqueId}@example.com`;
|
||||
await apiClient.registerUser(otherUserEmail, userPassword, 'Other Deals User');
|
||||
await getRequest()
|
||||
.post('/api/auth/register')
|
||||
.send({ email: otherUserEmail, password: userPassword, full_name: 'Other Deals User' });
|
||||
|
||||
const { responseBody: otherLoginData } = await poll(
|
||||
async () => {
|
||||
const response = await apiClient.loginUser(otherUserEmail, userPassword, false);
|
||||
const responseBody = response.ok ? await response.clone().json() : {};
|
||||
const response = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: otherUserEmail, password: userPassword, rememberMe: false });
|
||||
const responseBody = response.status === 200 ? response.body : {};
|
||||
return { response, responseBody };
|
||||
},
|
||||
(result) => result.response.ok,
|
||||
(result) => result.response.status === 200,
|
||||
{ timeout: 10000, interval: 1000, description: 'other user login' },
|
||||
);
|
||||
|
||||
@@ -395,32 +353,29 @@ describe('E2E Deals and Price Tracking Journey', () => {
|
||||
const otherUserId = otherLoginData.data.userprofile.user.user_id;
|
||||
|
||||
// Other user's watched items should be empty
|
||||
const otherWatchedResponse = await authedFetch('/users/watched-items', {
|
||||
method: 'GET',
|
||||
token: otherToken,
|
||||
});
|
||||
const otherWatchedResponse = await getRequest()
|
||||
.get('/api/users/watched-items')
|
||||
.set('Authorization', `Bearer ${otherToken}`);
|
||||
|
||||
expect(otherWatchedResponse.status).toBe(200);
|
||||
const otherWatchedData = await otherWatchedResponse.json();
|
||||
expect(otherWatchedData.data.length).toBe(0);
|
||||
expect(otherWatchedResponse.body.data.length).toBe(0);
|
||||
|
||||
// Other user's deals should be empty
|
||||
const otherDealsResponse = await authedFetch('/deals/best-watched-prices', {
|
||||
method: 'GET',
|
||||
token: otherToken,
|
||||
});
|
||||
const otherDealsResponse = await getRequest()
|
||||
.get('/api/deals/best-watched-prices')
|
||||
.set('Authorization', `Bearer ${otherToken}`);
|
||||
|
||||
expect(otherDealsResponse.status).toBe(200);
|
||||
const otherDealsData = await otherDealsResponse.json();
|
||||
expect(otherDealsData.data.length).toBe(0);
|
||||
expect(otherDealsResponse.body.data.length).toBe(0);
|
||||
|
||||
// Clean up other user
|
||||
await cleanupDb({ userIds: [otherUserId] });
|
||||
|
||||
// Step 11: Delete account
|
||||
const deleteAccountResponse = await apiClient.deleteUserAccount(userPassword, {
|
||||
tokenOverride: authToken,
|
||||
});
|
||||
const deleteAccountResponse = await getRequest()
|
||||
.delete('/api/users/account')
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.send({ password: userPassword });
|
||||
|
||||
expect(deleteAccountResponse.status).toBe(200);
|
||||
userId = null;
|
||||
|
||||
@@ -1,18 +1,22 @@
|
||||
// src/tests/e2e/flyer-upload.e2e.test.ts
|
||||
import { describe, it, expect, afterAll } from 'vitest';
|
||||
import supertest from 'supertest';
|
||||
import crypto from 'crypto';
|
||||
import * as apiClient from '../../services/apiClient';
|
||||
import { getPool } from '../../services/db/connection.db';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { cleanupDb } from '../utils/cleanup';
|
||||
import { poll } from '../utils/poll';
|
||||
import { getServerUrl } from '../setup/e2e-global-setup';
|
||||
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
|
||||
describe('E2E Flyer Upload and Processing Workflow', () => {
|
||||
// Create a getter function that returns supertest instance with the app
|
||||
const getRequest = () => supertest(getServerUrl());
|
||||
|
||||
const uniqueId = Date.now();
|
||||
const userEmail = `e2e-uploader-${uniqueId}@example.com`;
|
||||
const userPassword = 'StrongPassword123!';
|
||||
@@ -33,19 +37,20 @@ describe('E2E Flyer Upload and Processing Workflow', () => {
|
||||
|
||||
it('should allow a user to upload a flyer and wait for processing to complete', async () => {
|
||||
// 1. Register a new user
|
||||
const registerResponse = await apiClient.registerUser(
|
||||
userEmail,
|
||||
userPassword,
|
||||
'E2E Flyer Uploader',
|
||||
);
|
||||
const registerResponse = await getRequest().post('/api/auth/register').send({
|
||||
email: userEmail,
|
||||
password: userPassword,
|
||||
full_name: 'E2E Flyer Uploader',
|
||||
});
|
||||
expect(registerResponse.status).toBe(201);
|
||||
|
||||
// 2. Login to get the access token
|
||||
const loginResponse = await apiClient.loginUser(userEmail, userPassword, false);
|
||||
const loginResponse = await getRequest()
|
||||
.post('/api/auth/login')
|
||||
.send({ email: userEmail, password: userPassword, rememberMe: false });
|
||||
expect(loginResponse.status).toBe(200);
|
||||
const loginResponseBody = await loginResponse.json();
|
||||
authToken = loginResponseBody.data.token;
|
||||
userId = loginResponseBody.data.userprofile.user.user_id;
|
||||
authToken = loginResponse.body.data.token;
|
||||
userId = loginResponse.body.data.userprofile.user.user_id;
|
||||
expect(authToken).toBeDefined();
|
||||
|
||||
// 3. Prepare the flyer file
|
||||
@@ -69,29 +74,27 @@ describe('E2E Flyer Upload and Processing Workflow', () => {
|
||||
]);
|
||||
}
|
||||
|
||||
// Create a File object for the apiClient
|
||||
// FIX: The Node.js `Buffer` type can be incompatible with the web `File` API's
|
||||
// expected `BlobPart` type in some TypeScript configurations. Explicitly creating
|
||||
// a `Uint8Array` from the buffer ensures compatibility and resolves the type error.
|
||||
// `Uint8Array` is a valid `BufferSource`, which is a valid `BlobPart`.
|
||||
const flyerFile = new File([new Uint8Array(fileBuffer)], fileName, { type: 'image/jpeg' });
|
||||
|
||||
// Calculate checksum (required by the API)
|
||||
const checksum = crypto.createHash('sha256').update(fileBuffer).digest('hex');
|
||||
|
||||
// 4. Upload the flyer
|
||||
const uploadResponse = await apiClient.uploadAndProcessFlyer(flyerFile, checksum, authToken);
|
||||
const uploadResponse = await getRequest()
|
||||
.post('/api/flyers/upload')
|
||||
.set('Authorization', `Bearer ${authToken}`)
|
||||
.attach('flyer', fileBuffer, fileName)
|
||||
.field('checksum', checksum);
|
||||
|
||||
expect(uploadResponse.status).toBe(202);
|
||||
const uploadResponseBody = await uploadResponse.json();
|
||||
const jobId = uploadResponseBody.data.jobId;
|
||||
const jobId = uploadResponse.body.data.jobId;
|
||||
expect(jobId).toBeDefined();
|
||||
|
||||
// 5. Poll for job completion using the new utility
|
||||
const jobStatusResponse = await poll(
|
||||
async () => {
|
||||
const statusResponse = await apiClient.getJobStatus(jobId, authToken);
|
||||
return statusResponse.json();
|
||||
const statusResponse = await getRequest()
|
||||
.get(`/api/jobs/${jobId}`)
|
||||
.set('Authorization', `Bearer ${authToken}`);
|
||||
return statusResponse.body;
|
||||
},
|
||||
(responseBody) =>
|
||||
responseBody.data.state === 'completed' || responseBody.data.state === 'failed',
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user