Compare commits
58 Commits
v0.9.88
...
a14816c8ee
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a14816c8ee | ||
|
|
08b220e29c | ||
|
|
d41a3f1887 | ||
| 1f6cdc62d7 | |||
|
|
978c63bacd | ||
| 544eb7ae3c | |||
|
|
f6839f6e14 | ||
| 3fac29436a | |||
|
|
56f45c9301 | ||
| 83460abce4 | |||
|
|
1b084b2ba4 | ||
| 0ea034bdc8 | |||
|
|
fc9e27078a | ||
| fb8cbe8007 | |||
| f49f786c23 | |||
|
|
dd31141d4e | ||
| 8073094760 | |||
|
|
33a1e146ab | ||
| 4f8216db77 | |||
|
|
42d605d19f | ||
| 749350df7f | |||
|
|
ac085100fe | ||
| ce4ecd1268 | |||
|
|
a57cfc396b | ||
| 987badbf8d | |||
|
|
d38fcd21c1 | ||
| 6e36cc3b07 | |||
|
|
62a8a8bf4b | ||
| 96038cfcf4 | |||
|
|
981214fdd0 | ||
| 92b0138108 | |||
|
|
27f0255240 | ||
| 4e06dde9e1 | |||
|
|
b9a0e5b82c | ||
| bb7fe8dc2c | |||
|
|
81f1f2250b | ||
| c6c90bb615 | |||
|
|
60489a626b | ||
| 3c63e1ecbb | |||
|
|
acbcb39cbe | ||
| a87a0b6af1 | |||
|
|
abdc3cb6db | ||
| 7a1bd50119 | |||
|
|
87d75d0571 | ||
| faf2900c28 | |||
|
|
5258efc179 | ||
| 2a5cc5bb51 | |||
|
|
8eaee2844f | ||
| 440a19c3a7 | |||
| 4ae6d84240 | |||
|
|
5870e5c614 | ||
| 2e7ebbd9ed | |||
|
|
dc3fa21359 | ||
| 11aeac5edd | |||
|
|
f6c0c082bc | ||
| 4e22213cd1 | |||
|
|
9815eb3686 | ||
| 2bf4a7c1e6 |
@@ -87,7 +87,18 @@
|
||||
"Bash(docker ps:*)",
|
||||
"Bash(find:*)",
|
||||
"Bash(\"/c/Users/games3/.local/bin/uvx.exe\" markitdown-mcp --help)",
|
||||
"Bash(git stash:*)"
|
||||
"Bash(git stash:*)",
|
||||
"Bash(ping:*)",
|
||||
"Bash(tee:*)",
|
||||
"Bash(timeout 1800 podman exec flyer-crawler-dev npm run test:unit:*)",
|
||||
"mcp__filesystem__edit_file",
|
||||
"Bash(timeout 300 tail:*)",
|
||||
"mcp__filesystem__list_allowed_directories",
|
||||
"mcp__memory__add_observations",
|
||||
"Bash(ssh:*)",
|
||||
"mcp__redis__list",
|
||||
"Read(//d/gitea/bugsink-mcp/**)",
|
||||
"Bash(d:/nodejs/npm.cmd install)"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
27
.env.example
27
.env.example
@@ -41,6 +41,14 @@ FRONTEND_URL=http://localhost:3000
|
||||
# REQUIRED: Secret key for signing JWT tokens (generate a random 64+ character string)
|
||||
JWT_SECRET=your-super-secret-jwt-key-change-this-in-production
|
||||
|
||||
# OAuth Providers (Optional - enable social login)
|
||||
# Google OAuth - https://console.cloud.google.com/apis/credentials
|
||||
GOOGLE_CLIENT_ID=
|
||||
GOOGLE_CLIENT_SECRET=
|
||||
# GitHub OAuth - https://github.com/settings/developers
|
||||
GITHUB_CLIENT_ID=
|
||||
GITHUB_CLIENT_SECRET=
|
||||
|
||||
# ===================
|
||||
# AI/ML Services
|
||||
# ===================
|
||||
@@ -75,3 +83,22 @@ CLEANUP_WORKER_CONCURRENCY=10
|
||||
|
||||
# Worker lock duration in milliseconds (default: 2 minutes)
|
||||
WORKER_LOCK_DURATION=120000
|
||||
|
||||
# ===================
|
||||
# Error Tracking (ADR-015)
|
||||
# ===================
|
||||
# Sentry-compatible error tracking via Bugsink (self-hosted)
|
||||
# DSNs are created in Bugsink UI at http://localhost:8000 (dev) or your production URL
|
||||
# Backend DSN - for Express/Node.js errors
|
||||
SENTRY_DSN=
|
||||
# Frontend DSN - for React/browser errors (uses VITE_ prefix)
|
||||
VITE_SENTRY_DSN=
|
||||
# Environment name for error grouping (defaults to NODE_ENV)
|
||||
SENTRY_ENVIRONMENT=development
|
||||
VITE_SENTRY_ENVIRONMENT=development
|
||||
# Enable/disable error tracking (default: true)
|
||||
SENTRY_ENABLED=true
|
||||
VITE_SENTRY_ENABLED=true
|
||||
# Enable debug mode for SDK troubleshooting (default: false)
|
||||
SENTRY_DEBUG=false
|
||||
VITE_SENTRY_DEBUG=false
|
||||
|
||||
@@ -63,8 +63,8 @@ jobs:
|
||||
- name: Check for Production Database Schema Changes
|
||||
env:
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
run: |
|
||||
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
|
||||
@@ -98,6 +98,9 @@ jobs:
|
||||
VITE_APP_VERSION="$(date +'%Y%m%d-%H%M'):$(git rev-parse --short HEAD):$PACKAGE_VERSION" \
|
||||
VITE_APP_COMMIT_URL="$GITEA_SERVER_URL/${{ gitea.repository }}/commit/${{ gitea.sha }}" \
|
||||
VITE_APP_COMMIT_MESSAGE="$COMMIT_MESSAGE" \
|
||||
VITE_SENTRY_DSN="${{ secrets.VITE_SENTRY_DSN }}" \
|
||||
VITE_SENTRY_ENVIRONMENT="production" \
|
||||
VITE_SENTRY_ENABLED="true" \
|
||||
VITE_API_BASE_URL=/api VITE_API_KEY=${{ secrets.VITE_GOOGLE_GENAI_API_KEY }} npm run build
|
||||
|
||||
- name: Deploy Application to Production Server
|
||||
@@ -114,8 +117,8 @@ jobs:
|
||||
env:
|
||||
# --- Production Secrets Injection ---
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
# Explicitly use database 0 for production (test uses database 1)
|
||||
REDIS_URL: 'redis://localhost:6379/0'
|
||||
@@ -130,6 +133,15 @@ jobs:
|
||||
SMTP_USER: ''
|
||||
SMTP_PASS: ''
|
||||
SMTP_FROM_EMAIL: 'noreply@flyer-crawler.projectium.com'
|
||||
# OAuth Providers
|
||||
GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
|
||||
GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
|
||||
GITHUB_CLIENT_ID: ${{ secrets.GH_CLIENT_ID }}
|
||||
GITHUB_CLIENT_SECRET: ${{ secrets.GH_CLIENT_SECRET }}
|
||||
# Sentry/Bugsink Error Tracking (ADR-015)
|
||||
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
|
||||
SENTRY_ENVIRONMENT: 'production'
|
||||
SENTRY_ENABLED: 'true'
|
||||
run: |
|
||||
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
|
||||
echo "ERROR: One or more production database secrets (DB_HOST, DB_USER, DB_PASSWORD, DB_DATABASE_PROD) are not set."
|
||||
@@ -159,7 +171,7 @@ jobs:
|
||||
else
|
||||
echo "Version mismatch (Running: $RUNNING_VERSION -> Deployed: $NEW_VERSION) or app not running. Reloading PM2..."
|
||||
fi
|
||||
pm2 startOrReload ecosystem.config.cjs --env production --update-env && pm2 save
|
||||
pm2 startOrReload ecosystem.config.cjs --update-env && pm2 save
|
||||
echo "Production backend server reloaded successfully."
|
||||
else
|
||||
echo "Version $NEW_VERSION is already running. Skipping PM2 reload."
|
||||
|
||||
@@ -121,10 +121,11 @@ jobs:
|
||||
env:
|
||||
# --- Database credentials for the test suite ---
|
||||
# These are injected from Gitea secrets into the runner's environment.
|
||||
# CRITICAL: Use TEST-specific credentials that have CREATE privileges on the public schema.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_NAME: 'flyer-crawler-test' # Explicitly set for tests
|
||||
DB_USER: ${{ secrets.DB_USER_TEST }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
|
||||
|
||||
# --- Redis credentials for the test suite ---
|
||||
# CRITICAL: Use Redis database 1 to isolate tests from production (which uses db 0).
|
||||
@@ -240,7 +241,19 @@ jobs:
|
||||
# Run c8: read raw files from the temp dir, and output an Istanbul JSON report.
|
||||
# We only generate the 'json' report here because it's all nyc needs for merging.
|
||||
echo "Server coverage report about to be generated..."
|
||||
npx c8 report --exclude='**/*.test.ts' --exclude='**/tests/**' --exclude='**/mocks/**' --reporter=json --temp-directory .coverage/tmp/integration-server --reports-dir .coverage/integration-server
|
||||
npx c8 report \
|
||||
--include='src/**' \
|
||||
--exclude='**/*.test.ts' \
|
||||
--exclude='**/*.test.tsx' \
|
||||
--exclude='**/tests/**' \
|
||||
--exclude='**/mocks/**' \
|
||||
--exclude='hostexecutor/**' \
|
||||
--exclude='scripts/**' \
|
||||
--exclude='*.config.js' \
|
||||
--exclude='*.config.ts' \
|
||||
--reporter=json \
|
||||
--temp-directory .coverage/tmp/integration-server \
|
||||
--reports-dir .coverage/integration-server
|
||||
echo "Server coverage report generated. Verifying existence:"
|
||||
ls -l .coverage/integration-server/coverage-final.json
|
||||
|
||||
@@ -280,12 +293,18 @@ jobs:
|
||||
--reporter=html \
|
||||
--report-dir .coverage/ \
|
||||
--temp-dir "$NYC_SOURCE_DIR" \
|
||||
--include "src/**" \
|
||||
--exclude "**/*.test.ts" \
|
||||
--exclude "**/*.test.tsx" \
|
||||
--exclude "**/tests/**" \
|
||||
--exclude "**/mocks/**" \
|
||||
--exclude "**/index.tsx" \
|
||||
--exclude "**/vite-env.d.ts" \
|
||||
--exclude "**/vitest.setup.ts"
|
||||
--exclude "**/vitest.setup.ts" \
|
||||
--exclude "hostexecutor/**" \
|
||||
--exclude "scripts/**" \
|
||||
--exclude "*.config.js" \
|
||||
--exclude "*.config.ts"
|
||||
|
||||
# Re-enable secret masking for subsequent steps.
|
||||
echo "::secret-masking::"
|
||||
@@ -310,10 +329,11 @@ jobs:
|
||||
- name: Check for Test Database Schema Changes
|
||||
env:
|
||||
# Use test database credentials for this check.
|
||||
# CRITICAL: Use TEST-specific credentials that have CREATE privileges on the public schema.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # This is used by psql
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }} # This is used by the application
|
||||
DB_USER: ${{ secrets.DB_USER_TEST }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
|
||||
run: |
|
||||
# Fail-fast check to ensure secrets are configured in Gitea.
|
||||
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
|
||||
@@ -368,6 +388,9 @@ jobs:
|
||||
VITE_APP_VERSION="$(date +'%Y%m%d-%H%M'):$(git rev-parse --short HEAD):$PACKAGE_VERSION" \
|
||||
VITE_APP_COMMIT_URL="$GITEA_SERVER_URL/${{ gitea.repository }}/commit/${{ gitea.sha }}" \
|
||||
VITE_APP_COMMIT_MESSAGE="$COMMIT_MESSAGE" \
|
||||
VITE_SENTRY_DSN="${{ secrets.VITE_SENTRY_DSN_TEST }}" \
|
||||
VITE_SENTRY_ENVIRONMENT="test" \
|
||||
VITE_SENTRY_ENABLED="true" \
|
||||
VITE_API_BASE_URL="https://flyer-crawler-test.projectium.com/api" VITE_API_KEY=${{ secrets.VITE_GOOGLE_GENAI_API_KEY_TEST }} npm run build
|
||||
|
||||
- name: Deploy Application to Test Server
|
||||
@@ -406,9 +429,10 @@ jobs:
|
||||
# Your Node.js application will read these directly from `process.env`.
|
||||
|
||||
# Database Credentials
|
||||
# CRITICAL: Use TEST-specific credentials that have CREATE privileges on the public schema.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_TEST }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
|
||||
|
||||
# Redis Credentials (use database 1 to isolate from production)
|
||||
@@ -428,6 +452,10 @@ jobs:
|
||||
SMTP_USER: '' # Using MailHog, no auth needed
|
||||
SMTP_PASS: '' # Using MailHog, no auth needed
|
||||
SMTP_FROM_EMAIL: 'noreply@flyer-crawler-test.projectium.com'
|
||||
# Sentry/Bugsink Error Tracking (ADR-015)
|
||||
SENTRY_DSN: ${{ secrets.SENTRY_DSN_TEST }}
|
||||
SENTRY_ENVIRONMENT: 'test'
|
||||
SENTRY_ENABLED: 'true'
|
||||
|
||||
run: |
|
||||
# Fail-fast check to ensure secrets are configured in Gitea.
|
||||
@@ -451,10 +479,11 @@ jobs:
|
||||
echo "Cleaning up errored or stopped PM2 processes..."
|
||||
node -e "const exec = require('child_process').execSync; try { const list = JSON.parse(exec('pm2 jlist').toString()); list.forEach(p => { if (p.pm2_env.status === 'errored' || p.pm2_env.status === 'stopped') { console.log('Deleting ' + p.pm2_env.status + ' process: ' + p.name + ' (' + p.pm2_env.pm_id + ')'); try { exec('pm2 delete ' + p.pm2_env.pm_id); } catch(e) { console.error('Failed to delete ' + p.pm2_env.pm_id); } } }); } catch (e) { console.error('Error cleaning up processes:', e); }"
|
||||
|
||||
# Use `startOrReload` with the ecosystem file. This is the standard, idempotent way to deploy.
|
||||
# It will START the process if it's not running, or RELOAD it if it is.
|
||||
# Use `startOrReload` with the TEST ecosystem file. This starts test-specific processes
|
||||
# (flyer-crawler-api-test, flyer-crawler-worker-test, flyer-crawler-analytics-worker-test)
|
||||
# that run separately from production processes.
|
||||
# We also add `&& pm2 save` to persist the process list across server reboots.
|
||||
pm2 startOrReload ecosystem.config.cjs --env test --update-env && pm2 save
|
||||
pm2 startOrReload ecosystem-test.config.cjs --update-env && pm2 save
|
||||
echo "Test backend server reloaded successfully."
|
||||
|
||||
# After a successful deployment, update the schema hash in the database.
|
||||
|
||||
@@ -20,9 +20,9 @@ jobs:
|
||||
# Use production database credentials for this entire job.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_PORT: ${{ secrets.DB_PORT }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_NAME: ${{ secrets.DB_NAME_PROD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
|
||||
steps:
|
||||
- name: Validate Secrets
|
||||
|
||||
@@ -23,9 +23,9 @@ jobs:
|
||||
env:
|
||||
# Use production database credentials for this entire job.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }} # Used by the application
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
|
||||
@@ -23,9 +23,9 @@ jobs:
|
||||
env:
|
||||
# Use test database credentials for this entire job.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }} # Used by the application
|
||||
DB_USER: ${{ secrets.DB_USER_TEST }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
|
||||
@@ -22,8 +22,8 @@ jobs:
|
||||
env:
|
||||
# Use production database credentials for this entire job.
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
BACKUP_DIR: '/var/www/backups' # Define a dedicated directory for backups
|
||||
|
||||
|
||||
@@ -62,8 +62,8 @@ jobs:
|
||||
- name: Check for Production Database Schema Changes
|
||||
env:
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
run: |
|
||||
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
|
||||
@@ -113,8 +113,8 @@ jobs:
|
||||
env:
|
||||
# --- Production Secrets Injection ---
|
||||
DB_HOST: ${{ secrets.DB_HOST }}
|
||||
DB_USER: ${{ secrets.DB_USER }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
|
||||
DB_USER: ${{ secrets.DB_USER_PROD }}
|
||||
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
|
||||
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
|
||||
# Explicitly use database 0 for production (test uses database 1)
|
||||
REDIS_URL: 'redis://localhost:6379/0'
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -37,3 +37,4 @@ test-output.txt
|
||||
Thumbs.db
|
||||
.claude
|
||||
nul
|
||||
tmpclaude*
|
||||
|
||||
5
.nycrc.json
Normal file
5
.nycrc.json
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"text": {
|
||||
"maxCols": 200
|
||||
}
|
||||
}
|
||||
378
CLAUDE-MCP.md
Normal file
378
CLAUDE-MCP.md
Normal file
@@ -0,0 +1,378 @@
|
||||
# Claude Code MCP Configuration Guide
|
||||
|
||||
This document explains how to configure MCP (Model Context Protocol) servers for Claude Code, covering both the CLI and VS Code extension.
|
||||
|
||||
## The Two Config Files
|
||||
|
||||
Claude Code uses **two separate configuration files** for MCP servers. They must be kept in sync manually.
|
||||
|
||||
| File | Used By | Notes |
|
||||
| ------------------------- | ----------------------------- | ------------------------------------------- |
|
||||
| `~/.claude.json` | Claude CLI (`claude` command) | Requires `"type": "stdio"` in each server |
|
||||
| `~/.claude/settings.json` | VS Code Extension | Simpler format, supports `"disabled": true` |
|
||||
|
||||
**Important:** Changes to one file do NOT automatically sync to the other!
|
||||
|
||||
## File Locations (Windows)
|
||||
|
||||
```text
|
||||
C:\Users\<username>\.claude.json # CLI config
|
||||
C:\Users\<username>\.claude\settings.json # VS Code extension config
|
||||
```
|
||||
|
||||
## Config Format Differences
|
||||
|
||||
### VS Code Extension Format (`~/.claude/settings.json`)
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"server-name": {
|
||||
"command": "path/to/executable",
|
||||
"args": ["arg1", "arg2"],
|
||||
"env": {
|
||||
"ENV_VAR": "value"
|
||||
},
|
||||
"disabled": true // Optional - disable without removing
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### CLI Format (`~/.claude.json`)
|
||||
|
||||
The CLI config is a larger file with many settings. The `mcpServers` section is nested within it:
|
||||
|
||||
```json
|
||||
{
|
||||
"numStartups": 14,
|
||||
"installMethod": "global",
|
||||
// ... other settings ...
|
||||
"mcpServers": {
|
||||
"server-name": {
|
||||
"type": "stdio", // REQUIRED for CLI
|
||||
"command": "path/to/executable",
|
||||
"args": ["arg1", "arg2"],
|
||||
"env": {
|
||||
"ENV_VAR": "value"
|
||||
}
|
||||
}
|
||||
}
|
||||
// ... more settings ...
|
||||
}
|
||||
```
|
||||
|
||||
**Key difference:** CLI format requires `"type": "stdio"` in each server definition.
|
||||
|
||||
## Common MCP Server Examples
|
||||
|
||||
### Memory (Knowledge Graph)
|
||||
|
||||
```json
|
||||
// VS Code format
|
||||
"memory": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@modelcontextprotocol/server-memory"]
|
||||
}
|
||||
|
||||
// CLI format
|
||||
"memory": {
|
||||
"type": "stdio",
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@modelcontextprotocol/server-memory"],
|
||||
"env": {}
|
||||
}
|
||||
```
|
||||
|
||||
### Filesystem
|
||||
|
||||
```json
|
||||
// VS Code format
|
||||
"filesystem": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": [
|
||||
"c:\\Users\\<user>\\AppData\\Roaming\\npm\\node_modules\\@modelcontextprotocol\\server-filesystem\\dist\\index.js",
|
||||
"d:\\path\\to\\project"
|
||||
]
|
||||
}
|
||||
|
||||
// CLI format
|
||||
"filesystem": {
|
||||
"type": "stdio",
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": [
|
||||
"c:\\Users\\<user>\\AppData\\Roaming\\npm\\node_modules\\@modelcontextprotocol\\server-filesystem\\dist\\index.js",
|
||||
"d:\\path\\to\\project"
|
||||
],
|
||||
"env": {}
|
||||
}
|
||||
```
|
||||
|
||||
### Podman/Docker
|
||||
|
||||
```json
|
||||
// VS Code format
|
||||
"podman": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "podman-mcp-server@latest"],
|
||||
"env": {
|
||||
"DOCKER_HOST": "npipe:////./pipe/podman-machine-default"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Gitea
|
||||
|
||||
```json
|
||||
// VS Code format
|
||||
"gitea-myserver": {
|
||||
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
|
||||
"args": ["run", "-t", "stdio"],
|
||||
"env": {
|
||||
"GITEA_HOST": "https://gitea.example.com",
|
||||
"GITEA_ACCESS_TOKEN": "your-token-here"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Redis
|
||||
|
||||
```json
|
||||
// VS Code format
|
||||
"redis": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@modelcontextprotocol/server-redis", "redis://localhost:6379"]
|
||||
}
|
||||
```
|
||||
|
||||
### Bugsink (Error Tracking)
|
||||
|
||||
**Important:** Bugsink has a different API than Sentry. Use `bugsink-mcp`, NOT `sentry-selfhosted-mcp`.
|
||||
|
||||
**Note:** The `bugsink-mcp` npm package is NOT published. You must clone and build from source:
|
||||
|
||||
```bash
|
||||
# Clone and build bugsink-mcp
|
||||
git clone https://github.com/j-shelfwood/bugsink-mcp.git d:\gitea\bugsink-mcp
|
||||
cd d:\gitea\bugsink-mcp
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
```json
|
||||
// VS Code format (using locally built version)
|
||||
"bugsink": {
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "https://bugsink.example.com",
|
||||
"BUGSINK_TOKEN": "your-api-token"
|
||||
}
|
||||
}
|
||||
|
||||
// CLI format
|
||||
"bugsink": {
|
||||
"type": "stdio",
|
||||
"command": "d:\\nodejs\\node.exe",
|
||||
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
|
||||
"env": {
|
||||
"BUGSINK_URL": "https://bugsink.example.com",
|
||||
"BUGSINK_TOKEN": "your-api-token"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- GitHub: <https://github.com/j-shelfwood/bugsink-mcp>
|
||||
- Get token from Bugsink UI: Settings > API Tokens
|
||||
- **Do NOT use npx** - the package is not on npm
|
||||
|
||||
### Sentry (Cloud or Self-hosted)
|
||||
|
||||
For actual Sentry instances (not Bugsink), use:
|
||||
|
||||
```json
|
||||
"sentry": {
|
||||
"command": "D:\\nodejs\\npx.cmd",
|
||||
"args": ["-y", "@sentry/mcp-server"],
|
||||
"env": {
|
||||
"SENTRY_AUTH_TOKEN": "your-sentry-token"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Server Not Loading
|
||||
|
||||
1. **Check both config files** - Make sure the server is defined in both `~/.claude.json` AND `~/.claude/settings.json`
|
||||
|
||||
2. **Verify server order** - Servers load sequentially. Broken/slow servers can block others. Put important servers first.
|
||||
|
||||
3. **Check for timeout** - Each server has 30 seconds to connect. Slow npx downloads can cause timeouts.
|
||||
|
||||
4. **Fully restart VS Code** - Window reload is not enough. Close all VS Code windows and reopen.
|
||||
|
||||
### Verifying Configuration
|
||||
|
||||
**For CLI:**
|
||||
|
||||
```bash
|
||||
claude mcp list
|
||||
```
|
||||
|
||||
**For VS Code:**
|
||||
|
||||
1. Open VS Code
|
||||
2. View → Output
|
||||
3. Select "Claude" from the dropdown
|
||||
4. Look for MCP server connection logs
|
||||
|
||||
### Common Errors
|
||||
|
||||
| Error | Cause | Solution |
|
||||
| ------------------------------------ | ----------------------------- | --------------------------------------------------------------------------- |
|
||||
| `Connection timed out after 30000ms` | Server took too long to start | Move server earlier in config, or use pre-installed packages instead of npx |
|
||||
| `npm error 404 Not Found` | Package doesn't exist | Check package name spelling |
|
||||
| `The system cannot find the path` | Wrong executable path | Verify the command path exists |
|
||||
| `Connection closed` | Server crashed on startup | Check server logs, verify environment variables |
|
||||
|
||||
### Disabling Problem Servers
|
||||
|
||||
In `~/.claude/settings.json`, add `"disabled": true`:
|
||||
|
||||
```json
|
||||
"problem-server": {
|
||||
"command": "...",
|
||||
"args": ["..."],
|
||||
"disabled": true
|
||||
}
|
||||
```
|
||||
|
||||
**Note:** The CLI config (`~/.claude.json`) does not support the `disabled` flag. You must remove the server entirely from that file.
|
||||
|
||||
## Adding a New MCP Server
|
||||
|
||||
1. **Install/clone the MCP server** (if not using npx)
|
||||
|
||||
2. **Add to VS Code config** (`~/.claude/settings.json`):
|
||||
|
||||
```json
|
||||
"new-server": {
|
||||
"command": "path/to/command",
|
||||
"args": ["arg1", "arg2"],
|
||||
"env": { "VAR": "value" }
|
||||
}
|
||||
```
|
||||
|
||||
3. **Add to CLI config** (`~/.claude.json`) - find the `mcpServers` section:
|
||||
|
||||
```json
|
||||
"new-server": {
|
||||
"type": "stdio",
|
||||
"command": "path/to/command",
|
||||
"args": ["arg1", "arg2"],
|
||||
"env": { "VAR": "value" }
|
||||
}
|
||||
```
|
||||
|
||||
4. **Fully restart VS Code**
|
||||
|
||||
5. **Verify with `claude mcp list`**
|
||||
|
||||
## Quick Reference: Available MCP Servers
|
||||
|
||||
| Server | Package/Repo | Purpose |
|
||||
| ------------------- | -------------------------------------------------- | --------------------------- |
|
||||
| memory | `@modelcontextprotocol/server-memory` | Knowledge graph persistence |
|
||||
| filesystem | `@modelcontextprotocol/server-filesystem` | File system access |
|
||||
| redis | `@modelcontextprotocol/server-redis` | Redis cache inspection |
|
||||
| postgres | `@modelcontextprotocol/server-postgres` | PostgreSQL queries |
|
||||
| sequential-thinking | `@modelcontextprotocol/server-sequential-thinking` | Step-by-step reasoning |
|
||||
| podman | `podman-mcp-server` | Container management |
|
||||
| gitea | `gitea-mcp` (binary) | Gitea API access |
|
||||
| bugsink | `j-shelfwood/bugsink-mcp` (build from source) | Error tracking for Bugsink |
|
||||
| sentry | `@sentry/mcp-server` | Error tracking for Sentry |
|
||||
| playwright | `@anthropics/mcp-server-playwright` | Browser automation |
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Keep configs in sync** - When you change one file, update the other
|
||||
|
||||
2. **Order servers by importance** - Put essential servers (memory, filesystem) first
|
||||
|
||||
3. **Disable instead of delete** - Use `"disabled": true` in settings.json to troubleshoot
|
||||
|
||||
4. **Use node.exe directly** - For faster startup, install packages globally and use `node.exe` instead of `npx`
|
||||
|
||||
5. **Store sensitive data in memory** - Use the memory MCP to store API tokens and config for future sessions
|
||||
|
||||
---
|
||||
|
||||
## Future: MCP Launchpad
|
||||
|
||||
**Project:** <https://github.com/kenneth-liao/mcp-launchpad>
|
||||
|
||||
MCP Launchpad is a CLI tool that wraps multiple MCP servers into a single interface. Worth revisiting when:
|
||||
|
||||
- [ ] Windows support is stable (currently experimental)
|
||||
- [ ] Available as an MCP server itself (currently Bash-based)
|
||||
|
||||
**Why it's interesting:**
|
||||
|
||||
| Benefit | Description |
|
||||
| ---------------------- | -------------------------------------------------------------- |
|
||||
| Single config file | No more syncing `~/.claude.json` and `~/.claude/settings.json` |
|
||||
| Project-level configs | Drop `mcp.json` in any project for instant MCP setup |
|
||||
| Context window savings | One MCP server in context instead of 10+, reducing token usage |
|
||||
| Persistent daemon | Keeps server connections alive for faster repeated calls |
|
||||
| Tool search | Find tools across all servers with `mcpl search` |
|
||||
|
||||
**Current limitations:**
|
||||
|
||||
- Experimental Windows support
|
||||
- Requires Python 3.13+ and uv
|
||||
- Claude calls tools via Bash instead of native MCP integration
|
||||
- Different mental model (runtime discovery vs startup loading)
|
||||
|
||||
---
|
||||
|
||||
## Future: Graphiti (Advanced Knowledge Graph)
|
||||
|
||||
**Project:** <https://github.com/getzep/graphiti>
|
||||
|
||||
Graphiti provides temporal-aware knowledge graphs - it tracks not just facts, but _when_ they became true/outdated. Much more powerful than simple memory MCP, but requires significant infrastructure.
|
||||
|
||||
**Ideal setup:** Run on a Linux server, connect via HTTP from Windows:
|
||||
|
||||
```json
|
||||
// Windows client config (settings.json)
|
||||
"graphiti": {
|
||||
"type": "sse",
|
||||
"url": "http://linux-server:8000/mcp/"
|
||||
}
|
||||
```
|
||||
|
||||
**Linux server setup:**
|
||||
|
||||
```bash
|
||||
git clone https://github.com/getzep/graphiti.git
|
||||
cd graphiti/mcp_server
|
||||
docker compose up -d # Starts FalkorDB + MCP server on port 8000
|
||||
```
|
||||
|
||||
**Requirements:**
|
||||
|
||||
- Docker on Linux server
|
||||
- OpenAI API key (for embeddings)
|
||||
- Port 8000 open on LAN
|
||||
|
||||
**Benefits of remote deployment:**
|
||||
|
||||
- Heavy lifting (Neo4j/FalkorDB + embeddings) offloaded to Linux
|
||||
- Always-on server, Windows connects/disconnects freely
|
||||
- Multiple machines can share the same knowledge graph
|
||||
- Avoids Windows Docker/WSL2 complexity
|
||||
|
||||
---
|
||||
|
||||
\_Last updated: January 2026
|
||||
327
CLAUDE.md
327
CLAUDE.md
@@ -1,20 +1,68 @@
|
||||
# Claude Code Project Instructions
|
||||
|
||||
## Session Startup Checklist
|
||||
|
||||
**IMPORTANT**: At the start of every session, perform these steps:
|
||||
|
||||
1. **Check Memory First** - Use `mcp__memory__read_graph` or `mcp__memory__search_nodes` to recall:
|
||||
- Project-specific configurations and credentials
|
||||
- Previous work context and decisions
|
||||
- Infrastructure details (URLs, ports, access patterns)
|
||||
- Known issues and their solutions
|
||||
|
||||
2. **Review Recent Git History** - Check `git log --oneline -10` to understand recent changes
|
||||
|
||||
3. **Check Container Status** - Use `mcp__podman__container_list` to see what's running
|
||||
|
||||
---
|
||||
|
||||
## Project Instructions
|
||||
|
||||
### Things to Remember
|
||||
|
||||
Before writing any code:
|
||||
|
||||
1. State how you will verify this change works (test, bash command, browser check, etc.)
|
||||
|
||||
2. Write the test or verification step first
|
||||
|
||||
3. Then implement the code
|
||||
|
||||
4. Run verification and iterate until it passes
|
||||
|
||||
## Communication Style: Ask Before Assuming
|
||||
|
||||
**IMPORTANT**: When helping with tasks, **ask clarifying questions before making assumptions**. Do not assume:
|
||||
|
||||
- What steps the user has or hasn't completed
|
||||
- What the user already knows or has configured
|
||||
- What external services (OAuth providers, APIs, etc.) are already set up
|
||||
- What secrets or credentials have already been created
|
||||
|
||||
Instead, ask the user to confirm the current state before providing instructions or making recommendations. This prevents wasted effort and respects the user's existing work.
|
||||
|
||||
## Platform Requirement: Linux Only
|
||||
|
||||
**CRITICAL**: This application is designed to run **exclusively on Linux**. See [ADR-014](docs/adr/0014-containerization-and-deployment-strategy.md) for full details.
|
||||
|
||||
### Environment Terminology
|
||||
|
||||
- **Dev Container** (or just "dev"): The containerized Linux development environment (`flyer-crawler-dev`). This is where all development and testing should occur.
|
||||
- **Host**: The Windows machine running Podman/Docker and VS Code.
|
||||
|
||||
When instructions say "run in dev" or "run in the dev container", they mean executing commands inside the `flyer-crawler-dev` container.
|
||||
|
||||
### Test Execution Rules
|
||||
|
||||
1. **ALL tests MUST be executed on Linux** - either in the Dev Container or on a Linux host
|
||||
2. **NEVER run tests directly on Windows** - test results from Windows are unreliable
|
||||
3. **Always use the Dev Container for testing** when developing on Windows
|
||||
1. **ALL tests MUST be executed in the dev container** - the Linux container environment
|
||||
2. **NEVER run tests directly on Windows host** - test results from Windows are unreliable
|
||||
3. **Always use the dev container for testing** when developing on Windows
|
||||
|
||||
### How to Run Tests Correctly
|
||||
|
||||
```bash
|
||||
# If on Windows, first open VS Code and "Reopen in Container"
|
||||
# Then run tests inside the container:
|
||||
# Then run tests inside the dev container:
|
||||
npm test # Run all unit tests
|
||||
npm run test:unit # Run unit tests only
|
||||
npm run test:integration # Run integration tests (requires DB/Redis)
|
||||
@@ -22,13 +70,19 @@ npm run test:integration # Run integration tests (requires DB/Redis)
|
||||
|
||||
### Running Tests via Podman (from Windows host)
|
||||
|
||||
The command to run unit tests in the Linux container via podman:
|
||||
**Note:** This project has 2900+ unit tests. For AI-assisted development, pipe output to a file for easier processing.
|
||||
|
||||
The command to run unit tests in the dev container via podman:
|
||||
|
||||
```bash
|
||||
# Basic (output to terminal)
|
||||
podman exec -it flyer-crawler-dev npm run test:unit
|
||||
|
||||
# Recommended for AI processing: pipe to file
|
||||
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
|
||||
```
|
||||
|
||||
The command to run integration tests in the Linux container via podman:
|
||||
The command to run integration tests in the dev container via podman:
|
||||
|
||||
```bash
|
||||
podman exec -it flyer-crawler-dev npm run test:integration
|
||||
@@ -55,10 +109,10 @@ podman exec -it flyer-crawler-dev npm test -- --run src/hooks/useAuth.test.tsx
|
||||
## Development Workflow
|
||||
|
||||
1. Open project in VS Code
|
||||
2. Use "Reopen in Container" (Dev Containers extension required)
|
||||
3. Wait for container initialization to complete
|
||||
4. Run `npm test` to verify environment is working
|
||||
5. Make changes and run tests inside the container
|
||||
2. Use "Reopen in Container" (Dev Containers extension required) to enter the dev environment
|
||||
3. Wait for dev container initialization to complete
|
||||
4. Run `npm test` to verify the dev environment is working
|
||||
5. Make changes and run tests inside the dev container
|
||||
|
||||
## Code Change Verification
|
||||
|
||||
@@ -81,6 +135,26 @@ This prevents linting/type errors from being introduced into the codebase.
|
||||
| `npm run build` | Build for production |
|
||||
| `npm run type-check` | Run TypeScript type checking |
|
||||
|
||||
## Database Schema Files
|
||||
|
||||
**CRITICAL**: The database schema files must be kept in sync with each other. When making schema changes:
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------ | ----------------------------------------------------------- |
|
||||
| `sql/master_schema_rollup.sql` | Complete schema used by test database setup and reference |
|
||||
| `sql/initial_schema.sql` | Base schema without seed data, used as standalone reference |
|
||||
| `sql/migrations/*.sql` | Incremental migrations for production database updates |
|
||||
|
||||
**Maintenance Rules:**
|
||||
|
||||
1. **Keep `master_schema_rollup.sql` and `initial_schema.sql` in sync** - These files should contain the same table definitions
|
||||
2. **When adding columns via migration**, also add them to both `master_schema_rollup.sql` and `initial_schema.sql`
|
||||
3. **Migrations are for production deployments** - They use `ALTER TABLE` to add columns incrementally
|
||||
4. **Schema files are for fresh installs** - They define the complete table structure
|
||||
5. **Test database uses `master_schema_rollup.sql`** - If schema files are out of sync with migrations, tests will fail
|
||||
|
||||
**Example:** When `002_expiry_tracking.sql` adds `purchase_date` to `pantry_items`, that column must also exist in the `CREATE TABLE` statements in both `master_schema_rollup.sql` and `initial_schema.sql`.
|
||||
|
||||
## Known Integration Test Issues and Solutions
|
||||
|
||||
This section documents common test issues encountered in integration tests, their root causes, and solutions. These patterns recur frequently.
|
||||
@@ -172,22 +246,227 @@ cb(null, `${file.fieldname}-${uniqueSuffix}-${sanitizedOriginalName}`);
|
||||
|
||||
**Solution:** Use try/catch with graceful degradation or mock the external service checks.
|
||||
|
||||
## Secrets and Environment Variables
|
||||
|
||||
**CRITICAL**: This project uses **Gitea CI/CD secrets** for all sensitive configuration. There is NO `/etc/flyer-crawler/environment` file or similar local config file on the server.
|
||||
|
||||
### Server Directory Structure
|
||||
|
||||
| Path | Environment | Notes |
|
||||
| --------------------------------------------- | ----------- | ------------------------------------------------ |
|
||||
| `/var/www/flyer-crawler.projectium.com/` | Production | NO `.env` file - secrets injected via CI/CD only |
|
||||
| `/var/www/flyer-crawler-test.projectium.com/` | Test | Has `.env.test` file for test-specific config |
|
||||
|
||||
### How Secrets Work
|
||||
|
||||
1. **Gitea Secrets**: All secrets are stored in Gitea repository settings (Settings → Secrets)
|
||||
2. **CI/CD Injection**: Secrets are injected during deployment via `.gitea/workflows/deploy-to-prod.yml` and `deploy-to-test.yml`
|
||||
3. **PM2 Environment**: The CI/CD workflow passes secrets to PM2 via environment variables, which are then available to the application
|
||||
|
||||
### Key Files for Configuration
|
||||
|
||||
| File | Purpose |
|
||||
| ------------------------------------- | ---------------------------------------------------- |
|
||||
| `src/config/env.ts` | Centralized config with Zod schema validation |
|
||||
| `ecosystem.config.cjs` | PM2 process config - reads from `process.env` |
|
||||
| `.gitea/workflows/deploy-to-prod.yml` | Production deployment with secret injection |
|
||||
| `.gitea/workflows/deploy-to-test.yml` | Test deployment with secret injection |
|
||||
| `.env.example` | Template showing all available environment variables |
|
||||
| `.env.test` | Test environment overrides (only on test server) |
|
||||
|
||||
### Adding New Secrets
|
||||
|
||||
To add a new secret (e.g., `SENTRY_DSN`):
|
||||
|
||||
1. Add the secret to Gitea repository settings
|
||||
2. Update the relevant workflow file (e.g., `deploy-to-prod.yml`) to inject it:
|
||||
|
||||
```yaml
|
||||
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
|
||||
```
|
||||
|
||||
3. Update `ecosystem.config.cjs` to read it from `process.env`
|
||||
4. Update `src/config/env.ts` schema if validation is needed
|
||||
5. Update `.env.example` to document the new variable
|
||||
|
||||
### Current Gitea Secrets
|
||||
|
||||
**Shared (used by both environments):**
|
||||
|
||||
- `DB_HOST` - Database host (shared PostgreSQL server)
|
||||
- `JWT_SECRET` - Authentication
|
||||
- `GOOGLE_MAPS_API_KEY` - Google Maps
|
||||
- `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` - Google OAuth
|
||||
- `GH_CLIENT_ID`, `GH_CLIENT_SECRET` - GitHub OAuth
|
||||
|
||||
**Production-specific:**
|
||||
|
||||
- `DB_USER_PROD`, `DB_PASSWORD_PROD` - Production database credentials (`flyer_crawler_prod`)
|
||||
- `DB_DATABASE_PROD` - Production database name (`flyer-crawler`)
|
||||
- `REDIS_PASSWORD_PROD` - Redis password (uses database 0)
|
||||
- `VITE_GOOGLE_GENAI_API_KEY` - Gemini API key for production
|
||||
- `SENTRY_DSN`, `VITE_SENTRY_DSN` - Bugsink error tracking DSNs (production projects)
|
||||
|
||||
**Test-specific:**
|
||||
|
||||
- `DB_USER_TEST`, `DB_PASSWORD_TEST` - Test database credentials (`flyer_crawler_test`)
|
||||
- `DB_DATABASE_TEST` - Test database name (`flyer-crawler-test`)
|
||||
- `REDIS_PASSWORD_TEST` - Redis password (uses database 1 for isolation)
|
||||
- `VITE_GOOGLE_GENAI_API_KEY_TEST` - Gemini API key for test
|
||||
- `SENTRY_DSN_TEST`, `VITE_SENTRY_DSN_TEST` - Bugsink error tracking DSNs (test projects)
|
||||
|
||||
### Test Environment
|
||||
|
||||
The test environment (`flyer-crawler-test.projectium.com`) uses **both** Gitea CI/CD secrets and a local `.env.test` file:
|
||||
|
||||
- **Gitea secrets**: Injected during deployment via `.gitea/workflows/deploy-to-test.yml`
|
||||
- **`.env.test` file**: Located at `/var/www/flyer-crawler-test.projectium.com/.env.test` for local overrides
|
||||
- **Redis database 1**: Isolates test job queues from production (which uses database 0)
|
||||
- **PM2 process names**: Suffixed with `-test` (e.g., `flyer-crawler-api-test`)
|
||||
|
||||
### Database User Setup (Test Environment)
|
||||
|
||||
**CRITICAL**: The test database requires specific PostgreSQL permissions to be configured manually. Schema ownership alone is NOT sufficient - explicit privileges must be granted.
|
||||
|
||||
**Database Users:**
|
||||
|
||||
| User | Database | Purpose |
|
||||
| -------------------- | -------------------- | ---------- |
|
||||
| `flyer_crawler_prod` | `flyer-crawler` | Production |
|
||||
| `flyer_crawler_test` | `flyer-crawler-test` | Testing |
|
||||
|
||||
**Required Setup Commands** (run as `postgres` superuser):
|
||||
|
||||
```bash
|
||||
# Connect as postgres superuser
|
||||
sudo -u postgres psql
|
||||
|
||||
# Create the test database and user (if not exists)
|
||||
CREATE DATABASE "flyer-crawler-test";
|
||||
CREATE USER flyer_crawler_test WITH PASSWORD 'your-password-here';
|
||||
|
||||
# Grant ownership and privileges
|
||||
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
|
||||
\c "flyer-crawler-test"
|
||||
ALTER SCHEMA public OWNER TO flyer_crawler_test;
|
||||
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
|
||||
|
||||
# Create required extension (must be done by superuser)
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
```
|
||||
|
||||
**Why These Steps Are Necessary:**
|
||||
|
||||
1. **Schema ownership alone is insufficient** - PostgreSQL requires explicit `GRANT CREATE, USAGE` privileges even when the user owns the schema
|
||||
2. **uuid-ossp extension** - Required by the application for UUID generation; must be created by a superuser before the app can use it
|
||||
3. **Separate users for prod/test** - Prevents accidental cross-environment data access; each environment has its own credentials in Gitea secrets
|
||||
|
||||
**Verification:**
|
||||
|
||||
```bash
|
||||
# Check schema privileges (should show 'UC' for flyer_crawler_test)
|
||||
psql -d "flyer-crawler-test" -c "\dn+ public"
|
||||
|
||||
# Expected output:
|
||||
# Name | Owner | Access privileges
|
||||
# -------+--------------------+------------------------------------------
|
||||
# public | flyer_crawler_test | flyer_crawler_test=UC/flyer_crawler_test
|
||||
```
|
||||
|
||||
### Dev Container Environment
|
||||
|
||||
The dev container runs its own **local Bugsink instance** - it does NOT connect to the production Bugsink server:
|
||||
|
||||
- **Local Bugsink**: Runs at `http://localhost:8000` inside the container
|
||||
- **Pre-configured DSNs**: Set in `compose.dev.yml`, pointing to local instance
|
||||
- **Admin credentials**: `admin@localhost` / `admin`
|
||||
- **Isolated**: Dev errors stay local, don't pollute production/test dashboards
|
||||
- **No Gitea secrets needed**: Everything is self-contained in the container
|
||||
|
||||
---
|
||||
|
||||
## MCP Servers
|
||||
|
||||
The following MCP servers are configured for this project:
|
||||
|
||||
| Server | Purpose |
|
||||
| ------------------- | ---------------------------------------- |
|
||||
| gitea-projectium | Gitea API for gitea.projectium.com |
|
||||
| gitea-torbonium | Gitea API for gitea.torbonium.com |
|
||||
| podman | Container management |
|
||||
| filesystem | File system access |
|
||||
| fetch | Web fetching |
|
||||
| markitdown | Convert documents to markdown |
|
||||
| sequential-thinking | Step-by-step reasoning |
|
||||
| memory | Knowledge graph persistence |
|
||||
| postgres | Direct database queries (localhost:5432) |
|
||||
| playwright | Browser automation and testing |
|
||||
| redis | Redis cache inspection (localhost:6379) |
|
||||
| Server | Purpose |
|
||||
| --------------------- | ------------------------------------------- |
|
||||
| gitea-projectium | Gitea API for gitea.projectium.com |
|
||||
| gitea-torbonium | Gitea API for gitea.torbonium.com |
|
||||
| podman | Container management |
|
||||
| filesystem | File system access |
|
||||
| fetch | Web fetching |
|
||||
| markitdown | Convert documents to markdown |
|
||||
| sequential-thinking | Step-by-step reasoning |
|
||||
| memory | Knowledge graph persistence |
|
||||
| postgres | Direct database queries (localhost:5432) |
|
||||
| playwright | Browser automation and testing |
|
||||
| redis | Redis cache inspection (localhost:6379) |
|
||||
| sentry-selfhosted-mcp | Error tracking via Bugsink (localhost:8000) |
|
||||
|
||||
**Note:** MCP servers are currently only available in **Claude CLI**. Due to a bug in Claude VS Code extension, MCP servers do not work there yet.
|
||||
**Note:** MCP servers work in both **Claude CLI** and **Claude Code VS Code extension** (as of January 2026).
|
||||
|
||||
### Sentry/Bugsink MCP Server Setup (ADR-015)
|
||||
|
||||
To enable Claude Code to query and analyze application errors from Bugsink:
|
||||
|
||||
1. **Install the MCP server**:
|
||||
|
||||
```bash
|
||||
# Clone the sentry-selfhosted-mcp repository
|
||||
git clone https://github.com/ddfourtwo/sentry-selfhosted-mcp.git
|
||||
cd sentry-selfhosted-mcp
|
||||
npm install
|
||||
```
|
||||
|
||||
2. **Configure Claude Code** (add to `.claude/mcp.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"sentry-selfhosted-mcp": {
|
||||
"command": "node",
|
||||
"args": ["/path/to/sentry-selfhosted-mcp/dist/index.js"],
|
||||
"env": {
|
||||
"SENTRY_URL": "http://localhost:8000",
|
||||
"SENTRY_AUTH_TOKEN": "<get-from-bugsink-ui>",
|
||||
"SENTRY_ORG_SLUG": "flyer-crawler"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. **Get the auth token**:
|
||||
- Navigate to Bugsink UI at `http://localhost:8000`
|
||||
- Log in with admin credentials
|
||||
- Go to Settings > API Keys
|
||||
- Create a new API key with read access
|
||||
|
||||
4. **Available capabilities**:
|
||||
- List projects and issues
|
||||
- View detailed error events
|
||||
- Search by error message or stack trace
|
||||
- Update issue status (resolve, ignore)
|
||||
- Add comments to issues
|
||||
|
||||
### SSH Server Access
|
||||
|
||||
Claude Code can execute commands on the production server via SSH:
|
||||
|
||||
```bash
|
||||
# Basic command execution
|
||||
ssh root@projectium.com "command here"
|
||||
|
||||
# Examples:
|
||||
ssh root@projectium.com "systemctl status logstash"
|
||||
ssh root@projectium.com "pm2 list"
|
||||
ssh root@projectium.com "tail -50 /var/www/flyer-crawler.projectium.com/logs/app.log"
|
||||
```
|
||||
|
||||
**Use cases:**
|
||||
|
||||
- Managing Logstash, PM2, NGINX, Redis services
|
||||
- Viewing server logs
|
||||
- Deploying configuration changes
|
||||
- Checking service status
|
||||
|
||||
**Important:** SSH access requires the host machine to have SSH keys configured for `root@projectium.com`.
|
||||
|
||||
@@ -149,7 +149,7 @@ SELECT PostGIS_Full_Version();
|
||||
|
||||
Example output:
|
||||
|
||||
```
|
||||
```text
|
||||
PostgreSQL 14.19 (Ubuntu 14.19-0ubuntu0.22.04.1)
|
||||
POSTGIS="3.2.0 c3e3cc0" GEOS="3.10.2-CAPI-1.16.0" PROJ="8.2.1"
|
||||
```
|
||||
|
||||
@@ -204,8 +204,68 @@ pm2 restart flyer-crawler-api
|
||||
|
||||
---
|
||||
|
||||
## Error Tracking with Bugsink (ADR-015)
|
||||
|
||||
Bugsink is a self-hosted Sentry-compatible error tracking system. See [docs/adr/0015-application-performance-monitoring-and-error-tracking.md](docs/adr/0015-application-performance-monitoring-and-error-tracking.md) for the full architecture decision.
|
||||
|
||||
### Creating Bugsink Projects and DSNs
|
||||
|
||||
After Bugsink is installed and running, you need to create projects and obtain DSNs:
|
||||
|
||||
1. **Access Bugsink UI**: Navigate to `http://localhost:8000`
|
||||
|
||||
2. **Log in** with your admin credentials
|
||||
|
||||
3. **Create Backend Project**:
|
||||
- Click "Create Project"
|
||||
- Name: `flyer-crawler-backend`
|
||||
- Platform: Node.js
|
||||
- Copy the generated DSN (format: `http://<key>@localhost:8000/<project_id>`)
|
||||
|
||||
4. **Create Frontend Project**:
|
||||
- Click "Create Project"
|
||||
- Name: `flyer-crawler-frontend`
|
||||
- Platform: React
|
||||
- Copy the generated DSN
|
||||
|
||||
5. **Configure Environment Variables**:
|
||||
|
||||
```bash
|
||||
# Backend (server-side)
|
||||
export SENTRY_DSN=http://<backend-key>@localhost:8000/<backend-project-id>
|
||||
|
||||
# Frontend (client-side, exposed to browser)
|
||||
export VITE_SENTRY_DSN=http://<frontend-key>@localhost:8000/<frontend-project-id>
|
||||
|
||||
# Shared settings
|
||||
export SENTRY_ENVIRONMENT=production
|
||||
export VITE_SENTRY_ENVIRONMENT=production
|
||||
export SENTRY_ENABLED=true
|
||||
export VITE_SENTRY_ENABLED=true
|
||||
```
|
||||
|
||||
### Testing Error Tracking
|
||||
|
||||
Verify Bugsink is receiving events:
|
||||
|
||||
```bash
|
||||
npx tsx scripts/test-bugsink.ts
|
||||
```
|
||||
|
||||
This sends test error and info events. Check the Bugsink UI for:
|
||||
|
||||
- `BugsinkTestError` in the backend project
|
||||
- Info message "Test info message from test-bugsink.ts"
|
||||
|
||||
### Sentry SDK v10+ HTTP DSN Limitation
|
||||
|
||||
The Sentry SDK v10+ enforces HTTPS-only DSNs by default. Since Bugsink runs locally over HTTP, our implementation uses the Sentry Store API directly instead of the SDK's built-in transport. This is handled transparently by the `sentry.server.ts` and `sentry.client.ts` modules.
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Database Setup](DATABASE.md) - PostgreSQL and PostGIS configuration
|
||||
- [Authentication Setup](AUTHENTICATION.md) - OAuth provider configuration
|
||||
- [Installation Guide](INSTALL.md) - Local development setup
|
||||
- [Bare-Metal Server Setup](docs/BARE-METAL-SETUP.md) - Manual server installation guide
|
||||
|
||||
228
Dockerfile.dev
228
Dockerfile.dev
@@ -7,7 +7,7 @@
|
||||
#
|
||||
# Base: Ubuntu 22.04 (LTS) - matches production server
|
||||
# Node: v20.x (LTS) - matches production
|
||||
# Includes: PostgreSQL client, Redis CLI, build tools
|
||||
# Includes: PostgreSQL client, Redis CLI, build tools, Bugsink, Logstash
|
||||
# ============================================================================
|
||||
|
||||
FROM ubuntu:22.04
|
||||
@@ -21,16 +21,23 @@ ENV DEBIAN_FRONTEND=noninteractive
|
||||
# - curl: for downloading Node.js setup script and health checks
|
||||
# - git: for version control operations
|
||||
# - build-essential: for compiling native Node.js modules (node-gyp)
|
||||
# - python3: required by some Node.js build tools
|
||||
# - python3, python3-pip, python3-venv: for Bugsink
|
||||
# - postgresql-client: for psql CLI (database initialization)
|
||||
# - redis-tools: for redis-cli (health checks)
|
||||
# - gnupg, apt-transport-https: for Elastic APT repository (Logstash)
|
||||
# - openjdk-17-jre-headless: required by Logstash
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
git \
|
||||
build-essential \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-venv \
|
||||
postgresql-client \
|
||||
redis-tools \
|
||||
gnupg \
|
||||
apt-transport-https \
|
||||
openjdk-17-jre-headless \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ============================================================================
|
||||
@@ -39,6 +46,204 @@ RUN apt-get update && apt-get install -y \
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
|
||||
&& apt-get install -y nodejs
|
||||
|
||||
# ============================================================================
|
||||
# Install Logstash (Elastic APT Repository)
|
||||
# ============================================================================
|
||||
# ADR-015: Log aggregation for Pino and Redis logs → Bugsink
|
||||
RUN curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | gpg --dearmor -o /usr/share/keyrings/elastic-keyring.gpg \
|
||||
&& echo "deb [signed-by=/usr/share/keyrings/elastic-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-8.x.list \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y logstash \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ============================================================================
|
||||
# Install Bugsink (Python Package)
|
||||
# ============================================================================
|
||||
# ADR-015: Self-hosted Sentry-compatible error tracking
|
||||
# Create a virtual environment for Bugsink to avoid conflicts
|
||||
RUN python3 -m venv /opt/bugsink \
|
||||
&& /opt/bugsink/bin/pip install --upgrade pip \
|
||||
&& /opt/bugsink/bin/pip install bugsink gunicorn psycopg2-binary
|
||||
|
||||
# Create Bugsink directories and configuration
|
||||
RUN mkdir -p /var/log/bugsink /var/lib/bugsink /opt/bugsink/conf
|
||||
|
||||
# Create Bugsink configuration file (Django settings module)
|
||||
# This file is imported by bugsink-manage via DJANGO_SETTINGS_MODULE
|
||||
# Based on bugsink/conf_templates/docker.py.template but customized for our setup
|
||||
RUN echo 'import os\n\
|
||||
from urllib.parse import urlparse\n\
|
||||
\n\
|
||||
from bugsink.settings.default import *\n\
|
||||
from bugsink.settings.default import DATABASES, SILENCED_SYSTEM_CHECKS\n\
|
||||
from bugsink.conf_utils import deduce_allowed_hosts, deduce_script_name\n\
|
||||
\n\
|
||||
IS_DOCKER = True\n\
|
||||
\n\
|
||||
# Security settings\n\
|
||||
SECRET_KEY = os.getenv("SECRET_KEY")\n\
|
||||
DEBUG = os.getenv("DEBUG", "False").lower() in ("true", "1", "yes")\n\
|
||||
\n\
|
||||
# Silence cookie security warnings for dev (no HTTPS)\n\
|
||||
SILENCED_SYSTEM_CHECKS += ["security.W012", "security.W016"]\n\
|
||||
\n\
|
||||
# Database configuration from DATABASE_URL environment variable\n\
|
||||
if os.getenv("DATABASE_URL"):\n\
|
||||
DATABASE_URL = os.getenv("DATABASE_URL")\n\
|
||||
parsed = urlparse(DATABASE_URL)\n\
|
||||
\n\
|
||||
if parsed.scheme in ["postgres", "postgresql"]:\n\
|
||||
DATABASES["default"] = {\n\
|
||||
"ENGINE": "django.db.backends.postgresql",\n\
|
||||
"NAME": parsed.path.lstrip("/"),\n\
|
||||
"USER": parsed.username,\n\
|
||||
"PASSWORD": parsed.password,\n\
|
||||
"HOST": parsed.hostname,\n\
|
||||
"PORT": parsed.port or "5432",\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Snappea (background task runner) settings\n\
|
||||
SNAPPEA = {\n\
|
||||
"TASK_ALWAYS_EAGER": False,\n\
|
||||
"WORKAHOLIC": True,\n\
|
||||
"NUM_WORKERS": 2,\n\
|
||||
"PID_FILE": None,\n\
|
||||
}\n\
|
||||
DATABASES["snappea"]["NAME"] = "/tmp/snappea.sqlite3"\n\
|
||||
\n\
|
||||
# Site settings\n\
|
||||
_PORT = os.getenv("PORT", "8000")\n\
|
||||
BUGSINK = {\n\
|
||||
"BASE_URL": os.getenv("BASE_URL", f"http://localhost:{_PORT}"),\n\
|
||||
"SITE_TITLE": os.getenv("SITE_TITLE", "Flyer Crawler Error Tracking"),\n\
|
||||
"SINGLE_USER": os.getenv("SINGLE_USER", "True").lower() in ("true", "1", "yes"),\n\
|
||||
"SINGLE_TEAM": os.getenv("SINGLE_TEAM", "True").lower() in ("true", "1", "yes"),\n\
|
||||
"PHONEHOME": False,\n\
|
||||
}\n\
|
||||
\n\
|
||||
ALLOWED_HOSTS = deduce_allowed_hosts(BUGSINK["BASE_URL"])\n\
|
||||
\n\
|
||||
# Console email backend for dev\n\
|
||||
EMAIL_BACKEND = "bugsink.email_backends.QuietConsoleEmailBackend"\n\
|
||||
' > /opt/bugsink/conf/bugsink_conf.py
|
||||
|
||||
# Create Bugsink startup script
|
||||
# Uses DATABASE_URL environment variable (standard Docker approach per docs)
|
||||
RUN echo '#!/bin/bash\n\
|
||||
set -e\n\
|
||||
\n\
|
||||
# Build DATABASE_URL from individual env vars for flexibility\n\
|
||||
export DATABASE_URL="postgresql://${BUGSINK_DB_USER:-bugsink}:${BUGSINK_DB_PASSWORD:-bugsink_dev_password}@${BUGSINK_DB_HOST:-postgres}:${BUGSINK_DB_PORT:-5432}/${BUGSINK_DB_NAME:-bugsink}"\n\
|
||||
# SECRET_KEY is required by Bugsink/Django\n\
|
||||
export SECRET_KEY="${BUGSINK_SECRET_KEY:-dev-bugsink-secret-key-minimum-50-characters-for-security}"\n\
|
||||
\n\
|
||||
# Create superuser if not exists (for dev convenience)\n\
|
||||
if [ -n "$BUGSINK_ADMIN_EMAIL" ] && [ -n "$BUGSINK_ADMIN_PASSWORD" ]; then\n\
|
||||
export CREATE_SUPERUSER="${BUGSINK_ADMIN_EMAIL}:${BUGSINK_ADMIN_PASSWORD}"\n\
|
||||
fi\n\
|
||||
\n\
|
||||
# Wait for PostgreSQL to be ready\n\
|
||||
until pg_isready -h ${BUGSINK_DB_HOST:-postgres} -p ${BUGSINK_DB_PORT:-5432} -U ${BUGSINK_DB_USER:-bugsink}; do\n\
|
||||
echo "Waiting for PostgreSQL..."\n\
|
||||
sleep 2\n\
|
||||
done\n\
|
||||
\n\
|
||||
echo "PostgreSQL is ready. Starting Bugsink..."\n\
|
||||
echo "DATABASE_URL: postgresql://${BUGSINK_DB_USER}:***@${BUGSINK_DB_HOST}:${BUGSINK_DB_PORT}/${BUGSINK_DB_NAME}"\n\
|
||||
\n\
|
||||
# Change to config directory so bugsink_conf.py can be found\n\
|
||||
cd /opt/bugsink/conf\n\
|
||||
\n\
|
||||
# Run migrations\n\
|
||||
echo "Running database migrations..."\n\
|
||||
/opt/bugsink/bin/bugsink-manage migrate --noinput\n\
|
||||
\n\
|
||||
# Create superuser if CREATE_SUPERUSER is set (format: email:password)\n\
|
||||
if [ -n "$CREATE_SUPERUSER" ]; then\n\
|
||||
IFS=":" read -r ADMIN_EMAIL ADMIN_PASS <<< "$CREATE_SUPERUSER"\n\
|
||||
/opt/bugsink/bin/bugsink-manage shell -c "\n\
|
||||
from django.contrib.auth import get_user_model\n\
|
||||
User = get_user_model()\n\
|
||||
if not User.objects.filter(email='"'"'$ADMIN_EMAIL'"'"').exists():\n\
|
||||
User.objects.create_superuser('"'"'$ADMIN_EMAIL'"'"', '"'"'$ADMIN_PASS'"'"')\n\
|
||||
print('"'"'Superuser created'"'"')\n\
|
||||
else:\n\
|
||||
print('"'"'Superuser already exists'"'"')\n\
|
||||
" || true\n\
|
||||
fi\n\
|
||||
\n\
|
||||
# Start Bugsink with Gunicorn\n\
|
||||
echo "Starting Gunicorn on port ${BUGSINK_PORT:-8000}..."\n\
|
||||
exec /opt/bugsink/bin/gunicorn \\\n\
|
||||
--bind 0.0.0.0:${BUGSINK_PORT:-8000} \\\n\
|
||||
--workers ${BUGSINK_WORKERS:-2} \\\n\
|
||||
--access-logfile - \\\n\
|
||||
--error-logfile - \\\n\
|
||||
bugsink.wsgi:application\n\
|
||||
' > /usr/local/bin/start-bugsink.sh \
|
||||
&& chmod +x /usr/local/bin/start-bugsink.sh
|
||||
|
||||
# ============================================================================
|
||||
# Create Logstash Pipeline Configuration
|
||||
# ============================================================================
|
||||
# ADR-015: Pino and Redis logs → Bugsink
|
||||
RUN mkdir -p /etc/logstash/conf.d /app/logs
|
||||
|
||||
RUN echo 'input {\n\
|
||||
# Pino application logs\n\
|
||||
file {\n\
|
||||
path => "/app/logs/*.log"\n\
|
||||
codec => json\n\
|
||||
type => "pino"\n\
|
||||
tags => ["app"]\n\
|
||||
start_position => "beginning"\n\
|
||||
sincedb_path => "/var/lib/logstash/sincedb_pino"\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Redis logs\n\
|
||||
file {\n\
|
||||
path => "/var/log/redis/*.log"\n\
|
||||
type => "redis"\n\
|
||||
tags => ["redis"]\n\
|
||||
start_position => "beginning"\n\
|
||||
sincedb_path => "/var/lib/logstash/sincedb_redis"\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
filter {\n\
|
||||
# Pino error detection (level 50 = error, 60 = fatal)\n\
|
||||
if [type] == "pino" and [level] >= 50 {\n\
|
||||
mutate { add_tag => ["error"] }\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Redis error detection\n\
|
||||
if [type] == "redis" {\n\
|
||||
grok {\n\
|
||||
match => { "message" => "%%{POSINT:pid}:%%{WORD:role} %%{MONTHDAY} %%{MONTH} %%{TIME} %%{WORD:loglevel} %%{GREEDYDATA:redis_message}" }\n\
|
||||
}\n\
|
||||
if [loglevel] in ["WARNING", "ERROR"] {\n\
|
||||
mutate { add_tag => ["error"] }\n\
|
||||
}\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
output {\n\
|
||||
if "error" in [tags] {\n\
|
||||
http {\n\
|
||||
url => "http://localhost:8000/api/store/"\n\
|
||||
http_method => "post"\n\
|
||||
format => "json"\n\
|
||||
}\n\
|
||||
}\n\
|
||||
\n\
|
||||
# Debug output (comment out in production)\n\
|
||||
stdout { codec => rubydebug }\n\
|
||||
}\n\
|
||||
' > /etc/logstash/conf.d/bugsink.conf
|
||||
|
||||
# Create Logstash sincedb directory
|
||||
RUN mkdir -p /var/lib/logstash && chown -R logstash:logstash /var/lib/logstash
|
||||
|
||||
# ============================================================================
|
||||
# Set Working Directory
|
||||
# ============================================================================
|
||||
@@ -52,6 +257,25 @@ ENV NODE_ENV=development
|
||||
# Increase Node.js memory limit for large builds
|
||||
ENV NODE_OPTIONS='--max-old-space-size=8192'
|
||||
|
||||
# Bugsink defaults (ADR-015)
|
||||
ENV BUGSINK_DB_HOST=postgres
|
||||
ENV BUGSINK_DB_PORT=5432
|
||||
ENV BUGSINK_DB_NAME=bugsink
|
||||
ENV BUGSINK_DB_USER=bugsink
|
||||
ENV BUGSINK_DB_PASSWORD=bugsink_dev_password
|
||||
ENV BUGSINK_PORT=8000
|
||||
ENV BUGSINK_BASE_URL=http://localhost:8000
|
||||
ENV BUGSINK_ADMIN_EMAIL=admin@localhost
|
||||
ENV BUGSINK_ADMIN_PASSWORD=admin
|
||||
|
||||
# ============================================================================
|
||||
# Expose Ports
|
||||
# ============================================================================
|
||||
# 3000 - Vite frontend
|
||||
# 3001 - Express backend
|
||||
# 8000 - Bugsink error tracking
|
||||
EXPOSE 3000 3001 8000
|
||||
|
||||
# ============================================================================
|
||||
# Default Command
|
||||
# ============================================================================
|
||||
|
||||
@@ -103,6 +103,7 @@ You are now inside the Ubuntu container's shell.
|
||||
```
|
||||
|
||||
4. **Install Project Dependencies**:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
# This file defines the local development environment using Docker/Podman.
|
||||
#
|
||||
# Services:
|
||||
# - app: Node.js application (API + Frontend)
|
||||
# - app: Node.js application (API + Frontend + Bugsink + Logstash)
|
||||
# - postgres: PostgreSQL 15 with PostGIS extension
|
||||
# - redis: Redis for caching and job queues
|
||||
#
|
||||
@@ -18,6 +18,10 @@
|
||||
# VS Code Dev Containers:
|
||||
# This file is referenced by .devcontainer/devcontainer.json for seamless
|
||||
# VS Code integration. Open the project in VS Code and use "Reopen in Container".
|
||||
#
|
||||
# Bugsink (ADR-015):
|
||||
# Access error tracking UI at http://localhost:8000
|
||||
# Default login: admin@localhost / admin
|
||||
# ============================================================================
|
||||
|
||||
version: '3.8'
|
||||
@@ -43,6 +47,7 @@ services:
|
||||
ports:
|
||||
- '3000:3000' # Frontend (Vite default)
|
||||
- '3001:3001' # Backend API
|
||||
- '8000:8000' # Bugsink error tracking (ADR-015)
|
||||
environment:
|
||||
# Core settings
|
||||
- NODE_ENV=development
|
||||
@@ -62,6 +67,26 @@ services:
|
||||
- JWT_SECRET=dev-jwt-secret-change-in-production
|
||||
# Worker settings
|
||||
- WORKER_LOCK_DURATION=120000
|
||||
# Bugsink error tracking (ADR-015)
|
||||
- BUGSINK_DB_HOST=postgres
|
||||
- BUGSINK_DB_PORT=5432
|
||||
- BUGSINK_DB_NAME=bugsink
|
||||
- BUGSINK_DB_USER=bugsink
|
||||
- BUGSINK_DB_PASSWORD=bugsink_dev_password
|
||||
- BUGSINK_PORT=8000
|
||||
- BUGSINK_BASE_URL=http://localhost:8000
|
||||
- BUGSINK_ADMIN_EMAIL=admin@localhost
|
||||
- BUGSINK_ADMIN_PASSWORD=admin
|
||||
- BUGSINK_SECRET_KEY=dev-bugsink-secret-key-minimum-50-characters-for-security
|
||||
# Sentry SDK configuration (points to local Bugsink)
|
||||
- SENTRY_DSN=http://59a58583-e869-7697-f94a-cfa0337676a8@localhost:8000/1
|
||||
- VITE_SENTRY_DSN=http://d5fc5221-4266-ff2f-9af8-5689696072f3@localhost:8000/2
|
||||
- SENTRY_ENVIRONMENT=development
|
||||
- VITE_SENTRY_ENVIRONMENT=development
|
||||
- SENTRY_ENABLED=true
|
||||
- VITE_SENTRY_ENABLED=true
|
||||
- SENTRY_DEBUG=true
|
||||
- VITE_SENTRY_DEBUG=true
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -93,9 +118,10 @@ services:
|
||||
POSTGRES_INITDB_ARGS: '--encoding=UTF8 --locale=C'
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
# Mount the extensions init script to run on first database creation
|
||||
# The 00- prefix ensures it runs before any other init scripts
|
||||
# Mount init scripts to run on first database creation
|
||||
# Scripts run in alphabetical order: 00-extensions, 01-bugsink
|
||||
- ./sql/00-init-extensions.sql:/docker-entrypoint-initdb.d/00-init-extensions.sql:ro
|
||||
- ./sql/01-init-bugsink.sh:/docker-entrypoint-initdb.d/01-init-bugsink.sh:ro
|
||||
# Healthcheck ensures postgres is ready before app starts
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U postgres -d flyer_crawler_dev']
|
||||
|
||||
1347
docs/BARE-METAL-SETUP.md
Normal file
1347
docs/BARE-METAL-SETUP.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -2,17 +2,321 @@
|
||||
|
||||
**Date**: 2025-12-12
|
||||
|
||||
**Status**: Proposed
|
||||
**Status**: Accepted
|
||||
|
||||
**Updated**: 2026-01-11
|
||||
|
||||
## Context
|
||||
|
||||
While `ADR-004` established structured logging, the application lacks a high-level, aggregated view of its health, performance, and errors. It's difficult to spot trends, identify slow API endpoints, or be proactively notified of new types of errors.
|
||||
While `ADR-004` established structured logging with Pino, the application lacks a high-level, aggregated view of its health, performance, and errors. It's difficult to spot trends, identify slow API endpoints, or be proactively notified of new types of errors.
|
||||
|
||||
Key requirements:
|
||||
|
||||
1. **Self-hosted**: No external SaaS dependencies for error tracking
|
||||
2. **Sentry SDK compatible**: Leverage mature, well-documented SDKs
|
||||
3. **Lightweight**: Minimal resource overhead in the dev container
|
||||
4. **Production-ready**: Same architecture works on bare-metal production servers
|
||||
5. **AI-accessible**: MCP server integration for Claude Code and other AI tools
|
||||
|
||||
## Decision
|
||||
|
||||
We will integrate a dedicated Application Performance Monitoring (APM) and error tracking service like **Sentry**, **Datadog**, or **New Relic**. This will define how the service is integrated to automatically capture and report unhandled exceptions, performance data (e.g., transaction traces, database query times), and release health.
|
||||
We will implement a self-hosted error tracking stack using **Bugsink** as the Sentry-compatible backend, with the following components:
|
||||
|
||||
### 1. Error Tracking Backend: Bugsink
|
||||
|
||||
**Bugsink** is a lightweight, self-hosted Sentry alternative that:
|
||||
|
||||
- Runs as a single process (no Kafka, Redis, ClickHouse required)
|
||||
- Is fully compatible with Sentry SDKs
|
||||
- Supports ARM64 and AMD64 architectures
|
||||
- Can use SQLite (dev) or PostgreSQL (production)
|
||||
|
||||
**Deployment**:
|
||||
|
||||
- **Dev container**: Installed as a systemd service inside the container
|
||||
- **Production**: Runs as a systemd service on bare-metal, listening on localhost only
|
||||
- **Database**: Uses PostgreSQL with a dedicated `bugsink` user and `bugsink` database (same PostgreSQL instance as the main application)
|
||||
|
||||
### 2. Backend Integration: @sentry/node
|
||||
|
||||
The Express backend will integrate `@sentry/node` SDK to:
|
||||
|
||||
- Capture unhandled exceptions before PM2/process manager restarts
|
||||
- Report errors with full stack traces and context
|
||||
- Integrate with Pino logger for breadcrumbs
|
||||
- Track transaction performance (optional)
|
||||
|
||||
### 3. Frontend Integration: @sentry/react
|
||||
|
||||
The React frontend will integrate `@sentry/react` SDK to:
|
||||
|
||||
- Wrap the app in a Sentry Error Boundary
|
||||
- Capture unhandled JavaScript errors
|
||||
- Report errors with component stack traces
|
||||
- Track user session context
|
||||
- **Frontend Error Correlation**: The global API client (Axios/Fetch wrapper) MUST intercept 4xx/5xx responses. It MUST extract the `x-request-id` header (if present) and attach it to the Sentry scope as a tag `api_request_id` before re-throwing the error. This allows developers to copy the ID from Sentry and search for it in backend logs.
|
||||
|
||||
### 4. Log Aggregation: Logstash
|
||||
|
||||
**Logstash** parses application and infrastructure logs, forwarding error patterns to Bugsink:
|
||||
|
||||
- **Installation**: Installed inside the dev container (and on bare-metal prod servers)
|
||||
- **Inputs**:
|
||||
- Pino JSON logs from the Node.js application
|
||||
- Redis logs (connection errors, memory warnings, slow commands)
|
||||
- PostgreSQL function logs (future - see Implementation Steps)
|
||||
- **Filter**: Identifies error-level logs (5xx responses, unhandled exceptions, Redis errors)
|
||||
- **Output**: Sends to Bugsink via Sentry-compatible HTTP API
|
||||
|
||||
This provides a secondary error capture path for:
|
||||
|
||||
- Errors that occur before Sentry SDK initialization
|
||||
- Log-based errors that don't throw exceptions
|
||||
- Redis connection/performance issues
|
||||
- Database function errors and slow queries
|
||||
- Historical error analysis from log files
|
||||
|
||||
### 5. MCP Server Integration: sentry-selfhosted-mcp
|
||||
|
||||
For AI tool integration (Claude Code, Cursor, etc.), we use the open-source [sentry-selfhosted-mcp](https://github.com/ddfourtwo/sentry-selfhosted-mcp) server:
|
||||
|
||||
- **No code changes required**: Configurable via environment variables
|
||||
- **Capabilities**: List projects, get issues, view events, update status, add comments
|
||||
- **Configuration**:
|
||||
- `SENTRY_URL`: Points to Bugsink instance
|
||||
- `SENTRY_AUTH_TOKEN`: API token from Bugsink
|
||||
- `SENTRY_ORG_SLUG`: Organization identifier
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ Dev Container / Production Server │
|
||||
├─────────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────────────┐ ┌──────────────────┐ │
|
||||
│ │ Frontend │ │ Backend │ │
|
||||
│ │ (React) │ │ (Express) │ │
|
||||
│ │ @sentry/react │ │ @sentry/node │ │
|
||||
│ └────────┬─────────┘ └────────┬─────────┘ │
|
||||
│ │ │ │
|
||||
│ │ Sentry SDK Protocol │ │
|
||||
│ └───────────┬───────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────┐ │
|
||||
│ │ Bugsink │ │
|
||||
│ │ (localhost:8000) │◄──────────────────┐ │
|
||||
│ │ │ │ │
|
||||
│ │ PostgreSQL backend │ │ │
|
||||
│ └──────────────────────┘ │ │
|
||||
│ │ │
|
||||
│ ┌──────────────────────┐ │ │
|
||||
│ │ Logstash │───────────────────┘ │
|
||||
│ │ (Log Aggregator) │ Sentry Output │
|
||||
│ │ │ │
|
||||
│ │ Inputs: │ │
|
||||
│ │ - Pino app logs │ │
|
||||
│ │ - Redis logs │ │
|
||||
│ │ - PostgreSQL (future) │
|
||||
│ └──────────────────────┘ │
|
||||
│ ▲ ▲ ▲ │
|
||||
│ │ │ │ │
|
||||
│ ┌───────────┘ │ └───────────┐ │
|
||||
│ │ │ │ │
|
||||
│ ┌────┴─────┐ ┌─────┴────┐ ┌──────┴─────┐ │
|
||||
│ │ Pino │ │ Redis │ │ PostgreSQL │ │
|
||||
│ │ Logs │ │ Logs │ │ Logs (TBD) │ │
|
||||
│ └──────────┘ └──────────┘ └────────────┘ │
|
||||
│ │
|
||||
│ ┌──────────────────────┐ │
|
||||
│ │ PostgreSQL │ │
|
||||
│ │ ┌────────────────┐ │ │
|
||||
│ │ │ flyer_crawler │ │ (main app database) │
|
||||
│ │ ├────────────────┤ │ │
|
||||
│ │ │ bugsink │ │ (error tracking database) │
|
||||
│ │ └────────────────┘ │ │
|
||||
│ └──────────────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
External (Developer Machine):
|
||||
┌──────────────────────────────────────┐
|
||||
│ Claude Code / Cursor / VS Code │
|
||||
│ ┌────────────────────────────────┐ │
|
||||
│ │ sentry-selfhosted-mcp │ │
|
||||
│ │ (MCP Server) │ │
|
||||
│ │ │ │
|
||||
│ │ SENTRY_URL=http://localhost:8000
|
||||
│ │ SENTRY_AUTH_TOKEN=... │ │
|
||||
│ │ SENTRY_ORG_SLUG=... │ │
|
||||
│ └────────────────────────────────┘ │
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Default (Dev) |
|
||||
| ------------------ | ------------------------------ | -------------------------- |
|
||||
| `BUGSINK_DSN` | Sentry-compatible DSN for SDKs | Set after project creation |
|
||||
| `BUGSINK_ENABLED` | Enable/disable error reporting | `true` |
|
||||
| `BUGSINK_BASE_URL` | Bugsink web UI URL (internal) | `http://localhost:8000` |
|
||||
|
||||
### PostgreSQL Setup
|
||||
|
||||
```sql
|
||||
-- Create dedicated Bugsink database and user
|
||||
CREATE USER bugsink WITH PASSWORD 'bugsink_dev_password';
|
||||
CREATE DATABASE bugsink OWNER bugsink;
|
||||
GRANT ALL PRIVILEGES ON DATABASE bugsink TO bugsink;
|
||||
```
|
||||
|
||||
### Bugsink Configuration
|
||||
|
||||
```bash
|
||||
# Environment variables for Bugsink service
|
||||
SECRET_KEY=<random-50-char-string>
|
||||
DATABASE_URL=postgresql://bugsink:bugsink_dev_password@localhost:5432/bugsink
|
||||
BASE_URL=http://localhost:8000
|
||||
PORT=8000
|
||||
```
|
||||
|
||||
### Logstash Pipeline
|
||||
|
||||
```conf
|
||||
# /etc/logstash/conf.d/bugsink.conf
|
||||
|
||||
# === INPUTS ===
|
||||
input {
|
||||
# Pino application logs
|
||||
file {
|
||||
path => "/app/logs/*.log"
|
||||
codec => json
|
||||
type => "pino"
|
||||
tags => ["app"]
|
||||
}
|
||||
|
||||
# Redis logs
|
||||
file {
|
||||
path => "/var/log/redis/*.log"
|
||||
type => "redis"
|
||||
tags => ["redis"]
|
||||
}
|
||||
|
||||
# PostgreSQL logs (for function logging - future)
|
||||
# file {
|
||||
# path => "/var/log/postgresql/*.log"
|
||||
# type => "postgres"
|
||||
# tags => ["postgres"]
|
||||
# }
|
||||
}
|
||||
|
||||
# === FILTERS ===
|
||||
filter {
|
||||
# Pino error detection (level 50 = error, 60 = fatal)
|
||||
if [type] == "pino" and [level] >= 50 {
|
||||
mutate { add_tag => ["error"] }
|
||||
}
|
||||
|
||||
# Redis error detection
|
||||
if [type] == "redis" {
|
||||
grok {
|
||||
match => { "message" => "%{POSINT:pid}:%{WORD:role} %{MONTHDAY} %{MONTH} %{TIME} %{WORD:loglevel} %{GREEDYDATA:redis_message}" }
|
||||
}
|
||||
if [loglevel] in ["WARNING", "ERROR"] {
|
||||
mutate { add_tag => ["error"] }
|
||||
}
|
||||
}
|
||||
|
||||
# PostgreSQL function error detection (future)
|
||||
# if [type] == "postgres" {
|
||||
# # Parse PostgreSQL log format and detect ERROR/FATAL levels
|
||||
# }
|
||||
}
|
||||
|
||||
# === OUTPUT ===
|
||||
output {
|
||||
if "error" in [tags] {
|
||||
http {
|
||||
url => "http://localhost:8000/api/store/"
|
||||
http_method => "post"
|
||||
format => "json"
|
||||
# Sentry envelope format
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
1. **Update Dockerfile.dev**:
|
||||
- Install Bugsink (pip package or binary)
|
||||
- Install Logstash (Elastic APT repository)
|
||||
- Add systemd service files for both
|
||||
|
||||
2. **PostgreSQL initialization**:
|
||||
- Add Bugsink user/database creation to `sql/00-init-extensions.sql`
|
||||
|
||||
3. **Backend SDK integration**:
|
||||
- Install `@sentry/node`
|
||||
- Initialize in `server.ts` before Express app
|
||||
- Configure error handler middleware integration
|
||||
|
||||
4. **Frontend SDK integration**:
|
||||
- Install `@sentry/react`
|
||||
- Wrap `App` component with `Sentry.ErrorBoundary`
|
||||
- Configure in `src/index.tsx`
|
||||
|
||||
5. **Environment configuration**:
|
||||
- Add Bugsink variables to `src/config/env.ts`
|
||||
- Update `.env.example` and `compose.dev.yml`
|
||||
|
||||
6. **Logstash configuration**:
|
||||
- Create pipeline config for Pino → Bugsink
|
||||
- Configure Pino to write to log file in addition to stdout
|
||||
- Configure Redis log monitoring (connection errors, slow commands)
|
||||
|
||||
7. **MCP server documentation**:
|
||||
- Document `sentry-selfhosted-mcp` setup in CLAUDE.md
|
||||
|
||||
8. **PostgreSQL function logging** (future):
|
||||
- Configure PostgreSQL to log function execution errors
|
||||
- Add Logstash input for PostgreSQL logs
|
||||
- Define filter rules for function-level error detection
|
||||
- _Note: Ask for implementation details when this step is reached_
|
||||
|
||||
## Consequences
|
||||
|
||||
**Positive**: Provides critical observability into the application's real-world behavior. Enables proactive identification and resolution of performance bottlenecks and errors. Improves overall application reliability and user experience.
|
||||
**Negative**: Introduces a new third-party dependency and potential subscription costs. Requires initial setup and configuration of the APM/error tracking agent.
|
||||
### Positive
|
||||
|
||||
- **Full observability**: Aggregated view of errors, trends, and performance
|
||||
- **Self-hosted**: No external SaaS dependencies or subscription costs
|
||||
- **SDK compatibility**: Leverages mature Sentry SDKs with excellent documentation
|
||||
- **AI integration**: MCP server enables Claude Code to query and analyze errors
|
||||
- **Unified architecture**: Same setup works in dev container and production
|
||||
- **Lightweight**: Bugsink runs in a single process, unlike full Sentry (16GB+ RAM)
|
||||
|
||||
### Negative
|
||||
|
||||
- **Additional services**: Bugsink and Logstash add complexity to the container
|
||||
- **PostgreSQL overhead**: Additional database for error tracking
|
||||
- **Initial setup**: Requires configuration of multiple components
|
||||
- **Logstash learning curve**: Pipeline configuration requires Logstash knowledge
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. **Full Sentry self-hosted**: Rejected due to complexity (Kafka, Redis, ClickHouse, 16GB+ RAM minimum)
|
||||
2. **GlitchTip**: Considered, but Bugsink is lighter weight and easier to deploy
|
||||
3. **Sentry SaaS**: Rejected due to self-hosted requirement
|
||||
4. **Custom error aggregation**: Rejected in favor of proven Sentry SDK ecosystem
|
||||
|
||||
## References
|
||||
|
||||
- [Bugsink Documentation](https://www.bugsink.com/docs/)
|
||||
- [Bugsink Docker Install](https://www.bugsink.com/docs/docker-install/)
|
||||
- [@sentry/node Documentation](https://docs.sentry.io/platforms/javascript/guides/node/)
|
||||
- [@sentry/react Documentation](https://docs.sentry.io/platforms/javascript/guides/react/)
|
||||
- [sentry-selfhosted-mcp](https://github.com/ddfourtwo/sentry-selfhosted-mcp)
|
||||
- [Logstash Reference](https://www.elastic.co/guide/en/logstash/current/index.html)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user