Compare commits

..

39 Commits

Author SHA1 Message Date
Gitea Actions
a14816c8ee ci: Bump version to 0.11.0 for production release [skip ci] 2026-01-18 05:02:54 +05:00
Gitea Actions
08b220e29c ci: Bump version to 0.10.0 for production release [skip ci] 2026-01-18 04:50:17 +05:00
Gitea Actions
d41a3f1887 ci: Bump version to 0.9.115 [skip ci] 2026-01-18 04:10:18 +05:00
1f6cdc62d7 still fixin test
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 16m20s
2026-01-17 15:09:17 -08:00
Gitea Actions
978c63bacd ci: Bump version to 0.9.114 [skip ci] 2026-01-18 04:00:21 +05:00
544eb7ae3c still fixin test
Some checks failed
Deploy to Test Environment / deploy-to-test (push) Failing after 2m1s
2026-01-17 14:59:01 -08:00
Gitea Actions
f6839f6e14 ci: Bump version to 0.9.113 [skip ci] 2026-01-18 03:35:25 +05:00
3fac29436a still fixin test
Some checks failed
Deploy to Test Environment / deploy-to-test (push) Failing after 2m6s
2026-01-17 14:34:18 -08:00
Gitea Actions
56f45c9301 ci: Bump version to 0.9.112 [skip ci] 2026-01-18 03:19:53 +05:00
83460abce4 md fixin
Some checks failed
Deploy to Test Environment / deploy-to-test (push) Failing after 1m57s
2026-01-17 14:18:55 -08:00
Gitea Actions
1b084b2ba4 ci: Bump version to 0.9.111 [skip ci] 2026-01-18 02:56:20 +05:00
0ea034bdc8 push
Some checks failed
Deploy to Test Environment / deploy-to-test (push) Failing after 1m54s
2026-01-17 13:55:22 -08:00
Gitea Actions
fc9e27078a ci: Bump version to 0.9.110 [skip ci] 2026-01-18 02:41:36 +05:00
fb8cbe8007 update mcp and created new test user and reset passes
Some checks failed
Deploy to Test Environment / deploy-to-test (push) Failing after 1m56s
2026-01-17 13:40:31 -08:00
f49f786c23 fix: Add .env file loading to ecosystem-test.config.cjs
Allows test environment PM2 processes to load environment variables
from /var/www/flyer-crawler-test.projectium.com/.env file, enabling
manual restarts without requiring CI/CD to inject variables.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-17 13:38:15 -08:00
Gitea Actions
dd31141d4e ci: Bump version to 0.9.109 [skip ci] 2026-01-13 23:09:47 +05:00
8073094760 testing/staging fixin
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 16m15s
2026-01-13 10:08:28 -08:00
Gitea Actions
33a1e146ab ci: Bump version to 0.9.108 [skip ci] 2026-01-13 22:34:20 +05:00
4f8216db77 testing/staging fixin
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 15m55s
2026-01-13 09:33:38 -08:00
Gitea Actions
42d605d19f ci: Bump version to 0.9.107 [skip ci] 2026-01-13 22:06:39 +05:00
749350df7f testing/staging fixin
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 15m56s
2026-01-13 09:03:42 -08:00
Gitea Actions
ac085100fe ci: Bump version to 0.9.106 [skip ci] 2026-01-13 21:43:43 +05:00
ce4ecd1268 use port 3002 in test
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 16m13s
2026-01-13 08:42:34 -08:00
Gitea Actions
a57cfc396b ci: Bump version to 0.9.105 [skip ci] 2026-01-13 21:00:45 +05:00
987badbf8d use port 3002 in test
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 15m41s
2026-01-13 07:59:49 -08:00
Gitea Actions
d38fcd21c1 ci: Bump version to 0.9.104 [skip ci] 2026-01-13 08:11:38 +05:00
6e36cc3b07 logging + e2e test fixes
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 16m34s
2026-01-12 19:10:29 -08:00
Gitea Actions
62a8a8bf4b ci: Bump version to 0.9.103 [skip ci] 2026-01-13 06:39:39 +05:00
96038cfcf4 logging work - almost there
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 15m51s
2026-01-12 17:38:58 -08:00
Gitea Actions
981214fdd0 ci: Bump version to 0.9.102 [skip ci] 2026-01-13 06:27:55 +05:00
92b0138108 logging work - almost there
Some checks failed
Deploy to Test Environment / deploy-to-test (push) Failing after 1m2s
2026-01-12 17:26:59 -08:00
Gitea Actions
27f0255240 ci: Bump version to 0.9.101 [skip ci] 2026-01-13 05:57:55 +05:00
4e06dde9e1 logging work - almost there
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 15m30s
2026-01-12 16:57:18 -08:00
Gitea Actions
b9a0e5b82c ci: Bump version to 0.9.100 [skip ci] 2026-01-13 05:35:11 +05:00
bb7fe8dc2c logging work - almost there
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 15m28s
2026-01-12 16:34:18 -08:00
Gitea Actions
81f1f2250b ci: Bump version to 0.9.99 [skip ci] 2026-01-13 05:08:56 +05:00
c6c90bb615 more new feature fixes + sentry logging
All checks were successful
Deploy to Test Environment / deploy-to-test (push) Successful in 15m53s
2026-01-12 16:08:18 -08:00
Gitea Actions
60489a626b ci: Bump version to 0.9.98 [skip ci] 2026-01-13 05:05:59 +05:00
3c63e1ecbb more new feature fixes + sentry logging
Some checks failed
Deploy to Test Environment / deploy-to-test (push) Has been cancelled
2026-01-12 16:04:09 -08:00
53 changed files with 6063 additions and 417 deletions

View File

@@ -91,7 +91,14 @@
"Bash(ping:*)",
"Bash(tee:*)",
"Bash(timeout 1800 podman exec flyer-crawler-dev npm run test:unit:*)",
"mcp__filesystem__edit_file"
"mcp__filesystem__edit_file",
"Bash(timeout 300 tail:*)",
"mcp__filesystem__list_allowed_directories",
"mcp__memory__add_observations",
"Bash(ssh:*)",
"mcp__redis__list",
"Read(//d/gitea/bugsink-mcp/**)",
"Bash(d:/nodejs/npm.cmd install)"
]
}
}

View File

@@ -63,8 +63,8 @@ jobs:
- name: Check for Production Database Schema Changes
env:
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_USER: ${{ secrets.DB_USER_PROD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
run: |
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
@@ -117,8 +117,8 @@ jobs:
env:
# --- Production Secrets Injection ---
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_USER: ${{ secrets.DB_USER_PROD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
# Explicitly use database 0 for production (test uses database 1)
REDIS_URL: 'redis://localhost:6379/0'
@@ -171,7 +171,7 @@ jobs:
else
echo "Version mismatch (Running: $RUNNING_VERSION -> Deployed: $NEW_VERSION) or app not running. Reloading PM2..."
fi
pm2 startOrReload ecosystem.config.cjs --env production --update-env && pm2 save
pm2 startOrReload ecosystem.config.cjs --update-env && pm2 save
echo "Production backend server reloaded successfully."
else
echo "Version $NEW_VERSION is already running. Skipping PM2 reload."

View File

@@ -121,10 +121,11 @@ jobs:
env:
# --- Database credentials for the test suite ---
# These are injected from Gitea secrets into the runner's environment.
# CRITICAL: Use TEST-specific credentials that have CREATE privileges on the public schema.
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_NAME: 'flyer-crawler-test' # Explicitly set for tests
DB_USER: ${{ secrets.DB_USER_TEST }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
# --- Redis credentials for the test suite ---
# CRITICAL: Use Redis database 1 to isolate tests from production (which uses db 0).
@@ -328,10 +329,11 @@ jobs:
- name: Check for Test Database Schema Changes
env:
# Use test database credentials for this check.
# CRITICAL: Use TEST-specific credentials that have CREATE privileges on the public schema.
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # This is used by psql
DB_NAME: ${{ secrets.DB_DATABASE_TEST }} # This is used by the application
DB_USER: ${{ secrets.DB_USER_TEST }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
run: |
# Fail-fast check to ensure secrets are configured in Gitea.
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
@@ -427,9 +429,10 @@ jobs:
# Your Node.js application will read these directly from `process.env`.
# Database Credentials
# CRITICAL: Use TEST-specific credentials that have CREATE privileges on the public schema.
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_USER: ${{ secrets.DB_USER_TEST }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
# Redis Credentials (use database 1 to isolate from production)
@@ -476,10 +479,11 @@ jobs:
echo "Cleaning up errored or stopped PM2 processes..."
node -e "const exec = require('child_process').execSync; try { const list = JSON.parse(exec('pm2 jlist').toString()); list.forEach(p => { if (p.pm2_env.status === 'errored' || p.pm2_env.status === 'stopped') { console.log('Deleting ' + p.pm2_env.status + ' process: ' + p.name + ' (' + p.pm2_env.pm_id + ')'); try { exec('pm2 delete ' + p.pm2_env.pm_id); } catch(e) { console.error('Failed to delete ' + p.pm2_env.pm_id); } } }); } catch (e) { console.error('Error cleaning up processes:', e); }"
# Use `startOrReload` with the ecosystem file. This is the standard, idempotent way to deploy.
# It will START the process if it's not running, or RELOAD it if it is.
# Use `startOrReload` with the TEST ecosystem file. This starts test-specific processes
# (flyer-crawler-api-test, flyer-crawler-worker-test, flyer-crawler-analytics-worker-test)
# that run separately from production processes.
# We also add `&& pm2 save` to persist the process list across server reboots.
pm2 startOrReload ecosystem.config.cjs --env test --update-env && pm2 save
pm2 startOrReload ecosystem-test.config.cjs --update-env && pm2 save
echo "Test backend server reloaded successfully."
# After a successful deployment, update the schema hash in the database.

View File

@@ -20,9 +20,9 @@ jobs:
# Use production database credentials for this entire job.
DB_HOST: ${{ secrets.DB_HOST }}
DB_PORT: ${{ secrets.DB_PORT }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_NAME: ${{ secrets.DB_NAME_PROD }}
DB_USER: ${{ secrets.DB_USER_PROD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
steps:
- name: Validate Secrets

View File

@@ -23,9 +23,9 @@ jobs:
env:
# Use production database credentials for this entire job.
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
DB_NAME: ${{ secrets.DB_DATABASE_PROD }} # Used by the application
DB_USER: ${{ secrets.DB_USER_PROD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
steps:
- name: Checkout Code

View File

@@ -23,9 +23,9 @@ jobs:
env:
# Use test database credentials for this entire job.
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }} # Used by psql
DB_NAME: ${{ secrets.DB_DATABASE_TEST }} # Used by the application
DB_USER: ${{ secrets.DB_USER_TEST }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_TEST }}
DB_NAME: ${{ secrets.DB_DATABASE_TEST }}
steps:
- name: Checkout Code

View File

@@ -22,8 +22,8 @@ jobs:
env:
# Use production database credentials for this entire job.
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_USER: ${{ secrets.DB_USER_PROD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
BACKUP_DIR: '/var/www/backups' # Define a dedicated directory for backups

View File

@@ -62,8 +62,8 @@ jobs:
- name: Check for Production Database Schema Changes
env:
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_USER: ${{ secrets.DB_USER_PROD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
run: |
if [ -z "$DB_HOST" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_NAME" ]; then
@@ -113,8 +113,8 @@ jobs:
env:
# --- Production Secrets Injection ---
DB_HOST: ${{ secrets.DB_HOST }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_USER: ${{ secrets.DB_USER_PROD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
# Explicitly use database 0 for production (test uses database 1)
REDIS_URL: 'redis://localhost:6379/0'

1
.gitignore vendored
View File

@@ -37,3 +37,4 @@ test-output.txt
Thumbs.db
.claude
nul
tmpclaude*

378
CLAUDE-MCP.md Normal file
View File

@@ -0,0 +1,378 @@
# Claude Code MCP Configuration Guide
This document explains how to configure MCP (Model Context Protocol) servers for Claude Code, covering both the CLI and VS Code extension.
## The Two Config Files
Claude Code uses **two separate configuration files** for MCP servers. They must be kept in sync manually.
| File | Used By | Notes |
| ------------------------- | ----------------------------- | ------------------------------------------- |
| `~/.claude.json` | Claude CLI (`claude` command) | Requires `"type": "stdio"` in each server |
| `~/.claude/settings.json` | VS Code Extension | Simpler format, supports `"disabled": true` |
**Important:** Changes to one file do NOT automatically sync to the other!
## File Locations (Windows)
```text
C:\Users\<username>\.claude.json # CLI config
C:\Users\<username>\.claude\settings.json # VS Code extension config
```
## Config Format Differences
### VS Code Extension Format (`~/.claude/settings.json`)
```json
{
"mcpServers": {
"server-name": {
"command": "path/to/executable",
"args": ["arg1", "arg2"],
"env": {
"ENV_VAR": "value"
},
"disabled": true // Optional - disable without removing
}
}
}
```
### CLI Format (`~/.claude.json`)
The CLI config is a larger file with many settings. The `mcpServers` section is nested within it:
```json
{
"numStartups": 14,
"installMethod": "global",
// ... other settings ...
"mcpServers": {
"server-name": {
"type": "stdio", // REQUIRED for CLI
"command": "path/to/executable",
"args": ["arg1", "arg2"],
"env": {
"ENV_VAR": "value"
}
}
}
// ... more settings ...
}
```
**Key difference:** CLI format requires `"type": "stdio"` in each server definition.
## Common MCP Server Examples
### Memory (Knowledge Graph)
```json
// VS Code format
"memory": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-memory"]
}
// CLI format
"memory": {
"type": "stdio",
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-memory"],
"env": {}
}
```
### Filesystem
```json
// VS Code format
"filesystem": {
"command": "d:\\nodejs\\node.exe",
"args": [
"c:\\Users\\<user>\\AppData\\Roaming\\npm\\node_modules\\@modelcontextprotocol\\server-filesystem\\dist\\index.js",
"d:\\path\\to\\project"
]
}
// CLI format
"filesystem": {
"type": "stdio",
"command": "d:\\nodejs\\node.exe",
"args": [
"c:\\Users\\<user>\\AppData\\Roaming\\npm\\node_modules\\@modelcontextprotocol\\server-filesystem\\dist\\index.js",
"d:\\path\\to\\project"
],
"env": {}
}
```
### Podman/Docker
```json
// VS Code format
"podman": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "podman-mcp-server@latest"],
"env": {
"DOCKER_HOST": "npipe:////./pipe/podman-machine-default"
}
}
```
### Gitea
```json
// VS Code format
"gitea-myserver": {
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
"args": ["run", "-t", "stdio"],
"env": {
"GITEA_HOST": "https://gitea.example.com",
"GITEA_ACCESS_TOKEN": "your-token-here"
}
}
```
### Redis
```json
// VS Code format
"redis": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-redis", "redis://localhost:6379"]
}
```
### Bugsink (Error Tracking)
**Important:** Bugsink has a different API than Sentry. Use `bugsink-mcp`, NOT `sentry-selfhosted-mcp`.
**Note:** The `bugsink-mcp` npm package is NOT published. You must clone and build from source:
```bash
# Clone and build bugsink-mcp
git clone https://github.com/j-shelfwood/bugsink-mcp.git d:\gitea\bugsink-mcp
cd d:\gitea\bugsink-mcp
npm install
npm run build
```
```json
// VS Code format (using locally built version)
"bugsink": {
"command": "d:\\nodejs\\node.exe",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "https://bugsink.example.com",
"BUGSINK_TOKEN": "your-api-token"
}
}
// CLI format
"bugsink": {
"type": "stdio",
"command": "d:\\nodejs\\node.exe",
"args": ["d:\\gitea\\bugsink-mcp\\dist\\index.js"],
"env": {
"BUGSINK_URL": "https://bugsink.example.com",
"BUGSINK_TOKEN": "your-api-token"
}
}
```
- GitHub: <https://github.com/j-shelfwood/bugsink-mcp>
- Get token from Bugsink UI: Settings > API Tokens
- **Do NOT use npx** - the package is not on npm
### Sentry (Cloud or Self-hosted)
For actual Sentry instances (not Bugsink), use:
```json
"sentry": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@sentry/mcp-server"],
"env": {
"SENTRY_AUTH_TOKEN": "your-sentry-token"
}
}
```
## Troubleshooting
### Server Not Loading
1. **Check both config files** - Make sure the server is defined in both `~/.claude.json` AND `~/.claude/settings.json`
2. **Verify server order** - Servers load sequentially. Broken/slow servers can block others. Put important servers first.
3. **Check for timeout** - Each server has 30 seconds to connect. Slow npx downloads can cause timeouts.
4. **Fully restart VS Code** - Window reload is not enough. Close all VS Code windows and reopen.
### Verifying Configuration
**For CLI:**
```bash
claude mcp list
```
**For VS Code:**
1. Open VS Code
2. View → Output
3. Select "Claude" from the dropdown
4. Look for MCP server connection logs
### Common Errors
| Error | Cause | Solution |
| ------------------------------------ | ----------------------------- | --------------------------------------------------------------------------- |
| `Connection timed out after 30000ms` | Server took too long to start | Move server earlier in config, or use pre-installed packages instead of npx |
| `npm error 404 Not Found` | Package doesn't exist | Check package name spelling |
| `The system cannot find the path` | Wrong executable path | Verify the command path exists |
| `Connection closed` | Server crashed on startup | Check server logs, verify environment variables |
### Disabling Problem Servers
In `~/.claude/settings.json`, add `"disabled": true`:
```json
"problem-server": {
"command": "...",
"args": ["..."],
"disabled": true
}
```
**Note:** The CLI config (`~/.claude.json`) does not support the `disabled` flag. You must remove the server entirely from that file.
## Adding a New MCP Server
1. **Install/clone the MCP server** (if not using npx)
2. **Add to VS Code config** (`~/.claude/settings.json`):
```json
"new-server": {
"command": "path/to/command",
"args": ["arg1", "arg2"],
"env": { "VAR": "value" }
}
```
3. **Add to CLI config** (`~/.claude.json`) - find the `mcpServers` section:
```json
"new-server": {
"type": "stdio",
"command": "path/to/command",
"args": ["arg1", "arg2"],
"env": { "VAR": "value" }
}
```
4. **Fully restart VS Code**
5. **Verify with `claude mcp list`**
## Quick Reference: Available MCP Servers
| Server | Package/Repo | Purpose |
| ------------------- | -------------------------------------------------- | --------------------------- |
| memory | `@modelcontextprotocol/server-memory` | Knowledge graph persistence |
| filesystem | `@modelcontextprotocol/server-filesystem` | File system access |
| redis | `@modelcontextprotocol/server-redis` | Redis cache inspection |
| postgres | `@modelcontextprotocol/server-postgres` | PostgreSQL queries |
| sequential-thinking | `@modelcontextprotocol/server-sequential-thinking` | Step-by-step reasoning |
| podman | `podman-mcp-server` | Container management |
| gitea | `gitea-mcp` (binary) | Gitea API access |
| bugsink | `j-shelfwood/bugsink-mcp` (build from source) | Error tracking for Bugsink |
| sentry | `@sentry/mcp-server` | Error tracking for Sentry |
| playwright | `@anthropics/mcp-server-playwright` | Browser automation |
## Best Practices
1. **Keep configs in sync** - When you change one file, update the other
2. **Order servers by importance** - Put essential servers (memory, filesystem) first
3. **Disable instead of delete** - Use `"disabled": true` in settings.json to troubleshoot
4. **Use node.exe directly** - For faster startup, install packages globally and use `node.exe` instead of `npx`
5. **Store sensitive data in memory** - Use the memory MCP to store API tokens and config for future sessions
---
## Future: MCP Launchpad
**Project:** <https://github.com/kenneth-liao/mcp-launchpad>
MCP Launchpad is a CLI tool that wraps multiple MCP servers into a single interface. Worth revisiting when:
- [ ] Windows support is stable (currently experimental)
- [ ] Available as an MCP server itself (currently Bash-based)
**Why it's interesting:**
| Benefit | Description |
| ---------------------- | -------------------------------------------------------------- |
| Single config file | No more syncing `~/.claude.json` and `~/.claude/settings.json` |
| Project-level configs | Drop `mcp.json` in any project for instant MCP setup |
| Context window savings | One MCP server in context instead of 10+, reducing token usage |
| Persistent daemon | Keeps server connections alive for faster repeated calls |
| Tool search | Find tools across all servers with `mcpl search` |
**Current limitations:**
- Experimental Windows support
- Requires Python 3.13+ and uv
- Claude calls tools via Bash instead of native MCP integration
- Different mental model (runtime discovery vs startup loading)
---
## Future: Graphiti (Advanced Knowledge Graph)
**Project:** <https://github.com/getzep/graphiti>
Graphiti provides temporal-aware knowledge graphs - it tracks not just facts, but _when_ they became true/outdated. Much more powerful than simple memory MCP, but requires significant infrastructure.
**Ideal setup:** Run on a Linux server, connect via HTTP from Windows:
```json
// Windows client config (settings.json)
"graphiti": {
"type": "sse",
"url": "http://linux-server:8000/mcp/"
}
```
**Linux server setup:**
```bash
git clone https://github.com/getzep/graphiti.git
cd graphiti/mcp_server
docker compose up -d # Starts FalkorDB + MCP server on port 8000
```
**Requirements:**
- Docker on Linux server
- OpenAI API key (for embeddings)
- Port 8000 open on LAN
**Benefits of remote deployment:**
- Heavy lifting (Neo4j/FalkorDB + embeddings) offloaded to Linux
- Always-on server, Windows connects/disconnects freely
- Multiple machines can share the same knowledge graph
- Avoids Windows Docker/WSL2 complexity
---
\_Last updated: January 2026

118
CLAUDE.md
View File

@@ -1,5 +1,35 @@
# Claude Code Project Instructions
## Session Startup Checklist
**IMPORTANT**: At the start of every session, perform these steps:
1. **Check Memory First** - Use `mcp__memory__read_graph` or `mcp__memory__search_nodes` to recall:
- Project-specific configurations and credentials
- Previous work context and decisions
- Infrastructure details (URLs, ports, access patterns)
- Known issues and their solutions
2. **Review Recent Git History** - Check `git log --oneline -10` to understand recent changes
3. **Check Container Status** - Use `mcp__podman__container_list` to see what's running
---
## Project Instructions
### Things to Remember
Before writing any code:
1. State how you will verify this change works (test, bash command, browser check, etc.)
2. Write the test or verification step first
3. Then implement the code
4. Run verification and iterate until it passes
## Communication Style: Ask Before Assuming
**IMPORTANT**: When helping with tasks, **ask clarifying questions before making assumptions**. Do not assume:
@@ -40,10 +70,16 @@ npm run test:integration # Run integration tests (requires DB/Redis)
### Running Tests via Podman (from Windows host)
**Note:** This project has 2900+ unit tests. For AI-assisted development, pipe output to a file for easier processing.
The command to run unit tests in the dev container via podman:
```bash
# Basic (output to terminal)
podman exec -it flyer-crawler-dev npm run test:unit
# Recommended for AI processing: pipe to file
podman exec -it flyer-crawler-dev npm run test:unit 2>&1 | tee test-results.txt
```
The command to run integration tests in the dev container via podman:
@@ -257,7 +293,7 @@ To add a new secret (e.g., `SENTRY_DSN`):
**Shared (used by both environments):**
- `DB_HOST`, `DB_USER`, `DB_PASSWORD` - Database credentials
- `DB_HOST` - Database host (shared PostgreSQL server)
- `JWT_SECRET` - Authentication
- `GOOGLE_MAPS_API_KEY` - Google Maps
- `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET` - Google OAuth
@@ -265,14 +301,16 @@ To add a new secret (e.g., `SENTRY_DSN`):
**Production-specific:**
- `DB_DATABASE_PROD` - Production database name
- `DB_USER_PROD`, `DB_PASSWORD_PROD` - Production database credentials (`flyer_crawler_prod`)
- `DB_DATABASE_PROD` - Production database name (`flyer-crawler`)
- `REDIS_PASSWORD_PROD` - Redis password (uses database 0)
- `VITE_GOOGLE_GENAI_API_KEY` - Gemini API key for production
- `SENTRY_DSN`, `VITE_SENTRY_DSN` - Bugsink error tracking DSNs (production projects)
**Test-specific:**
- `DB_DATABASE_TEST` - Test database name
- `DB_USER_TEST`, `DB_PASSWORD_TEST` - Test database credentials (`flyer_crawler_test`)
- `DB_DATABASE_TEST` - Test database name (`flyer-crawler-test`)
- `REDIS_PASSWORD_TEST` - Redis password (uses database 1 for isolation)
- `VITE_GOOGLE_GENAI_API_KEY_TEST` - Gemini API key for test
- `SENTRY_DSN_TEST`, `VITE_SENTRY_DSN_TEST` - Bugsink error tracking DSNs (test projects)
@@ -286,6 +324,55 @@ The test environment (`flyer-crawler-test.projectium.com`) uses **both** Gitea C
- **Redis database 1**: Isolates test job queues from production (which uses database 0)
- **PM2 process names**: Suffixed with `-test` (e.g., `flyer-crawler-api-test`)
### Database User Setup (Test Environment)
**CRITICAL**: The test database requires specific PostgreSQL permissions to be configured manually. Schema ownership alone is NOT sufficient - explicit privileges must be granted.
**Database Users:**
| User | Database | Purpose |
| -------------------- | -------------------- | ---------- |
| `flyer_crawler_prod` | `flyer-crawler` | Production |
| `flyer_crawler_test` | `flyer-crawler-test` | Testing |
**Required Setup Commands** (run as `postgres` superuser):
```bash
# Connect as postgres superuser
sudo -u postgres psql
# Create the test database and user (if not exists)
CREATE DATABASE "flyer-crawler-test";
CREATE USER flyer_crawler_test WITH PASSWORD 'your-password-here';
# Grant ownership and privileges
ALTER DATABASE "flyer-crawler-test" OWNER TO flyer_crawler_test;
\c "flyer-crawler-test"
ALTER SCHEMA public OWNER TO flyer_crawler_test;
GRANT CREATE, USAGE ON SCHEMA public TO flyer_crawler_test;
# Create required extension (must be done by superuser)
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
```
**Why These Steps Are Necessary:**
1. **Schema ownership alone is insufficient** - PostgreSQL requires explicit `GRANT CREATE, USAGE` privileges even when the user owns the schema
2. **uuid-ossp extension** - Required by the application for UUID generation; must be created by a superuser before the app can use it
3. **Separate users for prod/test** - Prevents accidental cross-environment data access; each environment has its own credentials in Gitea secrets
**Verification:**
```bash
# Check schema privileges (should show 'UC' for flyer_crawler_test)
psql -d "flyer-crawler-test" -c "\dn+ public"
# Expected output:
# Name | Owner | Access privileges
# -------+--------------------+------------------------------------------
# public | flyer_crawler_test | flyer_crawler_test=UC/flyer_crawler_test
```
### Dev Container Environment
The dev container runs its own **local Bugsink instance** - it does NOT connect to the production Bugsink server:
@@ -317,7 +404,7 @@ The following MCP servers are configured for this project:
| redis | Redis cache inspection (localhost:6379) |
| sentry-selfhosted-mcp | Error tracking via Bugsink (localhost:8000) |
**Note:** MCP servers are currently only available in **Claude CLI**. Due to a bug in Claude VS Code extension, MCP servers do not work there yet.
**Note:** MCP servers work in both **Claude CLI** and **Claude Code VS Code extension** (as of January 2026).
### Sentry/Bugsink MCP Server Setup (ADR-015)
@@ -360,3 +447,26 @@ To enable Claude Code to query and analyze application errors from Bugsink:
- Search by error message or stack trace
- Update issue status (resolve, ignore)
- Add comments to issues
### SSH Server Access
Claude Code can execute commands on the production server via SSH:
```bash
# Basic command execution
ssh root@projectium.com "command here"
# Examples:
ssh root@projectium.com "systemctl status logstash"
ssh root@projectium.com "pm2 list"
ssh root@projectium.com "tail -50 /var/www/flyer-crawler.projectium.com/logs/app.log"
```
**Use cases:**
- Managing Logstash, PM2, NGINX, Redis services
- Viewing server logs
- Deploying configuration changes
- Checking service status
**Important:** SSH access requires the host machine to have SSH keys configured for `root@projectium.com`.

View File

@@ -149,7 +149,7 @@ SELECT PostGIS_Full_Version();
Example output:
```
```text
PostgreSQL 14.19 (Ubuntu 14.19-0ubuntu0.22.04.1)
POSTGIS="3.2.0 c3e3cc0" GEOS="3.10.2-CAPI-1.16.0" PROJ="8.2.1"
```

View File

@@ -244,19 +244,87 @@ For detailed information on secrets management, see [CLAUDE.md](../CLAUDE.md).
sudo npm install -g pm2
```
### Start Application with PM2
### PM2 Configuration Files
The application uses **separate ecosystem config files** for production and test environments:
| File | Purpose | Processes Started |
| --------------------------- | --------------------- | -------------------------------------------------------------------------------------------- |
| `ecosystem.config.cjs` | Production deployment | `flyer-crawler-api`, `flyer-crawler-worker`, `flyer-crawler-analytics-worker` |
| `ecosystem-test.config.cjs` | Test deployment | `flyer-crawler-api-test`, `flyer-crawler-worker-test`, `flyer-crawler-analytics-worker-test` |
**Key Points:**
- Production and test processes run **simultaneously** with distinct names
- Test processes use `NODE_ENV=test` which enables file logging
- Test processes use Redis database 1 (isolated from production which uses database 0)
- Both configs validate required environment variables but only warn (don't exit) if missing
### Start Production Application
```bash
cd /opt/flyer-crawler
npm run start:prod
cd /var/www/flyer-crawler.projectium.com
# Set required environment variables (usually done via CI/CD)
export DB_HOST=localhost
export JWT_SECRET=your-secret
export GEMINI_API_KEY=your-api-key
# ... other required variables
pm2 startOrReload ecosystem.config.cjs --update-env && pm2 save
```
This starts three processes:
This starts three production processes:
- `flyer-crawler-api` - Main API server (port 3001)
- `flyer-crawler-worker` - Background job worker
- `flyer-crawler-analytics-worker` - Analytics processing worker
### Start Test Application
```bash
cd /var/www/flyer-crawler-test.projectium.com
# Set required environment variables (usually done via CI/CD)
export DB_HOST=localhost
export DB_NAME=flyer-crawler-test
export JWT_SECRET=your-secret
export GEMINI_API_KEY=your-test-api-key
export REDIS_URL=redis://localhost:6379/1 # Use database 1 for isolation
# ... other required variables
pm2 startOrReload ecosystem-test.config.cjs --update-env && pm2 save
```
This starts three test processes (running alongside production):
- `flyer-crawler-api-test` - Test API server (port 3001 via different NGINX vhost)
- `flyer-crawler-worker-test` - Test background job worker
- `flyer-crawler-analytics-worker-test` - Test analytics worker
### Verify Running Processes
After starting both environments, you should see 6 application processes:
```bash
pm2 list
```
Expected output:
```text
┌────┬───────────────────────────────────┬──────────┬────────┬───────────┐
│ id │ name │ mode │ status │ cpu │
├────┼───────────────────────────────────┼──────────┼────────┼───────────┤
│ 0 │ flyer-crawler-api │ cluster │ online │ 0% │
│ 1 │ flyer-crawler-worker │ fork │ online │ 0% │
│ 2 │ flyer-crawler-analytics-worker │ fork │ online │ 0% │
│ 3 │ flyer-crawler-api-test │ fork │ online │ 0% │
│ 4 │ flyer-crawler-worker-test │ fork │ online │ 0% │
│ 5 │ flyer-crawler-analytics-worker-test│ fork │ online │ 0% │
└────┴───────────────────────────────────┴──────────┴────────┴───────────┘
```
### Configure PM2 Startup
```bash
@@ -275,6 +343,22 @@ pm2 set pm2-logrotate:retain 14
pm2 set pm2-logrotate:compress true
```
### Useful PM2 Commands
```bash
# View logs for a specific process
pm2 logs flyer-crawler-api-test --lines 50
# View environment variables for a process
pm2 env <process-id>
# Restart only test processes
pm2 restart flyer-crawler-api-test flyer-crawler-worker-test flyer-crawler-analytics-worker-test
# Delete all test processes (without affecting production)
pm2 delete flyer-crawler-api-test flyer-crawler-worker-test flyer-crawler-analytics-worker-test
```
---
## NGINX Reverse Proxy
@@ -796,16 +880,18 @@ Logstash aggregates logs from the application and infrastructure, forwarding err
### Step 1: Create Application Log Directory
Create the log directory and set appropriate permissions:
The flyer-crawler application automatically creates its log directory on startup, but you need to ensure proper permissions for Logstash to read the logs.
Create the log directories and set appropriate permissions:
```bash
# Create log directory for the flyer-crawler application
# Create log directory for the production application
sudo mkdir -p /var/www/flyer-crawler.projectium.com/logs
# Set ownership to the user running the application (typically the deploy user or www-data)
sudo chown -R $USER:$USER /var/www/flyer-crawler.projectium.com/logs
# Set ownership to root (since PM2 runs as root)
sudo chown -R root:root /var/www/flyer-crawler.projectium.com/logs
# Ensure logstash user can read the logs
# Make logs readable by logstash user
sudo chmod 755 /var/www/flyer-crawler.projectium.com/logs
```
@@ -813,26 +899,47 @@ For the test environment:
```bash
sudo mkdir -p /var/www/flyer-crawler-test.projectium.com/logs
sudo chown -R $USER:$USER /var/www/flyer-crawler-test.projectium.com/logs
sudo chown -R root:root /var/www/flyer-crawler-test.projectium.com/logs
sudo chmod 755 /var/www/flyer-crawler-test.projectium.com/logs
```
### Step 2: Configure Application to Write File Logs
### Step 2: Application File Logging (Already Configured)
The flyer-crawler application uses Pino for logging and currently outputs to stdout (captured by PM2). To enable file-based logging for Logstash, you would need to configure Pino to write to files.
The flyer-crawler application uses Pino for logging and is configured to write logs to files in production/test environments:
**Current Behavior:** Logs go to stdout → PM2 captures them → `~/.pm2/logs/`
**Log File Locations:**
**For Logstash Integration:** You would need to either:
| Environment | Log File Path |
| ------------- | --------------------------------------------------------- |
| Production | `/var/www/flyer-crawler.projectium.com/logs/app.log` |
| Test | `/var/www/flyer-crawler-test.projectium.com/logs/app.log` |
| Dev Container | `/app/logs/app.log` |
1. Configure Pino to write directly to files (requires code changes)
2. Use PM2's log files instead (located at `~/.pm2/logs/flyer-crawler-*.log`)
**How It Works:**
For now, we'll use PM2's log files which already exist:
- In production/test: Pino writes JSON logs to both stdout (for PM2) AND `logs/app.log` (for Logstash)
- In development: Pino uses pino-pretty for human-readable console output only
- The log directory is created automatically if it doesn't exist
- You can override the log directory with the `LOG_DIR` environment variable
**Verify Logging After Deployment:**
After deploying the application, verify that logs are being written:
```bash
# Check PM2 log location
ls -la ~/.pm2/logs/
# Check production logs
ls -la /var/www/flyer-crawler.projectium.com/logs/
tail -f /var/www/flyer-crawler.projectium.com/logs/app.log
# Check test logs
ls -la /var/www/flyer-crawler-test.projectium.com/logs/
tail -f /var/www/flyer-crawler-test.projectium.com/logs/app.log
```
You should see JSON-formatted log entries like:
```json
{ "level": 30, "time": 1704067200000, "msg": "Server started on port 3001", "module": "server" }
```
### Step 3: Install Logstash
@@ -861,14 +968,13 @@ Create the pipeline configuration file:
sudo nano /etc/logstash/conf.d/bugsink.conf
```
Add the following content (adjust paths as needed):
Add the following content:
```conf
input {
# PM2 application logs (Pino JSON format)
# PM2 stores logs in the home directory of the user running PM2
# Production application logs (Pino JSON format)
file {
path => "/root/.pm2/logs/flyer-crawler-api-out.log"
path => "/var/www/flyer-crawler.projectium.com/logs/app.log"
codec => json_lines
type => "pino"
tags => ["app", "production"]
@@ -876,18 +982,9 @@ input {
sincedb_path => "/var/lib/logstash/sincedb_pino_prod"
}
# PM2 error logs
# Test environment logs
file {
path => "/root/.pm2/logs/flyer-crawler-api-error.log"
type => "pm2-error"
tags => ["app", "production", "error"]
start_position => "end"
sincedb_path => "/var/lib/logstash/sincedb_pm2_error_prod"
}
# Test environment logs (if running on same server)
file {
path => "/root/.pm2/logs/flyer-crawler-api-test-out.log"
path => "/var/www/flyer-crawler-test.projectium.com/logs/app.log"
codec => json_lines
type => "pino"
tags => ["app", "test"]
@@ -895,21 +992,61 @@ input {
sincedb_path => "/var/lib/logstash/sincedb_pino_test"
}
# Redis logs
# Redis logs (shared by both environments)
file {
path => "/var/log/redis/redis-server.log"
type => "redis"
tags => ["redis"]
tags => ["infra", "redis", "production"]
start_position => "end"
sincedb_path => "/var/lib/logstash/sincedb_redis"
}
# NGINX error logs (production)
file {
path => "/var/log/nginx/error.log"
type => "nginx"
tags => ["infra", "nginx", "production"]
start_position => "end"
sincedb_path => "/var/lib/logstash/sincedb_nginx_error"
}
# NGINX access logs - for detecting 5xx errors (production)
file {
path => "/var/log/nginx/access.log"
type => "nginx_access"
tags => ["infra", "nginx", "production"]
start_position => "end"
sincedb_path => "/var/lib/logstash/sincedb_nginx_access"
}
# PM2 error logs - Production (plain text stack traces)
file {
path => "/home/gitea-runner/.pm2/logs/flyer-crawler-*-error.log"
exclude => "*-test-error.log"
type => "pm2"
tags => ["infra", "pm2", "production"]
start_position => "end"
sincedb_path => "/var/lib/logstash/sincedb_pm2_prod"
}
# PM2 error logs - Test
file {
path => "/home/gitea-runner/.pm2/logs/flyer-crawler-*-test-error.log"
type => "pm2"
tags => ["infra", "pm2", "test"]
start_position => "end"
sincedb_path => "/var/lib/logstash/sincedb_pm2_test"
}
}
filter {
# Pino error detection (level 50 = error, 60 = fatal)
# Pino log level detection
# Pino levels: 10=trace, 20=debug, 30=info, 40=warn, 50=error, 60=fatal
if [type] == "pino" and [level] {
if [level] >= 50 {
mutate { add_tag => ["error"] }
} else if [level] >= 40 {
mutate { add_tag => ["warning"] }
}
}
@@ -923,66 +1060,144 @@ filter {
}
}
# PM2 error logs are always errors
if [type] == "pm2-error" {
# NGINX error log detection (all entries are errors)
if [type] == "nginx" {
mutate { add_tag => ["error"] }
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} \[%{WORD:severity}\] %{GREEDYDATA:nginx_message}" }
}
}
# NGINX access log - detect 5xx errors
if [type] == "nginx_access" {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
if [response] =~ /^5\d{2}$/ {
mutate { add_tag => ["error"] }
}
}
# PM2 error log detection - tag lines with actual error indicators
if [type] == "pm2" {
if [message] =~ /Error:|error:|ECONNREFUSED|ENOENT|TypeError|ReferenceError|SyntaxError/ {
mutate { add_tag => ["error"] }
}
}
}
output {
# Only send errors to Bugsink
if "error" in [tags] {
# Production app errors -> flyer-crawler-backend (project 1)
if "error" in [tags] and "app" in [tags] and "production" in [tags] {
http {
url => "http://localhost:8000/api/1/store/"
http_method => "post"
format => "json"
headers => {
"X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=YOUR_BACKEND_DSN_KEY"
"X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=YOUR_PROD_BACKEND_DSN_KEY"
}
}
}
# Debug output (remove in production after confirming it works)
# Test app errors -> flyer-crawler-backend-test (project 3)
if "error" in [tags] and "app" in [tags] and "test" in [tags] {
http {
url => "http://localhost:8000/api/3/store/"
http_method => "post"
format => "json"
headers => {
"X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=YOUR_TEST_BACKEND_DSN_KEY"
}
}
}
# Production infrastructure errors (Redis, NGINX, PM2) -> flyer-crawler-infrastructure (project 5)
if "error" in [tags] and "infra" in [tags] and "production" in [tags] {
http {
url => "http://localhost:8000/api/5/store/"
http_method => "post"
format => "json"
headers => {
"X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=b083076f94fb461b889d5dffcbef43bf"
}
}
}
# Test infrastructure errors (PM2 test logs) -> flyer-crawler-test-infrastructure (project 6)
if "error" in [tags] and "infra" in [tags] and "test" in [tags] {
http {
url => "http://localhost:8000/api/6/store/"
http_method => "post"
format => "json"
headers => {
"X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=25020dd6c2b74ad78463ec90e90fadab"
}
}
}
# Debug output (uncomment to troubleshoot)
# stdout { codec => rubydebug }
}
```
**Important:** Replace `YOUR_BACKEND_DSN_KEY` with the key from your Bugsink backend DSN. The key is the part before the `@` symbol in the DSN URL.
**Bugsink Project DSNs:**
For example, if your DSN is:
| Project | DSN Key | Project ID |
| ----------------------------------- | ---------------------------------- | ---------- |
| `flyer-crawler-backend` | `911aef02b9a548fa8fabb8a3c81abfe5` | 1 |
| `flyer-crawler-frontend` | (used by app, not Logstash) | 2 |
| `flyer-crawler-backend-test` | `cdb99c314589431e83d4cc38a809449b` | 3 |
| `flyer-crawler-frontend-test` | (used by app, not Logstash) | 4 |
| `flyer-crawler-infrastructure` | `b083076f94fb461b889d5dffcbef43bf` | 5 |
| `flyer-crawler-test-infrastructure` | `25020dd6c2b74ad78463ec90e90fadab` | 6 |
```text
https://abc123def456@bugsink.yourdomain.com/1
```
**Note:** The DSN key is the part before `@` in the full DSN URL (e.g., `https://KEY@bugsink.projectium.com/PROJECT_ID`).
Then `YOUR_BACKEND_DSN_KEY` is `abc123def456`.
**Note on PM2 Logs:** PM2 error logs capture stack traces from stderr, which are valuable for debugging startup errors and uncaught exceptions. Production PM2 logs go to project 5 (infrastructure), test PM2 logs go to project 6 (test-infrastructure).
### Step 5: Create Logstash State Directory
### Step 5: Create Logstash State Directory and Fix Config Path
Logstash needs a directory to track which log lines it has already processed:
Logstash needs a directory to track which log lines it has already processed, and a symlink so it can find its config files:
```bash
# Create state directory for sincedb files
sudo mkdir -p /var/lib/logstash
sudo chown logstash:logstash /var/lib/logstash
# Create symlink so Logstash finds its config (avoids "Could not find logstash.yml" warning)
sudo ln -sf /etc/logstash /usr/share/logstash/config
```
### Step 6: Grant Logstash Access to PM2 Logs
### Step 6: Grant Logstash Access to Application Logs
Logstash runs as the `logstash` user and needs permission to read PM2 logs:
Logstash runs as the `logstash` user and needs permission to read log files:
```bash
# Add logstash user to the group that owns PM2 logs
# If PM2 runs as root:
sudo usermod -a -G root logstash
# Add logstash user to adm group (for nginx and redis logs)
sudo usermod -aG adm logstash
# Or, make PM2 logs world-readable (less secure but simpler)
sudo chmod 644 /root/.pm2/logs/*.log
# Make application log files readable (created automatically when app starts)
sudo chmod 644 /var/www/flyer-crawler.projectium.com/logs/app.log 2>/dev/null || echo "Production log file not yet created"
sudo chmod 644 /var/www/flyer-crawler-test.projectium.com/logs/app.log 2>/dev/null || echo "Test log file not yet created"
# For Redis logs
# Make Redis logs and directory readable
sudo chmod 755 /var/log/redis/
sudo chmod 644 /var/log/redis/redis-server.log
# Make NGINX logs readable
sudo chmod 644 /var/log/nginx/access.log /var/log/nginx/error.log
# Make PM2 logs and directories accessible
sudo chmod 755 /home/gitea-runner/
sudo chmod 755 /home/gitea-runner/.pm2/
sudo chmod 755 /home/gitea-runner/.pm2/logs/
sudo chmod 644 /home/gitea-runner/.pm2/logs/*.log
# Verify logstash group membership
groups logstash
```
**Note:** If PM2 runs as a different user, adjust the group accordingly.
**Note:** The application log files are created automatically when the application starts. Run the chmod commands after the first deployment.
### Step 7: Test Logstash Configuration

View File

@@ -42,9 +42,9 @@ jobs:
env:
DB_HOST: ${{ secrets.DB_HOST }}
DB_PORT: ${{ secrets.DB_PORT }}
DB_USER: ${{ secrets.DB_USER }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
DB_NAME: ${{ secrets.DB_NAME_PROD }}
DB_USER: ${{ secrets.DB_USER_PROD }}
DB_PASSWORD: ${{ secrets.DB_PASSWORD_PROD }}
DB_NAME: ${{ secrets.DB_DATABASE_PROD }}
steps:
- name: Validate Secrets

158
ecosystem-test.config.cjs Normal file
View File

@@ -0,0 +1,158 @@
// ecosystem-test.config.cjs
// PM2 configuration for the TEST environment only.
// NOTE: The filename must end with `.config.cjs` for PM2 to recognize it as a config file.
// This file defines test-specific apps that run alongside production apps.
//
// Test apps: flyer-crawler-api-test, flyer-crawler-worker-test, flyer-crawler-analytics-worker-test
//
// These apps:
// - Run from /var/www/flyer-crawler-test.projectium.com
// - Use NODE_ENV='staging' (enables file logging in logger.server.ts)
// - Use Redis database 1 (isolated from production which uses database 0)
// - Have distinct PM2 process names to avoid conflicts with production
// --- Load Environment Variables from .env file ---
// This allows PM2 to start without requiring the CI/CD pipeline to inject variables.
// The .env file should be created on the server with the required secrets.
// NOTE: We implement a simple .env parser since dotenv may not be installed.
const path = require('path');
const fs = require('fs');
const envPath = path.join('/var/www/flyer-crawler-test.projectium.com', '.env');
if (fs.existsSync(envPath)) {
console.log('[ecosystem-test.config.cjs] Loading environment from:', envPath);
const envContent = fs.readFileSync(envPath, 'utf8');
const lines = envContent.split('\n');
for (const line of lines) {
// Skip comments and empty lines
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
// Parse KEY=value
const eqIndex = trimmed.indexOf('=');
if (eqIndex > 0) {
const key = trimmed.substring(0, eqIndex);
let value = trimmed.substring(eqIndex + 1);
// Remove quotes if present
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
// Only set if not already in environment (don't override CI/CD vars)
if (!process.env[key]) {
process.env[key] = value;
}
}
}
console.log('[ecosystem-test.config.cjs] Environment loaded successfully');
} else {
console.warn('[ecosystem-test.config.cjs] No .env file found at:', envPath);
console.warn(
'[ecosystem-test.config.cjs] Environment variables must be provided by the shell or CI/CD.'
);
}
// --- Environment Variable Validation ---
// NOTE: We only WARN about missing secrets, not exit.
// Calling process.exit(1) prevents PM2 from reading the apps array.
// The actual application will fail to start if secrets are missing,
// which PM2 will handle with its restart logic.
const requiredSecrets = ['DB_HOST', 'JWT_SECRET', 'GEMINI_API_KEY'];
const missingSecrets = requiredSecrets.filter(key => !process.env[key]);
if (missingSecrets.length > 0) {
console.warn('\n[ecosystem.config.test.cjs] WARNING: The following environment variables are MISSING:');
missingSecrets.forEach(key => console.warn(` - ${key}`));
console.warn('[ecosystem.config.test.cjs] The application may fail to start if these are required.\n');
} else {
console.log('[ecosystem.config.test.cjs] Critical environment variables are present.');
}
// --- Shared Environment Variables ---
const sharedEnv = {
DB_HOST: process.env.DB_HOST,
DB_USER: process.env.DB_USER,
DB_PASSWORD: process.env.DB_PASSWORD,
DB_NAME: process.env.DB_NAME,
REDIS_URL: process.env.REDIS_URL,
REDIS_PASSWORD: process.env.REDIS_PASSWORD,
FRONTEND_URL: process.env.FRONTEND_URL,
JWT_SECRET: process.env.JWT_SECRET,
GEMINI_API_KEY: process.env.GEMINI_API_KEY,
GOOGLE_MAPS_API_KEY: process.env.GOOGLE_MAPS_API_KEY,
SMTP_HOST: process.env.SMTP_HOST,
SMTP_PORT: process.env.SMTP_PORT,
SMTP_SECURE: process.env.SMTP_SECURE,
SMTP_USER: process.env.SMTP_USER,
SMTP_PASS: process.env.SMTP_PASS,
SMTP_FROM_EMAIL: process.env.SMTP_FROM_EMAIL,
SENTRY_DSN: process.env.SENTRY_DSN,
SENTRY_ENVIRONMENT: process.env.SENTRY_ENVIRONMENT,
SENTRY_ENABLED: process.env.SENTRY_ENABLED,
};
module.exports = {
apps: [
// =========================================================================
// TEST APPS
// =========================================================================
{
// --- Test API Server ---
name: 'flyer-crawler-api-test',
script: './node_modules/.bin/tsx',
args: 'server.ts',
cwd: '/var/www/flyer-crawler-test.projectium.com',
max_memory_restart: '500M',
// Test environment: single instance (no cluster) to conserve resources
instances: 1,
exec_mode: 'fork',
kill_timeout: 5000,
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
max_restarts: 40,
exp_backoff_restart_delay: 100,
min_uptime: '10s',
env: {
NODE_ENV: 'staging',
PORT: 3002,
WORKER_LOCK_DURATION: '120000',
...sharedEnv,
},
},
{
// --- Test General Worker ---
name: 'flyer-crawler-worker-test',
script: './node_modules/.bin/tsx',
args: 'src/services/worker.ts',
cwd: '/var/www/flyer-crawler-test.projectium.com',
max_memory_restart: '1G',
kill_timeout: 10000,
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
max_restarts: 40,
exp_backoff_restart_delay: 100,
min_uptime: '10s',
env: {
NODE_ENV: 'staging',
...sharedEnv,
},
},
{
// --- Test Analytics Worker ---
name: 'flyer-crawler-analytics-worker-test',
script: './node_modules/.bin/tsx',
args: 'src/services/worker.ts',
cwd: '/var/www/flyer-crawler-test.projectium.com',
max_memory_restart: '1G',
kill_timeout: 10000,
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
max_restarts: 40,
exp_backoff_restart_delay: 100,
min_uptime: '10s',
env: {
NODE_ENV: 'staging',
...sharedEnv,
},
},
],
};

View File

@@ -2,18 +2,28 @@
// This file is the standard way to configure applications for PM2.
// It allows us to define all the settings for our application in one place.
// The .cjs extension is required because the project's package.json has "type": "module".
//
// IMPORTANT: This file defines SEPARATE apps for production and test environments.
// Production apps: flyer-crawler-api, flyer-crawler-worker, flyer-crawler-analytics-worker
// Test apps: flyer-crawler-api-test, flyer-crawler-worker-test, flyer-crawler-analytics-worker-test
//
// Use ecosystem-test.config.cjs for test deployments (contains only test apps).
// Use this file (ecosystem.config.cjs) for production deployments.
// --- Environment Variable Validation ---
// NOTE: We only WARN about missing secrets, not exit.
// Calling process.exit(1) prevents PM2 from reading the apps array.
// The actual application will fail to start if secrets are missing,
// which PM2 will handle with its restart logic.
const requiredSecrets = ['DB_HOST', 'JWT_SECRET', 'GEMINI_API_KEY'];
const missingSecrets = requiredSecrets.filter(key => !process.env[key]);
if (missingSecrets.length > 0) {
console.warn('\n[ecosystem.config.cjs] ⚠️ WARNING: The following environment variables are MISSING in the shell:');
console.warn('\n[ecosystem.config.cjs] WARNING: The following environment variables are MISSING:');
missingSecrets.forEach(key => console.warn(` - ${key}`));
console.warn('[ecosystem.config.cjs] The application may crash if these are required for startup.\n');
process.exit(1); // Fail fast so PM2 doesn't attempt to start a broken app
console.warn('[ecosystem.config.cjs] The application may fail to start if these are required.\n');
} else {
console.log('[ecosystem.config.cjs] Critical environment variables are present.');
console.log('[ecosystem.config.cjs] Critical environment variables are present.');
}
// --- Shared Environment Variables ---
@@ -35,125 +45,67 @@ const sharedEnv = {
SMTP_USER: process.env.SMTP_USER,
SMTP_PASS: process.env.SMTP_PASS,
SMTP_FROM_EMAIL: process.env.SMTP_FROM_EMAIL,
SENTRY_DSN: process.env.SENTRY_DSN,
SENTRY_ENVIRONMENT: process.env.SENTRY_ENVIRONMENT,
SENTRY_ENABLED: process.env.SENTRY_ENABLED,
};
module.exports = {
apps: [
// =========================================================================
// PRODUCTION APPS
// =========================================================================
{
// --- API Server ---
// --- Production API Server ---
name: 'flyer-crawler-api',
// Note: The process names below are referenced in .gitea/workflows/ for status checks.
script: './node_modules/.bin/tsx',
args: 'server.ts',
cwd: '/var/www/flyer-crawler.projectium.com',
max_memory_restart: '500M',
// Production Optimization: Run in cluster mode to utilize all CPU cores
instances: 'max',
exec_mode: 'cluster',
kill_timeout: 5000, // Allow 5s for graceful shutdown of API requests
kill_timeout: 5000,
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
// Restart Logic
max_restarts: 40,
exp_backoff_restart_delay: 100,
min_uptime: '10s',
// Production Environment Settings
env_production: {
env: {
NODE_ENV: 'production',
name: 'flyer-crawler-api',
cwd: '/var/www/flyer-crawler.projectium.com',
WORKER_LOCK_DURATION: '120000',
...sharedEnv,
},
// Test Environment Settings
env_test: {
NODE_ENV: 'test',
name: 'flyer-crawler-api-test',
cwd: '/var/www/flyer-crawler-test.projectium.com',
WORKER_LOCK_DURATION: '120000',
...sharedEnv,
},
// Development Environment Settings
env_development: {
NODE_ENV: 'development',
name: 'flyer-crawler-api-dev',
watch: true,
ignore_watch: ['node_modules', 'logs', '*.log', 'flyer-images', '.git'],
WORKER_LOCK_DURATION: '120000',
...sharedEnv,
},
},
{
// --- General Worker ---
// --- Production General Worker ---
name: 'flyer-crawler-worker',
script: './node_modules/.bin/tsx',
args: 'src/services/worker.ts',
cwd: '/var/www/flyer-crawler.projectium.com',
max_memory_restart: '1G',
kill_timeout: 10000, // Workers may need more time to complete a job
kill_timeout: 10000,
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
// Restart Logic
max_restarts: 40,
exp_backoff_restart_delay: 100,
min_uptime: '10s',
// Production Environment Settings
env_production: {
env: {
NODE_ENV: 'production',
name: 'flyer-crawler-worker',
cwd: '/var/www/flyer-crawler.projectium.com',
...sharedEnv,
},
// Test Environment Settings
env_test: {
NODE_ENV: 'test',
name: 'flyer-crawler-worker-test',
cwd: '/var/www/flyer-crawler-test.projectium.com',
...sharedEnv,
},
// Development Environment Settings
env_development: {
NODE_ENV: 'development',
name: 'flyer-crawler-worker-dev',
watch: true,
ignore_watch: ['node_modules', 'logs', '*.log', 'flyer-images', '.git'],
...sharedEnv,
},
},
{
// --- Analytics Worker ---
// --- Production Analytics Worker ---
name: 'flyer-crawler-analytics-worker',
script: './node_modules/.bin/tsx',
args: 'src/services/worker.ts',
cwd: '/var/www/flyer-crawler.projectium.com',
max_memory_restart: '1G',
kill_timeout: 10000,
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
// Restart Logic
max_restarts: 40,
exp_backoff_restart_delay: 100,
min_uptime: '10s',
// Production Environment Settings
env_production: {
env: {
NODE_ENV: 'production',
name: 'flyer-crawler-analytics-worker',
cwd: '/var/www/flyer-crawler.projectium.com',
...sharedEnv,
},
// Test Environment Settings
env_test: {
NODE_ENV: 'test',
name: 'flyer-crawler-analytics-worker-test',
cwd: '/var/www/flyer-crawler-test.projectium.com',
...sharedEnv,
},
// Development Environment Settings
env_development: {
NODE_ENV: 'development',
name: 'flyer-crawler-analytics-worker-dev',
watch: true,
ignore_watch: ['node_modules', 'logs', '*.log', 'flyer-images', '.git'],
...sharedEnv,
},
},

View File

@@ -0,0 +1,69 @@
# HTTPS Server Block (main)
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name flyer-crawler-test.projectium.com;
# SSL Configuration (managed by Certbot)
ssl_certificate /etc/letsencrypt/live/flyer-crawler-test.projectium.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/flyer-crawler-test.projectium.com/privkey.pem;
include /etc/letsencrypt/options-ssl-nginx.conf;
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
# Allow large file uploads (e.g., for flyers)
client_max_body_size 100M;
# Root directory for built application files
root /var/www/flyer-crawler-test.projectium.com;
index index.html;
# Deny access to all dotfiles
location ~ /\. {
deny all;
return 404;
}
# Coverage report (must come before generic location /)
location /coverage/ {
try_files $uri $uri/ =404;
}
# SPA fallback for React Router
location / {
try_files $uri $uri/ /index.html;
}
# Reverse proxy for backend API
location /api/ {
proxy_connect_timeout 300s;
proxy_send_timeout 300s;
proxy_read_timeout 300s;
proxy_pass http://localhost:3002;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
# Correct MIME type for .mjs files
location ~ \.mjs$ {
include /etc/nginx/mime.types;
default_type application/javascript;
}
# Security headers
add_header X-Frame-Options "SAMEORIGIN" always;
add_header X-XSS-Protection "1; mode=block" always;
add_header X-Content-Type-Options "nosniff" always;
}
# HTTP to HTTPS Redirect
server {
listen 80;
listen [::]:80;
server_name flyer-crawler-test.projectium.com;
return 301 https://$host$request_uri;
}

196
notes-to-ai5.txt Normal file
View File

@@ -0,0 +1,196 @@
What is the correct named pipe address for Podman on Windows if it's not using the default docker_engine pipe?
How do I configure the chrome-devtools-mcp server to connect to an existing Chrome instance using the browser_url input?
Unused Input: The browser_url input is defined but not used in any server arguments. If you intend to connect chrome-devtools-mcp to an existing Chrome instance, you would typically pass that URL in the arguments instead of launching a new headless instance.
Based on your current configuration and project setup (Gitea, Podman, PostgreSQL, Redis), here are several other valuable MCP servers available for VS Code.
Recommended for Your Stack
Since you are running a PostgreSQL and Redis stack via Podman, these servers would be immediately useful:
PostgreSQL (@modelcontextprotocol/server-postgres)
Purpose: Allows the AI to query your database directly (inspect schemas, run SELECT queries) without needing to shell into the container.
Config:
json
"postgres": {
"command": "npx",
"args": ["-y", "@modelcontextprotocol/server-postgres", "postgresql://postgres:postgres@localhost:5432/flyer_crawler_dev"]
}
Redis (@modelcontextprotocol/server-redis)
Purpose: Inspect and modify keys in your Redis instance.
Config:
json
"redis": {
"command": "npx",
"args": ["-y", "@modelcontextprotocol/server-redis", "redis://localhost:6379"]
}
General Utility Servers
These servers enhance the AI's core capabilities regardless of the specific project:
Memory (@modelcontextprotocol/server-memory)
Purpose: Gives the AI a persistent "knowledge graph" to remember context, user preferences, and project details across different chat sessions.
Config:
json
"memory": {
"command": "npx",
"args": ["-y", "@modelcontextprotocol/server-memory"]
}
Brave Search (@modelcontextprotocol/server-brave-search)
Purpose: Performs actual web searches (finding current docs, libraries, or solutions) rather than just fetching specific URLs like the fetch server does.
Config:
json
Show full code block
"brave-search": {
"command": "npx",
"args": ["-y", "@modelcontextprotocol/server-brave-search"],
"env": {
"BRAVE_API_KEY": "YOUR_API_KEY_HERE"
}
}
Sequential Thinking (@modelcontextprotocol/server-sequential-thinking)
Purpose: A tool that helps the AI break down complex problems into step-by-step thought processes. It is particularly useful for debugging difficult logic or architectural planning.
Config:
json
"sequential-thinking": {
"command": "npx",
"args": ["-y", "@modelcontextprotocol/server-sequential-thinking"]
}
Other Popular Servers
GitHub: (@modelcontextprotocol/server-github) - If you ever need to interact with GitHub.com in addition to Gitea.
Google Drive: (@modelcontextprotocol/server-google-drive) - Access files in Google Drive.
Slack: (@modelcontextprotocol/server-slack) - Read and send Slack messages.
Sentry: (@modelcontextprotocol/server-sentry) - Analyze error reports directly from Sentry.
How to Find More
The community maintains a comprehensive list of servers in the Awesome MCP Servers repository. You can find servers for almost any major tool (AWS, Azure, Linear, Jira, etc.).
To add any of these, you simply add a new entry to the mcpServers object in your mcp.json file, similar to how you configured podman or fetch.
Create a new ADR for "Worker Health Checks" to monitor stalled background jobs.
Storybook ! for UI
{
"mcpServers": {
"gitea-projectium": {
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
"args": ["run", "-t", "stdio"],
"env": {
"GITEA_HOST": "https://gitea.projectium.com",
"GITEA_ACCESS_TOKEN": "b111259253aa3cadcb6a37618de03bf388f6235a"
}
},
"gitea-torbonium": {
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
"args": ["run", "-t", "stdio"],
"env": {
"GITEA_HOST": "https://gitea.torbonium.com",
"GITEA_ACCESS_TOKEN": "563d01f9edc792b6dd09bf4cbd3a98bce45360a4"
}
},
"gitea-lan": {
"command": "d:\\gitea-mcp\\gitea-mcp.exe",
"args": ["run", "-t", "stdio"],
"env": {
"GITEA_HOST": "https://gitea.torbolan.com",
"GITEA_ACCESS_TOKEN": "YOUR_LAN_TOKEN_HERE"
},
"disabled": true
},
"podman": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "podman-mcp-server@latest"],
"env": {
"DOCKER_HOST": "npipe:////./pipe/podman-machine-default"
}
},
"filesystem": {
"command": "d:\\nodejs\\node.exe",
"args": [
"c:\\Users\\games3\\AppData\\Roaming\\npm\\node_modules\\@modelcontextprotocol\\server-filesystem\\dist\\index.js",
"d:\\gitea\\flyer-crawler.projectium.com\\flyer-crawler.projectium.com"
]
},
"fetch": {
"command": "C:\\Users\\games3\\.local\\bin\\uvx.exe",
"args": ["mcp-server-fetch"]
},
"chrome-devtools": {
"command": "D:\\nodejs\\npx.cmd",
"args": [
"chrome-devtools-mcp@latest",
"--headless",
"false",
"--isolated",
"false",
"--channel",
"stable"
],
"disabled": true
},
"markitdown": {
"command": "C:\\Users\\games3\\.local\\bin\\uvx.exe",
"args": ["markitdown-mcp"]
},
"sequential-thinking": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-sequential-thinking"]
},
"memory": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-memory"]
},
"postgres": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-postgres", "postgresql://postgres:postgres@localhost:5432/flyer_crawler_dev"]
},
"playwright": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@anthropics/mcp-server-playwright"]
},
"redis": {
"command": "D:\\nodejs\\npx.cmd",
"args": ["-y", "@modelcontextprotocol/server-redis", "redis://localhost:6379"]
}
}
}

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "flyer-crawler",
"version": "0.9.97",
"version": "0.11.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "flyer-crawler",
"version": "0.9.97",
"version": "0.11.0",
"dependencies": {
"@bull-board/api": "^6.14.2",
"@bull-board/express": "^6.14.2",

View File

@@ -1,7 +1,7 @@
{
"name": "flyer-crawler",
"private": true,
"version": "0.9.97",
"version": "0.11.0",
"type": "module",
"scripts": {
"dev": "concurrently \"npm:start:dev\" \"vite\"",

Some files were not shown because too many files have changed in this diff Show More