logging + e2e test fixes

docs/BARE-METAL-SETUP.md

@@ -968,14 +968,11 @@ Create the pipeline configuration file:
 sudo nano /etc/logstash/conf.d/bugsink.conf
 ```
 
-Next,
-
 Add the following content:
 
 ```conf
 input {
   # Production application logs (Pino JSON format)
-  # The flyer-crawler app writes JSON logs directly to this file
   file {
     path => "/var/www/flyer-crawler.projectium.com/logs/app.log"
     codec => json_lines
@@ -995,14 +992,51 @@ input {
     sincedb_path => "/var/lib/logstash/sincedb_pino_test"
   }
 
-  # Redis logs
+  # Redis logs (shared by both environments)
   file {
     path => "/var/log/redis/redis-server.log"
     type => "redis"
-    tags => ["redis"]
+    tags => ["infra", "redis", "production"]
     start_position => "end"
     sincedb_path => "/var/lib/logstash/sincedb_redis"
   }
+
+  # NGINX error logs (production)
+  file {
+    path => "/var/log/nginx/error.log"
+    type => "nginx"
+    tags => ["infra", "nginx", "production"]
+    start_position => "end"
+    sincedb_path => "/var/lib/logstash/sincedb_nginx_error"
+  }
+
+  # NGINX access logs - for detecting 5xx errors (production)
+  file {
+    path => "/var/log/nginx/access.log"
+    type => "nginx_access"
+    tags => ["infra", "nginx", "production"]
+    start_position => "end"
+    sincedb_path => "/var/lib/logstash/sincedb_nginx_access"
+  }
+
+  # PM2 error logs - Production (plain text stack traces)
+  file {
+    path => "/home/gitea-runner/.pm2/logs/flyer-crawler-*-error.log"
+    exclude => "*-test-error.log"
+    type => "pm2"
+    tags => ["infra", "pm2", "production"]
+    start_position => "end"
+    sincedb_path => "/var/lib/logstash/sincedb_pm2_prod"
+  }
+
+  # PM2 error logs - Test
+  file {
+    path => "/home/gitea-runner/.pm2/logs/flyer-crawler-*-test-error.log"
+    type => "pm2"
+    tags => ["infra", "pm2", "test"]
+    start_position => "end"
+    sincedb_path => "/var/lib/logstash/sincedb_pm2_test"
+  }
 }
 
 filter {
@@ -1025,59 +1059,142 @@ filter {
       mutate { add_tag => ["error"] }
     }
   }
+
+  # NGINX error log detection (all entries are errors)
+  if [type] == "nginx" {
+    mutate { add_tag => ["error"] }
+    grok {
+      match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} \[%{WORD:severity}\] %{GREEDYDATA:nginx_message}" }
+    }
+  }
+
+  # NGINX access log - detect 5xx errors
+  if [type] == "nginx_access" {
+    grok {
+      match => { "message" => "%{COMBINEDAPACHELOG}" }
+    }
+    if [response] =~ /^5\d{2}$/ {
+      mutate { add_tag => ["error"] }
+    }
+  }
+
+  # PM2 error log detection - tag lines with actual error indicators
+  if [type] == "pm2" {
+    if [message] =~ /Error:|error:|ECONNREFUSED|ENOENT|TypeError|ReferenceError|SyntaxError/ {
+      mutate { add_tag => ["error"] }
+    }
+  }
 }
 
 output {
-  # Only send errors to Bugsink
-  if "error" in [tags] {
+  # Production app errors -> flyer-crawler-backend (project 1)
+  if "error" in [tags] and "app" in [tags] and "production" in [tags] {
     http {
       url => "http://localhost:8000/api/1/store/"
       http_method => "post"
       format => "json"
       headers => {
-        "X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=YOUR_BACKEND_DSN_KEY"
+        "X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=YOUR_PROD_BACKEND_DSN_KEY"
       }
     }
   }
 
-  # Debug output (remove in production after confirming it works)
+  # Test app errors -> flyer-crawler-backend-test (project 3)
+  if "error" in [tags] and "app" in [tags] and "test" in [tags] {
+    http {
+      url => "http://localhost:8000/api/3/store/"
+      http_method => "post"
+      format => "json"
+      headers => {
+        "X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=YOUR_TEST_BACKEND_DSN_KEY"
+      }
+    }
+  }
+
+  # Production infrastructure errors (Redis, NGINX, PM2) -> flyer-crawler-infrastructure (project 5)
+  if "error" in [tags] and "infra" in [tags] and "production" in [tags] {
+    http {
+      url => "http://localhost:8000/api/5/store/"
+      http_method => "post"
+      format => "json"
+      headers => {
+        "X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=b083076f94fb461b889d5dffcbef43bf"
+      }
+    }
+  }
+
+  # Test infrastructure errors (PM2 test logs) -> flyer-crawler-test-infrastructure (project 6)
+  if "error" in [tags] and "infra" in [tags] and "test" in [tags] {
+    http {
+      url => "http://localhost:8000/api/6/store/"
+      http_method => "post"
+      format => "json"
+      headers => {
+        "X-Sentry-Auth" => "Sentry sentry_version=7, sentry_client=logstash/1.0, sentry_key=25020dd6c2b74ad78463ec90e90fadab"
+      }
+    }
+  }
+
+  # Debug output (uncomment to troubleshoot)
   # stdout { codec => rubydebug }
 }
 ```
 
-**Important:** Replace `YOUR_BACKEND_DSN_KEY` with the key from your Bugsink backend DSN. The key is the part before the `@` symbol in the DSN URL.
+**Bugsink Project DSNs:**
 
-For example, if your DSN is:
+| Project                             | DSN Key                            | Project ID |
+| ----------------------------------- | ---------------------------------- | ---------- |
+| `flyer-crawler-backend`             | `911aef02b9a548fa8fabb8a3c81abfe5` | 1          |
+| `flyer-crawler-frontend`            | (used by app, not Logstash)        | 2          |
+| `flyer-crawler-backend-test`        | `cdb99c314589431e83d4cc38a809449b` | 3          |
+| `flyer-crawler-frontend-test`       | (used by app, not Logstash)        | 4          |
+| `flyer-crawler-infrastructure`      | `b083076f94fb461b889d5dffcbef43bf` | 5          |
+| `flyer-crawler-test-infrastructure` | `25020dd6c2b74ad78463ec90e90fadab` | 6          |
 
-```text
-https://abc123def456@bugsink.yourdomain.com/1
-```
+**Note:** The DSN key is the part before `@` in the full DSN URL (e.g., `https://KEY@bugsink.projectium.com/PROJECT_ID`).
 
-Then `YOUR_BACKEND_DSN_KEY` is `abc123def456`.
+**Note on PM2 Logs:** PM2 error logs capture stack traces from stderr, which are valuable for debugging startup errors and uncaught exceptions. Production PM2 logs go to project 5 (infrastructure), test PM2 logs go to project 6 (test-infrastructure).
 
-### Step 5: Create Logstash State Directory
+### Step 5: Create Logstash State Directory and Fix Config Path
 
-Logstash needs a directory to track which log lines it has already processed:
+Logstash needs a directory to track which log lines it has already processed, and a symlink so it can find its config files:
 
 ```bash
+# Create state directory for sincedb files
 sudo mkdir -p /var/lib/logstash
 sudo chown logstash:logstash /var/lib/logstash
+
+# Create symlink so Logstash finds its config (avoids "Could not find logstash.yml" warning)
+sudo ln -sf /etc/logstash /usr/share/logstash/config
 ```
 
 ### Step 6: Grant Logstash Access to Application Logs
 
-Logstash runs as the `logstash` user and needs permission to read the application log files:
+Logstash runs as the `logstash` user and needs permission to read log files:
 
 ```bash
-# Make application log files readable by logstash
-# The directories were already set to 755 in Step 1
+# Add logstash user to adm group (for nginx and redis logs)
+sudo usermod -aG adm logstash
 
-# Ensure the log files themselves are readable (they should be created with 644 by default)
+# Make application log files readable (created automatically when app starts)
 sudo chmod 644 /var/www/flyer-crawler.projectium.com/logs/app.log 2>/dev/null || echo "Production log file not yet created"
 sudo chmod 644 /var/www/flyer-crawler-test.projectium.com/logs/app.log 2>/dev/null || echo "Test log file not yet created"
 
-# For Redis logs
+# Make Redis logs and directory readable
+sudo chmod 755 /var/log/redis/
 sudo chmod 644 /var/log/redis/redis-server.log
+
+# Make NGINX logs readable
+sudo chmod 644 /var/log/nginx/access.log /var/log/nginx/error.log
+
+# Make PM2 logs and directories accessible
+sudo chmod 755 /home/gitea-runner/
+sudo chmod 755 /home/gitea-runner/.pm2/
+sudo chmod 755 /home/gitea-runner/.pm2/logs/
+sudo chmod 644 /home/gitea-runner/.pm2/logs/*.log
+
+# Verify logstash group membership
+groups logstash
 ```
 
 **Note:** The application log files are created automatically when the application starts. Run the chmod commands after the first deployment.