debug: add PM2 crash debugging tools

scripts/analyze-pm2-crashes.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+# scripts/analyze-pm2-crashes.sh
+#
+# Analyzes PM2 logs to identify crash patterns and problematic projects
+
+set -e
+
+PM2_LOG="/home/gitea-runner/.pm2/pm2.log"
+
+echo "========================================="
+echo "PM2 CRASH ANALYSIS TOOL"
+echo "========================================="
+echo ""
+
+if [ ! -f "$PM2_LOG" ]; then
+    echo "❌ PM2 log file not found at: $PM2_LOG"
+    exit 1
+fi
+
+echo "Analyzing PM2 log file: $PM2_LOG"
+echo "Log file size: $(du -h "$PM2_LOG" | cut -f1)"
+echo "Last modified: $(stat -c %y "$PM2_LOG")"
+echo ""
+
+echo "========================================="
+echo "1. RECENT PM2 DAEMON RESTARTS"
+echo "========================================="
+grep -i "New PM2 Daemon started" "$PM2_LOG" | tail -5 || echo "No daemon restarts found"
+echo ""
+
+echo "========================================="
+echo "2. ENOENT / CWD ERRORS"
+echo "========================================="
+grep -i "ENOENT\|uv_cwd\|no such file or directory" "$PM2_LOG" | tail -20 || echo "No ENOENT errors found"
+echo ""
+
+echo "========================================="
+echo "3. PROCESS CRASH PATTERNS"
+echo "========================================="
+echo "Searching for app crash events..."
+grep -i "App \[.*\] exited\|App \[.*\] errored\|App \[.*\] crashed" "$PM2_LOG" | tail -20 || echo "No app crashes found"
+echo ""
+
+echo "========================================="
+echo "4. PROJECTS INVOLVED IN CRASHES"
+echo "========================================="
+echo "Extracting project names from crash logs..."
+grep -i "ENOENT\|crash\|error" "$PM2_LOG" | grep -oE "flyer-crawler[a-z-]*|stock-alert[a-z-]*" | sort | uniq -c | sort -rn || echo "No project names found in crashes"
+echo ""
+
+echo "========================================="
+echo "5. TIMELINE OF RECENT ERRORS (Last 50)"
+echo "========================================="
+grep -E "^[0-9]{4}-[0-9]{2}-[0-9]{2}" "$PM2_LOG" | grep -i "error\|crash\|ENOENT" | tail -50 || echo "No timestamped errors found"
+echo ""
+
+echo "========================================="
+echo "6. CURRENT PM2 STATE"
+echo "========================================="
+pm2 list
+echo ""
+
+echo "========================================="
+echo "7. PROCESSES WITH MISSING CWD"
+echo "========================================="
+pm2 jlist | jq -r '.[] | select(.pm2_env.pm_cwd) | "\(.name): \(.pm2_env.pm_cwd)"' | while read line; do
+    PROC_NAME=$(echo "$line" | cut -d: -f1)
+    CWD=$(echo "$line" | cut -d: -f2- | xargs)
+    if [ ! -d "$CWD" ]; then
+        echo "❌ $PROC_NAME - CWD missing: $CWD"
+    else
+        echo "✅ $PROC_NAME - CWD exists: $CWD"
+    fi
+done
+echo ""
+
+echo "========================================="
+echo "8. RECOMMENDATIONS"
+echo "========================================="
+echo ""
+
+# Count ENOENT errors
+ENOENT_COUNT=$(grep -c "ENOENT\|uv_cwd" "$PM2_LOG" 2>/dev/null || echo "0")
+if [ "$ENOENT_COUNT" -gt 0 ]; then
+    echo "⚠️  Found $ENOENT_COUNT ENOENT/CWD errors in logs"
+    echo "   This indicates processes losing their working directory during deployment"
+    echo "   Solution: Ensure PM2 processes are stopped BEFORE rsync --delete operations"
+    echo ""
+fi
+
+# Check for multiple projects
+FLYER_PROCESSES=$(pm2 jlist | jq '[.[] | select(.name | contains("flyer-crawler"))] | length' 2>/dev/null || echo "0")
+STOCK_PROCESSES=$(pm2 jlist | jq '[.[] | select(.name | contains("stock-alert"))] | length' 2>/dev/null || echo "0")
+
+if [ "$FLYER_PROCESSES" -gt 0 ] && [ "$STOCK_PROCESSES" -gt 0 ]; then
+    echo "ℹ️  Multiple projects detected:"
+    echo "   - Flyer-crawler: $FLYER_PROCESSES processes"
+    echo "   - Stock-alert: $STOCK_PROCESSES processes"
+    echo "   Recommendation: Ensure deployments don't interfere with each other"
+    echo ""
+fi
+
+echo "✅ Analysis complete"
+echo ""
+echo "To save this report:"
+echo "  bash scripts/analyze-pm2-crashes.sh > pm2-crash-report.txt"