Torben Sorensen
cd8ee92813
Some checks failed
Deploy to Test Environment / deploy-to-test (push) Has been cancelled
107 lines
3.8 KiB
Bash
107 lines
3.8 KiB
Bash
#!/bin/bash
|
||
# scripts/analyze-pm2-crashes.sh
|
||
#
|
||
# Analyzes PM2 logs to identify crash patterns and problematic projects
|
||
|
||
set -e
|
||
|
||
PM2_LOG="/home/gitea-runner/.pm2/pm2.log"
|
||
|
||
echo "========================================="
|
||
echo "PM2 CRASH ANALYSIS TOOL"
|
||
echo "========================================="
|
||
echo ""
|
||
|
||
if [ ! -f "$PM2_LOG" ]; then
|
||
echo "❌ PM2 log file not found at: $PM2_LOG"
|
||
exit 1
|
||
fi
|
||
|
||
echo "Analyzing PM2 log file: $PM2_LOG"
|
||
echo "Log file size: $(du -h "$PM2_LOG" | cut -f1)"
|
||
echo "Last modified: $(stat -c %y "$PM2_LOG")"
|
||
echo ""
|
||
|
||
echo "========================================="
|
||
echo "1. RECENT PM2 DAEMON RESTARTS"
|
||
echo "========================================="
|
||
grep -i "New PM2 Daemon started" "$PM2_LOG" | tail -5 || echo "No daemon restarts found"
|
||
echo ""
|
||
|
||
echo "========================================="
|
||
echo "2. ENOENT / CWD ERRORS"
|
||
echo "========================================="
|
||
grep -i "ENOENT\|uv_cwd\|no such file or directory" "$PM2_LOG" | tail -20 || echo "No ENOENT errors found"
|
||
echo ""
|
||
|
||
echo "========================================="
|
||
echo "3. PROCESS CRASH PATTERNS"
|
||
echo "========================================="
|
||
echo "Searching for app crash events..."
|
||
grep -i "App \[.*\] exited\|App \[.*\] errored\|App \[.*\] crashed" "$PM2_LOG" | tail -20 || echo "No app crashes found"
|
||
echo ""
|
||
|
||
echo "========================================="
|
||
echo "4. PROJECTS INVOLVED IN CRASHES"
|
||
echo "========================================="
|
||
echo "Extracting project names from crash logs..."
|
||
grep -i "ENOENT\|crash\|error" "$PM2_LOG" | grep -oE "flyer-crawler[a-z-]*|stock-alert[a-z-]*" | sort | uniq -c | sort -rn || echo "No project names found in crashes"
|
||
echo ""
|
||
|
||
echo "========================================="
|
||
echo "5. TIMELINE OF RECENT ERRORS (Last 50)"
|
||
echo "========================================="
|
||
grep -E "^[0-9]{4}-[0-9]{2}-[0-9]{2}" "$PM2_LOG" | grep -i "error\|crash\|ENOENT" | tail -50 || echo "No timestamped errors found"
|
||
echo ""
|
||
|
||
echo "========================================="
|
||
echo "6. CURRENT PM2 STATE"
|
||
echo "========================================="
|
||
pm2 list
|
||
echo ""
|
||
|
||
echo "========================================="
|
||
echo "7. PROCESSES WITH MISSING CWD"
|
||
echo "========================================="
|
||
pm2 jlist | jq -r '.[] | select(.pm2_env.pm_cwd) | "\(.name): \(.pm2_env.pm_cwd)"' | while read line; do
|
||
PROC_NAME=$(echo "$line" | cut -d: -f1)
|
||
CWD=$(echo "$line" | cut -d: -f2- | xargs)
|
||
if [ ! -d "$CWD" ]; then
|
||
echo "❌ $PROC_NAME - CWD missing: $CWD"
|
||
else
|
||
echo "✅ $PROC_NAME - CWD exists: $CWD"
|
||
fi
|
||
done
|
||
echo ""
|
||
|
||
echo "========================================="
|
||
echo "8. RECOMMENDATIONS"
|
||
echo "========================================="
|
||
echo ""
|
||
|
||
# Count ENOENT errors
|
||
ENOENT_COUNT=$(grep -c "ENOENT\|uv_cwd" "$PM2_LOG" 2>/dev/null || echo "0")
|
||
if [ "$ENOENT_COUNT" -gt 0 ]; then
|
||
echo "⚠️ Found $ENOENT_COUNT ENOENT/CWD errors in logs"
|
||
echo " This indicates processes losing their working directory during deployment"
|
||
echo " Solution: Ensure PM2 processes are stopped BEFORE rsync --delete operations"
|
||
echo ""
|
||
fi
|
||
|
||
# Check for multiple projects
|
||
FLYER_PROCESSES=$(pm2 jlist | jq '[.[] | select(.name | contains("flyer-crawler"))] | length' 2>/dev/null || echo "0")
|
||
STOCK_PROCESSES=$(pm2 jlist | jq '[.[] | select(.name | contains("stock-alert"))] | length' 2>/dev/null || echo "0")
|
||
|
||
if [ "$FLYER_PROCESSES" -gt 0 ] && [ "$STOCK_PROCESSES" -gt 0 ]; then
|
||
echo "ℹ️ Multiple projects detected:"
|
||
echo " - Flyer-crawler: $FLYER_PROCESSES processes"
|
||
echo " - Stock-alert: $STOCK_PROCESSES processes"
|
||
echo " Recommendation: Ensure deployments don't interfere with each other"
|
||
echo ""
|
||
fi
|
||
|
||
echo "✅ Analysis complete"
|
||
echo ""
|
||
echo "To save this report:"
|
||
echo " bash scripts/analyze-pm2-crashes.sh > pm2-crash-report.txt"
|