starrocks/starrocks-debug-skills/cpu-saturation

# Snapshot CPU breakdown
top -b -n 1 | head -5
# Output: %Cpu(s): 85.2 us,  8.3 sy,  0.0 ni,  3.1 id,  2.1 wa,  0.0 hi,  1.3 si,  0.0 st
 
# Per-process CPU (10-second sample)
pidstat -u 2 5
 
# Load average vs CPU count
uptime
# Load average: 64.2, 58.1, 52.3 → compare to: nproc (e.g., 32 CPUs → overloaded)
nproc

# Pull all BE CPU task metrics
curl -s "http://<be_host>:<be_http_port>/metrics" | grep "starrocks_be_cpu"
 
# Loop across all BEs to find the hotspot node
for be in <be1> <be2> <be3>; do
    echo "=== $be ==="
    curl -s "http://$be:8040/metrics" | grep "starrocks_be_cpu_task"
done

# FE Java process CPU
top -b -n 1 | grep java
 
# Per-thread breakdown for FE JVM
top -Hp $(pgrep -f 'StarRocksFE\|fe.pid') | head -20
# Convert thread ID (decimal) to hex → cross-reference with jstack
 
# FE thread dump
jstack $(cat fe/bin/fe.pid) > /tmp/fe_jstack.log
grep -c "RUNNABLE" /tmp/fe_jstack.log
# High RUNNABLE count relative to vCPUs → planning storm or GC
 
# GC pressure
grep "GC" fe/log/fe.gc.log | tail -20

# Live hot functions (needs root or perf permission)
perf top -p $(pidof starrocks_be) -d 5
# Format: [overhead%]  [symbol_name]  [DSO]
# Focus on the top 5 functions; the prefix identifies the module
 
# Quick thread snapshot without perf
pstack $(pidof starrocks_be) > /tmp/be_pstack_$(date +%s).log
grep -oE "starrocks::[A-Za-z:]+" /tmp/be_pstack_$(date +%s).log | sort | uniq -c | sort -rn | head -20

CPU saturation observed (load average high or top shows 100% on a node)
  │
  ├─ Step 1: Which process owns the CPU?
  │     → see Section 1
  │
  ├─ Process = starrocks_be
  │     └─ Step 2: Which BE module?
  │           → see Section 2
  │           ├─ Queries  → Section 3a + query/SKILL.md
  │           ├─ Import   → Section 3b + import/SKILL.md
  │           ├─ Compaction → Section 3c + compaction/SKILL.md
  │           └─ Background task / unknown → Section 3d
  │
  ├─ Process = java (FE)
  │     → FE Full GC or planning storm → Section 4 + node/SKILL.md
  │
  └─ Process = non-StarRocks (backup agent, kernel, etc.)
        → Section 5: OS-level attribution

# Snapshot: top CPU consumers by process
top -b -n 1 | head -20
 
# More detail: per-thread breakdown for a single process
top -Hp $(pidof starrocks_be) | head -30
 
# Or use pidstat for sustained view
pidstat -u 2 10

-- Any active long-running queries?
SHOW PROC '/current_queries';
-- Look for QueryTime > 30s or MemUsageBytes > 1GB
 
-- Any active imports?
SELECT LABEL, STATE, PROGRESS, TYPE, LOAD_START_TIME
FROM information_schema.loads
WHERE STATE IN ('LOADING', 'PREPARED', 'PREPARED')
ORDER BY LOAD_START_TIME ASC;

# CPU breakdown by task type (query / load / compaction / clone / schema change)
curl -s http://<BE_IP>:<BE_HTTP_PORT>/metrics | grep "starrocks_be_cpu"

# Live hot functions (needs root or perf permission)
perf top -p $(pidof starrocks_be) -d 5
 
# Or: pstack for a quick thread snapshot without perf
pstack $(pidof starrocks_be) > /tmp/be_pstack_$(date +%s).log

# Find queries with highest scan rows in audit log (last 1 hour)
python3 scripts/analyze_logs.py "$(date -d '1 hour ago' '+%Y-%m-%d %H:%M:%S')" "$(date '+%Y-%m-%d %H:%M:%S')" "ScanRows" 5 fe.audit.log
 
# Or find by query execution time
grep 'QueryTime' fe.audit.log | awk -F'QueryTime=' '{print $2}' | cut -d'|' -f1 | sort -rn | head -10

SET is_report_success = true;
-- Re-run the query, then check profile:
-- http://<fe_ip>:8030 -> Queries -> Profile

# Check import thread CPU contribution
curl -s http://<BE_IP>:<BE_HTTP_PORT>/metrics | grep "starrocks_be_cpu_task_load"
 
# Check active imports
curl -s http://<BE_IP>:<BE_HTTP_PORT>/api/running_load | python3 -m json.tool 2>/dev/null