starrocks/starrocks-debug-skills/node

# Per-module memory (most useful for OOM investigation)
curl -s http://<BE_IP>:<BE_HTTP_PORT>/mem_tracker
 
# All BE memory gauge metrics
curl -s http://<BE_IP>:<BE_HTTP_PORT>/metrics | grep "^starrocks_be_.*_mem_bytes"
 
# tcmalloc allocator status
curl -s http://<BE_IP>:<BE_HTTP_PORT>/memz
 
# Memory-heavy queries from mem_tracker
curl -s http://<BE_IP>:<BE_HTTP_PORT>/mem_tracker | grep "query"

# Large allocation events (include query_id)
grep "large memory alloc" be.WARNING
 
# OOM Killer evidence
dmesg | grep -i "oom\|killed process\|out of memory" | tail -30
 
# Last entries before crash
tail -200 be.WARNING
tail -200 be.ERROR

# Full GC frequency and pause duration
grep "Full GC\|GC pause" fe.gc.log | tail -50
 
# Old Gen occupancy trend
grep "Old" fe.gc.log | tail -30
 
# Queries running at FE OOM shutdown
grep "FE ShutDown! Running Query" fe.WARNING
 
# Memory audit post-incident (top N by FE memory)
python3 scripts/analyze_logs.py "2025-04-15 00:00:00" "2025-04-15 01:00:00" "QueryFEAllocatedMemory" 3 fe.audit.log
 
# Memory audit post-incident (top N by BE memory)
python3 scripts/analyze_logs.py "2025-04-15 00:00:00" "2025-04-15 01:00:00" "MemCostBytes" 3 fe.audit.log

# Version 3.3+
grep "LockManager" fe.log | tail -50
 
# Version 3.2 and earlier
grep "DeadlockChecker" fe.log | tail -50
 
# Capture thread dump for manual analysis
jstack -l <fe_pid> > /tmp/fe_jstack.log
 
# Find blocked threads
grep "parking to wait for\|BLOCKED" /tmp/fe_jstack.log | head -30

# Check whether BE process is alive
SHOW BACKENDS;  -- Alive column; LastHeartbeat timestamp
 
# Check whether FE is alive
SHOW FRONTENDS;  -- Alive column
 
# If BE is down — check OOM Killer
dmesg | grep -i "oom\|killed process\|out of memory" | tail -20

# OOM: Linux kernel killed the process
dmesg | grep -i "killed process" | grep "starrocks_be"
# → If present: Cause A (BE OOM)
 
# Crash: core dump or signal in log
ls -lh /proc/sys/kernel/core_pattern   # find core dump location
tail -50 be.WARNING
# → Look for "Segmentation fault", "SIGABRT", "Fatal signal"
# → If present: Cause B (BE crash)

# Deadlock: look for LockManager JSON in fe.log
grep "LockManager\|DeadlockChecker" fe.log | tail -5
# → If present: Cause C (FE deadlock)
 
# Full GC: check GC log for recent Full GC
grep "Full GC" fe.gc.log | tail -5
# → If recent Full GC with pause > 5s: Cause D (FE Full GC)
 
# OOM shutdown: look for shutdown message
grep "FE ShutDown! Running Query" fe.WARNING | tail -5
# → If present: Cause E (FE OOM)

# Step 1: Find time window of memory spike via monitoring (Grafana or RSS trend)
# Step 2: Find large allocation events in that window
grep "large memory alloc" be.WARNING
# Output includes query_id and allocation size in bytes
 
# Step 3: Match query_id to SQL
grep "<query_id>" fe.audit.log
# Or check live queries if incident is ongoing:

SHOW PROC '/current_queries';
-- Column: memUsageBytes — sort descending to find the largest consumer

# Step 4: Check overall module breakdown
curl -s http://<BE_IP>:<BE_HTTP_PORT>/mem_tracker
# Look for which module (query, load, compaction, metadata) dominates

# Confirm core dump location
cat /proc/sys/kernel/core_pattern
 
# Check file descriptor limit
cat /proc/<be_pid>/limits | grep "open files"
 
# Review log immediately before crash
tail -200 be.WARNING
tail -200 be.ERROR
 
# If core dump present, get stack trace
gdb starrocks_be <core_file> -batch -ex "thread apply all bt" 2>/dev/null | head -200

# Version 3.3+
grep "LockManager" fe.log | tail -20
 
# Version 3.2 and earlier
grep "DeadlockChecker" fe.log | tail -20

# Capture thread dump
jstack -l <fe_pid> > /tmp/fe_jstack.log
 
# Find threads waiting for locks
grep "parking to wait for" /tmp/fe_jstack.log
 
# Find the lock holder for a given lock address
grep -A 5 "Locked ownable synchronizers" /tmp/fe_jstack.log | grep "<lock_address>"
 
# Identify long-running threads: convert PID from top -Hp to hex
top -Hp <fe_pid> -b -n 1 > top_output.txt
printf "%x\n" <thread_pid>   # e.g., 19700 → 4cf4
grep "nid=0x4cf4" /tmp/fe_jstack.log

# Check GC frequency and pause duration
grep "Full GC" fe.gc.log | tail -30
 
# Check Old Gen occupancy trend
grep "Old" fe.gc.log | tail -30
 
# Check if FE shut down due to OOM
grep "FE ShutDown! Running Query" fe.WARNING
 
# Locate flame graph profiles (v3.3.6+)
ls -lh fe/log/proc_profile/
# Files are .tgz archives containing HTML flame graphs
 
# Audit which SQL consumed most FE memory post-incident
python3 scripts/analyze_logs.py "2025-04-15 00:00:00" "2025-04-15 01:00:00" "QueryFEAllocatedMemory" 3 fe.audit.log

-- Find the query_id from be.WARNING large memory alloc entry
KILL QUERY '<query_id>';

-- Session level
SET query_mem_limit = 17179869184;  -- 16 GB example
 
-- Global level
SET GLOBAL query_mem_limit = 17179869184;

SET enable_query_spill = true;
SET spill_mem_table_size = 268435456;  -- 256 MB spill trigger

# Set mem_limit in be.conf to leave OS headroom (e.g., 80% of physical RAM)
mem_limit = 80%
 
# Restart BE to apply

# Confirm BE rejoins cluster
# SHOW BACKENDS — wait for Alive=true