Profiling Operations Runbook¶

# 1. Check current CPU regression ratio
curl -s "http://localhost:9090/api/v1/query?query=job:service_cpu_regression_ratio:5m_vs_24h" | jq '.data.result'

# 2. View CPU profile in Grafana Pyroscope
# Open: http://localhost:3002/grafana/d/hsi-profiling
# Select the affected service from dropdown (live names: "backend", "nemotron-backend")

# 3. Compare current vs baseline flame graphs
# Enable "Comparison" mode in the dashboard
# Look for new hot functions or significantly increased function times

# 4. Check for recent deployments
git log --oneline --since="24 hours ago"

# 5. Check if workload increased
curl -s "http://localhost:9090/api/v1/query?query=rate(hsi_detections_processed_total[1h])" | jq

# Identify the problematic commit using flame graph comparison
# Roll back to previous version if needed
git checkout <previous-sha>
podman compose -f docker-compose.prod.yml build --no-cache [service]
podman compose -f docker-compose.prod.yml up -d [service]

# 1. Check current memory regression ratio
curl -s "http://localhost:9090/api/v1/query?query=job:service_memory_regression_ratio:current_vs_6h" | jq '.data.result'

# 2. Check memory growth rate
curl -s "http://localhost:9090/api/v1/query?query=job:service_memory_bytes:deriv1h" | jq '.data.result'

# 3. Check memory profile in Pyroscope
# Select "Memory Bytes" or "Memory Allocations" profile type
# (backend has PYROSCOPE_MEMORY_ENABLED=true, so allocation profiles exist for it)
# Look for functions allocating large amounts

# 4. Check container memory limits
podman stats --no-stream [container_name]

# 5. For Python services, check for common leak patterns
podman exec [container] python -c "import tracemalloc; tracemalloc.start()"

# Restart service as immediate mitigation
podman compose -f docker-compose.prod.yml restart [service]

# Schedule investigation of leak source

# 1. Check projected memory
curl -s "http://localhost:9090/api/v1/query?query=job:service_memory_bytes:predicted_24h" | jq '.data.result'

# 2. Check growth rate (bytes/hour)
curl -s "http://localhost:9090/api/v1/query?query=job:service_memory_bytes:deriv1h" | jq '.data.result'

# 3. Analyze memory allocation profile over time
# In Grafana Pyroscope, compare memory profiles from:
# - 6 hours ago
# - Current
# Look for functions with significantly more allocations

# 4. For Python: enable memory profiling
podman exec [container] python -c "
import tracemalloc
tracemalloc.start()
# ... run suspect code ...
snapshot = tracemalloc.take_snapshot()
for stat in snapshot.statistics('lineno')[:10]:
    print(stat)
"

# Set up scheduled restarts until fix is deployed
# Add to crontab or systemd timer:
# 0 */4 * * * podman compose -f docker-compose.prod.yml restart [service]

# 1. Check current latency
curl -s "http://localhost:9090/api/v1/query?query=job:backend_api_latency:p99_5m" | jq '.data.result'

# 2. Check database query latency
curl -s "http://localhost:9090/api/v1/query?query=histogram_quantile(0.99,rate(hsi_db_query_duration_seconds_bucket[5m]))" | jq

# 3. Check Redis latency
curl -s "http://localhost:9090/api/v1/query?query=redis_slowlog_length" | jq

# 4. Check CPU usage (may be contention)
curl -s "http://localhost:9090/api/v1/query?query=job:backend_cpu_seconds:rate5m" | jq

# 5. View backend flame graph for hot paths
# Open http://localhost:3002/grafana/d/hsi-profiling
# Select "nemotron-backend" (SDK) or "backend" (py-spy)

# Check for long-running queries
podman exec postgres psql -U hsi -c "SELECT * FROM pg_stat_activity WHERE state = 'active';"

# Check for missing indexes
podman exec postgres psql -U hsi -c "EXPLAIN ANALYZE [slow_query];"

# Check slow log
podman exec redis redis-cli slowlog get 10

# Check memory usage
podman exec redis redis-cli info memory

# 1. Triton inference latency + throughput for yolo26
curl -s "http://localhost:9090/api/v1/query" --data-urlencode \
  'query=sum(rate(nv_inference_request_success_total{model="yolo26"}[5m]))' | jq

# 2. Check GPU utilization (dcgm-exporter)
curl -s "http://localhost:9090/api/v1/query?query=DCGM_FI_DEV_GPU_UTIL" | jq

# 3. Check GPU temperature (throttling?)
curl -s "http://localhost:9090/api/v1/query?query=DCGM_FI_DEV_GPU_TEMP" | jq

# 4. Check the router still reports the model ready
curl -s http://localhost:8090/yolo26/health | jq .
curl -s http://localhost:8090/health | jq '.models.yolo26'

# 5. Check backend-attributed detector latency
curl -s "http://localhost:9090/api/v1/query" --data-urlencode \
  'query=rate(hsi_ai_request_duration_seconds_sum{ai_service="yolo26"}[5m]) / rate(hsi_ai_request_duration_seconds_count{ai_service="yolo26"}[5m])' | jq

# Restart the gateway to reinitialize Triton/TensorRT
podman compose -f docker-compose.prod.yml restart ai-gateway
# Triton warm-up can take up to its 180s health start_period

# 1. Check which services are affected
curl -s "http://localhost:9090/api/v1/query?query=job:service_cpu_regression_ratio:5m_vs_24h>1.3" | jq '.data.result[].metric.job'

# 2. Check host-level metrics
podman stats --no-stream

# 3. Check for noisy neighbor (other processes)
top -b -n 1 | head -20

# 4. Check disk I/O (may cause CPU wait)
iostat -x 1 5

# 5. Check network issues
netstat -s | grep -i error

# Check if Pyroscope container is running
podman ps | grep pyroscope

# Check container health
podman inspect pyroscope --format='{{.State.Health.Status}}'

# Check container logs
podman logs pyroscope --tail 100

# Test internal connectivity (from the backend container)
podman exec backend python -c "import httpx; print(httpx.get('http://pyroscope:4040/ready', timeout=5).status_code)"

# Restart Pyroscope
podman compose -f docker-compose.prod.yml restart pyroscope

# If restart fails, recreate container
podman compose -f docker-compose.prod.yml up -d --force-recreate pyroscope

# Verify recovery
curl http://localhost:4040/ready

# Check py-spy processes in backend
podman exec backend ps aux | grep py-spy

# Check profiler log for errors
podman exec backend cat /app/data/logs/profiler.log

# Check profile interval (default 30s, set via PROFILE_INTERVAL)
podman exec backend env | grep -E "PROFILE_INTERVAL|PYROSCOPE"

# Add to docker-compose.override.yml under backend.environment:
#   - PROFILE_INTERVAL=60      (default is 30)
podman compose -f docker-compose.prod.yml up -d backend

# Add to docker-compose.override.yml under backend.environment:
#   - PYROSCOPE_ENABLED=false   (turns off both the py-spy loop and the SDK)
podman compose -f docker-compose.prod.yml up -d backend

echo "PYROSCOPE_ENABLED=false" >> .env
podman compose -f docker-compose.prod.yml up -d

# Is the profiler script running in backend?
podman exec backend pgrep -fa pyroscope-profiler.sh

# Profiler log (launched by backend/entrypoint.sh)
podman exec backend tail -50 /app/data/logs/profiler.log
# Expect lines like: "[pyroscope-profiler] ... Starting profiler for backend"

# Pyroscope actually receiving anything?
curl -s http://localhost:4040/api/services/status 2>/dev/null | head -c 400

# For backend (SDK-based), check initialization in app logs
podman logs backend 2>&1 | grep -i "pyroscope profiling"

# Restart the service to reinitialize the profiler
podman compose -f docker-compose.prod.yml restart backend

# Verify profiler started
podman exec backend tail -5 /app/data/logs/profiler.log

# Check the SDK dependency is installed in the image
podman exec backend python -c "import pyroscope; print(pyroscope.__name__)"

# Restart backend (SDK initializes during app startup, backend/core/telemetry.py)
podman compose -f docker-compose.prod.yml restart backend

# Verify initialization
podman logs backend 2>&1 | grep -i pyroscope

# Check volume usage
podman volume inspect pyroscope_data

# Check container disk usage (image is busybox-based — invoke df via busybox)
podman exec pyroscope busybox df -h /data

# Check retention settings currently mounted in the container (NEM-3928)
podman exec pyroscope cat /etc/pyroscope/config.yml

# Check compactor status
podman logs pyroscope 2>&1 | grep -i "compactor\|retention\|cleanup"

# Option 1: Wait for automatic cleanup (recommended)
# The compactor runs every 15 minutes and will delete old blocks
# when disk space drops below thresholds. Check logs:
podman logs pyroscope 2>&1 | grep -i "deleting\|cleanup\|retention"

# Option 2: Reduce retention period for faster cleanup
# Edit monitoring/pyroscope/pyroscope-config.yml:
# Change: compactor_blocks_retention_period: 720h
# To:     compactor_blocks_retention_period: 168h  # 7 days

# Restart Pyroscope to apply new config
podman compose -f docker-compose.prod.yml restart pyroscope

# Option 3: Force immediate compaction
# Trigger a compaction cycle by restarting Pyroscope
podman compose -f docker-compose.prod.yml restart pyroscope

# Option 4: If urgent, clear all data (last resort)
podman compose -f docker-compose.prod.yml stop pyroscope
podman volume rm pyroscope_data  # WARNING: Deletes all profiling history
podman compose -f docker-compose.prod.yml up -d pyroscope

# Check current base version (pinned in the Dockerfile)
grep "grafana/pyroscope" monitoring/pyroscope/Dockerfile

# Review release notes for breaking changes
# https://github.com/grafana/pyroscope/releases

# 1. Update the base image tag in monitoring/pyroscope/Dockerfile
#    e.g. FROM docker.io/grafana/pyroscope:1.18.0 -> 1.19.0

# 2. Backup configuration
cp monitoring/pyroscope/pyroscope-config.yml monitoring/pyroscope/pyroscope-config.yml.bak

# 3. Rebuild (always --no-cache for infra changes) and recreate
podman compose -f docker-compose.prod.yml build --no-cache pyroscope
podman compose -f docker-compose.prod.yml up -d pyroscope

# 4. Verify health
curl http://localhost:4040/ready

# Revert monitoring/pyroscope/Dockerfile to the previous tag
podman compose -f docker-compose.prod.yml build --no-cache pyroscope
podman compose -f docker-compose.prod.yml up -d pyroscope

RUN uv tool install py-spy && \
    cp -L /root/.local/bin/py-spy /usr/local/bin/py-spy && \
    chmod +x /usr/local/bin/py-spy

COPY --chmod=755 scripts/pyroscope-profiler.sh /usr/local/bin/pyroscope-profiler.sh

if [ "${PYROSCOPE_ENABLED:-true}" = "true" ]; then
    nohup /usr/local/bin/pyroscope-profiler.sh "my-service" \
        "${PYROSCOPE_URL:-http://pyroscope:4040}" "${PROFILE_INTERVAL:-30}" \
        >> /app/data/logs/profiler.log 2>&1 &
fi

my-new-service:
  labels:
    pyroscope.profile: 'true'
    pyroscope.service: 'my-new-service'
  environment:
    - PYROSCOPE_ENABLED=${PYROSCOPE_ENABLED:-true}
    - PYROSCOPE_URL=http://pyroscope:4040

podman compose -f docker-compose.prod.yml build --no-cache my-new-service
podman compose -f docker-compose.prod.yml up -d my-new-service

podman exec my-new-service cat /app/data/logs/profiler.log
# Should see: "Starting profiler for my-new-service"

# 1. Check if datasource exists (creds default to admin/admin unless GF_ADMIN_USER/GF_ADMIN_PASSWORD override in .env)
curl -s http://admin:admin@localhost:3002/grafana/api/datasources | jq '.[].name' # pragma: allowlist secret

# 2. If missing, add via provisioning
cat > monitoring/grafana/provisioning/datasources/pyroscope.yml << 'EOF'
apiVersion: 1
datasources:
  - name: Pyroscope
    uid: pyroscope
    type: grafana-pyroscope-datasource
    url: http://pyroscope:4040
    access: proxy
    isDefault: false
EOF

# 3. Restart Grafana
podman compose -f docker-compose.prod.yml restart grafana

#!/bin/bash
# Check Pyroscope health and alert if down

if ! curl -sf http://localhost:4040/ready > /dev/null 2>&1; then
    echo "ALERT: Pyroscope is not responding"
    # Add alerting integration here
    exit 1
fi

echo "OK: Pyroscope is healthy"

#!/bin/bash
# Check if profiles are being collected (data within last 5 minutes)

# Live profile names in the current topology: "backend" (py-spy) and
# "nemotron-backend" (SDK). Retired names (ai-yolo26, ai-florence, ai-clip)
# stopped reporting at the gateway consolidation.
SERVICES="backend nemotron-backend"

for service in $SERVICES; do
    # Check for data in last 5 minutes
    RESULT=$(curl -s "http://localhost:4040/pyroscope/render?query=${service}&from=now-5m&until=now&format=json" | jq '.flamebearer.numTicks // 0')

    if [ "$RESULT" -eq 0 ]; then
        echo "WARNING: No recent profiles for $service"
    else
        echo "OK: $service has recent profile data"
    fi
done

# Pyroscope 1.18.0 configuration (NEM-3928)
# Comprehensive retention policy to prevent disk bloat

# Storage configuration
storage:
  backend: filesystem
  filesystem:
    dir: /data

# Server configuration
server:
  http_listen_port: 4040

# PyroscopeDB retention policy (NEM-3928)
# Disk-based retention to prevent storage exhaustion
pyroscopedb:
  min_free_disk_gb: 10 # Delete oldest when below 10GB free
  min_disk_available_percentage: 0.05 # Or below 5% free
  enforcement_interval: 5m # Check every 5 minutes
  max_block_duration: 1h # Must match block_ranges period

# Compactor configuration
compactor:
  data_dir: /data/compactor
  compaction_interval: 2h # Compact every 2 hours
  cleanup_interval: 15m # Apply retention every 15 minutes
  deletion_delay: 2h # Safety buffer before deletion
  max_opening_blocks_concurrency: 4

# Limits configuration
limits:
  compactor_blocks_retention_period: 720h # 30 days max retention
  ingestion_rate_mb: 4 # Max ingestion rate
  ingestion_burst_size_mb: 8 # Burst allowance
  max_label_names_per_series: 30
  max_label_value_length: 2048

#!/bin/bash
# Starts profiler in background if enabled, then runs main command
set -e

if [ "${PYROSCOPE_ENABLED:-true}" = "true" ] && [ -n "$SERVICE_NAME" ]; then
    nohup /usr/local/bin/pyroscope-profiler.sh "$SERVICE_NAME" \
        "${PYROSCOPE_URL:-http://pyroscope:4040}" \
        "${PROFILE_INTERVAL:-30}" >> /tmp/profiler.log 2>&1 &
fi

exec "$@"