AI Services GHCR Deployment¶

# 1. Pull latest backend and frontend from GHCR
podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/backend:latest
podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/frontend:latest

# 2. Build the AI containers locally (first time only, ~10-15 min)
podman compose -f docker-compose.prod.yml build ai-gateway ai-llm

# 3. Download models (manifest: models.yml)
./ai/download_models.sh

# 4. Start the full stack
podman compose -f docker-compose.prod.yml up -d

# 5. Verify deployment
curl -s http://localhost:8000/api/system/health/ready | jq
curl -s http://localhost:8090/health | jq       # ai-gateway
curl -s http://localhost:8091/health | jq       # ai-llm

# Build and start the gateway + LLM
podman compose -f docker-compose.prod.yml build ai-gateway ai-llm
podman compose -f docker-compose.prod.yml up -d ai-gateway ai-llm

# Verify
curl http://localhost:8090/health  # ai-gateway (all Triton models)
curl http://localhost:8091/health  # Nemotron

podman compose -f docker-compose.prod.yml build ai-gateway

volumes:
  - ${AI_MODELS_PATH:-/export/ai_models}/model-zoo:/models/zoo:ro
  - ${AI_MODELS_PATH:-/export/ai_models}/triton:/models/cache
  - ${AI_MODELS_PATH:-/export/ai_models}/quantized:/models/quantized:ro
  - triton-kernel-cache:/root/.nv
  - triton-tmp-cache:/tmp
  - ${HF_CACHE_PATH:-/home/ubuntu/.cache/huggingface}:/root/.cache/huggingface

podman compose -f docker-compose.prod.yml build ai-llm

volumes:
  - ${AI_MODELS_PATH:-/export/ai_models}/nemotron/nemotron-3-nano-30b-a3b-q4km:/models:ro
  - llama-cache:/home/llama/.cache
  - llama-nv-cache:/home/llama/.nv

mkdir -p /export/ai_models/nemotron/nemotron-3-nano-30b-a3b-q4km
cd /export/ai_models/nemotron/nemotron-3-nano-30b-a3b-q4km
wget https://huggingface.co/nvidia/Nemotron-3-Nano-30B-A3B-GGUF/resolve/main/Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf

# ai-gateway
devices:
  - nvidia.com/gpu=all
environment:
  - CUDA_VISIBLE_DEVICES=${GPU_AI_SERVICES:-1}
# ai-llm
devices:
  - nvidia.com/gpu=${GPU_LLM:-0}
environment:
  - CUDA_VISIBLE_DEVICES=${GPU_LLM:-0}

podman run --rm --device nvidia.com/gpu=all \
  docker.io/nvidia/cuda:12.0-base-ubuntu22.04 nvidia-smi

# Clone the repository
git clone https://github.com/mikesvoboda/nemotron-v3-home-security-intelligence.git
cd nemotron-v3-home-security-intelligence

# Run setup script (creates .env, installs deps)
python setup.py

# Pull backend/frontend from GHCR
podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/backend:latest
podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/frontend:latest

# Build AI containers locally
podman compose -f docker-compose.prod.yml build ai-gateway ai-llm

# Download models (see Model Downloads above)
./ai/download_models.sh

# Start all services
podman compose -f docker-compose.prod.yml up -d

# Build core AI containers
podman compose -f docker-compose.prod.yml build ai-gateway ai-llm

# Download Nemotron model (or run ./ai/download_models.sh)
mkdir -p /export/ai_models/nemotron/nemotron-3-nano-30b-a3b-q4km
wget -O /export/ai_models/nemotron/nemotron-3-nano-30b-a3b-q4km/Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf \
  https://huggingface.co/nvidia/Nemotron-3-Nano-30B-A3B-GGUF/resolve/main/Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf

# Start core services only
podman compose -f docker-compose.prod.yml up -d \
  postgres redis backend frontend ai-gateway ai-llm

git clone https://github.com/mikesvoboda/nemotron-v3-home-security-intelligence.git
cd nemotron-v3-home-security-intelligence
podman compose -f docker-compose.prod.yml build ai-gateway ai-llm
podman compose -f docker-compose.prod.yml up -d ai-gateway ai-llm

GPU_HOST=10.0.0.50  # Your GPU host IP
AI_GATEWAY_URL=http://${GPU_HOST}:8090
NEMOTRON_URL=http://${GPU_HOST}:8091

podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/backend:latest
podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/frontend:latest

podman compose -f docker-compose.prod.yml up -d postgres redis backend frontend

# Pull latest images
podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/backend:latest
podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/frontend:latest

# Recreate containers with new images
podman compose -f docker-compose.prod.yml up -d backend frontend

# Pull latest source code
git pull origin main

# Rebuild AI containers (--no-cache: cached layers hold stale code)
podman compose -f docker-compose.prod.yml build --no-cache ai-gateway ai-llm

# Recreate containers
podman compose -f docker-compose.prod.yml up -d ai-gateway ai-llm

# Deploy specific commit version
SHA=abc123
podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/backend:${SHA}
podman pull ghcr.io/mikesvoboda/nemotron-v3-home-security-intelligence/frontend:${SHA}
IMAGE_TAG=${SHA} podman compose -f docker-compose.ghcr.yml up -d   # ghcr stack only

curl http://localhost:8000/api/system/health/ready   # Backend

# AI services
curl http://localhost:8090/health                    # ai-gateway (aggregate)
curl http://localhost:8090/yolo26/health             # per-router
curl http://localhost:8091/health                    # Nemotron

#!/bin/bash
echo "=== Health Check ==="

services=(
  "backend:http://localhost:8000/api/system/health/ready"
  "ai-gateway:http://localhost:8090/health"
  "ai-gateway-yolo26:http://localhost:8090/yolo26/health"
  "ai-gateway-florence:http://localhost:8090/florence/health"
  "ai-gateway-clip:http://localhost:8090/clip/health"
  "ai-gateway-enrichment:http://localhost:8090/enrichment/health"
  "ai-gateway-enrich-lt:http://localhost:8090/enrich-lt/health"
  "ai-llm:http://localhost:8091/health"
)

for svc in "${services[@]}"; do
  name="${svc%%:*}"
  url="${svc#*:}"
  if curl -sf "$url" > /dev/null 2>&1; then
    echo "[OK] $name"
  else
    echo "[FAIL] $name ($url)"
  fi
done

# Watch GPU memory and utilization
watch -n 1 nvidia-smi

# Per-process VRAM usage
nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv

# Check container logs
podman compose -f docker-compose.prod.yml logs ai-gateway
podman compose -f docker-compose.prod.yml logs ai-llm

# Triton model-load failures show up first in the gateway log
podman logs ai-gateway 2>&1 | grep -iE "error|failed|model"

# Check if model files exist
ls -la /export/ai_models/nemotron/
ls -la /export/ai_models/model-zoo/

# Verify nvidia-container-toolkit is installed
nvidia-ctk --version

# Regenerate CDI spec (Podman)
sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml

# Test GPU access
podman run --rm --device nvidia.com/gpu=all \
  docker.io/nvidia/cuda:12.0-base-ubuntu22.04 nvidia-smi

# Check current VRAM usage
nvidia-smi

# Reduce GPU layers for Nemotron (in .env)
GPU_LAYERS=25  # default is `auto` (all layers on GPU)

# Move the gateway to a second GPU instead of shrinking the LLM
GPU_AI_SERVICES=1   # already the default two-GPU split

podman compose -f docker-compose.prod.yml logs -f ai-llm