Nemotron Prompting Best Practices¶

<|im_start|>system
{system message}
<|im_end|>
<|im_start|>user
{user message}
<|im_end|>
<|im_start|>assistant
{model response begins here}

prompt = """<|im_start|>system
You are a home security analyst for a residential property.
Output ONLY valid JSON. No preamble, no explanation.<|im_end|>
<|im_start|>user
Camera: Front Door
Time: 14:30:00 to 14:31:00

DETECTIONS:
- 14:30:05: person detected (confidence: 0.92)

Analyze the risk and output JSON with risk_score (0-100), risk_level, summary, and reasoning.<|im_end|>
<|im_start|>assistant
"""

payload = {
    "prompt": prompt,
    "temperature": 0.7,
    "top_p": 0.95,
    "max_tokens": 1536,
    "stop": ["<|im_end|>", "<|im_start|>"]  # ChatML terminators
}

SYSTEM_PROMPT_WITH_REASONING = """detailed thinking on

You are a home security analyst for a residential property.

REASONING INSTRUCTIONS:
1. First, output your reasoning in <think>...</think> tags
2. Consider: time of day, location, object types, household context
3. Evaluate each factor systematically before determining the risk score
4. After </think>, output ONLY valid JSON with no additional text

Output format after </think>:
{"risk_score": N, "risk_level": "level", "summary": "...", "reasoning": "..."}"""

<think>
Let me analyze this detection systematically:

1. Time: 14:30 - normal business hours
2. Location: Front door - entry point
3. Detection: Single person with high confidence (0.92)
4. Context: Daytime delivery is common at this hour

Risk factors:
- Normal hours: low risk modifier
- Single person: not a group, lower risk
- No suspicious behavior detected

Final assessment: Likely a delivery or expected visitor.
</think>
{"risk_score": 15, "risk_level": "low", "summary": "Person detected at front door during daytime", "reasoning": "Single person detection during normal business hours at front entrance. Pattern consistent with delivery or expected visitor."}

from backend.services.nemotron_analyzer import extract_reasoning_and_response

text = "<think>Analyzing the scene...</think>{\"risk_score\": 25}"
reasoning, json_response = extract_reasoning_and_response(text)

# reasoning = "Analyzing the scene..."
# json_response = '{"risk_score": 25}'

from backend.api.schemas.llm_response import RISK_ANALYSIS_JSON_SCHEMA

# The schema defines:
# - risk_score: integer 0-100
# - risk_level: enum ["low", "medium", "high", "critical"]
# - summary: string (max 200 chars)
# - reasoning: string
# - entities: array of detected entities
# - recommended_action: enum

response = client.chat.completions.create(
    model="nemotron",
    messages=[...],
    extra_body={
        "nvext": {
            "guided_json": RISK_ANALYSIS_JSON_SCHEMA
        }
    }
)

from backend.services.guided_constraints import (
    RISK_LEVEL_CHOICES,       # ["low", "medium", "high", "critical"]
    RECOMMENDED_ACTION_CHOICES,  # ["none", "review_later", "review_soon", "alert_homeowner", "immediate_response"]
    ENTITY_TYPE_CHOICES,      # ["person", "vehicle", "animal", "object"]
    THREAT_LEVEL_CHOICES,     # ["low", "medium", "high"]
)

# Use in nvext namespace
config = {
    "nvext": {
        "guided_choice": RISK_LEVEL_CHOICES
    }
}

from backend.services.guided_constraints import get_guided_regex_config

# Risk score: 0-100
config = get_guided_regex_config("risk_score")
# Returns: {'nvext': {'guided_regex': '[0-9]|[1-9][0-9]|100'}}

# Threat level score: 0-4
config = get_guided_regex_config("threat_level_score")
# Returns: {'nvext': {'guided_regex': '[0-4]'}}

# .env
NEMOTRON_USE_GUIDED_JSON=true
NEMOTRON_GUIDED_JSON_FALLBACK=true  # Fall back to regex parsing if guided_json unsupported

CALIBRATED_SYSTEM_PROMPT = """You are a home security analyst for a residential property.

CRITICAL PRINCIPLE: Most detections are NOT threats. Residents, family members,
delivery workers, and pets represent normal household activity. Your job is to
identify genuine anomalies, not flag everyday life.

CALIBRATION: In a typical day, expect:
- 80% of events to be LOW risk (0-29): Normal activity
- 15% to be MEDIUM risk (30-59): Worth noting but not alarming
- 4% to be HIGH risk (60-84): Genuinely suspicious, warrants review
- 1% to be CRITICAL (85-100): Immediate threats only

If you're scoring >20% of events as HIGH or CRITICAL, you are miscalibrated.

Output ONLY valid JSON. No preamble, no explanation."""

## SCORING REFERENCE

| Scenario                    | Score  | Reasoning               |
| --------------------------- | ------ | ----------------------- |
| Resident arriving home      | 5-15   | Expected activity       |
| Delivery driver at door     | 15-25  | Normal service visit    |
| Unknown person on sidewalk  | 20-35  | Public area, passive    |
| Unknown person lingering    | 45-60  | Warrants attention      |
| Person testing door handles | 70-85  | Clear suspicious intent |
| Active break-in or violence | 85-100 | Immediate threat        |

risk_score = (threat_level * 25) + (apparent_intent * 15) + (time_context * 10)

from backend.services.threat_categories import ThreatCategory

# Categories ordered by severity
ThreatCategory.VIOLENCE           # Physical violence or fighting
ThreatCategory.WEAPON_VISIBLE     # Firearm, knife, or weapon visible
ThreatCategory.CRIMINAL_PLANNING  # Evidence of planning criminal activity
ThreatCategory.THREAT_INTIMIDATION # Threatening or intimidating behavior
ThreatCategory.FRAUD_DECEPTION    # Deceptive behavior (impersonation)
ThreatCategory.ILLEGAL_ACTIVITY   # General illegal activity detected
ThreatCategory.PROPERTY_DAMAGE    # Vandalism or destruction
ThreatCategory.TRESPASSING        # Unauthorized entry or presence
ThreatCategory.THEFT_ATTEMPT      # Taking property without permission
ThreatCategory.SURVEILLANCE_CASING # Reconnaissance or casing
ThreatCategory.NONE               # No threat detected

from backend.services.threat_categories import get_category_prompt_section

prompt_section = get_category_prompt_section()
# Returns formatted markdown list of all categories with descriptions

data/synthetic/
├── normal/           # Low-risk scenarios (80% of training)
│   ├── delivery_driver_*/
│   ├── resident_arrival_*/
│   ├── pet_activity_*/
│   └── vehicle_parking_*/
├── suspicious/       # Medium-risk scenarios (15%)
│   ├── loitering_*/
│   └── casing_*/
└── threats/          # High/critical scenarios (5%)
    ├── break_in_*/
    └── vandalism_*/

scenario_name_timestamp/
├── expected_labels.json   # Ground truth with expected risk scores
├── scenario_spec.json     # Full scenario specification
├── metadata.json          # Generation metadata
├── generation_prompt.txt  # Prompt used for media generation
└── media/                 # Generated images/videos (optional)
    ├── 001.png
    └── 002.png

{
  "detections": [{ "class": "person", "min_confidence": 0.8, "count": 1 }],
  "risk": {
    "min_score": 0,
    "max_score": 15,
    "level": "low",
    "expected_factors": []
  },
  "florence_caption": {
    "must_contain": ["person", "package"],
    "must_not_contain": ["suspicious", "threat"]
  }
}

from backend.evaluation.prompt_eval_dataset import (
    load_synthetic_eval_dataset,
    get_samples_by_category,
    get_scenario_summary,
)

# Load all samples
samples = load_synthetic_eval_dataset()
print(f"Loaded {len(samples)} samples")

# Group by category
by_category = get_samples_by_category(samples)
for cat, cat_samples in by_category.items():
    print(f"  {cat}: {len(cat_samples)} samples")

# Get summary statistics
summary = get_scenario_summary(samples)

from backend.evaluation.prompt_evaluator import (
    evaluate_prediction,
    evaluate_batch,
    calculate_metrics,
    summarize_results,
)

# Evaluate a single prediction
result = evaluate_prediction(
    sample=sample,
    actual_score=25,
    actual_level="low"
)

print(f"Score in range: {result.score_in_range}")
print(f"Level match: {result.level_match}")
print(f"Deviation: {result.deviation}")

# Batch evaluation
results = evaluate_batch(samples, predictions)
metrics = calculate_metrics(results)

print(f"Score accuracy: {metrics['accuracy']:.1%}")
print(f"Level accuracy: {metrics['level_accuracy']:.1%}")
print(f"Combined accuracy: {metrics['combined_accuracy']:.1%}")

cp data/synthetic/normal/delivery_driver_*/media/* /export/foscam/synthetic_camera/

from backend.config.prompt_ab_config import (
    PromptExperiment,
    get_experiment,
    list_experiments,
)

# Get a predefined experiment
experiment = get_experiment("rubric_vs_current")
print(f"Testing: {experiment.control_prompt_key} vs {experiment.variant_prompt_key}")
print(f"Traffic split: {experiment.traffic_split:.0%} to variant")

# Create a custom experiment
custom = PromptExperiment(
    name="cot_reasoning_test",
    description="Test chain-of-thought reasoning impact",
    control_prompt_key="calibrated_system",
    variant_prompt_key="reasoning_enabled",
    traffic_split=0.1,  # 10% to variant
    metrics=[
        "json_parse_success_rate",
        "score_in_range_accuracy",
        "level_match_accuracy",
        "response_latency_ms",
    ]
)

from backend.evaluation.ab_experiment_runner import (
    select_variant,
    analyze_experiment,
    summarize_results,
)

# Select variant for a request
prompt_key = select_variant(experiment)

# After collecting data, analyze results
results = analyze_experiment(
    control_scores=[0.82, 0.85, 0.79, 0.88, 0.83],
    variant_scores=[0.91, 0.89, 0.93, 0.87, 0.92],
    alpha=0.05  # Significance level
)

# Check statistical significance
if results.is_significant:
    print(f"Significant! p-value: {results.p_value:.4f}")
    print(f"Effect size (Cohen's d): {results.effect_size:.2f}")

# Generate summary
print(summarize_results(results))