Prompt Management Architecture¶

┌─────────────────────────────────────────────────────────────────────┐
│                         Frontend                                     │
├─────────────────────────────────────────────────────────────────────┤
│  PromptPlayground.tsx    │  PromptABTest.tsx    │  ABTestStats.tsx  │
│  - Model editors         │  - Split-view UI     │  - Statistics     │
│  - Diff preview          │  - Delta indicators  │  - Comparison     │
│  - Test configuration    │  - Result panels     │  - Charts         │
└─────────────────────────────────────────────────────────────────────┘
                                    │
                                    ▼
┌─────────────────────────────────────────────────────────────────────┐
│                         Backend API                                  │
├─────────────────────────────────────────────────────────────────────┤
│  /api/ai-audit/prompts                                               │
│  - GET    /           - Get all prompts                              │
│  - GET    /{model}    - Get prompt for model                         │
│  - PUT    /{model}    - Update prompt (creates new version)          │
│  - GET    /export     - Export all configurations                    │
│  - POST   /import     - Import configurations                        │
│  - POST   /import/preview - Preview import changes                   │
│  - GET    /history    - Get version history                          │
│  - POST   /history/{id} - Restore version                            │
│  - POST   /test       - Test prompt against event                    │
└─────────────────────────────────────────────────────────────────────┘
                                    │
                                    ▼
┌─────────────────────────────────────────────────────────────────────┐
│                         Services                                     │
├─────────────────────────────────────────────────────────────────────┤
│  PromptService          │  PromptABTester       │  PromptEvaluator  │
│  - CRUD operations      │  - Traffic splitting  │  - Batch eval     │
│  - Version management   │  - A/B selection      │  - Comparison     │
│  - Import/Export        │  - Metrics recording  │  - Correlation    │
│                         │                       │                   │
│  PromptShadowRunner     │  PromptRollbackChecker                    │
│  - Shadow execution     │  - Performance check                      │
│  - Result comparison    │  - Auto rollback                          │
└─────────────────────────────────────────────────────────────────────┘
                                    │
                                    ▼
┌─────────────────────────────────────────────────────────────────────┐
│                         Database                                     │
├─────────────────────────────────────────────────────────────────────┤
│  prompt_versions                                                     │
│  - id, model, version, config_json, created_at, created_by          │
│  - change_description, is_active, row_version                        │
│  - Indexes: model, model+version, model+is_active, created_at       │
│  - Unique constraint: (model, version)                               │
└─────────────────────────────────────────────────────────────────────┘

class PromptVersion(Base):
    __tablename__ = "prompt_versions"

    id: Mapped[int]                    # Primary key
    model: Mapped[str]                 # Enum: nemotron, florence2, yolo_world, xclip, fashion_clip
    version: Mapped[int]               # Version number (auto-incremented per model)
    created_at: Mapped[datetime]       # When this version was created
    created_by: Mapped[str | None]     # Optional user identifier
    config_json: Mapped[str]           # JSON configuration (see below)
    change_description: Mapped[str | None]  # Optional description of changes
    is_active: Mapped[bool]            # True for current active version
    row_version: Mapped[int]           # Optimistic locking counter

{
  "system_prompt": "You are a security analysis AI...",
  "temperature": 0.7,
  "max_tokens": 2048
}

{
  "vqa_queries": ["What is the person doing?", "What objects are they carrying?"]
}

{
  "object_classes": ["knife", "gun", "package"],
  "confidence_threshold": 0.35
}

{
  "action_classes": ["loitering", "running away", "fighting"]
}

{
  "clothing_categories": ["dark hoodie", "face mask"],
  "suspicious_indicators": ["all black", "gloves at night"]
}

class PromptUpdateRequest(BaseModel):
    config: dict[str, Any]                    # New configuration
    change_description: str | None = None     # Optional change description
    expected_version: int | None = None       # For optimistic locking

def validate_config_for_model(model: AIModelEnum, config: dict) -> list[str]:
    """Validate configuration for a specific model."""

class PromptService:
    async def get_all_prompts(self, session: AsyncSession) -> dict[str, dict]:
        """Get active configurations for all models."""

    async def get_prompt_for_model(self, session: AsyncSession, model: str) -> dict:
        """Get active configuration for a specific model."""

    async def update_prompt_for_model(
        self,
        session: AsyncSession,
        model: str,
        config: dict,
        change_description: str | None = None,
        created_by: str | None = None,
        expected_version: int | None = None,
    ) -> PromptVersion:
        """Create new version with updated configuration."""

    async def get_version_history(
        self,
        session: AsyncSession,
        model: str | None = None,
        limit: int = 50,
        offset: int = 0,
    ) -> tuple[list[PromptVersion], int]:
        """Get paginated version history."""

    async def restore_version(
        self,
        session: AsyncSession,
        version_id: int,
    ) -> PromptVersion:
        """Restore a previous version (creates new version with old config)."""

@dataclass
class ABTestConfig:
    control_version: int       # Current production version
    treatment_version: int     # New version being tested
    traffic_split: float       # 0.0 to 1.0 - fraction to treatment
    model: str                 # Model name
    enabled: bool = True

class PromptABTester:
    def select_prompt_version(self) -> tuple[int, bool]:
        """Select version based on traffic split. Returns (version, is_treatment)."""

    async def record_prompt_execution(
        self,
        version: int,
        latency_seconds: float,
        risk_score: int,
    ) -> None:
        """Record metrics for analysis."""

@dataclass
class ShadowModeConfig:
    enabled: bool
    control_version: int
    shadow_version: int
    model: str
    log_comparisons: bool = True

@dataclass
class ShadowComparisonResult:
    control_result: dict
    shadow_result: dict | None
    risk_score_diff: float
    control_latency_ms: float
    shadow_latency_ms: float
    shadow_error: str | None

class PromptShadowRunner:
    async def run_shadow_comparison(self, context: str) -> ShadowComparisonResult:
        """Run both prompts and compare results."""

@dataclass
class RollbackConfig:
    enabled: bool = True
    max_latency_increase_pct: float = 50.0   # Rollback if latency increases 50%+
    max_score_variance: float = 15.0         # Rollback if score variance > 15
    min_samples: int = 100                   # Minimum samples before checking
    evaluation_window_hours: int = 1

class PromptRollbackChecker:
    async def check_rollback_needed(self, metrics) -> RollbackCheckResult:
        """Check if rollback should be triggered."""

    async def execute_rollback(
        self,
        session: AsyncSession,
        ab_config,
        reason: str,
    ) -> RollbackExecutionResult:
        """Execute rollback to control version."""

@dataclass
class EvaluationResults:
    total_events: int
    average_score_diff: float | None
    score_variance: float | None
    average_latency_ms: float
    score_correlation: float | None

class PromptEvaluator:
    async def create_evaluation_batch(
        self,
        session: AsyncSession,
        hours_back: int = 24,
        sample_size: int = 100,
    ) -> EvaluationBatch:
        """Create batch of historical events for evaluation."""

    async def evaluate_prompt_version(
        self,
        session: AsyncSession,
        prompt_version: int,
        batch: EvaluationBatch,
    ) -> EvaluationResults:
        """Evaluate prompt version against event batch."""

    async def compare_prompt_versions(
        self,
        session: AsyncSession,
        version_a: int,
        version_b: int,
        batch: EvaluationBatch,
    ) -> VersionComparisonResult:
        """Compare two versions and recommend best."""

interface PromptPlaygroundProps {
  isOpen: boolean;
  onClose: () => void;
  recommendation?: AiAuditRecommendationItem | null;
  sourceEventId?: number | null;
  enrichedSuggestion?: EnrichedSuggestion | null;
  initialShowDiffPreview?: boolean;
}

// Prompts and editing
const [prompts, setPrompts] = useState<AllPromptsResponse | null>(null);
const [editedConfigs, setEditedConfigs] = useState<Record<string, Record<string, unknown>>>({});

// A/B testing
const [showABTest, setShowABTest] = useState(false);
const [abTestResults, setAbTestResults] = useState<ABTestResult[]>([]);
const [isRunningABTest, setIsRunningABTest] = useState(false);

interface PromptABTestProps {
  originalPrompt: string;
  modifiedPrompt: string;
  results: ABTestResult[];
  isRunning: boolean;
  onRunTest: (eventId: number) => void;
  onRunRandomTests: (count: number) => void;
  onPromoteB: () => void;
}

// Green (improvement): B is less alarming (delta <= -5)
// Red (regression): B is more alarming (delta >= +5)
// Gray (neutral): Within threshold (-4 to +4)

export function calculateStats(results: ABTestResult[]): ABTestStats {
  return {
    totalTests: results.length,
    avgScoreDelta: /* average of all score deltas */,
    improvementRate: /* percentage where B was better */,
  };
}

// Apply a suggestion to a prompt
export function applySuggestion(
  originalPrompt: string,
  suggestion: EnrichedSuggestion
): InsertionResult;

// Generate visual diff between prompts
export function generateDiff(
  originalPrompt: string,
  modifiedPrompt: string,
  contextLines?: number
): DiffLine[];

// Check if suggestion is already applied
export function isSuggestionApplied(prompt: string, suggestion: EnrichedSuggestion): boolean;

export interface ABTestService {
  runTest(
    eventId: number,
    originalPrompt: string,
    modifiedPrompt: string,
    modelConfig: ModelConfig
  ): Promise<ABTestResult>;

  runRandomTests(
    count: number,
    originalPrompt: string,
    modifiedPrompt: string,
    modelConfig: ModelConfig
  ): Promise<ABTestResult[]>;

  getAvailableEvents(limit?: number): Promise<EventSummary[]>;
}

# backend/models/prompt_version.py
class AIModel(str, Enum):
    # ... existing models ...
    NEW_MODEL = "new_model"

# backend/api/schemas/prompt_management.py
class NewModelConfig(BaseModel):
    custom_field: str = Field(..., min_length=1)
    threshold: float = Field(0.5, ge=0.0, le=1.0)

validators: dict[AIModelEnum, tuple[type[BaseModel], dict[str, str]]] = {
    # ... existing models ...
    AIModelEnum.NEW_MODEL: (NewModelConfig, {"custom_field": "custom_field"}),
}

# backend/services/prompt_service.py
DEFAULT_CONFIGS: dict[str, dict[str, Any]] = {
    # ... existing models ...
    AIModelEnum.NEW_MODEL.value: {
        "custom_field": "default value",
        "threshold": 0.5,
    },
}

// frontend/src/components/ai/PromptPlayground.tsx
const MODEL_CONFIGS: ModelConfig[] = [
  // ... existing models ...
  {
    name: 'new_model',
    displayName: 'New Model',
    description: 'Description of the model',
  },
];

// Add case in renderModelEditor()
case 'new_model':
  return (
    <div className="space-y-4">
      {/* Custom editor UI */}
    </div>
  );

alembic revision --autogenerate -m "add new_model enum value"

## Role
You are a security analysis AI for a home monitoring system.

## Input Context
- Detection data: {detections}
- Time context: {time_context}
- Weather: {weather}
- Cross-camera data: {cross_camera_data}

## Analysis Instructions
1. Analyze all detections in context
2. Consider time of day and normal patterns
3. Evaluate for security concerns

## Output Format
Respond with valid JSON:
{
  "risk_score": 0-100,
  "risk_level": "low|medium|high|critical",
  "reasoning": "explanation",
  "recommended_actions": ["action1", "action2"]
}

## Risk Level Criteria
- LOW (0-29): Normal activity
- MEDIUM (30-59): Unusual but not threatening
- HIGH (60-84): Concerning activity
- CRITICAL (85-100): Potential threat

# Recorded on each prompt execution
record_prompt_latency(version: str, latency_seconds: float)

# Recorded on shadow mode comparison
record_shadow_comparison(model: str)

# Recorded on rollback
record_prompt_rollback(model: str, reason: str)

# Run prompt service tests
uv run pytest backend/tests/unit/services/test_prompt_service.py -v
uv run pytest backend/tests/unit/services/test_prompt_ab_testing.py -v
uv run pytest backend/tests/unit/services/test_prompt_version_service.py -v

# Run API route tests
uv run pytest backend/tests/unit/api/routes/test_prompt_management.py -v
uv run pytest backend/tests/unit/api/routes/test_prompt_config.py -v

cd frontend
npm test -- --grep "PromptPlayground"
npm test -- --grep "promptDiff"
npm test -- --grep "abTestService"