Delete caller-model routing + preflight admin UI
Backend (~273 lines):
- Removed GET/POST /api/caller-models endpoints (strategy/pool/map/fallback config)
- Removed POST /api/caller-models/{caller_key} override endpoint
- Removed GET /api/show/preflight diagnostics endpoint
Frontend:
- HTML: dropped Preflight header button, Caller Models settings panel,
Preflight modal, and caller-model-badge/override from info panel.
- JS (~328 lines): removed loadCallerModels, saveCallerModels,
updateCallerModelUI/Badges, overrideCallerModel, showCallerModelBadge,
populateCallerModelOverride, runPreflight + render helpers,
MODEL_ABBREVS, CALLER_STYLES, and all their listeners and init calls.
- CSS (~184 lines): removed .cm-*, .preflight-*, .caller-model-*,
.model-tag, .info-badge.model, .caller-model-override rules.
Session fields (caller_model_strategy/pool/map/fallback/models,
caller_styles, caller_shapes) still present — they wire into shape
picking and checkpoint restore and get deleted in the next commits.
This commit is contained in:
-275
@@ -9884,281 +9884,6 @@ async def set_show_theme(data: dict):
|
||||
return {"theme": session.show_theme}
|
||||
|
||||
|
||||
# --- Caller Model Routing ---
|
||||
|
||||
@app.get("/api/caller-models")
|
||||
async def get_caller_models():
|
||||
"""Get current caller model routing config and per-caller assignments."""
|
||||
assignments = {}
|
||||
for key in CALLER_BASES:
|
||||
name = CALLER_BASES[key].get("name", key)
|
||||
model = session.caller_models.get(key)
|
||||
assignments[key] = {"name": name, "model": model or "(default)"}
|
||||
return {
|
||||
"strategy": session.caller_model_strategy,
|
||||
"pool": session.caller_model_pool,
|
||||
"map": session.caller_model_map,
|
||||
"fallback": session.caller_model_fallback,
|
||||
"assignments": assignments,
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/caller-models")
|
||||
async def set_caller_models(data: dict):
|
||||
"""Update caller model routing strategy, pool, map, or fallback."""
|
||||
if "strategy" in data:
|
||||
strategy = data["strategy"]
|
||||
if strategy not in ("single", "cycle", "style_matched"):
|
||||
raise HTTPException(400, f"Invalid strategy: {strategy}")
|
||||
session.caller_model_strategy = strategy
|
||||
print(f"[CallerModel] Strategy set to: {strategy}")
|
||||
if "pool" in data:
|
||||
pool = data["pool"]
|
||||
if not isinstance(pool, list) or not pool:
|
||||
raise HTTPException(400, "pool must be a non-empty list of model IDs")
|
||||
session.caller_model_pool = pool
|
||||
print(f"[CallerModel] Pool set to: {pool}")
|
||||
if "map" in data:
|
||||
session.caller_model_map = data["map"]
|
||||
print(f"[CallerModel] Style map set: {len(data['map'])} entries")
|
||||
if "fallback" in data:
|
||||
session.caller_model_fallback = data["fallback"]
|
||||
print(f"[CallerModel] Fallback set to: {data['fallback']}")
|
||||
# Clear existing assignments so new strategy takes effect
|
||||
if "strategy" in data or "pool" in data or "map" in data:
|
||||
session.caller_models.clear()
|
||||
session._caller_model_cycle_idx = 0
|
||||
print(f"[CallerModel] Cleared caller assignments (new config)")
|
||||
_save_checkpoint()
|
||||
return await get_caller_models()
|
||||
|
||||
|
||||
@app.post("/api/caller-models/{caller_key}")
|
||||
async def set_caller_model_override(caller_key: str, data: dict):
|
||||
"""Override the model for a specific caller mid-show."""
|
||||
if caller_key not in CALLER_BASES:
|
||||
raise HTTPException(404, f"Unknown caller key: {caller_key}")
|
||||
model = data.get("model", "").strip()
|
||||
if not model:
|
||||
# Clear override
|
||||
session.caller_models.pop(caller_key, None)
|
||||
name = CALLER_BASES[caller_key].get("name", caller_key)
|
||||
print(f"[CallerModel] Cleared override for {name}")
|
||||
else:
|
||||
session.caller_models[caller_key] = model
|
||||
name = CALLER_BASES[caller_key].get("name", caller_key)
|
||||
print(f"[CallerModel] Override {name} → {model}")
|
||||
_save_checkpoint()
|
||||
return {"caller_key": caller_key, "model": session.caller_models.get(caller_key, "(default)")}
|
||||
|
||||
|
||||
# --- Show Preflight ---
|
||||
|
||||
@app.get("/api/show/preflight")
|
||||
async def show_preflight(test_responses: bool = False):
|
||||
"""Run diagnostic checks before a show goes live."""
|
||||
from .services.tts import VOICE_PROFILES
|
||||
|
||||
checks = {}
|
||||
|
||||
# --- 1. Model diversity ---
|
||||
caller_assignments = []
|
||||
model_counts: dict[str, int] = {}
|
||||
for key, base in CALLER_BASES.items():
|
||||
raw_style = session.caller_styles.get(key, "")
|
||||
style_key = _normalize_style_key(raw_style) if raw_style else ""
|
||||
# Look up what model they'd get without persisting
|
||||
model = session.caller_model_map.get(style_key) if style_key else None
|
||||
if not model:
|
||||
model = session.caller_model_fallback
|
||||
caller_assignments.append({
|
||||
"name": base.get("name", key),
|
||||
"style": style_key or "(none)",
|
||||
"model": model,
|
||||
})
|
||||
model_counts[model] = model_counts.get(model, 0) + 1
|
||||
|
||||
total_callers = len(caller_assignments)
|
||||
max_same = max(model_counts.values()) if model_counts else 0
|
||||
diversity_status = "fail" if total_callers > 0 and max_same / total_callers > 0.5 else "pass"
|
||||
checks["model_diversity"] = {
|
||||
"status": diversity_status,
|
||||
"details": {
|
||||
"callers": caller_assignments,
|
||||
"model_distribution": model_counts,
|
||||
"max_same_model_pct": round(max_same / total_callers * 100) if total_callers else 0,
|
||||
},
|
||||
}
|
||||
|
||||
# --- 2. Theme penetration ---
|
||||
if session.show_theme:
|
||||
theme_words = [w.lower() for w in session.show_theme.split() if len(w) > 2]
|
||||
hits = []
|
||||
misses = []
|
||||
for key, base in CALLER_BASES.items():
|
||||
bg = session.caller_backgrounds.get(key)
|
||||
if not isinstance(bg, CallerBackground):
|
||||
misses.append(base.get("name", key))
|
||||
continue
|
||||
searchable = f"{bg.natural_description} {bg.situation_summary}".lower()
|
||||
if any(tw in searchable for tw in theme_words):
|
||||
hits.append(base.get("name", key))
|
||||
else:
|
||||
misses.append(base.get("name", key))
|
||||
total_bg = len(hits) + len(misses)
|
||||
hit_pct = round(len(hits) / total_bg * 100) if total_bg else 0
|
||||
theme_status = "warn" if total_bg > 0 and hit_pct < 40 else "pass"
|
||||
checks["theme_penetration"] = {
|
||||
"status": theme_status,
|
||||
"details": {
|
||||
"theme": session.show_theme,
|
||||
"connected": hits,
|
||||
"not_connected": misses,
|
||||
"penetration_pct": hit_pct,
|
||||
},
|
||||
}
|
||||
else:
|
||||
checks["theme_penetration"] = {"status": "skip", "details": {"reason": "No theme set"}}
|
||||
|
||||
# --- 3. Voice-age alignment ---
|
||||
mismatches = []
|
||||
alignments = []
|
||||
for key, base in CALLER_BASES.items():
|
||||
bg = session.caller_backgrounds.get(key)
|
||||
if not isinstance(bg, CallerBackground):
|
||||
continue
|
||||
voice_name = base.get("voice", "")
|
||||
profile = VOICE_PROFILES.get(voice_name, {})
|
||||
age_feel = profile.get("age_feel", "unknown")
|
||||
is_mismatch = (bg.age >= 50 and age_feel == "young") or (bg.age < 25 and age_feel == "mature")
|
||||
entry = {
|
||||
"name": base.get("name", key),
|
||||
"age": bg.age,
|
||||
"voice": voice_name,
|
||||
"voice_age_feel": age_feel,
|
||||
"mismatch": is_mismatch,
|
||||
}
|
||||
alignments.append(entry)
|
||||
if is_mismatch:
|
||||
mismatches.append(entry)
|
||||
|
||||
voice_status = "warn" if mismatches else "pass"
|
||||
checks["voice_age_alignment"] = {
|
||||
"status": voice_status,
|
||||
"details": {
|
||||
"callers": alignments,
|
||||
"mismatches": len(mismatches),
|
||||
},
|
||||
}
|
||||
|
||||
# --- 4. Response coherence (optional) ---
|
||||
if test_responses:
|
||||
# Test ALL callers, not just a sample — we want confidence every caller works
|
||||
test_callers = []
|
||||
for key, base in CALLER_BASES.items():
|
||||
raw_style = session.caller_styles.get(key, "")
|
||||
style_key = _normalize_style_key(raw_style) if raw_style else ""
|
||||
model = session.caller_model_map.get(style_key) if style_key else None
|
||||
if not model:
|
||||
model = session.caller_model_fallback
|
||||
test_callers.append((key, base, model))
|
||||
|
||||
coherence_results = []
|
||||
coherence_fail = False
|
||||
|
||||
# Run all tests in parallel for speed
|
||||
async def _test_caller(key, base, model):
|
||||
slim_caller = session.caller_backgrounds.get(key, {})
|
||||
prompt = get_caller_prompt(slim_caller)
|
||||
# Simulate a real 3-exchange conversation
|
||||
max_tok, _ = _pick_response_budget(session.caller_shapes.get(key, "standard"))
|
||||
messages = [
|
||||
{"role": "user", "content": "Hey welcome to the show, what's going on tonight?"},
|
||||
]
|
||||
try:
|
||||
# First response
|
||||
r1 = await llm_service.generate(
|
||||
messages=messages, system_prompt=prompt,
|
||||
max_tokens=max_tok, category="caller_dialog",
|
||||
caller_name=base.get("name", ""), model_override=model,
|
||||
)
|
||||
r1 = await _retry_if_too_short(
|
||||
r1, llm_service, messages, prompt,
|
||||
max_tok, base.get("name", ""), model_override=model,
|
||||
)
|
||||
r1_words = len(r1.split()) if r1 else 0
|
||||
|
||||
# Second exchange — host follow-up
|
||||
messages.append({"role": "assistant", "content": r1})
|
||||
messages.append({"role": "user", "content": "Wait, tell me more about that."})
|
||||
r2 = await llm_service.generate(
|
||||
messages=messages, system_prompt=prompt,
|
||||
max_tokens=max_tok, category="caller_dialog",
|
||||
caller_name=base.get("name", ""), model_override=model,
|
||||
)
|
||||
r2 = await _retry_if_too_short(
|
||||
r2, llm_service, messages, prompt,
|
||||
max_tok, base.get("name", ""), model_override=model,
|
||||
)
|
||||
r2_words = len(r2.split()) if r2 else 0
|
||||
|
||||
avg_words = (r1_words + r2_words) // 2
|
||||
passed = avg_words >= MIN_RESPONSE_WORDS and r1_words >= 30 and r2_words >= 30
|
||||
return {
|
||||
"name": base.get("name", key),
|
||||
"model": model,
|
||||
"word_count": avg_words,
|
||||
"r1_words": r1_words,
|
||||
"r2_words": r2_words,
|
||||
"pass": passed,
|
||||
"snippet": (r1[:150] + "...") if r1 and len(r1) > 150 else r1,
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"name": base.get("name", key),
|
||||
"model": model,
|
||||
"error": str(e),
|
||||
"pass": False,
|
||||
}
|
||||
|
||||
results = await asyncio.gather(*[_test_caller(k, b, m) for k, b, m in test_callers])
|
||||
for r in results:
|
||||
coherence_results.append(r)
|
||||
if not r.get("pass"):
|
||||
coherence_fail = True
|
||||
|
||||
checks["response_coherence"] = {
|
||||
"status": "fail" if coherence_fail else "pass",
|
||||
"details": {"results": coherence_results},
|
||||
}
|
||||
else:
|
||||
checks["response_coherence"] = {"status": "skip", "details": {"reason": "Use ?test_responses=true to enable"}}
|
||||
|
||||
# --- Overall status ---
|
||||
statuses = [c["status"] for c in checks.values()]
|
||||
if "fail" in statuses:
|
||||
overall = "fail"
|
||||
elif "warn" in statuses:
|
||||
overall = "warn"
|
||||
else:
|
||||
overall = "pass"
|
||||
|
||||
# Summary
|
||||
issues = []
|
||||
if checks["model_diversity"]["status"] == "fail":
|
||||
issues.append(f"Model diversity: {checks['model_diversity']['details']['max_same_model_pct']}% on same model")
|
||||
if checks["theme_penetration"]["status"] == "warn":
|
||||
issues.append(f"Theme penetration: only {checks['theme_penetration']['details']['penetration_pct']}% connected")
|
||||
if checks["voice_age_alignment"]["status"] == "warn":
|
||||
issues.append(f"Voice-age mismatches: {len(mismatches)}")
|
||||
if checks.get("response_coherence", {}).get("status") == "fail":
|
||||
failed = [r["name"] for r in checks["response_coherence"]["details"]["results"] if not r.get("pass")]
|
||||
issues.append(f"Response coherence failed: {', '.join(failed)}")
|
||||
summary = "; ".join(issues) if issues else "All checks passed"
|
||||
|
||||
return {"status": overall, "checks": checks, "summary": summary}
|
||||
|
||||
|
||||
# --- Cost Tracking Endpoints ---
|
||||
|
||||
@app.get("/api/costs")
|
||||
|
||||
Reference in New Issue
Block a user