LLM: per-model params for caller_dialog + longer timeouts

Different models need different tuning for natural conversation:
Qwen gets high frequency penalty to fight repetition loops, Llama
gets warmer temp to reduce terseness, Grok/Mistral/DeepSeek/Kimi get
slightly warmer than Sonnet defaults. Bumps base httpx timeout from
10s to 30s and fallback per-call timeout from 8s to 20s.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-09 23:39:45 -06:00
co-authored by Claude Opus 4.6
parent c087c0362a
commit 54ad04ae09
+27 -6
View File
@@ -58,7 +58,7 @@ class LLMService:
@property @property
def client(self) -> httpx.AsyncClient: def client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed: if self._client is None or self._client.is_closed:
self._client = httpx.AsyncClient(timeout=10.0) self._client = httpx.AsyncClient(timeout=30.0)
return self._client return self._client
def update_settings( def update_settings(
@@ -317,7 +317,7 @@ class LLMService:
if model == self.openrouter_model: if model == self.openrouter_model:
continue # Already tried continue # Already tried
print(f"[LLM] Falling back to {model}...") print(f"[LLM] Falling back to {model}...")
result = await self._call_openrouter_once(messages, model, timeout=8.0, max_tokens=max_tokens, category=category, caller_name=caller_name) result = await self._call_openrouter_once(messages, model, timeout=20.0, max_tokens=max_tokens, category=category, caller_name=caller_name)
if result is not None: if result is not None:
return result return result
@@ -325,18 +325,39 @@ class LLMService:
print("[LLM] All models failed, using canned response") print("[LLM] All models failed, using canned response")
return "Sorry, I totally blanked out for a second. What were you saying?" return "Sorry, I totally blanked out for a second. What were you saying?"
async def _call_openrouter_once(self, messages: list[dict], model: str, timeout: float = 10.0, max_tokens: Optional[int] = None, response_format: Optional[dict] = None, category: str = "unknown", caller_name: str = "") -> str | None: # Per-model parameter overrides for caller_dialog category.
# Different models need different tuning for natural conversation:
# - Qwen: high freq penalty to fight phrase-level repetition loops
# - Llama: high temp + low freq penalty to reduce terseness
# - Grok/Mistral/DeepSeek/Kimi: slightly warmer than Sonnet defaults
_CALLER_DIALOG_MODEL_PARAMS = {
"anthropic/claude-sonnet-4.6": {"temperature": 0.65, "frequency_penalty": 0.3, "presence_penalty": 0.15},
"x-ai/grok-4.1-fast": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1},
"x-ai/grok-4": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1},
"qwen/qwen3-235b-a22b": {"temperature": 0.6, "frequency_penalty": 0.5, "presence_penalty": 0.2},
"mistralai/mistral-large-2512": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1},
"deepseek/deepseek-chat-v3-0324": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1},
"moonshotai/kimi-k2": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1},
"meta-llama/llama-3.3-70b-instruct": {"temperature": 0.8, "frequency_penalty": 0.1, "presence_penalty": 0.1},
}
async def _call_openrouter_once(self, messages: list[dict], model: str, timeout: float = 20.0, max_tokens: Optional[int] = None, response_format: Optional[dict] = None, category: str = "unknown", caller_name: str = "") -> str | None:
"""Single attempt to call OpenRouter. Returns None on failure (not a fallback string).""" """Single attempt to call OpenRouter. Returns None on failure (not a fallback string)."""
start_time = time.time() start_time = time.time()
try: try:
# Use per-model params for caller dialog, defaults for everything else
if category == "caller_dialog" and model in self._CALLER_DIALOG_MODEL_PARAMS:
params = self._CALLER_DIALOG_MODEL_PARAMS[model]
else:
params = {"temperature": 0.65, "frequency_penalty": 0.3, "presence_penalty": 0.15}
payload = { payload = {
"model": model, "model": model,
"messages": messages, "messages": messages,
"max_tokens": max_tokens or 500, "max_tokens": max_tokens or 500,
"temperature": 0.65, "temperature": params["temperature"],
"top_p": 0.9, "top_p": 0.9,
"frequency_penalty": 0.3, "frequency_penalty": params["frequency_penalty"],
"presence_penalty": 0.15, "presence_penalty": params["presence_penalty"],
} }
if response_format: if response_format:
payload["response_format"] = response_format payload["response_format"] = response_format