From 54ad04ae099fccd5b72ee7c4cd8199820091e908 Mon Sep 17 00:00:00 2001 From: tcpsyn Date: Thu, 9 Apr 2026 23:39:45 -0600 Subject: [PATCH] LLM: per-model params for caller_dialog + longer timeouts Different models need different tuning for natural conversation: Qwen gets high frequency penalty to fight repetition loops, Llama gets warmer temp to reduce terseness, Grok/Mistral/DeepSeek/Kimi get slightly warmer than Sonnet defaults. Bumps base httpx timeout from 10s to 30s and fallback per-call timeout from 8s to 20s. Co-Authored-By: Claude Opus 4.6 (1M context) --- backend/services/llm.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/backend/services/llm.py b/backend/services/llm.py index 772ae26..15f8763 100644 --- a/backend/services/llm.py +++ b/backend/services/llm.py @@ -58,7 +58,7 @@ class LLMService: @property def client(self) -> httpx.AsyncClient: if self._client is None or self._client.is_closed: - self._client = httpx.AsyncClient(timeout=10.0) + self._client = httpx.AsyncClient(timeout=30.0) return self._client def update_settings( @@ -317,7 +317,7 @@ class LLMService: if model == self.openrouter_model: continue # Already tried print(f"[LLM] Falling back to {model}...") - result = await self._call_openrouter_once(messages, model, timeout=8.0, max_tokens=max_tokens, category=category, caller_name=caller_name) + result = await self._call_openrouter_once(messages, model, timeout=20.0, max_tokens=max_tokens, category=category, caller_name=caller_name) if result is not None: return result @@ -325,18 +325,39 @@ class LLMService: print("[LLM] All models failed, using canned response") return "Sorry, I totally blanked out for a second. What were you saying?" - async def _call_openrouter_once(self, messages: list[dict], model: str, timeout: float = 10.0, max_tokens: Optional[int] = None, response_format: Optional[dict] = None, category: str = "unknown", caller_name: str = "") -> str | None: + # Per-model parameter overrides for caller_dialog category. + # Different models need different tuning for natural conversation: + # - Qwen: high freq penalty to fight phrase-level repetition loops + # - Llama: high temp + low freq penalty to reduce terseness + # - Grok/Mistral/DeepSeek/Kimi: slightly warmer than Sonnet defaults + _CALLER_DIALOG_MODEL_PARAMS = { + "anthropic/claude-sonnet-4.6": {"temperature": 0.65, "frequency_penalty": 0.3, "presence_penalty": 0.15}, + "x-ai/grok-4.1-fast": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1}, + "x-ai/grok-4": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1}, + "qwen/qwen3-235b-a22b": {"temperature": 0.6, "frequency_penalty": 0.5, "presence_penalty": 0.2}, + "mistralai/mistral-large-2512": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1}, + "deepseek/deepseek-chat-v3-0324": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1}, + "moonshotai/kimi-k2": {"temperature": 0.7, "frequency_penalty": 0.2, "presence_penalty": 0.1}, + "meta-llama/llama-3.3-70b-instruct": {"temperature": 0.8, "frequency_penalty": 0.1, "presence_penalty": 0.1}, + } + + async def _call_openrouter_once(self, messages: list[dict], model: str, timeout: float = 20.0, max_tokens: Optional[int] = None, response_format: Optional[dict] = None, category: str = "unknown", caller_name: str = "") -> str | None: """Single attempt to call OpenRouter. Returns None on failure (not a fallback string).""" start_time = time.time() try: + # Use per-model params for caller dialog, defaults for everything else + if category == "caller_dialog" and model in self._CALLER_DIALOG_MODEL_PARAMS: + params = self._CALLER_DIALOG_MODEL_PARAMS[model] + else: + params = {"temperature": 0.65, "frequency_penalty": 0.3, "presence_penalty": 0.15} payload = { "model": model, "messages": messages, "max_tokens": max_tokens or 500, - "temperature": 0.65, + "temperature": params["temperature"], "top_p": 0.9, - "frequency_penalty": 0.3, - "presence_penalty": 0.15, + "frequency_penalty": params["frequency_penalty"], + "presence_penalty": params["presence_penalty"], } if response_format: payload["response_format"] = response_format