Tune Inworld TTS: temperature, text normalization, emotional register mapping
Changes to backend/services/tts.py: - Add temperature: 0.9 and applyTextNormalization: "ON" to Inworld payload (text normalization auto-speaks "$5,432", "Dr.", phone numbers, dates, etc.) - Add _emotional_register_to_params() mapping caller's emotional_register to (temperature, speed_adjust) across 5 families: sadness/grief, anger/aggression, manic/excited, nervous/earnest, gruff/restrained. - generate_speech_inworld() and generate_speech() now accept optional emotional_register kwarg; all providers take it via the dispatch lambdas (only inworld uses it; others ignore) - Clamp speed to 0.5-1.5 after both emotional-register and per-text adjustments In backend/main.py: plumb emotional_register through the two caller-dialog TTS calls (auto-respond and ai-respond) by reading from the slim caller background dict. Devon and cohost/announcer calls pass empty string and hit the default (0.9, 0.0) branch — no behavior change.
This commit is contained in:
+8
-2
@@ -4141,9 +4141,12 @@ async def _trigger_ai_auto_respond(accumulated_text: str):
|
||||
broadcast_chat(ai_name, response)
|
||||
|
||||
broadcast_event("ai_status", {"text": f"{ai_name} is speaking..."})
|
||||
_caller_bg = session.caller_backgrounds.get(session.current_caller_key) or {}
|
||||
_emotional_register = _caller_bg.get("emotional_register", "") if isinstance(_caller_bg, dict) else ""
|
||||
try:
|
||||
audio_bytes = await generate_speech(response, session.caller["voice"], "none",
|
||||
provider_override=session.caller.get("tts_provider"))
|
||||
provider_override=session.caller.get("tts_provider"),
|
||||
emotional_register=_emotional_register)
|
||||
except Exception as e:
|
||||
print(f"[Auto-Respond] TTS failed: {e}")
|
||||
broadcast_event("ai_done")
|
||||
@@ -4253,11 +4256,14 @@ async def ai_respond():
|
||||
ai_name = caller["name"]
|
||||
ai_voice = caller["voice"]
|
||||
ai_tts_provider = caller.get("tts_provider")
|
||||
_caller_bg = session.caller_backgrounds.get(session.current_caller_key) or {}
|
||||
ai_emotional_register = _caller_bg.get("emotional_register", "") if isinstance(_caller_bg, dict) else ""
|
||||
|
||||
# TTS — outside the lock so other requests aren't blocked
|
||||
try:
|
||||
audio_bytes = await generate_speech(response, ai_voice, "none",
|
||||
provider_override=ai_tts_provider)
|
||||
provider_override=ai_tts_provider,
|
||||
emotional_register=ai_emotional_register)
|
||||
except Exception as e:
|
||||
print(f"[AI-Respond] TTS failed: {e}")
|
||||
broadcast_event("ai_done")
|
||||
|
||||
+45
-14
@@ -710,7 +710,33 @@ def _detect_speech_rate(text: str, base_speed: float) -> float:
|
||||
return base_speed
|
||||
|
||||
|
||||
async def generate_speech_inworld(text: str, voice_id: str) -> tuple[np.ndarray, int]:
|
||||
def _emotional_register_to_params(emotional_register: str) -> tuple[float, float]:
|
||||
"""Map caller's emotional_register description to (temperature, speed_adjust).
|
||||
speed_adjust is added to the base speed BEFORE clamping to 0.5-1.5.
|
||||
Returns (0.9, 0.0) default for unknown/empty registers."""
|
||||
if not emotional_register:
|
||||
return (0.9, 0.0)
|
||||
er = emotional_register.lower()
|
||||
# Sadness/grief family
|
||||
if any(kw in er for kw in ["fractured", "grief", "hollow", "sad", "mournful", "somber", "quietly desperate"]):
|
||||
return (0.85, -0.1)
|
||||
# Anger/aggression family
|
||||
if any(kw in er for kw in ["angry", "combative", "furious", "aggressive", "sharp intelligence"]):
|
||||
return (1.0, 0.1)
|
||||
# Manic/excited family
|
||||
if any(kw in er for kw in ["caffeinated", "manic", "giddy", "vibrating", "adrenaline", "frantic", "hyper-articulate"]):
|
||||
return (1.0, 0.15)
|
||||
# Nervous/earnest family
|
||||
if any(kw in er for kw in ["earnest", "nervous", "unsettled", "anxious", "tentative", "quietly unsettled"]):
|
||||
return (0.9, 0.0)
|
||||
# Gruff/restrained family
|
||||
if any(kw in er for kw in ["gruff", "measured", "precise", "stoic", "restrained", "deadpan", "forthright"]):
|
||||
return (0.85, -0.05)
|
||||
# Default
|
||||
return (0.9, 0.0)
|
||||
|
||||
|
||||
async def generate_speech_inworld(text: str, voice_id: str, emotional_register: str = "") -> tuple[np.ndarray, int]:
|
||||
"""Generate speech using Inworld TTS API (high quality, natural voices)"""
|
||||
import httpx
|
||||
import base64
|
||||
@@ -727,9 +753,10 @@ async def generate_speech_inworld(text: str, voice_id: str) -> tuple[np.ndarray,
|
||||
if not api_key:
|
||||
raise RuntimeError("INWORLD_API_KEY not set in environment")
|
||||
|
||||
base_speed = INWORLD_SPEED_OVERRIDES.get(voice, DEFAULT_INWORLD_SPEED)
|
||||
speed = _detect_speech_rate(text, base_speed)
|
||||
print(f"[Inworld TTS] Voice: {voice}, Speed: {speed:.2f} (base {base_speed}), Text: {text[:50]}...")
|
||||
temperature, speed_adjust = _emotional_register_to_params(emotional_register)
|
||||
base_speed = INWORLD_SPEED_OVERRIDES.get(voice, DEFAULT_INWORLD_SPEED) + speed_adjust
|
||||
speed = max(0.5, min(1.5, _detect_speech_rate(text, base_speed)))
|
||||
print(f"[Inworld TTS] Voice: {voice}, Speed: {speed:.2f} (base {base_speed:.2f}), Temp: {temperature}, Text: {text[:50]}...")
|
||||
|
||||
url = "https://api.inworld.ai/tts/v1/voice"
|
||||
headers = {
|
||||
@@ -740,6 +767,8 @@ async def generate_speech_inworld(text: str, voice_id: str) -> tuple[np.ndarray,
|
||||
"text": text,
|
||||
"voiceId": voice,
|
||||
"modelId": "inworld-tts-1.5-max",
|
||||
"temperature": temperature,
|
||||
"applyTextNormalization": "ON",
|
||||
"audioConfig": {
|
||||
"audioEncoding": "LINEAR16",
|
||||
"sampleRateHertz": 48000,
|
||||
@@ -797,14 +826,14 @@ def pick_caller_tts_provider() -> str | None:
|
||||
|
||||
|
||||
_TTS_PROVIDERS = {
|
||||
"kokoro": lambda text, vid: generate_speech_kokoro(text, vid),
|
||||
"f5tts": lambda text, vid: generate_speech_f5tts(text, vid),
|
||||
"inworld": lambda text, vid: generate_speech_inworld(text, vid),
|
||||
"chattts": lambda text, vid: generate_speech_chattts(text, vid),
|
||||
"styletts2": lambda text, vid: generate_speech_styletts2(text, vid),
|
||||
"bark": lambda text, vid: generate_speech_bark(text, vid),
|
||||
"vits": lambda text, vid: generate_speech_vits(text, vid),
|
||||
"elevenlabs": lambda text, vid: generate_speech_elevenlabs(text, vid),
|
||||
"kokoro": lambda text, vid, er: generate_speech_kokoro(text, vid),
|
||||
"f5tts": lambda text, vid, er: generate_speech_f5tts(text, vid),
|
||||
"inworld": lambda text, vid, er: generate_speech_inworld(text, vid, emotional_register=er),
|
||||
"chattts": lambda text, vid, er: generate_speech_chattts(text, vid),
|
||||
"styletts2": lambda text, vid, er: generate_speech_styletts2(text, vid),
|
||||
"bark": lambda text, vid, er: generate_speech_bark(text, vid),
|
||||
"vits": lambda text, vid, er: generate_speech_vits(text, vid),
|
||||
"elevenlabs": lambda text, vid, er: generate_speech_elevenlabs(text, vid),
|
||||
}
|
||||
|
||||
TTS_MAX_RETRIES = 2
|
||||
@@ -816,7 +845,8 @@ async def generate_speech(
|
||||
voice_id: str,
|
||||
phone_quality: str = "normal",
|
||||
apply_filter: bool = True,
|
||||
provider_override: str = None
|
||||
provider_override: str = None,
|
||||
emotional_register: str = "",
|
||||
) -> bytes:
|
||||
"""
|
||||
Generate speech from text with automatic retry on failure.
|
||||
@@ -827,6 +857,7 @@ async def generate_speech(
|
||||
phone_quality: Quality of phone filter ("none" to disable)
|
||||
apply_filter: Whether to apply phone filter
|
||||
provider_override: Override the global TTS provider for this call
|
||||
emotional_register: Caller's emotional register (used by Inworld for temp/speed tuning)
|
||||
|
||||
Returns:
|
||||
Raw PCM audio bytes (16-bit signed int, 24kHz)
|
||||
@@ -845,7 +876,7 @@ async def generate_speech(
|
||||
async with asyncio.timeout(20):
|
||||
for attempt in range(TTS_MAX_RETRIES):
|
||||
try:
|
||||
audio, sample_rate = await gen_fn(text, voice_id)
|
||||
audio, sample_rate = await gen_fn(text, voice_id, emotional_register)
|
||||
cost_tracker.record_tts_call(provider, voice_id, len(text))
|
||||
if attempt > 0:
|
||||
print(f"[TTS] Succeeded on retry {attempt}")
|
||||
|
||||
Reference in New Issue
Block a user