diff --git a/agent/tts_provider.py b/agent/tts_provider.py index c38469a3d2..873947e96d 100644 --- a/agent/tts_provider.py +++ b/agent/tts_provider.py @@ -34,7 +34,10 @@ class TTSProvider(CatalogProviderBase): and make ``stream(text, format="pcm", ...)`` yield raw int16 little-endian mono PCM at that rate. Both are read (with :meth:`is_available`) each time a reply resolves its streamer, so they may be properties reflecting live state; a - missing rate falls back to per-sentence :meth:`synthesize`. + missing rate falls back to per-sentence :meth:`synthesize`. Unlike + :meth:`synthesize`, ``stream()`` may run concurrently for consecutive + sentences of one reply (the speaker prefetches up to 3), so it must be + thread-safe. """ streams_pcm: bool = False diff --git a/tests/tools/test_tts_plugin_dispatch.py b/tests/tools/test_tts_plugin_dispatch.py index 519825f875..c8936c5d01 100644 --- a/tests/tools/test_tts_plugin_dispatch.py +++ b/tests/tools/test_tts_plugin_dispatch.py @@ -268,6 +268,8 @@ def test_opted_in_plugin_streams_pcm_in_the_configured_voice(monkeypatch, stream (_PCMPlugin(streams_pcm=False), {}), (_PCMPlugin(rate=None), {}), (_PCMPlugin(rate=0), {}), + (_PCMPlugin(rate=0.5), {}), + (_PCMPlugin(rate=float("nan")), {}), (_PCMPlugin(available=False), {}), (_PCMPlugin(), {"providers": {"fake-pcm": {"type": "command", "command": "say {input_path}"}}}), ]) diff --git a/tools/tts_streaming.py b/tools/tts_streaming.py index cd72c31ecc..9facab3039 100644 --- a/tools/tts_streaming.py +++ b/tools/tts_streaming.py @@ -175,10 +175,9 @@ class _PluginPCMStreamer(StreamingTTSProvider): """A plugin ``TTSProvider`` that opted into raw PCM (``streams_pcm``; see ``tools.tts_tool_plugins._plugin_pcm_streaming_provider``) behind this ABC.""" - def __init__(self, provider: Any, tts_config: Dict): + def __init__(self, provider: Any, sample_rate: int, tts_config: Dict): super().__init__(tts_config, tts_config.get(provider.name) or {}) - self._provider = provider - self.sample_rate = int(provider.stream_sample_rate) + self._provider, self.sample_rate = provider, sample_rate @staticmethod def available() -> bool: @@ -191,8 +190,8 @@ class _PluginPCMStreamer(StreamingTTSProvider): def _plugin_streamer(name: str, tts_config: Dict) -> Optional[StreamingTTSProvider]: - provider = _plugin_pcm_streaming_provider(name, tts_config) - return _PluginPCMStreamer(provider, tts_config) if provider is not None else None + found = _plugin_pcm_streaming_provider(name, tts_config) + return _PluginPCMStreamer(*found, tts_config) if found is not None else None def resolve_streaming_provider( diff --git a/tools/tts_tool_plugins.py b/tools/tts_tool_plugins.py index 2d2624dbcf..10c7e14c45 100644 --- a/tools/tts_tool_plugins.py +++ b/tools/tts_tool_plugins.py @@ -7,11 +7,13 @@ tool module stays importable without the plugin machinery). from __future__ import annotations import logging -from typing import Any, Dict, Optional +import math +from typing import Any, Dict, Optional, Tuple from tools.tts_command_provider import ( BUILTIN_TTS_PROVIDERS, DEFAULT_COMMAND_TTS_OUTPUT_FORMAT, _get_named_provider_config, _is_command_provider_config) +from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER logger = logging.getLogger("tools.tts_tool") @@ -35,7 +37,7 @@ def _plugin_route_key(provider: str, tts_config: Dict[str, Any]) -> Optional[str """Normalized *provider* when a plugin may service it, else None: built-in names never reach the registry and a same-named ``type: command`` provider wins (config is more local than a plugin).""" key = (provider or "").lower().strip() - if not key or key in BUILTIN_TTS_PROVIDERS: + if not key or key in BUILTIN_TTS_PROVIDERS or key == NOUS_MANAGED_PROVIDER: # nous = the openai path return None if _is_command_provider_config(_get_named_provider_config(tts_config, key)): return None @@ -52,12 +54,12 @@ def _plugin_voice_kwargs(tts_config: Dict[str, Any]) -> Dict[str, Any]: "speed": float(speed) if isinstance(speed, (int, float)) else None} -def _plugin_pcm_streaming_provider(provider: str, tts_config: Dict[str, Any]): - """The plugin that services *provider* when it opted into raw-PCM streaming, else None. +def _plugin_pcm_streaming_provider(provider: str, tts_config: Dict[str, Any]) -> Optional[Tuple[Any, int]]: + """``(plugin, sample_rate)`` when the plugin servicing *provider* opted into raw-PCM streaming. - Opt-in = ``streams_pcm`` truthy AND a positive ``stream_sample_rate`` AND ``is_available()`` — - all read at resolve time, so per-profile state (key present, user opted out) belongs there. - A missing rate is refused loudly: PCM played at a guessed rate garbles speech. + Opt-in = ``streams_pcm`` truthy AND ``stream_sample_rate`` a finite number >= 1 Hz AND + ``is_available()`` — read once per resolve, so per-profile state (key present, user opted out) + belongs there. A bad rate is refused loudly: PCM played at a guessed rate garbles speech. """ key = _plugin_route_key(provider, tts_config) if key is None: @@ -67,14 +69,14 @@ def _plugin_pcm_streaming_provider(provider: str, tts_config: Dict[str, Any]): if plugin is None or not plugin.streams_pcm or not plugin.is_available(): return None rate = plugin.stream_sample_rate - except Exception as exc: # noqa: BLE001 — a broken plugin falls back to per-sentence synthesis - logger.debug("plugin TTS streaming lookup failed for '%s': %s", key, exc) + except Exception: # third-party plugin code: a broken plugin falls back to per-sentence synthesis + logger.debug("plugin TTS streaming lookup failed for '%s'", key, exc_info=True) return None - if isinstance(rate, bool) or not isinstance(rate, (int, float)) or rate <= 0: - logger.warning("TTS provider '%s' sets streams_pcm without a positive stream_sample_rate; " - "using per-sentence synthesis instead", key) + if isinstance(rate, bool) or not isinstance(rate, (int, float)) or not math.isfinite(rate) or rate < 1: + logger.warning("TTS provider '%s' sets streams_pcm without a valid stream_sample_rate (%r); " + "using per-sentence synthesis instead", key, rate) return None - return plugin + return plugin, int(rate) def _dispatch_to_plugin_provider(text: str, output_path: str, provider: str, tts_config: Dict[str, Any]) -> Optional[str]: diff --git a/website/docs/developer-guide/streaming-tts.md b/website/docs/developer-guide/streaming-tts.md index 497011631c..dd8280b94a 100644 --- a/website/docs/developer-guide/streaming-tts.md +++ b/website/docs/developer-guide/streaming-tts.md @@ -98,6 +98,9 @@ plugin streams first. Under `tts.streaming.provider: auto` it is tried only afte the built-in priority list. `streams_pcm`, `stream_sample_rate` and `is_available()` are read every time a streamer is resolved. If the rate is missing or the provider reports unavailable, Hermes keeps per-sentence synthesis. +As with the built-in streamers, the speaker pipeline prefetches up to three +sentences, so a plugin's `stream()` must tolerate concurrent calls (the sync +path serializes `synthesize()`; this path does not). ## Gateway streaming (platform adapters) diff --git a/website/docs/user-guide/features/tts.md b/website/docs/user-guide/features/tts.md index 167989fe8c..9536836ea9 100644 --- a/website/docs/user-guide/features/tts.md +++ b/website/docs/user-guide/features/tts.md @@ -469,7 +469,7 @@ Override these on your provider class for richer integration: - `list_models()` → list of `{id, display, languages, max_text_length}` dicts. - `get_setup_schema()` → return `{name, badge, tag, env_vars: [{key, prompt, url}]}` to power the picker row in `hermes tools` / `hermes setup`. Without this, the plugin still works but its row in the picker is minimal. - `stream(text, *, voice, model, format, **extra)` → iterator yielding audio bytes (default raises `NotImplementedError`). -- `streams_pcm = True` + `stream_sample_rate` (Hz) → join the streaming voice path (CLI/TUI voice mode, desktop read-aloud, gateway streaming audio). Hermes then calls `stream(text, format="pcm", voice=..., model=..., speed=...)` with the same `tts.voice` / `tts.model` / `tts.speed` that `synthesize()` gets, and expects raw int16 little-endian mono PCM at that rate. Both attributes and `is_available()` are read each time a reply starts, so they can be properties that reflect live state. Without a positive `stream_sample_rate`, or when `is_available()` is `False`, Hermes keeps synthesizing one sentence at a time. +- `streams_pcm = True` + `stream_sample_rate` (Hz) → join the streaming voice path (CLI/TUI voice mode, desktop read-aloud, gateway streaming audio). Hermes then calls `stream(text, format="pcm", voice=..., model=..., speed=...)` with the same `tts.voice` / `tts.model` / `tts.speed` that `synthesize()` gets, and expects raw int16 little-endian mono PCM at that rate. Both attributes and `is_available()` are read each time a reply starts, so they can be properties that reflect live state. Unlike `synthesize()`, `stream()` can be called for up to three consecutive sentences at once while earlier audio plays, so it must be thread-safe. Without a positive `stream_sample_rate`, or when `is_available()` is `False`, Hermes keeps synthesizing one sentence at a time. - `voice_compatible` property → set `True` if your output is Opus-compatible and the gateway should deliver it as a voice bubble (default `False` = regular audio attachment). - `warm()` / `release()` → called when a surface toggles speech output on / when the last lease across surfaces is released, while your provider is the configured `tts.provider` — preload or unload a local model server here. Both default to no-ops; exceptions are logged at debug and never fail the toggle.