Files
hermes-agent/hermes_cli/cli_voice_mixin.py
Teknium 00adf69d92 feat(stt): live transcription while you speak on CLI, TUI and Desktop (stt.streaming)
Voice input was record-then-upload on every surface: nothing showed while the user
talked, and the transcript arrived only after the whole take uploaded. With
`stt.streaming: true` the openai, xai and elevenlabs providers now stream microphone
PCM to their realtime endpoints while the user speaks. Partial text renders live and
the final transcript is ready at end of speech.

- tools/transcription_streaming.py: one session API (push_audio / end_audio /
  finalize / cancel, an on_partial callback, a worker thread that owns the socket)
  with three wires, all verified live: the OpenAI Realtime `type: transcription`
  session (gpt-live-transcribe, 16->24 kHz resample since the endpoint refuses
  < 24 kHz), xAI wss /v1/stt with interim results, and ElevenLabs Scribe v2
  realtime. Plugin providers opt in through
  TranscriptionProvider.streaming_capable / open_stream_session.
- tools/voice_mode.py: AudioRecorder taps its capture callback into the session and
  parks it under the WAV path. transcribe_recording uses the live transcript and
  falls back to the file on any failure or empty result, so CLI push-to-talk, the
  continuous VAD loop and the TUI gateway gain streaming with no call-site changes.
- Classic CLI and Ink TUI show partial text in the input placeholder while recording
  (new `voice.partial` gateway event, contract regenerated). Desktop dictation
  streams PCM to the new /api/audio/transcribe-stream socket (provider keys stay on
  the host, and the host resamples) and shows partial text in the dictation pill.
  The recorded blob stays the fallback.

The plugin streaming contract (streaming_capable / open_stream_session /
TranscriptionStreamSession) and the transcribe-stream socket shape come from #103173.

Co-authored-by: Vinicius <timelord1025@gmail.com>
2026-10-06 12:50:31 -07:00

844 lines
40 KiB
Python

"""Voice mode (recording, STT, TTS, full-duplex barge-in) and wake-word listener handlers for the interactive CLI
Mixin split out of ``cli.py``; bound onto ``HermesCLI`` via the MRO. cli.py-internal
symbols are imported LAZILY inside each method (``from cli import ...``) — the mixin
never imports ``cli`` at module load time (import cycle).
"""
from __future__ import annotations
import json
import os
import re
import tempfile
import threading
import time
from agent.i18n import t
from hermes_constants import is_termux as _is_termux_environment
from typing import Optional
def _config_section(name: str) -> dict:
"""``load_config()[name]`` coerced to a dict.
Shape-safe: a hand-edited ``voice: true`` / ``voice: cmd+b`` leaves the section as a
non-dict; return {} so callers fall back to documented defaults instead of crashing on
``.get()``. Config load failures also yield {}.
"""
try:
from hermes_cli.config import load_config
section = load_config().get(name)
return section if isinstance(section, dict) else {}
except Exception:
return {}
def _numeric_or(value, default):
"""``value`` if it is a real int/float, else ``default``.
``bool`` is excluded — it subclasses int, so a hand-edited ``silence_threshold: true``
would otherwise be forwarded as ``1`` instead of falling back.
"""
return value if isinstance(value, (int, float)) and not isinstance(value, bool) else default
def _unlink_quietly(path) -> None:
try:
if path and os.path.isfile(path):
os.unlink(path)
except OSError:
pass
class CLIVoiceMixin:
"""Voice mode (recording, STT, TTS, full-duplex barge-in) and wake-word listener handlers for the interactive CLI"""
def _voice_invalidate(self) -> None:
"""Repaint the TUI (audio level indicator / status) when the app is live."""
if hasattr(self, '_app') and self._app:
self._app.invalidate()
def _voice_start_recording(self):
"""Start capturing audio from the microphone."""
from cli import _ACCENT, _DIM, _RST, _cprint
if getattr(self, '_should_exit', False):
return
from tools.voice_mode import create_audio_recorder, check_voice_requirements
reqs = check_voice_requirements()
if not reqs["audio_available"]:
if _is_termux_environment():
# These RuntimeErrors are human copy: callers print ``{e}`` straight to the terminal.
if "Termux:API Android app is not installed" in reqs.get("details", ""):
raise RuntimeError(t("cli.voice.err_termux_app_missing"))
raise RuntimeError(t("cli.voice.err_mic_access"))
# check_voice_requirements already asked PM to enable audio-io; its detail line
# says why that did not happen (lazy installs off, needs restart, platform gate).
raise RuntimeError(t("cli.voice.err_audio_capture") + "\n" + reqs.get("details", ""))
if not reqs.get("stt_available", reqs.get("stt_key_set")):
raise RuntimeError(t("cli.voice.err_stt_provider"))
# Prevent double-start from concurrent threads (atomic check-and-set)
with self._voice_lock:
if self._voice_recording:
return
self._voice_recording = True
voice_cfg = _config_section("voice")
# Recorder creation can fail (no input device, PortAudio init). Reset the flag on
# failure or every future voice start is silently skipped by the guard above.
if self._voice_recorder is None:
try:
self._voice_recorder = create_audio_recorder()
except Exception:
with self._voice_lock:
self._voice_recording = False
raise
# Config-driven silence params, numeric-guarded against YAML scalar corruption.
rec = self._voice_recorder
self._voice_live_text = ""
rec.on_live_partial = self._voice_on_live_partial
rec._silence_threshold = _numeric_or(voice_cfg.get("silence_threshold"), 200)
rec._silence_duration = _numeric_or(voice_cfg.get("silence_duration"), 3.0)
# voice.max_recording_seconds — hard cap on one recording; explicit <= 0 disables it.
_max_rec = _numeric_or(voice_cfg.get("max_recording_seconds"), None)
rec._max_recording_seconds = (_max_rec if _max_rec > 0 else 0.0) if _max_rec is not None else 120.0
def _on_silence():
"""Called by AudioRecorder when silence is detected after speech."""
with self._voice_lock:
if not self._voice_recording:
return
_cprint(f"\n{_DIM}{t('cli.voice.silence_autostop')}{_RST}")
self._voice_invalidate()
self._voice_stop_and_transcribe()
# Audio cue: single beep BEFORE starting stream (avoid CoreAudio conflict)
self._voice_beep(frequency=880, count=1)
try:
self._voice_recorder.start(on_silence_stop=_on_silence)
except Exception:
with self._voice_lock:
self._voice_recording = False
raise
_label = self._voice_record_key_label()
if getattr(self._voice_recorder, "supports_silence_autostop", True):
_recording_hint = t("cli.voice.hint_continuous", shortcut=_label)
elif _is_termux_environment():
_recording_hint = t("cli.voice.hint_termux", shortcut=_label)
else:
_recording_hint = t("cli.voice.hint_stop", shortcut=_label)
_cprint(f"\n{_ACCENT}{t('cli.voice.recording')}{_RST} {_DIM}({_recording_hint}){_RST}")
# Periodically refresh prompt to update audio level indicator
def _refresh_level():
while True:
with self._voice_lock:
still_recording = self._voice_recording
if not still_recording:
break
self._voice_invalidate()
time.sleep(0.15)
threading.Thread(target=_refresh_level, daemon=True).start()
def _voice_beep(self, *, frequency: int, count: int) -> None:
"""Play a record start/stop beep when enabled; never raises."""
if self._voice_beeps_enabled():
try:
from tools.voice_mode import play_beep
play_beep(frequency=frequency, count=count)
except Exception:
pass
def _voice_stt_model(self) -> Optional[str]:
"""STT model override from config, or None for the provider default.
For the local provider, prefer stt.local.model (default ``base``) so the CLI passes
a real model name into the local STT backend.
"""
stt_config = _config_section("stt")
if str(stt_config.get("provider") or "").strip().lower() == "local":
local_config = stt_config.get("local") or {}
if not isinstance(local_config, dict):
local_config = {}
return local_config.get("model") or "base"
return stt_config.get("model")
def _voice_stt_provider(self) -> str:
"""Configured STT provider name (lowercased), or empty string."""
return str(_config_section("stt").get("provider") or "").strip().lower()
def _voice_restart_recording_async(self) -> None:
"""Restart continuous-mode recording off-thread (start() can block)."""
from cli import _DIM, _RST, _cprint
def _restart_recording():
try:
self._voice_start_recording()
self._voice_invalidate()
except Exception as e:
_cprint(f"{_DIM}{t('cli.voice.autorestart_failed', error=e)}{_RST}")
threading.Thread(target=_restart_recording, daemon=True).start()
def _voice_on_live_partial(self, text: str) -> None:
"""Live STT partial (``stt.streaming``): shown in the input placeholder while recording."""
self._voice_live_text = text
self._voice_invalidate()
def _voice_stop_and_transcribe(self):
"""Stop recording, transcribe via STT, and queue the transcript as input."""
from cli import _DIM, _RST, _VoiceInputMessage, _cprint
# Atomic guard; _voice_processing is set immediately so concurrent Ctrl+B presses
# don't race into the START path while recorder.stop() holds its lock.
with self._voice_lock:
if not self._voice_recording:
return
self._voice_recording = False
self._voice_processing = True
submitted = False
transcription_failed = False
wav_path = None
try:
if self._voice_recorder is None:
return
wav_path = self._voice_recorder.stop()
# Audio cue: double beep after stream stopped (no CoreAudio conflict)
self._voice_beep(frequency=660, count=2)
if wav_path is None:
_cprint(f"{_DIM}{t('cli.voice.no_speech')}{_RST}")
return
self._voice_invalidate()
stt_model = self._voice_stt_model()
if self._voice_stt_provider() == "local":
_cprint(f"{_DIM}{t('cli.voice.preparing_stt', model=stt_model)}{_RST}")
else:
_cprint(f"{_DIM}{t('cli.voice.transcribing')}{_RST}")
from tools.voice_mode_transcript import is_voice_stop_phrase
from tools.voice_mode import transcribe_recording
result = transcribe_recording(wav_path, model=stt_model)
if result.get("success") and result.get("transcript", "").strip():
transcript = result["transcript"].strip()
if is_voice_stop_phrase(transcript):
# Bare "stop" (or configured phrase) ends the voice chat, not a turn.
_cprint(f"{_DIM}{t('cli.voice.stop_phrase')}{_RST}")
self._disable_voice_mode()
return
self._attached_images.clear()
self._voice_invalidate()
self._pending_input.put(_VoiceInputMessage(transcript))
submitted = True
elif result.get("success"):
_cprint(f"{_DIM}{t('cli.voice.no_speech')}{_RST}")
else:
_cprint(f"\n{_DIM}{t('cli.voice.transcription_failed', error=result.get('error') or t('cli.shared.unknown_error'))}{_RST}")
transcription_failed = True
except Exception as e:
_cprint(f"\n{_DIM}{t('cli.voice.processing_error', error=e)}{_RST}")
transcription_failed = wav_path is not None
finally:
with self._voice_lock:
self._voice_processing = False
self._voice_live_text = ""
self._voice_invalidate()
# On failure keep the source recording so long dictation is not lost.
try:
if wav_path and os.path.isfile(wav_path):
if transcription_failed:
_cprint(f"{_DIM}{t('cli.voice.recording_preserved', path=wav_path)}{_RST}")
else:
os.unlink(wav_path)
except Exception:
pass
# Three consecutive no-speech cycles end continuous mode (no infinite restart
# loop). While the agent is mid-turn or TTS is speaking the user is CORRECTLY
# silent — those cycles must not count, or a multi-minute tool run ends the voice
# chat under the user (stop phrase and barge-in still work during the hold).
stop_continuous_restart = False
_tts_done = getattr(self, "_voice_tts_done", None)
_activity_hold = bool(
getattr(self, "_agent_running", False)
or (_tts_done is not None and not _tts_done.is_set()))
if submitted:
self._no_speech_count = 0
elif not _activity_hold:
self._no_speech_count = getattr(self, '_no_speech_count', 0) + 1
if self._no_speech_count >= 3:
self._voice_continuous = False
self._no_speech_count = 0
_cprint(f"{_DIM}{t('cli.voice.no_speech_stop')}{_RST}")
stop_continuous_restart = True
# No transcript but continuous mode active: restart so the user can keep talking
# (when a transcript IS submitted, process_loop restarts after chat()).
if (
self._voice_continuous
and not submitted
and not self._voice_recording
and not stop_continuous_restart):
self._voice_restart_recording_async()
def _voice_speak_response_async(self, text: str) -> None:
"""Schedule TTS and mark it pending before continuous recording can restart."""
if not self._voice_tts or not text:
return
self._voice_tts_done.clear()
threading.Thread(target=self._voice_speak_response, args=(text,), daemon=True).start()
# Barge-in safety net for speak calls outside a chat turn (the agent-turn listener
# armed in chat() normally covers playback); idempotent via _voice_fd_active.
if self._voice_continuous:
threading.Thread(target=self._voice_full_duplex_listener, daemon=True).start()
def _voice_speak_response(self, text: str):
"""Speak the agent's response aloud using TTS (runs in background thread)."""
from cli import _DIM, _RST, _cprint, logger
if not self._voice_tts:
return
self._voice_tts_done.clear()
try:
from tools.tts_tool import text_to_speech_tool
from tools.voice_mode import play_audio_file
# Shared cleaner strips markdown/emoji/⋗ blocks/verifier footer; the TTS tool owns
# provider request limits and long-form chunking.
try:
from tools.tts_text_normalize import prepare_spoken_text
tts_text = prepare_spoken_text(text, max_chars=None)
except Exception:
# Legacy fallback pipeline — keep voice replies best-effort.
tts_text = re.sub(r'```[\s\S]*?```', ' ', text) # fenced code blocks
tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text) # [text](url) -> text
tts_text = re.sub(r'https?://\S+', '', tts_text) # URLs
tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text) # bold
tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text) # italic
tts_text = re.sub(r'`(.+?)`', r'\1', tts_text) # inline code
tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE) # headers
tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE) # list items
tts_text = re.sub(r'---+', '', tts_text) # horizontal rules
tts_text = re.sub(r'\n{3,}', '\n\n', tts_text) # excessive newlines
tts_text = tts_text.strip()
if not tts_text:
return
self._voice_last_tts_text = tts_text
# MP3 for CLI playback (afplay doesn't handle OGG well); the TTS tool may
# auto-convert MP3->OGG but the original MP3 remains.
out_dir = os.path.join(tempfile.gettempdir(), "hermes_voice")
os.makedirs(out_dir, exist_ok=True)
mp3_path = os.path.join(out_dir, f"tts_{time.strftime('%Y%m%d_%H%M%S')}.mp3")
raw_result = text_to_speech_tool(text=tts_text, output_path=mp3_path)
try:
tts_result = json.loads(raw_result) if isinstance(raw_result, str) else {}
except Exception:
tts_result = {}
# The tool result is authoritative — chunked long-form output returns several files.
play_paths = tts_result.get("file_paths") or [tts_result.get("file_path") or mp3_path]
for play_path in play_paths if tts_result.get("success") else []:
if os.path.isfile(play_path) and os.path.getsize(play_path) > 0:
play_audio_file(play_path)
# Clean up all generated files (play_paths + mp3_path + ogg variant)
for path in set(play_paths + [mp3_path, mp3_path.rsplit(".", 1)[0] + ".ogg"]):
_unlink_quietly(path)
except Exception as e:
logger.warning("Voice TTS playback failed: %s", e)
_cprint(f"{_DIM}{t('cli.voice.tts_failed', error=e)}{_RST}")
finally:
self._voice_tts_done.set()
def _voice_full_duplex_listener(self) -> None:
"""Full-duplex agent-turn listener: mic live for the WHOLE turn.
Armed at utterance-submit (chat() start in continuous voice mode), disarmed when agent
finished + TTS played, so the user can interject during generation too. Generation
phase: speech interrupts the turn via ``self.agent.interrupt()`` (same seam as
Ctrl+C); playback phase: speech cuts TTS. Either way the captured utterance is
submitted as the next message; the stop phrase ends the voice chat in BOTH phases.
"""
from cli import _DIM, _RST, _cprint, logger
fd_active = getattr(self, "_voice_fd_active", None)
if fd_active is None:
fd_active = threading.Event()
self._voice_fd_active = fd_active
if fd_active.is_set():
return # one listener owns the mic for this turn
fd_active.set()
try:
from hermes_cli.config import load_config
voice_cfg = load_config().get("voice") or {}
if not (isinstance(voice_cfg, dict) and voice_cfg.get("barge_in", True)):
return
from tools.voice_mode import full_duplex_listen, is_audio_output_active, stop_playback
try:
_mult = float(voice_cfg.get("barge_in_threshold_multiplier", 0) or 0)
except (TypeError, ValueError):
_mult = 0.0
try:
_grace_ms = int(float(voice_cfg.get("barge_in_grace_seconds", 0.5)) * 1000)
except (TypeError, ValueError):
_grace_ms = 500
tts_done = getattr(self, "_voice_tts_done", None)
def _should_stop() -> bool:
if not (getattr(self, "_voice_mode", False) and getattr(self, "_voice_continuous", False)):
return True
if getattr(self, "_agent_running", False):
return False
# Agent finished — keep listening until TTS fully played.
if tts_done is not None and not tts_done.is_set():
return False
return not is_audio_output_active()
def _on_trigger(phase: str) -> None:
# Latch BEFORE cutting anything: suppresses process_loop's auto-restart until
# the capture is submitted.
self._voice_barge_capture.set()
self._voice_barge_phase = phase
_pipe_stop = getattr(self, "_voice_tts_stop", None)
if phase == "playback":
logger.debug("TTS CUT: full-duplex listener tripped during playback")
from tools.tts_streaming import mark_speech_interrupted
mark_speech_interrupted()
if _pipe_stop is not None:
_pipe_stop.set()
stop_playback()
else:
# Generation phase: no audio to cut — interrupt the in-flight agent turn.
logger.debug(
"full-duplex listener tripped during generation — "
"interrupting agent turn")
if _pipe_stop is not None:
_pipe_stop.set() # never let the stale reply speak
try:
if self.agent is not None and getattr(self, "_agent_running", False):
_cprint(f"\n{_DIM}{t('cli.voice.interjection')}{_RST}")
self.agent.interrupt()
except Exception as e:
logger.debug("voice interjection interrupt failed: %s", e)
wav_path = full_duplex_listen(
_should_stop, is_playing=is_audio_output_active, on_trigger=_on_trigger,
multiplier=_mult or None, grace_ms=max(0, _grace_ms))
if wav_path and self._voice_barge_capture.is_set():
self._voice_submit_barge_utterance(wav_path)
else:
self._voice_barge_capture.clear()
except Exception as e:
self._voice_barge_capture.clear()
logger.debug("Voice full-duplex listener failed: %s", e)
finally:
fd_active.clear()
def _voice_submit_barge_utterance(self, wav_path: str) -> None:
"""Transcribe a barge-captured interruption and queue it as the next turn."""
from cli import _DIM, _RST, _VoiceInputMessage, _cprint, logger
submitted = False
try:
from tools.voice_mode import transcribe_recording
result = transcribe_recording(wav_path, model=self._voice_stt_model())
transcript = (result.get("transcript") or "").strip() if result.get("success") else ""
if transcript:
from tools.voice_mode_transcript import is_voice_stop_phrase
if is_voice_stop_phrase(transcript):
_cprint(f"\n{_DIM}{t('cli.voice.stop_phrase')}{_RST}")
self._disable_voice_mode()
return
# Fail-closed echo guard: playback-phase capture has no echo cancellation, so
# a close match for what Hermes just spoke is speaker bleed, not a user turn.
if getattr(self, "_voice_barge_phase", None) == "playback":
from tools.voice_mode_transcript import is_tts_echo
if is_tts_echo(transcript, getattr(self, "_voice_last_tts_text", "")):
logger.debug(
"Dropping playback-phase barge transcript as TTS echo: %r", transcript)
_cprint(f"\n{_DIM}{t('cli.voice.echo_ignored')}{_RST}")
return
self._pending_input.put(_VoiceInputMessage(transcript))
submitted = True
elif not result.get("success"):
_cprint(f"\n{_DIM}{t('cli.voice.transcription_failed', error=result.get('error') or t('cli.shared.unknown_error'))}{_RST}")
except Exception as e:
_cprint(f"\n{_DIM}{t('cli.voice.processing_error', error=e)}{_RST}")
finally:
_unlink_quietly(wav_path)
self._voice_barge_capture.clear()
self._voice_barge_phase = None
# No usable transcript: hand the mic back to the normal loop.
if not submitted and self._voice_mode and self._voice_continuous and not self._voice_recording:
self._voice_restart_recording_async()
def _voice_beeps_enabled(self) -> bool:
"""Return whether CLI voice mode should play record start/stop beeps."""
try:
from utils import is_truthy_value # handles quoted YAML "false" (bool() would not)
return is_truthy_value(_config_section("voice").get("beep_enabled", True), default=True)
except Exception:
return True
def _enable_voice_mode(self):
"""Enable voice mode after checking requirements."""
from cli import _ACCENT, _BOLD, _DIM, _RST, _cprint
if self._voice_mode:
_cprint(f"{_DIM}{t('cli.voice.already_enabled')}{_RST}")
return
from tools.voice_mode import check_voice_requirements, detect_audio_environment
env_check = detect_audio_environment()
if not env_check["available"]:
_cprint(f"\n{_ACCENT}{t('cli.voice.unavailable_header')}{_RST}")
for warning in env_check["warnings"]:
_cprint(f" {_DIM}{warning}{_RST}")
return
reqs = check_voice_requirements()
if not reqs["available"]:
_cprint(f"\n{_ACCENT}{t('cli.voice.requirements_header')}{_RST}")
for line in reqs["details"].split("\n"):
_cprint(f" {_DIM}{line}{_RST}")
if reqs["missing_packages"]:
if _is_termux_environment():
_cprint(f"\n {_BOLD}{t('cli.voice.termux_install_cmd')}{_RST}")
_cprint(f" {_DIM}{t('cli.voice.termux_install_hint')}{_RST}")
return
with self._voice_lock:
self._voice_mode = True
if _config_section("voice").get("auto_tts", False):
with self._voice_lock:
self._voice_tts = True
# The voice-mode instruction is injected as a user message prefix (not a system
# prompt change) to avoid invalidating the prompt cache — see _voice_message_prefix.
tts_status = t("cli.voice.tts_enabled_suffix") if self._voice_tts else ""
if self._voice_tts:
self._tts_lease_async(True) # warm the engine so the first reply isn't dead air
# Startup-pinned label so the advertised shortcut always matches the live
# prompt_toolkit binding (live config would drift after a mid-session edit).
# See #19835.
_cprint(f"\n{_ACCENT}{t('cli.voice.enabled', tts_status=tts_status)}{_RST}")
_cprint(f" {_DIM}{t('cli.voice.record_key_hint', shortcut=self._voice_record_key_label())}{_RST}")
# Spoken-stop hint from voice.stop_phrases (first entry); "" when disabled.
try:
from tools.voice_mode_transcript import voice_stop_hint
_stop_hint = voice_stop_hint()
except Exception:
_stop_hint = ""
if _stop_hint:
_cprint(f" {_DIM}{_stop_hint}{_RST}")
_cprint(f" {_DIM}{t('cli.voice.tts_toggle_hint')}{_RST}")
_cprint(f" {_DIM}{t('cli.voice.disable_hint')}{_RST}")
def _typed_voice_stop(self, user_input) -> bool:
"""Typed bare stop phrase during an active voice chat ends the chat (mirrors the spoken
one; outside voice mode "stop" passes through to the agent). Exact-match via
``is_voice_stop_phrase``, so longer messages containing "stop" are never swallowed.
Saying "stop" ends the voice chat (PR #73106); TYPING the same bare stop phrase while voice mode is
on must behave identically instead of sending "stop" to the agent as a turn.
"""
from cli import _DIM, _RST, _cprint
if not isinstance(user_input, str):
return False
with self._voice_lock:
voice_on = self._voice_mode or self._voice_continuous
if not voice_on:
return False
try:
from tools.voice_mode_transcript import is_voice_stop_phrase
if not is_voice_stop_phrase(user_input):
return False
except Exception:
return False
_cprint(f"\n{_DIM}{t('cli.voice.stop_phrase_typed')}{_RST}")
self._disable_voice_mode()
return True
def _disable_voice_mode(self):
"""Disable voice mode, cancel any active recording, and stop TTS."""
from cli import _DIM, _RST, _cprint, logger
with self._voice_lock:
if self._voice_recording and self._voice_recorder:
self._voice_recorder.cancel()
self._voice_recording = False
recorder = self._voice_recorder
self._voice_mode = False
self._voice_tts = False
self._voice_continuous = False
# Release the TTS lease so a resident local model (piper/kittentts) can be freed.
self._tts_lease_async(False)
# Shut down the persistent audio stream in background
if recorder is not None:
def _bg_shutdown(rec=recorder):
try:
rec.shutdown()
except Exception:
pass
threading.Thread(target=_bg_shutdown, daemon=True).start()
self._voice_recorder = None
# Stop any active TTS playback (file player + streaming pipeline)
try:
if self._voice_tts_stop is not None:
logger.info("TTS CUT: _disable_voice_mode setting stop event")
self._voice_tts_stop.set()
from tools.voice_mode import stop_playback
stop_playback()
except Exception:
pass
self._voice_tts_done.set()
_cprint(f"\n{_DIM}{t('cli.voice.disabled')}{_RST}")
def _maybe_start_wake_word(self):
"""Start the wake-word listener at CLI startup if this surface is eligible."""
try:
from tools.wake_word import wake_surface_enabled
if not wake_surface_enabled("cli"):
return
except Exception:
return
self._start_wake_word_listener(announce=True)
def _start_wake_word_listener(self, announce: bool = False) -> bool:
"""Build + start the hotword detector. Returns True on success."""
from cli import _ACCENT, _DIM, _RST, _cprint
say = _cprint if announce else (lambda *_a: None)
try:
from tools.wake_word import (
check_wake_word_requirements, load_wake_word_config, owns_listener, start_listening)
except Exception as e:
say(f"{_DIM}{t('cli.voice.wake_unavailable', error=e)}{_RST}")
return False
if getattr(self, "_wake_word_active", False) and owns_listener(self):
say(f"{_DIM}{t('cli.voice.wake_already')}{_RST}")
return True
self._wake_word_active = False
cfg = load_wake_word_config()
reqs = check_wake_word_requirements(cfg)
if not reqs["available"]:
say(f"\n{_ACCENT}{t('cli.voice.wake_requirements_header')}{_RST}")
if reqs.get("hint"):
say(f" {_DIM}{reqs['hint']}{_RST}")
return False
if not reqs.get("deps_available", True):
# Fresh install: the engine constructor lazy-installs its deps (onnxruntime is
# a large wheel) — tell the user why this is slow.
say(f"{_DIM}{t('cli.voice.wake_installing')}{_RST}")
self._wake_start_new_session = bool(cfg.get("start_new_session", True))
try:
start_listening(self._on_wake_word, owner=self, config=cfg)
except Exception as e:
say(f"\n{_DIM}{t('cli.voice.wake_start_failed', error=e)}{_RST}")
return False
self._wake_word_active = True
self._wake_suspended = False
import cli as _cli
_cli._cli_wake_owner = self
self._start_wake_watchdog()
say(f"\n{_ACCENT}{t('cli.voice.wake_listening')}{_RST} "
f"{_DIM}{t('cli.voice.wake_listening_hint', phrase=reqs['phrase'])}{_RST}")
return True
def _stop_wake_word_listener(self, announce: bool = False):
"""Stop and tear down the hotword detector."""
from cli import _DIM, _RST, _cprint
import cli as _cli
was_active = getattr(self, "_wake_word_active", False)
self._wake_word_active = False
self._wake_suspended = False
try:
from tools.wake_word import stop_listening
stop_listening(owner=self)
except Exception:
pass
if _cli._cli_wake_owner is self:
_cli._cli_wake_owner = None
if announce:
_cprint(f"{_DIM}{t('cli.voice.wake_stopped' if was_active else 'cli.voice.wake_not_running')}{_RST}")
def _on_wake_word(self):
"""Fired after the detector hears the wake phrase."""
from cli import _ACCENT, _DIM, _RST, _cprint, logger
if getattr(self, "_should_exit", False):
return
# Ignore wake while a turn is in flight or the mic is already in use.
if self._agent_running or self._voice_recording or getattr(self, "_voice_processing", False):
return
# Release the mic so STT can capture the command utterance.
try:
from tools.wake_word import pause_listening
if not pause_listening(owner=self):
self._wake_word_active = False
return
except Exception as e:
logger.debug("wake word pause failed: %s", e)
return
self._wake_suspended = True
# The CLI is single-profile: a phrase enrolled by ANOTHER profile can't be routed
# here — print the switch command and re-arm rather than answer as the wrong profile.
try:
from tools.wake_word import get_last_match
_match = get_last_match()
except Exception:
_match = None
if _match and _match[1]:
from tools.wake_word import _active_profile_name
if _match[1] != _active_profile_name():
_cprint(f"\n{_DIM}{t('cli.voice.wake_profile_hint', profile=_match[1])}{_RST}")
self._wake_suspended = True # watchdog resumes the listener
return
_cprint(f"\n{_ACCENT}{t('cli.voice.wake_detected')}{_RST}")
if getattr(self, "_app", None):
try:
self._app.invalidate()
except Exception:
pass
if getattr(self, "_wake_start_new_session", True):
try:
self.new_session(silent=True)
except Exception as e:
logger.debug("wake word new_session failed: %s", e)
# Single-utterance capture; VAD auto-stop transcribes and queues for process_loop.
with self._voice_lock:
self._voice_mode = True
self._voice_continuous = False
try:
self._voice_start_recording()
except Exception as e:
_cprint(f"{_DIM}{t('cli.voice.wake_capture_failed', error=e)}{_RST}")
def _start_wake_watchdog(self):
"""Resume the paused detector when the CLI returns to a stable idle."""
from cli import logger
if getattr(self, "_wake_watchdog_started", False):
return
self._wake_watchdog_started = True
def _loop():
idle_polls = 0
try:
while getattr(self, "_wake_word_active", False) and not getattr(self, "_should_exit", False):
time.sleep(0.25)
if not getattr(self, "_wake_suspended", False):
idle_polls = 0
continue
busy = (
self._agent_running
or self._voice_recording
or getattr(self, "_voice_processing", False)
or not self._pending_input.empty())
if busy:
idle_polls = 0
continue
# Require a few consecutive idle polls (~0.75s) so we don't resume in
# the gap between VAD stop and the agent starting.
idle_polls += 1
if idle_polls >= 3:
idle_polls = 0
try:
from tools.wake_word import resume_listening
if resume_listening(owner=self):
self._wake_suspended = False
else:
self._wake_word_active = False
except Exception as e:
logger.debug("wake word resume failed: %s", e)
finally:
self._wake_watchdog_started = False
threading.Thread(target=_loop, daemon=True, name="wake-watchdog").start()
def _show_wake_word_status(self):
"""Show current wake-word listener status."""
from cli import _ACCENT, _BOLD, _DIM, _RST, _cprint
from tools.wake_word import (
audio_is_silent, check_wake_word_requirements, is_listening, load_wake_word_config,
owns_listener)
cfg = load_wake_word_config()
reqs = check_wake_word_requirements(cfg)
owned = owns_listener(self)
# ``state`` is the internal id; only its display label is localized.
state = "listening" if owned and is_listening() else "paused" if owned else "off"
_cprint(f"\n{_BOLD}{t('cli.voice.wake_status_title')}{_RST}")
_cprint(t("cli.voice.label_state", state=t(f"cli.voice.state_{state}")))
_cprint(t("cli.voice.label_phrase", phrase=reqs['phrase']))
_cprint(t("cli.voice.label_provider", provider=reqs['provider']))
_cprint(t("cli.voice.label_surface", surface=cfg.get('surface', 'auto')))
_cprint(t("cli.voice.label_new_session",
value=t("cli.shared.label_yes" if cfg.get('start_new_session', True) else "cli.shared.label_no")))
if state == "listening" and audio_is_silent():
_cprint(f" {_ACCENT}{t('cli.voice.mic_silent_warning')}{_RST}")
_cprint(f" {_DIM}{t('cli.voice.mic_silent_macos_hint')}{_RST}")
if not reqs["available"] and reqs.get("hint"):
_cprint(f" {_DIM}{reqs['hint']}{_RST}")
if not owned:
_cprint(f" {_DIM}{t('cli.voice.wake_enable_hint')}{_RST}")
def _tts_lease_async(self, active: bool) -> None:
"""Acquire/release this CLI's TTS engine lease in the background.
Acquiring pre-loads the configured provider so the first reply starts hot; releasing
lets the last-holder path unload resident local models. Never blocks or fails the toggle.
"""
from cli import logger
def _run():
try:
from tools.tts_tool_lifecycle import acquire_tts_lease, release_tts_lease
if active:
acquire_tts_lease("cli:voice-tts")
else:
release_tts_lease("cli:voice-tts")
except Exception as e:
logger.debug("voice: tts lease active=%s failed: %s", active, e)
threading.Thread(target=_run, name="tts-lease-cli", daemon=True).start()
def _toggle_voice_tts(self):
"""Toggle TTS output for voice mode."""
from cli import _ACCENT, _DIM, _RST, _cprint
if not self._voice_mode:
_cprint(f"{_DIM}{t('cli.voice.enable_first_hint')}{_RST}")
return
with self._voice_lock:
self._voice_tts = not self._voice_tts
status = t("cli.shared.label_enabled" if self._voice_tts else "cli.shared.label_disabled")
if self._voice_tts:
from tools.tts_tool import check_tts_requirements
if not check_tts_requirements():
_cprint(f"{_DIM}{t('cli.voice.tts_no_provider')}{_RST}")
self._tts_lease_async(self._voice_tts) # warm-up / release signal for the TTS engine
_cprint(f"{_ACCENT}{t('cli.voice.tts_status', status=status)}{_RST}")
def _show_voice_status(self):
"""Show current voice mode status."""
from cli import _BOLD, _RST, _cprint
from tools.voice_mode import check_voice_requirements
reqs = check_voice_requirements()
_on, _off = t("cli.voice.state_on"), t("cli.voice.state_off")
_cprint(f"\n{_BOLD}{t('cli.voice.status_title')}{_RST}")
_cprint(t("cli.voice.label_mode", value=_on if self._voice_mode else _off))
_cprint(t("cli.voice.label_tts", value=_on if self._voice_tts else _off))
_cprint(t("cli.voice.label_recording",
value=t("cli.voice.state_yes") if self._voice_recording else t("cli.shared.label_no")))
# Startup-pinned label so /voice status always matches the live prompt_toolkit
# binding (live config would drift after a mid-session config edit).
# See #19835.
_cprint(t("cli.voice.label_record_key", shortcut=self._voice_record_key_label()))
_cprint(f"\n {_BOLD}{t('cli.voice.label_requirements')}{_RST}")
for line in reqs["details"].split("\n"):
_cprint(f" {line}")