From 4d37eddc2f5716b5d97a461eb864ac5fdc2cd6e7 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:41:57 -0700 Subject: [PATCH] refactor(tools): unify STT/TTS command-provider config helpers, REST STT flow, dispatch tables --- tools/transcription_audio.py | 74 +++----- tools/transcription_cloud.py | 161 +++++++++--------- tools/transcription_command.py | 168 ++++++------------ tools/transcription_common.py | 21 +-- tools/transcription_local.py | 73 +++----- tools/transcription_tools.py | 158 ++++++++--------- tools/tts_command_provider.py | 300 ++++++++++++--------------------- tools/tts_streaming.py | 165 +++++------------- tools/tts_text_normalize.py | 187 ++++++++------------ 9 files changed, 476 insertions(+), 831 deletions(-) diff --git a/tools/transcription_audio.py b/tools/transcription_audio.py index 07e45a6734..069a3d957b 100644 --- a/tools/transcription_audio.py +++ b/tools/transcription_audio.py @@ -2,10 +2,9 @@ Binary discovery, the shared ffmpeg m4a encode (transcode + silence trim), source/format validation, WeChat .silk decoding, CAF conversion and the -best-effort cloud pre-upload silence trim. - -Split out of ``tools/transcription_tools.py``, which re-imports every name (patch -surface) and is imported lazily here so origin patches still intercept. +best-effort cloud pre-upload silence trim. Every name is re-imported by +``tools/transcription_tools.py`` (patch surface), which is imported lazily here +so origin patches still intercept. """ from __future__ import annotations @@ -61,17 +60,11 @@ def _run_quiet(command: list, *, timeout: float, env: Optional[dict] = None) -> # Shared encode profile for every STT-bound m4a (transcode and silence-trim): # 16 kHz mono 32 kbps AAC, faststart. One owner so codec/bitrate never drift. -_STT_M4A_ENCODE_ARGS = ( - "-vn", "-ac", "1", "-ar", "16000", - "-c:a", "aac", "-b:a", "32k", "-movflags", "+faststart", -) +_STT_M4A_ENCODE_ARGS = ("-vn", "-ac", "1", "-ar", "16000", "-c:a", "aac", "-b:a", "32k", "-movflags", "+faststart") def _run_ffmpeg_stt_encode(ffmpeg: str, input_path: str, output_path: str, *, audio_filter: Optional[str] = None) -> None: - """Run the shared STT m4a encode, optionally with an ``-af`` filter. - - Raises on failure — callers own the error semantics (transcode reports, trim swallows). - """ + """Run the shared STT m4a encode, optionally with an ``-af`` filter. Raises on failure (callers own the semantics).""" command = [ffmpeg, "-y", "-i", input_path] if audio_filter: command += ["-af", audio_filter] @@ -80,11 +73,10 @@ def _run_ffmpeg_stt_encode(ffmpeg: str, input_path: str, output_path: str, *, au def _transcode_audio_for_stt(file_path: str, work_dir: str) -> tuple[Optional[str], Optional[str]]: - """Transcode to a compact 16 kHz mono AAC/m4a for STT upload. + """Transcode to a compact 16 kHz mono AAC/m4a for STT upload; ``(converted_path, None)`` or ``(None, error)``. Newer OpenAI models reject containers ``whisper-1`` accepted (notably Ogg/Opus voice notes) and gateway downloads may carry a misleading extension. - Returns ``(converted_path, None)`` or ``(None, error)``. """ from tools.transcription_tools import _find_ffmpeg_binary, _run_ffmpeg_stt_encode ffmpeg = _find_ffmpeg_binary() @@ -110,16 +102,13 @@ def _validate_audio_file_size(audio_path: Path, *, enforce_size_limit: bool = Tr except OSError as e: return _error_result(f"Failed to access file: {e}") if enforce_size_limit and file_size > MAX_FILE_SIZE: - return _error_result( - f"File too large: {file_size / (1024*1024):.1f}MB (max {MAX_FILE_SIZE / (1024*1024):.0f}MB)" - ) + return _error_result(f"File too large: {file_size / (1024*1024):.1f}MB (max {MAX_FILE_SIZE / (1024*1024):.0f}MB)") return None def _validate_audio_source_file(file_path: str, *, enforce_size_limit: bool = True) -> Optional[Dict[str, Any]]: """Validate source path safety (and optionally size) before any decoder runs.""" audio_path = Path(file_path) - if os.path.islink(audio_path): return _error_result(f"Path is a symbolic link: {file_path}") if not audio_path.exists(): @@ -137,9 +126,7 @@ def _validate_audio_file(file_path: str, *, enforce_size_limit: bool = True) -> suffix = Path(file_path).suffix if suffix.lower() not in SUPPORTED_FORMATS: - return _error_result( - f"Unsupported format: {suffix}. Supported: {', '.join(sorted(SUPPORTED_FORMATS))}" - ) + return _error_result(f"Unsupported format: {suffix}. Supported: {', '.join(sorted(SUPPORTED_FORMATS))}") return None @@ -150,8 +137,7 @@ def _prepare_audio_for_transcription(file_path: str) -> tuple[Optional[str], Opt if audio_path.suffix.lower() != ".silk": return file_path, None, None if not _HAS_PILK: - # pilk is a tiny silk-v3 codec binding — lazy-install on first .silk - # voice note instead of bloating the base install. + # pilk is a tiny silk-v3 codec binding — lazy-installed on first .silk voice note. _lazy_ensure_quietly("stt.silk") if not _safe_find_spec("pilk"): return None, None, _error_result( @@ -221,23 +207,22 @@ def _convert_caf_to_wav(file_path: str) -> Optional[str]: # ---- Cloud pre-upload silence trim -------------------------------------- # -# Local faster-whisper gets Silero VAD; cloud providers get the raw file, so -# every second of silence is paid for twice (upload + per-minute billing) and -# cloud Whisper hallucinates on it. Before uploading to a built-in cloud -# provider we collapse long pauses with ffmpeg's silenceremove, keeping -# ``stt.cloud_trim_keep_ms`` of each pause so word boundaries survive. -# Purely best-effort — ANY of these uploads the original untouched: -# ``stt.cloud_trim_silence: false``, ffmpeg/ffprobe missing, trim failure or -# timeout, a ~empty result (the provider, not a dB heuristic, decides "no -# speech"), or <10% saving. Command-type and plugin providers are NOT trimmed: -# they may wrap local CLIs that want the original bytes. +# Local faster-whisper gets Silero VAD; cloud providers get the raw file, so silence +# is paid for twice (upload + per-minute billing) and cloud Whisper hallucinates on +# it. Before uploading to a built-in cloud provider we collapse long pauses with +# ffmpeg's silenceremove, keeping ``stt.cloud_trim_keep_ms`` of each pause so word +# boundaries survive. Purely best-effort — ANY of these uploads the original: +# ``stt.cloud_trim_silence: false``, ffmpeg/ffprobe missing, trim failure/timeout, a +# ~empty result (the provider, not a dB heuristic, decides "no speech"), or <10% +# saving. Command-type and plugin providers are NOT trimmed: they may wrap local +# CLIs that want the original bytes. _CLOUD_TRIM_THRESHOLD_DB_DEFAULT = -40 # audio below this level counts as silence _CLOUD_TRIM_KEEP_MS_DEFAULT = 300 # how much of each pause survives the trim _CLOUD_TRIM_MIN_SAVING = 0.10 # use the trimmed file only when >=10% shorter _CLOUD_TRIM_MIN_RESULT_SECONDS = 0.3 # all-silence guard floor: never upload ~empty audio -# Below this the trim can't pay for itself (several providers bill a 10s -# minimum per request) and the encode would sit on the synchronous voice-note path. +# Below this the trim can't pay for itself (several providers bill a 10s minimum) +# and the encode would sit on the synchronous voice-note path. _CLOUD_TRIM_MIN_INPUT_SECONDS = 12.0 @@ -251,12 +236,7 @@ def _probe_audio_duration(file_path: str) -> Optional[float]: ffprobe = _find_ffprobe_binary() if not ffprobe: return None - command = [ - ffprobe, "-v", "error", - "-show_entries", "format=duration", - "-of", "default=noprint_wrappers=1:nokey=1", - file_path, - ] + command = [ffprobe, "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", file_path] try: return float(_run_quiet(command, timeout=30).stdout.strip()) except Exception: # noqa: BLE001 - probe is best-effort @@ -274,11 +254,9 @@ def _cloud_trim_settings(stt_config: Dict[str, Any]) -> tuple[bool, int, int]: def _trim_silence_for_cloud_stt(file_path: str, stt_config: Dict[str, Any]) -> Optional[str]: - """Return a silence-trimmed copy of *file_path* for cloud upload, or None. + """Return a silence-trimmed copy of *file_path* for cloud upload, or None (= upload the original). - ``None`` always means "upload the original" (disabled, tools missing, clip - too short, trim failed, mostly silence, or not enough saving). On success - the caller owns deleting the returned file's parent directory. + On success the caller owns deleting the returned file's parent directory. """ from tools.transcription_tools import _find_ffmpeg_binary, _probe_audio_duration, _run_ffmpeg_stt_encode enabled, threshold_db, keep_ms = _cloud_trim_settings(stt_config) @@ -301,8 +279,7 @@ def _trim_silence_for_cloud_stt(file_path: str, stt_config: Dict[str, Any]) -> O return None keep_seconds = keep_ms / 1000.0 - # start_periods=1 strips leading silence; stop_periods=-1 collapses every - # interior/trailing silence, keeping ``keep_seconds`` of each pause. + # start_periods=1 strips leading silence; stop_periods=-1 collapses every interior/trailing silence. filter_expr = ( f"silenceremove=" f"start_periods=1:start_threshold={threshold_db}dB:start_silence={keep_seconds}:" @@ -310,8 +287,7 @@ def _trim_silence_for_cloud_stt(file_path: str, stt_config: Dict[str, Any]) -> O ) work_dir = tempfile.mkdtemp(prefix="hermes-stt-trim-") trimmed_path = os.path.join(work_dir, f"{Path(file_path).stem or 'audio'}-trimmed.m4a") - # Scale the all-silence guard with keep_ms: an output consisting solely - # of kept pause must never be uploaded as "speech". + # Scale the all-silence guard with keep_ms: output that is solely kept pause must never upload as "speech". min_result_seconds = max(_CLOUD_TRIM_MIN_RESULT_SECONDS, 2 * keep_seconds) keep_result = False try: diff --git a/tools/transcription_cloud.py b/tools/transcription_cloud.py index c81f008a6d..620d7f5dda 100644 --- a/tools/transcription_cloud.py +++ b/tools/transcription_cloud.py @@ -2,10 +2,9 @@ OpenAI-SDK-shaped backends (groq, openai, deepinfra), Mistral Voxtral, the REST multipart backends (xAI, ElevenLabs), and OpenAI audio credential resolution -(config > keyless local server > env > managed Nous gateway). - -Split out of ``tools/transcription_tools.py``, which re-imports every name (patch -surface) and is imported lazily here so origin patches still intercept. +(config > keyless local server > env > managed Nous gateway). Every name is +re-imported by ``tools/transcription_tools.py`` (patch surface), which is +imported lazily here so origin patches still intercept. """ from __future__ import annotations @@ -14,7 +13,7 @@ import logging import re import tempfile from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Callable, Dict, Optional from urllib.parse import urljoin from utils import is_truthy_value @@ -35,12 +34,6 @@ def _has_xai_stt_credentials() -> bool: return bool(resolve_xai_http_credentials().get("api_key")) -def _close_client(client: Any) -> None: - close = getattr(client, "close", None) - if callable(close): - close() - - def _with_openai_client(api_key: str, base_url: Optional[str], file_path: str, log_label: str, body): """Run ``body(client)`` against a fresh OpenAI SDK client (30s timeout, no SDK retries). @@ -53,7 +46,9 @@ def _with_openai_client(api_key: str, base_url: Optional[str], file_path: str, l try: return body(client) finally: - _close_client(client) + close = getattr(client, "close", None) + if callable(close): + close() except Exception as e: return _openai_sdk_failure(e, file_path, log_label) @@ -78,16 +73,23 @@ def _openai_sdk_failure(exc: BaseException, file_path: str, log_label: str) -> D APIError = APIConnectionError = APITimeoutError = () if isinstance(exc, PermissionError): return _error_result(f"Permission denied: {file_path}") - if isinstance(exc, APIConnectionError): - return _error_result(f"Connection error: {exc}") - if isinstance(exc, APITimeoutError): - return _error_result(f"Request timeout: {exc}") - if isinstance(exc, APIError): - return _error_result(f"API error: {exc}") + for cls, label in ((APIConnectionError, "Connection error"), (APITimeoutError, "Request timeout"), (APIError, "API error")): + if isinstance(exc, cls): + return _error_result(f"{label}: {exc}") logger.error("%s transcription failed: %s", log_label, exc, exc_info=True) return _error_result(f"Transcription failed: {exc}") +def _sdk_prompt_kwargs(language: Optional[str], prompt: Optional[str]) -> Dict[str, Any]: + """``language``/``prompt`` create-kwargs, each only when set so the bare request stays byte-identical.""" + kwargs: Dict[str, Any] = {} + if language: + kwargs["language"] = language + if prompt: + kwargs["prompt"] = prompt + return kwargs + + def _transcribe_groq( file_path: str, model_name: str, @@ -104,7 +106,6 @@ def _transcribe_groq( api_key = _resolve_provider_key("GROQ_API_KEY", "groq") if not api_key: return _error_result("GROQ_API_KEY not set") - if not _HAS_OPENAI: return _error_result("openai package not installed") @@ -116,25 +117,12 @@ def _transcribe_groq( language = language or _resolve_stt_language("groq") def _run(client): - create_kwargs = { - "model": model_name, - "response_format": "text", - } - if language: - create_kwargs["language"] = language - if prompt: - # Only sent when set so the no-hook, no-config request stays byte-identical. - create_kwargs["prompt"] = prompt + create_kwargs = {"model": model_name, "response_format": "text", **_sdk_prompt_kwargs(language, prompt)} with open(file_path, "rb") as audio_file: - transcription = client.audio.transcriptions.create( - file=audio_file, - **create_kwargs, - ) - + transcription = client.audio.transcriptions.create(file=audio_file, **create_kwargs) transcript_text = str(transcription).strip() logger.info("Transcribed %s via Groq API (%s, lang=%s, %d chars)", Path(file_path).name, model_name, language or "auto", len(transcript_text)) - return _ok_result(transcript_text, "groq") return _with_openai_client(api_key, GROQ_BASE_URL, file_path, "Groq", _run) @@ -222,7 +210,6 @@ def _transcribe_openai( "Transcribed %s via %s (%s, %d chars)", Path(file_path).name, provider_label, model_name, len(transcript_text), ) - return _ok_result(transcript_text, provider_label) return _with_openai_client(api_key, base_url, file_path, provider_label, _run) @@ -247,17 +234,13 @@ def _transcribe_mistral( with Mistral(api_key=api_key) as client: with open(file_path, "rb") as audio_file: + # Language: hook override > stt.mistral.language > stt.language > env > auto. + language = language or _resolve_stt_language("mistral") complete_kwargs: Dict[str, Any] = { "model": model_name, "file": {"content": audio_file, "file_name": Path(file_path).name}, + **_sdk_prompt_kwargs(language, prompt), } - # Language: hook override > stt.mistral.language > stt.language > env > auto. - language = language or _resolve_stt_language("mistral") - if language: - complete_kwargs["language"] = language - if prompt: - # Only sent when set so the no-hook, no-config request stays byte-identical. - complete_kwargs["prompt"] = prompt result = client.audio.transcriptions.complete(**complete_kwargs) transcript_text = _extract_transcript_text(result) @@ -271,6 +254,9 @@ def _transcribe_mistral( return _cloud_failure(e, file_path, "Mistral transcription", type(e).__name__) +# ---- REST multipart backends (xAI, ElevenLabs) ---------------------------- + + def _post_audio_multipart(url: str, headers: Dict[str, str], file_path: str, data: Dict[str, str]): import requests @@ -281,22 +267,18 @@ def _post_audio_multipart(url: str, headers: Dict[str, str], file_path: str, dat ) -def _http_error_detail(response, extract) -> str: - """``extract(json_body)`` -> detail string, falling back to the first 300 chars of the body.""" - try: - return extract(response.json()) or response.text[:300] - except Exception: - return response.text[:300] - - def _rest_transcript(response, label: str, extract_detail, extract_text): """Turn a multipart STT response into ``(text, body, None)`` or ``(None, None, error_envelope)``. - Non-200 -> ``"