diff --git a/tests/tools/test_transcription_chunking.py b/tests/tools/test_transcription_chunking.py new file mode 100644 index 0000000000..801e4710f0 --- /dev/null +++ b/tests/tools/test_transcription_chunking.py @@ -0,0 +1,69 @@ +"""Cloud STT inputs over the provider's upload cap are fitted, not rejected (real ffmpeg, real files).""" + +from __future__ import annotations + +import shutil +import struct +import wave +from pathlib import Path +from unittest.mock import patch + +import pytest + +import tools.transcription_chunking as chunking +import tools.transcription_tools as tt +from tools.transcription_audio import _probe_audio_duration + +_HAS_FFMPEG = bool(shutil.which("ffmpeg")) and bool(shutil.which("ffprobe")) +pytestmark = pytest.mark.skipif(not _HAS_FFMPEG, reason="ffmpeg/ffprobe not installed") + + +def _tone_with_pauses(path: Path, seconds: int, rate: int = 16000) -> None: + """Alternating 1 s tone / 1 s silence, so silencedetect finds a pause every 2 s.""" + frames = bytearray() + for second in range(seconds): + amplitude = 6000 if second % 2 == 0 else 0 + frames += struct.pack(f"<{rate}h", *([amplitude, -amplitude] * (rate // 2))) + with wave.open(str(path), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(rate) + wav.writeframes(bytes(frames)) + + +def _run(path: Path, provider: str, *, model: str | None = None): + """Run the real transcribe_audio with only the provider's network call replaced.""" + uploads: list = [] + + def handler(file_path, model_name, **_kw): + uploads.append({"suffix": Path(file_path).suffix, "bytes": Path(file_path).stat().st_size, + "seconds": _probe_audio_duration(file_path), "model": model_name}) + return {"success": True, "transcript": f"part{len(uploads)}", "provider": provider} + + with patch.object(tt, "_load_stt_config", return_value={"provider": provider, "cloud_trim_silence": False}), \ + patch.object(tt, "_get_provider", return_value=provider), \ + patch.object(tt, f"_transcribe_{provider}", side_effect=handler): + return tt.transcribe_audio(str(path), model=model), uploads + + +def test_oversized_wav_is_reencoded_into_one_upload(tmp_path, monkeypatch): + """A WAV over the byte cap that compresses under it goes up once, as m4a, with no seams.""" + monkeypatch.setitem(chunking._PROVIDER_MAX_BYTES, "groq", 400_000) + wav = tmp_path / "note.wav" + _tone_with_pauses(wav, 20) # 640 KB of PCM, ~80 KB as 32 kbps AAC + result, uploads = _run(wav, "groq") + assert result == {"success": True, "transcript": "part1", "provider": "groq"} + assert len(uploads) == 1 and uploads[0]["suffix"] == ".m4a" and uploads[0]["bytes"] < 400_000 + + +def test_over_duration_cap_splits_in_order_under_the_cap(tmp_path, monkeypatch): + """A model duration cap forces segments: each fits the cap, transcripts join in order, all audio is kept.""" + monkeypatch.setitem(chunking._OPENAI_MODEL_MAX_SECONDS, "gpt-4o-transcribe", 10.0) + wav = tmp_path / "meeting.wav" + _tone_with_pauses(wav, 30) + result, uploads = _run(wav, "openai", model="gpt-4o-transcribe") + assert result["success"] is True + assert result["segments"] == len(uploads) >= 3 + assert result["transcript"] == " ".join(f"part{i}" for i in range(1, len(uploads) + 1)) + assert all(u["model"] == "gpt-4o-transcribe" and u["seconds"] <= 10.0 for u in uploads) + assert sum(u["seconds"] for u in uploads) == pytest.approx(30.0, abs=0.5) diff --git a/tests/tools/test_transcription_plugin_dispatch.py b/tests/tools/test_transcription_plugin_dispatch.py index 0f78025073..9b570efea5 100644 --- a/tests/tools/test_transcription_plugin_dispatch.py +++ b/tests/tools/test_transcription_plugin_dispatch.py @@ -177,7 +177,8 @@ class TestTranscribeAudioE2E: # Plugin was never called assert provider.last_call is None - def test_oversized_plugin_file_is_rejected_before_dispatch(self, tmp_path): + def test_unsplittable_oversized_plugin_file_never_reaches_the_plugin(self, tmp_path): + """Over the upload cap with no way to fit it (ffmpeg missing) -> refused, plugin untouched.""" from unittest.mock import patch provider = _FakeProvider(name="openrouter") @@ -190,7 +191,8 @@ class TestTranscribeAudioE2E: with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ - patch("tools.transcription_tools._get_provider", return_value="openrouter"): + patch("tools.transcription_tools._get_provider", return_value="openrouter"), \ + patch("tools.transcription_chunking._find_ffmpeg_binary", return_value=None): result = transcription_tools.transcribe_audio(str(audio_path)) assert result["success"] is False diff --git a/tools/transcription_chunking.py b/tools/transcription_chunking.py new file mode 100644 index 0000000000..75c63db757 --- /dev/null +++ b/tools/transcription_chunking.py @@ -0,0 +1,156 @@ +"""Per-provider upload limits and long-audio splitting for cloud STT. + +Every cloud transcription endpoint caps one request by size, and some models also cap it by +duration. ``transcribe_audio`` asks :func:`exceeds_upload_limit` before dispatch. An +oversized input is re-encoded to the compact STT m4a first, which is the vendors' own advice +and fits most recordings into a single request with no seams. When it still does not fit, +the file is cut into segments at silences close to the limit. Each segment is dispatched on +its own and the transcripts are joined in order. +""" + +from __future__ import annotations + +import logging +import math +import os +import re +from tempfile import TemporaryDirectory +from typing import Any, Callable, Dict, List, Optional, Tuple + +from tools.transcription_audio import ( + _find_ffmpeg_binary, _probe_audio_duration, _run_quiet, _transcode_audio_for_stt) +from tools.transcription_common import MAX_FILE_SIZE, _error_result, _ok_result + +logger = logging.getLogger("tools.transcription_tools") + +_MB = 1024 * 1024 +# Vendor-documented per-request upload caps. Providers not listed here (DeepInfra's +# OpenAI-compatible endpoint, command and plugin providers) get the OpenAI-shaped default. +_PROVIDER_MAX_BYTES = { + "openai": 25 * _MB, # developers.openai.com/api/docs/guides/speech-to-text: "Files can be up to 25 MB" + "groq": 25 * _MB, # console.groq.com/docs/speech-to-text: 25 MB free tier (100 MB dev tier) + "mistral": 500 * _MB, # docs.mistral.ai/resources/known-limitations: 500 MB, 60 minutes + "xai": 500 * _MB, # docs.x.ai speech-to-text: "Max 500 MB" (413 above it) + "elevenlabs": 5 * 1024 * _MB - 1, # elevenlabs.io speech-to-text convert: "must be less than 5.0GB" +} +_PROVIDER_MAX_SECONDS = {"mistral": 3600.0} +# OpenAI per-model duration caps, all measured live (the vendor docs give none in seconds): +# - gpt-4o-transcribe / gpt-4o-mini-transcribe have a documented 2,000 max output tokens. 1322 s of +# dense speech came back cut off at output_tokens=2048, and the API refuses more than 1400 s +# outright. 450 s keeps fast speech (~200 wpm) under the token ceiling. +# - whisper-1 answers in ~3% of real time (600 s -> 17.8 s, 1200 s -> 55 s), so 600 s segments +# finish well inside the default 60 s ``stt.openai.timeout``. +# - gpt-transcribe returned 1322 s complete in 16.7 s and accepted 1558 s, so it gets no cap. +_OPENAI_MODEL_MAX_SECONDS = {"gpt-4o-transcribe": 450.0, "gpt-4o-mini-transcribe": 450.0, "whisper-1": 600.0} +# Margin under each cap so container overhead and duration rounding never tip a segment over. +_SEGMENT_HEADROOM = 0.9 +# How far back from the ideal cut a silence may sit and still be used as the cut point. +_MAX_SILENCE_LOOKBACK_SECONDS = 60.0 + + +def upload_limit(provider: str, model: Optional[str]) -> Tuple[int, Optional[float]]: + """``(max_bytes, max_seconds or None)`` for one request to *provider* with *model*.""" + max_seconds = _PROVIDER_MAX_SECONDS.get(provider) + if provider == "openai": + max_seconds = _OPENAI_MODEL_MAX_SECONDS.get(model or "") + return _PROVIDER_MAX_BYTES.get(provider, MAX_FILE_SIZE), max_seconds + + +def _fits(path: str, limit: Tuple[int, Optional[float]], duration: Optional[float]) -> bool: + max_bytes, max_seconds = limit + if os.path.getsize(path) > max_bytes: + return False + # An unknown duration (no ffprobe) is left for the vendor to judge. + return max_seconds is None or duration is None or duration <= max_seconds + + +def exceeds_upload_limit(path: str, limit: Tuple[int, Optional[float]]) -> bool: + """True when *path* is over the byte cap, or over a duration cap ffprobe can measure.""" + if os.path.getsize(path) > limit[0]: + return True + return limit[1] is not None and not _fits(path, limit, _probe_audio_duration(path)) + + +def _too_large(path: str, max_bytes: int, reason: str) -> Dict[str, Any]: + size_mb = os.path.getsize(path) / _MB + return _error_result( + f"File too large: {size_mb:.1f}MB (max {max_bytes / _MB:.0f}MB) and it could not be split: {reason}") + + +def _silence_midpoints(ffmpeg: str, path: str) -> List[float]: + """Midpoints (seconds) of the pauses ffmpeg's silencedetect finds; [] on any failure.""" + try: + result = _run_quiet([ffmpeg, "-hide_banner", "-nostats", "-i", path, + "-af", "silencedetect=noise=-35dB:d=0.4", "-f", "null", "-"], timeout=300) + except Exception as exc: # noqa: BLE001 - silence-aware cuts are an optimisation + logger.debug("silencedetect failed for %s: %s", path, exc) + return [] + starts = [float(v) for v in re.findall(r"silence_start: (-?[\d.]+)", result.stderr)] + ends = [float(v) for v in re.findall(r"silence_end: ([\d.]+)", result.stderr)] + return [(max(start, 0.0) + end) / 2 for start, end in zip(starts, ends)] + + +def _cut_points(duration: float, target: float, silences: List[float]) -> List[float]: + """Cut times giving segments no longer than *target*, each at the latest pause before the ideal cut.""" + cuts: List[float] = [] + position = 0.0 + lookback = min(_MAX_SILENCE_LOOKBACK_SECONDS, target / 4) + while duration - position > target: + ideal = position + target + nearby = [s for s in silences if ideal - lookback <= s <= ideal and s > position] + position = max(nearby) if nearby else ideal + cuts.append(position) + return cuts + + +def _split(ffmpeg: str, path: str, work_dir: str, cuts: List[float]) -> List[str]: + pattern = os.path.join(work_dir, "part%03d.m4a") + _run_quiet([ffmpeg, "-y", "-loglevel", "error", "-i", path, "-vn", "-c", "copy", "-f", "segment", + "-segment_times", ",".join(f"{c:.3f}" for c in cuts), "-reset_timestamps", "1", pattern], + timeout=300) + return sorted(os.path.join(work_dir, name) for name in os.listdir(work_dir) if name.startswith("part")) + + +def transcribe_oversized( + path: str, limit: Tuple[int, Optional[float]], dispatch: Callable[[str], Dict[str, Any]], +) -> Dict[str, Any]: + """Fit *path* under *limit* (re-encode, else split at silences) and run *dispatch* per piece.""" + from tools.voice_mode_transcript import is_whisper_hallucination + + max_bytes, max_seconds = limit + ffmpeg = _find_ffmpeg_binary() + if not ffmpeg: + return _too_large(path, max_bytes, "ffmpeg was not found") + with TemporaryDirectory(prefix="hermes-stt-split-", ignore_cleanup_errors=True) as work_dir: + compact, error = _transcode_audio_for_stt(path, work_dir) + if error or not compact: + return _too_large(path, max_bytes, error or "re-encoding produced no file") + duration = _probe_audio_duration(compact) + if _fits(compact, limit, duration): + logger.info("Re-encoded oversized audio %s to %.1fMB for a single STT upload", + os.path.basename(path), os.path.getsize(compact) / _MB) + return dispatch(compact) + if not duration: + return _too_large(path, max_bytes, "its duration could not be read") + target = max_bytes * _SEGMENT_HEADROOM * duration / os.path.getsize(compact) + if max_seconds: + target = min(target, max_seconds * _SEGMENT_HEADROOM) + parts = _split(ffmpeg, compact, work_dir, _cut_points(duration, target, _silence_midpoints(ffmpeg, compact))) + logger.info("Transcribing %s (%.0fs) in %d segments of <= %.0fs", + os.path.basename(path), duration, len(parts), math.ceil(target)) + transcripts: List[str] = [] + provider = "" + for index, part in enumerate(parts, start=1): + result = dispatch(part) + provider = result.get("provider") or provider + if result.get("no_speech"): + continue + if not result.get("success"): + return _error_result(f"Segment {index}/{len(parts)} failed: {result.get('error', 'unknown error')}", + provider=provider) + text = (result.get("transcript") or "").strip() + if text and not is_whisper_hallucination(text): + transcripts.append(text) + if not transcripts: + return _error_result("Transcription returned an empty transcript", no_speech=True, provider=provider) + return {**_ok_result(" ".join(transcripts), provider), "segments": len(parts)} diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 00a4f776f4..065ca88c61 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -29,7 +29,8 @@ from tools.transcription_common import ( normalize_xai_stt_model, _error_result, _get_stt_section, _ok_result) from tools.transcription_audio import ( _convert_caf_to_wav, _prepare_audio_for_transcription, _trim_silence_for_cloud_stt, - _validate_audio_file, _validate_audio_file_size, _validate_audio_source_file) + _validate_audio_file, _validate_audio_source_file) +from tools.transcription_chunking import exceeds_upload_limit, transcribe_oversized, upload_limit from tools.transcription_local import ( _get_idle_unload_seconds, _has_local_command, _join_confident_segments, _load_local_whisper_model, _looks_like_cuda_lib_error, _normalize_local_model, @@ -443,28 +444,35 @@ def _transcribe_prepared_audio( if not is_stt_enabled(stt_config): return _error_result("STT is disabled in config.yaml (stt.enabled: false).") provider = _get_provider(stt_config) + remote = not _is_local_stt_provider(provider, stt_config) with ExitStack() as cleanup: - if not _is_local_stt_provider(provider, stt_config): - error = _validate_audio_file_size(Path(file_path)) - if error: - return error - # Never overwrite a neighboring WAV or leave converted voice notes behind. - if Path(file_path).suffix.lower() == ".caf": - work_dir = cleanup.enter_context( - TemporaryDirectory(prefix="hermes-caf-", ignore_cleanup_errors=True) - ) - file_path = _convert_caf_to_wav(file_path, work_dir) - if not file_path: - return _error_result("CAF audio could not be converted to WAV.") + # Never overwrite a neighboring WAV or leave converted voice notes behind. + if remote and Path(file_path).suffix.lower() == ".caf": + work_dir = cleanup.enter_context(TemporaryDirectory(prefix="hermes-caf-", ignore_cleanup_errors=True)) + file_path = _convert_caf_to_wav(file_path, work_dir) + if not file_path: + return _error_result("CAF audio could not be converted to WAV.") # Best-effort pre-upload silence trim for built-in cloud providers. if provider in CLOUD_STT_PROVIDERS: trimmed = _trim_silence_for_cloud_stt(file_path, stt_config) if trimmed: file_path = trimmed cleanup.callback(shutil.rmtree, os.path.dirname(trimmed), ignore_errors=True) + if remote: + limit = upload_limit(provider, _limit_model_name(provider, stt_config, model)) + if exceeds_upload_limit(file_path, limit): + # Hints are resolved once for the whole recording, not once per segment. + hints = _resolve_dispatch_hints(file_path, provider, stt_config, model, source) + return transcribe_oversized( + file_path, limit, lambda part: _route_stt_provider(part, provider, stt_config, *hints)) return _dispatch_stt_provider(file_path, provider, stt_config, model, source) +def _limit_model_name(provider: str, stt_config: Dict[str, Any], model: Optional[str]) -> Optional[str]: + """Model whose duration cap applies to an upload (built-ins resolve config/defaults).""" + return _builtin_model_name(provider, stt_config, model) if provider in _BUILTIN_MODEL_KEYS else model + + # Built-in provider -> (stt section, config key, default, treat-empty-as-missing). "local_command" # shares ``stt.local``; deepinfra uses the live catalog when empty. _BUILTIN_MODEL_KEYS = { @@ -488,10 +496,10 @@ def _builtin_model_name(provider: str, stt_config: Dict[str, Any], model: Option return normalize_xai_stt_model(resolved) if provider == "xai" else resolved -def _dispatch_stt_provider( - file_path: str, provider: str, stt_config: Dict[str, Any], model: Optional[str] = None, - source: Optional[str] = None) -> Dict[str, Any]: - """Route *file_path* to the handler for *provider* (built-in > command > plugin).""" +def _resolve_dispatch_hints( + file_path: str, provider: str, stt_config: Dict[str, Any], model: Optional[str], source: Optional[str], +) -> tuple[Optional[str], Optional[str], Optional[str]]: + """``(model, language, prompt)`` for one recording: static ``stt.prompt`` plus ``pre_transcription`` hooks.""" # Static ``stt.prompt`` is the base; hook results mutate on top (last hook to set a field wins). prompt = stt_config.get("prompt") prompt = prompt if isinstance(prompt, str) and prompt.strip() else None @@ -500,7 +508,20 @@ def _dispatch_stt_provider( file_path=file_path, provider=provider, model=model, language=_get_stt_section(stt_config, provider).get("language"), prompt=prompt, source=source, ) - prompt = _enforce_prompt_length_limit(prompt, provider) + return model, language, _enforce_prompt_length_limit(prompt, provider) + + +def _dispatch_stt_provider( + file_path: str, provider: str, stt_config: Dict[str, Any], model: Optional[str] = None, + source: Optional[str] = None) -> Dict[str, Any]: + """Route *file_path* to the handler for *provider* (built-in > command > plugin).""" + return _route_stt_provider( + file_path, provider, stt_config, *_resolve_dispatch_hints(file_path, provider, stt_config, model, source)) + + +def _route_stt_provider( + file_path: str, provider: str, stt_config: Dict[str, Any], model: Optional[str], + language: Optional[str], prompt: Optional[str]) -> Dict[str, Any]: if provider in BUILTIN_STT_PROVIDERS: # Looked up in this module at call time so tests may patch ``_transcribe_*``. handler = globals()[f"_transcribe_{provider}"] @@ -570,7 +591,7 @@ def transcribe_audio( def transcribe_audio_local_fallback(file_path: str, model: Optional[str] = None) -> Dict[str, Any]: """Try an already-installed local STT backend without changing config: passive inbound-media recovery after the configured provider failed — never lazy-installs or falls through to cloud.""" - error = _validate_audio_file(file_path) + error = _validate_audio_file(file_path, enforce_size_limit=False) if error: return error local_model = model or (_load_stt_config().get("local") or {}).get("model", DEFAULT_LOCAL_MODEL) diff --git a/tools/voice_mode.py b/tools/voice_mode.py index 8e2afbcca8..5044885d10 100644 --- a/tools/voice_mode.py +++ b/tools/voice_mode.py @@ -886,14 +886,10 @@ def create_audio_recorder() -> AudioRecorder | TermuxAudioRecorder: def transcribe_recording(wav_path: str, model: Optional[str] = None) -> Dict[str, Any]: """Transcribe a WAV via ``transcribe_audio()``, filtering Whisper hallucinations; returns ``{success, transcript[, error]}``.""" - from tools.transcription_common import MAX_FILE_SIZE from tools.transcription_tools import transcribe_audio + # transcribe_audio fits oversized recordings under the provider's upload cap itself. result = transcribe_audio(wav_path, model=model, source="voice_mode") - # Only chunk when the provider itself reports "File too large" — local - # providers have no upload cap and never return this error. - if not result.get("success") and "File too large" in result.get("error", ""): - result = _transcribe_wav_in_chunks(wav_path, model=model, max_file_size=MAX_FILE_SIZE) # A configured stop phrase always survives: "bye"/"okay" overlap the # hallucination blocklist, and swallowing them would make "bye" fail to end the chat. if result.get("success"): @@ -908,67 +904,6 @@ def transcribe_recording(wav_path: str, model: Optional[str] = None) -> Dict[str return result -def _transcribe_wav_in_chunks(wav_path: str, *, model: Optional[str], max_file_size: int) -> Dict[str, Any]: - """Split an oversized WAV into provider-sized chunks and join transcripts.""" - from tools.transcription_tools import transcribe_audio - - chunk_paths, transcripts = [], [] - try: - chunk_paths = _split_wav_for_transcription(wav_path, max_file_size=max_file_size) - if not chunk_paths: - return {"success": False, "transcript": "", "error": "No audio chunks were created"} - logger.info("Transcribing oversized WAV in %d chunks: %s", len(chunk_paths), wav_path) - for index, chunk_path in enumerate(chunk_paths, start=1): - result = transcribe_audio(chunk_path, model=model, source="voice_mode") - if not result.get("success"): - error = result.get("error", "Unknown transcription error") - return {"success": False, "transcript": "", - "error": f"Chunk {index}/{len(chunk_paths)} failed: {error}"} - transcript = result.get("transcript", "").strip() - if transcript and not is_whisper_hallucination(transcript): - transcripts.append(transcript) - return {"success": True, "transcript": " ".join(transcripts).strip(), - "provider": result.get("provider"), "chunks": len(chunk_paths)} - except Exception as e: - logger.error("Chunked transcription failed for %s: %s", wav_path, e, exc_info=True) - return {"success": False, "transcript": "", "error": f"Chunked transcription failed: {e}"} - finally: - for chunk_path in chunk_paths: - _unlink_quietly(chunk_path) - - -def _split_wav_for_transcription(wav_path: str, *, max_file_size: int) -> List[str]: - """Write WAV chunks small enough to pass the shared STT file-size gate.""" - os.makedirs(_TEMP_DIR, exist_ok=True) - chunk_paths: List[str] = [] - with wave.open(wav_path, "rb") as source: - params = source.getparams() - block_align = max(1, params.nchannels * params.sampwidth) - max_data_bytes = max_file_size - 64 * 1024 # header reserve - if max_data_bytes < block_align: - raise ValueError("STT max_file_size is too small for WAV chunking") - frames_per_chunk = max(1, max_data_bytes // block_align) - index = 0 - while True: - frames = source.readframes(frames_per_chunk) - if not frames: - break - index += 1 - with tempfile.NamedTemporaryFile( - prefix=f"{os.path.splitext(os.path.basename(wav_path))[0]}_chunk{index:03d}_", - suffix=".wav", dir=_TEMP_DIR, delete=False) as temp: - chunk_path = temp.name - try: - with wave.open(chunk_path, "wb") as chunk: - chunk.setparams(params._replace(nframes=0)) - chunk.writeframes(frames) - chunk_paths.append(chunk_path) - except Exception: - _unlink_quietly(chunk_path) - raise - return chunk_paths - - # ── Audio playback (interruptable) ── _active_playback: Optional[subprocess.Popen] = None # so stop_playback can interrupt it _playback_lock = threading.Lock() diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 255caa41c4..57a0dd2330 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -2473,6 +2473,8 @@ Provider behavior: Cloud providers (groq, openai, mistral, xai, elevenlabs, deepinfra) get a **pre-upload silence trim** by default when `ffmpeg` is installed: long pauses in a voice note are collapsed client-side before the file uploads, keeping `cloud_trim_keep_ms` of each pause so natural pacing survives. Shorter audio means faster uploads, lower per-audio-minute billing, and fewer silence hallucinations from the remote model. Clips shorter than 12 seconds skip the trim entirely (savings can't matter there, and several providers bill a per-request minimum anyway). The trim is best-effort — if ffmpeg is missing, the trim fails, the clip is mostly silence, or trimming would save less than ~10%, the original file is uploaded untouched. Set `stt.cloud_trim_silence: false` to always upload the original (e.g. when transcribing music or ambient audio through a cloud provider). Command-type and plugin providers never get trimmed audio. +Recordings over the provider's per-request upload limit (25 MB for OpenAI/Groq/DeepInfra, 500 MB for Mistral and xAI, about 5 GB for ElevenLabs, plus per-model length limits for OpenAI's `gpt-4o-transcribe` family and `whisper-1`) are re-encoded to compact AAC and, if still too large, split at pauses and transcribed piece by piece. See [Long recordings and upload limits](features/voice-mode.md#long-recordings-and-upload-limits). + An explicitly selected `stt.provider` is honored strictly — if it's unavailable, transcription errors with guidance to run `hermes tools` rather than switching providers. Only when no provider has ever been selected does Hermes auto-detect in this order: `local` → `groq` → `openai`. Groq and OpenAI model overrides are environment-driven: diff --git a/website/docs/user-guide/features/voice-mode.md b/website/docs/user-guide/features/voice-mode.md index 7a13d23367..5e64aec341 100644 --- a/website/docs/user-guide/features/voice-mode.md +++ b/website/docs/user-guide/features/voice-mode.md @@ -470,7 +470,7 @@ stt: # passes its path to the agent as part of the # inbound message, useful for custom pipelines # (diarization, alignment, archival, etc.) - provider: "local" # "local" (free) | "groq" | "openai" | "mistral" | "xai" + provider: "local" # "local" (free) | "groq" | "openai" | "mistral" | "xai" | "elevenlabs" | "deepinfra" local: model: "base" # tiny, base, small, medium, large-v3 language: "" # optional ISO-639-1 hint; blank = use HERMES_LOCAL_STT_LANGUAGE if set, else auto-detect @@ -542,6 +542,10 @@ DISCORD_ALLOWED_USERS=... Provider priority (automatic fallback): **local** > **groq** > **openai** +### Long recordings and upload limits + +Cloud providers cap a single request: OpenAI and Groq accept 25 MB, Mistral 500 MB (60 minutes), xAI 500 MB and ElevenLabs just under 5 GB. Some OpenAI models also have a practical length limit per request: about 7.5 minutes for `gpt-4o-transcribe` and `gpt-4o-mini-transcribe` (their 2,000-token output ceiling) and 10 minutes for `whisper-1` (so each request finishes inside the default 60 s timeout). When a voice note, audio attachment or voice-mode recording is over the active provider's limit, Hermes first re-encodes it to compact 16 kHz mono AAC, which fits most recordings into one request. If it is still too large, Hermes splits it at pauses, transcribes the pieces in order and joins the text. Local providers have no upload limit and are never split. ffmpeg is required for both steps. + ### TTS Provider Comparison | Provider | Quality | Cost | Latency | Key Required |