From 9bfbeb7cd04d45100423ec9eaed5d85e539bc309 Mon Sep 17 00:00:00 2001 From: tk-pkm111 <133480534+tk-pkm111@users.noreply.github.com> Date: Sat, 29 Aug 2026 17:36:23 +0900 Subject: [PATCH] fix(desktop): mark previous reply spoken before submitting the next voice turn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the voice conversation loop, the normal (silence-detected) submit path set `awaitingSpokenResponseRef` and submitted without consuming the reply that had just finished playing. The turn-drive effect then treated that reply as unspoken, opened speech for it, fell back to whole-text playback and spoke the OLD answer while the model was still thinking — until the new reply arrived and barged in. The barge-in submit path already calls `consumePendingResponse()` before `onSubmit`; give the normal path the same treatment. A submitted turn makes any existing reply stale by definition. Adds a regression test that fails on main (`expected [ 'submit' ] to deeply equal [ 'consume', 'submit' ]`) and passes with the fix. Co-Authored-By: Claude Fable 5 --- ...se-voice-conversation-stale-reply.test.tsx | 138 ++++++++++++++++++ .../composer/hooks/use-voice-conversation.ts | 8 + 2 files changed, 146 insertions(+) create mode 100644 apps/desktop/src/app/chat/composer/hooks/use-voice-conversation-stale-reply.test.tsx diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation-stale-reply.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation-stale-reply.test.tsx new file mode 100644 index 0000000000..d3cb97be06 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation-stale-reply.test.tsx @@ -0,0 +1,138 @@ +import { act, cleanup, renderHook, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { MicRecording } from './use-mic-recorder' +import { useVoiceConversation } from './use-voice-conversation' + +// Regression: after a reply finished playing, the NEXT voice turn must not +// re-speak that reply while the model is thinking. The normal (silence-detected) +// submit path has to mark the previous reply spoken BEFORE submitting — the +// barge path already did — otherwise the turn-drive effect sees `awaiting` plus +// an "unspoken" previous reply and plays it through the whole-text fallback +// until the new answer arrives and barges in. + +const playSpeechText = vi.fn(async (_text: string, _options: unknown) => true) + +vi.mock('@/lib/voice-barge-in', () => ({ + monitorSpeechDuringPlayback: () => vi.fn() +})) + +vi.mock('@/lib/voice-playback', () => ({ + markVoicePlaybackInterrupted: vi.fn(), + playSpeechText: (text: string, options: unknown) => playSpeechText(text, options), + startSpeechStream: vi.fn(async () => null), + stopVoicePlayback: vi.fn() +})) + +vi.mock('@/lib/thinking-sound', () => ({ + startThinkingSound: vi.fn(), + stopThinkingSound: vi.fn() +})) + +const micHandle = { + cancel: vi.fn(), + start: vi.fn(async () => undefined), + stop: vi.fn<() => Promise>(async () => null) +} + +vi.mock('./use-mic-recorder', () => ({ + useMicRecorder: () => ({ handle: micHandle, level: 0, recording: false }) +})) + +vi.mock('@/i18n', () => ({ + useI18n: () => ({ + t: { + notifications: { + voice: { + configureSpeechToText: 'configure STT', + couldNotStartSession: 'could not start', + microphoneFailed: 'mic failed', + playbackFailed: 'playback failed', + transcriptionFailed: 'transcription failed', + unavailable: 'unavailable' + } + } + } + }) +})) + +vi.mock('@/store/notifications', () => ({ + notify: vi.fn(), + notifyError: vi.fn() +})) + +describe('useVoiceConversation stale reply on next turn', () => { + beforeEach(() => { + vi.clearAllMocks() + micHandle.start.mockResolvedValue(undefined) + micHandle.stop.mockResolvedValue(null) + }) + + afterEach(cleanup) + + it('marks the previous reply spoken before submitting so it is not re-spoken while thinking', async () => { + const calls: string[] = [] + + // The previous turn's answer still sits at the bottom of the thread and — + // as in the field — is NOT yet marked spoken when the user's next + // utterance lands. It stays "pending to speak" until consumed. + let staleReplyUnspoken = true + + const consumePendingResponse = vi.fn(() => { + calls.push('consume') + staleReplyUnspoken = false + }) + + // Mirrors the real app: submitting a turn makes the agent busy. + const onBusyChange: { current: (busy: boolean) => void } = { current: () => undefined } + + const onSubmit = vi.fn(async () => { + calls.push('submit') + onBusyChange.current(true) + }) + + const hook = renderHook( + ({ busy }: { busy: boolean }) => + useVoiceConversation({ + busy, + consumePendingResponse, + enabled: true, + onInterrupt: vi.fn(), + onStopWord: vi.fn(), + onSubmit, + onTranscribeAudio: vi.fn(async () => 'what should we do today'), + pendingResponse: () => (staleReplyUnspoken ? { id: 'reply-1', pending: false, text: 'old answer' } : null) + }), + { initialProps: { busy: false } } + ) + + onBusyChange.current = busy => hook.rerender({ busy }) + + await act(async () => { + await hook.result.current.start() + }) + await waitFor(() => expect(hook.result.current.status).toBe('listening')) + + // start() consumes once so an already-present reply is never read when the + // conversation opens; only the submit-time ordering is under test here. + calls.length = 0 + + micHandle.stop.mockResolvedValueOnce({ + audio: new Blob(['q'], { type: 'audio/webm' }), + durationMs: 900, + heardSpeech: true + }) + + await act(async () => { + hook.result.current.stopTurn() + }) + await waitFor(() => expect(onSubmit).toHaveBeenCalledTimes(1)) + + // Consumed strictly before the new prompt goes out (barge-path parity). + expect(calls).toEqual(['consume', 'submit']) + + // And the stale reply never reaches the whole-text fallback player. + await waitFor(() => expect(hook.result.current.status).toBe('thinking')) + expect(playSpeechText).not.toHaveBeenCalled() + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts index ae61ea6f93..82ac8871de 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts @@ -240,6 +240,13 @@ export function useVoiceConversation({ awaitingSpokenResponseRef.current = true dropSpeechSession() + // The reply we just finished playing is stale the moment a new turn + // is submitted. Mark it spoken BEFORE submit (barge path parity): + // otherwise the turn-drive effect sees `awaiting` + an "unspoken" + // previous reply and re-speaks it via the whole-text fallback while + // the model is still thinking — the old answer plays until the new + // one arrives and barges in. + consumePendingResponse() await onSubmit(transcript) setStatus('thinking') } catch (error) { @@ -256,6 +263,7 @@ export function useVoiceConversation({ } }, [ + consumePendingResponse, handle, onFatalError, onSubmit,