Speech to speech: Mute functionality

CHANGELOG.md

@@ -151,6 +151,7 @@ Breaking changes in this release:
 - 👷🏻 Added `npm run build-browser` script for building test harness package only, in PR [#5667](https://github.com/microsoft/BotFramework-WebChat/pull/5667), by [@compulim](https://github.com/compulim)
 - Added pull-based capabilities system for dynamically discovering adapter capabilities at runtime, in PR [#5679](https://github.com/microsoft/BotFramework-WebChat/pull/5679), by [@pranavjoshi001](https://github.com/pranavjoshi001)
 - Added Speech-to-Speech (S2S) support for real-time voice conversations, in PR [#5654](https://github.com/microsoft/BotFramework-WebChat/pull/5654), by [@pranavjoshi](https://github.com/pranavjoshi001)
+- Added core mute/unmute functionality for speech-to-speech via `useRecorder` hook (silent chunks keep server connection alive), in PR [#5688](https://github.com/microsoft/BotFramework-WebChat/pull/5688), by [@pranavjoshi](https://github.com/pranavjoshi001)
 
 ### Changed
 

packages/api/src/boot/hook.ts

@@ -39,6 +39,7 @@ export {
   useMarkActivityAsSpoken,
   useMarkActivityKeyAsRead,
   useMarkAllAsAcknowledged,
+  useMuteVoice,
   useNotifications,
   usePerformCardAction,
   usePonyfill,
@@ -74,6 +75,7 @@ export {
   useTrackException,
   useTrackTiming,
   useUIState,
+  useUnmuteVoice,
   useUserID,
   useUsername,
   useVoiceSelector,

packages/api/src/hooks/Composer.tsx

@@ -13,6 +13,7 @@ import {
   dismissNotification,
   emitTypingIndicator,
   markActivity,
+  muteVoiceRecording,
   postActivity,
   sendEvent,
   sendFiles,
@@ -35,6 +36,7 @@ import {
   stopSpeakingActivity,
   stopVoiceRecording,
   submitSendBox,
+  unmuteVoiceRecording,
   type DirectLineJSBotConnection,
   type GlobalScopePonyfill,
   type OneOrMany,
@@ -381,6 +383,14 @@ const ComposerCore = ({
     dispatch(stopVoiceRecording());
   }, [dispatch, voiceHandlers]);
 
+  const muteVoice = useCallback(() => {
+    dispatch(muteVoiceRecording());
+  }, [dispatch]);
+
+  const unmuteVoice = useCallback(() => {
+    dispatch(unmuteVoiceRecording());
+  }, [dispatch]);
+
   const patchedLocalizedStrings = useMemo(
     () => mergeStringsOverrides(getAllLocalizedStrings()[normalizeLanguage(locale)], locale, overrideLocalizedStrings),
     [locale, overrideLocalizedStrings]
@@ -563,6 +573,7 @@ const ComposerCore = ({
       language: locale,
       localizedGlobalizeState: [localizedGlobalize],
       localizedStrings: patchedLocalizedStrings,
+      muteVoice,
       onTelemetry,
       renderMarkdown,
       scrollToEndButtonRenderer,
@@ -575,6 +586,7 @@ const ComposerCore = ({
       trackDimension,
       typingIndicatorRenderer: patchedTypingIndicatorRenderer,
       uiState,
+      unmuteVoice,
       userID,
       username
     }),
@@ -585,6 +597,7 @@ const ComposerCore = ({
       hoistedDispatchers,
       locale,
       localizedGlobalize,
+      muteVoice,
       onTelemetry,
       patchedActivityStatusRenderer,
       patchedAttachmentForScreenReaderRenderer,
@@ -604,6 +617,7 @@ const ComposerCore = ({
       telemetryDimensionsRef,
       trackDimension,
       uiState,
+      unmuteVoice,
       userID,
       username
     ]

packages/api/src/hooks/index.ts

@@ -37,6 +37,7 @@ import useLocalizer from './useLocalizer';
 import useMarkActivityAsSpoken from './useMarkActivityAsSpoken';
 import useMarkActivityKeyAsRead from './useMarkActivityKeyAsRead';
 import useMarkAllAsAcknowledged from './useMarkAllAsAcknowledged';
+import useMuteVoice from './useMuteVoice';
 import useNotifications from './useNotifications';
 import usePerformCardAction from './usePerformCardAction';
 import usePonyfill from './usePonyfill';
@@ -71,6 +72,7 @@ import useTrackEvent from './useTrackEvent';
 import useTrackException from './useTrackException';
 import useTrackTiming from './useTrackTiming';
 import useUIState from './useUIState';
+import useUnmuteVoice from './useUnmuteVoice';
 import useUserID from './useUserID';
 import useUsername from './useUsername';
 import useVoiceSelector from './useVoiceSelector';
@@ -119,6 +121,7 @@ export {
   useMarkActivityAsSpoken,
   useMarkActivityKeyAsRead,
   useMarkAllAsAcknowledged,
+  useMuteVoice,
   useNotifications,
   usePerformCardAction,
   usePonyfill,
@@ -153,6 +156,7 @@ export {
   useTrackException,
   useTrackTiming,
   useUIState,
+  useUnmuteVoice,
   useUserID,
   useUsername,
   useVoiceSelector,

packages/api/src/hooks/internal/WebChatAPIContext.ts

@@ -44,6 +44,7 @@ export type WebChatAPIContextType = {
   localizedGlobalizeState?: PrecompiledGlobalize[];
   localizedStrings?: { [language: string]: LocalizedStrings };
   markActivity?: (activity: { id: string }, name: string, value?: any) => void;
+  muteVoice?: () => void;
   onCardAction?: PerformCardAction;
   onTelemetry?: (event: TelemetryMeasurementEvent) => void;
   postActivity?: (activity: WebChatActivity) => Observable<string>;
@@ -81,6 +82,7 @@ export type WebChatAPIContextType = {
   trackDimension?: (name: string, data: any) => void;
   typingIndicatorRenderer?: any; // TODO
   uiState: 'blueprint' | 'disabled' | undefined;
+  unmuteVoice?: () => void;
   userID?: string;
   username?: string;
 };

packages/api/src/hooks/useMuteVoice.ts

@@ -0,0 +1,9 @@
+import useWebChatAPIContext from './internal/useWebChatAPIContext';
+
+/**
+ * Hook to mute voice mode (stops microphone input but keeps connection alive with silent chunks).
+ * The session remains active and can be unmuted to resume listening.
+ */
+export default function useMuteVoice(): () => void {
+  return useWebChatAPIContext().muteVoice;
+}

packages/api/src/hooks/useUnmuteVoice.ts

@@ -0,0 +1,9 @@
+import useWebChatAPIContext from './internal/useWebChatAPIContext';
+
+/**
+ * Hook to unmute voice mode (resumes microphone input after muting).
+ * This reactivates speech-to-speech listening.
+ */
+export default function useUnmuteVoice(): () => void {
+  return useWebChatAPIContext().unmuteVoice;
+}

packages/api/src/providers/SpeechToSpeech/private/VoiceRecorderBridge.tsx

@@ -11,6 +11,7 @@ export function VoiceRecorderBridge(): null {
   const [voiceState] = useVoiceState();
   const postVoiceActivity = usePostVoiceActivity();
 
+  const muted = voiceState === 'muted';
   // Derive recording state from voiceState - recording is active when not idle
   const recording = voiceState !== 'idle';
 
@@ -29,7 +30,13 @@ export function VoiceRecorderBridge(): null {
     [postVoiceActivity]
   );
 
-  const { record } = useRecorder(handleAudioChunk);
+  const { record, mute } = useRecorder(handleAudioChunk);
+
+  useEffect(() => {
+    if (muted) {
+      return mute();
+    }
+  }, [mute, muted]);
 
   useEffect(() => {
     if (recording) {

packages/api/src/providers/SpeechToSpeech/private/useRecorder.spec.tsx

@@ -36,13 +36,16 @@ const mockWorkletNode = {
   port: mockWorkletPort
 };
 
+const mockSourceNode = {
+  connect: jest.fn(),
+  disconnect: jest.fn()
+};
+
 const mockAudioContext = {
   audioWorklet: {
     addModule: jest.fn().mockResolvedValue(undefined)
   },
-  createMediaStreamSource: jest.fn(() => ({
-    connect: jest.fn()
-  })),
+  createMediaStreamSource: jest.fn(() => mockSourceNode),
   destination: {},
   resume: jest.fn().mockResolvedValue(undefined),
   state: 'running'
@@ -218,4 +221,74 @@ describe('useRecorder', () => {
       });
     });
   });
+
+  test('should return mute function', () => {
+    render(<HookApp onAudioChunk={onAudioChunk} />);
+    expect(typeof hookData?.mute).toBe('function');
+  });
+
+  test('should send MUTE command and stop media stream when mute is called', async () => {
+    render(<HookApp onAudioChunk={onAudioChunk} />);
+
+    // Start recording first
+    act(() => {
+      hookData?.record();
+    });
+
+    await waitFor(() => {
+      expect(mockWorkletPort.postMessage).toHaveBeenCalledWith({ command: 'START' });
+    });
+
+    // Clear mocks to isolate mute behavior
+    mockWorkletPort.postMessage.mockClear();
+    mockTrack.stop.mockClear();
+    mockSourceNode.disconnect.mockClear();
+
+    // Call mute
+    act(() => {
+      hookData?.mute();
+    });
+
+    // Should send MUTE command to worklet
+    expect(mockWorkletPort.postMessage).toHaveBeenCalledWith({ command: 'MUTE' });
+    // Should stop media stream tracks (mic indicator OFF)
+    expect(mockTrack.stop).toHaveBeenCalledTimes(1);
+    // Should disconnect source node
+    expect(mockSourceNode.disconnect).toHaveBeenCalledTimes(1);
+  });
+
+  test('should return unmute function from mute() that sends UNMUTE and restarts media stream', async () => {
+    render(<HookApp onAudioChunk={onAudioChunk} />);
+
+    // Start recording first
+    act(() => {
+      hookData?.record();
+    });
+
+    await waitFor(() => {
+      expect(mockWorkletPort.postMessage).toHaveBeenCalledWith({ command: 'START' });
+    });
+
+    // Call mute and get unmute function
+    let unmute: (() => void) | undefined;
+    act(() => {
+      unmute = hookData?.mute();
+    });
+
+    // Clear mocks to isolate unmute behavior
+    mockWorkletPort.postMessage.mockClear();
+    mockMediaDevices.getUserMedia.mockClear();
+
+    // Call unmute
+    act(() => {
+      unmute?.();
+    });
+
+    // Should send UNMUTE command to worklet
+    expect(mockWorkletPort.postMessage).toHaveBeenCalledWith({ command: 'UNMUTE' });
+    // Should restart media stream
+    await waitFor(() => {
+      expect(mockMediaDevices.getUserMedia).toHaveBeenCalledTimes(1);
+    });
+  });
 });