Hello I am using button click It is you NextJS SDK We interrupt it in the middle of the speech Can you look into our code? ```node 'use client'; import { RefObject, useEffect, useRef, useState } from "react"; import StreamingAvatar, { StreamingEvents, AvatarQuality, VoiceEmotion, TaskMode, TaskType, type StartAvatarResponse, } from "@heygen/streaming-avatar"; import { useGlobalVaribles } from "@/providers/GlobalVaribles"; import { Card } from "@/data/cards"; // import { useSpeechRecognition } from "../speechToText"; // import { startRecording, stopRecording } from "../speechToText"; interface UseHeygenAvatarProps { card: Card; knowledgeId: string | undefined; language: string; setLanguage: (lang: string) => void; setCurrentLoadingStep: (step: number) => void; } export function useHeygenAvatar({ card, knowledgeId, language, setCurrentLoadingStep, setLanguage, }: UseHeygenAvatarProps) { const [stream, setStream] = useState(); const [data, setData] = useState(); const [captionText, setCaptionText] = useState(""); const [isUserTalking, setIsUserTalking] = useState(false); const [isAvatarTalking, setIsAvatarTalking] = useState(false); const [isMicAllowed, setIsMicAllowed] = useState(false); const [isMicLoading, setIsMicLoading] = useState(false); const [isLoadingSession, setIsLoadingSession] = useState(false); const [isLoadingRepeat, setIsLoadingRepeat] = useState(false); const [debug, setDebug] = useState(); const [chatMode, setChatMode] = useState("text_mode"); const avatar = useRef(null); const heygenTimeoutRef = useRef(null); const sessionStarted = useRef(false); // Recording Variables const streamRef = useRef(null); const recRef = useRef(null); const chunksRef = useRef([]); const heygenTimeoutPlaceHolder = 1000 * 60 * 10; // 10 mins const sessionId = useRef(null); const { isUsingLangflow, setStreamIsVideo, setAvatarIsGivingIntro } = useGlobalVaribles() const EndSessionAfterAmountOfTime = (time: number) => { if (heygenTimeoutRef.current) { clearTimeout(heygenTimeoutRef.current); } heygenTimeoutRef.current = setTimeout(async () => { await endSession(); }, time); }; const fetchAccessToken = async () => { const res = await fetch("/api/get-access-token", { method: "POST", headers: { "Content-Type": "application/json" }, }); const data = await res.json(); if (!data.token) throw new Error("Fetch"); return data.token; }; const getHeygenRemainingQuota = async () => { const res = await fetch("/api/get-remaining-qouta"); const data = await res.json(); // if (!data.token) throw new Error("Fetch"); return data.remainingQuotaMinutes; }; /** * Gets the preferred MIME type for MediaRecorder */ const getPreferredMimeType = (): string => { const preferred = "audio/webm;codecs=opus"; return MediaRecorder.isTypeSupported(preferred) ? preferred : (MediaRecorder as any).defaultMimeType || ""; }; /** * Sets up the MediaRecorder for Arabic audio recording */ const setupArabicMicrophone = async (): Promise => { console.log("Starting microphone for Arabic"); const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1 }, }); streamRef.current = stream; const mimeType = getPreferredMimeType(); const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); recRef.current = recorder; chunksRef.current = []; recorder.ondataavailable = (e: BlobEvent) => { if (e.data && e.data.size) { chunksRef.current.push(e.data); } }; recorder.onerror = (e) => { console.error("Media Recorder Error: ", e); try { recorder.stop(); } catch (error) { console.error("Error stopping recorder:", error); } }; // Start recording recorder.start(); }; /** * Stops the recorder and waits for it to finish, ensuring all data is captured */ const stopRecorder = async (recorder: MediaRecorder): Promise => { // Save mimeType before stopping const mimeType = recorder.mimeType; // Wait for recorder to stop const waitRecorderStopped = new Promise((resolve) => { recorder.onstop = () => { resolve(mimeType); }; }); try { // Request any remaining data before stopping (ensures final data is captured) if (recorder.state === "recording") { recorder.requestData(); // Small delay to ensure data event is processed await new Promise(resolve => setTimeout(resolve, 50)); } recorder.stop(); } catch (error) { console.error("Error stopping recorder:", error); // Return mimeType even if stop fails return mimeType; } return await waitRecorderStopped; }; /** * Cleans up recording resources (stream, recorder, chunks) */ const cleanupArabicRecording = (): void => { if (streamRef.current) { streamRef.current.getTracks().forEach((track) => track.stop()); streamRef.current = null; } recRef.current = null; chunksRef.current = []; }; /** * Creates a Blob from recorded chunks */ const createAudioBlob = (chunks: BlobPart[], mimeType: string): Blob => { if (chunks.length === 0) { console.warn("No audio chunks to create blob from"); return new Blob([], { type: mimeType }); } return new Blob(chunks, { type: mimeType }); }; /** * Stops Arabic recording and processes the transcription */ const stopArabicRecording = async (): Promise => { const recorder = recRef.current; if (!recorder) { console.warn("No recorder found to stop"); return; } if (recorder.state === "inactive") { console.warn("Recorder is already stopped"); // cleanupArabicRecording(); return; } console.log("Closing microphone for Arabic"); // Stop recorder and get mimeType const mimeType = await stopRecorder(recorder); // Get chunks and create blob before cleanup const chunks = [...chunksRef.current]; const audioBlob = createAudioBlob(chunks, mimeType); // Cleanup resources // cleanupArabicRecording(); // Only transcribe if we have audio data if (audioBlob.size > 0) { try { const transcription = await transcribeAudio(audioBlob, language); if (transcription) { console.log("Transcription:", transcription); const res = await fetch("/api/chatgpt", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ prompt: transcription + " Reply in:" + language }), }); // Get the stream reader const reader = res.body?.getReader(); if (!reader) throw new Error("No reader available"); const decoder = new TextDecoder(); let buffer = ""; let fullText = ""; // Continuously read the stream while (true) { const { done, value } = await reader.read(); if (done) break; const chunk = decoder.decode(value, { stream: true }); buffer += chunk; let lastSentenceEnd = Math.max( buffer.lastIndexOf("."), buffer.lastIndexOf("!"), buffer.lastIndexOf("?"), ); if (lastSentenceEnd !== -1) { const toSpeak = buffer.slice(0, lastSentenceEnd + 1); buffer = buffer.slice(lastSentenceEnd + 1); fullText += toSpeak; if (toSpeak.trim()) { await handleSpeak(toSpeak, TaskType.REPEAT); } } } // console.log("Speak:"); } } catch (error) { console.error("Error transcribing audio:", error); setDebug(error instanceof Error ? error.message : "Failed to transcribe audio"); } } else { console.warn("No audio data recorded"); } }; /** * Handles microphone permission for Arabic language */ const handleArabicMicPermission = async (allow: boolean): Promise => { if (allow) { await setupArabicMicrophone(); } else { await stopArabicRecording(); } }; /** * Handles microphone permission for non-Arabic languages */ const handleStandardMicPermission = (allow: boolean): void => { if (allow) { avatar.current?.unmuteInputAudio(); } else { avatar.current?.muteInputAudio(); } }; /** * Main handler for microphone permission */ const handleMicPermission = async (allow: boolean) => { setIsMicLoading(true); try { if (allow) { handleInterrupt(); } if (!allow) { setIsMicAllowed(false); setIsMicLoading(false); } if (language === "ar") { console.log(`Allowing microphone for Arabic: ${allow}`); await handleArabicMicPermission(allow); } else { handleStandardMicPermission(allow); } setIsMicAllowed(allow); } catch (error) { console.error("Error handling mic permission:", error); setDebug(error instanceof Error ? error.message : "Unknown error"); } finally { } }; async function handleInterrupt() { if (!avatar.current) { console.log("Avatar API not initialized"); return; } console.log("Interrupting avatar"); // Check if stream is ready (connection is established) if (stream) { await avatar.current.interrupt(); } else { console.log('WebSocket not ready yet, skipping interrupt'); } } const startSession = async () => { console.log("Start Session") sessionStarted.current = true; setIsLoadingSession(true); setCurrentLoadingStep(0); try { const quota = await getHeygenRemainingQuota(); if (quota < 25000) { setDebug("Not enough quota"); return; } setCurrentLoadingStep(1); const token = await fetchAccessToken(); console setCurrentLoadingStep(2); avatar.current = new StreamingAvatar({ token }); avatar.current.on(StreamingEvents.AVATAR_START_TALKING, async () => { handleMicPermission(false); setIsAvatarTalking(true) EndSessionAfterAmountOfTime(heygenTimeoutPlaceHolder); }); avatar.current.on(StreamingEvents.AVATAR_TALKING_MESSAGE, (message) => { // setCaptionText((prev) => `${prev} ${message.detail.message}`); EndSessionAfterAmountOfTime(message.timeStamp + 1000 * 12); console.log(message.detail.message); }); avatar.current.on(StreamingEvents.AVATAR_STOP_TALKING, async () => { avatar.current?.startVoiceChat({ isInputAudioMuted: true }); setIsAvatarTalking(false) setAvatarIsGivingIntro(false) setCaptionText(""); EndSessionAfterAmountOfTime(heygenTimeoutPlaceHolder); }); avatar.current.on(StreamingEvents.STREAM_READY, (event) => { setStream(event.detail); EndSessionAfterAmountOfTime(heygenTimeoutPlaceHolder); }); avatar.current.on(StreamingEvents.USER_START, () => { console.log("User Started Talking"); // handleInterrupt(); setIsUserTalking(true); setCaptionText(""); }); avatar.current.on(StreamingEvents.USER_STOP, () => { setIsUserTalking(false); }); avatar.current.on(StreamingEvents.STREAM_DISCONNECTED, async () => { await endSession(); }); setCurrentLoadingStep(3); let lang = language; if (lang == "ar") { lang = "en"; } const res = await avatar.current.createStartAvatar({ avatarName: card.id, knowledgeId, quality: AvatarQuality.High, voice: { rate: 1.1, emotion: VoiceEmotion.EXCITED, }, language: lang, disableIdleTimeout: true, }); setCurrentLoadingStep(4); await avatar.current?.startVoiceChat({ isInputAudioMuted: true }); setData(res); console.log(res) setChatMode("voice_mode"); setCurrentLoadingStep(5); await handleMicPermission(false); // await wait(1500); setDebug(""); setCurrentLoadingStep(6); setStreamIsVideo(false) } catch (error) { console.error("StartSession Error:", error); setDebug( error instanceof Error ? error.message : "Failed to start the session" ); } finally { setIsLoadingSession(false); } }; const handleSpeak = async (text: string, talkType?: TaskType) => { setIsLoadingRepeat(true); if (isAvatarTalking) handleInterrupt(); // if text = "Hello, How are you" // result the same "Hello, How are you" try { await avatar.current?.speak({ text, taskType: talkType, taskMode: TaskMode.ASYNC, }); } catch (error) { setDebug(error instanceof Error ? error.message : "Unknown error"); } finally { setIsLoadingRepeat(false); } }; const endSession = async () => { avatar.current?.closeVoiceChat(); avatar.current?.stopAvatar(); avatar.current = null; setStream(undefined); setCaptionText(""); setIsMicAllowed(false); setStreamIsVideo(true) sessionStarted.current = false; }; const transcribeAudio = async (audioBlob: Blob, language = "en"): Promise => { console.log("TranscribeAudio.."); const formData = new FormData(); formData.append("audio", audioBlob, "recording.wav"); formData.append("language", language); try { const res = await fetch("/api/transcribe", { method: "POST", body: formData, }); const data: { text?: string; error?: string } = await res.json(); if (!res.ok) { console.error("Transcription failed:", data.error || "Unknown error"); return null; } const transcription = data.text || ""; console.log("Transcription:", transcription); return transcription; } catch (err) { console.error("Request failed:", err); return null; } }; return { stream, avatar, data, captionText, chatMode, isMicAllowed, isMicLoading, isUserTalking, isLoadingSession, isLoadingRepeat, handleInterrupt, debug, startSession, endSession, restartSession, handleSpeak, handleMicPermission, startWhisperMic, stopWhisperMic, setCaptionText, setChatMode, }; } ```

on Interrupting the avatar I gte: Failed to execute 'send' on 'WebSocket': Still in CONNECTING state