Discussions
Using HeyGen api in React+tsx
7 months ago by Dhruti Joshi
My team and I have been trying to use HeyGen api to render the avatar and communicate with it in real-time,
while the audio is working fine, the video is stuck in the first frame, since we would be using HeyGen avatar in integration with our company's agent, it's is something that we require solved.
Below is the code we have -
import StreamingAvatar, { AvatarQuality, StreamingEvents, TaskType } from '@heygen/streaming-avatar';
import React, { useRef, useState } from 'react';
const AvatarPanel: React.FC = () => {
const videoRef = useRef<HTMLVideoElement>(null);
const [avatar, setAvatar] = useState<StreamingAvatar | null>(null);
const [sessionData, setSessionData] = useState<any>(null);
const [input, setInput] = useState('');
const [sessionActive, setSessionActive] = useState(false);
// Helper function to fetch access token
async function fetchAccessToken(): Promise<string> {
// const apiKey = process.env.REACT_APP_HEYGEN_API_KEY;
const apiKey = "API_KEY_HERE";
const response = await fetch('https://api.heygen.com/v1/streaming.create_token', {
method: 'POST',
headers: { 'x-api-key': apiKey },
});
const { data } = await response.json();
return data.token;
}
// Initialize streaming avatar session
async function initializeAvatarSession() {
const token = await fetchAccessToken();
const avatarInstance = new StreamingAvatar({ token });
avatarInstance.on(StreamingEvents.STREAM_READY, handleStreamReady);
avatarInstance.on(StreamingEvents.STREAM_DISCONNECTED, handleStreamDisconnected);
const session = await avatarInstance.createStartAvatar({
quality: AvatarQuality.High,
avatarName: 'Thaddeus_ProfessionalLook2_public',
});
setAvatar(avatarInstance);
setSessionData(session);
setSessionActive(true);
}
// Handle when avatar stream is ready
function handleStreamReady(event: any) {
if (event.detail && videoRef.current) {
videoRef.current.srcObject = event.detail;
videoRef.current.onloadedmetadata = () => {
console.log('Video metadata loaded, playing video.', videoRef?.current?.srcObject);
videoRef.current?.play().catch(console.error);
};
}
}
// Handle stream disconnection
function handleStreamDisconnected() {
if (videoRef.current) {
videoRef.current.srcObject = null;
}
setSessionActive(false);
setAvatar(null);
setSessionData(null);
}
// End the avatar session
async function terminateAvatarSession() {
if (!avatar || !sessionData) return;
await avatar.stopAvatar();
if (videoRef.current) {
videoRef.current.srcObject = null;
}
setAvatar(null);
setSessionData(null);
setSessionActive(false);
}
// Handle speaking event
async function handleSpeak() {
if (avatar && input) {
await avatar.speak({
text: input,
// task_type: TaskType.REPEAT,
});
setInput('');
}
}
return (
<div>
<video ref={videoRef} id="avatarVideo" width={400} autoPlay playsInline>
<track kind="captions" label="Captions" />
</video>
<div>
<button id="startSession" onClick={initializeAvatarSession} disabled={sessionActive}>
Start Session
</button>
<button id="endSession" onClick={terminateAvatarSession} disabled={!sessionActive}>
End Session
</button>
</div>
<div>
<input
id="userInput"
type="text"
value={input}
onChange={(e) => setInput(e.target.value)}
placeholder="Type something to speak"
/>
<button id="speakButton" onClick={handleSpeak} disabled={!sessionActive || !input}>
Speak
</button>
</div>
</div>
);
};
export default AvatarPanel;