Discussions

Ask a Question
Back to all

Using HeyGen api in React+tsx

My team and I have been trying to use HeyGen api to render the avatar and communicate with it in real-time,
while the audio is working fine, the video is stuck in the first frame, since we would be using HeyGen avatar in integration with our company's agent, it's is something that we require solved.

Below is the code we have -

import StreamingAvatar, { AvatarQuality, StreamingEvents, TaskType } from '@heygen/streaming-avatar';
import React, { useRef, useState } from 'react';

const AvatarPanel: React.FC = () => {
  const videoRef = useRef<HTMLVideoElement>(null);
  const [avatar, setAvatar] = useState<StreamingAvatar | null>(null);
  const [sessionData, setSessionData] = useState<any>(null);
  const [input, setInput] = useState('');
  const [sessionActive, setSessionActive] = useState(false);

  // Helper function to fetch access token
  async function fetchAccessToken(): Promise<string> {
    // const apiKey = process.env.REACT_APP_HEYGEN_API_KEY;
    const apiKey = "API_KEY_HERE";
    const response = await fetch('https://api.heygen.com/v1/streaming.create_token', {
      method: 'POST',
      headers: { 'x-api-key': apiKey },
    });
    const { data } = await response.json();
    return data.token;
  }

  // Initialize streaming avatar session
  async function initializeAvatarSession() {
    const token = await fetchAccessToken();
    const avatarInstance = new StreamingAvatar({ token });

    avatarInstance.on(StreamingEvents.STREAM_READY, handleStreamReady);
    avatarInstance.on(StreamingEvents.STREAM_DISCONNECTED, handleStreamDisconnected);

    const session = await avatarInstance.createStartAvatar({
      quality: AvatarQuality.High,
      avatarName: 'Thaddeus_ProfessionalLook2_public',
    });

    setAvatar(avatarInstance);
    setSessionData(session);
    setSessionActive(true);
  }

  // Handle when avatar stream is ready
  function handleStreamReady(event: any) {
    if (event.detail && videoRef.current) {
      videoRef.current.srcObject = event.detail;
      videoRef.current.onloadedmetadata = () => {
        console.log('Video metadata loaded, playing video.', videoRef?.current?.srcObject);
        videoRef.current?.play().catch(console.error);
      };
    }
  }

  // Handle stream disconnection
  function handleStreamDisconnected() {
    if (videoRef.current) {
      videoRef.current.srcObject = null;
    }
    setSessionActive(false);
    setAvatar(null);
    setSessionData(null);
  }

  // End the avatar session
  async function terminateAvatarSession() {
    if (!avatar || !sessionData) return;
    await avatar.stopAvatar();
    if (videoRef.current) {
      videoRef.current.srcObject = null;
    }
    setAvatar(null);
    setSessionData(null);
    setSessionActive(false);
  }

  // Handle speaking event
  async function handleSpeak() {
    if (avatar && input) {
      await avatar.speak({
        text: input,
        // task_type: TaskType.REPEAT,
      });
      setInput('');
    }
  }

  return (
    <div>
      <video ref={videoRef} id="avatarVideo" width={400} autoPlay playsInline>
        <track kind="captions" label="Captions" />
      </video>
      <div>
        <button id="startSession" onClick={initializeAvatarSession} disabled={sessionActive}>
          Start Session
        </button>
        <button id="endSession" onClick={terminateAvatarSession} disabled={!sessionActive}>
          End Session
        </button>
      </div>
      <div>
        <input
          id="userInput"
          type="text"
          value={input}
          onChange={(e) => setInput(e.target.value)}
          placeholder="Type something to speak"
        />
        <button id="speakButton" onClick={handleSpeak} disabled={!sessionActive || !input}>
          Speak
        </button>
      </div>
    </div>
  );
};

export default AvatarPanel;