Creating a Realtime AI Voice and Text Conversational Session in Javascript/Typescript with Gabber

This guide explains how to leverage Gabber's React SDK to enable spoken or text-based conversations with an AI persona. It builds on the steps we followed in the API setup file.

This tutorial is based on the React SDK and this sample app.

Step 1: Set Up the Conversation Page

Create a new page that initializes a session with the necessary persona and scenario parameters. In this case, we're getting the persona and scenario parameters from the URL params.

import { generateUserToken } from "@/actions";
import { ChatPage } from "@/app/Components/ChatPage";

export default async function Page({ 
  searchParams 
}: { 
  searchParams: { personaId: string; scenarioId: string }
}) {
  const usageToken = await generateUserToken();

  if (!searchParams.personaId || !searchParams.scenarioId) {
    return <div>Missing required parameters</div>;
  }

  return (
    <div className="h-screen">
      <ChatPage 
        personaId={searchParams.personaId} 
        usageToken={usageToken.token}
        scenarioId={searchParams.scenarioId}
      />
    </div>
  );
}

Step 2: Configure the Chat Page

The ChatPage component sets up the session using RealtimeSessionEngineProvider and provides the chat interface via ApiProvider.

"use client"
import { ApiProvider, RealtimeSessionEngineProvider } from "gabber-client-react";
import { ChatContainer } from "./ChatContainer";

interface ChatPageProps {
  personaId: string;
  usageToken: string;
  scenarioId: string;
}

const SFW_LLM = "21892bb9-9809-4b6f-8c3e-e40093069f04"; // This is the default LLM for the SFW, you can use a different LLM by fetching the ID from the Gabber dashboard.

export function ChatPage({ persona, usageToken, scenarioId }: ChatPageProps) {
  return (
    <RealtimeSessionEngineProvider connectionOpts={{
        token: usageToken,
        config: {
          generative: {
            persona: persona.id,
            scenario: scenarioId,
            llm: SFW_LLM
          },
          general: {},
          input: { interruptable: true, parallel_listening: true },
          output: {
            stream_transcript: true,
            speech_synthesis_enabled: true
          }
        },
      }}>
      <ApiProvider usageToken={usageToken}>
        <ChatContainer persona={persona} />
      </ApiProvider>
    </RealtimeSessionEngineProvider>
  );
}

Step 3: Build the Chat Container

The ChatContainer component displays the chat interface, including headers, messages, and input fields.

import React from 'react';
import { ChatHeader } from './ChatHeader';
import { MessageList } from './MessageList';
import { ChatInput } from './ChatInput';
import { useChat } from '../../hooks/useChat';

interface Persona {
  id: string;
  name: string;
  description: string;
  gender: 'male' | 'female';
  voiceId: string;
}

interface ChatContainerProps {
  persona: Persona;
}

export const ChatContainer = ({ persona }: ChatContainerProps) => {
  const { messages, sendMessage, isLoading } = useChat();

  return (
    <div className="flex flex-col h-full bg-gray-50">
      <ChatHeader persona={{
        name: persona.name,
        imageUrl: undefined
      }} />
      <MessageList messages={messages} />
      <ChatInput onSendMessage={sendMessage} isLoading={isLoading} />
    </div>
  );
};

Step 4: Implement Chat Input

The ChatInput component handles text input and microphone toggling for voice input.

import React, { useState } from 'react';
import { Mic, MicOff, Send } from 'lucide-react';
import { useRealtimeSessionEngine } from 'gabber-client-react';

export const ChatInput = ({ onSendMessage, isLoading }: { onSendMessage: (message: string) => void; isLoading?: boolean }) => {
  const [message, setMessage] = useState('');
  const { sendChatMessage, isRecording, setMicrophoneEnabled, microphoneEnabled, startAudio } = useRealtimeSessionEngine();

  const handleSubmit = async (e: React.FormEvent) => {
    e.preventDefault();
    if (message.trim()) {
      try {
        await sendChatMessage({ text: message });
        onSendMessage(message);
        setMessage('');
      } catch (error) {
        console.error('Error sending message:', error);
      }
    }
  };

  const toggleMicrophone = async () => {
    try {
      if (!microphoneEnabled) {
        await startAudio();
      }
      setMicrophoneEnabled(!microphoneEnabled);
    } catch (error) {
      console.error('Error toggling microphone:', error);
    }
  };

  return (
    <form onSubmit={handleSubmit} className="p-4 bg-white border-t">
      <div className="flex items-center space-x-2">
        <button
          type="button"
          onClick={toggleMicrophone}
          className={`p-2 rounded-full transition-colors ${
            microphoneEnabled ? 'bg-red-500 text-white hover:bg-red-600' : 'bg-gray-200 text-gray-600 hover:bg-gray-300'
          }`}
        >
          {microphoneEnabled ? <MicOff className="w-5 h-5" /> : <Mic className="w-5 h-5" />}
        </button>
        <input
          type="text"
          value={message}
          onChange={(e) => setMessage(e.target.value)}
          placeholder={microphoneEnabled ? "Listening..." : "Type a message..."}
          disabled={microphoneEnabled}
          className="flex-1 p-2 border rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:bg-gray-100 disabled:text-gray-500 text-gray-900"
        />
        <button
          type="submit"
          disabled={isLoading || !message.trim()}
          className="p-2 bg-blue-500 text-white rounded-full disabled:opacity-50 hover:bg-blue-600"
        >
          <Send className="w-5 h-5" />
        </button>
      </div>

      {isRecording && (
        <div className="flex items-center gap-2 mt-2">
          <div className="w-2 h-2 bg-red-500 rounded-full animate-pulse" />
          <span className="text-sm text-gray-500">Recording...</span>
        </div>
      )}
    </form>
  );
};

Step 5: Render Messages

Create 2 components, Message and MessageList, to display chat messages in the conversation interface.

Message Component

The Message component renders individual chat messages with styling based on the sender (agent or user).

import React from 'react';
import { formatRelativeTime } from '../../utils/dateUtils';

interface MessageProps {
  message: {
    content: string;
    isAgent: boolean;
    timestamp: Date;
  };
}

export const Message = ({ message }: MessageProps) => {
  const { content, isAgent, timestamp } = message;
  
  return (
    <div className={`flex ${isAgent ? 'justify-start' : 'justify-end'}`}>
      <div
        className={`max-w-[70%] rounded-lg px-4 py-2 ${
          isAgent
            ? 'bg-white text-gray-800'
            : 'bg-blue-500 text-white'
        }`}
      >
        <p className="text-sm">{content}</p>
        <span className="text-xs opacity-75 mt-1 block">
          {formatRelativeTime(timestamp)}
        </span>
      </div>
    </div>
  );
};

Message List Component

The MessageList component displays a list of messages and ensures auto-scrolling behavior.

import React, { useRef, useEffect } from 'react';
import { Message } from './Message';
import { useAutoScroll } from '../../hooks/useAutoScroll';

interface MessageListProps {
  messages: Array<{
    id: string;
    content: string;
    isAgent: boolean;
    timestamp: Date;
  }>;
}

export const MessageList = ({ messages }: MessageListProps) => {
  const messagesEndRef = useRef<HTMLDivElement>(null);
  const { scrollToBottom, isAtBottom } = useAutoScroll(messagesEndRef as React.RefObject<HTMLElement>);

  useEffect(() => {
    if (isAtBottom) {
      scrollToBottom();
    }
  }, [messages, isAtBottom, scrollToBottom]);

  return (
    <div className="flex-1 overflow-y-auto p-4 space-y-4">
      {messages.map((message) => (
        <Message key={message.id} message={message} />
      ))}
      <div ref={messagesEndRef} />
    </div>
  );
};

With these steps, you can build a dynamic chat interface powered by Gabber's AI capabilities.