Skip to main content

Client SDK Realtime Session

Realtime sessions integrated into your application use one of Gabber's Client SDKs. There are two steps involved:

  1. Generating connection details
  2. Connecting to a Realtime Session

Generating Connection Details​

Realtime sessions can be started using the startRealtimeSession API.

A successful response contains connection_details which are used by a Gabber Client SDK to connect to a WebRTC session.

example connection details:

{
    "url": "<connection url to initiate the Gabber WebRTC session>",
    "token": "<connection token to authenticate the Gabber WebRTC"
}

Here's a minimal example Express server that starts a session and returns connection details:

import express, { Request, Response } from 'express';
import axios from 'axios';
import dotenv from 'dotenv';
import cors from 'cors';

dotenv.config();

const app = express();
const port = 4000;

var corsOptions = {
  origin: 'http://localhost:3000', // Allow example app to make requests
}

app.use(cors(corsOptions));

const axiosInstance = axios.create({
  baseURL: 'https://app.gabber.dev',
  headers: {
    'x-api-key': process.env.GABBER_API_KEY,
    'Content-Type': 'application/json',
  },
  timeout: 5000, // optional: add a timeout
});

// Middleware to parse JSON request bodies
app.use(express.json());

// Endpoint to generate a token
app.get('/connection_details', async (req: Request, res: Response) => {
  try {
    const response = await axiosInstance.post('api/v1/realtime/start', {
        config: {
            general: {
                time_limit_s: 120, // (optional)
                save_messages: true
            },
            input: {
                interruptable: true, // Whether the human can interrupt the AI
                parallel_listening: true // Whether the AI will listen while it's speaking (only relevant when interruption is false)
            },
            generative: {
                llm: "gabber_llm_id", // (required) Which LLM to use,
                voice_override: "gabber_voice_id", // (optional). Voice to use for the realtime session. Will override the persona voice is using a persona.
                persona: "gabber_persona_id", // (optional). Persona to use for this realtime session.
                scenario: "gabber_scenario_id", // (optional). Scenario to use for this realtime session.
                context: "gabber_memory_context_id", // (optional). Memory context to use
                tool_definitions: [
                    "gabber_tool_id_1",
                    "gabber_tool_id_2",
                    //etc.
                ] // List of Gabber Tools available to this session
            },
            output: {
                stream_transcript: true, // Stream the AI response transcript
                speech_synthesis_enabled: true, // Whether the AI produces audio or just text
                answer_message: null // The message the AI will respond with. If null, the human must speak first before getting a response.
            }
        },
    });

    return res.json(response.data.connection_details);
  } catch (error) {
    console.error(error);
    return res.status(500).json({ error: 'An error occurred' });
  }
});

// Start the server
app.listen(port, () => {
    console.log(`Server is running on http://localhost:${port}`);
});

Connecting to a Realtime Session​

A Gabber Client SDK is used to connect to a Realtime Session once connection details have been generated.

Install SDK​

npm install gabber-client-core

If using react, install the React SDK as well:

npm install gabber-client-react

Connect​

import React, { useEffect, useState } from 'react';
import { RealtimeSessionEngineProvider } from 'gabber-client-react';
import { SDKConnectOptions } from 'gabber-client-core';

function App() {
    const [connectionDetails, setConnectionDetails] = useState<RealtimeSessionConnectionDetails | null>(null);

    const connect = async () => {
        const response = await fetch('http://localhost:4000/connection_details');
        const connectionDetails = await response.json();
        setConnectionDetails(connectionDetails);
    }

    if (!connectionDetails) {
        return (
            <div>
                <button onClick={connect}></button>
            </div>
        );
        return null
    }

    return (
        <RealtimeSessionEngineProvider connectionOpts={{connection_details: connectionDetails}}>
            <AppInner />
        </RealtimeSessionEngineProvider>
    )
}

function AppInner() {
    const {
        id,
        agentState,
        connectionState,
        lastError,
        agentVolumeBands,
        agentVolume,
        userVolumeBands,
        userVolume,
        remainingSeconds,
        messages,
        sendChatMessage,
        transcription,
        microphoneEnabled,
        setMicrophoneEnabled,
        canPlayAudio,
        startAudio,
    } = useRealtimeSessionEngine();

    return (
        <div>
            <button onClick={async () => {
                await setMicrophoneEnabled(!microphoneEnabled);
            }}>Toggle Mute</button>
        </div>
    )
}