Building an AI Contact Center with FreeSWITCH and OpenAI Realtime API

<!-- Route inbound DID to AI agent context -->
<context name="public">
  <extension name="ai_agent_did">
    <!-- Match your DID number -->
    <condition field="destination_number" expression="^(18005551234)$">
      <!-- Set audio params for OpenAI Realtime (16kHz mono required) -->
      <action application="set" data="media_bug_answer_req=true"/>
      <action application="set" data="record_sample_rate=16000"/>

      <!-- Answer the call -->
      <action application="answer"/>

      <!-- Play a brief hold tone while bridge connects (optional) -->
      <action application="playback" data="silence_stream://500"/>

      <!-- Stream audio to our Node.js AI bridge via mod_audio_stream -->
      <!-- mod_audio_stream sends raw PCM to ws://localhost:3001/audio -->
      <action application="audio_stream" data="wss://localhost:3001/audio async full"/>

      <!-- If bridge hangs up cleanly, execute hangup -->
      <action application="hangup"/>
    </condition>
  </extension>
</context>

<!-- Sofia SIP profile — accept inbound SIP trunk calls -->
<!-- In conf/sip_profiles/external.xml: -->
<!-- <param name="context" value="public"/> -->

# Install build dependencies
apt-get install -y libjson-c-dev libwebsockets-dev

# Clone and build mod_audio_stream
git clone https://github.com/nicholasgasior/mod_audio_stream
cd mod_audio_stream
make
make install

# Load in modules.conf.xml
# <load module="mod_audio_stream"/>

# Reload modules in fs_cli
freeswitch> reload mod_audio_stream

const WebSocket = require('ws');
const OpenAI = require('openai');

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

// WebSocket server — accepts connections from FreeSWITCH mod_audio_stream
const wss = new WebSocket.Server({ port: 3001, path: '/audio' });

wss.on('connection', async (fsSocket) => {
  console.log('[Bridge] FreeSWITCH connected');

  // Open OpenAI Realtime session
  const realtimeWs = new WebSocket(
    'wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01',
    {
      headers: {
        Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
        'OpenAI-Beta': 'realtime=v1',
      },
    }
  );

  realtimeWs.on('open', () => {
    console.log('[Bridge] OpenAI Realtime connected');

    // Configure the session
    realtimeWs.send(JSON.stringify({
      type: 'session.update',
      session: {
        modalities: ['text', 'audio'],
        instructions: `You are a helpful customer service agent for Acme Corp.
Be concise — phone responses should be under 30 words unless the caller asks for detail.
You have access to tools: lookup_account, create_ticket, transfer_to_agent.`,
        voice: 'alloy',
        input_audio_format: 'pcm16',
        output_audio_format: 'pcm16',
        input_audio_transcription: { model: 'whisper-1' },
        turn_detection: {
          type: 'server_vad',
          threshold: 0.5,
          prefix_padding_ms: 300,
          silence_duration_ms: 600,
        },
        tools: [
          {
            type: 'function',
            name: 'lookup_account',
            description: 'Look up a customer account by phone number',
            parameters: {
              type: 'object',
              properties: { phone: { type: 'string', description: 'Caller phone number' } },
              required: ['phone'],
            },
          },
          {
            type: 'function',
            name: 'transfer_to_agent',
            description: 'Transfer the call to a human agent when the caller requests it or the AI cannot help',
            parameters: {
              type: 'object',
              properties: { reason: { type: 'string' }, department: { type: 'string' } },
              required: ['reason'],
            },
          },
        ],
      },
    }));
  });

  // ── FreeSWITCH → OpenAI: relay incoming audio ────────────────
  fsSocket.on('message', (audioChunk) => {
    if (realtimeWs.readyState === WebSocket.OPEN) {
      realtimeWs.send(JSON.stringify({
        type: 'input_audio_buffer.append',
        audio: audioChunk.toString('base64'),
      }));
    }
  });

  // ── OpenAI → FreeSWITCH: relay AI audio response ─────────────
  realtimeWs.on('message', async (data) => {
    const event = JSON.parse(data.toString());

    switch (event.type) {
      case 'response.audio.delta':
        // Stream AI audio back to FreeSWITCH
        if (fsSocket.readyState === WebSocket.OPEN) {
          fsSocket.send(Buffer.from(event.delta, 'base64'));
        }
        break;

      case 'response.function_call_arguments.done':
        // Handle tool calls
        await handleToolCall(event, realtimeWs, fsSocket);
        break;

      case 'input_speech_started':
        // Caller is speaking — interrupt AI audio
        console.log('[Bridge] Caller interruption detected');
        break;

      case 'error':
        console.error('[Bridge] Realtime API error:', event.error);
        break;
    }
  });

  fsSocket.on('close', () => {
    console.log('[Bridge] FreeSWITCH disconnected — closing OpenAI session');
    realtimeWs.close();
  });
});

async function handleToolCall(event, realtimeWs, fsSocket) {
  const args = JSON.parse(event.arguments);

  if (event.name === 'transfer_to_agent') {
    console.log(`[Bridge] Transferring to human agent: ${args.reason}`);
    // Signal FreeSWITCH to bridge to agent queue via ESL (out-of-band)
    // Your ESL client would issue: api uuid_transfer <uuid> 1001 xml default
    realtimeWs.send(JSON.stringify({
      type: 'conversation.item.create',
      item: {
        type: 'function_call_output',
        call_id: event.call_id,
        output: JSON.stringify({ status: 'transferring', estimated_wait: '2 minutes' }),
      },
    }));
    realtimeWs.send(JSON.stringify({ type: 'response.create' }));
  }
}

async function handleToolCall(event, realtimeWs) {
  const args = JSON.parse(event.arguments);
  let output;

  switch (event.name) {
    case 'lookup_account':
      try {
        // Query your CRM — replace with actual Salesforce/HubSpot API call
        const account = await crmClient.findByPhone(args.phone);
        output = account
          ? {
              found: true,
              name: account.name,
              account_id: account.id,
              plan: account.subscription_plan,
              open_tickets: account.open_tickets,
              last_interaction: account.last_contact_date,
            }
          : { found: false, message: 'No account found for this number' };
      } catch (err) {
        output = { found: false, error: 'CRM lookup failed' };
      }
      break;

    case 'create_ticket':
      const ticket = await crmClient.createTicket({
        caller_phone: args.phone,
        subject: args.subject,
        description: args.description,
        priority: args.priority || 'normal',
        channel: 'phone',
      });
      output = { ticket_id: ticket.id, estimated_resolution: '24 hours' };
      break;

    default:
      output = { error: 'Unknown tool' };
  }

  // Send tool result back to Realtime API
  realtimeWs.send(JSON.stringify({
    type: 'conversation.item.create',
    item: {
      type: 'function_call_output',
      call_id: event.call_id,
      output: JSON.stringify(output),
    },
  }));

  // Tell the model to continue the response with the tool result
  realtimeWs.send(JSON.stringify({ type: 'response.create' }));
}