Sending generated audio through Twilio

1
// src/app.ts
2
import { ElevenLabsClient } from '@elevenlabs/elevenlabs-js';
3
import 'dotenv/config';
4
import express, { Response } from 'express';
5
import ExpressWs from 'express-ws';
6
import { Readable } from 'stream';
7
import VoiceResponse from 'twilio/lib/twiml/VoiceResponse';
8
import { type WebSocket } from 'ws';
9
10
const app = ExpressWs(express()).app;
11
const PORT: number = parseInt(process.env.PORT || '5000');
12
13
const elevenlabs = new ElevenLabsClient();
14
const voiceId = 'aMSt68OGf4xUZAnLpTU8';
15
const outputFormat = 'ulaw_8000';
16
const text = 'This is a test. You can now hang up. Thank you.';
17
18
function startApp() {
19
  app.post('/call/incoming', (_, res: Response) => {
20
    const twiml = new VoiceResponse();
21
22
    twiml.connect().stream({
23
      url: `wss://${process.env.SERVER_DOMAIN}/call/connection`,
24
    });
25
26
    res.writeHead(200, { 'Content-Type': 'text/xml' });
27
    res.end(twiml.toString());
28
  });
29
30
  app.ws('/call/connection', (ws: WebSocket) => {
31
    ws.on('message', async (data: string) => {
32
      const message: {
33
        event: string;
34
        start?: { streamSid: string; callSid: string };
35
      } = JSON.parse(data);
36
37
      if (message.event === 'start' && message.start) {
38
        const streamSid = message.start.streamSid;
39
        const response = await elevenlabs.textToSpeech.convert(voiceId, {
40
          modelId: 'eleven_flash_v2_5',
41
          outputFormat: outputFormat,
42
          text,
43
        });
44
45
        const readableStream = Readable.from(response);
46
        const audioArrayBuffer = await streamToArrayBuffer(readableStream);
47
48
        ws.send(
49
          JSON.stringify({
50
            streamSid,
51
            event: 'media',
52
            media: {
53
              payload: Buffer.from(audioArrayBuffer as any).toString('base64'),
54
            },
55
          })
56
        );
57
      }
58
    });
59
60
    ws.on('error', console.error);
61
  });
62
63
  app.listen(PORT, () => {
64
    console.log(`Local: http://localhost:${PORT}`);
65
    console.log(`Remote: https://${process.env.SERVER_DOMAIN}`);
66
  });
67
}
68
69
function streamToArrayBuffer(readableStream: Readable) {
70
  return new Promise((resolve, reject) => {
71
    const chunks: Buffer[] = [];
72
73
    readableStream.on('data', (chunk) => {
74
      chunks.push(chunk);
75
    });
76
77
    readableStream.on('end', () => {
78
      resolve(Buffer.concat(chunks).buffer);
79
    });
80
81
    readableStream.on('error', reject);
82
  });
83
}
84
85
startApp();

1
function startApp() {
2
  app.post('/call/incoming', (_, res: Response) => {
3
    const twiml = new VoiceResponse();
4
5
    twiml.connect().stream({
6
      url: `wss://${process.env.SERVER_DOMAIN}/call/connection`,
7
    });
8
9
    res.writeHead(200, { 'Content-Type': 'text/xml' });
10
    res.end(twiml.toString());
11
  });

1
  app.ws('/call/connection', (ws: WebSocket) => {
2
    ws.on('message', async (data: string) => {
3
      const message: {
4
        event: string;
5
        start?: { streamSid: string; callSid: string };
6
      } = JSON.parse(data);
7
8
      if (message.event === 'start' && message.start) {
9
        const streamSid = message.start.streamSid;
10
        const response = await elevenlabs.textToSpeech.convert(voiceId, {
11
          modelId: 'eleven_flash_v2_5',
12
          outputFormat: outputFormat,
13
          text,
14
        });

1
const readableStream = Readable.from(response);
2
const audioArrayBuffer = await streamToArrayBuffer(readableStream);
3
4
ws.send(
5
  JSON.stringify({
6
    streamSid,
7
    event: 'media',
8
    media: {
9
      payload: Buffer.from(audioArrayBuffer as any).toString('base64'),
10
    },
11
  })
12
);