Graph calling bot

1
public class SpeechService
2
{
3
    private readonly AppSettings _settings;
4
    private readonly ILogger _logger;
5
    private ClientWebSocket _ws;
6
    private bool _started;
7
    private bool _connecting;
8
9
    public event EventHandler<MediaStreamEventArgs> SendMediaBuffer; // agent audio -> call
10
    public event EventHandler FlushMedia;                            // barge-in: drop queued audio
11
12
    public SpeechService(AppSettings settings, ILogger logger) { _settings = settings; _logger = logger; }
13
14
    // Caller audio -> ElevenLabs
15
    public async Task AppendAudioBuffer(AudioMediaBuffer buffer)
16
    {
17
        if (!_started)
18
        {
19
            if (_connecting) return;            // a connect attempt is already in flight
20
            _connecting = true;
21
            try { await Connect(); _started = true; }
22
            catch (Exception ex) { _logger.Error(ex, "ElevenLabs connect failed; retry on next frame"); return; }
23
            finally { _connecting = false; }
24
        }
25
        if (_ws?.State != WebSocketState.Open || buffer.Length <= 0) return;
26
        var pcm = new byte[buffer.Length];
27
        Marshal.Copy(buffer.Data, pcm, 0, (int)buffer.Length);
28
        var msg = JsonSerializer.Serialize(new { user_audio_chunk = Convert.ToBase64String(pcm) });
29
        await _ws.SendAsync(Encoding.UTF8.GetBytes(msg), WebSocketMessageType.Text, true, default);
30
    }
31
32
    private async Task Connect()
33
    {
34
        _ws = new ClientWebSocket();
35
        // ElevenLabsOrigin: wss://api.elevenlabs.io, or a residency host (.eu./.in./.sg.)
36
        var url = $"{_settings.ElevenLabsOrigin}/v1/convai/conversation?agent_id={_settings.ElevenLabsAgentId}";
37
        await _ws.ConnectAsync(new Uri(url), default);
38
        await _ws.SendAsync(Encoding.UTF8.GetBytes(
39
            JsonSerializer.Serialize(new { type = "conversation_initiation_client_data" })),
40
            WebSocketMessageType.Text, true, default);
41
        _ = Task.Run(ReceiveLoop);
42
    }
43
44
    private async Task ReceiveLoop()
45
    {
46
        var buf = new byte[32768]; var sb = new StringBuilder();
47
        while (_ws.State == WebSocketState.Open)
48
        {
49
            sb.Clear(); WebSocketReceiveResult r;
50
            do { r = await _ws.ReceiveAsync(buf, default); sb.Append(Encoding.UTF8.GetString(buf, 0, r.Count)); }
51
            while (!r.EndOfMessage);
52
53
            using var doc = JsonDocument.Parse(sb.ToString());
54
            var type = doc.RootElement.GetProperty("type").GetString();
55
            if (type == "audio")            // ElevenLabs audio -> call
56
                Emit(Convert.FromBase64String(doc.RootElement
57
                    .GetProperty("audio_event").GetProperty("audio_base_64").GetString()));
58
            else if (type == "ping")
59
                await _ws.SendAsync(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(new {
60
                    type = "pong", event_id = doc.RootElement.GetProperty("ping_event").GetProperty("event_id").GetInt32() })),
61
                    WebSocketMessageType.Text, true, default);
62
            else if (type == "interruption")  // barge-in: drop any agent audio still queued
63
                FlushMedia?.Invoke(this, EventArgs.Empty);
64
        }
65
    }
66
67
    // slice PCM into 20 ms / 640-byte frames the media platform expects
68
    private void Emit(byte[] pcm)
69
    {
70
        var all = new List<AudioMediaBuffer>(); long tick = DateTime.Now.Ticks;
71
        for (int off = 0; off < pcm.Length; off += 640)
72
        {
73
            var frame = new byte[640];
74
            Array.Copy(pcm, off, frame, 0, Math.Min(640, pcm.Length - off));
75
            all.AddRange(Utilities.CreateAudioMediaBuffers(frame, tick, _logger));
76
            tick += 20 * 10000;
77
        }
78
        if (all.Count > 0) SendMediaBuffer?.Invoke(this, new MediaStreamEventArgs { AudioMediaBuffers = all });
79
    }
80
}

1
public class ChatBot : ActivityHandler
2
{
3
    private readonly AppSettings _settings;
4
    public ChatBot(AppSettings settings) => _settings = settings;
5
6
    protected override async Task OnMessageActivityAsync(
7
        ITurnContext<IMessageActivity> turn, CancellationToken ct)
8
    {
9
        var reply = await AskAgent(turn.Activity.Text, ct);
10
        await turn.SendActivityAsync(MessageFactory.Text(reply), ct);
11
    }
12
13
    private async Task<string> AskAgent(string text, CancellationToken ct)
14
    {
15
        using var ws = new ClientWebSocket();
16
        var url = $"{_settings.ElevenLabsOrigin}/v1/convai/conversation?agent_id={_settings.ElevenLabsAgentId}";
17
        await ws.ConnectAsync(new Uri(url), ct);
18
        // Suppress the agent's greeting: with no override, the first agent_response is the
19
        // configured first message, not the answer to this user_message.
20
        await Send(ws, new
21
        {
22
            type = "conversation_initiation_client_data",
23
            conversation_config_override = new { agent = new { first_message = "" } },
24
        }, ct);
25
        await Send(ws, new { type = "user_message", text }, ct);
26
27
        var buf = new byte[16384]; var sb = new StringBuilder();
28
        while (ws.State == WebSocketState.Open)
29
        {
30
            sb.Clear(); WebSocketReceiveResult r;
31
            do { r = await ws.ReceiveAsync(buf, ct); sb.Append(Encoding.UTF8.GetString(buf, 0, r.Count)); }
32
            while (!r.EndOfMessage);
33
34
            using var doc = JsonDocument.Parse(sb.ToString());
35
            switch (doc.RootElement.GetProperty("type").GetString())
36
            {
37
                case "agent_response":
38
                    return doc.RootElement.GetProperty("agent_response_event")
39
                              .GetProperty("agent_response").GetString();
40
                case "ping":
41
                    await Send(ws, new { type = "pong", event_id = doc.RootElement
42
                        .GetProperty("ping_event").GetProperty("event_id").GetInt32() }, ct);
43
                    break;
44
            }
45
        }
46
        return "Sorry, I couldn't reach the agent.";
47
    }
48
49
    private static Task Send(ClientWebSocket ws, object msg, CancellationToken ct) =>
50
        ws.SendAsync(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(msg)),
51
            WebSocketMessageType.Text, true, ct);
52
}