Swift SDK

Conversational AI SDK: deploy customized, interactive voice agents in your Swift applications.

Installation

Add the ElevenLabs Swift SDK to your project using Swift Package Manager:

1

Add the Package Dependency

1dependencies: [ .package(url: "https://github.com/elevenlabs/elevenlabs-swift-sdk.git",
2from: "2.0.0") ]

Or using Xcode:

  1. Open your project in Xcode
  2. Go to File > Add Package Dependencies...
  3. Enter the repository URL: https://github.com/elevenlabs/elevenlabs-swift-sdk.git
  4. Select version 2.0.0 or later
2

Import the SDK

1import ElevenLabs

Ensure you add NSMicrophoneUsageDescription to your Info.plist to explain microphone access to users. The SDK requires iOS 14.0+ / macOS 11.0+ and Swift 5.9+.

Quick Start

Get started with a simple conversation in just a few lines:

1import ElevenLabs
2
3// Start a conversation with your agent
4let conversation = try await ElevenLabs.startConversation(
5 agentId: "your-agent-id",
6 config: ConversationConfig()
7)
8
9// Observe conversation state and messages
10conversation.$state
11 .sink { state in
12 print("Connection state: \(state)")
13 }
14 .store(in: &cancellables)
15
16conversation.$messages
17 .sink { messages in
18 for message in messages {
19 print("\(message.role): \(message.content)")
20 }
21 }
22 .store(in: &cancellables)
23
24// Send messages and control the conversation
25try await conversation.sendMessage("Hello!")
26try await conversation.toggleMute()
27await conversation.endConversation()

Authentication

There are two ways to authenticate and start a conversation:

For public agents, use the agent ID directly:

1let conversation = try await ElevenLabs.startConversation(
2 agentId: "your-public-agent-id",
3 config: ConversationConfig()
4)

Core Features

Reactive Conversation Management

The SDK provides a modern Conversation class with @Published properties for reactive UI updates:

1@MainActor
2class ConversationManager: ObservableObject {
3 @Published var conversation: Conversation?
4 private var cancellables = Set<AnyCancellable>()
5
6 func startConversation(agentId: String) async throws {
7 let config = ConversationConfig(
8 conversationOverrides: ConversationOverrides(textOnly: false)
9 )
10
11 conversation = try await ElevenLabs.startConversation(
12 agentId: agentId,
13 config: config
14 )
15
16 setupObservers()
17 }
18
19 private func setupObservers() {
20 guard let conversation else { return }
21
22 // Monitor connection state
23 conversation.$state
24 .sink { state in print("State: \(state)") }
25 .store(in: &cancellables)
26
27 // Monitor messages
28 conversation.$messages
29 .sink { messages in print("Messages: \(messages.count)") }
30 .store(in: &cancellables)
31 }
32}

Voice and Text Modes

1// Voice conversation (default)
2let voiceConfig = ConversationConfig(
3 conversationOverrides: ConversationOverrides(textOnly: false)
4)
5
6// Text-only conversation
7let textConfig = ConversationConfig(
8 conversationOverrides: ConversationOverrides(textOnly: true)
9)

Audio Controls

1// Microphone control
2try await conversation.toggleMute()
3try await conversation.setMuted(true)
4
5// Check microphone state
6let isMuted = conversation.isMuted
7
8// Access audio tracks for advanced use cases
9let inputTrack = conversation.inputTrack
10let agentAudioTrack = conversation.agentAudioTrack

Client Tools

Client Tools allow you to register custom functions that can be called by your AI agent during conversations. The new SDK provides improved parameter handling and error management.

Handling Tool Calls

Handle tool calls from your agent with full parameter support:

1private func handleToolCall(_ toolCall: ClientToolCallEvent) async {
2 do {
3 let parameters = try toolCall.getParameters()
4 let result = await executeClientTool(
5 name: toolCall.toolName,
6 parameters: parameters
7 )
8
9 if toolCall.expectsResponse {
10 try await conversation?.sendToolResult(
11 for: toolCall.toolCallId,
12 result: result
13 )
14 } else {
15 conversation?.markToolCallCompleted(toolCall.toolCallId)
16 }
17 } catch {
18 // Handle tool execution errors
19 if toolCall.expectsResponse {
20 try? await conversation?.sendToolResult(
21 for: toolCall.toolCallId,
22 result: ["error": error.localizedDescription],
23 isError: true
24 )
25 }
26 }
27}
28
29// Example tool implementation
30func executeClientTool(name: String, parameters: [String: Any]) async -> [String: Any] {
31 switch name {
32 case "get_weather":
33 guard let location = parameters["location"] as? String else {
34 return ["error": "Missing location parameter"]
35 }
36 // Fetch weather data
37 return ["temperature": "22°C", "condition": "Sunny"]
38
39 case "send_email":
40 guard let recipient = parameters["recipient"] as? String,
41 let subject = parameters["subject"] as? String else {
42 return ["error": "Missing required parameters"]
43 }
44 // Send email logic
45 return ["status": "sent", "messageId": "12345"]
46
47 default:
48 return ["error": "Unknown tool: \(name)"]
49 }
50}

Remember to setup your agent with the client-tools in the ElevenLabs UI. See the Client Tools documentation for setup instructions.

Connection State Management

Monitor the conversation state to handle different connection phases:

1conversation.$state
2 .sink { state in
3 switch state {
4 case .idle:
5 // Not connected
6 break
7 case .connecting:
8 // Show connecting indicator
9 break
10 case .active(let callInfo):
11 // Connected to agent: \(callInfo.agentId)
12 break
13 case .ended(let reason):
14 // Handle disconnection: \(reason)
15 break
16 case .error(let error):
17 // Handle error: \(error)
18 break
19 }
20 }
21 .store(in: &cancellables)

Agent State Monitoring

Track when the agent is listening or speaking:

1conversation.$agentState
2 .sink { state in
3 switch state {
4 case .listening:
5 // Agent is listening, show listening indicator
6 break
7 case .speaking:
8 // Agent is speaking, show speaking indicator
9 break
10 }
11 }
12 .store(in: &cancellables)

Message Handling

Send text messages and monitor the conversation:

1// Send a text message
2try await conversation.sendMessage("Hello, how can you help me today?")
3
4// Monitor all messages in the conversation
5conversation.$messages
6 .sink { messages in
7 for message in messages {
8 switch message.role {
9 case .user:
10 print("User: \(message.content)")
11 case .agent:
12 print("Agent: \(message.content)")
13 }
14 }
15 }
16 .store(in: &cancellables)

Session Management

1// End the conversation
2await conversation.endConversation()
3
4// Check if conversation is active
5let isActive = conversation.state.isActive

SwiftUI Integration

Here’s a comprehensive SwiftUI example using the new SDK:

1import SwiftUI
2import ElevenLabs
3import Combine
4
5struct ConversationView: View {
6 @StateObject private var viewModel = ConversationViewModel()
7
8 var body: some View {
9 VStack(spacing: 20) {
10 // Connection status
11 Text(viewModel.connectionStatus)
12 .font(.headline)
13 .foregroundColor(viewModel.isConnected ? .green : .red)
14
15 // Chat messages
16 ScrollView {
17 LazyVStack(alignment: .leading, spacing: 8) {
18 ForEach(viewModel.messages, id: \.id) { message in
19 MessageBubble(message: message)
20 }
21 }
22 }
23 .frame(maxHeight: 400)
24
25 // Controls
26 HStack(spacing: 16) {
27 Button(viewModel.isConnected ? "End" : "Start") {
28 Task {
29 if viewModel.isConnected {
30 await viewModel.endConversation()
31 } else {
32 await viewModel.startConversation()
33 }
34 }
35 }
36 .buttonStyle(.borderedProminent)
37
38 Button(viewModel.isMuted ? "Unmute" : "Mute") {
39 Task { await viewModel.toggleMute() }
40 }
41 .buttonStyle(.bordered)
42 .disabled(!viewModel.isConnected)
43
44 Button("Send Message") {
45 Task { await viewModel.sendTestMessage() }
46 }
47 .buttonStyle(.bordered)
48 .disabled(!viewModel.isConnected)
49 }
50
51 // Agent state indicator
52 if viewModel.isConnected {
53 HStack {
54 Circle()
55 .fill(viewModel.agentState == .speaking ? .blue : .gray)
56 .frame(width: 10, height: 10)
57 Text(viewModel.agentState == .speaking ? "Agent speaking" : "Agent listening")
58 .font(.caption)
59 }
60 }
61 }
62 .padding()
63 }
64}
65
66struct MessageBubble: View {
67 let message: Message
68
69 var body: some View {
70 HStack {
71 if message.role == .user { Spacer() }
72
73 VStack(alignment: .leading) {
74 Text(message.role == .user ? "You" : "Agent")
75 .font(.caption)
76 .foregroundColor(.secondary)
77 Text(message.content)
78 .padding()
79 .background(message.role == .user ? Color.blue : Color.gray.opacity(0.3))
80 .foregroundColor(message.role == .user ? .white : .primary)
81 .cornerRadius(12)
82 }
83
84 if message.role == .agent { Spacer() }
85 }
86 }
87}
88
89@MainActor
90class ConversationViewModel: ObservableObject {
91 @Published var messages: [Message] = []
92 @Published var isConnected = false
93 @Published var isMuted = false
94 @Published var agentState: AgentState = .listening
95 @Published var connectionStatus = "Disconnected"
96
97 private var conversation: Conversation?
98 private var cancellables = Set<AnyCancellable>()
99
100 func startConversation() async {
101 do {
102 conversation = try await ElevenLabs.startConversation(
103 agentId: "your-agent-id",
104 config: ConversationConfig()
105 )
106 setupObservers()
107 } catch {
108 print("Failed to start conversation: \(error)")
109 connectionStatus = "Failed to connect"
110 }
111 }
112
113 func endConversation() async {
114 await conversation?.endConversation()
115 conversation = nil
116 cancellables.removeAll()
117 }
118
119 func toggleMute() async {
120 try? await conversation?.toggleMute()
121 }
122
123 func sendTestMessage() async {
124 try? await conversation?.sendMessage("Hello from the app!")
125 }
126
127 private func setupObservers() {
128 guard let conversation else { return }
129
130 conversation.$messages
131 .assign(to: &$messages)
132
133 conversation.$state
134 .map { state in
135 switch state {
136 case .idle: return "Disconnected"
137 case .connecting: return "Connecting..."
138 case .active: return "Connected"
139 case .ended: return "Ended"
140 case .error: return "Error"
141 }
142 }
143 .assign(to: &$connectionStatus)
144
145 conversation.$state
146 .map { $0.isActive }
147 .assign(to: &$isConnected)
148
149 conversation.$isMuted
150 .assign(to: &$isMuted)
151
152 conversation.$agentState
153 .assign(to: &$agentState)
154 }
155}