1
{
2
  "text": "This is a sample text ",
3
  "voice_settings": {
4
    "stability": 0.8,
5
    "similarity_boost": 0.8
6
  },
7
  "generation_config": {
8
    "chunk_length_schedule": [120, 160, 250, 290]
9
  },
10
  "xi_api_key": "<XI API Key>",
11
  "authorization": "Bearer <Authorization Token>"
12
}

1
import asyncio
2
import websockets
3
import json
4
import base64
5
import shutil
6
import os
7
import subprocess
8
from openai import AsyncOpenAI
9
10
# Define API keys and voice ID
11
OPENAI_API_KEY = '<OPENAI_API_KEY>'
12
ELEVENLABS_API_KEY = '<ELEVENLABS_API_KEY>'
13
VOICE_ID = '21m00Tcm4TlvDq8ikWAM'
14
15
# Set OpenAI API key
16
aclient = AsyncOpenAI(api_key=OPENAI_API_KEY)
17
18
def is_installed(lib_name):
19
    return shutil.which(lib_name) is not None
20
21
22
async def text_chunker(chunks):
23
    """Split text into chunks, ensuring to not break sentences."""
24
    splitters = (".", ",", "?", "!", ";", ":", "—", "-", "(", ")", "[", "]", "}", " ")
25
    buffer = ""
26
27
    async for text in chunks:
28
        if buffer.endswith(splitters):
29
            yield buffer + " "
30
            buffer = text
31
        elif text.startswith(splitters):
32
            yield buffer + text[0] + " "
33
            buffer = text[1:]
34
        else:
35
            buffer += text
36
37
    if buffer:
38
        yield buffer + " "
39
40
41
async def stream(audio_stream):
42
    """Stream audio data using mpv player."""
43
    if not is_installed("mpv"):
44
        raise ValueError(
45
            "mpv not found, necessary to stream audio. "
46
            "Install instructions: https://mpv.io/installation/"
47
        )
48
49
    mpv_process = subprocess.Popen(
50
        ["mpv", "--no-cache", "--no-terminal", "--", "fd://0"],
51
        stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
52
    )
53
54
    print("Started streaming audio")
55
    async for chunk in audio_stream:
56
        if chunk:
57
            mpv_process.stdin.write(chunk)
58
            mpv_process.stdin.flush()
59
60
    if mpv_process.stdin:
61
        mpv_process.stdin.close()
62
    mpv_process.wait()
63
64
65
async def text_to_speech_input_streaming(voice_id, text_iterator):
66
    """Send text to ElevenLabs API and stream the returned audio."""
67
    uri = f"wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input?model_id=eleven_flash_v2_5"
68
69
    async with websockets.connect(uri) as websocket:
70
        await websocket.send(json.dumps({
71
            "text": " ",
72
            "voice_settings": {"stability": 0.5, "similarity_boost": 0.8},
73
            "xi_api_key": ELEVENLABS_API_KEY,
74
        }))
75
76
        async def listen():
77
            """Listen to the websocket for audio data and stream it."""
78
            while True:
79
                try:
80
                    message = await websocket.recv()
81
                    data = json.loads(message)
82
                    if data.get("audio"):
83
                        yield base64.b64decode(data["audio"])
84
                    elif data.get('isFinal'):
85
                        break
86
                except websockets.exceptions.ConnectionClosed:
87
                    print("Connection closed")
88
                    break
89
90
        listen_task = asyncio.create_task(stream(listen()))
91
92
        async for text in text_chunker(text_iterator):
93
            await websocket.send(json.dumps({"text": text}))
94
95
        await websocket.send(json.dumps({"text": ""}))
96
97
        await listen_task
98
99
100
async def chat_completion(query):
101
    """Retrieve text from OpenAI and pass it to the text-to-speech function."""
102
    response = await aclient.chat.completions.create(model='gpt-4', messages=[{'role': 'user', 'content': query}],
103
    temperature=1, stream=True)
104
105
    async def text_iterator():
106
        async for chunk in response:
107
            delta = chunk.choices[0].delta
108
            yield delta.content
109
110
    await text_to_speech_input_streaming(VOICE_ID, text_iterator())
111
112
113
# Main execution
114
if __name__ == "__main__":
115
    user_query = "Hello, tell me a very long story."
116
    asyncio.run(chat_completion(user_query))

1
import asyncio
2
import websockets
3
import json
4
import base64
5
6
# Define API keys and voice ID
7
ELEVENLABS_API_KEY = "INSERT HERE" <- INSERT YOUR API KEY HERE
8
VOICE_ID = 'nPczCjzI2devNBz1zQrb' #Brian
9
10
def calculate_word_start_times(alignment_info):
11
    # Alignment start times are indexed from the start of the audio chunk that generated them
12
    # In order to analyse runtime over the entire response we keep a cumulative count of played audio
13
    full_alignment = {'chars': [], 'charStartTimesMs': [], 'charDurationsMs': []}
14
    cumulative_run_time = 0
15
    for old_dict in alignment_info:
16
        full_alignment['chars'].extend([" "] + old_dict['chars'])
17
        full_alignment['charDurationsMs'].extend([old_dict['charStartTimesMs'][0]] + old_dict['charDurationsMs'])
18
        full_alignment['charStartTimesMs'].extend([0] + [time+cumulative_run_time for time in old_dict['charStartTimesMs']])
19
        cumulative_run_time += sum(old_dict['charDurationsMs'])
20
    
21
    # We now have the start times of every character relative to the entire audio output
22
    zipped_start_times = list(zip(full_alignment['chars'], full_alignment['charStartTimesMs']))
23
    # Get the start time of every character that appears after a space and match this to the word
24
    words = ''.join(full_alignment['chars']).split(" ")
25
    word_start_times = list(zip(words, [0] + [zipped_start_times[i+1][1] for (i, (a,b)) in enumerate(zipped_start_times) if a == ' ']))
26
    print(f"total duration:{cumulative_run_time}")
27
    print(word_start_times)
28
29
30
async def text_to_speech_alignment_example(voice_id, text_to_send):
31
    """Send text to ElevenLabs API and stream the returned audio and alignment information."""
32
    uri = f"wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input?model_id=eleven_flash_v2_5"
33
    async with websockets.connect(uri) as websocket:
34
        await websocket.send(json.dumps({
35
            "text": " ",
36
            "voice_settings": {"stability": 0.5, "similarity_boost": 0.8, "use_speaker_boost": False},
37
            "generation_config": {
38
                "chunk_length_schedule": [120, 160, 250, 290]
39
            },
40
            "xi_api_key": ELEVENLABS_API_KEY,
41
        }))
42
43
        async def text_iterator(text):
44
            """Split text into chunks to mimic streaming from an LLM or similar"""
45
            split_text = text.split(" ")
46
            words = 0
47
            to_send = ""
48
            for chunk in split_text:
49
                to_send += chunk  + ' '
50
                words += 1
51
                if words >= 10:
52
                    print(to_send)
53
                    yield to_send
54
                    words = 0
55
                    to_send = ""
56
            yield to_send
57
58
        async def listen():
59
            """Listen to the websocket for audio data and write it to a file."""
60
            audio_chunks = []
61
            alignment_info = []
62
            received_final_chunk = False
63
            print("Listening for chunks from ElevenLabs...")
64
            while not received_final_chunk:
65
                try:
66
                    message = await websocket.recv()
67
                    data = json.loads(message)
68
                    if data.get("audio"):
69
                        audio_chunks.append(base64.b64decode(data["audio"]))
70
                    if data.get("alignment"):
71
                        alignment_info.append(data.get("alignment"))
72
                    if data.get('isFinal'):
73
                        received_final_chunk = True
74
                        break
75
                except websockets.exceptions.ConnectionClosed:
76
                    print("Connection closed")
77
                    break
78
            print("Writing audio to file")
79
            with open("output_file.mp3", "wb") as f:        
80
                f.write(b''.join(audio_chunks))
81
82
            calculate_word_start_times(alignment_info)
83
84
85
        listen_task = asyncio.create_task(listen())
86
87
        async for text in text_iterator(text_to_send):
88
            await websocket.send(json.dumps({"text": text}))
89
        await websocket.send(json.dumps({"text": " ", "flush": True}))
90
        await listen_task
91
92
93
# Main execution
94
if __name__ == "__main__":
95
    text_to_send = "The twilight sun cast its warm golden hues upon the vast rolling fields, saturating the landscape with an ethereal glow."
96
    asyncio.run(text_to_speech_alignment_example(VOICE_ID, text_to_send))