Data Collection and Analysis with Conversational AI in Next.js

1
import { ElevenLabsClient } from 'elevenlabs';
2
import { NextResponse } from 'next/server';
3
4
export async function GET() {
5
  const agentId = process.env.ELEVENLABS_AGENT_ID;
6
  if (!agentId) {
7
    throw Error('ELEVENLABS_AGENT_ID is not set');
8
  }
9
  try {
10
    const client = new ElevenLabsClient();
11
    const response = await client.conversationalAi.getSignedUrl({
12
      agent_id: agentId,
13
    });
14
    return NextResponse.json({ signedUrl: response.signed_url });
15
  } catch (error) {
16
    console.error('Error:', error);
17
    return NextResponse.json({ error: 'Failed to get signed URL' }, { status: 500 });
18
  }
19
}

1
import { useConversation } from '@11labs/react';
2
3
async function getSignedUrl(): Promise<string> {
4
  const response = await fetch('/api/signed-url');
5
  if (!response.ok) {
6
    throw Error('Failed to get signed url');
7
  }
8
  const data = await response.json();
9
  return data.signedUrl;
10
}
11
12
export default function Home() {
13
  // ...
14
  const [currentStep, setCurrentStep] = useState<
15
    'initial' | 'training' | 'voice' | 'email' | 'ready'
16
  >('initial');
17
  const [conversationId, setConversationId] = useState('');
18
  const [userName, setUserName] = useState('');
19
20
  const conversation = useConversation({
21
    onConnect: () => console.log('Connected'),
22
    onDisconnect: () => console.log('Disconnected'),
23
    onMessage: (message: string) => console.log('Message:', message),
24
    onError: (error: Error) => console.error('Error:', error),
25
  });
26
27
  const startConversation = useCallback(async () => {
28
    try {
29
      // Request microphone permission
30
      await navigator.mediaDevices.getUserMedia({ audio: true });
31
      // Start the conversation with your agent
32
      const signedUrl = await getSignedUrl();
33
      const convId = await conversation.startSession({
34
        signedUrl,
35
        dynamicVariables: {
36
          user_name: userName,
37
        },
38
        clientTools: {
39
          set_ui_state: ({ step }: { step: string }): string => {
40
            // Allow agent to navigate the UI.
41
            setCurrentStep(step as 'initial' | 'training' | 'voice' | 'email' | 'ready');
42
            return `Navigated to ${step}`;
43
          },
44
        },
45
      });
46
      setConversationId(convId);
47
      console.log('Conversation ID:', convId);
48
    } catch (error) {
49
      console.error('Failed to start conversation:', error);
50
    }
51
  }, [conversation, userName]);
52
  const stopConversation = useCallback(async () => {
53
    await conversation.endSession();
54
  }, [conversation]);
55
  // ...
56
}

1
const convId = await conversation.startSession({
2
  signedUrl,
3
  dynamicVariables: {
4
    user_name: userName,
5
  },
6
  clientTools: {
7
    set_ui_state: ({ step }: { step: string }): string => {
8
      // Allow agent to navigate the UI.
9
      setCurrentStep(step as 'initial' | 'training' | 'voice' | 'email' | 'ready');
10
      return `Navigated to ${step}`;
11
    },
12
  },
13
});

1
'use server';
2
3
import { Redis } from '@upstash/redis';
4
import { ElevenLabsClient } from 'elevenlabs';
5
import { redirect } from 'next/navigation';
6
import { after } from 'next/server';
7
8
// Initialize Redis
9
const redis = Redis.fromEnv();
10
11
const elevenLabsClient = new ElevenLabsClient({
12
  apiKey: process.env.ELEVENLABS_API_KEY,
13
});
14
15
export async function uploadFormData(formData: FormData) {
16
  const knowledgeBase: Array<{
17
    id: string;
18
    type: 'file' | 'url';
19
    name: string;
20
  }> = [];
21
  const files = formData.getAll('file-upload') as File[];
22
  const email = formData.get('email-input');
23
  const urls = formData.getAll('url-input');
24
  const conversationId = formData.get('conversation-id');
25
26
  after(async () => {
27
    // Upload files as background job
28
    // Create knowledge base entries
29
    // Loop trhough files and create knowledge base entries
30
    for (const file of files) {
31
      if (file.size > 0) {
32
        const response = await elevenLabsClient.conversationalAi.addToKnowledgeBase({ file });
33
        if (response.id) {
34
          knowledgeBase.push({
35
            id: response.id,
36
            type: 'file',
37
            name: file.name,
38
          });
39
        }
40
      }
41
    }
42
    // Append all urls
43
    for (const url of urls) {
44
      const response = await elevenLabsClient.conversationalAi.addToKnowledgeBase({
45
        url: url as string,
46
      });
47
      if (response.id) {
48
        knowledgeBase.push({
49
          id: response.id,
50
          type: 'url',
51
          name: `url for ${conversationId}`,
52
        });
53
      }
54
    }
55
56
    // Store knowledge base IDs and conversation ID in database.
57
    const redisRes = await redis.set(
58
      conversationId as string,
59
      JSON.stringify({ email, knowledgeBase })
60
    );
61
    console.log({ redisRes });
62
  });
63
64
  redirect('/success');
65
}

1
import { Redis } from '@upstash/redis';
2
import crypto from 'crypto';
3
import { ElevenLabsClient } from 'elevenlabs';
4
import { NextResponse } from 'next/server';
5
import type { NextRequest } from 'next/server';
6
import { Resend } from 'resend';
7
8
import { EmailTemplate } from '@/components/email/post-call-webhook-email';
9
10
// Initialize Redis
11
const redis = Redis.fromEnv();
12
// Initialize Resend
13
const resend = new Resend(process.env.RESEND_API_KEY);
14
15
const elevenLabsClient = new ElevenLabsClient({
16
  apiKey: process.env.ELEVENLABS_API_KEY,
17
});
18
19
export async function GET() {
20
  return NextResponse.json({ status: 'webhook listening' }, { status: 200 });
21
}
22
23
export async function POST(req: NextRequest) {
24
  const secret = process.env.ELEVENLABS_CONVAI_WEBHOOK_SECRET; // Add this to your env variables
25
  const { event, error } = await constructWebhookEvent(req, secret);
26
  if (error) {
27
    return NextResponse.json({ error: error }, { status: 401 });
28
  }
29
30
  if (event.type === 'post_call_transcription') {
31
    const { conversation_id, analysis, agent_id } = event.data;
32
33
    if (
34
      agent_id === process.env.ELEVENLABS_AGENT_ID &&
35
      analysis.evaluation_criteria_results.all_data_provided?.result === 'success' &&
36
      analysis.data_collection_results.voice_description?.value
37
    ) {
38
      try {
39
        // Design the voice
40
        const voicePreview = await elevenLabsClient.textToVoice.createPreviews({
41
          voice_description: analysis.data_collection_results.voice_description.value,
42
          text: 'The night air carried whispers of betrayal, thick as London fog. I adjusted my cufflinks - after all, even spies must maintain appearances, especially when the game is afoot.',
43
        });
44
        const voice = await elevenLabsClient.textToVoice.createVoiceFromPreview({
45
          voice_name: `voice-${conversation_id}`,
46
          voice_description: `Voice for ${conversation_id}`,
47
          generated_voice_id: voicePreview.previews[0].generated_voice_id,
48
        });
49
50
        // Get the knowledge base from redis
51
        const redisRes = await getRedisDataWithRetry(conversation_id);
52
        if (!redisRes) throw new Error('Conversation data not found!');
53
        // Handle agent creation
54
        const agent = await elevenLabsClient.conversationalAi.createAgent({
55
          name: `Agent for ${conversation_id}`,
56
          conversation_config: {
57
            tts: { voice_id: voice.voice_id },
58
            agent: {
59
              prompt: {
60
                prompt:
61
                  analysis.data_collection_results.agent_description?.value ??
62
                  'You are a helpful assistant.',
63
                knowledge_base: redisRes.knowledgeBase,
64
              },
65
              first_message: 'Hello, how can I help you today?',
66
            },
67
          },
68
        });
69
        console.log('Agent created', { agent: agent.agent_id });
70
        // Send email to user
71
        console.log('Sending email to', redisRes.email);
72
        await resend.emails.send({
73
          from: process.env.RESEND_FROM_EMAIL!,
74
          to: redisRes.email,
75
          subject: 'Your Conversational AI agent is ready to chat!',
76
          react: EmailTemplate({ agentId: agent.agent_id }),
77
        });
78
      } catch (error) {
79
        console.error(error);
80
        return NextResponse.json({ error }, { status: 500 });
81
      }
82
    }
83
  }
84
85
  return NextResponse.json({ received: true }, { status: 200 });
86
}
87
88
const constructWebhookEvent = async (req: NextRequest, secret?: string) => {
89
  const body = await req.text();
90
  const signature_header = req.headers.get('ElevenLabs-Signature');
91
92
  if (!signature_header) {
93
    return { event: null, error: 'Missing signature header' };
94
  }
95
96
  const headers = signature_header.split(',');
97
  const timestamp = headers.find((e) => e.startsWith('t='))?.substring(2);
98
  const signature = headers.find((e) => e.startsWith('v0='));
99
100
  if (!timestamp || !signature) {
101
    return { event: null, error: 'Invalid signature format' };
102
  }
103
104
  // Validate timestamp
105
  const reqTimestamp = Number(timestamp) * 1000;
106
  const tolerance = Date.now() - 30 * 60 * 1000;
107
  if (reqTimestamp < tolerance) {
108
    return { event: null, error: 'Request expired' };
109
  }
110
111
  // Validate hash
112
  const message = `${timestamp}.${body}`;
113
114
  if (!secret) {
115
    return { event: null, error: 'Webhook secret not configured' };
116
  }
117
118
  const digest = 'v0=' + crypto.createHmac('sha256', secret).update(message).digest('hex');
119
120
  if (signature !== digest) {
121
    return { event: null, error: 'Invalid signature' };
122
  }
123
124
  const event = JSON.parse(body);
125
  return { event, error: null };
126
};
127
128
async function getRedisDataWithRetry(
129
  conversationId: string,
130
  maxRetries = 5
131
): Promise<{
132
  email: string;
133
  knowledgeBase: Array<{
134
    id: string;
135
    type: 'file' | 'url';
136
    name: string;
137
  }>;
138
} | null> {
139
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
140
    try {
141
      const data = await redis.get(conversationId);
142
      return data as any;
143
    } catch (error) {
144
      if (attempt === maxRetries) throw error;
145
      console.log(`Redis get attempt ${attempt} failed, retrying...`);
146
      await new Promise((resolve) => setTimeout(resolve, 1000));
147
    }
148
  }
149
  return null;
150
}

1
// ...
2
3
export async function POST(req: NextRequest) {
4
  const secret = process.env.ELEVENLABS_CONVAI_WEBHOOK_SECRET;
5
  const { event, error } = await constructWebhookEvent(req, secret);
6
  // ...
7
}
8
9
// ...
10
const constructWebhookEvent = async (req: NextRequest, secret?: string) => {
11
  const body = await req.text();
12
  const signature_header = req.headers.get('ElevenLabs-Signature');
13
14
  if (!signature_header) {
15
    return { event: null, error: 'Missing signature header' };
16
  }
17
18
  const headers = signature_header.split(',');
19
  const timestamp = headers.find((e) => e.startsWith('t='))?.substring(2);
20
  const signature = headers.find((e) => e.startsWith('v0='));
21
22
  if (!timestamp || !signature) {
23
    return { event: null, error: 'Invalid signature format' };
24
  }
25
26
  // Validate timestamp
27
  const reqTimestamp = Number(timestamp) * 1000;
28
  const tolerance = Date.now() - 30 * 60 * 1000;
29
  if (reqTimestamp < tolerance) {
30
    return { event: null, error: 'Request expired' };
31
  }
32
33
  // Validate hash
34
  const message = `${timestamp}.${body}`;
35
36
  if (!secret) {
37
    return { event: null, error: 'Webhook secret not configured' };
38
  }
39
40
  const digest = 'v0=' + crypto.createHmac('sha256', secret).update(message).digest('hex');
41
42
  if (signature !== digest) {
43
    return { event: null, error: 'Invalid signature' };
44
  }
45
46
  const event = JSON.parse(body);
47
  return { event, error: null };
48
};
49
50
async function getRedisDataWithRetry(
51
  conversationId: string,
52
  maxRetries = 5
53
): Promise<{
54
  email: string;
55
  knowledgeBase: Array<{
56
    id: string;
57
    type: 'file' | 'url';
58
    name: string;
59
  }>;
60
} | null> {
61
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
62
    try {
63
      const data = await redis.get(conversationId);
64
      return data as any;
65
    } catch (error) {
66
      if (attempt === maxRetries) throw error;
67
      console.log(`Redis get attempt ${attempt} failed, retrying...`);
68
      await new Promise((resolve) => setTimeout(resolve, 1000));
69
    }
70
  }
71
  return null;
72
}

1
// ...
2
3
// Design the voice
4
const voicePreview = await elevenLabsClient.textToVoice.createPreviews({
5
  voice_description: analysis.data_collection_results.voice_description.value,
6
  text: 'The night air carried whispers of betrayal, thick as London fog. I adjusted my cufflinks - after all, even spies must maintain appearances, especially when the game is afoot.',
7
});
8
const voice = await elevenLabsClient.textToVoice.createVoiceFromPreview({
9
  voice_name: `voice-${conversation_id}`,
10
  voice_description: `Voice for ${conversation_id}`,
11
  generated_voice_id: voicePreview.previews[0].generated_voice_id,
12
});
13
14
// ...

1
// ...
2
3
// Get the knowledge base from redis
4
const redisRes = await getRedisDataWithRetry(conversation_id);
5
if (!redisRes) throw new Error('Conversation data not found!');
6
// ...
7
8
async function getRedisDataWithRetry(
9
  conversationId: string,
10
  maxRetries = 5
11
): Promise<{
12
  email: string;
13
  knowledgeBase: Array<{
14
    id: string;
15
    type: 'file' | 'url';
16
    name: string;
17
  }>;
18
} | null> {
19
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
20
    try {
21
      const data = await redis.get(conversationId);
22
      return data as any;
23
    } catch (error) {
24
      if (attempt === maxRetries) throw error;
25
      console.log(`Redis get attempt ${attempt} failed, retrying...`);
26
      await new Promise((resolve) => setTimeout(resolve, 1000));
27
    }
28
  }
29
  return null;
30
}

1
// ...
2
3
// Handle agent creation
4
const agent = await elevenLabsClient.conversationalAi.createAgent({
5
  name: `Agent for ${conversation_id}`,
6
  conversation_config: {
7
    tts: { voice_id: voice.voice_id },
8
    agent: {
9
      prompt: {
10
        prompt:
11
          analysis.data_collection_results.agent_description?.value ??
12
          'You are a helpful assistant.',
13
        knowledge_base: redisRes.knowledgeBase,
14
      },
15
      first_message: 'Hello, how can I help you today?',
16
    },
17
  },
18
});
19
console.log('Agent created', { agent: agent.agent_id });
20
21
// ...

1
import { Resend } from 'resend';
2
3
import { EmailTemplate } from '@/components/email/post-call-webhook-email';
4
5
// ...
6
7
// Send email to user
8
console.log('Sending email to', redisRes.email);
9
await resend.emails.send({
10
  from: process.env.RESEND_FROM_EMAIL!,
11
  to: redisRes.email,
12
  subject: 'Your Conversational AI agent is ready to chat!',
13
  react: EmailTemplate({ agentId: agent.agent_id }),
14
});
15
16
// ...

1
import {
2
  Body,
3
  Button,
4
  Container,
5
  Head,
6
  Html,
7
  Section,
8
  Text,
9
  Tailwind,
10
} from '@react-email/components';
11
import * as React from 'react';
12
13
const EmailTemplate = (props: any) => {
14
  const { agentId } = props;
15
  return (
16
    <Html>
17
      <Head />
18
      <Tailwind>
19
        <Body className="bg-[#151516] font-sans">
20
          <Container className="mx-auto my-[40px] max-w-[600px] rounded-[8px] bg-[#0a1929] p-[20px]">
21
            {/* Top Section */}
22
            <Section className="mb-[32px] mt-[32px] text-center">
23
              <Text className="m-0 text-[28px] font-bold text-[#9c27b0]">
24
                Your Conversational AI agent is ready to chat!
25
              </Text>
26
            </Section>
27
28
            {/* Content Area with Icon */}
29
            <Section className="mb-[32px] text-center">
30
              {/* Circle Icon with Checkmark */}
31
              <div className="mx-auto mb-[24px] flex h-[80px] w-[80px] items-center justify-center rounded-full bg-gradient-to-r from-[#9c27b0] to-[#3f51b5]">
32
                <div className="text-[40px] text-white">✓</div>
33
              </div>
34
35
              {/* Descriptive Text */}
36
              <Text className="mb-[24px] text-[18px] text-white">
37
                Your Conversational AI agent is ready to chat!
38
              </Text>
39
            </Section>
40
41
            {/* Call to Action Button */}
42
            <Section className="mb-[32px] text-center">
43
              <Button
44
                href={`https://elevenlabs.io/app/talk-to?agent_id=${agentId}`}
45
                className="box-border rounded-[8px] bg-[#9c27b0] px-[40px] py-[20px] text-[24px] font-bold text-white no-underline"
46
              >
47
                Chat now!
48
              </Button>
49
            </Section>
50
51
            {/* Footer */}
52
            <Section className="mt-[40px] border-t border-[#2d3748] pt-[20px] text-center">
53
              <Text className="m-0 text-[14px] text-white">
54
                Powered by{' '}
55
                <a
56
                  href="https://elevenlabs.io/conversational-ai"
57
                  target="_blank"
58
                  rel="noopener noreferrer"
59
                  className="underline transition-colors hover:text-gray-400"
60
                >
61
                  ElevenLabs Conversational AI
62
                </a>
63
              </Text>
64
            </Section>
65
          </Container>
66
        </Body>
67
      </Tailwind>
68
    </Html>
69
  );
70
};
71
72
export { EmailTemplate };