diff --git a/.env.example b/.env.example index 615daa8..c54b936 100644 --- a/.env.example +++ b/.env.example @@ -15,7 +15,23 @@ PORT=3000 NODE_ENV=development ENCRYPTION_KEY=change-me-32-char-encryption-key! -# LLM (à configurer plus tard) -# LLM_API_KEY= -# STT_API_KEY= -# TTS_API_KEY= +# Auth +JWT_SECRET=dev-secret-change-in-production + +# STT (Deepgram) +DEEPGRAM_API_KEY= + +# LLM — choisir le provider : "anthropic" ou "openai" +LLM_PROVIDER=anthropic + +# Anthropic (utilisé si LLM_PROVIDER=anthropic) +ANTHROPIC_API_KEY= +ANTHROPIC_MODEL=claude-sonnet-4-20250514 + +# OpenAI (utilisé si LLM_PROVIDER=openai) +# OPENAI_API_KEY= +# OPENAI_MODEL=gpt-4o + +# TTS (ElevenLabs) +ELEVENLABS_API_KEY= +ELEVENLABS_VOICE_ID=pFZP5JQG7iQjIQuC4Bku diff --git a/apps/backend/package.json b/apps/backend/package.json index dc1ce3e..46d281c 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -22,6 +22,7 @@ "migration:revert": "pnpm typeorm migration:revert -d src/config/typeorm.config.ts" }, "dependencies": { + "@anthropic-ai/sdk": "^0.80.0", "@deepgram/sdk": "^5.0.0", "@nestjs/common": "^11.1.17", "@nestjs/config": "^4.0.3", @@ -36,7 +37,9 @@ "class-transformer": "^0.5.1", "class-validator": "^0.15.1", "dotenv": "^17.3.1", + "elevenlabs": "^1.59.0", "ioredis": "^5.10.1", + "openai": "^6.33.0", "passport": "^0.7.0", "passport-jwt": "^4.0.1", "pg": "^8.20.0", diff --git a/apps/backend/src/adapters/inbound/websocket/robot.gateway.ts b/apps/backend/src/adapters/inbound/websocket/robot.gateway.ts index 888e4be..d27b22e 100644 --- a/apps/backend/src/adapters/inbound/websocket/robot.gateway.ts +++ b/apps/backend/src/adapters/inbound/websocket/robot.gateway.ts @@ -7,12 +7,13 @@ import { MessageBody, ConnectedSocket, } from '@nestjs/websockets'; -import { Inject, Logger } from '@nestjs/common'; +import { Inject, Logger, forwardRef } from '@nestjs/common'; import { JwtService } from '@nestjs/jwt'; import { Server, Socket } from 'socket.io'; import { DeviceService } from '../../../core/services/device.service'; import { JwtPayload } from '../rest/auth/strategies/jwt.strategy'; import { IConversationPort, CONVERSATION_PORT } from '../../../core/ports/inbound/conversation.port'; +import { IDeviceGatewayPort } from '../../../core/ports/outbound/device-gateway.port'; interface AuthenticatedSocket extends Socket { data: { @@ -32,7 +33,7 @@ type RobotState = 'listening' | 'thinking' | 'speaking' | 'idle'; namespace: '/ws/robot', cors: { origin: '*' }, }) -export class RobotGateway implements OnGatewayConnection, OnGatewayDisconnect { +export class RobotGateway implements OnGatewayConnection, OnGatewayDisconnect, IDeviceGatewayPort { @WebSocketServer() server!: Server; @@ -42,7 +43,7 @@ export class RobotGateway implements OnGatewayConnection, OnGatewayDisconnect { constructor( private readonly jwtService: JwtService, private readonly deviceService: DeviceService, - @Inject(CONVERSATION_PORT) private readonly conversationPort: IConversationPort, + @Inject(forwardRef(() => CONVERSATION_PORT)) private readonly conversationPort: IConversationPort, ) { } async handleConnection(client: AuthenticatedSocket) { @@ -112,13 +113,7 @@ export class RobotGateway implements OnGatewayConnection, OnGatewayDisconnect { @SubscribeMessage('speech_end') async handleSpeechEnd(@ConnectedSocket() client: AuthenticatedSocket) { this.logger.log(`Speech ended on device ${client.data.deviceId}`); - client.emit('status', { state: 'thinking' as RobotState }); - - const transcription = await this.conversationPort.stopListening(client.data.deviceId); - this.logger.log(`Transcription: "${transcription}"`); - - // TODO: plus tard, le ConversationService enverra la réponse LLM+TTS - client.emit('status', { state: 'idle' as RobotState }); + await this.conversationPort.stopListening(client.data.deviceId); } @SubscribeMessage('user_interrupt') @@ -131,7 +126,8 @@ export class RobotGateway implements OnGatewayConnection, OnGatewayDisconnect { // --- Helpers --- sendAudioChunk(deviceId: string, chunk: Buffer) { - this.connectedDevices.get(deviceId)?.emit('audio_chunk', { data: chunk }); + const base64 = chunk.toString('base64'); + this.connectedDevices.get(deviceId)?.emit('audio_chunk', { data: base64 }); } sendStatus(deviceId: string, state: RobotState) { diff --git a/apps/backend/src/adapters/outbound/llm/anthropic.adapter.ts b/apps/backend/src/adapters/outbound/llm/anthropic.adapter.ts new file mode 100644 index 0000000..4e52b40 --- /dev/null +++ b/apps/backend/src/adapters/outbound/llm/anthropic.adapter.ts @@ -0,0 +1,114 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import Anthropic from '@anthropic-ai/sdk'; +import { + ILLMPort, + LLMMessage, + LLMResponse, + LLMToolCall, + LLMToolDefinition, +} from '../../../core/ports/outbound/llm.port'; + +@Injectable() +export class AnthropicAdapter implements ILLMPort { + private readonly logger = new Logger(AnthropicAdapter.name); + private readonly client: Anthropic; + private readonly model: string; + + constructor(private readonly configService: ConfigService) { + const apiKey = this.configService.get('ANTHROPIC_API_KEY'); + if (!apiKey) { + throw new Error('ANTHROPIC_API_KEY is not set'); + } + this.client = new Anthropic({ apiKey }); + this.model = this.configService.get('ANTHROPIC_MODEL', 'claude-sonnet-4-20250514'); + } + + async chat(messages: LLMMessage[], tools?: LLMToolDefinition[]): Promise { + const systemMessage = messages.find((m) => m.role === 'system'); + const conversationMessages = messages + .filter((m) => m.role !== 'system') + .map((m) => this.toAnthropicMessage(m)); + + const params: Anthropic.MessageCreateParams = { + model: this.model, + max_tokens: 1024, + messages: conversationMessages, + }; + + if (systemMessage) { + params.system = systemMessage.content; + } + + if (tools && tools.length > 0) { + params.tools = tools.map((t) => ({ + name: t.name, + description: t.description, + input_schema: t.parameters as Anthropic.Tool.InputSchema, + })); + } + + const response = await this.client.messages.create(params); + + let content = ''; + const toolCalls: LLMToolCall[] = []; + + for (const block of response.content) { + if (block.type === 'text') { + content += block.text; + } else if (block.type === 'tool_use') { + toolCalls.push({ + id: block.id, + name: block.name, + arguments: JSON.stringify(block.input), + }); + } + } + + this.logger.debug( + `LLM response: ${content.substring(0, 100)}... (${response.usage.input_tokens}+${response.usage.output_tokens} tokens)`, + ); + + return { + content: content || null, + toolCalls, + tokensUsed: response.usage.input_tokens + response.usage.output_tokens, + }; + } + + private toAnthropicMessage(msg: LLMMessage): Anthropic.MessageParam { + if (msg.role === 'tool') { + return { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: msg.toolCallId!, + content: msg.content, + }, + ], + }; + } + + if (msg.role === 'assistant' && msg.toolCalls && msg.toolCalls.length > 0) { + const content: Anthropic.ContentBlockParam[] = []; + if (msg.content) { + content.push({ type: 'text', text: msg.content }); + } + for (const tc of msg.toolCalls) { + content.push({ + type: 'tool_use', + id: tc.id, + name: tc.name, + input: JSON.parse(tc.arguments), + }); + } + return { role: 'assistant', content }; + } + + return { + role: msg.role as 'user' | 'assistant', + content: msg.content, + }; + } +} diff --git a/apps/backend/src/adapters/outbound/llm/openai.adapter.ts b/apps/backend/src/adapters/outbound/llm/openai.adapter.ts new file mode 100644 index 0000000..d0a7e33 --- /dev/null +++ b/apps/backend/src/adapters/outbound/llm/openai.adapter.ts @@ -0,0 +1,99 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import OpenAI from 'openai'; +import { + ILLMPort, + LLMMessage, + LLMResponse, + LLMToolCall, + LLMToolDefinition, +} from '../../../core/ports/outbound/llm.port'; + +@Injectable() +export class OpenAIAdapter implements ILLMPort { + private readonly logger = new Logger(OpenAIAdapter.name); + private readonly client: OpenAI; + private readonly model: string; + + constructor(private readonly configService: ConfigService) { + const apiKey = this.configService.get('OPENAI_API_KEY'); + if (!apiKey) { + throw new Error('OPENAI_API_KEY is not set'); + } + this.client = new OpenAI({ apiKey }); + this.model = this.configService.get('OPENAI_MODEL', 'gpt-4o'); + } + + async chat(messages: LLMMessage[], tools?: LLMToolDefinition[]): Promise { + const openaiMessages = messages.map((m) => this.toOpenAIMessage(m)); + + const params: OpenAI.ChatCompletionCreateParams = { + model: this.model, + max_tokens: 1024, + messages: openaiMessages, + }; + + if (tools && tools.length > 0) { + params.tools = tools.map((t) => ({ + type: 'function' as const, + function: { + name: t.name, + description: t.description, + parameters: t.parameters, + }, + })); + } + + const response = await this.client.chat.completions.create(params); + const choice = response.choices[0]; + + const content = choice.message.content ?? ''; + const toolCalls: LLMToolCall[] = (choice.message.tool_calls ?? []) + .filter((tc): tc is OpenAI.ChatCompletionMessageToolCall & { type: 'function' } => tc.type === 'function') + .map((tc) => ({ + id: tc.id, + name: tc.function.name, + arguments: tc.function.arguments, + })); + + this.logger.debug( + `LLM response: ${content.substring(0, 100)}... (${response.usage?.total_tokens ?? 0} tokens)`, + ); + + return { + content: content || null, + toolCalls, + tokensUsed: response.usage?.total_tokens ?? 0, + }; + } + + private toOpenAIMessage(msg: LLMMessage): OpenAI.ChatCompletionMessageParam { + if (msg.role === 'tool') { + return { + role: 'tool', + tool_call_id: msg.toolCallId!, + content: msg.content, + }; + } + + if (msg.role === 'assistant' && msg.toolCalls && msg.toolCalls.length > 0) { + return { + role: 'assistant', + content: msg.content || null, + tool_calls: msg.toolCalls.map((tc) => ({ + id: tc.id, + type: 'function' as const, + function: { + name: tc.name, + arguments: tc.arguments, + }, + })), + }; + } + + return { + role: msg.role as 'system' | 'user' | 'assistant', + content: msg.content, + }; + } +} diff --git a/apps/backend/src/adapters/outbound/stt/deepgram.adapter.ts b/apps/backend/src/adapters/outbound/stt/deepgram.adapter.ts index 818fb03..6c11c47 100644 --- a/apps/backend/src/adapters/outbound/stt/deepgram.adapter.ts +++ b/apps/backend/src/adapters/outbound/stt/deepgram.adapter.ts @@ -1,14 +1,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { DeepgramClient } from '@deepgram/sdk'; -import { ISTTPort, TranscriptionResult } from '../../../core/ports/outbound/stt.port'; +import { ISTTPort, ISTTStream, TranscriptionResult } from '../../../core/ports/outbound/stt.port'; @Injectable() export class DeepgramAdapter implements ISTTPort { private readonly logger = new Logger(DeepgramAdapter.name); private readonly deepgram: DeepgramClient; private readonly apiKey: string; - private socket: any = null; constructor(private readonly configService: ConfigService) { const apiKey = this.configService.get('DEEPGRAM_API_KEY'); @@ -19,7 +18,7 @@ export class DeepgramAdapter implements ISTTPort { this.deepgram = new DeepgramClient({ apiKey }); } - async startStream(onResult: (result: TranscriptionResult) => void): Promise { + async openStream(onResult: (result: TranscriptionResult) => void): Promise { const socket = await this.deepgram.listen.v1.connect({ model: 'nova-3', language: 'fr', @@ -56,22 +55,16 @@ export class DeepgramAdapter implements ISTTPort { socket.connect(); await socket.waitForOpen(); - this.socket = socket; this.logger.log('Deepgram stream opened'); - } - sendAudio(chunk: Buffer): void { - if (!this.socket) { - this.logger.warn('No active Deepgram stream, ignoring audio chunk'); - return; - } - this.socket.sendMedia(chunk); - } - - async endStream(): Promise { - if (!this.socket) return; - this.socket.close(); - this.socket = null; + return { + sendAudio(chunk: Buffer): void { + socket.sendMedia(chunk); + }, + async close(): Promise { + socket.close(); + }, + }; } async transcribe(audioChunk: Buffer, sampleRate: number): Promise { diff --git a/apps/backend/src/adapters/outbound/tts/elevenlabs.adapter.ts b/apps/backend/src/adapters/outbound/tts/elevenlabs.adapter.ts new file mode 100644 index 0000000..1946db7 --- /dev/null +++ b/apps/backend/src/adapters/outbound/tts/elevenlabs.adapter.ts @@ -0,0 +1,53 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { ElevenLabsClient } from 'elevenlabs'; +import { ITTSPort } from '../../../core/ports/outbound/tts.port'; + +@Injectable() +export class ElevenLabsAdapter implements ITTSPort { + private readonly logger = new Logger(ElevenLabsAdapter.name); + private readonly client: ElevenLabsClient; + private readonly defaultVoiceId: string; + + constructor(private readonly configService: ConfigService) { + const apiKey = this.configService.get('ELEVENLABS_API_KEY'); + if (!apiKey) { + throw new Error('ELEVENLABS_API_KEY is not set'); + } + this.client = new ElevenLabsClient({ apiKey }); + this.defaultVoiceId = this.configService.get('ELEVENLABS_VOICE_ID', 'pFZP5JQG7iQjIQuC4Bku'); + } + + async synthesize(text: string, voice?: string): Promise { + const stream = await this.client.textToSpeech.convert(voice || this.defaultVoiceId, { + text, + model_id: 'eleven_multilingual_v2', + output_format: 'pcm_16000', + }); + + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.from(chunk)); + } + + return Buffer.concat(chunks); + } + + async synthesizeStream( + text: string, + voice?: string, + onChunk?: (chunk: Buffer) => void, + ): Promise { + const stream = await this.client.textToSpeech.convertAsStream(voice || this.defaultVoiceId, { + text, + model_id: 'eleven_multilingual_v2', + output_format: 'pcm_16000', + }); + + for await (const chunk of stream) { + const buffer = Buffer.from(chunk); + this.logger.debug(`TTS chunk: ${buffer.length} bytes`); + onChunk?.(buffer); + } + } +} diff --git a/apps/backend/src/app.module.ts b/apps/backend/src/app.module.ts index 1ec1a8b..bdf07d7 100644 --- a/apps/backend/src/app.module.ts +++ b/apps/backend/src/app.module.ts @@ -1,4 +1,3 @@ - import { Module } from '@nestjs/common'; import { ConfigModule, ConfigService } from '@nestjs/config'; import { join } from 'path'; @@ -22,8 +21,14 @@ import { AuthController } from './adapters/inbound/rest/auth/auth.controller'; import { DeviceController } from './adapters/inbound/rest/device/device.controller'; import { RobotGateway } from './adapters/inbound/websocket/robot.gateway'; import { DeepgramAdapter } from './adapters/outbound/stt/deepgram.adapter'; +import { AnthropicAdapter } from './adapters/outbound/llm/anthropic.adapter'; +import { OpenAIAdapter } from './adapters/outbound/llm/openai.adapter'; +import { ElevenLabsAdapter } from './adapters/outbound/tts/elevenlabs.adapter'; import { CONVERSATION_PORT } from './core/ports/inbound/conversation.port'; import { STT_PORT } from './core/ports/outbound/stt.port'; +import { LLM_PORT } from './core/ports/outbound/llm.port'; +import { TTS_PORT } from './core/ports/outbound/tts.port'; +import { DEVICE_GATEWAY_PORT } from './core/ports/outbound/device-gateway.port'; @Module({ imports: [ @@ -63,6 +68,25 @@ import { STT_PORT } from './core/ports/outbound/stt.port'; provide: STT_PORT, useClass: DeepgramAdapter, }, + { + provide: LLM_PORT, + inject: [ConfigService], + useFactory: (configService: ConfigService) => { + const provider = configService.get('LLM_PROVIDER', 'anthropic'); + if (provider === 'openai') { + return new OpenAIAdapter(configService); + } + return new AnthropicAdapter(configService); + }, + }, + { + provide: TTS_PORT, + useClass: ElevenLabsAdapter, + }, + { + provide: DEVICE_GATEWAY_PORT, + useExisting: RobotGateway, + }, ], }) -export class AppModule { } +export class AppModule {} diff --git a/apps/backend/src/core/ports/outbound/device-gateway.port.ts b/apps/backend/src/core/ports/outbound/device-gateway.port.ts new file mode 100644 index 0000000..b3ebbe4 --- /dev/null +++ b/apps/backend/src/core/ports/outbound/device-gateway.port.ts @@ -0,0 +1,8 @@ +export interface IDeviceGatewayPort { + sendAudioChunk(deviceId: string, chunk: Buffer): void; + sendStatus(deviceId: string, state: 'listening' | 'thinking' | 'speaking' | 'idle'): void; + sendNotification(deviceId: string, payload: Record): void; + isDeviceConnected(deviceId: string): boolean; +} + +export const DEVICE_GATEWAY_PORT = Symbol('DEVICE_GATEWAY_PORT'); diff --git a/apps/backend/src/core/ports/outbound/stt.port.ts b/apps/backend/src/core/ports/outbound/stt.port.ts index 1658a7d..93ad2a3 100644 --- a/apps/backend/src/core/ports/outbound/stt.port.ts +++ b/apps/backend/src/core/ports/outbound/stt.port.ts @@ -4,11 +4,14 @@ export interface TranscriptionResult { isFinal: boolean; } -export interface ISTTPort { - transcribe(audioChunk: Buffer, sampleRate: number): Promise; - startStream(onResult: (result: TranscriptionResult) => void): Promise; - endStream(): Promise; +export interface ISTTStream { sendAudio(chunk: Buffer): void; + close(): Promise; +} + +export interface ISTTPort { + openStream(onResult: (result: TranscriptionResult) => void): Promise; + transcribe(audioChunk: Buffer, sampleRate: number): Promise; } export const STT_PORT = Symbol('STT_PORT'); diff --git a/apps/backend/src/core/services/conversation.service.ts b/apps/backend/src/core/services/conversation.service.ts index f935927..0333122 100644 --- a/apps/backend/src/core/services/conversation.service.ts +++ b/apps/backend/src/core/services/conversation.service.ts @@ -1,76 +1,139 @@ -import { Inject, Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable, Logger, forwardRef } from '@nestjs/common'; import { IConversationPort } from '../ports/inbound/conversation.port'; -import { ISTTPort, STT_PORT, TranscriptionResult } from '../ports/outbound/stt.port'; +import { ISTTPort, ISTTStream, STT_PORT, TranscriptionResult } from '../ports/outbound/stt.port'; +import { ILLMPort, LLM_PORT, LLMMessage } from '../ports/outbound/llm.port'; +import { ITTSPort, TTS_PORT } from '../ports/outbound/tts.port'; +import { IDeviceGatewayPort, DEVICE_GATEWAY_PORT } from '../ports/outbound/device-gateway.port'; interface ActiveSession { - deviceId: string; - transcription: string; + deviceId: string; + transcription: string; + messages: LLMMessage[]; + sttStream: ISTTStream | null; } +const SYSTEM_PROMPT = `Tu es Ti-Pote, un petit robot de bureau animatronique, chaleureux et serviable. +Tu parles en français. Tu es concis dans tes réponses car elles seront lues à voix haute. +Tu as une personnalité enjouée mais tu restes utile et pertinent. +Réponds en 1 à 3 phrases maximum.`; + @Injectable() export class ConversationService implements IConversationPort { - private readonly logger = new Logger(ConversationService.name); - private readonly activeSessions = new Map(); + private readonly logger = new Logger(ConversationService.name); + private readonly activeSessions = new Map(); - constructor( - @Inject(STT_PORT) private readonly sttPort: ISTTPort, - ) { } + constructor( + @Inject(STT_PORT) private readonly sttPort: ISTTPort, + @Inject(LLM_PORT) private readonly llmPort: ILLMPort, + @Inject(TTS_PORT) private readonly ttsPort: ITTSPort, + @Inject(forwardRef(() => DEVICE_GATEWAY_PORT)) private readonly deviceGateway: IDeviceGatewayPort, + ) {} - async startListening(deviceId: string): Promise { - this.logger.log(`Start listening for device ${deviceId}`); + async startListening(deviceId: string): Promise { + this.logger.log(`Start listening for device ${deviceId}`); - const session: ActiveSession = { - deviceId, - transcription: '', - }; - - this.activeSessions.set(deviceId, session); - - await this.sttPort.startStream((result: TranscriptionResult) => { - this.logger.debug(`STT [${deviceId}]: "${result.text}" (final: ${result.isFinal}, confidence: ${result.confidence})`); - - if (result.isFinal) { - session.transcription += result.text + ' '; - } - }); + // Close previous STT stream if any + const existing = this.activeSessions.get(deviceId); + if (existing?.sttStream) { + await existing.sttStream.close(); } - processAudioChunk(deviceId: string, chunk: Buffer, sampleRate: number): void { - const session = this.activeSessions.get(deviceId); - if (!session) { - this.logger.warn(`No active session for device ${deviceId}, ignoring audio chunk`); - return; - } + const session: ActiveSession = { + deviceId, + transcription: '', + messages: existing?.messages ?? [], + sttStream: null, + }; - this.sttPort.sendAudio(chunk); + this.activeSessions.set(deviceId, session); + + const sttStream = await this.sttPort.openStream((result: TranscriptionResult) => { + this.logger.debug( + `STT [${deviceId}]: "${result.text}" (final: ${result.isFinal}, confidence: ${result.confidence})`, + ); + + if (result.isFinal) { + session.transcription += result.text + ' '; + } + }); + + session.sttStream = sttStream; + } + + processAudioChunk(deviceId: string, chunk: Buffer, sampleRate: number): void { + const session = this.activeSessions.get(deviceId); + if (!session?.sttStream) { + this.logger.warn(`No active STT stream for device ${deviceId}, ignoring audio chunk`); + return; } - async stopListening(deviceId: string): Promise { - const session = this.activeSessions.get(deviceId); - if (!session) { - this.logger.warn(`No active session for device ${deviceId}`); - return null; - } + session.sttStream.sendAudio(chunk); + } - await this.sttPort.endStream(); - - const finalText = session.transcription.trim() || null; - this.activeSessions.delete(deviceId); - - this.logger.log(`Final transcription for ${deviceId}: "${finalText}"`); - - // TODO: plus tard, envoyer finalText au LLM via ILLMPort - - return finalText; + async stopListening(deviceId: string): Promise { + const session = this.activeSessions.get(deviceId); + if (!session) { + this.logger.warn(`No active session for device ${deviceId}`); + return null; } - interrupt(deviceId: string): void { - const session = this.activeSessions.get(deviceId); - if (!session) return; - - this.logger.log(`Interrupting session for device ${deviceId}`); - this.activeSessions.delete(deviceId); - - // TODO: plus tard, couper aussi le TTS en cours + if (session.sttStream) { + await session.sttStream.close(); + session.sttStream = null; } + + const finalText = session.transcription.trim() || null; + this.logger.log(`Final transcription for ${deviceId}: "${finalText}"`); + + if (!finalText) { + return null; + } + + session.messages.push({ role: 'user', content: finalText }); + session.transcription = ''; + + try { + this.deviceGateway.sendStatus(deviceId, 'thinking'); + + const llmMessages: LLMMessage[] = [ + { role: 'system', content: SYSTEM_PROMPT }, + ...session.messages, + ]; + + const llmResponse = await this.llmPort.chat(llmMessages); + const responseText = llmResponse.content ?? ''; + this.logger.log(`LLM response for ${deviceId}: "${responseText}"`); + + session.messages.push({ role: 'assistant', content: responseText }); + + if (responseText) { + this.deviceGateway.sendStatus(deviceId, 'speaking'); + + const audioBuffer = await this.ttsPort.synthesize(responseText); + this.logger.debug(`TTS complete: ${audioBuffer.length} bytes`); + this.deviceGateway.sendAudioChunk(deviceId, audioBuffer); + } + + this.deviceGateway.sendStatus(deviceId, 'idle'); + } catch (error) { + this.logger.error(`Error processing conversation for ${deviceId}:`, error); + this.deviceGateway.sendStatus(deviceId, 'idle'); + } + + return finalText; + } + + interrupt(deviceId: string): void { + const session = this.activeSessions.get(deviceId); + if (!session) return; + + this.logger.log(`Interrupting session for device ${deviceId}`); + + if (session.sttStream) { + session.sttStream.close(); + session.sttStream = null; + } + + this.activeSessions.delete(deviceId); + } } diff --git a/apps/simulator/src/App.tsx b/apps/simulator/src/App.tsx index 5199cf3..e7d7ae9 100644 --- a/apps/simulator/src/App.tsx +++ b/apps/simulator/src/App.tsx @@ -21,8 +21,10 @@ const STATE_LABELS: Record = { function App() { const [serverUrl, setServerUrl] = useState('http://localhost:3000'); const [deviceToken, setDeviceToken] = useState(''); + const [conversationActive, setConversationActive] = useState(false); const { state, connected, logs, connect, disconnect, emit, clearLogs } = useSocket(); const logsEndRef = useRef(null); + const prevStateRef = useRef('disconnected'); const onAudioChunk = useCallback( (chunk: ArrayBuffer, sampleRate: number) => { @@ -37,6 +39,17 @@ function App() { const { recording, start: startMic, stop: stopMic } = useMicrophone({ onAudioChunk, onSpeechEnd }); + // Auto-restart listening when Ti-Pote finishes speaking + useEffect(() => { + const prevState = prevStateRef.current; + prevStateRef.current = state; + + if (conversationActive && state === 'idle' && (prevState === 'speaking' || prevState === 'thinking')) { + emit('wake_word_detected'); + startMic(); + } + }, [state, conversationActive, emit, startMic]); + useEffect(() => { logsEndRef.current?.scrollIntoView({ behavior: 'smooth' }); }, [logs]); @@ -44,6 +57,7 @@ function App() { const handleConnect = () => { if (connected) { disconnect(); + setConversationActive(false); } else { if (!deviceToken.trim()) return alert('Device token requis'); connect(serverUrl, deviceToken.trim()); @@ -51,11 +65,13 @@ function App() { }; const handleWakeWord = () => { + setConversationActive(true); emit('wake_word_detected'); if (!recording) startMic(); }; const handleInterrupt = () => { + setConversationActive(false); emit('user_interrupt'); if (recording) stopMic(); }; diff --git a/apps/simulator/src/hooks/useAudioPlayer.ts b/apps/simulator/src/hooks/useAudioPlayer.ts new file mode 100644 index 0000000..a4d2304 --- /dev/null +++ b/apps/simulator/src/hooks/useAudioPlayer.ts @@ -0,0 +1,60 @@ +import { useRef, useCallback } from 'react'; + +const SAMPLE_RATE = 16000; + +export function useAudioPlayer() { + const contextRef = useRef(null); + + const getContext = useCallback(() => { + if (!contextRef.current || contextRef.current.state === 'closed') { + contextRef.current = new AudioContext({ sampleRate: SAMPLE_RATE }); + } + if (contextRef.current.state === 'suspended') { + contextRef.current.resume(); + } + return contextRef.current; + }, []); + + const playChunk = useCallback( + (base64: string) => { + const ctx = getContext(); + + // Decode base64 → bytes + const binaryStr = atob(base64); + const bytes = new Uint8Array(binaryStr.length); + for (let i = 0; i < binaryStr.length; i++) { + bytes[i] = binaryStr.charCodeAt(i); + } + + // Ensure even byte count for Int16 + const evenLength = bytes.length - (bytes.length % 2); + const int16 = new Int16Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + evenLength)); + + // Int16 PCM → Float32 + const float32 = new Float32Array(int16.length); + for (let i = 0; i < int16.length; i++) { + float32[i] = int16[i] / 0x7fff; + } + + const buffer = ctx.createBuffer(1, float32.length, SAMPLE_RATE); + buffer.copyToChannel(float32, 0); + + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(ctx.destination); + source.start(); + }, + [getContext], + ); + + const flush = useCallback(() => {}, []); + + const stop = useCallback(() => { + if (contextRef.current) { + contextRef.current.close(); + contextRef.current = null; + } + }, []); + + return { playChunk, flush, stop }; +} diff --git a/apps/simulator/src/hooks/useMicrophone.ts b/apps/simulator/src/hooks/useMicrophone.ts index c98dc21..74bbb6c 100644 --- a/apps/simulator/src/hooks/useMicrophone.ts +++ b/apps/simulator/src/hooks/useMicrophone.ts @@ -3,16 +3,51 @@ import { useRef, useState, useCallback } from 'react'; interface UseMicrophoneOptions { onAudioChunk: (chunk: ArrayBuffer, sampleRate: number) => void; onSpeechEnd: () => void; + /** Silence duration in ms before triggering speech end (default: 1500) */ + silenceTimeout?: number; + /** RMS threshold below which audio is considered silence (default: 0.01) */ + silenceThreshold?: number; } -export function useMicrophone({ onAudioChunk, onSpeechEnd }: UseMicrophoneOptions) { +export function useMicrophone({ + onAudioChunk, + onSpeechEnd, + silenceTimeout = 1500, + silenceThreshold = 0.01, +}: UseMicrophoneOptions) { const [recording, setRecording] = useState(false); const streamRef = useRef(null); const contextRef = useRef(null); const processorRef = useRef(null); + const silenceTimerRef = useRef | null>(null); + const hasSpeechRef = useRef(false); + const stoppedRef = useRef(false); + + const clearSilenceTimer = useCallback(() => { + if (silenceTimerRef.current) { + clearTimeout(silenceTimerRef.current); + silenceTimerRef.current = null; + } + }, []); + + const cleanup = useCallback(() => { + clearSilenceTimer(); + processorRef.current?.disconnect(); + contextRef.current?.close(); + streamRef.current?.getTracks().forEach((t) => t.stop()); + processorRef.current = null; + contextRef.current = null; + streamRef.current = null; + hasSpeechRef.current = false; + stoppedRef.current = false; + setRecording(false); + }, [clearSilenceTimer]); const start = useCallback(async () => { try { + stoppedRef.current = false; + hasSpeechRef.current = false; + const stream = await navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true }, }); @@ -22,19 +57,42 @@ export function useMicrophone({ onAudioChunk, onSpeechEnd }: UseMicrophoneOption contextRef.current = context; const source = context.createMediaStreamSource(stream); - // 4096 samples per chunk at 16kHz = ~256ms per chunk const processor = context.createScriptProcessor(4096, 1, 1); processorRef.current = processor; processor.onaudioprocess = (e) => { + if (stoppedRef.current) return; + const float32 = e.inputBuffer.getChannelData(0); - // Convert Float32 to Int16 PCM + + // Calculate RMS volume + let sum = 0; + for (let i = 0; i < float32.length; i++) { + sum += float32[i] * float32[i]; + } + const rms = Math.sqrt(sum / float32.length); + + // Convert and send audio const int16 = new Int16Array(float32.length); for (let i = 0; i < float32.length; i++) { const s = Math.max(-1, Math.min(1, float32[i])); int16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; } onAudioChunk(int16.buffer, context.sampleRate); + + // VAD logic + if (rms > silenceThreshold) { + hasSpeechRef.current = true; + clearSilenceTimer(); + } else if (hasSpeechRef.current && !silenceTimerRef.current) { + // Speech detected before, now silence — start countdown + silenceTimerRef.current = setTimeout(() => { + if (stoppedRef.current) return; + stoppedRef.current = true; + cleanup(); + onSpeechEnd(); + }, silenceTimeout); + } }; source.connect(processor); @@ -43,18 +101,14 @@ export function useMicrophone({ onAudioChunk, onSpeechEnd }: UseMicrophoneOption } catch (err) { console.error('Microphone access denied:', err); } - }, [onAudioChunk]); + }, [onAudioChunk, onSpeechEnd, silenceTimeout, silenceThreshold, clearSilenceTimer, cleanup]); const stop = useCallback(() => { - processorRef.current?.disconnect(); - contextRef.current?.close(); - streamRef.current?.getTracks().forEach((t) => t.stop()); - processorRef.current = null; - contextRef.current = null; - streamRef.current = null; - setRecording(false); + if (stoppedRef.current) return; + stoppedRef.current = true; + cleanup(); onSpeechEnd(); - }, [onSpeechEnd]); + }, [onSpeechEnd, cleanup]); return { recording, start, stop }; } diff --git a/apps/simulator/src/hooks/useSocket.ts b/apps/simulator/src/hooks/useSocket.ts index 3b7b74b..083df99 100644 --- a/apps/simulator/src/hooks/useSocket.ts +++ b/apps/simulator/src/hooks/useSocket.ts @@ -1,5 +1,6 @@ import { useRef, useState, useCallback } from 'react'; import { io, Socket } from 'socket.io-client'; +import { useAudioPlayer } from './useAudioPlayer'; export type RobotState = 'disconnected' | 'idle' | 'listening' | 'thinking' | 'speaking'; @@ -15,6 +16,7 @@ export function useSocket() { const [state, setState] = useState('disconnected'); const [connected, setConnected] = useState(false); const [logs, setLogs] = useState([]); + const audioPlayer = useAudioPlayer(); const addLog = useCallback((direction: LogEntry['direction'], event: string, data?: string) => { setLogs((prev) => [...prev.slice(-200), { timestamp: new Date(), direction, event, data }]); @@ -50,11 +52,16 @@ export function useSocket() { socket.on('status', (payload: { state: RobotState }) => { setState(payload.state); addLog('in', 'status', payload.state); + if (payload.state === 'idle') { + audioPlayer.flush(); + } }); - socket.on('audio_chunk', (payload: { data: ArrayBuffer }) => { - addLog('in', 'audio_chunk', `${payload.data?.byteLength ?? 0} bytes`); - // TODO: play audio through speakers + socket.on('audio_chunk', (payload: { data: string }) => { + addLog('in', 'audio_chunk', `${payload.data?.length ?? 0} chars (base64)`); + if (payload.data) { + audioPlayer.playChunk(payload.data); + } }); socket.on('notification', (payload: Record) => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index da8350f..d86b7ef 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,6 +10,9 @@ importers: apps/backend: dependencies: + '@anthropic-ai/sdk': + specifier: ^0.80.0 + version: 0.80.0 '@deepgram/sdk': specifier: ^5.0.0 version: 5.0.0 @@ -52,9 +55,15 @@ importers: dotenv: specifier: ^17.3.1 version: 17.3.1 + elevenlabs: + specifier: ^1.59.0 + version: 1.59.0 ioredis: specifier: ^5.10.1 version: 5.10.1 + openai: + specifier: ^6.33.0 + version: 6.33.0(ws@8.18.3) passport: specifier: ^0.7.0 version: 0.7.0 @@ -196,6 +205,15 @@ packages: resolution: {integrity: sha512-J4Jarr0SohdrHcb40gTL4wGPCQ952IMWF1G/MSAQfBAPvA9ZKApYhpxcY7PmehVePve+ujpus1dGsJ7dPxz8Kg==} engines: {node: ^18.19.1 || ^20.11.1 || >=22.0.0, npm: ^6.11.0 || ^7.5.6 || >=8.0.0, yarn: '>= 1.13.0'} + '@anthropic-ai/sdk@0.80.0': + resolution: {integrity: sha512-WeXLn7zNVk3yjeshn+xZHvld6AoFUOR3Sep6pSoHho5YbSi6HwcirqgPA5ccFuW8QTVJAAU7N8uQQC6Wa9TG+g==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + '@babel/code-frame@7.29.0': resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} engines: {node: '>=6.9.0'} @@ -346,6 +364,10 @@ packages: peerDependencies: '@babel/core': ^7.0.0-0 + '@babel/runtime@7.29.2': + resolution: {integrity: sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==} + engines: {node: '>=6.9.0'} + '@babel/template@7.28.6': resolution: {integrity: sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==} engines: {node: '>=6.9.0'} @@ -1468,6 +1490,10 @@ packages: '@xtuc/long@4.2.2': resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==} + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} @@ -1586,6 +1612,9 @@ packages: array-timsort@1.0.3: resolution: {integrity: sha512-/+3GRL7dDAGEfM6TseQk/U+mi18TU2Ms9I3UlLdUMhz2hbvGNTKdj9xniwXfUqgYhHxRx0+8UnKkvlNwVU+cWQ==} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + available-typed-arrays@1.0.7: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} @@ -1788,6 +1817,13 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + command-exists@1.2.9: + resolution: {integrity: sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==} + commander@2.20.3: resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} @@ -1889,6 +1925,10 @@ packages: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + denque@2.1.0: resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} engines: {node: '>=0.10'} @@ -1941,6 +1981,10 @@ packages: electron-to-chromium@1.5.328: resolution: {integrity: sha512-QNQ5l45DzYytThO21403XN3FvK0hOkWDG8viNf6jqS42msJ8I4tGDSpBCgvDRRPnkffafiwAym2X2eHeGD2V0w==} + elevenlabs@1.59.0: + resolution: {integrity: sha512-OVKOd+lxNya8h4Rn5fcjv00Asd+DGWfTT6opGrQ16sTI+1HwdLn/kYtjl8tRMhDXbNmksD/9SBRKjb9neiUuVg==} + deprecated: This package has moved to @elevenlabs/elevenlabs-js + emittery@0.13.1: resolution: {integrity: sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==} engines: {node: '>=12'} @@ -1988,6 +2032,10 @@ packages: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + esbuild@0.27.4: resolution: {integrity: sha512-Rq4vbHnYkK5fws5NF7MYTU68FPRE1ajX7heQ/8QXXWqNgqqJ/GkmmyxIzUnf2Sr/bakf8l54716CcMGHYhMrrQ==} engines: {node: '>=18'} @@ -2087,6 +2135,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + events@3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} @@ -2179,6 +2231,18 @@ packages: typescript: '>3.6.0' webpack: ^5.11.0 + form-data-encoder@4.1.0: + resolution: {integrity: sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==} + engines: {node: '>= 18'} + + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + + formdata-node@6.0.3: + resolution: {integrity: sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==} + engines: {node: '>= 18'} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -2566,6 +2630,10 @@ packages: json-parse-even-better-errors@2.3.1: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} @@ -2869,6 +2937,15 @@ packages: node-emoji@1.11.0: resolution: {integrity: sha512-wo2DpQkQp7Sjm2A0cq+sN7EHKO6Sl0ctXeBdFZrL9T9+UywORbufTcTZxom8YqpLQt/FqNMUkOpkZrJVYSKD3A==} + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + node-gyp-build@4.8.4: resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} hasBin: true @@ -2910,6 +2987,18 @@ packages: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} + openai@6.33.0: + resolution: {integrity: sha512-xAYN1W3YsDXJWA5F277135YfkEk6H7D3D6vWwRhJ3OEkzRgcyK8z/P5P9Gyi/wB4N8kK9kM5ZjprfvyHagKmpw==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -3096,6 +3185,10 @@ packages: resolution: {integrity: sha512-oG4T3wCbfeuvljnyAzhBvpN45E8iOTXCU/TD3zXW80HA3dQ4ahdqMkWGiPWZvjpQwlbyHrPTWUAqUzGzv4l1JQ==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -3135,6 +3228,10 @@ packages: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -3439,6 +3536,12 @@ packages: resolution: {integrity: sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==} engines: {node: '>=14.16'} + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + ts-api-utils@2.5.0: resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} engines: {node: '>=18.12'} @@ -3634,6 +3737,9 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + url-join@4.0.1: + resolution: {integrity: sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -3713,6 +3819,9 @@ packages: wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + webpack-node-externals@3.0.0: resolution: {integrity: sha512-LnL6Z3GGDPht/AigwRh2dvL9PQPFQ8skEpVrWZXLWBYmqcaojHNN0onvHzie6rq7EWKrrBfPYqNEzTJgiwEQDQ==} engines: {node: '>=6'} @@ -3731,6 +3840,9 @@ packages: webpack-cli: optional: true + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + which-typed-array@1.1.20: resolution: {integrity: sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg==} engines: {node: '>= 0.4'} @@ -3869,6 +3981,10 @@ snapshots: transitivePeerDependencies: - chokidar + '@anthropic-ai/sdk@0.80.0': + dependencies: + json-schema-to-ts: 3.1.1 + '@babel/code-frame@7.29.0': dependencies: '@babel/helper-validator-identifier': 7.28.5 @@ -4033,6 +4149,8 @@ snapshots: '@babel/core': 7.29.0 '@babel/helper-plugin-utils': 7.28.6 + '@babel/runtime@7.29.2': {} + '@babel/template@7.28.6': dependencies: '@babel/code-frame': 7.29.0 @@ -5220,6 +5338,10 @@ snapshots: '@xtuc/long@4.2.2': {} + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + accepts@1.3.8: dependencies: mime-types: 2.1.35 @@ -5321,6 +5443,8 @@ snapshots: array-timsort@1.0.3: {} + asynckit@0.4.0: {} + available-typed-arrays@1.0.7: dependencies: possible-typed-array-names: 1.1.0 @@ -5547,6 +5671,12 @@ snapshots: color-name@1.1.4: {} + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + command-exists@1.2.9: {} + commander@2.20.3: {} commander@4.1.1: {} @@ -5626,6 +5756,8 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 + delayed-stream@1.0.0: {} + denque@2.1.0: {} depd@2.0.0: {} @@ -5662,6 +5794,20 @@ snapshots: electron-to-chromium@1.5.328: {} + elevenlabs@1.59.0: + dependencies: + command-exists: 1.2.9 + execa: 5.1.1 + form-data: 4.0.5 + form-data-encoder: 4.1.0 + formdata-node: 6.0.3 + node-fetch: 2.7.0 + qs: 6.15.0 + readable-stream: 4.7.0 + url-join: 4.0.1 + transitivePeerDependencies: + - encoding + emittery@0.13.1: {} emoji-regex@8.0.0: {} @@ -5720,6 +5866,13 @@ snapshots: dependencies: es-errors: 1.3.0 + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + esbuild@0.27.4: optionalDependencies: '@esbuild/aix-ppc64': 0.27.4 @@ -5846,6 +5999,8 @@ snapshots: etag@1.8.1: {} + event-target-shim@5.0.1: {} + events@3.3.0: {} execa@5.1.1: @@ -5991,6 +6146,18 @@ snapshots: typescript: 5.9.3 webpack: 5.104.1 + form-data-encoder@4.1.0: {} + + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + + formdata-node@6.0.3: {} + forwarded@0.2.0: {} fresh@2.0.0: {} @@ -6559,6 +6726,11 @@ snapshots: json-parse-even-better-errors@2.3.1: {} + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.29.2 + ts-algebra: 2.0.0 + json-schema-traverse@0.4.1: {} json-schema-traverse@1.0.0: {} @@ -6799,6 +6971,10 @@ snapshots: dependencies: lodash: 4.17.23 + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + node-gyp-build@4.8.4: {} node-int64@0.4.0: {} @@ -6829,6 +7005,10 @@ snapshots: dependencies: mimic-fn: 2.1.0 + openai@6.33.0(ws@8.18.3): + optionalDependencies: + ws: 8.18.3 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -7001,6 +7181,8 @@ snapshots: ansi-styles: 5.2.0 react-is: 18.3.1 + process@0.11.10: {} + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 @@ -7038,6 +7220,14 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + readdirp@4.1.2: {} redis-errors@1.2.0: {} @@ -7383,6 +7573,10 @@ snapshots: '@tokenizer/token': 0.3.0 ieee754: 1.2.1 + tr46@0.0.3: {} + + ts-algebra@2.0.0: {} + ts-api-utils@2.5.0(typescript@5.8.3): dependencies: typescript: 5.8.3 @@ -7554,6 +7748,8 @@ snapshots: dependencies: punycode: 2.3.1 + url-join@4.0.1: {} + util-deprecate@1.0.2: {} utils-merge@1.0.1: {} @@ -7599,6 +7795,8 @@ snapshots: dependencies: defaults: 1.0.4 + webidl-conversions@3.0.1: {} + webpack-node-externals@3.0.0: {} webpack-sources@3.3.4: {} @@ -7635,6 +7833,11 @@ snapshots: - esbuild - uglify-js + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + which-typed-array@1.1.20: dependencies: available-typed-arrays: 1.0.7