From 9ee09afa77d9408dd14468e1ddd26f4de22da11d Mon Sep 17 00:00:00 2001 From: ordinarthur <@arthurbarre.js@gmail.com> Date: Tue, 14 Apr 2026 01:40:52 +0200 Subject: [PATCH] on avance --- .../adapters/inbound/websocket/app.gateway.ts | 204 +++++++++++++ .../inbound/websocket/robot.gateway.ts | 29 ++ apps/backend/src/app.module.ts | 2 + .../src/core/services/device.service.ts | 12 + .../core/services/health-telemetry.service.ts | 16 + apps/frontend/package.json | 1 + apps/frontend/src/context/AuthContext.tsx | 6 + apps/frontend/src/hooks/useDeviceSocket.ts | 281 ++++++++++++++++++ apps/frontend/src/lib/socket.ts | 72 +++++ apps/frontend/src/pages/DashboardPage.tsx | 4 +- apps/frontend/src/pages/DeviceDetailPage.tsx | 177 ++++++++--- .../src/config/hardware.config.ts | 2 +- apps/robot-client/src/main.ts | 50 ++++ .../src/services/orchestrator.service.ts | 26 +- .../src/services/wake-word.service.ts | 79 ++++- .../src/transport/cloud-socket.ts | 14 + apps/robot-client/src/utils/sounds.ts | 80 +++++ pnpm-lock.yaml | 3 + tools/pi-gen-tipote/deploy.sh | 7 +- 19 files changed, 1013 insertions(+), 52 deletions(-) create mode 100644 apps/backend/src/adapters/inbound/websocket/app.gateway.ts create mode 100644 apps/frontend/src/hooks/useDeviceSocket.ts create mode 100644 apps/frontend/src/lib/socket.ts create mode 100644 apps/robot-client/src/utils/sounds.ts diff --git a/apps/backend/src/adapters/inbound/websocket/app.gateway.ts b/apps/backend/src/adapters/inbound/websocket/app.gateway.ts new file mode 100644 index 0000000..7b66a41 --- /dev/null +++ b/apps/backend/src/adapters/inbound/websocket/app.gateway.ts @@ -0,0 +1,204 @@ +import { + WebSocketGateway, + WebSocketServer, + OnGatewayConnection, + OnGatewayDisconnect, + SubscribeMessage, + MessageBody, + ConnectedSocket, +} from '@nestjs/websockets'; +import { Inject, Logger, forwardRef } from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import { OnEvent } from '@nestjs/event-emitter'; +import { Server, Socket } from 'socket.io'; +import { JwtPayload } from '../rest/auth/strategies/jwt.strategy'; +import { RobotGateway } from './robot.gateway'; +import { DeviceService } from '../../../core/services/device.service'; + +/** + * WebSocket gateway for the desktop/web app. + * + * Authenticates users via JWT (same tokens as REST API). + * Broadcasts real-time events: + * - `health_report` — device health telemetry + * - `device_status` — device online/offline changes + * - `device_log` — live log entries + */ + +interface AppSocket extends Socket { + data: { + userId: string; + homeId: string; + }; +} + +@WebSocketGateway({ + namespace: '/ws/app', + cors: { origin: '*' }, +}) +export class AppGateway implements OnGatewayConnection, OnGatewayDisconnect { + @WebSocketServer() + server!: Server; + + private readonly logger = new Logger(AppGateway.name); + + constructor( + private readonly jwtService: JwtService, + @Inject(forwardRef(() => RobotGateway)) + private readonly robotGateway: RobotGateway, + private readonly deviceService: DeviceService, + ) {} + + // ── Connection lifecycle ── + + async handleConnection(client: AppSocket) { + try { + const token = + client.handshake.auth?.token || + client.handshake.headers?.authorization?.replace('Bearer ', ''); + + if (!token) { + this.logger.warn('App WS rejected: no token'); + client.disconnect(); + return; + } + + const payload = this.jwtService.verify(token); + if (payload.type !== 'user') { + this.logger.warn('App WS rejected: not a user token'); + client.disconnect(); + return; + } + + client.data.userId = payload.sub; + client.data.homeId = payload.homeId; + + // Join a room scoped to the user's home so we can broadcast per-home + await client.join(`home:${payload.homeId}`); + + this.logger.log(`App client connected: user ${payload.sub} (home ${payload.homeId})`); + } catch { + this.logger.warn('App WS rejected: invalid token'); + client.disconnect(); + } + } + + handleDisconnect(client: AppSocket) { + const userId = client.data?.userId; + if (userId) { + this.logger.log(`App client disconnected: user ${userId}`); + } + } + + // ── Subscribe to device for targeted updates ── + + @SubscribeMessage('subscribe_device') + handleSubscribeDevice( + @ConnectedSocket() client: AppSocket, + @MessageBody() data: { deviceId: string }, + ) { + void client.join(`device:${data.deviceId}`); + this.logger.debug(`User ${client.data.userId} subscribed to device ${data.deviceId}`); + } + + @SubscribeMessage('unsubscribe_device') + handleUnsubscribeDevice( + @ConnectedSocket() client: AppSocket, + @MessageBody() data: { deviceId: string }, + ) { + void client.leave(`device:${data.deviceId}`); + } + + // ── Commands from desktop app → robot ── + + @SubscribeMessage('trigger_conversation') + async handleTriggerConversation( + @ConnectedSocket() client: AppSocket, + @MessageBody() data: { deviceId: string }, + ) { + // Verify the user owns this device + const device = await this.deviceService.findById(data.deviceId); + if (!device || device.homeId !== client.data.homeId) { + return { error: 'Device not found' }; + } + + const sent = this.robotGateway.sendRemoteTrigger(data.deviceId); + if (!sent) { + return { error: 'Device not connected' }; + } + + this.logger.log(`User ${client.data.userId} triggered conversation on ${data.deviceId}`); + return { ok: true }; + } + + @SubscribeMessage('set_trigger_mode') + async handleSetTriggerMode( + @ConnectedSocket() client: AppSocket, + @MessageBody() data: { deviceId: string; mode: string }, + ) { + if (!['wakeword', 'keyboard'].includes(data.mode)) { + return { error: 'Invalid mode. Must be "wakeword" or "keyboard".' }; + } + + const device = await this.deviceService.findById(data.deviceId); + if (!device || device.homeId !== client.data.homeId) { + return { error: 'Device not found' }; + } + + const sent = this.robotGateway.sendSetTriggerMode(data.deviceId, data.mode); + if (!sent) { + return { error: 'Device not connected' }; + } + + // Broadcast the mode change to all subscribed app clients + this.server.to(`device:${data.deviceId}`).emit('trigger_mode_changed', { + deviceId: data.deviceId, + mode: data.mode, + }); + + this.logger.log(`User ${client.data.userId} set trigger mode to ${data.mode} on ${data.deviceId}`); + return { ok: true }; + } + + // ── Event listeners (from NestJS EventEmitter) ── + + @OnEvent('device.health_report') + handleHealthReportEvent(payload: { + deviceId: string; + homeId: string; + report: Record; + alerts: string[]; + }) { + // Broadcast to all app clients subscribed to this device + this.server.to(`device:${payload.deviceId}`).emit('health_report', { + deviceId: payload.deviceId, + report: payload.report, + alerts: payload.alerts, + }); + } + + @OnEvent('device.status_changed') + handleDeviceStatusEvent(payload: { + deviceId: string; + homeId: string; + status: string; + }) { + // Broadcast to the entire home (all clients see device go online/offline) + this.server.to(`home:${payload.homeId}`).emit('device_status', { + deviceId: payload.deviceId, + status: payload.status, + }); + } + + @OnEvent('device.log') + handleDeviceLogEvent(payload: { + deviceId: string; + level: number; + msg: string; + loggerName: string | null; + context: Record | null; + loggedAt: string; + }) { + this.server.to(`device:${payload.deviceId}`).emit('device_log', payload); + } +} diff --git a/apps/backend/src/adapters/inbound/websocket/robot.gateway.ts b/apps/backend/src/adapters/inbound/websocket/robot.gateway.ts index d3925bd..cfaa163 100644 --- a/apps/backend/src/adapters/inbound/websocket/robot.gateway.ts +++ b/apps/backend/src/adapters/inbound/websocket/robot.gateway.ts @@ -175,4 +175,33 @@ export class RobotGateway implements OnGatewayConnection, OnGatewayDisconnect, I isDeviceConnected(deviceId: string): boolean { return this.connectedDevices.has(deviceId); } + + /** + * Send a remote trigger to a connected device (starts a conversation). + */ + sendRemoteTrigger(deviceId: string): boolean { + const socket = this.connectedDevices.get(deviceId); + if (!socket) return false; + this.logger.log(`Remote trigger sent to device ${deviceId}`); + socket.emit('remote_trigger'); + return true; + } + + /** + * Send a trigger mode change to a connected device. + */ + sendSetTriggerMode(deviceId: string, mode: string): boolean { + const socket = this.connectedDevices.get(deviceId); + if (!socket) return false; + this.logger.log(`Set trigger mode to ${mode} on device ${deviceId}`); + socket.emit('set_trigger_mode', { mode }); + return true; + } + + /** + * Get the homeId for a connected device. + */ + getDeviceHomeId(deviceId: string): string | null { + return this.connectedDevices.get(deviceId)?.data.homeId ?? null; + } } diff --git a/apps/backend/src/app.module.ts b/apps/backend/src/app.module.ts index bc4bb02..c4327f6 100644 --- a/apps/backend/src/app.module.ts +++ b/apps/backend/src/app.module.ts @@ -27,6 +27,7 @@ import { HealthController } from './adapters/inbound/rest/health/health.controll import { LogsController } from './adapters/inbound/rest/logs/logs.controller'; import { PairingService } from './core/services/pairing.service'; import { RobotGateway } from './adapters/inbound/websocket/robot.gateway'; +import { AppGateway } from './adapters/inbound/websocket/app.gateway'; import { DeepgramAdapter } from './adapters/outbound/stt/deepgram.adapter'; import { AnthropicAdapter } from './adapters/outbound/llm/anthropic.adapter'; import { OpenAIAdapter } from './adapters/outbound/llm/openai.adapter'; @@ -75,6 +76,7 @@ import { CACHE_PORT } from './core/ports/outbound/cache.port'; PairingService, JwtStrategy, RobotGateway, + AppGateway, HealthTelemetryService, LogIngestionService, { diff --git a/apps/backend/src/core/services/device.service.ts b/apps/backend/src/core/services/device.service.ts index 425badd..ebc2e02 100644 --- a/apps/backend/src/core/services/device.service.ts +++ b/apps/backend/src/core/services/device.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; +import { EventEmitter2 } from '@nestjs/event-emitter'; import * as bcrypt from 'bcrypt'; import { Device, DeviceStatus } from '../domain/entities/device.entity'; @@ -9,6 +10,7 @@ export class DeviceService { constructor( @InjectRepository(Device) private readonly deviceRepository: Repository, + private readonly events: EventEmitter2, ) {} async findById(id: string): Promise { @@ -43,5 +45,15 @@ export class DeviceService { status, lastSeenAt: new Date(), }); + + // Broadcast status change to app clients + const device = await this.deviceRepository.findOne({ where: { id }, select: ['homeId'] }); + if (device) { + this.events.emit('device.status_changed', { + deviceId: id, + homeId: device.homeId, + status, + }); + } } } diff --git a/apps/backend/src/core/services/health-telemetry.service.ts b/apps/backend/src/core/services/health-telemetry.service.ts index 89d86dd..04c7b45 100644 --- a/apps/backend/src/core/services/health-telemetry.service.ts +++ b/apps/backend/src/core/services/health-telemetry.service.ts @@ -1,7 +1,9 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; +import { EventEmitter2 } from '@nestjs/event-emitter'; import { HealthReport } from '../domain/entities/health-report.entity'; +import { Device } from '../domain/entities/device.entity'; import { IHealthTelemetryPort, HealthReportPayload, @@ -23,6 +25,9 @@ export class HealthTelemetryService implements IHealthTelemetryPort { constructor( @InjectRepository(HealthReport) private readonly repo: Repository, + @InjectRepository(Device) + private readonly deviceRepo: Repository, + private readonly events: EventEmitter2, ) {} async ingestReport(deviceId: string, payload: HealthReportPayload): Promise { @@ -52,6 +57,17 @@ export class HealthTelemetryService implements IHealthTelemetryPort { } else { this.logger.debug({ deviceId }, 'Health report ingested'); } + + // Broadcast to connected app clients via EventEmitter → AppGateway + const device = await this.deviceRepo.findOne({ where: { id: deviceId }, select: ['homeId'] }); + if (device) { + this.events.emit('device.health_report', { + deviceId, + homeId: device.homeId, + report: payload, + alerts, + }); + } } async getLatestReports(deviceId: string, limit = 20): Promise { diff --git a/apps/frontend/package.json b/apps/frontend/package.json index 3249f76..2b4da28 100644 --- a/apps/frontend/package.json +++ b/apps/frontend/package.json @@ -16,6 +16,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^6.26.0", + "socket.io-client": "^4.8.3", "zustand": "^4.5.5" }, "devDependencies": { diff --git a/apps/frontend/src/context/AuthContext.tsx b/apps/frontend/src/context/AuthContext.tsx index 6874673..e935a8c 100644 --- a/apps/frontend/src/context/AuthContext.tsx +++ b/apps/frontend/src/context/AuthContext.tsx @@ -14,6 +14,7 @@ import { type Me, type RegisterInput, } from '../lib/api'; +import { getSocket, disconnectSocket } from '../lib/socket'; type AuthStatus = 'loading' | 'authenticated' | 'unauthenticated'; @@ -46,6 +47,8 @@ export function AuthProvider({ children }: { children: ReactNode }) { if (cancelled) return; setUser(me); setStatus('authenticated'); + // Connect WebSocket on session restore + getSocket().catch(() => {}); } catch { if (cancelled) return; setUser(null); @@ -63,6 +66,8 @@ export function AuthProvider({ children }: { children: ReactNode }) { const me = await api.me(); setUser(me); setStatus('authenticated'); + // Connect WebSocket after login + getSocket().catch(() => {}); }, []); const register = useCallback(async (input: RegisterInput) => { @@ -73,6 +78,7 @@ export function AuthProvider({ children }: { children: ReactNode }) { }, []); const logout = useCallback(async () => { + disconnectSocket(); await api.logout(); setUser(null); setStatus('unauthenticated'); diff --git a/apps/frontend/src/hooks/useDeviceSocket.ts b/apps/frontend/src/hooks/useDeviceSocket.ts new file mode 100644 index 0000000..fe29d59 --- /dev/null +++ b/apps/frontend/src/hooks/useDeviceSocket.ts @@ -0,0 +1,281 @@ +/** + * React hooks for real-time device updates via WebSocket. + */ + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { getSocket } from '../lib/socket'; +import type { HealthReport, LogEntry } from '../lib/api'; +import type { Socket } from 'socket.io-client'; + +export type TriggerMode = 'wakeword' | 'keyboard'; + +// ─── Level label mapping (pino levels) ──────────────────────────── + +const LEVEL_LABELS: Record = { + 10: 'trace', + 20: 'debug', + 30: 'info', + 40: 'warn', + 50: 'error', + 60: 'fatal', +}; + +// ─── useDeviceHealth ────────────────────────────────────────────── + +/** + * Subscribe to real-time health reports for a device. + * Returns the latest report, the history, and current alerts. + */ +export function useDeviceHealth(deviceId: string | undefined) { + const [latest, setLatest] = useState(null); + const [reports, setReports] = useState([]); + const [alerts, setAlerts] = useState([]); + const [connected, setConnected] = useState(false); + const socketRef = useRef(null); + + useEffect(() => { + if (!deviceId) return; + + let cancelled = false; + + (async () => { + try { + const sock = await getSocket(); + if (cancelled) return; + + socketRef.current = sock; + setConnected(sock.connected); + + // Subscribe to this device's updates + sock.emit('subscribe_device', { deviceId }); + + const onHealthReport = (data: { + deviceId: string; + report: HealthReport; + alerts: string[]; + }) => { + if (data.deviceId !== deviceId) return; + + setLatest(data.report); + setAlerts(data.alerts); + setReports((prev) => { + const next = [data.report, ...prev]; + return next.slice(0, 50); // Keep last 50 + }); + }; + + const onConnect = () => setConnected(true); + const onDisconnect = () => setConnected(false); + + sock.on('health_report', onHealthReport); + sock.on('connect', onConnect); + sock.on('disconnect', onDisconnect); + + return () => { + sock.emit('unsubscribe_device', { deviceId }); + sock.off('health_report', onHealthReport); + sock.off('connect', onConnect); + sock.off('disconnect', onDisconnect); + }; + } catch (err) { + console.warn('Failed to connect device socket:', err); + } + })(); + + return () => { + cancelled = true; + if (socketRef.current) { + socketRef.current.emit('unsubscribe_device', { deviceId }); + } + }; + }, [deviceId]); + + return { latest, reports, alerts, connected }; +} + +// ─── useDeviceStatus ────────────────────────────────────────────── + +/** + * Subscribe to device status changes (online/offline) for all devices in the home. + * Returns a map of deviceId → status. + */ +export function useDeviceStatus() { + const [statuses, setStatuses] = useState>({}); + + useEffect(() => { + let cancelled = false; + let sock: Socket | null = null; + + (async () => { + try { + sock = await getSocket(); + if (cancelled) return; + + const onStatus = (data: { deviceId: string; status: string }) => { + setStatuses((prev) => ({ ...prev, [data.deviceId]: data.status })); + }; + + sock.on('device_status', onStatus); + + return () => { + sock?.off('device_status', onStatus); + }; + } catch { + // Not authenticated yet + } + })(); + + return () => { + cancelled = true; + }; + }, []); + + return statuses; +} + +// ─── useDeviceLogs ──────────────────────────────────────────────── + +/** + * Subscribe to real-time log entries for a device. + */ +export function useDeviceLogs(deviceId: string | undefined, minLevel = 0) { + const [logs, setLogs] = useState([]); + const socketRef = useRef(null); + + const clearLogs = useCallback(() => setLogs([]), []); + + useEffect(() => { + if (!deviceId) return; + + let cancelled = false; + + (async () => { + try { + const sock = await getSocket(); + if (cancelled) return; + + socketRef.current = sock; + sock.emit('subscribe_device', { deviceId }); + + const onLog = (data: { + deviceId: string; + level: number; + msg: string; + loggerName: string | null; + context: Record | null; + loggedAt: string; + }) => { + if (data.deviceId !== deviceId) return; + if (data.level < minLevel) return; + + const entry: LogEntry = { + level: data.level, + levelLabel: LEVEL_LABELS[data.level] ?? 'info', + time: new Date(data.loggedAt).getTime(), + msg: data.msg, + name: data.loggerName ?? undefined, + ...(data.context ?? {}), + }; + + setLogs((prev) => { + const next = [entry, ...prev]; + return next.slice(0, 500); // Keep last 500 + }); + }; + + sock.on('device_log', onLog); + + return () => { + sock.emit('unsubscribe_device', { deviceId }); + sock.off('device_log', onLog); + }; + } catch { + // Not authenticated + } + })(); + + return () => { + cancelled = true; + if (socketRef.current) { + socketRef.current.emit('unsubscribe_device', { deviceId }); + } + }; + }, [deviceId, minLevel]); + + return { logs, clearLogs }; +} + +// ─── useDeviceCommands ──────────────────────────────────────────── + +/** + * Send commands to a device via WebSocket. + */ +export function useDeviceCommands(deviceId: string | undefined) { + const [triggerMode, setTriggerMode] = useState('keyboard'); + const [triggering, setTriggering] = useState(false); + + // Listen for trigger mode changes + useEffect(() => { + if (!deviceId) return; + + let cancelled = false; + + (async () => { + try { + const sock = await getSocket(); + if (cancelled) return; + + const onModeChanged = (data: { deviceId: string; mode: string }) => { + if (data.deviceId !== deviceId) return; + setTriggerMode(data.mode as TriggerMode); + }; + + sock.on('trigger_mode_changed', onModeChanged); + + return () => { + sock.off('trigger_mode_changed', onModeChanged); + }; + } catch { + // Not connected + } + })(); + + return () => { + cancelled = true; + }; + }, [deviceId]); + + const triggerConversation = useCallback(async () => { + if (!deviceId) return; + setTriggering(true); + try { + const sock = await getSocket(); + sock.emit('trigger_conversation', { deviceId }, (response: { ok?: boolean; error?: string }) => { + if (response?.error) { + console.warn('Trigger failed:', response.error); + } + setTriggering(false); + }); + } catch { + setTriggering(false); + } + }, [deviceId]); + + const changeTriggerMode = useCallback(async (mode: TriggerMode) => { + if (!deviceId) return; + try { + const sock = await getSocket(); + sock.emit('set_trigger_mode', { deviceId, mode }, (response: { ok?: boolean; error?: string }) => { + if (response?.error) { + console.warn('Set trigger mode failed:', response.error); + } else { + setTriggerMode(mode); + } + }); + } catch { + // Not connected + } + }, [deviceId]); + + return { triggerMode, triggerConversation, triggering, changeTriggerMode }; +} diff --git a/apps/frontend/src/lib/socket.ts b/apps/frontend/src/lib/socket.ts new file mode 100644 index 0000000..5ca3e91 --- /dev/null +++ b/apps/frontend/src/lib/socket.ts @@ -0,0 +1,72 @@ +/** + * Singleton Socket.IO client for the app WebSocket (`/ws/app`). + * + * Provides real-time events from the backend: + * - `health_report` — device health telemetry + * - `device_status` — device online/offline changes + * - `device_log` — live log entries + */ + +import { io, type Socket } from 'socket.io-client'; +import { storage } from './storage'; + +const BASE_URL = + (import.meta.env.VITE_API_URL as string | undefined)?.replace(/\/$/, '') || + 'http://localhost:3000'; + +const ACCESS_KEY = 'auth.accessToken'; + +let socket: Socket | null = null; + +/** + * Get or create the singleton socket connection. + * Connects to /ws/app with the current user JWT. + */ +export async function getSocket(): Promise { + if (socket?.connected) return socket; + + const token = await storage.get(ACCESS_KEY); + if (!token) throw new Error('No auth token for WebSocket'); + + if (socket) { + // Reconnect with fresh token + socket.auth = { token }; + socket.connect(); + return socket; + } + + socket = io(`${BASE_URL}/ws/app`, { + auth: { token }, + transports: ['websocket'], + autoConnect: true, + reconnection: true, + reconnectionAttempts: Infinity, + reconnectionDelay: 2000, + reconnectionDelayMax: 10000, + }); + + socket.on('connect', () => { + console.log('[ws/app] connected'); + }); + + socket.on('disconnect', (reason) => { + console.log('[ws/app] disconnected:', reason); + }); + + socket.on('connect_error', (err) => { + console.warn('[ws/app] connection error:', err.message); + }); + + return socket; +} + +/** + * Disconnect and destroy the singleton socket. + * Call this on logout. + */ +export function disconnectSocket(): void { + if (socket) { + socket.disconnect(); + socket = null; + } +} diff --git a/apps/frontend/src/pages/DashboardPage.tsx b/apps/frontend/src/pages/DashboardPage.tsx index 5b73706..8894e0f 100644 --- a/apps/frontend/src/pages/DashboardPage.tsx +++ b/apps/frontend/src/pages/DashboardPage.tsx @@ -3,9 +3,11 @@ import { Link } from 'react-router-dom'; import { Button, Card, StatusBadge } from '../components/ui'; import { useAuth } from '../context/AuthContext'; import { api, ApiError, type DeviceSummary } from '../lib/api'; +import { useDeviceStatus } from '../hooks/useDeviceSocket'; export function DashboardPage() { const { user, logout } = useAuth(); + const statusMap = useDeviceStatus(); const [devices, setDevices] = useState(null); const [error, setError] = useState(null); const [loading, setLoading] = useState(true); @@ -101,7 +103,7 @@ export function DashboardPage() { {d.id}

- +
diff --git a/apps/frontend/src/pages/DeviceDetailPage.tsx b/apps/frontend/src/pages/DeviceDetailPage.tsx index 2bf2a5d..bad8f3e 100644 --- a/apps/frontend/src/pages/DeviceDetailPage.tsx +++ b/apps/frontend/src/pages/DeviceDetailPage.tsx @@ -8,6 +8,7 @@ import { type HealthReport, type LogEntry, } from '../lib/api'; +import { useDeviceHealth, useDeviceLogs, useDeviceStatus, useDeviceCommands } from '../hooks/useDeviceSocket'; // ─── Level helpers ──────────────────────────────────────────────── @@ -36,20 +37,50 @@ export function DeviceDetailPage() { // Device info const [device, setDevice] = useState(null); + const [error, setError] = useState(null); - // Health - const [reports, setReports] = useState([]); - const [alerts, setAlerts] = useState([]); + // Remote commands + const { + triggerMode, + triggerConversation, + triggering, + changeTriggerMode, + } = useDeviceCommands(deviceId); + + // Real-time health via WebSocket + const { + latest: wsLatest, + reports: wsReports, + alerts: wsAlerts, + connected: wsConnected, + } = useDeviceHealth(deviceId); + + // Real-time device status via WebSocket + const statusMap = useDeviceStatus(); + const liveStatus = deviceId ? statusMap[deviceId] : undefined; + + // Initial health data loaded from REST (hydration) + const [initialReports, setInitialReports] = useState([]); + const [initialAlerts, setInitialAlerts] = useState([]); const [healthLoading, setHealthLoading] = useState(true); - // Logs - const [logs, setLogs] = useState([]); - const [logsTotal, setLogsTotal] = useState(0); - const [logsLoading, setLogsLoading] = useState(false); + // Merge: WS reports on top of initial REST data + const reports = wsReports.length > 0 ? wsReports : initialReports; + const alerts = wsAlerts.length > 0 ? wsAlerts : initialAlerts; + const latest = wsLatest ?? reports[0] ?? null; + + // Logs — real-time via WebSocket + initial load from REST const [logLevel, setLogLevel] = useState(0); const [logSearch, setLogSearch] = useState(''); + const { logs: wsLogs } = useDeviceLogs(deviceId, logLevel); + const [initialLogs, setInitialLogs] = useState([]); + const [logsTotal, setLogsTotal] = useState(0); + const [logsLoading, setLogsLoading] = useState(false); - const [error, setError] = useState(null); + // Merge: WS logs on top, then initial + const logs = [...wsLogs, ...initialLogs.filter( + (il) => !wsLogs.some((wl) => wl.time === il.time && wl.msg === il.msg), + )]; // ── Fetch device info ── @@ -64,31 +95,42 @@ export function DeviceDetailPage() { .catch(() => {}); }, [deviceId]); - // ── Fetch health data ── - - const fetchHealth = useCallback(async () => { - if (!deviceId) return; - setHealthLoading(true); - setError(null); - try { - const [reportsRes, alertsRes] = await Promise.all([ - api.getHealthReports(deviceId, 20), - api.getHealthAlerts(deviceId), - ]); - setReports(reportsRes.reports); - setAlerts(alertsRes.alerts); - } catch (err) { - setError(err instanceof ApiError ? err.message : 'Erreur réseau'); - } finally { - setHealthLoading(false); + // Update device status from WebSocket + useEffect(() => { + if (liveStatus && device) { + setDevice((d) => d ? { ...d, status: liveStatus as DeviceSummary['status'] } : d); } - }, [deviceId]); + }, [liveStatus]); + + // ── Hydrate health from REST on first load ── useEffect(() => { - if (tab === 'health') fetchHealth(); - }, [tab, fetchHealth]); + if (!deviceId || tab !== 'health') return; + let cancelled = false; - // ── Fetch logs ── + (async () => { + setHealthLoading(true); + setError(null); + try { + const [reportsRes, alertsRes] = await Promise.all([ + api.getHealthReports(deviceId, 20), + api.getHealthAlerts(deviceId), + ]); + if (!cancelled) { + setInitialReports(reportsRes.reports); + setInitialAlerts(alertsRes.alerts); + } + } catch (err) { + if (!cancelled) setError(err instanceof ApiError ? err.message : 'Erreur réseau'); + } finally { + if (!cancelled) setHealthLoading(false); + } + })(); + + return () => { cancelled = true; }; + }, [deviceId, tab]); + + // ── Hydrate logs from REST on first load ── const fetchLogs = useCallback(async () => { if (!deviceId) return; @@ -99,7 +141,7 @@ export function DeviceDetailPage() { search: logSearch || undefined, limit: 100, }); - setLogs(res.logs); + setInitialLogs(res.logs); setLogsTotal(res.total); } catch (err) { setError(err instanceof ApiError ? err.message : 'Erreur réseau'); @@ -132,8 +174,6 @@ export function DeviceDetailPage() { }); } - const latest = reports[0] ?? null; - // ─── Render ───────────────────────────────────────────────────── return ( @@ -154,9 +194,64 @@ export function DeviceDetailPage() {

{deviceId}

- {device && } +
+ {wsConnected && ( + + + live + + )} + {device && } +
+ {/* ── Remote controls ── */} + {device?.status === 'online' && ( + +
+ + + Déclenche une conversation à distance + +
+ +
+ Mode : +
+ + +
+
+ + )} + {/* Tabs */}
+ {wsConnected ? ( + Mise à jour en direct + ) : ( + + )}
diff --git a/apps/robot-client/src/config/hardware.config.ts b/apps/robot-client/src/config/hardware.config.ts index 3fea598..80d68c3 100644 --- a/apps/robot-client/src/config/hardware.config.ts +++ b/apps/robot-client/src/config/hardware.config.ts @@ -73,7 +73,7 @@ export function loadHardwareConfig(): HardwareConfig { wakeWord: { pythonPath: process.env.WAKEWORD_PYTHON_PATH || 'python3', scriptPath: process.env.WAKEWORD_SCRIPT_PATH || './scripts/wake_word.py', - modelName: process.env.WAKEWORD_MODEL || 'hey_ti_pote', + modelName: process.env.WAKEWORD_MODEL || 'hey_jarvis', threshold: parseFloat(process.env.WAKEWORD_THRESHOLD || '0.75'), }, serial: { diff --git a/apps/robot-client/src/main.ts b/apps/robot-client/src/main.ts index 88c4f0d..2ecfdeb 100644 --- a/apps/robot-client/src/main.ts +++ b/apps/robot-client/src/main.ts @@ -16,6 +16,7 @@ import { type ITriggerService } from './services/trigger.interface.js'; import { SetupFlow } from './setup/index.js'; import { HardwareService, Emotion } from './hardware/index.js'; import { createLogger, setLogForwarder } from './utils/index.js'; +import { SOUND_TRIGGER } from './utils/sounds.js'; const logger = createLogger('main', 'info'); @@ -165,6 +166,55 @@ async function main(): Promise { logForwarder.start(); // Flush logs to backend every 5s orchestrator.start(); + // ── Remote trigger from desktop app ── + + cloudSocket.on('remote_trigger', async () => { + logger.info('🎯 Remote trigger from app — starting conversation'); + try { + await audioService.play(SOUND_TRIGGER); + } catch (err) { + logger.warn({ err }, 'Failed to play trigger beep'); + } + orchestrator.handleTriggerDetected(); + }); + + // ── Hot-swap trigger mode from desktop app ── + + cloudSocket.on('set_trigger_mode', async (event: { mode: string }) => { + const newMode = event.mode as 'wakeword' | 'keyboard'; + if (!['wakeword', 'keyboard'].includes(newMode)) { + logger.warn({ mode: event.mode }, 'Invalid trigger mode from remote'); + return; + } + + if (newMode === resolvedConfig.triggerMode) { + logger.info({ mode: newMode }, 'Trigger mode already set'); + return; + } + + logger.info({ from: resolvedConfig.triggerMode, to: newMode }, 'Switching trigger mode'); + + // Stop current trigger + trigger.stop(); + + // Create new trigger + if (newMode === 'wakeword') { + trigger = new WakeWordService( + hardwareConfig.wakeWord, + hardwareConfig.audio, + audioBackend === 'esp32' ? hardwareService : null, + ); + } else { + trigger = new KeyboardTriggerService(); + } + + // Rewire orchestrator with new trigger + orchestrator.swapTrigger(trigger); + resolvedConfig.triggerMode = newMode; + + logger.info({ mode: newMode }, 'Trigger mode switched'); + }); + if (resolvedConfig.triggerMode === 'wakeword') { logger.info('Ti-Pote is ready! Say "Hey Ti-Pote" to start a conversation.'); } else { diff --git a/apps/robot-client/src/services/orchestrator.service.ts b/apps/robot-client/src/services/orchestrator.service.ts index 4aabbe5..15ea840 100644 --- a/apps/robot-client/src/services/orchestrator.service.ts +++ b/apps/robot-client/src/services/orchestrator.service.ts @@ -39,7 +39,7 @@ export class OrchestratorService extends EventEmitter { constructor( private readonly cloudSocket: CloudSocket, private readonly audioService: AudioService, - private readonly trigger: ITriggerService, + private trigger: ITriggerService, private readonly audioConfig: AudioConfig, ) { super(); @@ -83,9 +83,10 @@ export class OrchestratorService extends EventEmitter { } /** - * Handle trigger detection (wake word or Enter key). + * Handle trigger detection (wake word, Enter key, or remote command). + * Public so that external callers (e.g. CloudSocket remote trigger) can invoke it. */ - private async handleTriggerDetected(): Promise { + async handleTriggerDetected(): Promise { if (this.state !== 'idle') { this.logger.debug({ state: this.state }, 'Trigger detected but not idle, ignoring'); return; @@ -314,6 +315,25 @@ export class OrchestratorService extends EventEmitter { this.logger.info('💤 Idle — waiting for wake word...'); } + /** + * Hot-swap the trigger service (e.g., switching between wakeword and keyboard). + */ + swapTrigger(newTrigger: ITriggerService): void { + this.logger.info('Swapping trigger service'); + + // Wire up the new trigger's detected event + newTrigger.on('detected', () => { + this.handleTriggerDetected(); + }); + + this.trigger = newTrigger; + + // Start the new trigger if we're idle + if (this.state === 'idle') { + newTrigger.start(); + } + } + /** * Handle user interrupt (e.g., button press, or second wake word). */ diff --git a/apps/robot-client/src/services/wake-word.service.ts b/apps/robot-client/src/services/wake-word.service.ts index 74ee7b1..c93249d 100644 --- a/apps/robot-client/src/services/wake-word.service.ts +++ b/apps/robot-client/src/services/wake-word.service.ts @@ -38,16 +38,38 @@ export class WakeWordService extends EventEmitter { private _isPaused = false; private _streamClosed = false; private readonly usesHardware: boolean; + private restartCount = 0; + private readonly maxRestarts = 5; + /** Set once on first EPIPE — prevents log spam from in-flight chunks. */ + private _pipeBroken = false; + /** Chunk counter for debug — logs every N chunks to confirm audio flows. */ + private _chunkCount = 0; /** Latched forwarder so we can detach it on stop / error. */ private readonly forwardMicChunk = (chunk: Buffer): void => { + if (this._pipeBroken) return; if (!this.process || !this.process.stdin || this.process.stdin.destroyed) return; - this.process.stdin.write(chunk, (err) => { - if (err && (err as NodeJS.ErrnoException).code === 'EPIPE') { - this.logger.warn('Wake word process stdin pipe broken — detaching audio'); + this._chunkCount++; + if (this._chunkCount % 500 === 0) { + this.logger.info({ chunks: this._chunkCount, paused: this._isPaused }, 'Wake word audio flowing'); + } + try { + this.process.stdin.write(chunk, (err) => { + if (err && (err as NodeJS.ErrnoException).code === 'EPIPE') { + if (!this._pipeBroken) { + this._pipeBroken = true; + this.logger.warn('Wake word stdin pipe broken — detaching audio'); + this.detachHardware(); + } + } + }); + } catch (err) { + if (!this._pipeBroken) { + this._pipeBroken = true; + this.logger.warn('Wake word stdin write failed — detaching audio'); this.detachHardware(); } - }); + } }; constructor( @@ -105,6 +127,20 @@ export class WakeWordService extends EventEmitter { this._isListening = true; this._isPaused = false; + this._pipeBroken = false; + + // Catch EPIPE on stdin so it doesn't bubble as uncaughtException + this.process.stdin?.on('error', (err) => { + if ((err as NodeJS.ErrnoException).code === 'EPIPE') { + if (!this._pipeBroken) { + this._pipeBroken = true; + this.logger.debug('Wake word stdin EPIPE (process exited)'); + this.detachHardware(); + } + } else { + this.logger.error({ err }, 'Wake word stdin error'); + } + }); this.process.stdout?.on('data', (data: Buffer) => { const lines = data.toString().trim().split('\n'); @@ -129,12 +165,12 @@ export class WakeWordService extends EventEmitter { this.emit('ready'); } else if (msg === 'PAUSED') { this._streamClosed = false; - this.logger.debug('Wake word paused'); + this.logger.info('⏸️ Wake word paused'); } else if (msg === 'STREAM_CLOSED') { this._streamClosed = true; this.logger.debug('Wake word audio stream closed'); } else if (msg === 'RESUMED') { - this.logger.debug('Wake word resumed'); + this.logger.info('▶️ Wake word resumed (Python confirmed)'); } else if (msg === 'STREAM_REOPENED') { this.logger.debug('Wake word audio stream reopened'); } else if (msg.startsWith('Loading wake word model')) { @@ -162,17 +198,40 @@ export class WakeWordService extends EventEmitter { this.detachHardware(); this.process = null; if (code !== 0 && code !== null) { - this.logger.warn({ code }, 'Wake word process exited unexpectedly'); + this.restartCount++; + if (this.restartCount > this.maxRestarts) { + this.logger.error( + { code, restarts: this.restartCount }, + 'Wake word exceeded max restarts — giving up. Fix the issue and restart the service.', + ); + this.emit('error', new Error(`Wake word crashed ${this.restartCount} times`)); + return; + } + this.logger.warn({ code, restart: this.restartCount }, 'Wake word process exited unexpectedly'); setTimeout(() => { this.logger.info('Restarting wake word detection...'); this.start(); }, 2000); + } else { + // Clean exit — reset counter + this.restartCount = 0; } }); - // In ESP32 mode, start piping mic audio from the UART. - if (this.usesHardware && this.hardware) { - this.hardware.on('audio_up', this.forwardMicChunk); + // In ESP32 mode, catch errors on the control pipe (fd 3) and start piping mic audio. + if (this.usesHardware) { + const controlPipe = this.process.stdio[3] as unknown as + | (NodeJS.WritableStream & { on?: (event: string, cb: (err: Error) => void) => void }) + | null; + if (controlPipe?.on) { + controlPipe.on('error', (err: Error) => { + this.logger.debug({ err: (err as NodeJS.ErrnoException).code }, 'Control pipe error (process exited)'); + }); + } + + if (this.hardware) { + this.hardware.on('audio_up', this.forwardMicChunk); + } } } diff --git a/apps/robot-client/src/transport/cloud-socket.ts b/apps/robot-client/src/transport/cloud-socket.ts index e8a37f2..d378662 100644 --- a/apps/robot-client/src/transport/cloud-socket.ts +++ b/apps/robot-client/src/transport/cloud-socket.ts @@ -29,6 +29,8 @@ export interface CloudSocketEvents { response_text: (event: ResponseTextEvent) => void; audio_chunk: (event: AudioChunkEvent) => void; notification: (payload: Record) => void; + remote_trigger: () => void; + set_trigger_mode: (event: { mode: string }) => void; error: (error: Error) => void; } @@ -123,6 +125,18 @@ export class CloudSocket extends EventEmitter { this.logger.info({ payload }, 'Notification received'); this.emit('notification', payload); }); + + // ── Remote control events from desktop app (via AppGateway → RobotGateway) ── + + this.socket.on('remote_trigger', () => { + this.logger.info('Remote trigger received from app'); + this.emit('remote_trigger'); + }); + + this.socket.on('set_trigger_mode', (event: { mode: string }) => { + this.logger.info({ mode: event.mode }, 'Trigger mode change from app'); + this.emit('set_trigger_mode', event); + }); }); } diff --git a/apps/robot-client/src/utils/sounds.ts b/apps/robot-client/src/utils/sounds.ts new file mode 100644 index 0000000..3e88fb5 --- /dev/null +++ b/apps/robot-client/src/utils/sounds.ts @@ -0,0 +1,80 @@ +/** + * Pre-generated notification sounds for the robot. + * + * All sounds are raw S16LE mono PCM at 16 kHz, ready to pass to AudioService.play(). + * Generated once at import time — zero runtime cost per playback. + */ + +const SAMPLE_RATE = 16_000; + +/** + * Generate a sine wave tone with attack/release envelope to prevent speaker clicks. + */ +function generateTone(freqHz: number, durationMs: number, amplitude = 0.5): Buffer { + const sampleCount = Math.floor((SAMPLE_RATE * durationMs) / 1000); + const buf = Buffer.alloc(sampleCount * 2); // 16-bit = 2 bytes per sample + const amp = Math.max(0, Math.min(1, amplitude)) * 32767; + const twoPiF = (2 * Math.PI * freqHz) / SAMPLE_RATE; + + // 5ms linear attack/release envelope + const rampSamples = Math.floor((SAMPLE_RATE * 5) / 1000); + + for (let i = 0; i < sampleCount; i++) { + let env = 1; + if (i < rampSamples) env = i / rampSamples; + else if (i > sampleCount - rampSamples) env = (sampleCount - i) / rampSamples; + + const s = Math.round(Math.sin(i * twoPiF) * amp * env); + buf.writeInt16LE(Math.max(-32768, Math.min(32767, s)), i * 2); + } + return buf; +} + +/** + * Concatenate multiple PCM buffers with silence gaps between them. + */ +function concat(parts: Buffer[], gapMs = 0): Buffer { + if (gapMs <= 0) return Buffer.concat(parts); + + const gapBytes = Math.floor((SAMPLE_RATE * gapMs) / 1000) * 2; + const silence = Buffer.alloc(gapBytes); + const result: Buffer[] = []; + + for (let i = 0; i < parts.length; i++) { + result.push(parts[i]); + if (i < parts.length - 1) result.push(silence); + } + + return Buffer.concat(result); +} + +// ── Pre-generated sounds ────────────────────────────────────────── + +/** + * Short rising two-tone chirp: "bi-bip!" (80ms + 80ms) + * Played when a remote trigger starts a conversation. + */ +export const SOUND_TRIGGER = concat( + [ + generateTone(660, 80, 0.4), // E5 + generateTone(880, 80, 0.4), // A5 + ], + 30, // 30ms gap +); + +/** + * Single soft beep for generic notifications. + */ +export const SOUND_NOTIFY = generateTone(440, 120, 0.3); + +/** + * Three descending tones for errors. + */ +export const SOUND_ERROR = concat( + [ + generateTone(880, 100, 0.35), + generateTone(660, 100, 0.35), + generateTone(440, 150, 0.35), + ], + 40, +); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5bb65ee..f8a9519 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -170,6 +170,9 @@ importers: react-router-dom: specifier: ^6.26.0 version: 6.30.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + socket.io-client: + specifier: ^4.8.3 + version: 4.8.3 zustand: specifier: ^4.5.5 version: 4.5.7(@types/react@18.3.28)(react@18.3.1) diff --git a/tools/pi-gen-tipote/deploy.sh b/tools/pi-gen-tipote/deploy.sh index 3268bfb..cb0f167 100755 --- a/tools/pi-gen-tipote/deploy.sh +++ b/tools/pi-gen-tipote/deploy.sh @@ -81,11 +81,15 @@ $SSH "sudo mkdir -p /opt/tipote && sudo chown ${PI_USER}:${PI_USER} /opt/tipote" echo "▸ [5/7] Uploading files..." -# Sync dist/ and package.json +# Sync dist/, scripts/, and package.json rsync -az --delete \ "$ROBOT_CLIENT_DIR/dist/" \ "${PI_USER}@${PI_HOST}:/opt/tipote/dist/" +rsync -az --delete \ + "$ROBOT_CLIENT_DIR/scripts/wake_word.py" \ + "${PI_USER}@${PI_HOST}:/opt/tipote/scripts/" + $SCP "$ROBOT_CLIENT_DIR/package.json" "${PI_USER}@${PI_HOST}:/opt/tipote/package.json" # Install production deps on Pi @@ -115,6 +119,7 @@ HARDWARE_SERIAL_ENABLED=true HARDWARE_SERIAL_PORT=/dev/serial0 HARDWARE_SERIAL_BAUD_RATE=921600 WAKEWORD_PYTHON_PATH=/opt/tipote/.venv/bin/python3 +WAKEWORD_MODEL=hey_jarvis NODE_ENV=production LOG_LEVEL=info ENVEOF