on avance
This commit is contained in:
ordinarthur
parent
3b993894e4
commit
9ee09afa77
204
apps/backend/src/adapters/inbound/websocket/app.gateway.ts
Normal file
204
apps/backend/src/adapters/inbound/websocket/app.gateway.ts
Normal file
@ -0,0 +1,204 @@
|
||||
import {
|
||||
WebSocketGateway,
|
||||
WebSocketServer,
|
||||
OnGatewayConnection,
|
||||
OnGatewayDisconnect,
|
||||
SubscribeMessage,
|
||||
MessageBody,
|
||||
ConnectedSocket,
|
||||
} from '@nestjs/websockets';
|
||||
import { Inject, Logger, forwardRef } from '@nestjs/common';
|
||||
import { JwtService } from '@nestjs/jwt';
|
||||
import { OnEvent } from '@nestjs/event-emitter';
|
||||
import { Server, Socket } from 'socket.io';
|
||||
import { JwtPayload } from '../rest/auth/strategies/jwt.strategy';
|
||||
import { RobotGateway } from './robot.gateway';
|
||||
import { DeviceService } from '../../../core/services/device.service';
|
||||
|
||||
/**
|
||||
* WebSocket gateway for the desktop/web app.
|
||||
*
|
||||
* Authenticates users via JWT (same tokens as REST API).
|
||||
* Broadcasts real-time events:
|
||||
* - `health_report` — device health telemetry
|
||||
* - `device_status` — device online/offline changes
|
||||
* - `device_log` — live log entries
|
||||
*/
|
||||
|
||||
interface AppSocket extends Socket {
|
||||
data: {
|
||||
userId: string;
|
||||
homeId: string;
|
||||
};
|
||||
}
|
||||
|
||||
@WebSocketGateway({
|
||||
namespace: '/ws/app',
|
||||
cors: { origin: '*' },
|
||||
})
|
||||
export class AppGateway implements OnGatewayConnection, OnGatewayDisconnect {
|
||||
@WebSocketServer()
|
||||
server!: Server;
|
||||
|
||||
private readonly logger = new Logger(AppGateway.name);
|
||||
|
||||
constructor(
|
||||
private readonly jwtService: JwtService,
|
||||
@Inject(forwardRef(() => RobotGateway))
|
||||
private readonly robotGateway: RobotGateway,
|
||||
private readonly deviceService: DeviceService,
|
||||
) {}
|
||||
|
||||
// ── Connection lifecycle ──
|
||||
|
||||
async handleConnection(client: AppSocket) {
|
||||
try {
|
||||
const token =
|
||||
client.handshake.auth?.token ||
|
||||
client.handshake.headers?.authorization?.replace('Bearer ', '');
|
||||
|
||||
if (!token) {
|
||||
this.logger.warn('App WS rejected: no token');
|
||||
client.disconnect();
|
||||
return;
|
||||
}
|
||||
|
||||
const payload = this.jwtService.verify<JwtPayload>(token);
|
||||
if (payload.type !== 'user') {
|
||||
this.logger.warn('App WS rejected: not a user token');
|
||||
client.disconnect();
|
||||
return;
|
||||
}
|
||||
|
||||
client.data.userId = payload.sub;
|
||||
client.data.homeId = payload.homeId;
|
||||
|
||||
// Join a room scoped to the user's home so we can broadcast per-home
|
||||
await client.join(`home:${payload.homeId}`);
|
||||
|
||||
this.logger.log(`App client connected: user ${payload.sub} (home ${payload.homeId})`);
|
||||
} catch {
|
||||
this.logger.warn('App WS rejected: invalid token');
|
||||
client.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
handleDisconnect(client: AppSocket) {
|
||||
const userId = client.data?.userId;
|
||||
if (userId) {
|
||||
this.logger.log(`App client disconnected: user ${userId}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Subscribe to device for targeted updates ──
|
||||
|
||||
@SubscribeMessage('subscribe_device')
|
||||
handleSubscribeDevice(
|
||||
@ConnectedSocket() client: AppSocket,
|
||||
@MessageBody() data: { deviceId: string },
|
||||
) {
|
||||
void client.join(`device:${data.deviceId}`);
|
||||
this.logger.debug(`User ${client.data.userId} subscribed to device ${data.deviceId}`);
|
||||
}
|
||||
|
||||
@SubscribeMessage('unsubscribe_device')
|
||||
handleUnsubscribeDevice(
|
||||
@ConnectedSocket() client: AppSocket,
|
||||
@MessageBody() data: { deviceId: string },
|
||||
) {
|
||||
void client.leave(`device:${data.deviceId}`);
|
||||
}
|
||||
|
||||
// ── Commands from desktop app → robot ──
|
||||
|
||||
@SubscribeMessage('trigger_conversation')
|
||||
async handleTriggerConversation(
|
||||
@ConnectedSocket() client: AppSocket,
|
||||
@MessageBody() data: { deviceId: string },
|
||||
) {
|
||||
// Verify the user owns this device
|
||||
const device = await this.deviceService.findById(data.deviceId);
|
||||
if (!device || device.homeId !== client.data.homeId) {
|
||||
return { error: 'Device not found' };
|
||||
}
|
||||
|
||||
const sent = this.robotGateway.sendRemoteTrigger(data.deviceId);
|
||||
if (!sent) {
|
||||
return { error: 'Device not connected' };
|
||||
}
|
||||
|
||||
this.logger.log(`User ${client.data.userId} triggered conversation on ${data.deviceId}`);
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
@SubscribeMessage('set_trigger_mode')
|
||||
async handleSetTriggerMode(
|
||||
@ConnectedSocket() client: AppSocket,
|
||||
@MessageBody() data: { deviceId: string; mode: string },
|
||||
) {
|
||||
if (!['wakeword', 'keyboard'].includes(data.mode)) {
|
||||
return { error: 'Invalid mode. Must be "wakeword" or "keyboard".' };
|
||||
}
|
||||
|
||||
const device = await this.deviceService.findById(data.deviceId);
|
||||
if (!device || device.homeId !== client.data.homeId) {
|
||||
return { error: 'Device not found' };
|
||||
}
|
||||
|
||||
const sent = this.robotGateway.sendSetTriggerMode(data.deviceId, data.mode);
|
||||
if (!sent) {
|
||||
return { error: 'Device not connected' };
|
||||
}
|
||||
|
||||
// Broadcast the mode change to all subscribed app clients
|
||||
this.server.to(`device:${data.deviceId}`).emit('trigger_mode_changed', {
|
||||
deviceId: data.deviceId,
|
||||
mode: data.mode,
|
||||
});
|
||||
|
||||
this.logger.log(`User ${client.data.userId} set trigger mode to ${data.mode} on ${data.deviceId}`);
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
// ── Event listeners (from NestJS EventEmitter) ──
|
||||
|
||||
@OnEvent('device.health_report')
|
||||
handleHealthReportEvent(payload: {
|
||||
deviceId: string;
|
||||
homeId: string;
|
||||
report: Record<string, unknown>;
|
||||
alerts: string[];
|
||||
}) {
|
||||
// Broadcast to all app clients subscribed to this device
|
||||
this.server.to(`device:${payload.deviceId}`).emit('health_report', {
|
||||
deviceId: payload.deviceId,
|
||||
report: payload.report,
|
||||
alerts: payload.alerts,
|
||||
});
|
||||
}
|
||||
|
||||
@OnEvent('device.status_changed')
|
||||
handleDeviceStatusEvent(payload: {
|
||||
deviceId: string;
|
||||
homeId: string;
|
||||
status: string;
|
||||
}) {
|
||||
// Broadcast to the entire home (all clients see device go online/offline)
|
||||
this.server.to(`home:${payload.homeId}`).emit('device_status', {
|
||||
deviceId: payload.deviceId,
|
||||
status: payload.status,
|
||||
});
|
||||
}
|
||||
|
||||
@OnEvent('device.log')
|
||||
handleDeviceLogEvent(payload: {
|
||||
deviceId: string;
|
||||
level: number;
|
||||
msg: string;
|
||||
loggerName: string | null;
|
||||
context: Record<string, unknown> | null;
|
||||
loggedAt: string;
|
||||
}) {
|
||||
this.server.to(`device:${payload.deviceId}`).emit('device_log', payload);
|
||||
}
|
||||
}
|
||||
@ -175,4 +175,33 @@ export class RobotGateway implements OnGatewayConnection, OnGatewayDisconnect, I
|
||||
isDeviceConnected(deviceId: string): boolean {
|
||||
return this.connectedDevices.has(deviceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a remote trigger to a connected device (starts a conversation).
|
||||
*/
|
||||
sendRemoteTrigger(deviceId: string): boolean {
|
||||
const socket = this.connectedDevices.get(deviceId);
|
||||
if (!socket) return false;
|
||||
this.logger.log(`Remote trigger sent to device ${deviceId}`);
|
||||
socket.emit('remote_trigger');
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a trigger mode change to a connected device.
|
||||
*/
|
||||
sendSetTriggerMode(deviceId: string, mode: string): boolean {
|
||||
const socket = this.connectedDevices.get(deviceId);
|
||||
if (!socket) return false;
|
||||
this.logger.log(`Set trigger mode to ${mode} on device ${deviceId}`);
|
||||
socket.emit('set_trigger_mode', { mode });
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the homeId for a connected device.
|
||||
*/
|
||||
getDeviceHomeId(deviceId: string): string | null {
|
||||
return this.connectedDevices.get(deviceId)?.data.homeId ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
@ -27,6 +27,7 @@ import { HealthController } from './adapters/inbound/rest/health/health.controll
|
||||
import { LogsController } from './adapters/inbound/rest/logs/logs.controller';
|
||||
import { PairingService } from './core/services/pairing.service';
|
||||
import { RobotGateway } from './adapters/inbound/websocket/robot.gateway';
|
||||
import { AppGateway } from './adapters/inbound/websocket/app.gateway';
|
||||
import { DeepgramAdapter } from './adapters/outbound/stt/deepgram.adapter';
|
||||
import { AnthropicAdapter } from './adapters/outbound/llm/anthropic.adapter';
|
||||
import { OpenAIAdapter } from './adapters/outbound/llm/openai.adapter';
|
||||
@ -75,6 +76,7 @@ import { CACHE_PORT } from './core/ports/outbound/cache.port';
|
||||
PairingService,
|
||||
JwtStrategy,
|
||||
RobotGateway,
|
||||
AppGateway,
|
||||
HealthTelemetryService,
|
||||
LogIngestionService,
|
||||
{
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
import * as bcrypt from 'bcrypt';
|
||||
import { Device, DeviceStatus } from '../domain/entities/device.entity';
|
||||
|
||||
@ -9,6 +10,7 @@ export class DeviceService {
|
||||
constructor(
|
||||
@InjectRepository(Device)
|
||||
private readonly deviceRepository: Repository<Device>,
|
||||
private readonly events: EventEmitter2,
|
||||
) {}
|
||||
|
||||
async findById(id: string): Promise<Device | null> {
|
||||
@ -43,5 +45,15 @@ export class DeviceService {
|
||||
status,
|
||||
lastSeenAt: new Date(),
|
||||
});
|
||||
|
||||
// Broadcast status change to app clients
|
||||
const device = await this.deviceRepository.findOne({ where: { id }, select: ['homeId'] });
|
||||
if (device) {
|
||||
this.events.emit('device.status_changed', {
|
||||
deviceId: id,
|
||||
homeId: device.homeId,
|
||||
status,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
import { HealthReport } from '../domain/entities/health-report.entity';
|
||||
import { Device } from '../domain/entities/device.entity';
|
||||
import {
|
||||
IHealthTelemetryPort,
|
||||
HealthReportPayload,
|
||||
@ -23,6 +25,9 @@ export class HealthTelemetryService implements IHealthTelemetryPort {
|
||||
constructor(
|
||||
@InjectRepository(HealthReport)
|
||||
private readonly repo: Repository<HealthReport>,
|
||||
@InjectRepository(Device)
|
||||
private readonly deviceRepo: Repository<Device>,
|
||||
private readonly events: EventEmitter2,
|
||||
) {}
|
||||
|
||||
async ingestReport(deviceId: string, payload: HealthReportPayload): Promise<void> {
|
||||
@ -52,6 +57,17 @@ export class HealthTelemetryService implements IHealthTelemetryPort {
|
||||
} else {
|
||||
this.logger.debug({ deviceId }, 'Health report ingested');
|
||||
}
|
||||
|
||||
// Broadcast to connected app clients via EventEmitter → AppGateway
|
||||
const device = await this.deviceRepo.findOne({ where: { id: deviceId }, select: ['homeId'] });
|
||||
if (device) {
|
||||
this.events.emit('device.health_report', {
|
||||
deviceId,
|
||||
homeId: device.homeId,
|
||||
report: payload,
|
||||
alerts,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async getLatestReports(deviceId: string, limit = 20): Promise<HealthReportPayload[]> {
|
||||
|
||||
@ -16,6 +16,7 @@
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
"react-router-dom": "^6.26.0",
|
||||
"socket.io-client": "^4.8.3",
|
||||
"zustand": "^4.5.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@ -14,6 +14,7 @@ import {
|
||||
type Me,
|
||||
type RegisterInput,
|
||||
} from '../lib/api';
|
||||
import { getSocket, disconnectSocket } from '../lib/socket';
|
||||
|
||||
type AuthStatus = 'loading' | 'authenticated' | 'unauthenticated';
|
||||
|
||||
@ -46,6 +47,8 @@ export function AuthProvider({ children }: { children: ReactNode }) {
|
||||
if (cancelled) return;
|
||||
setUser(me);
|
||||
setStatus('authenticated');
|
||||
// Connect WebSocket on session restore
|
||||
getSocket().catch(() => {});
|
||||
} catch {
|
||||
if (cancelled) return;
|
||||
setUser(null);
|
||||
@ -63,6 +66,8 @@ export function AuthProvider({ children }: { children: ReactNode }) {
|
||||
const me = await api.me();
|
||||
setUser(me);
|
||||
setStatus('authenticated');
|
||||
// Connect WebSocket after login
|
||||
getSocket().catch(() => {});
|
||||
}, []);
|
||||
|
||||
const register = useCallback(async (input: RegisterInput) => {
|
||||
@ -73,6 +78,7 @@ export function AuthProvider({ children }: { children: ReactNode }) {
|
||||
}, []);
|
||||
|
||||
const logout = useCallback(async () => {
|
||||
disconnectSocket();
|
||||
await api.logout();
|
||||
setUser(null);
|
||||
setStatus('unauthenticated');
|
||||
|
||||
281
apps/frontend/src/hooks/useDeviceSocket.ts
Normal file
281
apps/frontend/src/hooks/useDeviceSocket.ts
Normal file
@ -0,0 +1,281 @@
|
||||
/**
|
||||
* React hooks for real-time device updates via WebSocket.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { getSocket } from '../lib/socket';
|
||||
import type { HealthReport, LogEntry } from '../lib/api';
|
||||
import type { Socket } from 'socket.io-client';
|
||||
|
||||
export type TriggerMode = 'wakeword' | 'keyboard';
|
||||
|
||||
// ─── Level label mapping (pino levels) ────────────────────────────
|
||||
|
||||
const LEVEL_LABELS: Record<number, string> = {
|
||||
10: 'trace',
|
||||
20: 'debug',
|
||||
30: 'info',
|
||||
40: 'warn',
|
||||
50: 'error',
|
||||
60: 'fatal',
|
||||
};
|
||||
|
||||
// ─── useDeviceHealth ──────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Subscribe to real-time health reports for a device.
|
||||
* Returns the latest report, the history, and current alerts.
|
||||
*/
|
||||
export function useDeviceHealth(deviceId: string | undefined) {
|
||||
const [latest, setLatest] = useState<HealthReport | null>(null);
|
||||
const [reports, setReports] = useState<HealthReport[]>([]);
|
||||
const [alerts, setAlerts] = useState<string[]>([]);
|
||||
const [connected, setConnected] = useState(false);
|
||||
const socketRef = useRef<Socket | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!deviceId) return;
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const sock = await getSocket();
|
||||
if (cancelled) return;
|
||||
|
||||
socketRef.current = sock;
|
||||
setConnected(sock.connected);
|
||||
|
||||
// Subscribe to this device's updates
|
||||
sock.emit('subscribe_device', { deviceId });
|
||||
|
||||
const onHealthReport = (data: {
|
||||
deviceId: string;
|
||||
report: HealthReport;
|
||||
alerts: string[];
|
||||
}) => {
|
||||
if (data.deviceId !== deviceId) return;
|
||||
|
||||
setLatest(data.report);
|
||||
setAlerts(data.alerts);
|
||||
setReports((prev) => {
|
||||
const next = [data.report, ...prev];
|
||||
return next.slice(0, 50); // Keep last 50
|
||||
});
|
||||
};
|
||||
|
||||
const onConnect = () => setConnected(true);
|
||||
const onDisconnect = () => setConnected(false);
|
||||
|
||||
sock.on('health_report', onHealthReport);
|
||||
sock.on('connect', onConnect);
|
||||
sock.on('disconnect', onDisconnect);
|
||||
|
||||
return () => {
|
||||
sock.emit('unsubscribe_device', { deviceId });
|
||||
sock.off('health_report', onHealthReport);
|
||||
sock.off('connect', onConnect);
|
||||
sock.off('disconnect', onDisconnect);
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('Failed to connect device socket:', err);
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (socketRef.current) {
|
||||
socketRef.current.emit('unsubscribe_device', { deviceId });
|
||||
}
|
||||
};
|
||||
}, [deviceId]);
|
||||
|
||||
return { latest, reports, alerts, connected };
|
||||
}
|
||||
|
||||
// ─── useDeviceStatus ──────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Subscribe to device status changes (online/offline) for all devices in the home.
|
||||
* Returns a map of deviceId → status.
|
||||
*/
|
||||
export function useDeviceStatus() {
|
||||
const [statuses, setStatuses] = useState<Record<string, string>>({});
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
let sock: Socket | null = null;
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
sock = await getSocket();
|
||||
if (cancelled) return;
|
||||
|
||||
const onStatus = (data: { deviceId: string; status: string }) => {
|
||||
setStatuses((prev) => ({ ...prev, [data.deviceId]: data.status }));
|
||||
};
|
||||
|
||||
sock.on('device_status', onStatus);
|
||||
|
||||
return () => {
|
||||
sock?.off('device_status', onStatus);
|
||||
};
|
||||
} catch {
|
||||
// Not authenticated yet
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
return statuses;
|
||||
}
|
||||
|
||||
// ─── useDeviceLogs ────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Subscribe to real-time log entries for a device.
|
||||
*/
|
||||
export function useDeviceLogs(deviceId: string | undefined, minLevel = 0) {
|
||||
const [logs, setLogs] = useState<LogEntry[]>([]);
|
||||
const socketRef = useRef<Socket | null>(null);
|
||||
|
||||
const clearLogs = useCallback(() => setLogs([]), []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!deviceId) return;
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const sock = await getSocket();
|
||||
if (cancelled) return;
|
||||
|
||||
socketRef.current = sock;
|
||||
sock.emit('subscribe_device', { deviceId });
|
||||
|
||||
const onLog = (data: {
|
||||
deviceId: string;
|
||||
level: number;
|
||||
msg: string;
|
||||
loggerName: string | null;
|
||||
context: Record<string, unknown> | null;
|
||||
loggedAt: string;
|
||||
}) => {
|
||||
if (data.deviceId !== deviceId) return;
|
||||
if (data.level < minLevel) return;
|
||||
|
||||
const entry: LogEntry = {
|
||||
level: data.level,
|
||||
levelLabel: LEVEL_LABELS[data.level] ?? 'info',
|
||||
time: new Date(data.loggedAt).getTime(),
|
||||
msg: data.msg,
|
||||
name: data.loggerName ?? undefined,
|
||||
...(data.context ?? {}),
|
||||
};
|
||||
|
||||
setLogs((prev) => {
|
||||
const next = [entry, ...prev];
|
||||
return next.slice(0, 500); // Keep last 500
|
||||
});
|
||||
};
|
||||
|
||||
sock.on('device_log', onLog);
|
||||
|
||||
return () => {
|
||||
sock.emit('unsubscribe_device', { deviceId });
|
||||
sock.off('device_log', onLog);
|
||||
};
|
||||
} catch {
|
||||
// Not authenticated
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (socketRef.current) {
|
||||
socketRef.current.emit('unsubscribe_device', { deviceId });
|
||||
}
|
||||
};
|
||||
}, [deviceId, minLevel]);
|
||||
|
||||
return { logs, clearLogs };
|
||||
}
|
||||
|
||||
// ─── useDeviceCommands ────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Send commands to a device via WebSocket.
|
||||
*/
|
||||
export function useDeviceCommands(deviceId: string | undefined) {
|
||||
const [triggerMode, setTriggerMode] = useState<TriggerMode>('keyboard');
|
||||
const [triggering, setTriggering] = useState(false);
|
||||
|
||||
// Listen for trigger mode changes
|
||||
useEffect(() => {
|
||||
if (!deviceId) return;
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const sock = await getSocket();
|
||||
if (cancelled) return;
|
||||
|
||||
const onModeChanged = (data: { deviceId: string; mode: string }) => {
|
||||
if (data.deviceId !== deviceId) return;
|
||||
setTriggerMode(data.mode as TriggerMode);
|
||||
};
|
||||
|
||||
sock.on('trigger_mode_changed', onModeChanged);
|
||||
|
||||
return () => {
|
||||
sock.off('trigger_mode_changed', onModeChanged);
|
||||
};
|
||||
} catch {
|
||||
// Not connected
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [deviceId]);
|
||||
|
||||
const triggerConversation = useCallback(async () => {
|
||||
if (!deviceId) return;
|
||||
setTriggering(true);
|
||||
try {
|
||||
const sock = await getSocket();
|
||||
sock.emit('trigger_conversation', { deviceId }, (response: { ok?: boolean; error?: string }) => {
|
||||
if (response?.error) {
|
||||
console.warn('Trigger failed:', response.error);
|
||||
}
|
||||
setTriggering(false);
|
||||
});
|
||||
} catch {
|
||||
setTriggering(false);
|
||||
}
|
||||
}, [deviceId]);
|
||||
|
||||
const changeTriggerMode = useCallback(async (mode: TriggerMode) => {
|
||||
if (!deviceId) return;
|
||||
try {
|
||||
const sock = await getSocket();
|
||||
sock.emit('set_trigger_mode', { deviceId, mode }, (response: { ok?: boolean; error?: string }) => {
|
||||
if (response?.error) {
|
||||
console.warn('Set trigger mode failed:', response.error);
|
||||
} else {
|
||||
setTriggerMode(mode);
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
// Not connected
|
||||
}
|
||||
}, [deviceId]);
|
||||
|
||||
return { triggerMode, triggerConversation, triggering, changeTriggerMode };
|
||||
}
|
||||
72
apps/frontend/src/lib/socket.ts
Normal file
72
apps/frontend/src/lib/socket.ts
Normal file
@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Singleton Socket.IO client for the app WebSocket (`/ws/app`).
|
||||
*
|
||||
* Provides real-time events from the backend:
|
||||
* - `health_report` — device health telemetry
|
||||
* - `device_status` — device online/offline changes
|
||||
* - `device_log` — live log entries
|
||||
*/
|
||||
|
||||
import { io, type Socket } from 'socket.io-client';
|
||||
import { storage } from './storage';
|
||||
|
||||
const BASE_URL =
|
||||
(import.meta.env.VITE_API_URL as string | undefined)?.replace(/\/$/, '') ||
|
||||
'http://localhost:3000';
|
||||
|
||||
const ACCESS_KEY = 'auth.accessToken';
|
||||
|
||||
let socket: Socket | null = null;
|
||||
|
||||
/**
|
||||
* Get or create the singleton socket connection.
|
||||
* Connects to /ws/app with the current user JWT.
|
||||
*/
|
||||
export async function getSocket(): Promise<Socket> {
|
||||
if (socket?.connected) return socket;
|
||||
|
||||
const token = await storage.get<string>(ACCESS_KEY);
|
||||
if (!token) throw new Error('No auth token for WebSocket');
|
||||
|
||||
if (socket) {
|
||||
// Reconnect with fresh token
|
||||
socket.auth = { token };
|
||||
socket.connect();
|
||||
return socket;
|
||||
}
|
||||
|
||||
socket = io(`${BASE_URL}/ws/app`, {
|
||||
auth: { token },
|
||||
transports: ['websocket'],
|
||||
autoConnect: true,
|
||||
reconnection: true,
|
||||
reconnectionAttempts: Infinity,
|
||||
reconnectionDelay: 2000,
|
||||
reconnectionDelayMax: 10000,
|
||||
});
|
||||
|
||||
socket.on('connect', () => {
|
||||
console.log('[ws/app] connected');
|
||||
});
|
||||
|
||||
socket.on('disconnect', (reason) => {
|
||||
console.log('[ws/app] disconnected:', reason);
|
||||
});
|
||||
|
||||
socket.on('connect_error', (err) => {
|
||||
console.warn('[ws/app] connection error:', err.message);
|
||||
});
|
||||
|
||||
return socket;
|
||||
}
|
||||
|
||||
/**
|
||||
* Disconnect and destroy the singleton socket.
|
||||
* Call this on logout.
|
||||
*/
|
||||
export function disconnectSocket(): void {
|
||||
if (socket) {
|
||||
socket.disconnect();
|
||||
socket = null;
|
||||
}
|
||||
}
|
||||
@ -3,9 +3,11 @@ import { Link } from 'react-router-dom';
|
||||
import { Button, Card, StatusBadge } from '../components/ui';
|
||||
import { useAuth } from '../context/AuthContext';
|
||||
import { api, ApiError, type DeviceSummary } from '../lib/api';
|
||||
import { useDeviceStatus } from '../hooks/useDeviceSocket';
|
||||
|
||||
export function DashboardPage() {
|
||||
const { user, logout } = useAuth();
|
||||
const statusMap = useDeviceStatus();
|
||||
const [devices, setDevices] = useState<DeviceSummary[] | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
@ -101,7 +103,7 @@ export function DashboardPage() {
|
||||
{d.id}
|
||||
</p>
|
||||
</div>
|
||||
<StatusBadge status={d.status} />
|
||||
<StatusBadge status={(statusMap[d.id] as DeviceSummary['status']) ?? d.status} />
|
||||
</div>
|
||||
<dl className="mt-4 flex flex-wrap gap-x-4 gap-y-1 text-xs text-slate-400">
|
||||
<div>
|
||||
|
||||
@ -8,6 +8,7 @@ import {
|
||||
type HealthReport,
|
||||
type LogEntry,
|
||||
} from '../lib/api';
|
||||
import { useDeviceHealth, useDeviceLogs, useDeviceStatus, useDeviceCommands } from '../hooks/useDeviceSocket';
|
||||
|
||||
// ─── Level helpers ────────────────────────────────────────────────
|
||||
|
||||
@ -36,20 +37,50 @@ export function DeviceDetailPage() {
|
||||
|
||||
// Device info
|
||||
const [device, setDevice] = useState<DeviceSummary | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// Health
|
||||
const [reports, setReports] = useState<HealthReport[]>([]);
|
||||
const [alerts, setAlerts] = useState<string[]>([]);
|
||||
// Remote commands
|
||||
const {
|
||||
triggerMode,
|
||||
triggerConversation,
|
||||
triggering,
|
||||
changeTriggerMode,
|
||||
} = useDeviceCommands(deviceId);
|
||||
|
||||
// Real-time health via WebSocket
|
||||
const {
|
||||
latest: wsLatest,
|
||||
reports: wsReports,
|
||||
alerts: wsAlerts,
|
||||
connected: wsConnected,
|
||||
} = useDeviceHealth(deviceId);
|
||||
|
||||
// Real-time device status via WebSocket
|
||||
const statusMap = useDeviceStatus();
|
||||
const liveStatus = deviceId ? statusMap[deviceId] : undefined;
|
||||
|
||||
// Initial health data loaded from REST (hydration)
|
||||
const [initialReports, setInitialReports] = useState<HealthReport[]>([]);
|
||||
const [initialAlerts, setInitialAlerts] = useState<string[]>([]);
|
||||
const [healthLoading, setHealthLoading] = useState(true);
|
||||
|
||||
// Logs
|
||||
const [logs, setLogs] = useState<LogEntry[]>([]);
|
||||
const [logsTotal, setLogsTotal] = useState(0);
|
||||
const [logsLoading, setLogsLoading] = useState(false);
|
||||
// Merge: WS reports on top of initial REST data
|
||||
const reports = wsReports.length > 0 ? wsReports : initialReports;
|
||||
const alerts = wsAlerts.length > 0 ? wsAlerts : initialAlerts;
|
||||
const latest = wsLatest ?? reports[0] ?? null;
|
||||
|
||||
// Logs — real-time via WebSocket + initial load from REST
|
||||
const [logLevel, setLogLevel] = useState(0);
|
||||
const [logSearch, setLogSearch] = useState('');
|
||||
const { logs: wsLogs } = useDeviceLogs(deviceId, logLevel);
|
||||
const [initialLogs, setInitialLogs] = useState<LogEntry[]>([]);
|
||||
const [logsTotal, setLogsTotal] = useState(0);
|
||||
const [logsLoading, setLogsLoading] = useState(false);
|
||||
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
// Merge: WS logs on top, then initial
|
||||
const logs = [...wsLogs, ...initialLogs.filter(
|
||||
(il) => !wsLogs.some((wl) => wl.time === il.time && wl.msg === il.msg),
|
||||
)];
|
||||
|
||||
// ── Fetch device info ──
|
||||
|
||||
@ -64,10 +95,20 @@ export function DeviceDetailPage() {
|
||||
.catch(() => {});
|
||||
}, [deviceId]);
|
||||
|
||||
// ── Fetch health data ──
|
||||
// Update device status from WebSocket
|
||||
useEffect(() => {
|
||||
if (liveStatus && device) {
|
||||
setDevice((d) => d ? { ...d, status: liveStatus as DeviceSummary['status'] } : d);
|
||||
}
|
||||
}, [liveStatus]);
|
||||
|
||||
const fetchHealth = useCallback(async () => {
|
||||
if (!deviceId) return;
|
||||
// ── Hydrate health from REST on first load ──
|
||||
|
||||
useEffect(() => {
|
||||
if (!deviceId || tab !== 'health') return;
|
||||
let cancelled = false;
|
||||
|
||||
(async () => {
|
||||
setHealthLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
@ -75,20 +116,21 @@ export function DeviceDetailPage() {
|
||||
api.getHealthReports(deviceId, 20),
|
||||
api.getHealthAlerts(deviceId),
|
||||
]);
|
||||
setReports(reportsRes.reports);
|
||||
setAlerts(alertsRes.alerts);
|
||||
} catch (err) {
|
||||
setError(err instanceof ApiError ? err.message : 'Erreur réseau');
|
||||
} finally {
|
||||
setHealthLoading(false);
|
||||
if (!cancelled) {
|
||||
setInitialReports(reportsRes.reports);
|
||||
setInitialAlerts(alertsRes.alerts);
|
||||
}
|
||||
}, [deviceId]);
|
||||
} catch (err) {
|
||||
if (!cancelled) setError(err instanceof ApiError ? err.message : 'Erreur réseau');
|
||||
} finally {
|
||||
if (!cancelled) setHealthLoading(false);
|
||||
}
|
||||
})();
|
||||
|
||||
useEffect(() => {
|
||||
if (tab === 'health') fetchHealth();
|
||||
}, [tab, fetchHealth]);
|
||||
return () => { cancelled = true; };
|
||||
}, [deviceId, tab]);
|
||||
|
||||
// ── Fetch logs ──
|
||||
// ── Hydrate logs from REST on first load ──
|
||||
|
||||
const fetchLogs = useCallback(async () => {
|
||||
if (!deviceId) return;
|
||||
@ -99,7 +141,7 @@ export function DeviceDetailPage() {
|
||||
search: logSearch || undefined,
|
||||
limit: 100,
|
||||
});
|
||||
setLogs(res.logs);
|
||||
setInitialLogs(res.logs);
|
||||
setLogsTotal(res.total);
|
||||
} catch (err) {
|
||||
setError(err instanceof ApiError ? err.message : 'Erreur réseau');
|
||||
@ -132,8 +174,6 @@ export function DeviceDetailPage() {
|
||||
});
|
||||
}
|
||||
|
||||
const latest = reports[0] ?? null;
|
||||
|
||||
// ─── Render ─────────────────────────────────────────────────────
|
||||
|
||||
return (
|
||||
@ -154,9 +194,64 @@ export function DeviceDetailPage() {
|
||||
<p className="font-mono text-xs text-slate-500">{deviceId}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
{wsConnected && (
|
||||
<span className="flex items-center gap-1.5 text-xs text-emerald-400">
|
||||
<span className="h-1.5 w-1.5 animate-pulse rounded-full bg-emerald-400" />
|
||||
live
|
||||
</span>
|
||||
)}
|
||||
{device && <StatusBadge status={device.status} />}
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{/* ── Remote controls ── */}
|
||||
{device?.status === 'online' && (
|
||||
<Card className="flex items-center justify-between gap-4 p-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<Button
|
||||
onClick={() => void triggerConversation()}
|
||||
loading={triggering}
|
||||
className="gap-2"
|
||||
>
|
||||
<span className="text-lg leading-none">🎙</span>
|
||||
Parler
|
||||
</Button>
|
||||
<span className="text-xs text-slate-500">
|
||||
Déclenche une conversation à distance
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-xs text-slate-400">Mode :</span>
|
||||
<div className="flex overflow-hidden rounded-lg border border-slate-700">
|
||||
<button
|
||||
onClick={() => void changeTriggerMode('keyboard')}
|
||||
className={[
|
||||
'px-3 py-1.5 text-xs font-medium transition-colors',
|
||||
triggerMode === 'keyboard'
|
||||
? 'bg-brand-500/20 text-brand-400'
|
||||
: 'text-slate-400 hover:bg-slate-800',
|
||||
].join(' ')}
|
||||
>
|
||||
Manuel
|
||||
</button>
|
||||
<button
|
||||
onClick={() => void changeTriggerMode('wakeword')}
|
||||
className={[
|
||||
'px-3 py-1.5 text-xs font-medium transition-colors border-l border-slate-700',
|
||||
triggerMode === 'wakeword'
|
||||
? 'bg-brand-500/20 text-brand-400'
|
||||
: 'text-slate-400 hover:bg-slate-800',
|
||||
].join(' ')}
|
||||
>
|
||||
Wake word
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* Tabs */}
|
||||
<div className="flex gap-1 rounded-xl bg-slate-900/50 p-1">
|
||||
<button
|
||||
@ -268,9 +363,19 @@ export function DeviceDetailPage() {
|
||||
<h3 className="text-sm font-medium text-slate-300">
|
||||
Historique ({reports.length} rapports)
|
||||
</h3>
|
||||
<Button variant="ghost" className="text-xs" onClick={() => void fetchHealth()}>
|
||||
{wsConnected ? (
|
||||
<span className="text-xs text-emerald-400/70">Mise à jour en direct</span>
|
||||
) : (
|
||||
<Button variant="ghost" className="text-xs" onClick={() => {
|
||||
setHealthLoading(true);
|
||||
api.getHealthReports(deviceId!, 20).then((r) => {
|
||||
setInitialReports(r.reports);
|
||||
setHealthLoading(false);
|
||||
}).catch(() => setHealthLoading(false));
|
||||
}}>
|
||||
Rafraîchir
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
<div className="max-h-48 overflow-y-auto">
|
||||
<table className="w-full text-xs">
|
||||
|
||||
@ -73,7 +73,7 @@ export function loadHardwareConfig(): HardwareConfig {
|
||||
wakeWord: {
|
||||
pythonPath: process.env.WAKEWORD_PYTHON_PATH || 'python3',
|
||||
scriptPath: process.env.WAKEWORD_SCRIPT_PATH || './scripts/wake_word.py',
|
||||
modelName: process.env.WAKEWORD_MODEL || 'hey_ti_pote',
|
||||
modelName: process.env.WAKEWORD_MODEL || 'hey_jarvis',
|
||||
threshold: parseFloat(process.env.WAKEWORD_THRESHOLD || '0.75'),
|
||||
},
|
||||
serial: {
|
||||
|
||||
@ -16,6 +16,7 @@ import { type ITriggerService } from './services/trigger.interface.js';
|
||||
import { SetupFlow } from './setup/index.js';
|
||||
import { HardwareService, Emotion } from './hardware/index.js';
|
||||
import { createLogger, setLogForwarder } from './utils/index.js';
|
||||
import { SOUND_TRIGGER } from './utils/sounds.js';
|
||||
|
||||
const logger = createLogger('main', 'info');
|
||||
|
||||
@ -165,6 +166,55 @@ async function main(): Promise<void> {
|
||||
logForwarder.start(); // Flush logs to backend every 5s
|
||||
orchestrator.start();
|
||||
|
||||
// ── Remote trigger from desktop app ──
|
||||
|
||||
cloudSocket.on('remote_trigger', async () => {
|
||||
logger.info('🎯 Remote trigger from app — starting conversation');
|
||||
try {
|
||||
await audioService.play(SOUND_TRIGGER);
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'Failed to play trigger beep');
|
||||
}
|
||||
orchestrator.handleTriggerDetected();
|
||||
});
|
||||
|
||||
// ── Hot-swap trigger mode from desktop app ──
|
||||
|
||||
cloudSocket.on('set_trigger_mode', async (event: { mode: string }) => {
|
||||
const newMode = event.mode as 'wakeword' | 'keyboard';
|
||||
if (!['wakeword', 'keyboard'].includes(newMode)) {
|
||||
logger.warn({ mode: event.mode }, 'Invalid trigger mode from remote');
|
||||
return;
|
||||
}
|
||||
|
||||
if (newMode === resolvedConfig.triggerMode) {
|
||||
logger.info({ mode: newMode }, 'Trigger mode already set');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ from: resolvedConfig.triggerMode, to: newMode }, 'Switching trigger mode');
|
||||
|
||||
// Stop current trigger
|
||||
trigger.stop();
|
||||
|
||||
// Create new trigger
|
||||
if (newMode === 'wakeword') {
|
||||
trigger = new WakeWordService(
|
||||
hardwareConfig.wakeWord,
|
||||
hardwareConfig.audio,
|
||||
audioBackend === 'esp32' ? hardwareService : null,
|
||||
);
|
||||
} else {
|
||||
trigger = new KeyboardTriggerService();
|
||||
}
|
||||
|
||||
// Rewire orchestrator with new trigger
|
||||
orchestrator.swapTrigger(trigger);
|
||||
resolvedConfig.triggerMode = newMode;
|
||||
|
||||
logger.info({ mode: newMode }, 'Trigger mode switched');
|
||||
});
|
||||
|
||||
if (resolvedConfig.triggerMode === 'wakeword') {
|
||||
logger.info('Ti-Pote is ready! Say "Hey Ti-Pote" to start a conversation.');
|
||||
} else {
|
||||
|
||||
@ -39,7 +39,7 @@ export class OrchestratorService extends EventEmitter {
|
||||
constructor(
|
||||
private readonly cloudSocket: CloudSocket,
|
||||
private readonly audioService: AudioService,
|
||||
private readonly trigger: ITriggerService,
|
||||
private trigger: ITriggerService,
|
||||
private readonly audioConfig: AudioConfig,
|
||||
) {
|
||||
super();
|
||||
@ -83,9 +83,10 @@ export class OrchestratorService extends EventEmitter {
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle trigger detection (wake word or Enter key).
|
||||
* Handle trigger detection (wake word, Enter key, or remote command).
|
||||
* Public so that external callers (e.g. CloudSocket remote trigger) can invoke it.
|
||||
*/
|
||||
private async handleTriggerDetected(): Promise<void> {
|
||||
async handleTriggerDetected(): Promise<void> {
|
||||
if (this.state !== 'idle') {
|
||||
this.logger.debug({ state: this.state }, 'Trigger detected but not idle, ignoring');
|
||||
return;
|
||||
@ -314,6 +315,25 @@ export class OrchestratorService extends EventEmitter {
|
||||
this.logger.info('💤 Idle — waiting for wake word...');
|
||||
}
|
||||
|
||||
/**
|
||||
* Hot-swap the trigger service (e.g., switching between wakeword and keyboard).
|
||||
*/
|
||||
swapTrigger(newTrigger: ITriggerService): void {
|
||||
this.logger.info('Swapping trigger service');
|
||||
|
||||
// Wire up the new trigger's detected event
|
||||
newTrigger.on('detected', () => {
|
||||
this.handleTriggerDetected();
|
||||
});
|
||||
|
||||
this.trigger = newTrigger;
|
||||
|
||||
// Start the new trigger if we're idle
|
||||
if (this.state === 'idle') {
|
||||
newTrigger.start();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle user interrupt (e.g., button press, or second wake word).
|
||||
*/
|
||||
|
||||
@ -38,16 +38,38 @@ export class WakeWordService extends EventEmitter {
|
||||
private _isPaused = false;
|
||||
private _streamClosed = false;
|
||||
private readonly usesHardware: boolean;
|
||||
private restartCount = 0;
|
||||
private readonly maxRestarts = 5;
|
||||
/** Set once on first EPIPE — prevents log spam from in-flight chunks. */
|
||||
private _pipeBroken = false;
|
||||
/** Chunk counter for debug — logs every N chunks to confirm audio flows. */
|
||||
private _chunkCount = 0;
|
||||
|
||||
/** Latched forwarder so we can detach it on stop / error. */
|
||||
private readonly forwardMicChunk = (chunk: Buffer): void => {
|
||||
if (this._pipeBroken) return;
|
||||
if (!this.process || !this.process.stdin || this.process.stdin.destroyed) return;
|
||||
this._chunkCount++;
|
||||
if (this._chunkCount % 500 === 0) {
|
||||
this.logger.info({ chunks: this._chunkCount, paused: this._isPaused }, 'Wake word audio flowing');
|
||||
}
|
||||
try {
|
||||
this.process.stdin.write(chunk, (err) => {
|
||||
if (err && (err as NodeJS.ErrnoException).code === 'EPIPE') {
|
||||
this.logger.warn('Wake word process stdin pipe broken — detaching audio');
|
||||
if (!this._pipeBroken) {
|
||||
this._pipeBroken = true;
|
||||
this.logger.warn('Wake word stdin pipe broken — detaching audio');
|
||||
this.detachHardware();
|
||||
}
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
if (!this._pipeBroken) {
|
||||
this._pipeBroken = true;
|
||||
this.logger.warn('Wake word stdin write failed — detaching audio');
|
||||
this.detachHardware();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
constructor(
|
||||
@ -105,6 +127,20 @@ export class WakeWordService extends EventEmitter {
|
||||
|
||||
this._isListening = true;
|
||||
this._isPaused = false;
|
||||
this._pipeBroken = false;
|
||||
|
||||
// Catch EPIPE on stdin so it doesn't bubble as uncaughtException
|
||||
this.process.stdin?.on('error', (err) => {
|
||||
if ((err as NodeJS.ErrnoException).code === 'EPIPE') {
|
||||
if (!this._pipeBroken) {
|
||||
this._pipeBroken = true;
|
||||
this.logger.debug('Wake word stdin EPIPE (process exited)');
|
||||
this.detachHardware();
|
||||
}
|
||||
} else {
|
||||
this.logger.error({ err }, 'Wake word stdin error');
|
||||
}
|
||||
});
|
||||
|
||||
this.process.stdout?.on('data', (data: Buffer) => {
|
||||
const lines = data.toString().trim().split('\n');
|
||||
@ -129,12 +165,12 @@ export class WakeWordService extends EventEmitter {
|
||||
this.emit('ready');
|
||||
} else if (msg === 'PAUSED') {
|
||||
this._streamClosed = false;
|
||||
this.logger.debug('Wake word paused');
|
||||
this.logger.info('⏸️ Wake word paused');
|
||||
} else if (msg === 'STREAM_CLOSED') {
|
||||
this._streamClosed = true;
|
||||
this.logger.debug('Wake word audio stream closed');
|
||||
} else if (msg === 'RESUMED') {
|
||||
this.logger.debug('Wake word resumed');
|
||||
this.logger.info('▶️ Wake word resumed (Python confirmed)');
|
||||
} else if (msg === 'STREAM_REOPENED') {
|
||||
this.logger.debug('Wake word audio stream reopened');
|
||||
} else if (msg.startsWith('Loading wake word model')) {
|
||||
@ -162,19 +198,42 @@ export class WakeWordService extends EventEmitter {
|
||||
this.detachHardware();
|
||||
this.process = null;
|
||||
if (code !== 0 && code !== null) {
|
||||
this.logger.warn({ code }, 'Wake word process exited unexpectedly');
|
||||
this.restartCount++;
|
||||
if (this.restartCount > this.maxRestarts) {
|
||||
this.logger.error(
|
||||
{ code, restarts: this.restartCount },
|
||||
'Wake word exceeded max restarts — giving up. Fix the issue and restart the service.',
|
||||
);
|
||||
this.emit('error', new Error(`Wake word crashed ${this.restartCount} times`));
|
||||
return;
|
||||
}
|
||||
this.logger.warn({ code, restart: this.restartCount }, 'Wake word process exited unexpectedly');
|
||||
setTimeout(() => {
|
||||
this.logger.info('Restarting wake word detection...');
|
||||
this.start();
|
||||
}, 2000);
|
||||
} else {
|
||||
// Clean exit — reset counter
|
||||
this.restartCount = 0;
|
||||
}
|
||||
});
|
||||
|
||||
// In ESP32 mode, start piping mic audio from the UART.
|
||||
if (this.usesHardware && this.hardware) {
|
||||
// In ESP32 mode, catch errors on the control pipe (fd 3) and start piping mic audio.
|
||||
if (this.usesHardware) {
|
||||
const controlPipe = this.process.stdio[3] as unknown as
|
||||
| (NodeJS.WritableStream & { on?: (event: string, cb: (err: Error) => void) => void })
|
||||
| null;
|
||||
if (controlPipe?.on) {
|
||||
controlPipe.on('error', (err: Error) => {
|
||||
this.logger.debug({ err: (err as NodeJS.ErrnoException).code }, 'Control pipe error (process exited)');
|
||||
});
|
||||
}
|
||||
|
||||
if (this.hardware) {
|
||||
this.hardware.on('audio_up', this.forwardMicChunk);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause wake word detection.
|
||||
|
||||
@ -29,6 +29,8 @@ export interface CloudSocketEvents {
|
||||
response_text: (event: ResponseTextEvent) => void;
|
||||
audio_chunk: (event: AudioChunkEvent) => void;
|
||||
notification: (payload: Record<string, unknown>) => void;
|
||||
remote_trigger: () => void;
|
||||
set_trigger_mode: (event: { mode: string }) => void;
|
||||
error: (error: Error) => void;
|
||||
}
|
||||
|
||||
@ -123,6 +125,18 @@ export class CloudSocket extends EventEmitter {
|
||||
this.logger.info({ payload }, 'Notification received');
|
||||
this.emit('notification', payload);
|
||||
});
|
||||
|
||||
// ── Remote control events from desktop app (via AppGateway → RobotGateway) ──
|
||||
|
||||
this.socket.on('remote_trigger', () => {
|
||||
this.logger.info('Remote trigger received from app');
|
||||
this.emit('remote_trigger');
|
||||
});
|
||||
|
||||
this.socket.on('set_trigger_mode', (event: { mode: string }) => {
|
||||
this.logger.info({ mode: event.mode }, 'Trigger mode change from app');
|
||||
this.emit('set_trigger_mode', event);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
80
apps/robot-client/src/utils/sounds.ts
Normal file
80
apps/robot-client/src/utils/sounds.ts
Normal file
@ -0,0 +1,80 @@
|
||||
/**
|
||||
* Pre-generated notification sounds for the robot.
|
||||
*
|
||||
* All sounds are raw S16LE mono PCM at 16 kHz, ready to pass to AudioService.play().
|
||||
* Generated once at import time — zero runtime cost per playback.
|
||||
*/
|
||||
|
||||
const SAMPLE_RATE = 16_000;
|
||||
|
||||
/**
|
||||
* Generate a sine wave tone with attack/release envelope to prevent speaker clicks.
|
||||
*/
|
||||
function generateTone(freqHz: number, durationMs: number, amplitude = 0.5): Buffer {
|
||||
const sampleCount = Math.floor((SAMPLE_RATE * durationMs) / 1000);
|
||||
const buf = Buffer.alloc(sampleCount * 2); // 16-bit = 2 bytes per sample
|
||||
const amp = Math.max(0, Math.min(1, amplitude)) * 32767;
|
||||
const twoPiF = (2 * Math.PI * freqHz) / SAMPLE_RATE;
|
||||
|
||||
// 5ms linear attack/release envelope
|
||||
const rampSamples = Math.floor((SAMPLE_RATE * 5) / 1000);
|
||||
|
||||
for (let i = 0; i < sampleCount; i++) {
|
||||
let env = 1;
|
||||
if (i < rampSamples) env = i / rampSamples;
|
||||
else if (i > sampleCount - rampSamples) env = (sampleCount - i) / rampSamples;
|
||||
|
||||
const s = Math.round(Math.sin(i * twoPiF) * amp * env);
|
||||
buf.writeInt16LE(Math.max(-32768, Math.min(32767, s)), i * 2);
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Concatenate multiple PCM buffers with silence gaps between them.
|
||||
*/
|
||||
function concat(parts: Buffer[], gapMs = 0): Buffer {
|
||||
if (gapMs <= 0) return Buffer.concat(parts);
|
||||
|
||||
const gapBytes = Math.floor((SAMPLE_RATE * gapMs) / 1000) * 2;
|
||||
const silence = Buffer.alloc(gapBytes);
|
||||
const result: Buffer[] = [];
|
||||
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
result.push(parts[i]);
|
||||
if (i < parts.length - 1) result.push(silence);
|
||||
}
|
||||
|
||||
return Buffer.concat(result);
|
||||
}
|
||||
|
||||
// ── Pre-generated sounds ──────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Short rising two-tone chirp: "bi-bip!" (80ms + 80ms)
|
||||
* Played when a remote trigger starts a conversation.
|
||||
*/
|
||||
export const SOUND_TRIGGER = concat(
|
||||
[
|
||||
generateTone(660, 80, 0.4), // E5
|
||||
generateTone(880, 80, 0.4), // A5
|
||||
],
|
||||
30, // 30ms gap
|
||||
);
|
||||
|
||||
/**
|
||||
* Single soft beep for generic notifications.
|
||||
*/
|
||||
export const SOUND_NOTIFY = generateTone(440, 120, 0.3);
|
||||
|
||||
/**
|
||||
* Three descending tones for errors.
|
||||
*/
|
||||
export const SOUND_ERROR = concat(
|
||||
[
|
||||
generateTone(880, 100, 0.35),
|
||||
generateTone(660, 100, 0.35),
|
||||
generateTone(440, 150, 0.35),
|
||||
],
|
||||
40,
|
||||
);
|
||||
3
pnpm-lock.yaml
generated
3
pnpm-lock.yaml
generated
@ -170,6 +170,9 @@ importers:
|
||||
react-router-dom:
|
||||
specifier: ^6.26.0
|
||||
version: 6.30.3(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
|
||||
socket.io-client:
|
||||
specifier: ^4.8.3
|
||||
version: 4.8.3
|
||||
zustand:
|
||||
specifier: ^4.5.5
|
||||
version: 4.5.7(@types/react@18.3.28)(react@18.3.1)
|
||||
|
||||
@ -81,11 +81,15 @@ $SSH "sudo mkdir -p /opt/tipote && sudo chown ${PI_USER}:${PI_USER} /opt/tipote"
|
||||
|
||||
echo "▸ [5/7] Uploading files..."
|
||||
|
||||
# Sync dist/ and package.json
|
||||
# Sync dist/, scripts/, and package.json
|
||||
rsync -az --delete \
|
||||
"$ROBOT_CLIENT_DIR/dist/" \
|
||||
"${PI_USER}@${PI_HOST}:/opt/tipote/dist/"
|
||||
|
||||
rsync -az --delete \
|
||||
"$ROBOT_CLIENT_DIR/scripts/wake_word.py" \
|
||||
"${PI_USER}@${PI_HOST}:/opt/tipote/scripts/"
|
||||
|
||||
$SCP "$ROBOT_CLIENT_DIR/package.json" "${PI_USER}@${PI_HOST}:/opt/tipote/package.json"
|
||||
|
||||
# Install production deps on Pi
|
||||
@ -115,6 +119,7 @@ HARDWARE_SERIAL_ENABLED=true
|
||||
HARDWARE_SERIAL_PORT=/dev/serial0
|
||||
HARDWARE_SERIAL_BAUD_RATE=921600
|
||||
WAKEWORD_PYTHON_PATH=/opt/tipote/.venv/bin/python3
|
||||
WAKEWORD_MODEL=hey_jarvis
|
||||
NODE_ENV=production
|
||||
LOG_LEVEL=info
|
||||
ENVEOF
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user