diff --git a/_tmp_945_290d62cdc9142d0226f30816d854ae86 b/_tmp_945_290d62cdc9142d0226f30816d854ae86 new file mode 100644 index 0000000..e69de29 diff --git a/_tmp_945_eb782ccbcda1bcca02bbc45f41d4c1b8 b/_tmp_945_eb782ccbcda1bcca02bbc45f41d4c1b8 new file mode 100644 index 0000000..e69de29 diff --git a/apps/robot-client/.env.dev b/apps/robot-client/.env.dev new file mode 100644 index 0000000..02236ff --- /dev/null +++ b/apps/robot-client/.env.dev @@ -0,0 +1,52 @@ +# ────────────────────────────────────────────── +# Ti-Pote Robot Client — Dev/Test Configuration +# Pour tester sur Raspberry Pi 3B+ avec casque USB Logitech Pro X +# ────────────────────────────────────────────── + +# Mode dev : test sur Pi avec USB audio +ROBOT_MODE=dev + +# Trigger mode : 'wakeword' (OpenWakeWord) ou 'keyboard' (Entrée = wake word) +# Utilise 'wakeword' si tu as installé OpenWakeWord (pip install openwakeword pyaudio) +# Utilise 'keyboard' pour tester sans OpenWakeWord +TRIGGER_MODE=keyboard + +# Device identification (à remplir après avoir enregistré le device sur le backend) +DEVICE_ID= +DEVICE_TOKEN= + +# Cloud backend URL (le backend NestJS tourne sur ton Mac ou un serveur) +CLOUD_URL=ws://192.168.1.XXX:3000 + +# Robot name +ROBOT_NAME=Ti-Pote-Dev + +# Log level debug pour le dev +LOG_LEVEL=debug + +# ── Audio (ALSA — casque USB Logitech Pro X) ── +# +# Pour trouver le bon device, lance sur le Pi : +# arecord -l → liste les devices de capture (micro) +# aplay -l → liste les devices de playback (speaker/casque) +# +# Le casque USB apparaîtra comme un device "USB Audio" ou "Logitech". +# Note le numéro de carte (card X) et de device (device Y), +# puis utilise : plughw:X,Y +# +# Exemple si le casque est card 1, device 0 : +AUDIO_CAPTURE_DEVICE=plughw:1,0 +AUDIO_PLAYBACK_DEVICE=plughw:1,0 + +# Sample rate 16kHz (doit matcher le backend) +AUDIO_SAMPLE_RATE=16000 + +# Chunks de 100ms +AUDIO_CHUNK_MS=100 + +# ── Wake Word ── +# Python du venv où openwakeword est installé +WAKEWORD_PYTHON_PATH=/home/ti-pote/.tipote-venv/bin/python3 +WAKEWORD_SCRIPT_PATH=./scripts/wake_word.py +WAKEWORD_MODEL=hey_jarvis +WAKEWORD_THRESHOLD=0.5 diff --git a/apps/robot-client/.env.example b/apps/robot-client/.env.example new file mode 100644 index 0000000..cbc6200 --- /dev/null +++ b/apps/robot-client/.env.example @@ -0,0 +1,55 @@ +# ────────────────────────────────────────────── +# Ti-Pote Robot Client — Configuration +# ────────────────────────────────────────────── + +# Mode: 'physical' (Raspberry Pi prod) | 'dev' (Pi + USB audio test) | 'simulator' (PC dev) +ROBOT_MODE=physical + +# Trigger mode: 'wakeword' (OpenWakeWord) | 'keyboard' (press Enter = wake word) +TRIGGER_MODE=wakeword + +# Device identification (set during first setup or manually) +DEVICE_ID= +DEVICE_TOKEN= + +# Cloud backend URL +CLOUD_URL=ws://localhost:3000 + +# Robot name +ROBOT_NAME=Ti-Pote + +# Log level: fatal, error, warn, info, debug, trace +LOG_LEVEL=info + +# ── Audio (ALSA) ── + +# ALSA device for microphone capture +# Examples: 'default', 'plughw:1,0', 'hw:1,0' +# Use 'arecord -l' to list available devices +AUDIO_CAPTURE_DEVICE=default + +# ALSA device for speaker playback +# Examples: 'default', 'plughw:0,0', 'hw:0,0' +# Use 'aplay -l' to list available devices +AUDIO_PLAYBACK_DEVICE=default + +# Audio sample rate in Hz (must match backend: 16000) +AUDIO_SAMPLE_RATE=16000 + +# Audio chunk duration in ms (how often chunks are sent to cloud) +AUDIO_CHUNK_MS=100 + +# ── Wake Word ── + +# Path to the Python binary (use venv python if openwakeword is in a virtualenv) +# Examples: 'python3', '/home/pi/.tipote-venv/bin/python3' +WAKEWORD_PYTHON_PATH=python3 + +# Path to the OpenWakeWord Python script +WAKEWORD_SCRIPT_PATH=./scripts/wake_word.py + +# Wake word model name (use 'hey_jarvis' as placeholder until custom model is trained) +WAKEWORD_MODEL=hey_jarvis + +# Wake word detection threshold (0.0 to 1.0, higher = less false positives) +WAKEWORD_THRESHOLD=0.5 diff --git a/apps/robot-client/package.json b/apps/robot-client/package.json new file mode 100644 index 0000000..b1d8c9f --- /dev/null +++ b/apps/robot-client/package.json @@ -0,0 +1,31 @@ +{ + "name": "@ti-pote/robot-client", + "version": "0.0.1", + "private": true, + "type": "module", + "description": "Ti-Pote Robot Client — Runs on Raspberry Pi Zero 2W", + "scripts": { + "dev": "tsx watch src/main.ts", + "build": "tsup src/main.ts --format esm --dts --clean", + "start": "node dist/main.js", + "lint": "eslint \"src/**/*.ts\" --fix", + "format": "prettier --write \"src/**/*.ts\"", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "socket.io-client": "^4.8.3", + "dotenv": "^17.3.1", + "pino": "^9.6.0", + "pino-pretty": "^13.0.0" + }, + "devDependencies": { + "typescript": "^5.8.3", + "tsx": "^4.21.0", + "tsup": "^8.5.0", + "@types/node": "^22.15.0", + "vitest": "^3.2.1", + "eslint": "^10.1.0", + "prettier": "^3.8.1" + } +} diff --git a/apps/robot-client/scripts/install.sh b/apps/robot-client/scripts/install.sh new file mode 100755 index 0000000..e6c44e3 --- /dev/null +++ b/apps/robot-client/scripts/install.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# ────────────────────────────────────────────── +# Ti-Pote Robot Client — Installation Script +# For Raspberry Pi Zero 2W running Raspberry Pi OS Lite (64-bit) +# ────────────────────────────────────────────── + +set -euo pipefail + +echo "╔══════════════════════════════════════╗" +echo "║ Ti-Pote Robot Client Installer ║" +echo "╚══════════════════════════════════════╝" + +# ── System packages ── + +echo "" +echo "→ Updating system packages..." +sudo apt update && sudo apt upgrade -y + +echo "→ Installing system dependencies..." +sudo apt install -y \ + git \ + curl \ + alsa-utils \ + python3 \ + python3-pip \ + python3-venv \ + portaudio19-dev \ + libatlas-base-dev + +# ── Node.js (LTS via nvm) ── + +echo "" +echo "→ Installing Node.js LTS..." +if ! command -v node &> /dev/null; then + curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - + sudo apt install -y nodejs + echo "Node.js $(node -v) installed" +else + echo "Node.js $(node -v) already installed" +fi + +# ── pnpm ── + +echo "" +echo "→ Installing pnpm..." +if ! command -v pnpm &> /dev/null; then + npm install -g pnpm + echo "pnpm installed" +else + echo "pnpm already installed" +fi + +# ── Python dependencies (wake word) ── + +echo "" +echo "→ Setting up Python virtual environment for wake word..." +VENV_DIR="$HOME/.tipote-venv" +if [ ! -d "$VENV_DIR" ]; then + python3 -m venv "$VENV_DIR" +fi +source "$VENV_DIR/bin/activate" + +pip install --upgrade pip +pip install openwakeword pyaudio numpy + +deactivate +echo "Python dependencies installed in $VENV_DIR" + +# ── I2S Audio setup (for INMP441 + MAX98357) ── + +echo "" +echo "→ Checking I2S audio configuration..." +CONFIG_FILE="/boot/firmware/config.txt" +if [ ! -f "$CONFIG_FILE" ]; then + CONFIG_FILE="/boot/config.txt" +fi + +if ! grep -q "dtoverlay=googlevoicehat-soundcard" "$CONFIG_FILE" 2>/dev/null; then + echo "" + echo "⚠ I2S audio overlay not configured." + echo " To enable I2S audio (INMP441 mic + MAX98357 amp), add to $CONFIG_FILE:" + echo "" + echo " # I2S audio for Ti-Pote" + echo " dtparam=i2s=on" + echo " dtoverlay=googlevoicehat-soundcard" + echo "" + echo " Then reboot the Pi." + echo "" + echo " Alternatively, if using USB audio, no changes are needed." +else + echo "I2S audio overlay already configured" +fi + +# ── Verify audio devices ── + +echo "" +echo "→ Available audio capture devices:" +arecord -l 2>/dev/null || echo " (no capture devices found — connect a microphone)" + +echo "" +echo "→ Available audio playback devices:" +aplay -l 2>/dev/null || echo " (no playback devices found — connect a speaker)" + +# ── Summary ── + +echo "" +echo "╔══════════════════════════════════════╗" +echo "║ Installation complete! ║" +echo "╚══════════════════════════════════════╝" +echo "" +echo "Next steps:" +echo " 1. Configure audio (see above)" +echo " 2. Copy .env.example to .env and fill in your device credentials" +echo " 3. Run: pnpm install" +echo " 4. Run: pnpm dev" +echo "" diff --git a/apps/robot-client/scripts/wake_word.py b/apps/robot-client/scripts/wake_word.py new file mode 100755 index 0000000..c91b589 --- /dev/null +++ b/apps/robot-client/scripts/wake_word.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +""" +Ti-Pote Wake Word Detection Script. + +Runs OpenWakeWord model continuously, listening on the specified ALSA device. +Prints "DETECTED" to stdout when the wake word is heard. + +Supports PAUSE/RESUME commands on stdin to temporarily stop/start listening +without reloading the model. When paused, the audio stream is closed so other +processes (arecord) can use the device. + +Usage: + python3 wake_word.py --model hey_jarvis --threshold 0.5 --device default --sample-rate 16000 + +Requirements: + pip install openwakeword pyaudio numpy +""" + +import argparse +import sys +import os +import signal +import select +import threading +import numpy as np + + +def main(): + parser = argparse.ArgumentParser(description='Ti-Pote Wake Word Detection') + parser.add_argument('--model', type=str, default='hey_jarvis', + help='Wake word model name (default: hey_jarvis as placeholder)') + parser.add_argument('--threshold', type=float, default=0.5, + help='Detection threshold (0.0-1.0)') + parser.add_argument('--device', type=str, default='default', + help='ALSA audio capture device') + parser.add_argument('--sample-rate', type=int, default=16000, + help='Audio sample rate in Hz') + args = parser.parse_args() + + try: + from openwakeword.model import Model + except ImportError: + print("ERROR: openwakeword not installed. Run: pip install openwakeword", file=sys.stderr) + sys.exit(1) + + try: + import pyaudio + except ImportError: + print("ERROR: pyaudio not installed. Run: pip install pyaudio", file=sys.stderr) + sys.exit(1) + + # ── Load the wake word model (one time only) ── + + print(f"Loading wake word model: {args.model}...", file=sys.stderr) + + import openwakeword + pretrained_paths = openwakeword.get_pretrained_model_paths() + model_path = None + for p in pretrained_paths: + basename = os.path.basename(p) + if basename.startswith(args.model): + model_path = p + break + + if model_path is None: + if os.path.isfile(args.model): + model_path = args.model + else: + print(f"ERROR: model '{args.model}' not found in pretrained models", file=sys.stderr) + print(f"Available models:", file=sys.stderr) + for p in pretrained_paths: + print(f" - {os.path.basename(p)}", file=sys.stderr) + sys.exit(1) + + print(f"Resolved model path: {model_path}", file=sys.stderr) + + try: + oww_model = Model(wakeword_model_paths=[model_path]) + except Exception as e: + print(f"ERROR loading model '{args.model}': {e}", file=sys.stderr) + sys.exit(1) + + print(f"Wake word model loaded: {args.model}", file=sys.stderr) + print(f"Threshold: {args.threshold}", file=sys.stderr) + print(f"Listening on device: {args.device}", file=sys.stderr) + + # ── Initialize PyAudio ── + + pa = pyaudio.PyAudio() + + # Find the device index + import re + device_index = None + if args.device != 'default': + try: + idx = int(args.device) + info = pa.get_device_info_by_index(idx) + if info.get('maxInputChannels', 0) > 0: + device_index = idx + print(f"Using device by index: [{idx}] {info['name']}", file=sys.stderr) + except (ValueError, IOError): + pass + + if device_index is None: + hw_match = re.search(r'(\d+),(\d+)', args.device) + hw_pattern = f"hw:{hw_match.group(1)},{hw_match.group(2)}" if hw_match else None + + for i in range(pa.get_device_count()): + info = pa.get_device_info_by_index(i) + if info.get('maxInputChannels', 0) <= 0: + continue + name = str(info.get('name', '')) + if (hw_pattern and hw_pattern in name) or args.device in name: + device_index = i + print(f"Matched device: [{i}] {name}", file=sys.stderr) + break + + if device_index is None: + print(f"WARNING: Device '{args.device}' not found, listing available inputs:", file=sys.stderr) + for i in range(pa.get_device_count()): + info = pa.get_device_info_by_index(i) + if info.get('maxInputChannels', 0) > 0: + print(f" [{i}] {info['name']}", file=sys.stderr) + print("Falling back to default device", file=sys.stderr) + + # ── Audio stream helpers ── + + chunk_size = 1280 # ~80ms at 16kHz (OpenWakeWord expects this) + stream = None + + def open_stream(): + nonlocal stream + stream = pa.open( + format=pyaudio.paInt16, + channels=1, + rate=args.sample_rate, + input=True, + frames_per_buffer=chunk_size, + input_device_index=device_index, + ) + + def close_stream(): + nonlocal stream + if stream is not None: + try: + stream.stop_stream() + stream.close() + except Exception: + pass + stream = None + + # ── Stdin command reader (PAUSE / RESUME) ── + + paused = False + running = True + lock = threading.Lock() + + def stdin_reader(): + nonlocal paused, running + while running: + try: + line = sys.stdin.readline() + if not line: # EOF + running = False + break + cmd = line.strip().upper() + with lock: + if cmd == 'PAUSE': + if not paused: + paused = True + print("PAUSED", file=sys.stderr, flush=True) + elif cmd == 'RESUME': + if paused: + paused = False + print("RESUMED", file=sys.stderr, flush=True) + elif cmd == 'QUIT': + running = False + break + except Exception: + break + + stdin_thread = threading.Thread(target=stdin_reader, daemon=True) + stdin_thread.start() + + # ── Signal handling ── + + def handle_signal(sig, frame): + nonlocal running + running = False + signal.signal(signal.SIGTERM, handle_signal) + signal.signal(signal.SIGINT, handle_signal) + + # ── Main loop ── + + open_stream() + print("READY", file=sys.stderr, flush=True) + + try: + while running: + with lock: + is_paused = paused + + if is_paused: + # Close the audio stream so arecord can use the device + if stream is not None: + close_stream() + print("STREAM_CLOSED", file=sys.stderr, flush=True) + # Wait a bit before checking again + import time + time.sleep(0.1) + continue + + # Reopen stream if it was closed (after resume) + if stream is None: + open_stream() + oww_model.reset() + print("STREAM_REOPENED", file=sys.stderr, flush=True) + + try: + audio_data = stream.read(chunk_size, exception_on_overflow=False) + except Exception as e: + print(f"Audio read error: {e}", file=sys.stderr) + close_stream() + import time + time.sleep(0.5) + continue + + audio_array = np.frombuffer(audio_data, dtype=np.int16) + + oww_model.predict(audio_array) + + for model_name, score in oww_model.prediction_buffer.items(): + if len(score) > 0 and score[-1] > args.threshold: + print("DETECTED", flush=True) + oww_model.reset() + break + + except KeyboardInterrupt: + pass + finally: + close_stream() + pa.terminate() + print("Wake word detection stopped", file=sys.stderr) + + +if __name__ == '__main__': + main() diff --git a/apps/robot-client/src/config/hardware.config.ts b/apps/robot-client/src/config/hardware.config.ts new file mode 100644 index 0000000..7fd4934 --- /dev/null +++ b/apps/robot-client/src/config/hardware.config.ts @@ -0,0 +1,57 @@ +export interface AudioConfig { + /** ALSA device for capture (e.g., 'plughw:1,0' or 'default') */ + captureDevice: string; + + /** ALSA device for playback (e.g., 'plughw:0,0' or 'default') */ + playbackDevice: string; + + /** Sample rate in Hz */ + sampleRate: number; + + /** Bits per sample */ + bitDepth: number; + + /** Number of audio channels */ + channels: number; + + /** Audio chunk duration in ms (how often we send chunks to cloud) */ + chunkDurationMs: number; +} + +export interface WakeWordConfig { + /** Path to Python binary (use venv python if openwakeword is in a venv) */ + pythonPath: string; + + /** Path to the OpenWakeWord Python script */ + scriptPath: string; + + /** Wake word model name or path */ + modelName: string; + + /** Detection threshold (0.0 to 1.0) */ + threshold: number; +} + +export interface HardwareConfig { + audio: AudioConfig; + wakeWord: WakeWordConfig; +} + +export function loadHardwareConfig(): HardwareConfig { + return { + audio: { + captureDevice: process.env.AUDIO_CAPTURE_DEVICE || 'default', + playbackDevice: process.env.AUDIO_PLAYBACK_DEVICE || 'default', + sampleRate: parseInt(process.env.AUDIO_SAMPLE_RATE || '16000', 10), + bitDepth: 16, + channels: 1, + chunkDurationMs: parseInt(process.env.AUDIO_CHUNK_MS || '100', 10), + }, + wakeWord: { + pythonPath: process.env.WAKEWORD_PYTHON_PATH || 'python3', + scriptPath: process.env.WAKEWORD_SCRIPT_PATH || './scripts/wake_word.py', + modelName: process.env.WAKEWORD_MODEL || 'hey_ti_pote', + threshold: parseFloat(process.env.WAKEWORD_THRESHOLD || '0.5'), + }, + }; +} diff --git a/apps/robot-client/src/config/index.ts b/apps/robot-client/src/config/index.ts new file mode 100644 index 0000000..10bddb1 --- /dev/null +++ b/apps/robot-client/src/config/index.ts @@ -0,0 +1,2 @@ +export { loadRobotConfig, type RobotConfig, type RobotMode, type TriggerMode } from './robot.config.js'; +export { loadHardwareConfig, type HardwareConfig, type AudioConfig, type WakeWordConfig } from './hardware.config.js'; diff --git a/apps/robot-client/src/config/robot.config.ts b/apps/robot-client/src/config/robot.config.ts new file mode 100644 index 0000000..35c6dc5 --- /dev/null +++ b/apps/robot-client/src/config/robot.config.ts @@ -0,0 +1,75 @@ +import { resolve } from 'node:path'; +import { config } from 'dotenv'; + +/** + * Robot operating modes: + * - 'physical': Production on Pi with I2S audio + * - 'dev': Test on Pi with USB audio (e.g., USB headset) + * - 'simulator': PC dev, no real audio (future: mock services) + */ +export type RobotMode = 'physical' | 'dev' | 'simulator'; + +/** + * How conversations are triggered: + * - 'wakeword': OpenWakeWord listens for "Hey Ti-Pote" (requires Python + openwakeword) + * - 'keyboard': Press Enter in the terminal to start a conversation + */ +export type TriggerMode = 'wakeword' | 'keyboard'; + +export interface RobotConfig { + /** Operating mode */ + mode: RobotMode; + + /** How conversations are triggered */ + triggerMode: TriggerMode; + + /** Unique device identifier (generated at first setup or from env) */ + deviceId: string; + + /** JWT device token for cloud authentication */ + deviceToken: string; + + /** Cloud backend WebSocket URL */ + cloudUrl: string; + + /** Robot display name */ + robotName: string; + + /** Log level */ + logLevel: string; +} + +export function loadRobotConfig(): RobotConfig { + // Load the right .env file based on ROBOT_MODE or --env flag + const envFile = process.env.ENV_FILE || (process.env.ROBOT_MODE === 'dev' ? '.env.dev' : '.env'); + config({ path: resolve(process.cwd(), envFile) }); + + const mode = (process.env.ROBOT_MODE || 'dev') as RobotMode; + + if (!['physical', 'dev', 'simulator'].includes(mode)) { + throw new Error(`Invalid ROBOT_MODE: ${mode}. Must be 'physical', 'dev', or 'simulator'.`); + } + + const triggerMode = (process.env.TRIGGER_MODE || 'wakeword') as TriggerMode; + if (!['wakeword', 'keyboard'].includes(triggerMode)) { + throw new Error(`Invalid TRIGGER_MODE: ${triggerMode}. Must be 'wakeword' or 'keyboard'.`); + } + + return { + mode, + triggerMode, + deviceId: requireEnv('DEVICE_ID'), + deviceToken: requireEnv('DEVICE_TOKEN'), + cloudUrl: process.env.CLOUD_URL || 'ws://localhost:3000', + robotName: process.env.ROBOT_NAME || 'Ti-Pote', + logLevel: process.env.LOG_LEVEL || 'info', + }; +} + +function requireEnv(key: string): string { + const value = process.env[key]; + if (!value) { + throw new Error(`Missing required environment variable: ${key}`); + } + return value; +} diff --git a/apps/robot-client/src/main.ts b/apps/robot-client/src/main.ts new file mode 100644 index 0000000..1836163 --- /dev/null +++ b/apps/robot-client/src/main.ts @@ -0,0 +1,151 @@ +import { loadRobotConfig, loadHardwareConfig } from './config/index.js'; +import { CloudSocket } from './transport/index.js'; +import { + AudioService, + WakeWordService, + KeyboardTriggerService, + HealthService, + OrchestratorService, + LocalStore, + WifiService, +} from './services/index.js'; +import { type ITriggerService } from './services/trigger.interface.js'; +import { SetupFlow } from './setup/index.js'; +import { createLogger } from './utils/index.js'; + +const logger = createLogger('main', 'info'); + +async function main(): Promise { + logger.info('╔══════════════════════════════════════╗'); + logger.info('║ Ti-Pote Robot Client v0.0.1 ║'); + logger.info('╚══════════════════════════════════════╝'); + + // ── Load configuration ── + + const robotConfig = loadRobotConfig(); + const hardwareConfig = loadHardwareConfig(); + const store = new LocalStore(); + + logger.info( + { mode: robotConfig.mode, triggerMode: robotConfig.triggerMode }, + 'Configuration loaded', + ); + + // ── Step 1: Ensure WiFi connectivity ── + // Only run the setup flow (captive portal) in physical/production mode. + // In dev mode, the Pi is already on the network (configured manually). + // In simulator mode, the dev machine is already on the network. + + if (robotConfig.mode === 'physical') { + const wifiService = new WifiService(); + const setupFlow = new SetupFlow(wifiService, store, robotConfig.robotName); + + const wifiReady = await setupFlow.run(); + if (!wifiReady) { + logger.fatal('WiFi setup failed — cannot continue without network'); + process.exit(1); + } + } else { + logger.info('Skipping WiFi setup (mode: %s)', robotConfig.mode); + } + + // ── Step 2: Resolve device credentials ── + // Use stored device credentials if available, fall back to env vars. + + const deviceId = store.device?.id || robotConfig.deviceId; + const deviceToken = store.device?.token || robotConfig.deviceToken; + + if (!deviceId || !deviceToken) { + logger.fatal( + 'No device credentials found. Register this device on the backend first, ' + + 'then set DEVICE_ID and DEVICE_TOKEN in your .env file.', + ); + process.exit(1); + } + + // Override config with resolved credentials + const resolvedConfig = { ...robotConfig, deviceId, deviceToken }; + + logger.info({ deviceId }, 'Device credentials resolved'); + + // ── Step 3: Initialize services ── + + const cloudSocket = new CloudSocket(resolvedConfig); + const audioService = new AudioService(hardwareConfig.audio); + const healthService = new HealthService(cloudSocket); + + // Choose trigger based on TRIGGER_MODE: + // wakeword → OpenWakeWord subprocess (requires Python + openwakeword) + // keyboard → Press Enter in terminal to talk + let trigger: ITriggerService; + + if (resolvedConfig.triggerMode === 'wakeword') { + logger.info('Trigger: wake word (OpenWakeWord)'); + trigger = new WakeWordService(hardwareConfig.wakeWord, hardwareConfig.audio); + } else { + logger.info('Trigger: keyboard (press Enter to talk)'); + trigger = new KeyboardTriggerService(); + } + + const orchestrator = new OrchestratorService( + cloudSocket, + audioService, + trigger, + hardwareConfig.audio, + ); + + // ── Step 4: Connect to cloud ── + + logger.info({ url: resolvedConfig.cloudUrl }, 'Connecting to cloud backend...'); + + try { + await cloudSocket.connect(); + logger.info('Cloud connection established'); + } catch (err) { + logger.fatal({ err }, 'Failed to connect to cloud backend'); + process.exit(1); + } + + // ── Step 5: Start services ── + + healthService.start(); + orchestrator.start(); + + if (resolvedConfig.triggerMode === 'wakeword') { + logger.info('Ti-Pote is ready! Say "Hey Ti-Pote" to start a conversation.'); + } else { + logger.info('Ti-Pote is ready! Press Enter to start a conversation.'); + } + + // ── Graceful shutdown ── + + const shutdown = async (signal: string) => { + logger.info({ signal }, 'Shutdown signal received'); + + await orchestrator.stop(); + healthService.stop(); + await audioService.destroy(); + await cloudSocket.disconnect(); + + logger.info('Goodbye!'); + process.exit(0); + }; + + process.on('SIGINT', () => shutdown('SIGINT')); + process.on('SIGTERM', () => shutdown('SIGTERM')); + + process.on('uncaughtException', (err) => { + logger.fatal({ err }, 'Uncaught exception'); + shutdown('uncaughtException'); + }); + + process.on('unhandledRejection', (reason) => { + logger.fatal({ reason }, 'Unhandled promise rejection'); + shutdown('unhandledRejection'); + }); +} + +main().catch((err) => { + logger.fatal({ err }, 'Fatal error during startup'); + process.exit(1); +}); diff --git a/apps/robot-client/src/services/audio.service.ts b/apps/robot-client/src/services/audio.service.ts new file mode 100644 index 0000000..c44bc73 --- /dev/null +++ b/apps/robot-client/src/services/audio.service.ts @@ -0,0 +1,203 @@ +import { ChildProcess, spawn } from 'node:child_process'; +import { EventEmitter } from 'node:events'; +import { type AudioConfig } from '../config/index.js'; +import { createLogger, type Logger } from '../utils/index.js'; + +export interface AudioServiceEvents { + /** Emitted when a raw PCM audio chunk is captured from the microphone */ + audio_chunk: (chunk: Buffer) => void; + /** Emitted when playback of a response finishes */ + playback_done: () => void; + /** Emitted on audio errors */ + error: (error: Error) => void; +} + +/** + * Audio service for Raspberry Pi. + * + * Uses ALSA tools (arecord/aplay) via child processes. + * Works with any ALSA-compatible audio device: + * - I2S (INMP441 mic, MAX98357 amp) connected directly to Pi GPIO + * - USB audio devices + * - Default system audio + * + * Audio format: PCM signed 16-bit little-endian, mono, 16kHz + */ +export class AudioService extends EventEmitter { + private captureProcess: ChildProcess | null = null; + private readonly logger: Logger; + private _isCapturing = false; + private _isPlaying = false; + private _stoppedManually = false; + + constructor(private readonly config: AudioConfig) { + super(); + this.logger = createLogger('audio', 'info'); + } + + get isCapturing(): boolean { + return this._isCapturing; + } + + get isPlaying(): boolean { + return this._isPlaying; + } + + /** + * Start capturing audio from the microphone. + * Emits 'audio_chunk' events with raw PCM buffers. + */ + startCapture(): void { + if (this._isCapturing) { + this.logger.warn('Already capturing audio'); + return; + } + + this.logger.info( + { device: this.config.captureDevice, sampleRate: this.config.sampleRate }, + 'Starting audio capture', + ); + + // arecord outputs raw PCM to stdout + // -D: ALSA device + // -f: format (S16_LE = signed 16-bit little-endian) + // -r: sample rate + // -c: channels + // -t: type (raw = no header) + // --buffer-size: in frames, controls latency + const bufferFrames = Math.floor(this.config.sampleRate * (this.config.chunkDurationMs / 1000)); + + this.captureProcess = spawn('arecord', [ + '-D', this.config.captureDevice, + '-f', 'S16_LE', + '-r', String(this.config.sampleRate), + '-c', String(this.config.channels), + '-t', 'raw', + '--buffer-size', String(bufferFrames * 4), + '--period-size', String(bufferFrames), + ], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + + this._isCapturing = true; + + let chunkCount = 0; + this.captureProcess.stdout?.on('data', (chunk: Buffer) => { + chunkCount++; + if (chunkCount === 1) { + this.logger.info({ bytes: chunk.length }, 'First audio chunk received from arecord'); + } + this.emit('audio_chunk', chunk); + }); + + this.captureProcess.stderr?.on('data', (data: Buffer) => { + const msg = data.toString().trim(); + if (msg) { + this.logger.debug({ msg }, 'arecord stderr'); + } + }); + + this.captureProcess.on('error', (err) => { + this.logger.error({ err }, 'arecord process error'); + this._isCapturing = false; + this.emit('error', new Error(`Audio capture failed: ${err.message}`)); + }); + + this.captureProcess.on('exit', (code) => { + this._isCapturing = false; + if (code !== null && code !== 0 && !this._stoppedManually) { + this.logger.warn({ code }, 'arecord exited with non-zero code'); + } + this._stoppedManually = false; + }); + } + + /** + * Stop capturing audio from the microphone. + */ + stopCapture(): void { + if (!this.captureProcess) return; + + this.logger.info('Stopping audio capture'); + this._stoppedManually = true; + this.captureProcess.kill('SIGTERM'); + this.captureProcess = null; + this._isCapturing = false; + } + + /** + * Play audio through the speaker. + * Accepts either raw PCM or WAV (with RIFF header) data. + * + * @returns Promise that resolves when playback is complete + */ + async play(audioBuffer: Buffer): Promise { + if (this._isPlaying) { + this.logger.warn('Already playing audio, queueing...'); + } + + this._isPlaying = true; + + const isWav = audioBuffer.length > 4 && audioBuffer.toString('ascii', 0, 4) === 'RIFF'; + + return new Promise((resolve, reject) => { + const args = isWav + ? ['-D', this.config.playbackDevice, '-t', 'wav', '-'] + : [ + '-D', this.config.playbackDevice, + '-f', 'S16_LE', + '-r', String(this.config.sampleRate), + '-c', String(this.config.channels), + '-t', 'raw', + '-', + ]; + + const playProcess = spawn('aplay', args, { + stdio: ['pipe', 'ignore', 'pipe'], + }); + + playProcess.stderr?.on('data', (data: Buffer) => { + const msg = data.toString().trim(); + if (msg && !msg.startsWith('Playing') && !msg.startsWith('Warning')) { + this.logger.error({ msg }, 'aplay stderr'); + } + }); + + playProcess.on('error', (err) => { + this._isPlaying = false; + reject(new Error(`Audio playback failed: ${err.message}`)); + }); + + playProcess.on('exit', (code) => { + this._isPlaying = false; + if (code === 0 || code === null) { + this.emit('playback_done'); + resolve(); + } else { + reject(new Error(`aplay exited with code ${code}`)); + } + }); + + // Write audio data to aplay's stdin and close it + playProcess.stdin?.write(audioBuffer); + playProcess.stdin?.end(); + }); + } + + /** + * Stop any currently playing audio. + */ + stopPlayback(): void { + // aplay is spawned per-play, so we can't easily stop it here + // For interrupt support, we'd track the play process + this._isPlaying = false; + } + + /** + * Clean up resources. + */ + async destroy(): Promise { + this.stopCapture(); + this.removeAllListeners(); + } +} diff --git a/apps/robot-client/src/services/health.service.ts b/apps/robot-client/src/services/health.service.ts new file mode 100644 index 0000000..1203f12 --- /dev/null +++ b/apps/robot-client/src/services/health.service.ts @@ -0,0 +1,57 @@ +import { EventEmitter } from 'node:events'; +import { createLogger, type Logger } from '../utils/index.js'; +import { type CloudSocket } from '../transport/cloud-socket.js'; + +/** + * Health monitoring service. + * Tracks connectivity, system health, and provides diagnostics. + */ +export class HealthService extends EventEmitter { + private readonly logger: Logger; + private heartbeatInterval: ReturnType | null = null; + private _isCloudConnected = false; + + constructor(private readonly cloudSocket: CloudSocket) { + super(); + this.logger = createLogger('health', 'info'); + + this.cloudSocket.on('connected', () => { + this._isCloudConnected = true; + this.logger.info('Cloud connection established'); + }); + + this.cloudSocket.on('disconnected', () => { + this._isCloudConnected = false; + this.logger.warn('Cloud connection lost'); + }); + } + + get isCloudConnected(): boolean { + return this._isCloudConnected; + } + + /** + * Start periodic health checks. + */ + start(intervalMs = 30_000): void { + this.logger.info({ intervalMs }, 'Starting health monitoring'); + + this.heartbeatInterval = setInterval(() => { + this.logger.debug({ + cloud: this._isCloudConnected, + uptime: process.uptime(), + memoryMB: Math.round(process.memoryUsage().heapUsed / 1024 / 1024), + }, 'Health check'); + }, intervalMs); + } + + /** + * Stop health monitoring. + */ + stop(): void { + if (this.heartbeatInterval) { + clearInterval(this.heartbeatInterval); + this.heartbeatInterval = null; + } + } +} diff --git a/apps/robot-client/src/services/index.ts b/apps/robot-client/src/services/index.ts new file mode 100644 index 0000000..4390a33 --- /dev/null +++ b/apps/robot-client/src/services/index.ts @@ -0,0 +1,8 @@ +export { AudioService } from './audio.service.js'; +export { WakeWordService } from './wake-word.service.js'; +export { KeyboardTriggerService } from './keyboard-trigger.service.js'; +export { HealthService } from './health.service.js'; +export { OrchestratorService } from './orchestrator.service.js'; +export { LocalStore } from './local-store.service.js'; +export { WifiService } from './wifi.service.js'; +export { type ITriggerService } from './trigger.interface.js'; diff --git a/apps/robot-client/src/services/keyboard-trigger.service.ts b/apps/robot-client/src/services/keyboard-trigger.service.ts new file mode 100644 index 0000000..fe964d0 --- /dev/null +++ b/apps/robot-client/src/services/keyboard-trigger.service.ts @@ -0,0 +1,91 @@ +import { EventEmitter } from 'node:events'; +import * as readline from 'node:readline'; +import { type ITriggerService } from './trigger.interface.js'; +import { createLogger, type Logger } from '../utils/index.js'; + +/** + * Keyboard trigger for dev/test mode. + * + * Replaces the wake word: press Enter in the terminal to start a conversation. + * Perfect for testing on a Pi over SSH without needing OpenWakeWord installed. + */ +export class KeyboardTriggerService extends EventEmitter implements ITriggerService { + private rl: readline.Interface | null = null; + private readonly logger: Logger; + private _isListening = false; + private _isPaused = false; + + constructor() { + super(); + this.logger = createLogger('keyboard-trigger', 'info'); + } + + get isListening(): boolean { + return this._isListening; + } + + /** + * Start listening for Enter key presses. + */ + start(): void { + if (this._isListening) return; + + this.rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + this._isListening = true; + this._isPaused = false; + + this.prompt(); + + this.rl.on('line', () => { + if (this._isPaused) { + this.logger.debug('Key press ignored — currently paused'); + return; + } + + this.logger.info('⏎ Enter pressed — triggering conversation'); + this.emit('detected'); + }); + + this.rl.on('close', () => { + this._isListening = false; + }); + } + + private prompt(): void { + if (this._isPaused || !this._isListening) return; + this.logger.info('──────────────────────────────────────'); + this.logger.info('Appuie sur Entrée pour parler à Ti-Pote...'); + } + + /** + * Pause: ignore key presses (e.g., while the robot is speaking). + */ + pause(): void { + this._isPaused = true; + } + + /** + * Resume after pause. + */ + resume(): void { + this._isPaused = false; + this.prompt(); + } + + /** + * Stop and clean up. + */ + stop(): void { + if (this.rl) { + this.rl.close(); + this.rl = null; + } + this._isListening = false; + this._isPaused = false; + this.removeAllListeners(); + } +} diff --git a/apps/robot-client/src/services/local-store.service.ts b/apps/robot-client/src/services/local-store.service.ts new file mode 100644 index 0000000..6357704 --- /dev/null +++ b/apps/robot-client/src/services/local-store.service.ts @@ -0,0 +1,169 @@ +import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { createLogger, type Logger } from '../utils/index.js'; + +/** + * What we persist on the SD card. + */ +export interface LocalStoreData { + /** WiFi credentials */ + wifi?: { + ssid: string; + password: string; + }; + + /** Device registration (received from backend on first setup) */ + device?: { + id: string; + token: string; + homeId?: string; + }; + + /** Robot preferences */ + preferences?: { + robotName?: string; + volume?: number; + wakeWordModel?: string; + language?: string; + }; + + /** Tracks first setup state */ + setupComplete?: boolean; +} + +const DEFAULT_STORE_PATH = '/home/pi/.tipote/config.json'; + +/** + * Simple JSON file store for persisting robot config on the SD card. + * + * Stores WiFi credentials, device token, and preferences. + * Survives reboots and code redeployments — unlike .env files, + * this is the robot's own "memory" of who it is and where it connects. + */ +export class LocalStore { + private data: LocalStoreData; + private readonly filePath: string; + private readonly logger: Logger; + + constructor(filePath?: string) { + this.logger = createLogger('local-store', 'info'); + this.filePath = filePath || process.env.STORE_PATH || DEFAULT_STORE_PATH; + this.data = this.load(); + } + + /** + * Load stored data from disk. Returns empty object if file doesn't exist. + */ + private load(): LocalStoreData { + try { + if (existsSync(this.filePath)) { + const raw = readFileSync(this.filePath, 'utf-8'); + const parsed = JSON.parse(raw) as LocalStoreData; + this.logger.info({ path: this.filePath }, 'Config loaded from disk'); + return parsed; + } + } catch (err) { + this.logger.error({ err }, 'Failed to load config file, starting fresh'); + } + return {}; + } + + /** + * Save current data to disk. + */ + private save(): void { + try { + const dir = dirname(this.filePath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + writeFileSync(this.filePath, JSON.stringify(this.data, null, 2), 'utf-8'); + this.logger.debug({ path: this.filePath }, 'Config saved to disk'); + } catch (err) { + this.logger.error({ err }, 'Failed to save config file'); + } + } + + // ── WiFi ── + + get wifi(): LocalStoreData['wifi'] { + return this.data.wifi; + } + + get hasWifi(): boolean { + return !!this.data.wifi?.ssid; + } + + setWifi(ssid: string, password: string): void { + this.data.wifi = { ssid, password }; + this.save(); + this.logger.info({ ssid }, 'WiFi credentials saved'); + } + + clearWifi(): void { + delete this.data.wifi; + this.save(); + this.logger.info('WiFi credentials cleared'); + } + + // ── Device ── + + get device(): LocalStoreData['device'] { + return this.data.device; + } + + get hasDevice(): boolean { + return !!this.data.device?.id && !!this.data.device?.token; + } + + setDevice(id: string, token: string, homeId?: string): void { + this.data.device = { id, token, homeId }; + this.save(); + this.logger.info({ deviceId: id }, 'Device credentials saved'); + } + + // ── Preferences ── + + get preferences(): LocalStoreData['preferences'] { + return this.data.preferences ?? {}; + } + + setPreference>( + key: K, + value: NonNullable[K], + ): void { + if (!this.data.preferences) this.data.preferences = {}; + this.data.preferences[key] = value; + this.save(); + } + + // ── Setup state ── + + get isSetupComplete(): boolean { + return this.data.setupComplete === true; + } + + markSetupComplete(): void { + this.data.setupComplete = true; + this.save(); + this.logger.info('Setup marked as complete'); + } + + // ── Reset ── + + /** + * Factory reset: clear all stored data. + */ + factoryReset(): void { + this.data = {}; + this.save(); + this.logger.info('Factory reset — all config cleared'); + } + + /** + * Get all stored data (for debug). + */ + getAll(): Readonly { + return { ...this.data }; + } +} diff --git a/apps/robot-client/src/services/orchestrator.service.ts b/apps/robot-client/src/services/orchestrator.service.ts new file mode 100644 index 0000000..c85c8d9 --- /dev/null +++ b/apps/robot-client/src/services/orchestrator.service.ts @@ -0,0 +1,336 @@ +import { EventEmitter } from 'node:events'; +import { type AudioConfig } from '../config/index.js'; +import { type CloudSocket, type RobotState, type ResponseTextEvent } from '../transport/cloud-socket.js'; +import { type AudioService } from './audio.service.js'; +import { type ITriggerService } from './trigger.interface.js'; +import { createLogger, type Logger } from '../utils/index.js'; + +/** + * Conversation orchestrator — the main brain of the robot client. + * + * Coordinates the full conversation flow: + * 1. Trigger detected (wake word or keyboard) → notify cloud, start audio capture + * 2. Audio chunks → forward to cloud for STT + * 3. User stops speaking → notify cloud + * 4. Cloud sends response (text + audio) → play audio through speaker + * 5. Playback done → resume trigger listening + * + * State machine: + * IDLE → (trigger) → LISTENING → (speech end) → THINKING → (response) → SPEAKING → IDLE + */ +export class OrchestratorService extends EventEmitter { + private readonly logger: Logger; + private state: RobotState = 'idle'; + + /** Timer for Voice Activity Detection (silence timeout) */ + private silenceTimer: ReturnType | null = null; + private readonly silenceTimeoutMs = 2000; // 2s of silence = speech end + private readonly initialGracePeriodMs = 3000; // 3s grace period before silence detection kicks in + + /** Track when the last audio chunk was received */ + private lastAudioChunkTime = 0; + private conversationStartTime = 0; + private hasDetectedSpeech = false; + + /** Buffer audio chunks until the cloud confirms it's ready (STT stream open) */ + private audioBuffer: Buffer[] = []; + private cloudReady = false; + + constructor( + private readonly cloudSocket: CloudSocket, + private readonly audioService: AudioService, + private readonly trigger: ITriggerService, + private readonly audioConfig: AudioConfig, + ) { + super(); + this.logger = createLogger('orchestrator', 'info'); + } + + /** + * Initialize the orchestrator: wire up all event handlers. + */ + start(): void { + this.logger.info('Starting conversation orchestrator'); + + // ── Trigger (wake word or keyboard) → start conversation ── + + this.trigger.on('detected', () => { + this.handleTriggerDetected(); + }); + + // ── Audio chunks from microphone ── + + this.audioService.on('audio_chunk', (chunk: Buffer) => { + this.handleAudioChunk(chunk); + }); + + // ── Cloud responses ── + + this.cloudSocket.on('status', (event) => { + this.handleStatusChange(event.state); + }); + + this.cloudSocket.on('response_text', (event) => { + this.handleResponse(event); + }); + + // ── Start trigger ── + + this.trigger.start(); + this.state = 'idle'; + + this.logger.info('Orchestrator ready — waiting for trigger'); + } + + /** + * Handle trigger detection (wake word or Enter key). + */ + private async handleTriggerDetected(): Promise { + if (this.state !== 'idle') { + this.logger.debug({ state: this.state }, 'Trigger detected but not idle, ignoring'); + return; + } + + this.logger.info('👂 Trigger detected — listening...'); + this.state = 'listening'; + + // Pause trigger and wait for it to fully release the audio device + await this.trigger.pause(); + + // Reset cloud readiness and audio buffer + this.cloudReady = false; + this.audioBuffer = []; + + // Notify cloud + this.cloudSocket.sendWakeWordDetected(); + + // Start capturing audio from the microphone + this.audioService.startCapture(); + + // Start silence detection + this.lastAudioChunkTime = Date.now(); + this.conversationStartTime = Date.now(); + this.hasDetectedSpeech = false; + this.startSilenceDetection(); + } + + /** + * Handle incoming audio chunks from the microphone. + */ + private handleAudioChunk(chunk: Buffer): void { + if (this.state !== 'listening') return; + + if (!this.cloudReady) { + // Cloud STT stream not ready yet — buffer chunks so we don't lose the first words + this.audioBuffer.push(chunk); + } else { + // Forward raw PCM to cloud for STT + this.cloudSocket.sendAudioChunk(chunk, this.audioConfig.sampleRate); + } + + // Update silence tracking + // Simple VAD: if we're receiving non-silent audio, reset the silence timer + if (this.isAudioSignificant(chunk)) { + if (!this.hasDetectedSpeech) { + this.logger.info('🗣️ Speech detected — listening...'); + this.hasDetectedSpeech = true; + } + this.lastAudioChunkTime = Date.now(); + } + } + + /** + * Basic Voice Activity Detection: check if audio chunk contains significant signal. + * Uses RMS (root mean square) amplitude threshold. + */ + private isAudioSignificant(chunk: Buffer, threshold = 200): boolean { + let sumSquares = 0; + const samples = chunk.length / 2; // 16-bit = 2 bytes per sample + + for (let i = 0; i < chunk.length - 1; i += 2) { + const sample = chunk.readInt16LE(i); + sumSquares += sample * sample; + } + + const rms = Math.sqrt(sumSquares / samples); + return rms > threshold; + } + + /** + * Start a timer that triggers speech_end after silence. + */ + private startSilenceDetection(): void { + this.clearSilenceTimer(); + + this.silenceTimer = setInterval(() => { + if (this.state !== 'listening') { + this.clearSilenceTimer(); + return; + } + + const timeSinceStart = Date.now() - this.conversationStartTime; + const silenceDuration = Date.now() - this.lastAudioChunkTime; + + // Don't end speech during the initial grace period (give user time to start talking) + if (!this.hasDetectedSpeech && timeSinceStart < this.initialGracePeriodMs) { + return; + } + + // If user never spoke and grace period is over, return to idle (conversation over) + if (!this.hasDetectedSpeech && timeSinceStart >= this.initialGracePeriodMs) { + this.logger.info('💤 No speech detected — back to sleep'); + this.clearSilenceTimer(); + this.audioService.stopCapture(); + this.returnToIdle(); + return; + } + + // Normal silence detection after speech was detected + if (silenceDuration >= this.silenceTimeoutMs) { + this.logger.info({ silenceDuration }, 'Silence detected — ending speech'); + this.handleSpeechEnd(); + } + }, 200); // Check every 200ms + } + + /** + * Handle end of user speech. + */ + private handleSpeechEnd(): void { + if (this.state !== 'listening') return; + + this.state = 'thinking'; + this.clearSilenceTimer(); + + // Stop audio capture + this.audioService.stopCapture(); + + // Notify cloud that speech has ended + this.cloudSocket.sendSpeechEnd(); + + this.logger.info('🤔 Thinking...'); + } + + /** + * Handle status changes from the cloud. + */ + private handleStatusChange(newState: RobotState): void { + this.logger.debug({ from: this.state, to: newState }, 'State change from cloud'); + + // Cloud confirmed STT stream is ready — flush buffered audio + if (newState === 'listening' && this.state === 'listening' && !this.cloudReady) { + this.cloudReady = true; + if (this.audioBuffer.length > 0) { + this.logger.info( + { bufferedChunks: this.audioBuffer.length }, + 'Cloud ready — flushing buffered audio', + ); + for (const chunk of this.audioBuffer) { + this.cloudSocket.sendAudioChunk(chunk, this.audioConfig.sampleRate); + } + this.audioBuffer = []; + } + } + + // The cloud drives the state machine for thinking → speaking → idle + // We only override our local state if it makes sense + if (newState === 'idle' && this.state !== 'idle') { + this.returnToIdle(); + } + } + + /** + * Handle a response from the cloud (text + audio). + */ + private async handleResponse(event: ResponseTextEvent): Promise { + this.logger.info('🔊 Speaking: %s', event.text); + this.state = 'speaking'; + + if (event.audio) { + try { + // Decode base64 WAV audio and play it + const audioBuffer = Buffer.from(event.audio, 'base64'); + await this.audioService.play(audioBuffer); + } catch (err) { + this.logger.error({ err }, 'Failed to play audio response'); + } + } + + // After playback, continue listening for more speech (continuous conversation) + this.continueListening(); + } + + /** + * Continue the conversation — go back to listening immediately after playback. + * No need to re-trigger (wake word or Enter), just start capturing audio again. + */ + private continueListening(): void { + this.logger.info('👂 Listening...'); + this.state = 'listening'; + + // Reset cloud readiness and audio buffer + this.cloudReady = false; + this.audioBuffer = []; + + // Notify cloud we're listening again (so it opens a new STT stream) + this.cloudSocket.sendWakeWordDetected(); + + // Start capturing audio again + this.audioService.startCapture(); + + // Reset silence detection + this.lastAudioChunkTime = Date.now(); + this.conversationStartTime = Date.now(); + this.hasDetectedSpeech = false; + this.startSilenceDetection(); + } + + /** + * Return to idle state: resume trigger listening. + */ + private returnToIdle(): void { + this.state = 'idle'; + this.clearSilenceTimer(); + + // Resume trigger + if (!this.trigger.isListening) { + this.trigger.start(); + } else { + this.trigger.resume(); + } + + this.logger.info('💤 Idle — waiting for wake word...'); + } + + /** + * Handle user interrupt (e.g., button press, or second wake word). + */ + interrupt(): void { + this.logger.info('User interrupt'); + + this.audioService.stopCapture(); + this.audioService.stopPlayback(); + this.clearSilenceTimer(); + + this.cloudSocket.sendUserInterrupt(); + this.returnToIdle(); + } + + private clearSilenceTimer(): void { + if (this.silenceTimer) { + clearInterval(this.silenceTimer); + this.silenceTimer = null; + } + } + + /** + * Stop the orchestrator and clean up resources. + */ + async stop(): Promise { + this.logger.info('Stopping orchestrator'); + this.clearSilenceTimer(); + this.audioService.stopCapture(); + this.trigger.stop(); + this.removeAllListeners(); + } +} diff --git a/apps/robot-client/src/services/trigger.interface.ts b/apps/robot-client/src/services/trigger.interface.ts new file mode 100644 index 0000000..fbed82b --- /dev/null +++ b/apps/robot-client/src/services/trigger.interface.ts @@ -0,0 +1,42 @@ +/** + * Common interface for conversation trigger services. + * + * Both WakeWordService and KeyboardTriggerService implement this interface, + * allowing the orchestrator to work with any trigger mechanism. + */ +export interface ITriggerService { + /** Whether the trigger is currently active and listening */ + readonly isListening: boolean; + + /** Start listening for the trigger */ + start(): void; + + /** Temporarily pause (e.g., while robot is speaking) */ + pause(): void | Promise; + + /** Resume after pause */ + resume(): void; + + /** Stop permanently and clean up resources */ + stop(): void; + + /** EventEmitter methods */ + on(event: 'detected', listener: () => void): this; + on(event: 'error', listener: (error: Error) => void): this; + emit(event: 'detected'): boolean; + emit(event: 'error', error: Error): boolean; +} + +/** + * Type guard to ensure a service implements ITriggerService. + */ +export function isTriggerService(obj: unknown): obj is ITriggerService { + return ( + obj !== null && + typeof obj === 'object' && + 'isListening' in obj && + 'start' in obj && + 'pause' in obj && + 'stop' in obj + ); +} diff --git a/apps/robot-client/src/services/wake-word.service.ts b/apps/robot-client/src/services/wake-word.service.ts new file mode 100644 index 0000000..9e6076a --- /dev/null +++ b/apps/robot-client/src/services/wake-word.service.ts @@ -0,0 +1,200 @@ +import { ChildProcess, spawn } from 'node:child_process'; +import { EventEmitter } from 'node:events'; +import { type WakeWordConfig, type AudioConfig } from '../config/index.js'; +import { createLogger, type Logger } from '../utils/index.js'; + +export interface WakeWordServiceEvents { + /** Emitted when the wake word is detected */ + detected: () => void; + /** Emitted when the engine is ready */ + ready: () => void; + /** Emitted on errors */ + error: (error: Error) => void; +} + +/** + * Wake word detection service. + * + * Runs OpenWakeWord as a **long-lived** Python subprocess. + * The model is loaded once at startup; pause/resume is handled via + * PAUSE/RESUME commands on stdin, so the audio device is released + * while arecord is capturing, then reclaimed when listening resumes. + */ +export class WakeWordService extends EventEmitter { + private process: ChildProcess | null = null; + private readonly logger: Logger; + private _isListening = false; + private _isPaused = false; + private _streamClosed = false; + + constructor( + private readonly wakeWordConfig: WakeWordConfig, + private readonly audioConfig: AudioConfig, + ) { + super(); + this.logger = createLogger('wake-word', 'info'); + } + + get isListening(): boolean { + return this._isListening && !this._isPaused; + } + + /** + * Start the wake word Python subprocess. + * The model is loaded once; subsequent pause/resume cycles are fast. + */ + start(): void { + if (this.process) { + // Process already running — just resume if paused + if (this._isPaused) { + this.resume(); + } + return; + } + + this.logger.info( + { model: this.wakeWordConfig.modelName, threshold: this.wakeWordConfig.threshold }, + 'Starting wake word detection', + ); + + this.process = spawn(this.wakeWordConfig.pythonPath, [ + this.wakeWordConfig.scriptPath, + '--model', this.wakeWordConfig.modelName, + '--threshold', String(this.wakeWordConfig.threshold), + '--device', this.audioConfig.captureDevice, + '--sample-rate', String(this.audioConfig.sampleRate), + ], { + stdio: ['pipe', 'pipe', 'pipe'], + }); + + this._isListening = true; + this._isPaused = false; + + // ── stdout: DETECTED events ── + this.process.stdout?.on('data', (data: Buffer) => { + const lines = data.toString().trim().split('\n'); + for (const line of lines) { + if (line.trim() === 'DETECTED') { + this.logger.info('🎙️ Wake word detected!'); + this.emit('detected'); + } else { + this.logger.debug({ line }, 'Wake word stdout'); + } + } + }); + + // ── stderr: status messages ── + this.process.stderr?.on('data', (data: Buffer) => { + const lines = data.toString().trim().split('\n'); + for (const line of lines) { + const msg = line.trim(); + if (!msg) continue; + + if (msg === 'READY') { + this.logger.info('🎤 Wake word engine ready — listening...'); + this.emit('ready'); + } else if (msg === 'PAUSED') { + this._streamClosed = false; + this.logger.debug('Wake word paused'); + } else if (msg === 'STREAM_CLOSED') { + this._streamClosed = true; + this.logger.debug('Wake word audio stream closed'); + } else if (msg === 'RESUMED') { + this.logger.debug('Wake word resumed'); + } else if (msg === 'STREAM_REOPENED') { + this.logger.debug('Wake word audio stream reopened'); + } else if (msg.startsWith('Loading wake word model')) { + this.logger.info('⏳ Loading wake word model...'); + } else if (msg.startsWith('Wake word model loaded')) { + this.logger.info('✅ Wake word model loaded'); + } else if (msg.startsWith('Matched device') || msg.startsWith('Using device')) { + this.logger.info(`🔊 ${msg}`); + } else { + this.logger.debug({ msg }, 'Wake word stderr'); + } + } + }); + + this.process.on('error', (err) => { + this._isListening = false; + this.logger.error({ err }, 'Wake word process error'); + this.emit('error', new Error(`Wake word process failed: ${err.message}`)); + }); + + this.process.on('exit', (code) => { + this._isListening = false; + this._isPaused = false; + this.process = null; + if (code !== 0 && code !== null) { + this.logger.warn({ code }, 'Wake word process exited unexpectedly'); + // Auto-restart after a short delay + setTimeout(() => { + this.logger.info('Restarting wake word detection...'); + this.start(); + }, 2000); + } + }); + } + + /** + * Pause wake word detection. + * Sends PAUSE command to Python subprocess which closes the audio stream, + * freeing the device for arecord. Returns a promise that resolves when + * the audio stream is confirmed closed. + */ + pause(): Promise { + if (!this.process || this._isPaused) return Promise.resolve(); + + this._isPaused = true; + this._streamClosed = false; + + this.process.stdin?.write('PAUSE\n'); + + // Wait for the stream to be closed (so arecord can use the device) + return new Promise((resolve) => { + const checkInterval = setInterval(() => { + if (this._streamClosed || !this.process) { + clearInterval(checkInterval); + resolve(); + } + }, 50); + + // Safety timeout + setTimeout(() => { + clearInterval(checkInterval); + resolve(); + }, 2000); + }); + } + + /** + * Resume wake word detection after pause. + * The Python subprocess reopens the audio stream (fast, no model reload). + */ + resume(): void { + if (!this.process || !this._isPaused) return; + + this._isPaused = false; + this.process.stdin?.write('RESUME\n'); + this.logger.info('🎤 Resuming wake word listening...'); + } + + /** + * Stop wake word detection permanently. + */ + stop(): void { + if (this.process) { + this.process.stdin?.write('QUIT\n'); + // Give it a moment to exit cleanly, then force kill + setTimeout(() => { + if (this.process) { + this.process.kill('SIGTERM'); + this.process = null; + } + }, 500); + } + this._isListening = false; + this._isPaused = false; + this.removeAllListeners(); + } +} diff --git a/apps/robot-client/src/services/wifi.service.ts b/apps/robot-client/src/services/wifi.service.ts new file mode 100644 index 0000000..9a9da77 --- /dev/null +++ b/apps/robot-client/src/services/wifi.service.ts @@ -0,0 +1,227 @@ +import { exec } from 'node:child_process'; +import { promisify } from 'node:util'; +import { createLogger, type Logger } from '../utils/index.js'; + +const execAsync = promisify(exec); + +export interface WifiNetwork { + ssid: string; + signal: number; // dBm + security: string; // e.g., 'WPA2', 'WPA3', 'Open' + frequency: number; // MHz +} + +/** + * WiFi management service for Raspberry Pi. + * + * Uses `nmcli` (NetworkManager) which is available on Raspberry Pi OS. + * Handles scanning, connecting, AP mode, and connectivity checking. + */ +export class WifiService { + private readonly logger: Logger; + + constructor() { + this.logger = createLogger('wifi', 'info'); + } + + /** + * Check if the Pi is currently connected to a WiFi network. + */ + async isConnected(): Promise { + try { + const { stdout } = await execAsync('nmcli -t -f TYPE,STATE device | grep wifi'); + return stdout.includes('connected') && !stdout.includes('disconnected'); + } catch { + return false; + } + } + + /** + * Get the current WiFi SSID, if connected. + */ + async currentSSID(): Promise { + try { + const { stdout } = await execAsync('iwgetid -r'); + const ssid = stdout.trim(); + return ssid || null; + } catch { + return null; + } + } + + /** + * Check internet connectivity by pinging a reliable host. + */ + async hasInternet(): Promise { + try { + await execAsync('ping -c 1 -W 3 8.8.8.8'); + return true; + } catch { + return false; + } + } + + /** + * Scan available WiFi networks. + * Returns a list sorted by signal strength (strongest first). + */ + async scan(): Promise { + try { + // Force a fresh scan + await execAsync('nmcli device wifi rescan').catch(() => { /* ignore */ }); + // Wait a moment for scan results + await new Promise((resolve) => setTimeout(resolve, 2000)); + + const { stdout } = await execAsync( + 'nmcli -t -f SSID,SIGNAL,SECURITY,FREQ device wifi list', + ); + + const networks: WifiNetwork[] = []; + const seen = new Set(); + + for (const line of stdout.trim().split('\n')) { + if (!line) continue; + const parts = line.split(':'); + if (parts.length < 4) continue; + + const ssid = parts[0]; + if (!ssid || seen.has(ssid)) continue; + seen.add(ssid); + + networks.push({ + ssid, + signal: parseInt(parts[1], 10), + security: parts[2] || 'Open', + frequency: parseInt(parts[3], 10), + }); + } + + // Sort by signal strength (higher = better) + networks.sort((a, b) => b.signal - a.signal); + + this.logger.info({ count: networks.length }, 'WiFi scan complete'); + return networks; + } catch (err) { + this.logger.error({ err }, 'WiFi scan failed'); + return []; + } + } + + /** + * Connect to a WiFi network. + * + * @returns true if connection succeeded + */ + async connect(ssid: string, password: string): Promise { + this.logger.info({ ssid }, 'Connecting to WiFi network...'); + + try { + // First, try to connect (this creates a connection profile automatically) + await execAsync( + `nmcli device wifi connect "${this.escapeShell(ssid)}" password "${this.escapeShell(password)}"`, + ); + + // Verify connection + const connected = await this.isConnected(); + if (connected) { + this.logger.info({ ssid }, 'WiFi connected successfully'); + return true; + } + + this.logger.warn({ ssid }, 'WiFi connection attempt returned but not connected'); + return false; + } catch (err) { + this.logger.error({ err, ssid }, 'WiFi connection failed'); + return false; + } + } + + /** + * Disconnect from current WiFi network. + */ + async disconnect(): Promise { + try { + await execAsync('nmcli device disconnect wlan0'); + this.logger.info('WiFi disconnected'); + } catch (err) { + this.logger.error({ err }, 'WiFi disconnect failed'); + } + } + + /** + * Start Access Point mode for the captive portal. + * Creates a hotspot named "Ti-Pote-XXXX". + * + * @param apName - Name of the hotspot (default: Ti-Pote-XXXX based on hostname) + */ + async startAP(apName?: string): Promise { + const name = apName || `Ti-Pote-${await this.getShortId()}`; + + this.logger.info({ apName: name }, 'Starting WiFi Access Point...'); + + try { + // Stop any existing hotspot + await execAsync('nmcli connection down Hotspot').catch(() => { /* ignore */ }); + await execAsync('nmcli connection delete Hotspot').catch(() => { /* ignore */ }); + + // Create and start an open hotspot (no password — easier for setup) + await execAsync( + `nmcli connection add type wifi ifname wlan0 con-name Hotspot autoconnect no ssid "${this.escapeShell(name)}" && ` + + `nmcli connection modify Hotspot 802-11-wireless.mode ap ipv4.method shared ipv4.addresses 192.168.4.1/24 && ` + + `nmcli connection up Hotspot`, + ); + + this.logger.info({ apName: name }, 'Access Point started'); + return true; + } catch (err) { + this.logger.error({ err }, 'Failed to start Access Point'); + return false; + } + } + + /** + * Stop Access Point mode. + */ + async stopAP(): Promise { + try { + await execAsync('nmcli connection down Hotspot'); + await execAsync('nmcli connection delete Hotspot').catch(() => { /* ignore */ }); + this.logger.info('Access Point stopped'); + } catch (err) { + this.logger.error({ err }, 'Failed to stop Access Point'); + } + } + + /** + * Get the Pi's IP address on the current network. + */ + async getIPAddress(): Promise { + try { + const { stdout } = await execAsync( + "hostname -I | awk '{print $1}'", + ); + return stdout.trim() || null; + } catch { + return null; + } + } + + /** + * Generate a short unique ID from the Pi's hostname or MAC address. + */ + private async getShortId(): Promise { + try { + const { stdout } = await execAsync("cat /sys/class/net/wlan0/address | tr -d ':'"); + return stdout.trim().slice(-4).toUpperCase(); + } catch { + return Math.random().toString(36).substring(2, 6).toUpperCase(); + } + } + + /** + * Escape a string for safe use in shell commands. + */ + private escapeShell(str: string): string { + return str.replace(/(['"\\$`!])/g, '\\$1'); + } +} diff --git a/apps/robot-client/src/setup/captive-portal.ts b/apps/robot-client/src/setup/captive-portal.ts new file mode 100644 index 0000000..9d32a4c --- /dev/null +++ b/apps/robot-client/src/setup/captive-portal.ts @@ -0,0 +1,224 @@ +import { createServer, type IncomingMessage, type ServerResponse, type Server } from 'node:http'; +import { type WifiService } from '../services/wifi.service.js'; +import { type LocalStore } from '../services/local-store.service.js'; +import { createLogger, type Logger } from '../utils/index.js'; +import { portalHTML } from './portal.html.js'; + +const PORTAL_PORT = 80; // Standard HTTP port — captive portals need this + +/** + * Captive Portal server. + * + * When the robot starts in AP mode, this HTTP server serves: + * - GET / → the setup HTML page + * - GET /api/wifi/scan → JSON list of available WiFi networks + * - POST /api/wifi/connect → attempt to connect to a WiFi network + * + * Also handles captive portal detection URLs from iOS/Android/Windows + * by redirecting them to the portal page. + */ +export class CaptivePortal { + private server: Server | null = null; + private readonly logger: Logger; + private resolveSetup: ((value: boolean) => void) | null = null; + + constructor( + private readonly wifiService: WifiService, + private readonly store: LocalStore, + private readonly robotName: string, + ) { + this.logger = createLogger('captive-portal', 'info'); + } + + /** + * Start the captive portal server. + * Returns a promise that resolves when WiFi is successfully configured. + */ + async start(): Promise { + return new Promise((resolve) => { + this.resolveSetup = resolve; + + this.server = createServer((req, res) => { + this.handleRequest(req, res); + }); + + this.server.listen(PORTAL_PORT, '0.0.0.0', () => { + this.logger.info({ port: PORTAL_PORT }, 'Captive portal running'); + }); + + this.server.on('error', (err: NodeJS.ErrnoException) => { + if (err.code === 'EACCES') { + this.logger.error( + 'Port 80 requires root. Run with sudo or use: sudo setcap cap_net_bind_service=+ep $(which node)', + ); + } else { + this.logger.error({ err }, 'Captive portal server error'); + } + resolve(false); + }); + }); + } + + /** + * Stop the captive portal server. + */ + async stop(): Promise { + return new Promise((resolve) => { + if (this.server) { + this.server.close(() => { + this.logger.info('Captive portal stopped'); + resolve(); + }); + } else { + resolve(); + } + }); + } + + private async handleRequest(req: IncomingMessage, res: ServerResponse): Promise { + const url = req.url || '/'; + const method = req.method || 'GET'; + + this.logger.debug({ method, url }, 'Request'); + + try { + // ── Captive portal detection URLs ── + // iOS, Android, Windows, etc. check these URLs to detect captive portals. + // Redirecting them triggers the captive portal popup on the user's device. + if (this.isCaptivePortalCheck(url)) { + res.writeHead(302, { Location: 'http://192.168.4.1/' }); + res.end(); + return; + } + + // ── API routes ── + + if (url === '/api/wifi/scan' && method === 'GET') { + await this.handleScan(res); + return; + } + + if (url === '/api/wifi/connect' && method === 'POST') { + await this.handleConnect(req, res); + return; + } + + // ── Portal HTML page ── + + res.writeHead(200, { + 'Content-Type': 'text/html; charset=utf-8', + 'Cache-Control': 'no-cache', + }); + res.end(portalHTML(this.robotName)); + } catch (err) { + this.logger.error({ err, url }, 'Request handler error'); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Internal server error' })); + } + } + + /** + * Check if the URL is a captive portal detection request. + */ + private isCaptivePortalCheck(url: string): boolean { + const captivePortalPaths = [ + '/generate_204', // Android + '/gen_204', // Android + '/hotspot-detect.html', // iOS / macOS + '/library/test/success.html', // iOS + '/ncsi.txt', // Windows + '/connecttest.txt', // Windows + '/redirect', // Windows + '/canonical.html', // Firefox + '/success.txt', // Firefox + ]; + return captivePortalPaths.some((path) => url.startsWith(path)); + } + + /** + * GET /api/wifi/scan — return available WiFi networks as JSON. + */ + private async handleScan(res: ServerResponse): Promise { + const networks = await this.wifiService.scan(); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(networks)); + } + + /** + * POST /api/wifi/connect — attempt to connect to a WiFi network. + */ + private async handleConnect(req: IncomingMessage, res: ServerResponse): Promise { + const body = await this.readBody(req); + + let ssid: string; + let password: string; + + try { + const parsed = JSON.parse(body); + ssid = parsed.ssid; + password = parsed.password; + } catch { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: false, error: 'Invalid request body' })); + return; + } + + if (!ssid) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: false, error: 'SSID is required' })); + return; + } + + this.logger.info({ ssid }, 'WiFi connection attempt from portal'); + + // Stop AP mode before attempting to connect + await this.wifiService.stopAP(); + + // Small delay to let AP go down cleanly + await new Promise((r) => setTimeout(r, 2000)); + + const success = await this.wifiService.connect(ssid, password); + + if (success) { + // Save credentials to local store + this.store.setWifi(ssid, password); + + // Respond with success (we may lose the connection since AP is down, + // but the page JavaScript handles this gracefully) + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ success: true })); + + // Notify the setup flow that WiFi is configured + if (this.resolveSetup) { + this.resolveSetup(true); + this.resolveSetup = null; + } + } else { + // Connection failed — restart AP so user can retry + await this.wifiService.startAP(`Ti-Pote`); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + success: false, + error: 'Connexion échouée. Vérifiez le mot de passe et réessayez.', + }), + ); + } + } + + /** + * Read the full request body as a string. + */ + private readBody(req: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let data = ''; + req.on('data', (chunk: Buffer) => { + data += chunk.toString(); + }); + req.on('end', () => resolve(data)); + req.on('error', reject); + }); + } +} diff --git a/apps/robot-client/src/setup/index.ts b/apps/robot-client/src/setup/index.ts new file mode 100644 index 0000000..5bd09e2 --- /dev/null +++ b/apps/robot-client/src/setup/index.ts @@ -0,0 +1,2 @@ +export { SetupFlow } from './setup-flow.js'; +export { CaptivePortal } from './captive-portal.js'; diff --git a/apps/robot-client/src/setup/portal.html.ts b/apps/robot-client/src/setup/portal.html.ts new file mode 100644 index 0000000..eb1cbb3 --- /dev/null +++ b/apps/robot-client/src/setup/portal.html.ts @@ -0,0 +1,346 @@ +/** + * Captive portal HTML page — served locally when the robot is in setup mode. + * + * Single-page app that: + * 1. Fetches available WiFi networks from the Pi's local API + * 2. Lets the user select their network and enter the password + * 3. Shows connection progress and result + * + * No external dependencies — must work offline (the user is on the robot's hotspot). + */ +export function portalHTML(robotName: string): string { + return /* html */ ` + + + + + ${robotName} — Configuration + + + + + +

${robotName}

+

Configuration WiFi

+ + +
+

Réseaux disponibles

+ +
    +

  • Recherche des réseaux...

    • +
+
+ + + + + + + + + + + + + + + +`; +} diff --git a/apps/robot-client/src/setup/setup-flow.ts b/apps/robot-client/src/setup/setup-flow.ts new file mode 100644 index 0000000..41c4b61 --- /dev/null +++ b/apps/robot-client/src/setup/setup-flow.ts @@ -0,0 +1,106 @@ +import { type WifiService } from '../services/wifi.service.js'; +import { type LocalStore } from '../services/local-store.service.js'; +import { CaptivePortal } from './captive-portal.js'; +import { createLogger, type Logger } from '../utils/index.js'; + +/** + * First-boot setup flow. + * + * Orchestrates the entire onboarding experience: + * + * Power on + * │ + * ▼ + * WiFi configured in store? + * ├── YES → Try to connect with stored credentials + * │ ├── Connected → Done ✓ + * │ └── Failed → Clear stored creds, fall through to AP mode + * │ + * └── NO → Start AP mode + Captive Portal + * └── User configures WiFi via portal + * ├── Connected → Save credentials → Done ✓ + * └── Failed → Retry via portal + */ +export class SetupFlow { + private readonly logger: Logger; + + constructor( + private readonly wifiService: WifiService, + private readonly store: LocalStore, + private readonly robotName: string, + ) { + this.logger = createLogger('setup', 'info'); + } + + /** + * Run the setup flow. Returns true when WiFi is connected and ready. + */ + async run(): Promise { + this.logger.info('Starting setup flow...'); + + // ── Step 1: Check if WiFi is already connected ── + + const alreadyConnected = await this.wifiService.isConnected(); + if (alreadyConnected) { + const ssid = await this.wifiService.currentSSID(); + this.logger.info({ ssid }, 'Already connected to WiFi — skipping setup'); + return true; + } + + // ── Step 2: Try stored credentials ── + + if (this.store.hasWifi) { + const { ssid, password } = this.store.wifi!; + this.logger.info({ ssid }, 'Trying stored WiFi credentials...'); + + const connected = await this.wifiService.connect(ssid, password); + if (connected) { + this.logger.info({ ssid }, 'Connected with stored credentials'); + return true; + } + + // Stored credentials failed — clear them and go to portal + this.logger.warn({ ssid }, 'Stored credentials failed — starting captive portal'); + this.store.clearWifi(); + } + + // ── Step 3: Start AP mode + Captive Portal ── + + this.logger.info('No WiFi configured — starting Access Point for setup'); + + const apStarted = await this.wifiService.startAP(`Ti-Pote`); + if (!apStarted) { + this.logger.error('Failed to start Access Point'); + return false; + } + + this.logger.info('╔══════════════════════════════════════════════╗'); + this.logger.info('║ Connectez-vous au WiFi "Ti-Pote" puis ║'); + this.logger.info('║ ouvrez http://192.168.4.1 dans votre ║'); + this.logger.info('║ navigateur pour configurer le réseau. ║'); + this.logger.info('╚══════════════════════════════════════════════╝'); + + const portal = new CaptivePortal(this.wifiService, this.store, this.robotName); + const wifiConfigured = await portal.start(); + + // Portal resolved — WiFi should be connected + await portal.stop(); + + if (wifiConfigured) { + this.logger.info('WiFi configured via captive portal'); + + // Verify internet connectivity + const hasInternet = await this.wifiService.hasInternet(); + if (hasInternet) { + this.logger.info('Internet connectivity verified'); + } else { + this.logger.warn('WiFi connected but no internet — cloud features may not work'); + } + + return true; + } + + this.logger.error('Setup flow ended without WiFi configuration'); + return false; + } +} diff --git a/apps/robot-client/src/transport/cloud-socket.ts b/apps/robot-client/src/transport/cloud-socket.ts new file mode 100644 index 0000000..c9d8d42 --- /dev/null +++ b/apps/robot-client/src/transport/cloud-socket.ts @@ -0,0 +1,188 @@ +import { io, Socket } from 'socket.io-client'; +import { EventEmitter } from 'node:events'; +import { type RobotConfig } from '../config/index.js'; +import { createLogger, type Logger } from '../utils/index.js'; + +// ── Types matching backend protocol ── + +export type RobotState = 'idle' | 'listening' | 'thinking' | 'speaking'; + +export interface StatusEvent { + state: RobotState; +} + +export interface ResponseTextEvent { + text: string; + audio?: string; // base64-encoded WAV +} + +export interface AudioChunkEvent { + data: string; // base64-encoded PCM +} + +// ── Events emitted by CloudSocket ── + +export interface CloudSocketEvents { + connected: () => void; + disconnected: (reason: string) => void; + status: (event: StatusEvent) => void; + response_text: (event: ResponseTextEvent) => void; + audio_chunk: (event: AudioChunkEvent) => void; + notification: (payload: Record) => void; + error: (error: Error) => void; +} + +/** + * WebSocket client for communication with the Ti-Pote backend cloud. + * + * Matches the protocol expected by RobotGateway: + * - Outbound: wake_word_detected, audio_chunk, speech_end, user_interrupt + * - Inbound: status, response_text, audio_chunk, notification + */ +export class CloudSocket extends EventEmitter { + private socket: Socket | null = null; + private readonly logger: Logger; + private reconnectAttempts = 0; + private readonly maxReconnectAttempts = 50; + private _currentState: RobotState = 'idle'; + + constructor(private readonly config: RobotConfig) { + super(); + this.logger = createLogger('cloud-socket', config.logLevel); + } + + get currentState(): RobotState { + return this._currentState; + } + + get isConnected(): boolean { + return this.socket?.connected ?? false; + } + + /** + * Connect to the backend cloud WebSocket. + */ + async connect(): Promise { + return new Promise((resolve, reject) => { + const url = `${this.config.cloudUrl}/ws/robot`; + + this.logger.info({ url }, 'Connecting to cloud backend...'); + + this.socket = io(url, { + auth: { token: this.config.deviceToken }, + transports: ['websocket'], + reconnection: true, + reconnectionDelay: 1000, + reconnectionDelayMax: 30000, + reconnectionAttempts: this.maxReconnectAttempts, + }); + + this.socket.on('connect', () => { + this.reconnectAttempts = 0; + this.logger.info('Connected to cloud backend'); + this.emit('connected'); + resolve(); + }); + + this.socket.on('connect_error', (err) => { + this.reconnectAttempts++; + this.logger.error({ err: err.message, attempt: this.reconnectAttempts }, 'Connection error'); + + if (this.reconnectAttempts >= this.maxReconnectAttempts) { + const error = new Error(`Failed to connect after ${this.maxReconnectAttempts} attempts`); + this.emit('error', error); + reject(error); + } + }); + + this.socket.on('disconnect', (reason) => { + this.logger.warn({ reason }, 'Disconnected from cloud backend'); + this._currentState = 'idle'; + this.emit('disconnected', reason); + }); + + // ── Inbound events from backend ── + + this.socket.on('status', (event: StatusEvent) => { + this._currentState = event.state; + this.logger.debug({ state: event.state }, 'Status update'); + this.emit('status', event); + }); + + this.socket.on('response_text', (event: ResponseTextEvent) => { + this.logger.info({ text: event.text, hasAudio: !!event.audio }, 'Response received'); + this.emit('response_text', event); + }); + + this.socket.on('audio_chunk', (event: AudioChunkEvent) => { + this.logger.debug('Audio chunk received from cloud'); + this.emit('audio_chunk', event); + }); + + this.socket.on('notification', (payload: Record) => { + this.logger.info({ payload }, 'Notification received'); + this.emit('notification', payload); + }); + }); + } + + /** + * Notify the backend that the wake word was detected. + */ + sendWakeWordDetected(): void { + if (!this.socket?.connected) { + this.logger.warn('Cannot send wake_word_detected: not connected'); + return; + } + this.logger.info('Sending wake_word_detected'); + this.socket.emit('wake_word_detected'); + } + + /** + * Send an audio chunk (raw PCM) to the backend for STT processing. + */ + sendAudioChunk(pcmBuffer: Buffer, sampleRate: number): void { + if (!this.socket?.connected) return; + + this.socket.emit('audio_chunk', { + data: pcmBuffer, + sampleRate, + }); + } + + /** + * Notify the backend that the user has finished speaking. + */ + sendSpeechEnd(): void { + if (!this.socket?.connected) { + this.logger.warn('Cannot send speech_end: not connected'); + return; + } + this.logger.info('Sending speech_end'); + this.socket.emit('speech_end'); + } + + /** + * Notify the backend that the user wants to interrupt the current response. + */ + sendUserInterrupt(): void { + if (!this.socket?.connected) { + this.logger.warn('Cannot send user_interrupt: not connected'); + return; + } + this.logger.info('Sending user_interrupt'); + this.socket.emit('user_interrupt'); + } + + /** + * Disconnect from the cloud backend. + */ + async disconnect(): Promise { + if (this.socket) { + this.socket.removeAllListeners(); + this.socket.disconnect(); + this.socket = null; + this.logger.info('Disconnected from cloud backend'); + } + } +} diff --git a/apps/robot-client/src/transport/index.ts b/apps/robot-client/src/transport/index.ts new file mode 100644 index 0000000..9526094 --- /dev/null +++ b/apps/robot-client/src/transport/index.ts @@ -0,0 +1 @@ +export { CloudSocket, type RobotState, type CloudSocketEvents } from './cloud-socket.js'; diff --git a/apps/robot-client/src/utils/index.ts b/apps/robot-client/src/utils/index.ts new file mode 100644 index 0000000..e3d8797 --- /dev/null +++ b/apps/robot-client/src/utils/index.ts @@ -0,0 +1 @@ +export { createLogger, type Logger } from './logger.js'; diff --git a/apps/robot-client/src/utils/logger.ts b/apps/robot-client/src/utils/logger.ts new file mode 100644 index 0000000..9eddbf7 --- /dev/null +++ b/apps/robot-client/src/utils/logger.ts @@ -0,0 +1,14 @@ +import pino from 'pino'; + +export function createLogger(name: string, level = 'info') { + return pino({ + name, + level, + transport: + process.env.NODE_ENV !== 'production' + ? { target: 'pino-pretty', options: { colorize: true } } + : undefined, + }); +} + +export type Logger = pino.Logger; diff --git a/apps/robot-client/tsconfig.json b/apps/robot-client/tsconfig.json new file mode 100644 index 0000000..f58eda9 --- /dev/null +++ b/apps/robot-client/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "tests"] +} diff --git a/package.json b/package.json index e1c26e5..fbdc6ea 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,11 @@ "scripts": { "dev": "pnpm --filter @ti-pote/backend dev", "dev:sim": "pnpm --filter simulator dev", + "dev:robot": "pnpm --filter @ti-pote/robot-client dev", "build": "pnpm --filter @ti-pote/backend build", + "build:robot": "pnpm --filter @ti-pote/robot-client build", "start": "pnpm --filter @ti-pote/backend start:prod", + "start:robot": "pnpm --filter @ti-pote/robot-client start", "lint": "pnpm -r lint", "format": "pnpm -r format" }