From 2cc12a8129d30e22423dea5f9e7ac7390519aa09 Mon Sep 17 00:00:00 2001 From: Alysson Machado Date: Wed, 16 Jul 2025 04:29:56 +0000 Subject: [PATCH] feat: Add TTS Voice Notifications and Claude Code Hooks Integration --- .env.example | 99 ++- README.md | 150 ++++- package-lock.json | 4 +- server/claude-cli.js | 10 + server/data/tts-settings.json | 36 ++ server/index.js | 59 ++ server/routes/audio.js | 390 ++++++++++++ server/routes/git.js | 2 - server/routes/settings.js | 545 ++++++++++++++++ server/routes/webhooks.js | 103 +++ server/utils/audioNotifications.js | 60 ++ src/components/AudioNotificationSettings.jsx | 625 +++++++++++++++++++ src/components/ChatInterface.jsx | 12 +- src/components/QuickSettingsPanel.jsx | 10 +- src/index.css | 181 +++--- src/services/audioService.js | 525 ++++++++++++++++ src/utils/websocket.js | 7 + test-audio.js | 49 ++ 18 files changed, 2752 insertions(+), 115 deletions(-) mode change 100755 => 100644 .env.example create mode 100644 server/data/tts-settings.json create mode 100644 server/routes/audio.js create mode 100644 server/routes/settings.js create mode 100644 server/routes/webhooks.js create mode 100644 server/utils/audioNotifications.js create mode 100644 src/components/AudioNotificationSettings.jsx create mode 100644 src/services/audioService.js create mode 100644 test-audio.js diff --git a/.env.example b/.env.example old mode 100755 new mode 100644 index 7cd2dd5b..deee5329 --- a/.env.example +++ b/.env.example @@ -1,12 +1,95 @@ -# Claude Code UI Environment Configuration -# Only includes variables that are actually used in the code +# Claude Code UI Configuration Example +# Copy this file to .env and configure your settings + +# Server Configuration +PORT=3008 +VITE_PORT=3009 + +# JWT Secret for authentication (generate a random string) +JWT_SECRET=your-jwt-secret-here + +# Optional API key for additional security +API_KEY=your-api-key-here # ============================================================================= -# SERVER CONFIGURATION +# TTS (Text-to-Speech) Configuration # ============================================================================= -# Backend server port (Express API + WebSocket server) -#API server -PORT=3008 -#Frontend port -VITE_PORT=3009 \ No newline at end of file +# Enable text-to-speech notifications for Claude events +ENABLE_TTS_NOTIFICATIONS=false + +# Enable ClaudeCodeUI notifications integration +ENABLE_CLAUDECODEUI_NOTIFICATIONS=true +CLAUDECODEUI_URL=http://localhost:3000 + +# ============================================================================= +# TTS Provider API Keys (Listed by priority - highest to lowest) +# ============================================================================= + +# ElevenLabs API Configuration (Highest Quality - Neural TTS) +ELEVENLABS_API_KEY=your-elevenlabs-api-key-here +ELEVENLABS_VOICE_ID=WejK3H1m7MI9CHnIjW9K +ELEVENLABS_MODEL=eleven_turbo_v2_5 + +# Deepgram API Configuration (High Quality, Fast - Aura TTS) +DEEPGRAM_API_KEY=your-deepgram-api-key-here +DEEPGRAM_VOICE_MODEL=aura-helios-en + +# OpenAI API Configuration (High Quality) +OPENAI_API_KEY=your-openai-api-key-here +OPENAI_TTS_VOICE=nova +OPENAI_TTS_MODEL=gpt-4o-mini-tts +OPENAI_TTS_INSTRUCTIONS="Speak in English with a cheerful, positive yet professional tone." + +# IBM Watson Text to Speech Configuration +IBM_WATSON_TTS_API_KEY=your-ibm-watson-api-key-here +IBM_WATSON_TTS_URL=your-ibm-watson-service-url-here +IBM_WATSON_TTS_VOICE=en-US_AllisonV3Voice + +# ============================================================================= +# Offline TTS Configuration (pyttsx3 - No API required) +# ============================================================================= + +# pyttsx3 Offline TTS Settings +PYTTSX3_RATE=180 +PYTTSX3_VOLUME=0.8 + +# ============================================================================= +# Personalization Settings +# ============================================================================= + +# Engineer name for personalized notifications (optional) +ENGINEER_NAME=YourName + +# Chance of including engineer name in notifications (0.0 to 1.0) +TTS_NAME_CHANCE=0.3 + +# ============================================================================= +# Database Configuration +# ============================================================================= + +# SQLite is default +DATABASE_URL=sqlite:./database/auth.db + +# ============================================================================= +# Development Configuration +# ============================================================================= + +NODE_ENV=development +DEBUG=false + +# ============================================================================= +# Available Deepgram Aura Voice Models +# ============================================================================= +# aura-helios-en (default) - Warm, confident male voice +# aura-luna-en - Smooth, professional female voice +# aura-stella-en - Clear, articulate female voice +# aura-athena-en - Authoritative, intelligent female voice +# aura-hera-en - Rich, expressive female voice +# aura-orion-en - Deep, resonant male voice +# aura-arcas-en - Friendly, approachable male voice +# aura-perseus-en - Strong, reliable male voice +# aura-angus-en - Distinctive, character-rich male voice +# aura-orpheus-en - Melodic, engaging male voice +# aura-electra-en - Dynamic, energetic female voice +# aura-zeus-en - Powerful, commanding male voice \ No newline at end of file diff --git a/README.md b/README.md index ad33afea..7a012942 100755 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ A desktop and mobile UI for [Claude Code](https://docs.anthropic.com/en/docs/cla - **File Explorer** - Interactive file tree with syntax highlighting and live editing - **Git Explorer** - View, stage and commit your changes. You can also switch branches - **Session Management** - Resume conversations, manage multiple sessions, and track history +- **TTS Voice Notifications** - Audio notifications with high-quality neural voices (ElevenLabs, Deepgram, OpenAI) +- **Claude Code Hooks Integration** - Comprehensive lifecycle hooks system for enhanced automation ## Quick Start @@ -64,7 +66,7 @@ npm install 3. **Configure environment:** ```bash cp .env.example .env -# Edit .env with your preferred settings +# Edit .env with your preferred settings and TTS API keys (optional) ``` 4. **Start the application:** @@ -99,6 +101,86 @@ To use Claude Code's full functionality, you'll need to manually enable tools: **Recommended approach**: Start with basic tools enabled and add more as needed. You can always adjust these settings later. +## Advanced Configuration + +### TTS Voice Notifications Setup + +ClaudeCodeUI supports multiple TTS providers for high-quality voice notifications. Configure your preferred provider: + +#### Environment Variables +```bash +# ElevenLabs (Premium Neural TTS) +ELEVENLABS_API_KEY=your_elevenlabs_key +ELEVENLABS_VOICE_ID=WejK3H1m7MI9CHnIjW9K # Default: Alice +ELEVENLABS_MODEL=eleven_turbo_v2_5 + +# Deepgram Aura (Fast, High-Quality) +DEEPGRAM_API_KEY=your_deepgram_key +DEEPGRAM_VOICE_MODEL=aura-helios-en # Options: aura-helios-en, aura-luna-en, etc. + +# OpenAI TTS +OPENAI_API_KEY=your_openai_key +OPENAI_TTS_VOICE=nova # Options: nova, alloy, echo, fable, onyx, shimmer +OPENAI_TTS_MODEL=gpt-4o-mini-tts +OPENAI_TTS_INSTRUCTIONS="Speak with a professional, friendly tone" + +# Personalization +ENGINEER_NAME=YourName +TTS_NAME_CHANCE=0.3 # 30% chance to include your name in notifications + +# ClaudeCodeUI Integration +ENABLE_CLAUDECODEUI_NOTIFICATIONS=true +CLAUDECODEUI_URL=http://localhost:3000 +``` + +#### Provider Priority +1. **ElevenLabs** - Highest quality neural voices +2. **Deepgram Aura** - Fast generation with natural voices +3. **OpenAI TTS** - High quality with voice instructions +4. **pyttsx3** - Offline fallback (no API key required) +5. **Browser TTS** - Final fallback using Web Speech API + +### Claude Code Hooks Configuration + +ClaudeCodeUI includes a complete hooks system in the `.claude/hooks/` directory: + +#### Hook Types +- **`notification.py`** - TTS notifications when Claude needs input +- **`pre_tool_use.py`** - Security validation before tool execution +- **`post_tool_use.py`** - Event logging after tool completion +- **`stop.py`** - AI-generated completion messages +- **`subagent_stop.py`** - Subagent completion notifications + +#### Hooks Configuration (.claude/settings.json) +```json +{ + "hooks": { + "notification": ".claude/hooks/notification.py --notify", + "pre_tool_use": ".claude/hooks/pre_tool_use.py", + "post_tool_use": ".claude/hooks/post_tool_use.py", + "stop": ".claude/hooks/stop.py", + "subagent_stop": ".claude/hooks/subagent_stop.py" + }, + "tools": { + "bash": "enabled", + "read": "enabled" + } +} +``` + +#### Testing Hooks +```bash +# Test notification hook +uv run .claude/hooks/notification.py --notify + +# Test TTS integration +echo '{"message": "Test notification"}' | uv run .claude/hooks/notification.py --notify + +# Check hook logs +tail -f logs/notification_hook.log +tail -f logs/claudecodeui_debug.log +``` + ## Usage Guide ### Core Features @@ -123,7 +205,38 @@ The UI automatically discovers Claude Code projects from `~/.claude/projects/` a - **File Operations** - Create, rename, delete files and directories #### Git Explorer - +- **Git Status** - View modified, staged, and untracked files with clear indicators +- **Branch Management** - Switch branches, view commit history, and track changes +- **Commit Operations** - Stage changes and create commits directly from the interface +- **Visual Diff** - Compare file changes with syntax-highlighted diff viewer + +#### TTS Voice Notifications +ClaudeCodeUI features an advanced Text-to-Speech system that provides audio notifications when Claude needs your input or completes tasks: + +- **Multiple TTS Providers**: Support for ElevenLabs, Deepgram Aura, OpenAI TTS, and offline pyttsx3 +- **High-Quality Neural Voices**: Professional-grade speech synthesis with natural-sounding voices +- **Backend Audio Streaming**: Server-side audio generation with seamless browser playback +- **Personalized Notifications**: Configurable engineer name integration with customizable frequency +- **Automatic Provider Selection**: Intelligent fallback from premium services to local synthesis +- **Real-time Configuration**: Update TTS settings and API keys through the web interface + +**Setup**: Navigate to Settings ā†’ Audio Notifications to configure your preferred TTS provider and voice settings. + +#### Claude Code Hooks Integration +ClaudeCodeUI includes a comprehensive hooks system that extends Claude Code's capabilities: + +- **Pre-Tool Use Hooks**: Security validation and command blocking before tool execution +- **Post-Tool Use Hooks**: Event logging and transcript processing after tool completion +- **Notification Hooks**: TTS audio alerts when Claude requests user input +- **Stop Hooks**: AI-generated completion messages with continuation control +- **Subagent Hooks**: Notifications when background tasks complete + +**Key Features**: +- **Security Layer**: Automatically blocks dangerous commands (`rm -rf`, sensitive file access) +- **Comprehensive Logging**: JSON-formatted event logs for debugging and audit trails +- **UV Single-File Scripts**: Self-contained Python scripts with embedded dependencies +- **Environment Integration**: Seamless .env configuration sharing between hooks and UI +- **Audio Streaming Pipeline**: Backend TTS generation with WebSocket delivery to browser #### Session Management - **Session Persistence** - All conversations automatically saved @@ -151,14 +264,18 @@ The UI automatically discovers Claude Code projects from `~/.claude/projects/` a ### Backend (Node.js + Express) - **Express Server** - RESTful API with static file serving -- **WebSocket Server** - Communication for chats and project refresh -- **Claude CLI Integration** - Process spawning and management +- **WebSocket Server** - Real-time communication for chats, notifications, and project updates +- **Claude CLI Integration** - Process spawning and PTY management - **Session Management** - JSONL parsing and conversation persistence -- **File System API** - Exposing file browser for projects +- **File System API** - Secure file browser for projects with path validation +- **TTS Audio Streaming** - Server-side audio generation and streaming endpoints +- **Hooks Integration** - UV script execution and environment synchronization ### Frontend (React + Vite) - **React 18** - Modern component architecture with hooks - **CodeMirror** - Advanced code editor with syntax highlighting +- **Audio Service** - WebSocket-based audio notification system with fallbacks +- **Real-time Updates** - Live project file monitoring and session synchronization @@ -191,6 +308,9 @@ We welcome contributions! Please follow these guidelines: - **Documentation** - Improve guides and API docs - **UI/UX improvements** - Better user experience - **Performance optimizations** - Make it faster +- **TTS Provider Integration** - Add support for new voice services +- **Hook Enhancements** - Extend the Claude Code hooks system +- **Audio/Voice Features** - Improve notification and accessibility features ## Troubleshooting @@ -212,6 +332,26 @@ d - Review server console logs for detailed error messages - Ensure you're not trying to access system directories outside project scope +#### TTS Notifications Not Working +**Problem**: Audio notifications not playing or voice generation failing +**Solutions**: +- **Check API Keys**: Verify TTS provider API keys in `.env` file +- **Test Provider**: Use Settings ā†’ Audio Notifications ā†’ Test button to verify configuration +- **Check Browser Audio**: Ensure browser allows audio playback (check for blocked autoplay) +- **Review Logs**: Check `logs/claudecodeui_debug.log` for TTS generation errors +- **Provider Fallback**: If premium provider fails, ensure pyttsx3 or browser TTS works +- **Network Issues**: Verify internet connection for cloud TTS providers + +#### Claude Code Hooks Issues +**Problem**: Hooks not executing or notification hooks failing +**Solutions**: +- **UV Installation**: Ensure `uv` is installed and available in PATH +- **Hook Permissions**: Check that hook scripts are executable (`chmod +x .claude/hooks/*.py`) +- **Environment Variables**: Verify `.env` file is properly loaded by hooks +- **Test Manually**: Run `uv run .claude/hooks/notification.py --notify` to test +- **Check Logs**: Review `logs/notification_hook.log` for hook execution details +- **Dependencies**: Ensure Python dependencies are available via UV script headers + ## License diff --git a/package-lock.json b/package-lock.json index 3d515197..0f65ef86 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "claude-code-ui", - "version": "1.2.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "claude-code-ui", - "version": "1.2.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "@anthropic-ai/claude-code": "^1.0.24", diff --git a/server/claude-cli.js b/server/claude-cli.js index 251f0473..f72ef002 100755 --- a/server/claude-cli.js +++ b/server/claude-cli.js @@ -2,6 +2,7 @@ import { spawn } from 'child_process'; import { promises as fs } from 'fs'; import path from 'path'; import os from 'os'; +import { createAudioNotification, isTTSEnabled } from './utils/audioNotifications.js'; let activeClaudeProcesses = new Map(); // Track active processes by session ID @@ -313,6 +314,15 @@ async function spawnClaude(command, options = {}, ws) { isNewSession: !sessionId && !!command // Flag to indicate this was a new session })); + // Send completion audio notification if enabled + if (isTTSEnabled()) { + const completionNotification = createAudioNotification('complete', '', { + exitCode: code, + sessionId: finalSessionId + }); + ws.send(JSON.stringify(completionNotification)); + } + // Clean up temporary image files if any if (claudeProcess.tempImagePaths && claudeProcess.tempImagePaths.length > 0) { for (const imagePath of claudeProcess.tempImagePaths) { diff --git a/server/data/tts-settings.json b/server/data/tts-settings.json new file mode 100644 index 00000000..d9251f67 --- /dev/null +++ b/server/data/tts-settings.json @@ -0,0 +1,36 @@ +{ + "enabled": true, + "provider": "deepgram", + "elevenlabs": { + "apiKey": "", + "voiceId": "WejK3H1m7MI9CHnIjW9K", + "model": "eleven_turbo_v2_5" + }, + "deepgram": { + "apiKey": "", + "voiceModel": "aura-stella-en" + }, + "openai": { + "apiKey": "", + "voice": "nova", + "model": "gpt-4o-mini-tts", + "instructions": "Speak in a cheerful, positive yet professional tone." + }, + "ibm_watson": { + "apiKey": "", + "apiUrl": "https://api.us-south.text-to-speech.watson.cloud.ibm.com/instances/77de050e-305c-4fff-8a7e-57023427ccf9", + "voice": "en-GB_CharlotteV3Voice" + }, + "pyttsx3": { + "rate": 180, + "volume": 0.8 + }, + "general": { + "engineerName": "Alysson", + "nameChance": 0.3, + "claudecodeui": { + "enabled": true, + "url": "https://3000-firebase-studio-1750510535256.cluster-hf4yr35cmnbd4vhbxvfvc6cp5q.cloudworkstations.dev" + } + } +} \ No newline at end of file diff --git a/server/index.js b/server/index.js index e5641158..e7997768 100755 --- a/server/index.js +++ b/server/index.js @@ -41,13 +41,40 @@ import { spawnClaude, abortClaudeSession } from './claude-cli.js'; import gitRoutes from './routes/git.js'; import authRoutes from './routes/auth.js'; import mcpRoutes from './routes/mcp.js'; +import webhookRoutes, { setConnectedClients } from './routes/webhooks.js'; +import audioRoutes, { setConnectedClients as setAudioConnectedClients } from './routes/audio.js'; +import settingsRoutes from './routes/settings.js'; import { initializeDatabase } from './database/db.js'; import { validateApiKey, authenticateToken, authenticateWebSocket } from './middleware/auth.js'; +import { createAudioNotification, isTTSEnabled } from './utils/audioNotifications.js'; + +// Broadcast audio notification to all connected chat clients +function broadcastAudioNotification(messageType, customMessage = '', metadata = {}) { + if (!isTTSEnabled()) return; + + const notification = createAudioNotification(messageType, customMessage, metadata); + + console.log(`šŸ”Š Broadcasting audio notification: ${notification.message}`); + + connectedClients.forEach(client => { + if (client.readyState === 1) { // WebSocket.OPEN + try { + client.send(JSON.stringify(notification)); + } catch (error) { + console.error('āŒ Error sending audio notification:', error.message); + } + } + }); +} // File system watcher for projects folder let projectsWatcher = null; const connectedClients = new Set(); +// Setup webhook integration with connected clients +setConnectedClients(connectedClients); +setAudioConnectedClients(connectedClients); + // Setup file system watcher for Claude projects folder using chokidar async function setupProjectsWatcher() { const chokidar = (await import('chokidar')).default; @@ -175,6 +202,15 @@ app.use('/api/git', authenticateToken, gitRoutes); // MCP API Routes (protected) app.use('/api/mcp', authenticateToken, mcpRoutes); +// Webhook API Routes (unprotected for external integrations) +app.use('/api/webhooks', webhookRoutes); + +// Audio API Routes (unprotected for TTS integration) +app.use('/api/audio', audioRoutes); + +// Settings API Routes (mixed protection) - specific routes first +app.use('/api/settings', settingsRoutes); // All settings routes (will handle auth internally) + // Static files served after API routes app.use(express.static(path.join(__dirname, '../dist'))); @@ -459,6 +495,13 @@ function handleChatConnection(ws) { console.log('šŸ’¬ User message:', data.command || '[Continue/Resume]'); console.log('šŸ“ Project:', data.options?.projectPath || 'Unknown'); console.log('šŸ”„ Session:', data.options?.sessionId ? 'Resume' : 'New'); + + // Send session start notification + broadcastAudioNotification('session_start', '', { + projectPath: data.options?.projectPath, + sessionId: data.options?.sessionId + }); + await spawnClaude(data.command, data.options, ws); } else if (data.type === 'abort-session') { console.log('šŸ›‘ Abort session request:', data.sessionId); @@ -468,6 +511,22 @@ function handleChatConnection(ws) { sessionId: data.sessionId, success })); + + // Send session end notification + broadcastAudioNotification('session_end', '', { sessionId: data.sessionId }); + } else if (data.type === 'trigger-audio-notification') { + // Allow manual audio notification triggering for testing/hooks integration + console.log('šŸ”Š Manual audio notification trigger:', data.messageType); + broadcastAudioNotification( + data.messageType || 'input', + data.customMessage || '', + data.metadata || {} + ); + ws.send(JSON.stringify({ + type: 'audio-notification-sent', + messageType: data.messageType, + success: true + })); } } catch (error) { console.error('āŒ Chat WebSocket error:', error.message); diff --git a/server/routes/audio.js b/server/routes/audio.js new file mode 100644 index 00000000..fa03a906 --- /dev/null +++ b/server/routes/audio.js @@ -0,0 +1,390 @@ +// Audio streaming routes for TTS integration +import express from 'express'; +import multer from 'multer'; +import path from 'path'; +import fs from 'fs/promises'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const router = express.Router(); + +// Configure multer for audio file uploads +const storage = multer.diskStorage({ + destination: async (req, file, cb) => { + const audioDir = path.join(__dirname, '../temp/audio'); + try { + await fs.mkdir(audioDir, { recursive: true }); + cb(null, audioDir); + } catch (error) { + cb(error); + } + }, + filename: (req, file, cb) => { + // Generate unique filename with timestamp + const timestamp = Date.now(); + const randomId = Math.random().toString(36).substring(2, 15); + const ext = path.extname(file.originalname) || '.mp3'; + cb(null, `tts-${timestamp}-${randomId}${ext}`); + } +}); + +const upload = multer({ + storage, + limits: { + fileSize: 10 * 1024 * 1024, // 10MB limit + }, + fileFilter: (req, file, cb) => { + // Accept audio files + const allowedMimes = [ + 'audio/mpeg', + 'audio/wav', + 'audio/mp3', + 'audio/ogg', + 'audio/webm' + ]; + + if (allowedMimes.includes(file.mimetype)) { + cb(null, true); + } else { + cb(new Error('Invalid file type. Only audio files are allowed.')); + } + } +}); + +// Store connected WebSocket clients (will be set by main server) +let connectedClients = new Set(); + +export function setConnectedClients(clients) { + connectedClients = clients; +} + +// Upload audio file for streaming +router.post('/upload', upload.single('audio'), async (req, res) => { + try { + if (!req.file) { + return res.status(400).json({ error: 'No audio file provided' }); + } + + const { messageType = 'input', message = '', metadata = {} } = req.body; + + // Generate public URL for the audio file + const audioUrl = `/api/audio/stream/${path.basename(req.file.filename)}`; + + console.log(`šŸŽµ Audio uploaded: ${req.file.filename} (${req.file.size} bytes)`); + + // Broadcast audio notification to connected clients + const notification = { + type: 'audio-notification', + messageType, + message, + audioUrl, + timestamp: new Date().toISOString(), + ttsEnabled: true, + source: 'backend-tts', + metadata: { + ...metadata, + audioFile: req.file.filename, + fileSize: req.file.size, + mimeType: req.file.mimetype + } + }; + + connectedClients.forEach(client => { + if (client.readyState === 1) { // WebSocket.OPEN + try { + client.send(JSON.stringify(notification)); + } catch (error) { + console.error('āŒ Error sending audio notification:', error.message); + } + } + }); + + res.json({ + success: true, + audioUrl, + filename: req.file.filename, + message: 'Audio uploaded and notification sent', + clientCount: connectedClients.size + }); + + } catch (error) { + console.error('āŒ Audio upload error:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +// Stream audio file +router.get('/stream/:filename', async (req, res) => { + try { + const { filename } = req.params; + const audioDir = path.join(__dirname, '../temp/audio'); + const filePath = path.join(audioDir, filename); + + // Security check: ensure filename doesn't contain path traversal + const normalizedPath = path.normalize(filePath); + if (!normalizedPath.startsWith(audioDir)) { + return res.status(403).json({ error: 'Access denied' }); + } + + // Check if file exists + try { + await fs.access(filePath); + } catch { + return res.status(404).json({ error: 'Audio file not found' }); + } + + // Get file stats + const stats = await fs.stat(filePath); + const fileSize = stats.size; + + // Set appropriate headers + const ext = path.extname(filename).toLowerCase(); + let mimeType = 'audio/mpeg'; // default + + switch (ext) { + case '.mp3': + mimeType = 'audio/mpeg'; + break; + case '.wav': + mimeType = 'audio/wav'; + break; + case '.ogg': + mimeType = 'audio/ogg'; + break; + case '.webm': + mimeType = 'audio/webm'; + break; + } + + res.set({ + 'Content-Type': mimeType, + 'Content-Length': fileSize, + 'Accept-Ranges': 'bytes', + 'Cache-Control': 'public, max-age=3600', // Cache for 1 hour + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET', + 'Access-Control-Allow-Headers': 'Range' + }); + + // Handle range requests for better audio streaming + const range = req.headers.range; + if (range) { + const parts = range.replace(/bytes=/, '').split('-'); + const start = parseInt(parts[0], 10); + const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1; + const chunkSize = (end - start) + 1; + + res.status(206).set({ + 'Content-Range': `bytes ${start}-${end}/${fileSize}`, + 'Content-Length': chunkSize + }); + + // Stream the requested range + const readStream = (await import('fs')).createReadStream(filePath, { start, end }); + readStream.pipe(res); + } else { + // Stream entire file + const readStream = (await import('fs')).createReadStream(filePath); + readStream.pipe(res); + } + + console.log(`šŸŽµ Streaming audio: ${filename}`); + + } catch (error) { + console.error('āŒ Audio streaming error:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +// Clean up old audio files +router.delete('/cleanup', async (req, res) => { + try { + const audioDir = path.join(__dirname, '../temp/audio'); + const maxAge = 24 * 60 * 60 * 1000; // 24 hours + const now = Date.now(); + + let cleanedCount = 0; + + try { + const files = await fs.readdir(audioDir); + + for (const file of files) { + const filePath = path.join(audioDir, file); + const stats = await fs.stat(filePath); + + if (now - stats.mtime.getTime() > maxAge) { + await fs.unlink(filePath); + cleanedCount++; + console.log(`šŸ—‘ļø Cleaned up old audio file: ${file}`); + } + } + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + res.json({ + success: true, + message: `Cleaned up ${cleanedCount} old audio files`, + cleanedCount + }); + + } catch (error) { + console.error('āŒ Audio cleanup error:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +// Get audio directory info +router.get('/info', async (req, res) => { + try { + const audioDir = path.join(__dirname, '../temp/audio'); + + let fileCount = 0; + let totalSize = 0; + + try { + const files = await fs.readdir(audioDir); + + for (const file of files) { + const filePath = path.join(audioDir, file); + const stats = await fs.stat(filePath); + fileCount++; + totalSize += stats.size; + } + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + res.json({ + audioDirectory: audioDir, + fileCount, + totalSize, + totalSizeMB: Math.round(totalSize / 1024 / 1024 * 100) / 100 + }); + + } catch (error) { + console.error('āŒ Audio info error:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +// Test notification endpoint (triggers backend TTS) +router.post('/test-notification', async (req, res) => { + try { + const { message = 'Audio notifications are working correctly', messageType = 'test' } = req.body; + + console.log('šŸŽµ Test notification requested:', message); + + // Try to generate actual TTS audio for the test + const { spawn } = await import('child_process'); + const path = await import('path'); + const fs = await import('fs/promises'); + + // Path to claudecodeui_notification script + const scriptPath = path.join(__dirname, '../../../.claude/hooks/utils/claudecodeui_notification.py'); + + try { + // Check if the script exists + await fs.access(scriptPath); + + // Call the script to generate and upload TTS audio + const childProcess = spawn('uv', ['run', scriptPath, messageType, message], { + stdio: 'pipe', + timeout: 15000 // 15 second timeout + }); + + let output = ''; + let error = ''; + + childProcess.stdout.on('data', (data) => { + output += data.toString(); + }); + + childProcess.stderr.on('data', (data) => { + error += data.toString(); + }); + + childProcess.on('close', (code) => { + if (code === 0) { + console.log('āœ… Backend TTS test completed successfully'); + console.log('šŸ“„ Output:', output); + } else { + console.error('āŒ Backend TTS test failed with code:', code); + console.error('šŸ“„ Error:', error); + + // Fallback to text notification + const fallbackNotification = { + type: 'audio-notification', + messageType, + message, + timestamp: new Date().toISOString(), + ttsEnabled: true, + source: 'test-fallback', + metadata: { + source: 'backend-test-fallback', + testMode: true, + fallbackReason: 'TTS generation failed' + } + }; + + connectedClients.forEach(client => { + if (client.readyState === 1) { + try { + client.send(JSON.stringify(fallbackNotification)); + } catch (error) { + console.error('āŒ Error sending fallback notification:', error.message); + } + } + }); + } + }); + + } catch (accessError) { + console.log('āš ļø TTS script not found, sending text notification:', accessError.message); + + // Fallback to text notification if script doesn't exist + const textNotification = { + type: 'audio-notification', + messageType, + message, + timestamp: new Date().toISOString(), + ttsEnabled: true, + source: 'test', + metadata: { + source: 'backend-test', + testMode: true, + fallbackReason: 'TTS script not available' + } + }; + + connectedClients.forEach(client => { + if (client.readyState === 1) { + try { + client.send(JSON.stringify(textNotification)); + } catch (error) { + console.error('āŒ Error sending test notification:', error.message); + } + } + }); + } + + res.json({ + success: true, + message: 'Test notification initiated', + clientCount: connectedClients.size, + method: 'backend-tts' + }); + + } catch (error) { + console.error('āŒ Test notification error:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +export default router; \ No newline at end of file diff --git a/server/routes/git.js b/server/routes/git.js index b56b3e4e..a82e04d4 100755 --- a/server/routes/git.js +++ b/server/routes/git.js @@ -56,7 +56,6 @@ router.get('/status', async (req, res) => { try { const projectPath = await getActualProjectPath(project); - console.log('Git status for project:', project, '-> path:', projectPath); // Validate git repository await validateGitRepository(projectPath); @@ -192,7 +191,6 @@ router.get('/branches', async (req, res) => { try { const projectPath = await getActualProjectPath(project); - console.log('Git branches for project:', project, '-> path:', projectPath); // Validate git repository await validateGitRepository(projectPath); diff --git a/server/routes/settings.js b/server/routes/settings.js new file mode 100644 index 00000000..5b7e6aae --- /dev/null +++ b/server/routes/settings.js @@ -0,0 +1,545 @@ +// Settings management for TTS and hook configurations +import express from 'express'; +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { authenticateToken } from '../middleware/auth.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const router = express.Router(); + +// Default TTS settings +const DEFAULT_TTS_SETTINGS = { + enabled: true, + provider: 'auto', // auto, elevenlabs, deepgram, openai, ibm_watson, pyttsx3 + elevenlabs: { + apiKey: '', + voiceId: 'WejK3H1m7MI9CHnIjW9K', + model: 'eleven_turbo_v2_5' + }, + deepgram: { + apiKey: '', + voiceModel: 'aura-helios-en' + }, + openai: { + apiKey: '', + voice: 'nova', + model: 'gpt-4o-mini-tts', + instructions: 'Speak in a cheerful, positive yet professional tone.' + }, + ibm_watson: { + apiKey: '', + apiUrl: '', + voice: 'en-US_MichaelV3Voice' + }, + pyttsx3: { + rate: 180, + volume: 0.8 + }, + general: { + engineerName: '', + nameChance: 0.3, + claudecodeui: { + enabled: true, + url: 'http://localhost:3000' + } + } +}; + +// Get settings file paths +function getSettingsPaths() { + const claudecodeUIRoot = path.resolve(__dirname, '../../'); + return { + envFile: path.join(claudecodeUIRoot, '.env'), + settingsFile: path.join(__dirname, '../data/tts-settings.json') + }; +} + +// Load current TTS settings from environment and stored config +async function loadTTSSettings() { + try { + const paths = getSettingsPaths(); + let settings = { ...DEFAULT_TTS_SETTINGS }; + + // Try to load stored settings + try { + const settingsData = await fs.readFile(paths.settingsFile, 'utf8'); + const storedSettings = JSON.parse(settingsData); + settings = { ...settings, ...storedSettings }; + } catch (error) { + if (error.code !== 'ENOENT') { + console.warn('Error loading TTS settings file:', error.message); + } + } + + // Override with environment variables + if (process.env.ELEVENLABS_API_KEY) { + settings.elevenlabs.apiKey = process.env.ELEVENLABS_API_KEY; + } + if (process.env.ELEVENLABS_VOICE_ID) { + settings.elevenlabs.voiceId = process.env.ELEVENLABS_VOICE_ID; + } + if (process.env.ELEVENLABS_MODEL) { + settings.elevenlabs.model = process.env.ELEVENLABS_MODEL; + } + + if (process.env.DEEPGRAM_API_KEY) { + settings.deepgram.apiKey = process.env.DEEPGRAM_API_KEY; + } + if (process.env.DEEPGRAM_VOICE_MODEL) { + settings.deepgram.voiceModel = process.env.DEEPGRAM_VOICE_MODEL; + } + + if (process.env.OPENAI_API_KEY) { + settings.openai.apiKey = process.env.OPENAI_API_KEY; + } + if (process.env.OPENAI_TTS_VOICE) { + settings.openai.voice = process.env.OPENAI_TTS_VOICE; + } + if (process.env.OPENAI_TTS_MODEL) { + settings.openai.model = process.env.OPENAI_TTS_MODEL; + } + if (process.env.OPENAI_TTS_INSTRUCTIONS) { + settings.openai.instructions = process.env.OPENAI_TTS_INSTRUCTIONS; + } + + if (process.env.IBM_API_KEY) { + settings.ibm_watson.apiKey = process.env.IBM_API_KEY; + } + if (process.env.IBM_API_URL) { + settings.ibm_watson.apiUrl = process.env.IBM_API_URL; + } + if (process.env.IBM_WATSON_VOICE) { + settings.ibm_watson.voice = process.env.IBM_WATSON_VOICE; + } + + if (process.env.PYTTSX3_RATE) { + settings.pyttsx3.rate = parseInt(process.env.PYTTSX3_RATE); + } + if (process.env.PYTTSX3_VOLUME) { + settings.pyttsx3.volume = parseFloat(process.env.PYTTSX3_VOLUME); + } + + if (process.env.ENGINEER_NAME) { + settings.general.engineerName = process.env.ENGINEER_NAME; + } + if (process.env.TTS_NAME_CHANCE) { + settings.general.nameChance = parseFloat(process.env.TTS_NAME_CHANCE); + } + if (process.env.ENABLE_CLAUDECODEUI_NOTIFICATIONS) { + settings.general.claudecodeui.enabled = process.env.ENABLE_CLAUDECODEUI_NOTIFICATIONS.toLowerCase() === 'true'; + } + if (process.env.CLAUDECODEUI_URL) { + settings.general.claudecodeui.url = process.env.CLAUDECODEUI_URL; + } + + return settings; + } catch (error) { + console.error('Error loading TTS settings:', error.message); + return DEFAULT_TTS_SETTINGS; + } +} + +// Save TTS settings to file and environment +async function saveTTSSettings(settings) { + try { + const paths = getSettingsPaths(); + + // Ensure data directory exists + await fs.mkdir(path.dirname(paths.settingsFile), { recursive: true }); + + // Create settings copy without API keys for JSON storage + const settingsForJson = JSON.parse(JSON.stringify(settings)); + settingsForJson.elevenlabs.apiKey = ''; + settingsForJson.deepgram.apiKey = ''; + settingsForJson.openai.apiKey = ''; + settingsForJson.ibm_watson.apiKey = ''; + + // Save settings to JSON file (without API keys) + await fs.writeFile(paths.settingsFile, JSON.stringify(settingsForJson, null, 2)); + + // Generate environment variables content (only non-sensitive settings) + const envContent = generateEnvContent(settings); + + // Update only non-API-key settings in .env file + await updateEnvFilePreservingAPIKeys(paths.envFile, envContent); + + console.log('āœ… TTS settings saved successfully (API keys preserved)'); + return true; + } catch (error) { + console.error('āŒ Error saving TTS settings:', error.message); + throw error; + } +} + +// Generate environment variables content from settings +function generateEnvContent(settings) { + const lines = []; + + lines.push('# TTS Configuration'); + + // NEVER write API keys - they should only be read from .env + if (settings.elevenlabs.voiceId) { + lines.push(`ELEVENLABS_VOICE_ID=${settings.elevenlabs.voiceId}`); + } + if (settings.elevenlabs.model) { + lines.push(`ELEVENLABS_MODEL=${settings.elevenlabs.model}`); + } + + if (settings.deepgram.voiceModel) { + lines.push(`DEEPGRAM_VOICE_MODEL=${settings.deepgram.voiceModel}`); + } + + if (settings.openai.voice) { + lines.push(`OPENAI_TTS_VOICE=${settings.openai.voice}`); + } + if (settings.openai.model) { + lines.push(`OPENAI_TTS_MODEL=${settings.openai.model}`); + } + if (settings.openai.instructions) { + lines.push(`OPENAI_TTS_INSTRUCTIONS=${settings.openai.instructions}`); + } + + if (settings.ibm_watson.apiUrl) { + lines.push(`IBM_API_URL=${settings.ibm_watson.apiUrl}`); + } + if (settings.ibm_watson.voice) { + lines.push(`IBM_WATSON_VOICE=${settings.ibm_watson.voice}`); + } + + if (settings.pyttsx3.rate) { + lines.push(`PYTTSX3_RATE=${settings.pyttsx3.rate}`); + } + if (settings.pyttsx3.volume) { + lines.push(`PYTTSX3_VOLUME=${settings.pyttsx3.volume}`); + } + + if (settings.general.engineerName) { + lines.push(`ENGINEER_NAME=${settings.general.engineerName}`); + } + if (settings.general.nameChance !== undefined) { + lines.push(`TTS_NAME_CHANCE=${settings.general.nameChance}`); + } + if (settings.general.claudecodeui.enabled !== undefined) { + lines.push(`ENABLE_CLAUDECODEUI_NOTIFICATIONS=${settings.general.claudecodeui.enabled ? 'true' : 'false'}`); + } + if (settings.general.claudecodeui.url) { + lines.push(`CLAUDECODEUI_URL=${settings.general.claudecodeui.url}`); + } + + return lines.join('\n') + '\n'; +} + +// Update environment file while preserving other variables +async function updateEnvFile(filePath, newContent) { + try { + let existingContent = ''; + try { + existingContent = await fs.readFile(filePath, 'utf8'); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + // Parse existing content + const existingLines = existingContent.split('\n'); + const ttsKeys = new Set([ + 'ELEVENLABS_API_KEY', 'ELEVENLABS_VOICE_ID', 'ELEVENLABS_MODEL', + 'DEEPGRAM_API_KEY', 'DEEPGRAM_VOICE_MODEL', + 'OPENAI_API_KEY', 'OPENAI_TTS_VOICE', 'OPENAI_TTS_MODEL', 'OPENAI_TTS_INSTRUCTIONS', + 'IBM_API_KEY', 'IBM_API_URL', 'IBM_WATSON_VOICE', + 'PYTTSX3_RATE', 'PYTTSX3_VOLUME', + 'ENGINEER_NAME', 'TTS_NAME_CHANCE', + 'ENABLE_CLAUDECODEUI_NOTIFICATIONS', 'CLAUDECODEUI_URL' + ]); + + // Filter out existing TTS-related lines and TTS comments + const filteredLines = existingLines.filter(line => { + const trimmed = line.trim(); + + // Remove TTS configuration comment lines + if (trimmed === '# TTS Configuration') return false; + + // Keep empty lines and other comments + if (!trimmed || (trimmed.startsWith('#') && trimmed !== '# TTS Configuration')) return true; + + // Remove TTS key-value pairs + const key = trimmed.split('=')[0]; + return !ttsKeys.has(key); + }); + + // Add new TTS content + const cleanedContent = filteredLines.join('\n').replace(/\n{3,}/g, '\n\n').trim(); + const finalContent = cleanedContent + '\n\n' + newContent; + + await fs.writeFile(filePath, finalContent); + } catch (error) { + console.error(`Error updating ${filePath}:`, error.message); + throw error; + } +} + +// Update env file while preserving API keys +async function updateEnvFilePreservingAPIKeys(filePath, newContent) { + try { + let existingContent = ''; + try { + existingContent = await fs.readFile(filePath, 'utf8'); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + // Parse existing content and preserve API keys + const existingLines = existingContent.split('\n'); + const apiKeyLines = existingLines.filter(line => { + const trimmed = line.trim(); + return trimmed.startsWith('ELEVENLABS_API_KEY=') || + trimmed.startsWith('DEEPGRAM_API_KEY=') || + trimmed.startsWith('OPENAI_API_KEY=') || + trimmed.startsWith('IBM_API_KEY='); + }); + + // Remove TTS settings but keep API keys and other content + const nonTTSKeys = new Set([ + 'ELEVENLABS_VOICE_ID', 'ELEVENLABS_MODEL', + 'DEEPGRAM_VOICE_MODEL', + 'OPENAI_TTS_VOICE', 'OPENAI_TTS_MODEL', 'OPENAI_TTS_INSTRUCTIONS', + 'IBM_API_URL', 'IBM_WATSON_VOICE', + 'PYTTSX3_RATE', 'PYTTSX3_VOLUME', + 'ENGINEER_NAME', 'TTS_NAME_CHANCE', + 'ENABLE_CLAUDECODEUI_NOTIFICATIONS', 'CLAUDECODEUI_URL' + ]); + + const filteredLines = existingLines.filter(line => { + const trimmed = line.trim(); + + // Keep empty lines and non-TTS comments + if (!trimmed || (trimmed.startsWith('#') && trimmed !== '# TTS Configuration')) return true; + + // Remove TTS Configuration section header to prevent duplicates + if (trimmed === '# TTS Configuration') return false; + + // Keep API keys + if (trimmed.startsWith('ELEVENLABS_API_KEY=') || + trimmed.startsWith('DEEPGRAM_API_KEY=') || + trimmed.startsWith('OPENAI_API_KEY=') || + trimmed.startsWith('IBM_API_KEY=')) return true; + + // Remove non-API TTS settings that will be replaced + const key = trimmed.split('=')[0]; + return !nonTTSKeys.has(key); + }); + + // Add new non-API content + const cleanedContent = filteredLines.join('\n').replace(/\n{3,}/g, '\n\n').trim(); + const finalContent = cleanedContent + '\n\n' + newContent; + + await fs.writeFile(filePath, finalContent); + } catch (error) { + console.error(`Error updating ${filePath}:`, error.message); + throw error; + } +} + +// Get current TTS settings (protected) +router.get('/tts', authenticateToken, async (req, res) => { + try { + const settings = await loadTTSSettings(); + + // Don't expose API keys in the response + const safeSettings = { + ...settings, + elevenlabs: { + ...settings.elevenlabs, + apiKey: settings.elevenlabs.apiKey ? '***masked***' : '' + }, + deepgram: { + ...settings.deepgram, + apiKey: settings.deepgram.apiKey ? '***masked***' : '' + }, + openai: { + ...settings.openai, + apiKey: settings.openai.apiKey ? '***masked***' : '' + }, + ibm_watson: { + ...settings.ibm_watson, + apiKey: settings.ibm_watson.apiKey ? '***masked***' : '' + } + }; + + res.json(safeSettings); + } catch (error) { + console.error('āŒ Error getting TTS settings:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +// Update TTS settings (protected) +router.put('/tts', authenticateToken, async (req, res) => { + try { + const currentSettings = await loadTTSSettings(); + const newSettings = { ...currentSettings, ...req.body }; + + // Remove masked API keys - these should only come from .env file + if (newSettings.elevenlabs?.apiKey === '***masked***') { + newSettings.elevenlabs.apiKey = currentSettings.elevenlabs.apiKey; + } + if (newSettings.deepgram?.apiKey === '***masked***') { + newSettings.deepgram.apiKey = currentSettings.deepgram.apiKey; + } + if (newSettings.openai?.apiKey === '***masked***') { + newSettings.openai.apiKey = currentSettings.openai.apiKey; + } + if (newSettings.ibm_watson?.apiKey === '***masked***') { + newSettings.ibm_watson.apiKey = currentSettings.ibm_watson.apiKey; + } + + // Validate settings + if (newSettings.elevenlabs && typeof newSettings.elevenlabs !== 'object') { + return res.status(400).json({ error: 'Invalid elevenlabs settings' }); + } + if (newSettings.deepgram && typeof newSettings.deepgram !== 'object') { + return res.status(400).json({ error: 'Invalid deepgram settings' }); + } + if (newSettings.openai && typeof newSettings.openai !== 'object') { + return res.status(400).json({ error: 'Invalid openai settings' }); + } + if (newSettings.ibm_watson && typeof newSettings.ibm_watson !== 'object') { + return res.status(400).json({ error: 'Invalid ibm_watson settings' }); + } + if (newSettings.pyttsx3 && typeof newSettings.pyttsx3 !== 'object') { + return res.status(400).json({ error: 'Invalid pyttsx3 settings' }); + } + + await saveTTSSettings(newSettings); + + res.json({ + success: true, + message: 'TTS settings updated successfully', + settings: newSettings + }); + } catch (error) { + console.error('āŒ Error updating TTS settings:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +// Test TTS with current settings (protected) +router.post('/tts/test', authenticateToken, async (req, res) => { + try { + const { message = 'TTS test message', provider = 'auto' } = req.body; + + // Import the ClaudeCodeUI notification utility + const { spawn } = await import('child_process'); + const path = await import('path'); + + // Path to the ClaudeCodeUI notification script + const claudecodeUIRoot = path.resolve(__dirname, '../../'); + const notificationScript = path.join(claudecodeUIRoot, '.claude/hooks/utils/claudecodeui_notification.py'); + + // Check if the script exists + const fs = await import('fs'); + if (!fs.existsSync(notificationScript)) { + return res.status(404).json({ + error: 'ClaudeCodeUI notification script not found', + path: notificationScript + }); + } + + // Spawn the notification script to trigger TTS + const child = spawn('uv', ['run', notificationScript, 'test', message], { + env: { + ...process.env, + ENABLE_CLAUDECODEUI_NOTIFICATIONS: 'true', + CLAUDECODEUI_URL: 'http://localhost:3000' + }, + timeout: 10000 + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data.toString(); + }); + + child.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + child.on('close', (code) => { + if (code === 0) { + console.log('āœ… TTS test completed successfully'); + } else { + console.log('āŒ TTS test failed with code:', code); + } + }); + + // Don't wait for completion, return immediately + res.json({ + success: true, + message: 'TTS test initiated with backend audio generation', + provider, + testMessage: message + }); + + } catch (error) { + console.error('āŒ Error testing TTS:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +// Get available TTS providers (no auth required for basic info) +router.get('/tts/providers', (req, res) => { + try { + const providers = [ + { + id: 'auto', + name: 'Auto (Best Available)', + description: 'Automatically selects the best available TTS provider' + }, + { + id: 'elevenlabs', + name: 'ElevenLabs', + description: 'High-quality neural TTS (requires API key)', + requiresApiKey: true + }, + { + id: 'deepgram', + name: 'Deepgram Aura', + description: 'Fast, high-quality Aura TTS models (requires API key)', + requiresApiKey: true + }, + { + id: 'openai', + name: 'OpenAI TTS', + description: 'OpenAI text-to-speech (requires API key)', + requiresApiKey: true + }, + { + id: 'ibm_watson', + name: 'IBM Watson', + description: 'IBM Watson neural voices (requires API key)', + requiresApiKey: true + }, + { + id: 'pyttsx3', + name: 'Offline TTS', + description: 'Local text-to-speech (no API key required)', + requiresApiKey: false + } + ]; + + res.json(providers); + } catch (error) { + console.error('āŒ Error getting TTS providers:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +export default router; \ No newline at end of file diff --git a/server/routes/webhooks.js b/server/routes/webhooks.js new file mode 100644 index 00000000..09c2efe6 --- /dev/null +++ b/server/routes/webhooks.js @@ -0,0 +1,103 @@ +// Webhook endpoints for external integrations (like claude-code-hooks) +import express from 'express'; + +const router = express.Router(); + +// Store connected WebSocket clients globally +let connectedClients = new Set(); + +// Function to set connected clients from main server +export function setConnectedClients(clients) { + connectedClients = clients; +} + +// Broadcast audio notification to all connected clients +function broadcastAudioNotification(messageType, customMessage = '', metadata = {}) { + const notification = { + type: 'audio-notification', + messageType, + message: customMessage || `Claude ${messageType} notification`, + timestamp: new Date().toISOString(), + ttsEnabled: true, + voice: 'nova', + metadata, + source: 'webhook' + }; + + console.log(`šŸ”Š Webhook broadcasting audio notification: ${notification.message}`); + + connectedClients.forEach(client => { + if (client.readyState === 1) { // WebSocket.OPEN + try { + client.send(JSON.stringify(notification)); + } catch (error) { + console.error('āŒ Error sending webhook audio notification:', error.message); + } + } + }); +} + +// Webhook endpoint for Claude Code hooks integration +router.post('/audio-notification', (req, res) => { + try { + const { messageType, message, metadata } = req.body; + + console.log('šŸŽÆ Webhook received audio notification request:', { + messageType, + message, + metadata + }); + + // Validate required fields + if (!messageType) { + return res.status(400).json({ + error: 'messageType is required', + example: { messageType: 'input', message: 'Claude needs your input', metadata: {} } + }); + } + + // Broadcast to connected clients + broadcastAudioNotification(messageType, message, metadata || {}); + + res.json({ + success: true, + message: 'Audio notification sent to connected clients', + clientCount: connectedClients.size + }); + + } catch (error) { + console.error('āŒ Webhook error:', error.message); + res.status(500).json({ error: error.message }); + } +}); + +// Health check endpoint +router.get('/health', (req, res) => { + res.json({ + status: 'ok', + connectedClients: connectedClients.size, + timestamp: new Date().toISOString() + }); +}); + +// Get notification types and examples +router.get('/notification-types', (req, res) => { + res.json({ + messageTypes: [ + { type: 'input', description: 'Claude needs user input', example: 'Claude is waiting for you' }, + { type: 'complete', description: 'Task completed', example: 'Task completed successfully' }, + { type: 'error', description: 'Error occurred', example: 'Something went wrong' }, + { type: 'session_start', description: 'New session started', example: 'New session started' }, + { type: 'session_end', description: 'Session ended', example: 'Session ended' } + ], + webhookUrl: '/api/webhooks/audio-notification', + method: 'POST', + bodyExample: { + messageType: 'input', + message: 'Your agent needs your input', + metadata: { source: 'claude-hooks', hookType: 'notification' } + } + }); +}); + +export default router; \ No newline at end of file diff --git a/server/utils/audioNotifications.js b/server/utils/audioNotifications.js new file mode 100644 index 00000000..5e5455fb --- /dev/null +++ b/server/utils/audioNotifications.js @@ -0,0 +1,60 @@ +// Audio notification utilities for claudecodeui +import { promises as fs } from 'fs'; +import path from 'path'; + +// Audio notification settings +const AUDIO_NOTIFICATION_SETTINGS = { + enableTTS: process.env.ENABLE_TTS_NOTIFICATIONS === 'true' || false, + openaiApiKey: process.env.OPENAI_API_KEY, + elevenLabsApiKey: process.env.ELEVENLABS_API_KEY, + defaultVoice: process.env.TTS_VOICE || 'nova', + engineerName: process.env.ENGINEER_NAME || '', + notificationChance: parseFloat(process.env.TTS_NAME_CHANCE || '0.3') +}; + +// Generate audio notification message +export function generateNotificationMessage(messageType = 'input', customMessage = '') { + if (customMessage) return customMessage; + + const messages = { + input: ['Your agent needs your input', 'Claude is waiting for you', 'Agent needs assistance'], + complete: ['Task completed successfully', 'Agent has finished', 'Work is done'], + error: ['There was an error', 'Something went wrong', 'Agent encountered an issue'], + session_start: ['New session started', 'Agent is ready', 'Claude is online'], + session_end: ['Session ended', 'Agent signed off', 'Claude is offline'] + }; + + const messageList = messages[messageType] || messages.input; + const baseMessage = messageList[Math.floor(Math.random() * messageList.length)]; + + // Add engineer name with configured probability + if (AUDIO_NOTIFICATION_SETTINGS.engineerName && Math.random() < AUDIO_NOTIFICATION_SETTINGS.notificationChance) { + return `${AUDIO_NOTIFICATION_SETTINGS.engineerName}, ${baseMessage.toLowerCase()}`; + } + + return baseMessage; +} + +// Create audio notification object +export function createAudioNotification(messageType, customMessage = '', metadata = {}) { + const notificationMessage = generateNotificationMessage(messageType, customMessage); + return { + type: 'audio-notification', + messageType, + message: notificationMessage, + timestamp: new Date().toISOString(), + ttsEnabled: AUDIO_NOTIFICATION_SETTINGS.enableTTS, + voice: AUDIO_NOTIFICATION_SETTINGS.defaultVoice, + metadata + }; +} + +// Check if TTS is enabled +export function isTTSEnabled() { + return AUDIO_NOTIFICATION_SETTINGS.enableTTS; +} + +// Get TTS settings +export function getTTSSettings() { + return { ...AUDIO_NOTIFICATION_SETTINGS }; +} \ No newline at end of file diff --git a/src/components/AudioNotificationSettings.jsx b/src/components/AudioNotificationSettings.jsx new file mode 100644 index 00000000..75556b20 --- /dev/null +++ b/src/components/AudioNotificationSettings.jsx @@ -0,0 +1,625 @@ +import { useState, useEffect } from 'react'; +import { audioService } from '../services/audioService'; + +export default function AudioNotificationSettings({ className = '' }) { + const [isEnabled, setIsEnabled] = useState(false); + const [settings, setSettings] = useState({ + volume: 0.7, + rate: 1.0, + pitch: 1.0, + fallbackToBeep: true + }); + const [voices, setVoices] = useState([]); + const [selectedVoice, setSelectedVoice] = useState(''); + const [isLoading, setIsLoading] = useState(false); + + // Backend TTS settings + const [backendTTSSettings, setBackendTTSSettings] = useState(null); + const [ttsProviders, setTTSProviders] = useState([]); + const [selectedProvider, setSelectedProvider] = useState('auto'); + const [isBackendTTSEnabled, setIsBackendTTSEnabled] = useState(true); + const [apiKeys, setApiKeys] = useState({ + elevenlabs: '', + deepgram: '', + openai: '', + ibm_watson: '' + }); + const [engineerName, setEngineerName] = useState(''); + const [nameChance, setNameChance] = useState(0.3); + + useEffect(() => { + // Load initial settings + setIsEnabled(audioService.getEnabled()); + setSettings(audioService.settings); + + // Load available voices + loadVoices(); + + // Load backend TTS settings + loadBackendTTSSettings(); + loadTTSProviders(); + + // Listen for voice changes + if ('speechSynthesis' in window) { + window.speechSynthesis.addEventListener('voiceschanged', loadVoices); + } + + return () => { + if ('speechSynthesis' in window) { + window.speechSynthesis.removeEventListener('voiceschanged', loadVoices); + } + }; + }, []); + + const loadVoices = () => { + if ('speechSynthesis' in window) { + const availableVoices = window.speechSynthesis.getVoices(); + const englishVoices = availableVoices.filter(voice => + voice.lang.startsWith('en') + ); + setVoices(englishVoices); + + if (audioService.settings.voice) { + setSelectedVoice(audioService.settings.voice.name); + } + } + }; + + const loadBackendTTSSettings = async () => { + try { + const response = await fetch('/api/settings/tts', { + headers: { + 'Authorization': `Bearer ${localStorage.getItem('auth-token')}` + } + }); + + if (response.ok) { + const settings = await response.json(); + setBackendTTSSettings(settings); + setSelectedProvider(settings.provider || 'auto'); + setIsBackendTTSEnabled(settings.enabled !== false); + setEngineerName(settings.general?.engineerName || ''); + setNameChance(settings.general?.nameChance || 0.3); + + // Don't show masked API keys, but indicate if they're set + setApiKeys({ + elevenlabs: settings.elevenlabs?.apiKey === '***masked***' ? 'SET' : '', + deepgram: settings.deepgram?.apiKey === '***masked***' ? 'SET' : '', + openai: settings.openai?.apiKey === '***masked***' ? 'SET' : '', + ibm_watson: settings.ibm_watson?.apiKey === '***masked***' ? 'SET' : '' + }); + } + } catch (error) { + console.error('Error loading backend TTS settings:', error); + } + }; + + const loadTTSProviders = async () => { + try { + const response = await fetch('/api/settings/tts/providers'); + + if (response.ok) { + const providers = await response.json(); + setTTSProviders(providers); + console.log('šŸŽµ Loaded TTS providers:', providers); + } else { + console.error('Failed to load TTS providers:', response.status); + } + } catch (error) { + console.error('Error loading TTS providers:', error); + } + }; + + const handleToggleEnabled = (enabled) => { + setIsEnabled(enabled); + audioService.setEnabled(enabled); + }; + + const handleSettingChange = (key, value) => { + const newSettings = { ...settings, [key]: value }; + setSettings(newSettings); + audioService.updateSettings(newSettings); + }; + + const handleVoiceChange = (voiceName) => { + setSelectedVoice(voiceName); + const voice = voices.find(v => v.name === voiceName); + if (voice) { + audioService.updateSettings({ voice }); + } + }; + + const saveBackendTTSSettings = async () => { + if (!backendTTSSettings) return; + + try { + setIsLoading(true); + + const updatedSettings = { + ...backendTTSSettings, + enabled: isBackendTTSEnabled, + provider: selectedProvider, + general: { + ...backendTTSSettings.general, + engineerName, + nameChance + } + }; + + // Only update API keys if they're actually changed (not just 'SET') + if (apiKeys.elevenlabs && apiKeys.elevenlabs !== 'SET') { + updatedSettings.elevenlabs = { + ...updatedSettings.elevenlabs, + apiKey: apiKeys.elevenlabs + }; + } + + if (apiKeys.deepgram && apiKeys.deepgram !== 'SET') { + updatedSettings.deepgram = { + ...updatedSettings.deepgram, + apiKey: apiKeys.deepgram + }; + } + + if (apiKeys.openai && apiKeys.openai !== 'SET') { + updatedSettings.openai = { + ...updatedSettings.openai, + apiKey: apiKeys.openai + }; + } + + if (apiKeys.ibm_watson && apiKeys.ibm_watson !== 'SET') { + updatedSettings.ibm_watson = { + ...updatedSettings.ibm_watson, + apiKey: apiKeys.ibm_watson + }; + } + + const response = await fetch('/api/settings/tts', { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${localStorage.getItem('auth-token')}` + }, + body: JSON.stringify(updatedSettings) + }); + + if (response.ok) { + // Update local state with the saved settings (preserve UI state) + setBackendTTSSettings(updatedSettings); + + // Only update API key display to show 'SET' status + setApiKeys({ + elevenlabs: updatedSettings.elevenlabs?.apiKey ? 'SET' : '', + deepgram: updatedSettings.deepgram?.apiKey ? 'SET' : '', + openai: updatedSettings.openai?.apiKey ? 'SET' : '', + ibm_watson: updatedSettings.ibm_watson?.apiKey ? 'SET' : '' + }); + + console.log('āœ… Backend TTS settings saved'); + } else { + console.error('āŒ Failed to save backend TTS settings'); + } + } catch (error) { + console.error('Error saving backend TTS settings:', error); + } finally { + setIsLoading(false); + } + }; + + const handleTestNotification = async () => { + setIsLoading(true); + try { + // Save current settings first to ensure backend gets the updated voice model + if (isBackendTTSEnabled) { + await saveBackendTTSSettings(); + + // Small delay to ensure settings are saved + await new Promise(resolve => setTimeout(resolve, 500)); + + const response = await fetch('/api/settings/tts/test', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${localStorage.getItem('auth-token')}` + }, + body: JSON.stringify({ + message: selectedProvider === 'deepgram' && backendTTSSettings?.deepgram?.voiceModel + ? `Testing Deepgram ${backendTTSSettings.deepgram.voiceModel.replace('aura-', '').replace('-en', '')} voice - Audio notifications are working correctly` + : 'Backend TTS test - Audio notifications are working correctly', + provider: selectedProvider + }) + }); + + if (response.ok) { + console.log('šŸŽµ Backend TTS test initiated with updated settings'); + } else { + throw new Error('Backend TTS test failed'); + } + } else { + // Fallback to browser TTS + await audioService.testNotification(); + } + } catch (error) { + console.error('Test notification failed, trying browser TTS:', error); + await audioService.testNotification(); + } finally { + setIsLoading(false); + } + }; + + return ( +
+
+
+

+ Audio Notifications +

+
+ ); +} \ No newline at end of file diff --git a/src/components/ChatInterface.jsx b/src/components/ChatInterface.jsx index 39be27ea..ac972cbf 100755 --- a/src/components/ChatInterface.jsx +++ b/src/components/ChatInterface.jsx @@ -242,12 +242,12 @@ const MessageComponent = memo(({ message, index, prevMessage, createDiff, onFile ); })()} {message.toolInput && message.toolName !== 'Edit' && (() => { - // Debug log to see what we're dealing with - console.log('Tool display - name:', message.toolName, 'input type:', typeof message.toolInput); + // Debug log to see what we're dealing with (commented out to reduce console noise) + // console.log('Tool display - name:', message.toolName, 'input type:', typeof message.toolInput); // Special handling for Write tool if (message.toolName === 'Write') { - console.log('Write tool detected, toolInput:', message.toolInput); + // console.log('Write tool detected, toolInput:', message.toolInput); try { let input; // Handle both JSON string and already parsed object @@ -257,7 +257,7 @@ const MessageComponent = memo(({ message, index, prevMessage, createDiff, onFile input = message.toolInput; } - console.log('Parsed Write input:', input); + // console.log('Parsed Write input:', input); if (input.file_path && input.content !== undefined) { return ( @@ -1570,7 +1570,7 @@ function ChatInterface({ selectedProject, selectedSession, ws, sendMessage, mess case 'claude-status': // Handle Claude working status messages - console.log('šŸ”” Received claude-status message:', latestMessage); + // console.log('šŸ”” Received claude-status message:', latestMessage); const statusData = latestMessage.data; if (statusData) { // Parse the status message to extract relevant information @@ -1601,7 +1601,7 @@ function ChatInterface({ selectedProject, selectedSession, ws, sendMessage, mess statusInfo.can_interrupt = statusData.can_interrupt; } - console.log('šŸ“Š Setting claude status:', statusInfo); + // console.log('šŸ“Š Setting claude status:', statusInfo); setClaudeStatus(statusInfo); setIsLoading(true); setCanAbortSession(statusInfo.can_interrupt); diff --git a/src/components/QuickSettingsPanel.jsx b/src/components/QuickSettingsPanel.jsx index 6a5d0918..9bafe15a 100755 --- a/src/components/QuickSettingsPanel.jsx +++ b/src/components/QuickSettingsPanel.jsx @@ -11,10 +11,12 @@ import { Mic, Brain, Sparkles, - FileText + FileText, + Volume2 } from 'lucide-react'; import DarkModeToggle from './DarkModeToggle'; import { useTheme } from '../contexts/ThemeContext'; +import AudioNotificationSettings from './AudioNotificationSettings'; const QuickSettingsPanel = ({ isOpen, @@ -142,6 +144,12 @@ const QuickSettingsPanel = ({
+ {/* Audio Notifications */} +
+

Audio Notifications

+ +
+ {/* Whisper Dictation Settings - HIDDEN */}

Whisper Dictation

diff --git a/src/index.css b/src/index.css index b5e97ba5..d6a65087 100755 --- a/src/index.css +++ b/src/index.css @@ -434,7 +434,9 @@ /* Mobile optimizations and components */ @layer components { /* Mobile touch optimization and safe areas */ - @media (max-width: 768px) { +} + +@media (max-width: 768px) { * { touch-action: manipulation; -webkit-tap-highlight-color: transparent; @@ -512,109 +514,106 @@ opacity: 1 !important; } - @media (hover: none) and (pointer: coarse) { - .group-hover\:opacity-100 { - opacity: 1 !important; - } - - .hover\:bg-gray-50:hover, - .hover\:bg-gray-100:hover, - .hover\:bg-red-200:hover, - .dark\:hover\:bg-gray-700:hover, - .dark\:hover\:bg-red-900\/50:hover { - background-color: inherit; - } - } +} + +@media (hover: none) and (pointer: coarse) { + .group-hover\:opacity-100 { + opacity: 1 !important; } - /* Touch device optimizations for all screen sizes */ - @media (hover: none) and (pointer: coarse) { - .touch\:opacity-100 { - opacity: 1 !important; - } - - /* Completely disable hover states on touch devices */ - * { - -webkit-tap-highlight-color: transparent !important; - } - - /* Preserve checkbox visibility on touch devices */ - input[type="checkbox"] { - -webkit-tap-highlight-color: rgba(0, 0, 0, 0.1) !important; - opacity: 1 !important; - } - - /* Only disable hover states for interactive elements, not containers */ - button:hover, - [role="button"]:hover, - .cursor-pointer:hover, - a:hover, - .hover\:bg-gray-50:hover, - .hover\:bg-gray-100:hover, - .hover\:text-gray-900:hover, - .hover\:opacity-100:hover { - background-color: inherit !important; - color: inherit !important; - opacity: inherit !important; - transform: inherit !important; - } - - /* Preserve backgrounds for containers and modals */ - .fixed:hover, - .modal:hover, - .bg-white:hover, - .bg-gray-800:hover, - .bg-gray-900:hover, - [class*="bg-"]:hover { - background-color: revert !important; - } - - /* Force buttons to be immediately clickable */ - button, [role="button"], .cursor-pointer { - cursor: pointer !important; - pointer-events: auto !important; - } - - /* Keep active states for immediate feedback */ - .active\:scale-\[0\.98\]:active, - .active\:scale-95:active { - transform: scale(0.98) !important; - } + .hover\:bg-gray-50:hover, + .hover\:bg-gray-100:hover, + .hover\:bg-red-200:hover, + .dark\:hover\:bg-gray-700:hover, + .dark\:hover\:bg-red-900\/50:hover { + background-color: inherit; } - /* Safe area support for iOS devices */ - .ios-bottom-safe { - padding-bottom: max(env(safe-area-inset-bottom), 12px); + .touch\:opacity-100 { + opacity: 1 !important; } - @media screen and (max-width: 768px) { - .chat-input-mobile { - padding-bottom: calc(60px + max(env(safe-area-inset-bottom), 12px)); - } + /* Completely disable hover states on touch devices */ + * { + -webkit-tap-highlight-color: transparent !important; } - - /* Text wrapping improvements */ - .chat-message { - word-wrap: break-word; - overflow-wrap: break-word; - hyphens: auto; + + /* Preserve checkbox visibility on touch devices */ + input[type="checkbox"] { + -webkit-tap-highlight-color: rgba(0, 0, 0, 0.1) !important; + opacity: 1 !important; } - - /* Force wrap long URLs and code */ - .chat-message pre, - .chat-message code { - white-space: pre-wrap !important; - word-break: break-all; - overflow-wrap: break-word; + + /* Only disable hover states for interactive elements, not containers */ + button:hover, + [role="button"]:hover, + .cursor-pointer:hover, + a:hover, + .hover\:bg-gray-50:hover, + .hover\:bg-gray-100:hover, + .hover\:text-gray-900:hover, + .hover\:opacity-100:hover { + background-color: inherit !important; + color: inherit !important; + opacity: inherit !important; + transform: inherit !important; + } + + /* Preserve backgrounds for containers and modals */ + .fixed:hover, + .modal:hover, + .bg-white:hover, + .bg-gray-800:hover, + .bg-gray-900:hover, + [class*="bg-"]:hover { + background-color: revert !important; + } + + /* Force buttons to be immediately clickable */ + button, [role="button"], .cursor-pointer { + cursor: pointer !important; + pointer-events: auto !important; } + + /* Keep active states for immediate feedback */ + .active\:scale-\[0\.98\]:active, + .active\:scale-95:active { + transform: scale(0.98) !important; + } +} - /* Prevent horizontal scroll in chat area */ - .chat-message * { - max-width: 100%; - box-sizing: border-box; +/* Safe area support for iOS devices */ +.ios-bottom-safe { + padding-bottom: max(env(safe-area-inset-bottom), 12px); +} + +@media screen and (max-width: 768px) { + .chat-input-mobile { + padding-bottom: calc(60px + max(env(safe-area-inset-bottom), 12px)); } } +/* Text wrapping improvements */ +.chat-message { + word-wrap: break-word; + overflow-wrap: break-word; + hyphens: auto; +} + +/* Force wrap long URLs and code */ +.chat-message pre, +.chat-message code { + white-space: pre-wrap !important; + word-break: break-all; + overflow-wrap: break-word; +} + +/* Prevent horizontal scroll in chat area */ +.chat-message * { + max-width: 100%; + box-sizing: border-box; +} + /* Hide markdown backticks in prose content */ .prose code::before, .prose code::after { diff --git a/src/services/audioService.js b/src/services/audioService.js new file mode 100644 index 00000000..0f22560e --- /dev/null +++ b/src/services/audioService.js @@ -0,0 +1,525 @@ +// Audio notification service for claudecodeui +class AudioService { + constructor() { + this.context = null; + this.isEnabled = true; + this.audioUnlocked = false; + this.settings = { + volume: 0.7, + rate: 1.0, + pitch: 1.0, + voice: null, + fallbackToBeep: true + }; + this.initializeAudio(); + this.setupMobileAudioUnlock(); + } + + async initializeAudio() { + try { + // Initialize Web Audio context for beep fallback + this.context = new (window.AudioContext || window.webkitAudioContext)(); + + // Initialize speech synthesis + if ('speechSynthesis' in window) { + this.speechSynthesis = window.speechSynthesis; + + // Load voices when they become available + if (this.speechSynthesis.getVoices().length === 0) { + this.speechSynthesis.addEventListener('voiceschanged', () => { + this.loadVoices(); + }); + } else { + this.loadVoices(); + } + } + } catch (error) { + console.warn('Could not initialize audio service:', error); + } + } + + // Setup mobile audio unlock on user interaction + setupMobileAudioUnlock() { + const isMobile = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent); + + if (isMobile) { + console.log('šŸ“± Setting up mobile audio unlock listeners'); + + const unlockAudio = async (event) => { + try { + console.log('šŸ“± User interaction detected, unlocking audio...'); + + if (this.context && this.context.state === 'suspended') { + await this.context.resume(); + console.log('šŸ”Š Audio context resumed'); + } + + // Play a silent buffer to unlock audio + if (this.context) { + const buffer = this.context.createBuffer(1, 1, 22050); + const source = this.context.createBufferSource(); + source.buffer = buffer; + source.connect(this.context.destination); + source.start(); + console.log('šŸ”Š Silent buffer played to unlock audio'); + } + + // Also try to unlock speech synthesis + if (this.speechSynthesis) { + try { + // Create a very short, silent utterance to unlock speech synthesis + const silentUtterance = new SpeechSynthesisUtterance(' '); + silentUtterance.volume = 0; + silentUtterance.rate = 10; + this.speechSynthesis.speak(silentUtterance); + console.log('šŸŽ¤ Speech synthesis unlocked'); + } catch (speechError) { + console.warn('Could not unlock speech synthesis:', speechError); + } + } + + this.audioUnlocked = true; + console.log('āœ… Mobile audio fully unlocked through user interaction'); + + } catch (error) { + console.warn('Could not unlock mobile audio:', error); + } + }; + + // Add persistent event listeners (don't remove them) + document.addEventListener('touchstart', unlockAudio, { passive: true }); + document.addEventListener('touchend', unlockAudio, { passive: true }); + document.addEventListener('click', unlockAudio, { passive: true }); + document.addEventListener('keydown', unlockAudio, { passive: true }); + + // Also listen for specific UI interactions that should unlock audio + const uiSelectors = ['button', 'input', 'select', 'textarea', '[role="button"]', '[onclick]']; + uiSelectors.forEach(selector => { + document.addEventListener('click', (event) => { + if (event.target.closest(selector)) { + unlockAudio(event); + } + }, { passive: true }); + }); + } + } + + loadVoices() { + const voices = this.speechSynthesis.getVoices(); + + // Prioritized list of natural-sounding English voices + const preferredVoices = [ + // High-quality neural voices + 'Microsoft Aria Online (Natural) - English (United States)', + 'Microsoft Jenny Online (Natural) - English (United States)', + 'Microsoft Guy Online (Natural) - English (United States)', + 'Microsoft Davis Online (Natural) - English (United States)', + 'Microsoft Ana Online (Natural) - English (United States)', + + // Google voices (often high quality) + 'Google US English', + 'Google UK English Female', + 'Google UK English Male', + + // System voices (macOS) + 'Samantha', + 'Alex', + 'Victoria', + 'Karen', + 'Daniel', + + // Microsoft standard voices + 'Microsoft Zira', + 'Microsoft David', + 'Microsoft Mark', + + // Chrome/Edge voices + 'Chrome OS US English', + 'Edge Aria', + 'Edge Jenny' + ]; + + // Find the best available voice + for (const voiceName of preferredVoices) { + const voice = voices.find(v => + v.name.includes(voiceName) || v.name === voiceName + ); + if (voice && voice.lang.startsWith('en')) { + this.settings.voice = voice; + console.log(`šŸŽ¤ Selected voice: ${voice.name} (${voice.lang})`); + break; + } + } + + // Fallback: find best English voice by language preference + if (!this.settings.voice) { + const englishVoices = voices.filter(v => v.lang.startsWith('en')); + + // Prefer US English, then UK English, then any English + const prioritizedLangs = ['en-US', 'en-GB', 'en']; + for (const lang of prioritizedLangs) { + const voice = englishVoices.find(v => v.lang.startsWith(lang)); + if (voice) { + this.settings.voice = voice; + console.log(`šŸŽ¤ Fallback voice: ${voice.name} (${voice.lang})`); + break; + } + } + } + + // Last resort: use first available voice + if (!this.settings.voice && voices.length > 0) { + this.settings.voice = voices[0]; + console.log(`šŸŽ¤ Last resort voice: ${voices[0].name} (${voices[0].lang})`); + } + } + + async speak(text, options = {}) { + if (!this.isEnabled) return; + + // Ensure audio context is active for mobile + await this.ensureAudioContextActive(); + + try { + // Try speech synthesis first + if (this.speechSynthesis && this.settings.voice) { + await this.speakWithSynthesis(text, options); + } else if (this.settings.fallbackToBeep) { + // Fallback to beep notification + this.playNotificationBeep(); + } + } catch (error) { + console.warn('Speech synthesis failed:', error); + if (this.settings.fallbackToBeep) { + this.playNotificationBeep(); + } + } + } + + async speakWithSynthesis(text, options = {}) { + return new Promise((resolve, reject) => { + try { + // Cancel any ongoing speech + this.speechSynthesis.cancel(); + + const utterance = new SpeechSynthesisUtterance(text); + utterance.voice = this.settings.voice; + utterance.volume = options.volume || this.settings.volume; + utterance.rate = options.rate || this.settings.rate; + utterance.pitch = options.pitch || this.settings.pitch; + + utterance.onend = () => resolve(); + utterance.onerror = (error) => reject(error); + + this.speechSynthesis.speak(utterance); + } catch (error) { + reject(error); + } + }); + } + + playNotificationBeep(frequency = 800, duration = 200) { + if (!this.context) return; + + try { + const oscillator = this.context.createOscillator(); + const gainNode = this.context.createGain(); + + oscillator.connect(gainNode); + gainNode.connect(this.context.destination); + + oscillator.frequency.setValueAtTime(frequency, this.context.currentTime); + oscillator.type = 'sine'; + + gainNode.gain.setValueAtTime(0, this.context.currentTime); + gainNode.gain.linearRampToValueAtTime(0.3, this.context.currentTime + 0.01); + gainNode.gain.exponentialRampToValueAtTime(0.001, this.context.currentTime + duration / 1000); + + oscillator.start(this.context.currentTime); + oscillator.stop(this.context.currentTime + duration / 1000); + } catch (error) { + console.warn('Could not play notification beep:', error); + } + } + + // Handle audio notification from WebSocket + async handleAudioNotification(notification) { + if (!notification || notification.type !== 'audio-notification') return; + + const { message, messageType, voice, metadata, audioUrl, source } = notification; + + // Reduced console logging for audio notifications (commented out to reduce console noise) + // console.log(`šŸ”Š Playing audio notification: ${message}`); + // console.log(`šŸŽµ Audio source: ${source || 'browser-tts'}`); + // console.log(`šŸ“± Audio unlock status: ${this.isAudioUnlocked()}`); + // console.log(`šŸ”— Audio URL provided: ${audioUrl ? 'YES' : 'NO'}`); + // console.log(`šŸ“‹ Full notification data:`, { message, messageType, audioUrl, source, metadata }); + + // If we have a backend-generated audio URL, play that instead of browser TTS + if (audioUrl && source === 'backend-tts') { + try { + // console.log(`šŸŽµ Playing backend TTS audio: ${audioUrl}`); + await this.playStreamedAudio(audioUrl, notification); + // console.log(`āœ… Backend TTS audio played successfully`); + return; + } catch (error) { + console.warn('šŸ”„ Backend audio failed, falling back to browser TTS:', error); + console.error('šŸ”„ Error details:', error); + // Continue to browser TTS fallback below + } + } + + // Fallback to browser TTS synthesis + // console.log('šŸŽ¤ Using browser TTS synthesis'); + + // Add contextual information based on message type + let enhancedMessage = message; + if (messageType === 'session_start' && metadata?.projectPath) { + const projectName = metadata.projectPath.split('/').pop() || 'project'; + enhancedMessage = `${message} for ${projectName}`; + } + + await this.speak(enhancedMessage, { voice }); + } + + // Check if audio context is suspended (mobile autoplay restriction) + async ensureAudioContextActive() { + if (this.context && this.context.state === 'suspended') { + try { + await this.context.resume(); + console.log('šŸ”Š Audio context resumed for mobile playback'); + } catch (error) { + console.warn('Could not resume audio context:', error); + } + } + } + + // Check if audio is fully unlocked and ready for mobile playback + isAudioUnlocked() { + const isMobile = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent); + + if (!isMobile) { + return true; // Desktop doesn't have autoplay restrictions for notifications + } + + const contextReady = !this.context || this.context.state === 'running'; + const audioUnlocked = this.audioUnlocked; + + // console.log(`šŸ“± Mobile audio status: unlocked=${audioUnlocked}, contextReady=${contextReady}`); + + return audioUnlocked && contextReady; + } + + // Play streamed audio from backend TTS with mobile support + async playStreamedAudio(audioUrl, notification = {}) { + if (!this.isEnabled) return; + + // Ensure audio context is active for mobile + await this.ensureAudioContextActive(); + + return new Promise((resolve, reject) => { + try { + // Create audio element + const audio = new Audio(); + + // Mobile-specific settings + const isMobile = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent); + + if (isMobile) { + // Allow mobile playback + audio.setAttribute('playsinline', 'true'); + audio.muted = false; // Ensure not muted + console.log('šŸ“± Mobile device detected, applying mobile audio settings'); + } + + // Set up event handlers + audio.onloadstart = () => { + console.log('šŸŽµ Started loading backend audio...'); + }; + + audio.oncanplay = () => { + console.log('šŸŽµ Backend audio ready to play'); + }; + + audio.onended = () => { + console.log('āœ… Backend audio playback complete'); + resolve(); + }; + + audio.onerror = (error) => { + console.error('āŒ Backend audio playback error:', error); + console.error('Audio error details:', { + code: audio.error?.code, + message: audio.error?.message, + type: error.type, + target: error.target + }); + reject(new Error(`Audio playback failed: ${error.message || audio.error?.message || 'Unknown error'}`)); + }; + + audio.onabort = () => { + console.warn('āš ļø Backend audio playback aborted'); + reject(new Error('Audio playback aborted')); + }; + + // Set audio properties + audio.volume = this.settings.volume || 0.7; + audio.preload = 'auto'; + + // Handle CORS and authentication if needed + audio.crossOrigin = 'anonymous'; + + // Start loading and playing + audio.src = audioUrl; + + // Mobile-friendly play attempt with retry + const attemptPlay = async () => { + try { + // If audio is not unlocked on mobile, try to unlock it first + if (isMobile && !this.audioUnlocked) { + console.log('šŸ“± Audio not unlocked yet, attempting unlock before playback...'); + await this.ensureAudioContextActive(); + + // Try to unlock with a silent buffer + try { + if (this.context) { + const buffer = this.context.createBuffer(1, 1, 22050); + const source = this.context.createBufferSource(); + source.buffer = buffer; + source.connect(this.context.destination); + source.start(); + this.audioUnlocked = true; + console.log('šŸ”Š Audio unlocked before playback attempt'); + } + } catch (unlockError) { + console.warn('Could not pre-unlock audio:', unlockError); + } + } + + const playPromise = audio.play(); + + if (playPromise !== undefined) { + await playPromise; + console.log('šŸŽµ Backend audio started playing successfully'); + } + } catch (playError) { + console.error('āŒ Backend audio play() failed:', playError); + + // For mobile, try multiple fallback approaches + if (isMobile && (playError.name === 'NotAllowedError' || playError.name === 'NotSupportedError')) { + console.log('šŸ“± Mobile autoplay blocked, trying fallback methods...'); + + // First, try browser TTS as it might work better on mobile + try { + console.log('šŸ”„ Falling back to browser TTS for mobile...'); + const message = notification.message || 'Notification'; + await this.speak(message); + resolve(); + return; + } catch (ttsError) { + console.warn('šŸ”„ Browser TTS also failed, trying beep fallback'); + } + + // Last resort: beep notification + try { + this.playNotificationBeep(); + console.log('šŸ”Š Played beep notification as final fallback'); + resolve(); + } catch (beepError) { + console.warn('šŸ”„ Even beep notification failed:', beepError); + resolve(); // Don't reject, just complete silently + } + } else { + reject(playError); + } + } + }; + + // Start play attempt + attemptPlay(); + + // Cleanup function + const cleanup = () => { + audio.onloadstart = null; + audio.oncanplay = null; + audio.onended = null; + audio.onerror = null; + audio.onabort = null; + audio.src = ''; + }; + + // Set up cleanup on completion + audio.addEventListener('ended', cleanup, { once: true }); + audio.addEventListener('error', cleanup, { once: true }); + audio.addEventListener('abort', cleanup, { once: true }); + + } catch (error) { + console.error('āŒ Error setting up backend audio playback:', error); + reject(error); + } + }); + } + + // Settings management + setEnabled(enabled) { + this.isEnabled = enabled; + localStorage.setItem('audioNotifications.enabled', enabled.toString()); + } + + getEnabled() { + const stored = localStorage.getItem('audioNotifications.enabled'); + return stored !== null ? stored === 'true' : this.isEnabled; + } + + updateSettings(newSettings) { + this.settings = { ...this.settings, ...newSettings }; + localStorage.setItem('audioNotifications.settings', JSON.stringify(this.settings)); + } + + loadSettings() { + const stored = localStorage.getItem('audioNotifications.settings'); + if (stored) { + try { + this.settings = { ...this.settings, ...JSON.parse(stored) }; + } catch (error) { + console.warn('Could not load audio settings:', error); + } + } + this.isEnabled = this.getEnabled(); + } + + // Test audio notification + async testNotification() { + // Try to test with backend TTS first + try { + const response = await fetch('/api/audio/test-notification', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + message: 'Audio notifications are working correctly', + messageType: 'test' + }) + }); + + if (response.ok) { + console.log('šŸŽµ Backend TTS test initiated'); + return; // Backend will handle the notification via WebSocket + } + } catch (error) { + console.warn('šŸ”„ Backend TTS test failed, using browser TTS:', error); + } + + // Fallback to browser TTS + await this.speak('Audio notifications are working correctly'); + } +} + +// Create singleton instance +export const audioService = new AudioService(); + +// Load settings on initialization +audioService.loadSettings(); + +export default audioService; \ No newline at end of file diff --git a/src/utils/websocket.js b/src/utils/websocket.js index f03fd002..272ed39c 100755 --- a/src/utils/websocket.js +++ b/src/utils/websocket.js @@ -1,4 +1,5 @@ import { useState, useEffect, useRef } from 'react'; +import { audioService } from '../services/audioService'; export function useWebSocket() { const [ws, setWs] = useState(null); @@ -67,6 +68,12 @@ export function useWebSocket() { websocket.onmessage = (event) => { try { const data = JSON.parse(event.data); + + // Handle audio notifications + if (data.type === 'audio-notification') { + audioService.handleAudioNotification(data); + } + setMessages(prev => [...prev, data]); } catch (error) { console.error('Error parsing WebSocket message:', error); diff --git a/test-audio.js b/test-audio.js new file mode 100644 index 00000000..69f64ae8 --- /dev/null +++ b/test-audio.js @@ -0,0 +1,49 @@ +// Simple test script for audio streaming functionality +import fetch from 'node-fetch'; + +async function testTTSProviders() { + try { + console.log('Testing TTS providers endpoint...'); + const response = await fetch('http://localhost:3000/api/settings/tts/providers'); + + if (response.ok) { + const providers = await response.json(); + console.log('āœ… TTS Providers loaded successfully:'); + console.log(JSON.stringify(providers, null, 2)); + } else { + console.error(`āŒ Failed to load TTS providers: ${response.status} ${response.statusText}`); + const errorText = await response.text(); + console.error('Error response:', errorText); + } + } catch (error) { + console.error('āŒ Network error:', error.message); + } +} + +async function testAudioEndpoints() { + try { + console.log('\nTesting audio info endpoint...'); + const response = await fetch('http://localhost:3000/api/audio/info'); + + if (response.ok) { + const info = await response.json(); + console.log('āœ… Audio info loaded successfully:'); + console.log(JSON.stringify(info, null, 2)); + } else { + console.error(`āŒ Failed to load audio info: ${response.status} ${response.statusText}`); + } + } catch (error) { + console.error('āŒ Network error:', error.message); + } +} + +async function runTests() { + console.log('šŸŽµ Audio Streaming System Test\n'); + + await testTTSProviders(); + await testAudioEndpoints(); + + console.log('\nšŸ Test completed'); +} + +runTests().catch(console.error); \ No newline at end of file