From 8db58dcb1e4b71d0cc3f9fca424e1f31c50238ff Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 7 Jul 2025 18:38:22 +0100 Subject: [PATCH] windows --- electron-builder.yml | 2 + package.json | 8 +- src/features/listen/listenService.js | 14 +++ src/features/listen/renderer/listenCapture.js | 116 +++++++++++++----- src/features/listen/stt/sttService.js | 15 +++ .../summary/repositories/sqlite.repository.js | 46 +++---- src/index.js | 13 +- 7 files changed, 158 insertions(+), 56 deletions(-) diff --git a/electron-builder.yml b/electron-builder.yml index 35e6ed5..79b81fb 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -44,6 +44,8 @@ win: - target: portable arch: x64 requestedExecutionLevel: asInvoker + # Disable code signing to avoid symbolic link issues on Windows + signAndEditExecutable: false # NSIS installer configuration for Windows nsis: diff --git a/package.json b/package.json index b399345..581acde 100644 --- a/package.json +++ b/package.json @@ -9,12 +9,14 @@ "start": "npm run build:renderer && electron-forge start", "package": "npm run build:renderer && electron-forge package", "make": "npm run build:renderer && electron-forge make", - "build": "npm run build:renderer && electron-builder --config electron-builder.yml --publish never", - "build:win": "npm run build:renderer && electron-builder --win --x64 --publish never", - "publish": "npm run build:renderer && electron-builder --config electron-builder.yml --publish always", + "build": "npm run build:all && electron-builder --config electron-builder.yml --publish never", + "build:win": "npm run build:all && electron-builder --win --x64 --publish never", + "publish": "npm run build:all && electron-builder --config electron-builder.yml --publish always", "lint": "eslint --ext .ts,.tsx,.js .", "postinstall": "electron-builder install-app-deps", "build:renderer": "node build.js", + "build:web": "cd pickleglass_web && npm run build && cd ..", + "build:all": "npm run build:renderer && npm run build:web", "watch:renderer": "node build.js --watch" }, "keywords": [ diff --git a/src/features/listen/listenService.js b/src/features/listen/listenService.js index a0dcde8..3b89aa2 100644 --- a/src/features/listen/listenService.js +++ b/src/features/listen/listenService.js @@ -219,6 +219,20 @@ class ListenService { } }); + ipcMain.handle('send-system-audio-content', async (event, { data, mimeType }) => { + try { + await this.sttService.sendSystemAudioContent(data, mimeType); + + // Send system audio data back to renderer for AEC reference (like macOS does) + this.sendToRenderer('system-audio-data', { data }); + + return { success: true }; + } catch (error) { + console.error('Error sending system audio:', error); + return { success: false, error: error.message }; + } + }); + ipcMain.handle('start-macos-audio', async () => { if (process.platform !== 'darwin') { return { success: false, error: 'macOS audio capture only available on macOS' }; diff --git a/src/features/listen/renderer/listenCapture.js b/src/features/listen/renderer/listenCapture.js index a4eef43..1c27f75 100644 --- a/src/features/listen/renderer/listenCapture.js +++ b/src/features/listen/renderer/listenCapture.js @@ -15,6 +15,8 @@ let micMediaStream = null; let screenshotInterval = null; let audioContext = null; let audioProcessor = null; +let systemAudioContext = null; +let systemAudioProcessor = null; let currentImageQuality = 'medium'; let lastScreenshotBase64 = null; @@ -345,6 +347,7 @@ function setupMicProcessing(micStream) { micProcessor.connect(micAudioContext.destination); audioProcessor = micProcessor; + return { context: micAudioContext, processor: micProcessor }; } function setupLinuxMicProcessing(micStream) { @@ -380,34 +383,40 @@ function setupLinuxMicProcessing(micStream) { audioProcessor = micProcessor; } -function setupWindowsLoopbackProcessing() { - // Setup audio processing for Windows loopback audio only - audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); - const source = audioContext.createMediaStreamSource(mediaStream); - audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); +function setupSystemAudioProcessing(systemStream) { + const systemAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); + const systemSource = systemAudioContext.createMediaStreamSource(systemStream); + const systemProcessor = systemAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); let audioBuffer = []; const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION; - audioProcessor.onaudioprocess = async e => { + systemProcessor.onaudioprocess = async e => { const inputData = e.inputBuffer.getChannelData(0); + if (!inputData || inputData.length === 0) return; + audioBuffer.push(...inputData); - // Process audio in chunks while (audioBuffer.length >= samplesPerChunk) { const chunk = audioBuffer.splice(0, samplesPerChunk); const pcmData16 = convertFloat32ToInt16(chunk); const base64Data = arrayBufferToBase64(pcmData16.buffer); - await ipcRenderer.invoke('send-audio-content', { - data: base64Data, - mimeType: 'audio/pcm;rate=24000', - }); + try { + await ipcRenderer.invoke('send-system-audio-content', { + data: base64Data, + mimeType: 'audio/pcm;rate=24000', + }); + } catch (error) { + console.error('Failed to send system audio:', error); + } } }; - source.connect(audioProcessor); - audioProcessor.connect(audioContext.destination); + systemSource.connect(systemProcessor); + systemProcessor.connect(systemAudioContext.destination); + + return { context: systemAudioContext, processor: systemProcessor }; } // --------------------------- @@ -534,7 +543,9 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu }); console.log('macOS microphone capture started'); - setupMicProcessing(micMediaStream); + const { context, processor } = setupMicProcessing(micMediaStream); + audioContext = context; + audioProcessor = processor; } catch (micErr) { console.warn('Failed to get microphone on macOS:', micErr); } @@ -577,27 +588,62 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu console.log('Linux screen capture started'); } else { - // Windows - use display media for audio, main process for screenshots + // Windows - capture mic and system audio separately using native loopback + console.log('Starting Windows capture with native loopback audio...'); + + // Start screen capture in main process for screenshots const screenResult = await ipcRenderer.invoke('start-screen-capture'); if (!screenResult.success) { throw new Error('Failed to start screen capture: ' + screenResult.error); } - mediaStream = await navigator.mediaDevices.getDisplayMedia({ - video: false, // We don't need video in renderer - audio: { - sampleRate: SAMPLE_RATE, - channelCount: 1, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - }, - }); + // Ensure STT sessions are initialized before starting audio capture + const sessionActive = await ipcRenderer.invoke('is-session-active'); + if (!sessionActive) { + throw new Error('STT sessions not initialized - please wait for initialization to complete'); + } - console.log('Windows capture started with loopback audio'); + // 1. Get user's microphone + try { + micMediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + sampleRate: SAMPLE_RATE, + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + video: false, + }); + console.log('Windows microphone capture started'); + const { context, processor } = setupMicProcessing(micMediaStream); + audioContext = context; + audioProcessor = processor; + } catch (micErr) { + console.warn('Could not get microphone access on Windows:', micErr); + } - // Setup audio processing for Windows loopback audio only - setupWindowsLoopbackProcessing(); + // 2. Get system audio using native Electron loopback + try { + mediaStream = await navigator.mediaDevices.getDisplayMedia({ + video: true, + audio: true // This will now use native loopback from our handler + }); + + // Verify we got audio tracks + const audioTracks = mediaStream.getAudioTracks(); + if (audioTracks.length === 0) { + throw new Error('No audio track in native loopback stream'); + } + + console.log('Windows native loopback audio capture started'); + const { context, processor } = setupSystemAudioProcessing(mediaStream); + systemAudioContext = context; + systemAudioProcessor = processor; + } catch (sysAudioErr) { + console.error('Failed to start Windows native loopback audio:', sysAudioErr); + // Continue without system audio + } } // Start capturing screenshots - check if manual mode @@ -626,21 +672,31 @@ function stopCapture() { screenshotInterval = null; } + // Clean up microphone resources if (audioProcessor) { audioProcessor.disconnect(); audioProcessor = null; } - if (audioContext) { audioContext.close(); audioContext = null; } + // Clean up system audio resources + if (systemAudioProcessor) { + systemAudioProcessor.disconnect(); + systemAudioProcessor = null; + } + if (systemAudioContext) { + systemAudioContext.close(); + systemAudioContext = null; + } + + // Stop and release media stream tracks if (mediaStream) { mediaStream.getTracks().forEach(track => track.stop()); mediaStream = null; } - if (micMediaStream) { micMediaStream.getTracks().forEach(t => t.stop()); micMediaStream = null; diff --git a/src/features/listen/stt/sttService.js b/src/features/listen/stt/sttService.js index 294bd74..1206d5f 100644 --- a/src/features/listen/stt/sttService.js +++ b/src/features/listen/stt/sttService.js @@ -300,6 +300,21 @@ class SttService { await this.mySttSession.sendRealtimeInput(payload); } + async sendSystemAudioContent(data, mimeType) { + const provider = await this.getAiProvider(); + const isGemini = provider === 'gemini'; + + if (!this.theirSttSession) { + throw new Error('Their STT session not active'); + } + + const payload = isGemini + ? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } } + : data; + + await this.theirSttSession.sendRealtimeInput(payload); + } + killExistingSystemAudioDump() { return new Promise(resolve => { console.log('Checking for existing SystemAudioDump processes...'); diff --git a/src/features/listen/summary/repositories/sqlite.repository.js b/src/features/listen/summary/repositories/sqlite.repository.js index b365090..008aa21 100644 --- a/src/features/listen/summary/repositories/sqlite.repository.js +++ b/src/features/listen/summary/repositories/sqlite.repository.js @@ -1,28 +1,30 @@ const sqliteClient = require('../../../../common/services/sqliteClient'); function saveSummary({ sessionId, tldr, text, bullet_json, action_json, model = 'gpt-4.1' }) { - const db = sqliteClient.getDb(); - const now = Math.floor(Date.now() / 1000); - const query = ` - INSERT INTO summaries (session_id, generated_at, model, text, tldr, bullet_json, action_json, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(session_id) DO UPDATE SET - generated_at=excluded.generated_at, - model=excluded.model, - text=excluded.text, - tldr=excluded.tldr, - bullet_json=excluded.bullet_json, - action_json=excluded.action_json, - updated_at=excluded.updated_at - `; - - try { - const result = db.prepare(query).run(sessionId, now, model, text, tldr, bullet_json, action_json, now); - return { changes: result.changes }; - } catch (err) { - console.error('Error saving summary:', err); - throw err; - } + return new Promise((resolve, reject) => { + try { + const db = sqliteClient.getDb(); + const now = Math.floor(Date.now() / 1000); + const query = ` + INSERT INTO summaries (session_id, generated_at, model, text, tldr, bullet_json, action_json, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + generated_at=excluded.generated_at, + model=excluded.model, + text=excluded.text, + tldr=excluded.tldr, + bullet_json=excluded.bullet_json, + action_json=excluded.action_json, + updated_at=excluded.updated_at + `; + + const result = db.prepare(query).run(sessionId, now, model, text, tldr, bullet_json, action_json, now); + resolve({ changes: result.changes }); + } catch (err) { + console.error('Error saving summary:', err); + reject(err); + } + }); } function getSummaryBySessionId(sessionId) { diff --git a/src/index.js b/src/index.js index 52c057f..340e91b 100644 --- a/src/index.js +++ b/src/index.js @@ -11,7 +11,7 @@ if (require('electron-squirrel-startup')) { process.exit(0); } -const { app, BrowserWindow, shell, ipcMain, dialog } = require('electron'); +const { app, BrowserWindow, shell, ipcMain, dialog, desktopCapturer, session } = require('electron'); const { createWindows } = require('./electron/windowManager.js'); const ListenService = require('./features/listen/listenService'); const { initializeFirebase } = require('./common/services/firebaseClient'); @@ -162,6 +162,17 @@ setupProtocolHandling(); app.whenReady().then(async () => { + // Setup native loopback audio capture for Windows + session.defaultSession.setDisplayMediaRequestHandler((request, callback) => { + desktopCapturer.getSources({ types: ['screen'] }).then((sources) => { + // Grant access to the first screen found with loopback audio + callback({ video: sources[0], audio: 'loopback' }); + }).catch((error) => { + console.error('Failed to get desktop capturer sources:', error); + callback({}); + }); + }); + // Initialize core services initializeFirebase();