cheykrym/glass
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

windows

Browse Source
This commit is contained in:
unknown 2025-07-07 18:38:22 +01:00
parent 6a0d8d35c4
commit 8db58dcb1e
7 changed files with 158 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
electron-builder.yml
View File

@ -44,6 +44,8 @@ win:
- target: portable
arch: x64
requestedExecutionLevel: asInvoker
# Disable code signing to avoid symbolic link issues on Windows
signAndEditExecutable: false
# NSIS installer configuration for Windows
nsis:

8
package.json
View File

@ -9,12 +9,14 @@
"start": "npm run build:renderer && electron-forge start",
"package": "npm run build:renderer && electron-forge package",
"make": "npm run build:renderer && electron-forge make",
"build": "npm run build:renderer && electron-builder --config electron-builder.yml --publish never",
"build:win": "npm run build:renderer && electron-builder --win --x64 --publish never",
"publish": "npm run build:renderer && electron-builder --config electron-builder.yml --publish always",
"build": "npm run build:all && electron-builder --config electron-builder.yml --publish never",
"build:win": "npm run build:all && electron-builder --win --x64 --publish never",
"publish": "npm run build:all && electron-builder --config electron-builder.yml --publish always",
"lint": "eslint --ext .ts,.tsx,.js .",
"postinstall": "electron-builder install-app-deps",
"build:renderer": "node build.js",
"build:web": "cd pickleglass_web && npm run build && cd ..",
"build:all": "npm run build:renderer && npm run build:web",
"watch:renderer": "node build.js --watch"
},
"keywords": [

14
src/features/listen/listenService.js
View File

@ -219,6 +219,20 @@ class ListenService {
}
});
ipcMain.handle('send-system-audio-content', async (event, { data, mimeType }) => {
try {
await this.sttService.sendSystemAudioContent(data, mimeType);
// Send system audio data back to renderer for AEC reference (like macOS does)
this.sendToRenderer('system-audio-data', { data });
return { success: true };
} catch (error) {
console.error('Error sending system audio:', error);
return { success: false, error: error.message };
}
});
ipcMain.handle('start-macos-audio', async () => {
if (process.platform !== 'darwin') {
return { success: false, error: 'macOS audio capture only available on macOS' };

116
src/features/listen/renderer/listenCapture.js
View File

@ -15,6 +15,8 @@ let micMediaStream = null;
let screenshotInterval = null;
let audioContext = null;
let audioProcessor = null;
let systemAudioContext = null;
let systemAudioProcessor = null;
let currentImageQuality = 'medium';
let lastScreenshotBase64 = null;
@ -345,6 +347,7 @@ function setupMicProcessing(micStream) {
micProcessor.connect(micAudioContext.destination);
audioProcessor = micProcessor;
return { context: micAudioContext, processor: micProcessor };
}
function setupLinuxMicProcessing(micStream) {
@ -380,34 +383,40 @@ function setupLinuxMicProcessing(micStream) {
audioProcessor = micProcessor;
}
function setupWindowsLoopbackProcessing() {
// Setup audio processing for Windows loopback audio only
audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
const source = audioContext.createMediaStreamSource(mediaStream);
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
function setupSystemAudioProcessing(systemStream) {
const systemAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
const systemSource = systemAudioContext.createMediaStreamSource(systemStream);
const systemProcessor = systemAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
let audioBuffer = [];
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
audioProcessor.onaudioprocess = async e => {
systemProcessor.onaudioprocess = async e => {
const inputData = e.inputBuffer.getChannelData(0);
if (!inputData || inputData.length === 0) return;
audioBuffer.push(...inputData);
// Process audio in chunks
while (audioBuffer.length >= samplesPerChunk) {
const chunk = audioBuffer.splice(0, samplesPerChunk);
const pcmData16 = convertFloat32ToInt16(chunk);
const base64Data = arrayBufferToBase64(pcmData16.buffer);
await ipcRenderer.invoke('send-audio-content', {
data: base64Data,
mimeType: 'audio/pcm;rate=24000',
});
try {
await ipcRenderer.invoke('send-system-audio-content', {
data: base64Data,
mimeType: 'audio/pcm;rate=24000',
});
} catch (error) {
console.error('Failed to send system audio:', error);
}
}
};
source.connect(audioProcessor);
audioProcessor.connect(audioContext.destination);
systemSource.connect(systemProcessor);
systemProcessor.connect(systemAudioContext.destination);
return { context: systemAudioContext, processor: systemProcessor };
}
// ---------------------------
@ -534,7 +543,9 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
});
console.log('macOS microphone capture started');
setupMicProcessing(micMediaStream);
const { context, processor } = setupMicProcessing(micMediaStream);
audioContext = context;
audioProcessor = processor;
} catch (micErr) {
console.warn('Failed to get microphone on macOS:', micErr);
}
@ -577,27 +588,62 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
console.log('Linux screen capture started');
} else {
// Windows - use display media for audio, main process for screenshots
// Windows - capture mic and system audio separately using native loopback
console.log('Starting Windows capture with native loopback audio...');
// Start screen capture in main process for screenshots
const screenResult = await ipcRenderer.invoke('start-screen-capture');
if (!screenResult.success) {
throw new Error('Failed to start screen capture: ' + screenResult.error);
}
mediaStream = await navigator.mediaDevices.getDisplayMedia({
video: false, // We don't need video in renderer
audio: {
sampleRate: SAMPLE_RATE,
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
// Ensure STT sessions are initialized before starting audio capture
const sessionActive = await ipcRenderer.invoke('is-session-active');
if (!sessionActive) {
throw new Error('STT sessions not initialized - please wait for initialization to complete');
}
console.log('Windows capture started with loopback audio');
// 1. Get user's microphone
try {
micMediaStream = await navigator.mediaDevices.getUserMedia({
audio: {
sampleRate: SAMPLE_RATE,
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
video: false,
});
console.log('Windows microphone capture started');
const { context, processor } = setupMicProcessing(micMediaStream);
audioContext = context;
audioProcessor = processor;
} catch (micErr) {
console.warn('Could not get microphone access on Windows:', micErr);
}
// Setup audio processing for Windows loopback audio only
setupWindowsLoopbackProcessing();
// 2. Get system audio using native Electron loopback
try {
mediaStream = await navigator.mediaDevices.getDisplayMedia({
video: true,
audio: true // This will now use native loopback from our handler
});
// Verify we got audio tracks
const audioTracks = mediaStream.getAudioTracks();
if (audioTracks.length === 0) {
throw new Error('No audio track in native loopback stream');
}
console.log('Windows native loopback audio capture started');
const { context, processor } = setupSystemAudioProcessing(mediaStream);
systemAudioContext = context;
systemAudioProcessor = processor;
} catch (sysAudioErr) {
console.error('Failed to start Windows native loopback audio:', sysAudioErr);
// Continue without system audio
}
}
// Start capturing screenshots - check if manual mode
@ -626,21 +672,31 @@ function stopCapture() {
screenshotInterval = null;
}
// Clean up microphone resources
if (audioProcessor) {
audioProcessor.disconnect();
audioProcessor = null;
}
if (audioContext) {
audioContext.close();
audioContext = null;
}
// Clean up system audio resources
if (systemAudioProcessor) {
systemAudioProcessor.disconnect();
systemAudioProcessor = null;
}
if (systemAudioContext) {
systemAudioContext.close();
systemAudioContext = null;
}
// Stop and release media stream tracks
if (mediaStream) {
mediaStream.getTracks().forEach(track => track.stop());
mediaStream = null;
}
if (micMediaStream) {
micMediaStream.getTracks().forEach(t => t.stop());
micMediaStream = null;

15
src/features/listen/stt/sttService.js
View File

@ -300,6 +300,21 @@ class SttService {
await this.mySttSession.sendRealtimeInput(payload);
}
async sendSystemAudioContent(data, mimeType) {
const provider = await this.getAiProvider();
const isGemini = provider === 'gemini';
if (!this.theirSttSession) {
throw new Error('Their STT session not active');
}
const payload = isGemini
? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } }
: data;
await this.theirSttSession.sendRealtimeInput(payload);
}
killExistingSystemAudioDump() {
return new Promise(resolve => {
console.log('Checking for existing SystemAudioDump processes...');

46
src/features/listen/summary/repositories/sqlite.repository.js
View File

@ -1,28 +1,30 @@
const sqliteClient = require('../../../../common/services/sqliteClient');
function saveSummary({ sessionId, tldr, text, bullet_json, action_json, model = 'gpt-4.1' }) {
const db = sqliteClient.getDb();
const now = Math.floor(Date.now() / 1000);
const query = `
INSERT INTO summaries (session_id, generated_at, model, text, tldr, bullet_json, action_json, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(session_id) DO UPDATE SET
generated_at=excluded.generated_at,
model=excluded.model,
text=excluded.text,
tldr=excluded.tldr,
bullet_json=excluded.bullet_json,
action_json=excluded.action_json,
updated_at=excluded.updated_at
`;
try {
const result = db.prepare(query).run(sessionId, now, model, text, tldr, bullet_json, action_json, now);
return { changes: result.changes };
} catch (err) {
console.error('Error saving summary:', err);
throw err;
}
return new Promise((resolve, reject) => {
try {
const db = sqliteClient.getDb();
const now = Math.floor(Date.now() / 1000);
const query = `
INSERT INTO summaries (session_id, generated_at, model, text, tldr, bullet_json, action_json, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(session_id) DO UPDATE SET
generated_at=excluded.generated_at,
model=excluded.model,
text=excluded.text,
tldr=excluded.tldr,
bullet_json=excluded.bullet_json,
action_json=excluded.action_json,
updated_at=excluded.updated_at
`;
const result = db.prepare(query).run(sessionId, now, model, text, tldr, bullet_json, action_json, now);
resolve({ changes: result.changes });
} catch (err) {
console.error('Error saving summary:', err);
reject(err);
}
});
}
function getSummaryBySessionId(sessionId) {

13
src/index.js
View File

@ -11,7 +11,7 @@ if (require('electron-squirrel-startup')) {
process.exit(0);
}
const { app, BrowserWindow, shell, ipcMain, dialog } = require('electron');
const { app, BrowserWindow, shell, ipcMain, dialog, desktopCapturer, session } = require('electron');
const { createWindows } = require('./electron/windowManager.js');
const ListenService = require('./features/listen/listenService');
const { initializeFirebase } = require('./common/services/firebaseClient');
@ -162,6 +162,17 @@ setupProtocolHandling();
app.whenReady().then(async () => {
// Setup native loopback audio capture for Windows
session.defaultSession.setDisplayMediaRequestHandler((request, callback) => {
desktopCapturer.getSources({ types: ['screen'] }).then((sources) => {
// Grant access to the first screen found with loopback audio
callback({ video: sources[0], audio: 'loopback' });
}).catch((error) => {
console.error('Failed to get desktop capturer sources:', error);
callback({});
});
});
// Initialize core services
initializeFirebase();