Merge remote-tracking branch 'origin/main' into pr-77

.github/workflows/build.yml

@@ -0,0 +1,34 @@
+name: Build & Verify
+
+on:
+  push:
+    branches: [ "main" ] # Runs on every push to main branch
+
+jobs:
+  build:
+    # Currently runs on macOS only, can add windows-latest later
+    runs-on: macos-latest
+
+    steps:
+      - name: 🚚 Checkout code
+        uses: actions/checkout@v4
+
+      - name: ⚙️ Setup Node.js environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20.x' # Node.js version compatible with project
+          cache: 'npm' # npm dependency caching for speed improvement
+
+      - name: 📦 Install root dependencies
+        run: npm install
+
+      - name: 🌐 Install and build web (Renderer) part
+        # Move to pickleglass_web directory and run commands
+        working-directory: ./pickleglass_web
+        run: |
+          npm install
+          npm run build
+
+      - name: 🖥️ Build Electron app
+        # Run Electron build script from root directory
+        run: npm run build

.gitignore

@@ -102,7 +102,6 @@ pickleglass_web/venv/
 node_modules/
 npm-debug.log
 yarn-error.log
-package-lock.json
 
 # Database
 data/*.db

electron-builder.yml

@@ -44,6 +44,8 @@ win:
         - target: portable
           arch: x64
     requestedExecutionLevel: asInvoker
+    # Disable code signing to avoid symbolic link issues on Windows
+    signAndEditExecutable: false
 
 # NSIS installer configuration for Windows
 nsis:

package.json

@@ -1,7 +1,7 @@
 {
     "name": "pickle-glass",
     "productName": "Glass",
-    "version": "0.2.0",
+    "version": "0.2.1",
     "description": "Cl*ely for Free",
     "main": "src/index.js",
     "scripts": {
@@ -9,12 +9,14 @@
         "start": "npm run build:renderer && electron-forge start",
         "package": "npm run build:renderer && electron-forge package",
         "make": "npm run build:renderer && electron-forge make",
-        "build": "npm run build:renderer && electron-builder --config electron-builder.yml --publish never",
-        "build:win": "npm run build:renderer && electron-builder --win --x64 --publish never", 
-        "publish": "npm run build:renderer && electron-builder --config electron-builder.yml --publish always",
+        "build": "npm run build:all && electron-builder --config electron-builder.yml --publish never",
+        "build:win": "npm run build:all && electron-builder --win --x64 --publish never", 
+        "publish": "npm run build:all && electron-builder --config electron-builder.yml --publish always",
         "lint": "eslint --ext .ts,.tsx,.js .",
         "postinstall": "electron-builder install-app-deps",
         "build:renderer": "node build.js",
+        "build:web": "cd pickleglass_web && npm run build && cd ..",
+        "build:all": "npm run build:renderer && npm run build:web",
         "watch:renderer": "node build.js --watch"
     },
     "keywords": [

src/features/listen/listenService.js

@@ -219,6 +219,20 @@ class ListenService {
             }
         });
 
+        ipcMain.handle('send-system-audio-content', async (event, { data, mimeType }) => {
+            try {
+                await this.sttService.sendSystemAudioContent(data, mimeType);
+                
+                // Send system audio data back to renderer for AEC reference (like macOS does)
+                this.sendToRenderer('system-audio-data', { data });
+                
+                return { success: true };
+            } catch (error) {
+                console.error('Error sending system audio:', error);
+                return { success: false, error: error.message };
+            }
+        });
+
         ipcMain.handle('start-macos-audio', async () => {
             if (process.platform !== 'darwin') {
                 return { success: false, error: 'macOS audio capture only available on macOS' };

src/features/listen/renderer/listenCapture.js

@@ -15,6 +15,8 @@ let micMediaStream = null;
 let screenshotInterval = null;
 let audioContext = null;
 let audioProcessor = null;
+let systemAudioContext = null;
+let systemAudioProcessor = null;
 let currentImageQuality = 'medium';
 let lastScreenshotBase64 = null;
 
@@ -345,6 +347,7 @@ function setupMicProcessing(micStream) {
     micProcessor.connect(micAudioContext.destination);
 
     audioProcessor = micProcessor;
+    return { context: micAudioContext, processor: micProcessor };
 }
 
 function setupLinuxMicProcessing(micStream) {
@@ -380,34 +383,40 @@ function setupLinuxMicProcessing(micStream) {
     audioProcessor = micProcessor;
 }
 
-function setupWindowsLoopbackProcessing() {
-    // Setup audio processing for Windows loopback audio only
-    audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
-    const source = audioContext.createMediaStreamSource(mediaStream);
-    audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
+function setupSystemAudioProcessing(systemStream) {
+    const systemAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
+    const systemSource = systemAudioContext.createMediaStreamSource(systemStream);
+    const systemProcessor = systemAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
 
     let audioBuffer = [];
     const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
 
-    audioProcessor.onaudioprocess = async e => {
+    systemProcessor.onaudioprocess = async e => {
         const inputData = e.inputBuffer.getChannelData(0);
+        if (!inputData || inputData.length === 0) return;
+        
         audioBuffer.push(...inputData);
 
-        // Process audio in chunks
         while (audioBuffer.length >= samplesPerChunk) {
             const chunk = audioBuffer.splice(0, samplesPerChunk);
             const pcmData16 = convertFloat32ToInt16(chunk);
             const base64Data = arrayBufferToBase64(pcmData16.buffer);
 
-            await ipcRenderer.invoke('send-audio-content', {
-                data: base64Data,
-                mimeType: 'audio/pcm;rate=24000',
-            });
+            try {
+                await ipcRenderer.invoke('send-system-audio-content', {
+                    data: base64Data,
+                    mimeType: 'audio/pcm;rate=24000',
+                });
+            } catch (error) {
+                console.error('Failed to send system audio:', error);
+            }
         }
     };
 
-    source.connect(audioProcessor);
-    audioProcessor.connect(audioContext.destination);
+    systemSource.connect(systemProcessor);
+    systemProcessor.connect(systemAudioContext.destination);
+
+    return { context: systemAudioContext, processor: systemProcessor };
 }
 
 // ---------------------------
@@ -534,7 +543,9 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
                 });
 
                 console.log('macOS microphone capture started');
-                setupMicProcessing(micMediaStream);
+                const { context, processor } = setupMicProcessing(micMediaStream);
+                audioContext = context;
+                audioProcessor = processor;
             } catch (micErr) {
                 console.warn('Failed to get microphone on macOS:', micErr);
             }
@@ -577,27 +588,62 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
 
             console.log('Linux screen capture started');
         } else {
-            // Windows - use display media for audio, main process for screenshots
+            // Windows - capture mic and system audio separately using native loopback
+            console.log('Starting Windows capture with native loopback audio...');
+
+            // Start screen capture in main process for screenshots
             const screenResult = await ipcRenderer.invoke('start-screen-capture');
             if (!screenResult.success) {
                 throw new Error('Failed to start screen capture: ' + screenResult.error);
             }
 
-            mediaStream = await navigator.mediaDevices.getDisplayMedia({
-                video: false, // We don't need video in renderer
-                audio: {
-                    sampleRate: SAMPLE_RATE,
-                    channelCount: 1,
-                    echoCancellation: true,
-                    noiseSuppression: true,
-                    autoGainControl: true,
-                },
-            });
+            // Ensure STT sessions are initialized before starting audio capture
+            const sessionActive = await ipcRenderer.invoke('is-session-active');
+            if (!sessionActive) {
+                throw new Error('STT sessions not initialized - please wait for initialization to complete');
+            }
 
-            console.log('Windows capture started with loopback audio');
+            // 1. Get user's microphone
+            try {
+                micMediaStream = await navigator.mediaDevices.getUserMedia({
+                    audio: {
+                        sampleRate: SAMPLE_RATE,
+                        channelCount: 1,
+                        echoCancellation: true,
+                        noiseSuppression: true,
+                        autoGainControl: true,
+                    },
+                    video: false,
+                });
+                console.log('Windows microphone capture started');
+                const { context, processor } = setupMicProcessing(micMediaStream);
+                audioContext = context;
+                audioProcessor = processor;
+            } catch (micErr) {
+                console.warn('Could not get microphone access on Windows:', micErr);
+            }
 
-            // Setup audio processing for Windows loopback audio only
-            setupWindowsLoopbackProcessing();
+            // 2. Get system audio using native Electron loopback
+            try {
+                mediaStream = await navigator.mediaDevices.getDisplayMedia({
+                    video: true,
+                    audio: true // This will now use native loopback from our handler
+                });
+                
+                // Verify we got audio tracks
+                const audioTracks = mediaStream.getAudioTracks();
+                if (audioTracks.length === 0) {
+                    throw new Error('No audio track in native loopback stream');
+                }
+                
+                console.log('Windows native loopback audio capture started');
+                const { context, processor } = setupSystemAudioProcessing(mediaStream);
+                systemAudioContext = context;
+                systemAudioProcessor = processor;
+            } catch (sysAudioErr) {
+                console.error('Failed to start Windows native loopback audio:', sysAudioErr);
+                // Continue without system audio
+            }
         }
 
         // Start capturing screenshots - check if manual mode
@@ -626,21 +672,31 @@ function stopCapture() {
         screenshotInterval = null;
     }
 
+    // Clean up microphone resources
     if (audioProcessor) {
         audioProcessor.disconnect();
         audioProcessor = null;
     }
-
     if (audioContext) {
         audioContext.close();
         audioContext = null;
     }
 
+    // Clean up system audio resources
+    if (systemAudioProcessor) {
+        systemAudioProcessor.disconnect();
+        systemAudioProcessor = null;
+    }
+    if (systemAudioContext) {
+        systemAudioContext.close();
+        systemAudioContext = null;
+    }
+
+    // Stop and release media stream tracks
     if (mediaStream) {
         mediaStream.getTracks().forEach(track => track.stop());
         mediaStream = null;
     }
-
     if (micMediaStream) {
         micMediaStream.getTracks().forEach(t => t.stop());
         micMediaStream = null;

src/features/listen/stt/sttService.js

@@ -319,6 +319,21 @@ class SttService {
         await this.mySttSession.sendRealtimeInput(payload);
     }
 
+    async sendSystemAudioContent(data, mimeType) {
+        const provider = await this.getAiProvider();
+        const isGemini = provider === 'gemini';
+
+        if (!this.theirSttSession) {
+            throw new Error('Their STT session not active');
+        }
+
+        const payload = isGemini
+            ? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } }
+            : data;
+        
+        await this.theirSttSession.sendRealtimeInput(payload);
+    }
+
     killExistingSystemAudioDump() {
         return new Promise(resolve => {
             console.log('Checking for existing SystemAudioDump processes...');

src/features/listen/summary/repositories/sqlite.repository.js

@@ -1,28 +1,30 @@
 const sqliteClient = require('../../../../common/services/sqliteClient');
 
 function saveSummary({ sessionId, tldr, text, bullet_json, action_json, model = 'gpt-4.1' }) {
-    const db = sqliteClient.getDb();
-    const now = Math.floor(Date.now() / 1000);
-    const query = `
-        INSERT INTO summaries (session_id, generated_at, model, text, tldr, bullet_json, action_json, updated_at) 
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-        ON CONFLICT(session_id) DO UPDATE SET
-            generated_at=excluded.generated_at,
-            model=excluded.model,
-            text=excluded.text,
-            tldr=excluded.tldr,
-            bullet_json=excluded.bullet_json,
-            action_json=excluded.action_json,
-            updated_at=excluded.updated_at
-    `;
+    return new Promise((resolve, reject) => {
+        try {
+            const db = sqliteClient.getDb();
+            const now = Math.floor(Date.now() / 1000);
+            const query = `
+                INSERT INTO summaries (session_id, generated_at, model, text, tldr, bullet_json, action_json, updated_at) 
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(session_id) DO UPDATE SET
+                    generated_at=excluded.generated_at,
+                    model=excluded.model,
+                    text=excluded.text,
+                    tldr=excluded.tldr,
+                    bullet_json=excluded.bullet_json,
+                    action_json=excluded.action_json,
+                    updated_at=excluded.updated_at
+            `;
             
-    try {
-        const result = db.prepare(query).run(sessionId, now, model, text, tldr, bullet_json, action_json, now);
-        return { changes: result.changes };
-    } catch (err) {
-        console.error('Error saving summary:', err);
-        throw err;
-    }
+            const result = db.prepare(query).run(sessionId, now, model, text, tldr, bullet_json, action_json, now);
+            resolve({ changes: result.changes });
+        } catch (err) {
+            console.error('Error saving summary:', err);
+            reject(err);
+        }
+    });
 }
 
 function getSummaryBySessionId(sessionId) {

src/features/listen/summary/summaryService.js

@@ -170,14 +170,18 @@ Keep all points concise and build upon previous analysis if provided.`,
             const structuredData = this.parseResponseText(responseText, this.previousAnalysisResult);
 
             if (this.currentSessionId) {
-                summaryRepository.saveSummary({
-                    sessionId: this.currentSessionId,
-                    text: responseText,
-                    tldr: structuredData.summary.join('\n'),
-                    bullet_json: JSON.stringify(structuredData.topic.bullets),
-                    action_json: JSON.stringify(structuredData.actions),
-                    model: modelInfo.model 
-                }).catch(err => console.error('[DB] Failed to save summary:', err));
+                try {
+                    summaryRepository.saveSummary({
+                        sessionId: this.currentSessionId,
+                        text: responseText,
+                        tldr: structuredData.summary.join('\n'),
+                        bullet_json: JSON.stringify(structuredData.topic.bullets),
+                        action_json: JSON.stringify(structuredData.actions),
+                        model: modelInfo.model
+                    });
+                } catch (err) {
+                    console.error('[DB] Failed to save summary:', err);
+                }
             }
 
             // 분석 결과 저장

src/index.js

@@ -11,7 +11,7 @@ if (require('electron-squirrel-startup')) {
     process.exit(0);
 }
 
-const { app, BrowserWindow, shell, ipcMain, dialog } = require('electron');
+const { app, BrowserWindow, shell, ipcMain, dialog, desktopCapturer, session } = require('electron');
 const { createWindows } = require('./electron/windowManager.js');
 const ListenService = require('./features/listen/listenService');
 const { initializeFirebase } = require('./common/services/firebaseClient');
@@ -168,15 +168,26 @@ setupProtocolHandling();
 
 app.whenReady().then(async () => {
 
+    // Setup native loopback audio capture for Windows
+    session.defaultSession.setDisplayMediaRequestHandler((request, callback) => {
+        desktopCapturer.getSources({ types: ['screen'] }).then((sources) => {
+            // Grant access to the first screen found with loopback audio
+            callback({ video: sources[0], audio: 'loopback' });
+        }).catch((error) => {
+            console.error('Failed to get desktop capturer sources:', error);
+            callback({});
+        });
+    });
+
     // Initialize core services
     initializeFirebase();
     
-    databaseInitializer.initialize()
-        .then(() => {
-            console.log('>>> [index.js] Database initialized successfully');
+    try {
+        await databaseInitializer.initialize();
+        console.log('>>> [index.js] Database initialized successfully');
         
-            // Clean up zombie sessions from previous runs first
-            sessionRepository.endAllActiveSessions();
+        // Clean up zombie sessions from previous runs first
+        sessionRepository.endAllActiveSessions();
 
             authService.initialize();
             //////// after_modelStateService ////////
@@ -191,10 +202,20 @@ app.whenReady().then(async () => {
             console.error('>>> [index.js] Database initialization failed - some features may not work', err);
         });
 
-    WEB_PORT = await startWebStack();
-    console.log('Web front-end listening on', WEB_PORT);
+        // Start web server and create windows ONLY after all initializations are successful
+        WEB_PORT = await startWebStack();
+        console.log('Web front-end listening on', WEB_PORT);
         
-    createWindows();
+        createWindows();
+
+    } catch (err) {
+        console.error('>>> [index.js] Database initialization failed - some features may not work', err);
+        // Optionally, show an error dialog to the user
+        dialog.showErrorBox(
+            'Application Error',
+            'A critical error occurred during startup. Some features might be disabled. Please restart the application.'
+        );
+    }
 
     initAutoUpdater();