From e86c2db4647c2664b8940b7184467b32b3621714 Mon Sep 17 00:00:00 2001 From: sanio Date: Fri, 11 Jul 2025 02:39:12 +0900 Subject: [PATCH] toggle listen button logic improved --- package-lock.json | 35 ----- src/app/PickleGlassApp.js | 138 +++++------------- src/electron/windowManager.js | 59 ++++---- src/features/listen/listenService.js | 3 + src/features/listen/renderer/listenCapture.js | 24 ++- src/features/listen/renderer/renderer.js | 130 ++--------------- src/features/listen/summary/SummaryView.js | 4 +- src/features/listen/summary/summaryService.js | 19 +-- src/index.js | 15 -- 9 files changed, 103 insertions(+), 324 deletions(-) diff --git a/package-lock.json b/package-lock.json index f3d0c5e..8f48150 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4172,28 +4172,6 @@ "node": ">= 8" } }, - "node_modules/fs-temp": { - "version": "1.2.1", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "random-path": "^0.1.0" - } - }, - "node_modules/fs-xattr": { - "version": "0.3.1", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "!win32" - ], - "engines": { - "node": ">=8.6.0" - } - }, "node_modules/fs.realpath": { "version": "1.0.0", "dev": true, @@ -5158,19 +5136,6 @@ "lru-cache": "6.0.0" } }, - "node_modules/macos-alias": { - "version": "0.2.12", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "dependencies": { - "nan": "^2.4.0" - } - }, "node_modules/make-fetch-happen": { "version": "10.2.1", "dev": true, diff --git a/src/app/PickleGlassApp.js b/src/app/PickleGlassApp.js index 2f51a11..b5aa9d0 100644 --- a/src/app/PickleGlassApp.js +++ b/src/app/PickleGlassApp.js @@ -68,12 +68,7 @@ export class PickleGlassApp extends LitElement { this.selectedScreenshotInterval = localStorage.getItem('selectedScreenshotInterval') || '5'; this.selectedImageQuality = localStorage.getItem('selectedImageQuality') || 'medium'; this._isClickThrough = false; - this.outlines = []; - this.analysisRequests = []; - window.pickleGlass.setStructuredData = data => { - this.updateStructuredData(data); - }; } connectedCallback() { @@ -82,14 +77,13 @@ export class PickleGlassApp extends LitElement { if (window.require) { const { ipcRenderer } = window.require('electron'); - ipcRenderer.on('update-status', (_, status) => this.setStatus(status)); ipcRenderer.on('click-through-toggled', (_, isEnabled) => { this._isClickThrough = isEnabled; }); - ipcRenderer.on('start-listening-session', () => { - console.log('Received start-listening-session command, calling handleListenClick.'); - this.handleListenClick(); - }); + // ipcRenderer.on('start-listening-session', () => { + // console.log('Received start-listening-session command, calling handleListenClick.'); + // this.handleListenClick(); + // }); } } @@ -97,16 +91,15 @@ export class PickleGlassApp extends LitElement { super.disconnectedCallback(); if (window.require) { const { ipcRenderer } = window.require('electron'); - ipcRenderer.removeAllListeners('update-status'); ipcRenderer.removeAllListeners('click-through-toggled'); - ipcRenderer.removeAllListeners('start-listening-session'); + // ipcRenderer.removeAllListeners('start-listening-session'); } } updated(changedProperties) { - if (changedProperties.has('isMainViewVisible') || changedProperties.has('currentView')) { - this.requestWindowResize(); - } + // if (changedProperties.has('isMainViewVisible') || changedProperties.has('currentView')) { + // this.requestWindowResize(); + // } if (changedProperties.has('currentView')) { const viewContainer = this.shadowRoot?.querySelector('.view-container'); @@ -136,57 +129,35 @@ export class PickleGlassApp extends LitElement { } } - setStatus(text) { - this.statusText = text; - } - async handleListenClick() { - if (window.require) { - const { ipcRenderer } = window.require('electron'); - const isActive = await ipcRenderer.invoke('is-session-active'); - if (isActive) { - console.log('Session is already active. No action needed.'); - return; - } - } + // async handleListenClick() { + // if (window.require) { + // const { ipcRenderer } = window.require('electron'); + // const isActive = await ipcRenderer.invoke('is-session-active'); + // // if (isActive) { + // // console.log('Session is already active. No action needed.'); + // // return; + // // } + // } - if (window.pickleGlass) { - await window.pickleGlass.initializeopenai(this.selectedProfile, this.selectedLanguage); - window.pickleGlass.startCapture(this.selectedScreenshotInterval, this.selectedImageQuality); - } + // if (window.pickleGlass) { + // // await window.pickleGlass.initializeopenai(this.selectedProfile, this.selectedLanguage); + // window.pickleGlass.startCapture(this.selectedScreenshotInterval, this.selectedImageQuality); + // } - // πŸ”„ Clear previous summary/analysis when a new listening session begins - this.structuredData = { - summary: [], - topic: { header: '', bullets: [] }, - actions: [], - followUps: [], - }; + // // πŸ”„ Clear previous summary/analysis when a new listening session begins + // this.structuredData = { + // summary: [], + // topic: { header: '', bullets: [] }, + // actions: [], + // followUps: [], + // }; - this.currentResponseIndex = -1; - this.startTime = Date.now(); - this.currentView = 'listen'; - this.isMainViewVisible = true; - } - - handleShowHideClick() { - this.isMainViewVisible = !this.isMainViewVisible; - } - - handleSettingsClick() { - this.currentView = 'settings'; - this.isMainViewVisible = true; - } - - handleHelpClick() { - this.currentView = 'help'; - this.isMainViewVisible = true; - } - - handleHistoryClick() { - this.currentView = 'history'; - this.isMainViewVisible = true; - } + // this.currentResponseIndex = -1; + // this.startTime = Date.now(); + // this.currentView = 'listen'; + // this.isMainViewVisible = true; + // } async handleClose() { if (window.require) { @@ -195,50 +166,8 @@ export class PickleGlassApp extends LitElement { } } - handleBackClick() { - this.currentView = 'listen'; - } - async handleSendText(message) { - if (window.pickleGlass) { - const result = await window.pickleGlass.sendTextMessage(message); - if (!result.success) { - console.error('Failed to send message:', result.error); - this.setStatus('Error sending message: ' + result.error); - } else { - this.setStatus('Message sent...'); - } - } - } - - // updateOutline(outline) { - // console.log('πŸ“ PickleGlassApp updateOutline:', outline); - // this.outlines = [...outline]; - // this.requestUpdate(); - // } - - // updateAnalysisRequests(requests) { - // console.log('πŸ“ PickleGlassApp updateAnalysisRequests:', requests); - // this.analysisRequests = [...requests]; - // this.requestUpdate(); - // } - - updateStructuredData(data) { - console.log('πŸ“ PickleGlassApp updateStructuredData:', data); - this.structuredData = data; - this.requestUpdate(); - - const assistantView = this.shadowRoot?.querySelector('assistant-view'); - if (assistantView) { - assistantView.structuredData = data; - console.log('βœ… Structured data passed to AssistantView'); - } - } - - handleResponseIndexChanged(e) { - this.currentResponseIndex = e.detail.index; - } render() { switch (this.currentView) { @@ -247,7 +176,6 @@ export class PickleGlassApp extends LitElement { .currentResponseIndex=${this.currentResponseIndex} .selectedProfile=${this.selectedProfile} .structuredData=${this.structuredData} - .onSendText=${message => this.handleSendText(message)} @response-index-changed=${e => (this.currentResponseIndex = e.detail.index)} >`; case 'ask': diff --git a/src/electron/windowManager.js b/src/electron/windowManager.js index 206c14a..d05e781 100644 --- a/src/electron/windowManager.js +++ b/src/electron/windowManager.js @@ -90,7 +90,7 @@ function createFeatureWindows(header, namesToCreate) { hasShadow: false, skipTaskbar: true, hiddenInMissionControl: true, - resizable: false, + resizable: true, webPreferences: { nodeIntegration: true, contextIsolation: false }, }; @@ -100,8 +100,8 @@ function createFeatureWindows(header, namesToCreate) { switch (name) { case 'listen': { const listen = new BrowserWindow({ - ...commonChildOptions, width:400,minWidth:400,maxWidth:400, - maxHeight:700, + ...commonChildOptions, width:400,minWidth:400,maxWidth:900, + maxHeight:900, }); listen.setContentProtection(isContentProtectionOn); listen.setVisibleOnAllWorkspaces(true,{visibleOnFullScreen:true}); @@ -472,18 +472,27 @@ function createWindows() { createFeatureWindows(windowPool.get('header')); } - const windowToToggle = windowPool.get(featureName); - if (windowToToggle) { - if (featureName === 'listen') { - const listenService = global.listenService; - if (listenService && listenService.isSessionActive()) { - console.log('[WindowManager] Listen session is active, closing it via toggle.'); - await listenService.closeSession(); - return; - } - } + if (featureName === 'listen') { console.log(`[WindowManager] Toggling feature: ${featureName}`); + const listenWindow = windowPool.get(featureName); + const listenService = global.listenService; + if (listenService && listenService.isSessionActive()) { + console.log('[WindowManager] Listen session is active, closing it via toggle.'); + await listenService.closeSession(); + return; + } + + if (listenWindow.isVisible()) { + listenWindow.webContents.send('window-hide-animation'); + } else { + listenWindow.show(); + updateLayout(); + // listenWindow.webContents.send('start-listening-session'); + listenWindow.webContents.send('window-show-animation'); + await listenService.initializeSession(); + // listenWindow.webContents.send('start-listening-session'); + } } if (featureName === 'ask') { @@ -561,31 +570,29 @@ function createWindows() { askWindow.webContents.send('window-show-animation'); askWindow.webContents.send('window-did-show'); } - } else { - const windowToToggle = windowPool.get(featureName); + } - if (windowToToggle) { - if (windowToToggle.isDestroyed()) { + if (featureName === 'settings') { + const settingsWindow = windowPool.get(featureName); + + if (settingsWindow) { + if (settingsWindow.isDestroyed()) { console.error(`Window ${featureName} is destroyed, cannot toggle`); return; } - if (windowToToggle.isVisible()) { + if (settingsWindow.isVisible()) { if (featureName === 'settings') { - windowToToggle.webContents.send('settings-window-hide-animation'); + settingsWindow.webContents.send('settings-window-hide-animation'); } else { - windowToToggle.webContents.send('window-hide-animation'); + settingsWindow.webContents.send('window-hide-animation'); } } else { try { - windowToToggle.show(); + settingsWindow.show(); updateLayout(); - if (featureName === 'listen') { - windowToToggle.webContents.send('start-listening-session'); - } - - windowToToggle.webContents.send('window-show-animation'); + settingsWindow.webContents.send('window-show-animation'); } catch (e) { console.error('Error showing window:', e); } diff --git a/src/features/listen/listenService.js b/src/features/listen/listenService.js index f676d93..a019f32 100644 --- a/src/features/listen/listenService.js +++ b/src/features/listen/listenService.js @@ -146,6 +146,7 @@ class ListenService { this.sendToRenderer('session-state-changed', { isActive: true }); this.sendToRenderer('update-status', 'Connected. Ready to listen.'); + // this.sendToRenderer('change-listen-capture-state', { status: "start" }); return true; } catch (error) { @@ -155,6 +156,7 @@ class ListenService { } finally { this.isInitializingSession = false; this.sendToRenderer('session-initializing', false); + this.sendToRenderer('change-listen-capture-state', { status: "start" }); } } @@ -194,6 +196,7 @@ class ListenService { this.sendToRenderer('session-state-changed', { isActive: false }); this.sendToRenderer('session-did-close'); + this.sendToRenderer('change-listen-capture-state', { status: "stop" }); console.log('Listen service session closed.'); return { success: true }; diff --git a/src/features/listen/renderer/listenCapture.js b/src/features/listen/renderer/listenCapture.js index 1607d0b..6c0637e 100644 --- a/src/features/listen/renderer/listenCapture.js +++ b/src/features/listen/renderer/listenCapture.js @@ -7,9 +7,15 @@ let aecPtr = 0; // Rust Aec* 1개만 μž¬μ‚¬μš© /** WASM λͺ¨λ“ˆ κ°€μ Έμ˜€κ³  1회 μ΄ˆκΈ°ν™” */ async function getAec () { - if (aecModPromise) return aecModPromise; // μΊμ‹œ + if (aecModPromise) { + console.log('[AEC] getAec: μΊμ‹œ=있음(μž¬μ‚¬μš©)'); + return aecModPromise; // μΊμ‹œ + } + + console.log('[AEC] getAec: μΊμ‹œ=μ—†μŒ β†’ λͺ¨λ“ˆ λ‘œλ“œ μ‹œμž‘'); aecModPromise = createAecModule().then((M) => { + console.log('[AEC] WASM λͺ¨λ“ˆ λ‘œλ“œ μ™„λ£Œ'); aecMod = M; // C 심볼 β†’ JS 래퍼 바인딩 (λ”± 1번) M.newPtr = M.cwrap('AecNew', 'number', @@ -18,7 +24,12 @@ async function getAec () { ['number','number','number','number','number']); M.destroy = M.cwrap('AecDestroy', null, ['number']); return M; - }); + }) + .catch(err => { + console.error('[AEC] WASM λͺ¨λ“ˆ λ‘œλ“œ μ‹€νŒ¨:', err); + throw err; // μƒμœ„μ—μ„œλ„ μž‘μ„ 수 있게 + }); + return aecModPromise; } @@ -132,6 +143,10 @@ function disposeAec () { } function runAecSync (micF32, sysF32) { + const modStat = aecMod?.HEAPU8 ? '있음' : 'μ—†μŒ'; // aecModκ°€ μ΄ˆκΈ°ν™”λ˜μ—ˆκ³  HEAP μ ‘κ·Ό κ°€λŠ₯? + const ptrStat = aecPtr ? '있음' : 'μ—†μŒ'; // newPtr 호좜 μ—¬λΆ€ + const heapStat = aecMod?.HEAPU8 ? '있음' : 'μ—†μŒ'; // HEAPU8 생성 μ—¬λΆ€ + console.log(`[AEC] mod:${modStat} ptr:${ptrStat} heap:${heapStat}`); if (!aecMod || !aecPtr || !aecMod.HEAPU8) return micF32; // 아직 λͺ¨λ“ˆ μ•ˆ 뜸 β†’ 패슀 const len = micF32.length; @@ -145,6 +160,7 @@ function runAecSync (micF32, sysF32) { const outF32 = float32FromInt16View(new Int16Array(heapBuf, out, len)); aecMod._free(mic.ptr); aecMod._free(echo.ptr); aecMod._free(out); + console.log(`[AEC] 적용 μ™„λ£Œ`); return outF32; } @@ -266,7 +282,7 @@ async function setupMicProcessing(micStream) { micProcessor.onaudioprocess = (e) => { const inputData = e.inputBuffer.getChannelData(0); audioBuffer.push(...inputData); - console.log('🎀 micProcessor.onaudioprocess'); + // console.log('🎀 micProcessor.onaudioprocess'); // samplesPerChunk(=2400) 만큼 λͺ¨μ΄λ©΄ 전솑 while (audioBuffer.length >= samplesPerChunk) { @@ -280,7 +296,7 @@ async function setupMicProcessing(micStream) { // **μŒμ„± ꡬ간일 λ•Œλ§Œ 런** processedChunk = runAecSync(new Float32Array(chunk), sysF32); - console.log('πŸ”Š Applied WASM-AEC (speex)'); + // console.log('πŸ”Š Applied WASM-AEC (speex)'); } else { console.log('πŸ”Š No system audio for AEC reference'); } diff --git a/src/features/listen/renderer/renderer.js b/src/features/listen/renderer/renderer.js index 6601b48..fe26e66 100644 --- a/src/features/listen/renderer/renderer.js +++ b/src/features/listen/renderer/renderer.js @@ -1,138 +1,30 @@ // renderer.js const { ipcRenderer } = require('electron'); const listenCapture = require('./listenCapture.js'); +const params = new URLSearchParams(window.location.search); +const isListenView = params.get('view') === 'listen'; -let realtimeConversationHistory = []; - -async function queryLoginState() { - const userState = await ipcRenderer.invoke('get-current-user'); - return userState; -} - -function pickleGlassElement() { - return document.getElementById('pickle-glass'); -} - -async function initializeopenai(profile = 'interview', language = 'en') { - // The API key is now handled in the main process from .env file. - // We just need to trigger the initialization. - try { - console.log(`Requesting OpenAI initialization with profile: ${profile}, language: ${language}`); - const success = await ipcRenderer.invoke('initialize-openai', profile, language); - if (success) { - // The status will be updated via 'update-status' event from the main process. - console.log('OpenAI initialization successful.'); - } else { - console.error('OpenAI initialization failed.'); - const appElement = pickleGlassElement(); - if (appElement && typeof appElement.setStatus === 'function') { - appElement.setStatus('Initialization Failed'); - } - } - } catch (error) { - console.error('Error during OpenAI initialization IPC call:', error); - const appElement = pickleGlassElement(); - if (appElement && typeof appElement.setStatus === 'function') { - appElement.setStatus('Error'); - } - } -} - -// Listen for status updates -ipcRenderer.on('update-status', (event, status) => { - console.log('Status update:', status); - pickleGlass.e().setStatus(status); -}); - -// Listen for real-time STT updates -ipcRenderer.on('stt-update', (event, data) => { - console.log('Renderer.js stt-update', data); - const { speaker, text, isFinal, isPartial, timestamp } = data; - - if (isPartial) { - console.log(`πŸ”„ [${speaker} - partial]: ${text}`); - } else if (isFinal) { - console.log(`βœ… [${speaker} - final]: ${text}`); - - const speakerText = speaker.toLowerCase(); - const conversationText = `${speakerText}: ${text.trim()}`; - - realtimeConversationHistory.push(conversationText); - - if (realtimeConversationHistory.length > 30) { - realtimeConversationHistory = realtimeConversationHistory.slice(-30); - } - - console.log(`πŸ“ Updated realtime conversation history: ${realtimeConversationHistory.length} texts`); - console.log(`πŸ“‹ Latest text: ${conversationText}`); - } - - if (pickleGlass.e() && typeof pickleGlass.e().updateRealtimeTranscription === 'function') { - pickleGlass.e().updateRealtimeTranscription({ - speaker, - text, - isFinal, - isPartial, - timestamp, - }); - } -}); - -ipcRenderer.on('update-structured-data', (_, structuredData) => { - console.log('πŸ“₯ Received structured data update:', structuredData); - window.pickleGlass.structuredData = structuredData; - window.pickleGlass.setStructuredData(structuredData); -}); - -window.pickleGlass.structuredData = { - summary: [], - topic: { header: '', bullets: [] }, - actions: [], -}; - -window.pickleGlass.setStructuredData = data => { - window.pickleGlass.structuredData = data; - pickleGlass.e()?.updateStructuredData?.(data); -}; - -function formatRealtimeConversationHistory() { - if (realtimeConversationHistory.length === 0) return 'No conversation history available.'; - - return realtimeConversationHistory.slice(-30).join('\n'); -} window.pickleGlass = { - initializeopenai, startCapture: listenCapture.startCapture, stopCapture: listenCapture.stopCapture, isLinux: listenCapture.isLinux, isMacOS: listenCapture.isMacOS, captureManualScreenshot: listenCapture.captureManualScreenshot, getCurrentScreenshot: listenCapture.getCurrentScreenshot, - e: pickleGlassElement, }; -// ------------------------------------------------------- -// πŸ”” React to session state changes from the main process -// When the session ends (isActive === false), ensure we stop -// all local capture pipelines (mic, screen, etc.). -// ------------------------------------------------------- -ipcRenderer.on('session-state-changed', (_event, { isActive }) => { - if (!isActive) { + +ipcRenderer.on('change-listen-capture-state', (_event, { status }) => { + if (!isListenView) { + console.log('[Renderer] Non-listen view: ignoring capture-state change'); + return; + } + if (status === "stop") { console.log('[Renderer] Session ended – stopping local capture'); listenCapture.stopCapture(); } else { - console.log('[Renderer] New session started – clearing in-memory history and summaries'); - - // Reset live conversation & analysis caches - realtimeConversationHistory = []; - - const blankData = { - summary: [], - topic: { header: '', bullets: [] }, - actions: [], - followUps: [], - }; - window.pickleGlass.setStructuredData(blankData); + console.log('[Renderer] Session initialized – starting local capture'); + listenCapture.startCapture(); } }); diff --git a/src/features/listen/summary/SummaryView.js b/src/features/listen/summary/SummaryView.js index 214f9bd..b500689 100644 --- a/src/features/listen/summary/SummaryView.js +++ b/src/features/listen/summary/SummaryView.js @@ -264,7 +264,7 @@ export class SummaryView extends LitElement { super.connectedCallback(); if (window.require) { const { ipcRenderer } = window.require('electron'); - ipcRenderer.on('update-structured-data', (event, data) => { + ipcRenderer.on('summary-update', (event, data) => { this.structuredData = data; this.requestUpdate(); }); @@ -275,7 +275,7 @@ export class SummaryView extends LitElement { super.disconnectedCallback(); if (window.require) { const { ipcRenderer } = window.require('electron'); - ipcRenderer.removeAllListeners('update-structured-data'); + ipcRenderer.removeAllListeners('summary-update'); } } diff --git a/src/features/listen/summary/summaryService.js b/src/features/listen/summary/summaryService.js index 53d3b79..a3dc065 100644 --- a/src/features/listen/summary/summaryService.js +++ b/src/features/listen/summary/summaryService.js @@ -27,23 +27,6 @@ class SummaryService { this.currentSessionId = sessionId; } - // async getApiKey() { - // const storedKey = await getStoredApiKey(); - // if (storedKey) { - // console.log('[SummaryService] Using stored API key'); - // return storedKey; - // } - - // const envKey = process.env.OPENAI_API_KEY; - // if (envKey) { - // console.log('[SummaryService] Using environment API key'); - // return envKey; - // } - - // console.error('[SummaryService] No API key found in storage or environment'); - // return null; - // } - sendToRenderer(channel, data) { BrowserWindow.getAllWindows().forEach(win => { if (!win.isDestroyed()) { @@ -327,7 +310,7 @@ Keep all points concise and build upon previous analysis if provided.`, .then(data => { if (data) { console.log('πŸ“€ Sending structured data to renderer'); - this.sendToRenderer('update-structured-data', data); + this.sendToRenderer('summary-update', data); // Notify callback if (this.onAnalysisComplete) { diff --git a/src/index.js b/src/index.js index 16be2e4..a63d8cb 100644 --- a/src/index.js +++ b/src/index.js @@ -396,21 +396,6 @@ function setupGeneralIpcHandlers() { const userRepository = require('./common/repositories/user'); const presetRepository = require('./common/repositories/preset'); - ipcMain.handle('save-api-key', (event, apiKey) => { - try { - // The adapter injects the UID and handles local/firebase logic. - // Assuming a default provider if not specified. - userRepository.saveApiKey(apiKey, 'openai'); - BrowserWindow.getAllWindows().forEach(win => { - win.webContents.send('api-key-updated'); - }); - return { success: true }; - } catch (error) { - console.error('IPC: Failed to save API key:', error); - return { success: false, error: error.message }; - } - }); - ipcMain.handle('get-user-presets', () => { // The adapter injects the UID. return presetRepository.getPresets();