cheykrym/glass
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

toggle listen button logic improved

Browse Source
This commit is contained in:
sanio 2025-07-11 02:39:12 +09:00
parent 3031d0d288
commit e86c2db464
9 changed files with 103 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
package-lock.json generated
View File

@ -4172,28 +4172,6 @@
"node": ">= 8"
}
},
"node_modules/fs-temp": {
"version": "1.2.1",
"dev": true,
"license": "MIT",
"optional": true,
"dependencies": {
"random-path": "^0.1.0"
}
},
"node_modules/fs-xattr": {
"version": "0.3.1",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"!win32"
],
"engines": {
"node": ">=8.6.0"
}
},
"node_modules/fs.realpath": {
"version": "1.0.0",
"dev": true,
@ -5158,19 +5136,6 @@
"lru-cache": "6.0.0"
}
},
"node_modules/macos-alias": {
"version": "0.2.12",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"dependencies": {
"nan": "^2.4.0"
}
},
"node_modules/make-fetch-happen": {
"version": "10.2.1",
"dev": true,

138
src/app/PickleGlassApp.js
View File

@ -68,12 +68,7 @@ export class PickleGlassApp extends LitElement {
this.selectedScreenshotInterval = localStorage.getItem('selectedScreenshotInterval') || '5';
this.selectedImageQuality = localStorage.getItem('selectedImageQuality') || 'medium';
this._isClickThrough = false;
this.outlines = [];
this.analysisRequests = [];
window.pickleGlass.setStructuredData = data => {
this.updateStructuredData(data);
};
}
connectedCallback() {
@ -82,14 +77,13 @@ export class PickleGlassApp extends LitElement {
if (window.require) {
const { ipcRenderer } = window.require('electron');
ipcRenderer.on('update-status', (_, status) => this.setStatus(status));
ipcRenderer.on('click-through-toggled', (_, isEnabled) => {
this._isClickThrough = isEnabled;
});
ipcRenderer.on('start-listening-session', () => {
console.log('Received start-listening-session command, calling handleListenClick.');
this.handleListenClick();
});
// ipcRenderer.on('start-listening-session', () => {
// console.log('Received start-listening-session command, calling handleListenClick.');
// this.handleListenClick();
// });
}
}
@ -97,16 +91,15 @@ export class PickleGlassApp extends LitElement {
super.disconnectedCallback();
if (window.require) {
const { ipcRenderer } = window.require('electron');
ipcRenderer.removeAllListeners('update-status');
ipcRenderer.removeAllListeners('click-through-toggled');
ipcRenderer.removeAllListeners('start-listening-session');
// ipcRenderer.removeAllListeners('start-listening-session');
}
}
updated(changedProperties) {
if (changedProperties.has('isMainViewVisible') || changedProperties.has('currentView')) {
this.requestWindowResize();
}
// if (changedProperties.has('isMainViewVisible') || changedProperties.has('currentView')) {
// this.requestWindowResize();
// }
if (changedProperties.has('currentView')) {
const viewContainer = this.shadowRoot?.querySelector('.view-container');
@ -136,57 +129,35 @@ export class PickleGlassApp extends LitElement {
}
}
setStatus(text) {
this.statusText = text;
}
async handleListenClick() {
if (window.require) {
const { ipcRenderer } = window.require('electron');
const isActive = await ipcRenderer.invoke('is-session-active');
if (isActive) {
console.log('Session is already active. No action needed.');
return;
}
}
// async handleListenClick() {
// if (window.require) {
// const { ipcRenderer } = window.require('electron');
// const isActive = await ipcRenderer.invoke('is-session-active');
// // if (isActive) {
// // console.log('Session is already active. No action needed.');
// // return;
// // }
// }
if (window.pickleGlass) {
await window.pickleGlass.initializeopenai(this.selectedProfile, this.selectedLanguage);
window.pickleGlass.startCapture(this.selectedScreenshotInterval, this.selectedImageQuality);
}
// if (window.pickleGlass) {
// // await window.pickleGlass.initializeopenai(this.selectedProfile, this.selectedLanguage);
// window.pickleGlass.startCapture(this.selectedScreenshotInterval, this.selectedImageQuality);
// }
// 🔄 Clear previous summary/analysis when a new listening session begins
this.structuredData = {
summary: [],
topic: { header: '', bullets: [] },
actions: [],
followUps: [],
};
// // 🔄 Clear previous summary/analysis when a new listening session begins
// this.structuredData = {
// summary: [],
// topic: { header: '', bullets: [] },
// actions: [],
// followUps: [],
// };
this.currentResponseIndex = -1;
this.startTime = Date.now();
this.currentView = 'listen';
this.isMainViewVisible = true;
}
handleShowHideClick() {
this.isMainViewVisible = !this.isMainViewVisible;
}
handleSettingsClick() {
this.currentView = 'settings';
this.isMainViewVisible = true;
}
handleHelpClick() {
this.currentView = 'help';
this.isMainViewVisible = true;
}
handleHistoryClick() {
this.currentView = 'history';
this.isMainViewVisible = true;
}
// this.currentResponseIndex = -1;
// this.startTime = Date.now();
// this.currentView = 'listen';
// this.isMainViewVisible = true;
// }
async handleClose() {
if (window.require) {
@ -195,50 +166,8 @@ export class PickleGlassApp extends LitElement {
}
}
handleBackClick() {
this.currentView = 'listen';
}
async handleSendText(message) {
if (window.pickleGlass) {
const result = await window.pickleGlass.sendTextMessage(message);
if (!result.success) {
console.error('Failed to send message:', result.error);
this.setStatus('Error sending message: ' + result.error);
} else {
this.setStatus('Message sent...');
}
}
}
// updateOutline(outline) {
// console.log('📝 PickleGlassApp updateOutline:', outline);
// this.outlines = [...outline];
// this.requestUpdate();
// }
// updateAnalysisRequests(requests) {
// console.log('📝 PickleGlassApp updateAnalysisRequests:', requests);
// this.analysisRequests = [...requests];
// this.requestUpdate();
// }
updateStructuredData(data) {
console.log('📝 PickleGlassApp updateStructuredData:', data);
this.structuredData = data;
this.requestUpdate();
const assistantView = this.shadowRoot?.querySelector('assistant-view');
if (assistantView) {
assistantView.structuredData = data;
console.log('✅ Structured data passed to AssistantView');
}
}
handleResponseIndexChanged(e) {
this.currentResponseIndex = e.detail.index;
}
render() {
switch (this.currentView) {
@ -247,7 +176,6 @@ export class PickleGlassApp extends LitElement {
.currentResponseIndex=${this.currentResponseIndex}
.selectedProfile=${this.selectedProfile}
.structuredData=${this.structuredData}
.onSendText=${message => this.handleSendText(message)}
@response-index-changed=${e => (this.currentResponseIndex = e.detail.index)}
></assistant-view>`;
case 'ask':

45
src/electron/windowManager.js
View File

@ -90,7 +90,7 @@ function createFeatureWindows(header, namesToCreate) {
hasShadow: false,
skipTaskbar: true,
hiddenInMissionControl: true,
resizable: false,
resizable: true,
webPreferences: { nodeIntegration: true, contextIsolation: false },
};
@ -100,8 +100,8 @@ function createFeatureWindows(header, namesToCreate) {
switch (name) {
case 'listen': {
const listen = new BrowserWindow({
...commonChildOptions, width:400,minWidth:400,maxWidth:400,
maxHeight:700,
...commonChildOptions, width:400,minWidth:400,maxWidth:900,
maxHeight:900,
});
listen.setContentProtection(isContentProtectionOn);
listen.setVisibleOnAllWorkspaces(true,{visibleOnFullScreen:true});
@ -472,18 +472,27 @@ function createWindows() {
createFeatureWindows(windowPool.get('header'));
}
const windowToToggle = windowPool.get(featureName);
if (windowToToggle) {
if (featureName === 'listen') {
console.log(`[WindowManager] Toggling feature: ${featureName}`);
const listenWindow = windowPool.get(featureName);
const listenService = global.listenService;
if (listenService && listenService.isSessionActive()) {
console.log('[WindowManager] Listen session is active, closing it via toggle.');
await listenService.closeSession();
return;
}
if (listenWindow.isVisible()) {
listenWindow.webContents.send('window-hide-animation');
} else {
listenWindow.show();
updateLayout();
// listenWindow.webContents.send('start-listening-session');
listenWindow.webContents.send('window-show-animation');
await listenService.initializeSession();
// listenWindow.webContents.send('start-listening-session');
}
console.log(`[WindowManager] Toggling feature: ${featureName}`);
}
if (featureName === 'ask') {
@ -561,31 +570,29 @@ function createWindows() {
askWindow.webContents.send('window-show-animation');
askWindow.webContents.send('window-did-show');
}
} else {
const windowToToggle = windowPool.get(featureName);
}
if (windowToToggle) {
if (windowToToggle.isDestroyed()) {
if (featureName === 'settings') {
const settingsWindow = windowPool.get(featureName);
if (settingsWindow) {
if (settingsWindow.isDestroyed()) {
console.error(`Window ${featureName} is destroyed, cannot toggle`);
return;
}
if (windowToToggle.isVisible()) {
if (settingsWindow.isVisible()) {
if (featureName === 'settings') {
windowToToggle.webContents.send('settings-window-hide-animation');
settingsWindow.webContents.send('settings-window-hide-animation');
} else {
windowToToggle.webContents.send('window-hide-animation');
settingsWindow.webContents.send('window-hide-animation');
}
} else {
try {
windowToToggle.show();
settingsWindow.show();
updateLayout();
if (featureName === 'listen') {
windowToToggle.webContents.send('start-listening-session');
}
windowToToggle.webContents.send('window-show-animation');
settingsWindow.webContents.send('window-show-animation');
} catch (e) {
console.error('Error showing window:', e);
}

3
src/features/listen/listenService.js
View File

@ -146,6 +146,7 @@ class ListenService {
this.sendToRenderer('session-state-changed', { isActive: true });
this.sendToRenderer('update-status', 'Connected. Ready to listen.');
// this.sendToRenderer('change-listen-capture-state', { status: "start" });
return true;
} catch (error) {
@ -155,6 +156,7 @@ class ListenService {
} finally {
this.isInitializingSession = false;
this.sendToRenderer('session-initializing', false);
this.sendToRenderer('change-listen-capture-state', { status: "start" });
}
}
@ -194,6 +196,7 @@ class ListenService {
this.sendToRenderer('session-state-changed', { isActive: false });
this.sendToRenderer('session-did-close');
this.sendToRenderer('change-listen-capture-state', { status: "stop" });
console.log('Listen service session closed.');
return { success: true };

22
src/features/listen/renderer/listenCapture.js
View File

@ -7,9 +7,15 @@ let aecPtr = 0; // Rust Aec* 1개만 재사용
/** WASM 모듈 가져오고 1회 초기화 */
async function getAec () {
if (aecModPromise) return aecModPromise; // 캐시
if (aecModPromise) {
console.log('[AEC] getAec: 캐시=있음(재사용)');
return aecModPromise; // 캐시
}
console.log('[AEC] getAec: 캐시=없음 → 모듈 로드 시작');
aecModPromise = createAecModule().then((M) => {
console.log('[AEC] WASM 모듈 로드 완료');
aecMod = M;
// C 심볼 → JS 래퍼 바인딩 (딱 1번)
M.newPtr = M.cwrap('AecNew', 'number',
@ -18,8 +24,13 @@ async function getAec () {
['number','number','number','number','number']);
M.destroy = M.cwrap('AecDestroy', null, ['number']);
return M;
})
.catch(err => {
console.error('[AEC] WASM 모듈 로드 실패:', err);
throw err; // 상위에서도 잡을 수 있게
});
return aecModPromise;
}
@ -132,6 +143,10 @@ function disposeAec () {
}
function runAecSync (micF32, sysF32) {
const modStat = aecMod?.HEAPU8 ? '있음' : '없음'; // aecMod가 초기화되었고 HEAP 접근 가능?
const ptrStat = aecPtr ? '있음' : '없음'; // newPtr 호출 여부
const heapStat = aecMod?.HEAPU8 ? '있음' : '없음'; // HEAPU8 생성 여부
console.log(`[AEC] mod:${modStat} ptr:${ptrStat} heap:${heapStat}`);
if (!aecMod || !aecPtr || !aecMod.HEAPU8) return micF32; // 아직 모듈 안 뜸 → 패스
const len = micF32.length;
@ -145,6 +160,7 @@ function runAecSync (micF32, sysF32) {
const outF32 = float32FromInt16View(new Int16Array(heapBuf, out, len));
aecMod._free(mic.ptr); aecMod._free(echo.ptr); aecMod._free(out);
console.log(`[AEC] 적용 완료`);
return outF32;
}
@ -266,7 +282,7 @@ async function setupMicProcessing(micStream) {
micProcessor.onaudioprocess = (e) => {
const inputData = e.inputBuffer.getChannelData(0);
audioBuffer.push(...inputData);
console.log('🎤 micProcessor.onaudioprocess');
// console.log('🎤 micProcessor.onaudioprocess');
// samplesPerChunk(=2400) 만큼 모이면 전송
while (audioBuffer.length >= samplesPerChunk) {
@ -280,7 +296,7 @@ async function setupMicProcessing(micStream) {
// **음성 구간일 때만 런**
processedChunk = runAecSync(new Float32Array(chunk), sysF32);
console.log('🔊 Applied WASM-AEC (speex)');
// console.log('🔊 Applied WASM-AEC (speex)');
} else {
console.log('🔊 No system audio for AEC reference');
}

130
src/features/listen/renderer/renderer.js
View File

@ -1,138 +1,30 @@
// renderer.js
const { ipcRenderer } = require('electron');
const listenCapture = require('./listenCapture.js');
const params = new URLSearchParams(window.location.search);
const isListenView = params.get('view') === 'listen';
let realtimeConversationHistory = [];
async function queryLoginState() {
const userState = await ipcRenderer.invoke('get-current-user');
return userState;
}
function pickleGlassElement() {
return document.getElementById('pickle-glass');
}
async function initializeopenai(profile = 'interview', language = 'en') {
// The API key is now handled in the main process from .env file.
// We just need to trigger the initialization.
try {
console.log(`Requesting OpenAI initialization with profile: ${profile}, language: ${language}`);
const success = await ipcRenderer.invoke('initialize-openai', profile, language);
if (success) {
// The status will be updated via 'update-status' event from the main process.
console.log('OpenAI initialization successful.');
} else {
console.error('OpenAI initialization failed.');
const appElement = pickleGlassElement();
if (appElement && typeof appElement.setStatus === 'function') {
appElement.setStatus('Initialization Failed');
}
}
} catch (error) {
console.error('Error during OpenAI initialization IPC call:', error);
const appElement = pickleGlassElement();
if (appElement && typeof appElement.setStatus === 'function') {
appElement.setStatus('Error');
}
}
}
// Listen for status updates
ipcRenderer.on('update-status', (event, status) => {
console.log('Status update:', status);
pickleGlass.e().setStatus(status);
});
// Listen for real-time STT updates
ipcRenderer.on('stt-update', (event, data) => {
console.log('Renderer.js stt-update', data);
const { speaker, text, isFinal, isPartial, timestamp } = data;
if (isPartial) {
console.log(`🔄 [${speaker} - partial]: ${text}`);
} else if (isFinal) {
console.log(`✅ [${speaker} - final]: ${text}`);
const speakerText = speaker.toLowerCase();
const conversationText = `${speakerText}: ${text.trim()}`;
realtimeConversationHistory.push(conversationText);
if (realtimeConversationHistory.length > 30) {
realtimeConversationHistory = realtimeConversationHistory.slice(-30);
}
console.log(`📝 Updated realtime conversation history: ${realtimeConversationHistory.length} texts`);
console.log(`📋 Latest text: ${conversationText}`);
}
if (pickleGlass.e() && typeof pickleGlass.e().updateRealtimeTranscription === 'function') {
pickleGlass.e().updateRealtimeTranscription({
speaker,
text,
isFinal,
isPartial,
timestamp,
});
}
});
ipcRenderer.on('update-structured-data', (_, structuredData) => {
console.log('📥 Received structured data update:', structuredData);
window.pickleGlass.structuredData = structuredData;
window.pickleGlass.setStructuredData(structuredData);
});
window.pickleGlass.structuredData = {
summary: [],
topic: { header: '', bullets: [] },
actions: [],
};
window.pickleGlass.setStructuredData = data => {
window.pickleGlass.structuredData = data;
pickleGlass.e()?.updateStructuredData?.(data);
};
function formatRealtimeConversationHistory() {
if (realtimeConversationHistory.length === 0) return 'No conversation history available.';
return realtimeConversationHistory.slice(-30).join('\n');
}
window.pickleGlass = {
initializeopenai,
startCapture: listenCapture.startCapture,
stopCapture: listenCapture.stopCapture,
isLinux: listenCapture.isLinux,
isMacOS: listenCapture.isMacOS,
captureManualScreenshot: listenCapture.captureManualScreenshot,
getCurrentScreenshot: listenCapture.getCurrentScreenshot,
e: pickleGlassElement,
};
// -------------------------------------------------------
// 🔔 React to session state changes from the main process
// When the session ends (isActive === false), ensure we stop
// all local capture pipelines (mic, screen, etc.).
// -------------------------------------------------------
ipcRenderer.on('session-state-changed', (_event, { isActive }) => {
if (!isActive) {
ipcRenderer.on('change-listen-capture-state', (_event, { status }) => {
if (!isListenView) {
console.log('[Renderer] Non-listen view: ignoring capture-state change');
return;
}
if (status === "stop") {
console.log('[Renderer] Session ended – stopping local capture');
listenCapture.stopCapture();
} else {
console.log('[Renderer] New session started – clearing in-memory history and summaries');
// Reset live conversation & analysis caches
realtimeConversationHistory = [];
const blankData = {
summary: [],
topic: { header: '', bullets: [] },
actions: [],
followUps: [],
};
window.pickleGlass.setStructuredData(blankData);
console.log('[Renderer] Session initialized – starting local capture');
listenCapture.startCapture();
}
});

4
src/features/listen/summary/SummaryView.js
View File

@ -264,7 +264,7 @@ export class SummaryView extends LitElement {
super.connectedCallback();
if (window.require) {
const { ipcRenderer } = window.require('electron');
ipcRenderer.on('update-structured-data', (event, data) => {
ipcRenderer.on('summary-update', (event, data) => {
this.structuredData = data;
this.requestUpdate();
});
@ -275,7 +275,7 @@ export class SummaryView extends LitElement {
super.disconnectedCallback();
if (window.require) {
const { ipcRenderer } = window.require('electron');
ipcRenderer.removeAllListeners('update-structured-data');
ipcRenderer.removeAllListeners('summary-update');
}
}

19
src/features/listen/summary/summaryService.js
View File

@ -27,23 +27,6 @@ class SummaryService {
this.currentSessionId = sessionId;
}
// async getApiKey() {
// const storedKey = await getStoredApiKey();
// if (storedKey) {
// console.log('[SummaryService] Using stored API key');
// return storedKey;
// }
// const envKey = process.env.OPENAI_API_KEY;
// if (envKey) {
// console.log('[SummaryService] Using environment API key');
// return envKey;
// }
// console.error('[SummaryService] No API key found in storage or environment');
// return null;
// }
sendToRenderer(channel, data) {
BrowserWindow.getAllWindows().forEach(win => {
if (!win.isDestroyed()) {
@ -327,7 +310,7 @@ Keep all points concise and build upon previous analysis if provided.`,
.then(data => {
if (data) {
console.log('📤 Sending structured data to renderer');
this.sendToRenderer('update-structured-data', data);
this.sendToRenderer('summary-update', data);
// Notify callback
if (this.onAnalysisComplete) {

15
src/index.js
View File

@ -396,21 +396,6 @@ function setupGeneralIpcHandlers() {
const userRepository = require('./common/repositories/user');
const presetRepository = require('./common/repositories/preset');
ipcMain.handle('save-api-key', (event, apiKey) => {
try {
// The adapter injects the UID and handles local/firebase logic.
// Assuming a default provider if not specified.
userRepository.saveApiKey(apiKey, 'openai');
BrowserWindow.getAllWindows().forEach(win => {
win.webContents.send('api-key-updated');
});
return { success: true };
} catch (error) {
console.error('IPC: Failed to save API key:', error);
return { success: false, error: error.message };
}
});
ipcMain.handle('get-user-presets', () => {
// The adapter injects the UID.
return presetRepository.getPresets();