cheykrym/glass
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/pickle-com/glass

Browse Source
This commit is contained in:
sanio 2025-07-08 21:35:24 +09:00
commit 4b8b097ce2
15 changed files with 239 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/build.yml vendored
View File

@ -31,4 +31,15 @@ jobs:
- name: 🖥️ Build Electron app
# Run Electron build script from root directory
run: npm run build
run: npm run build
- name: 🚨 Send failure notification to Slack
if: failure()
uses: rtCamp/action-slack-notify@v2
env:
SLACK_CHANNEL: general
SLACK_TITLE: "🚨 Build Failed"
SLACK_MESSAGE: "😭 Build failed for `${{ github.repository }}` repo on main branch."
SLACK_COLOR: 'danger'
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "aec"]
path = aec
url = https://github.com/samtiz/aec.git

1
README.md
View File

@ -121,7 +121,6 @@ We have a list of [help wanted](https://github.com/pickle-com/glass/issues?q=is%
| 🚧 WIP | Local LLM Support | Supporting Local LLM to power AI answers |
| 🚧 WIP | AEC Improvement | Transcription is not working occasionally |
| 🚧 WIP | Firebase Data Storage Issue | Session & ask should be saved in firebase for signup users |
| 🚧 WIP | Login Issue | Currently breaking when switching between local and sign-in mode |
| 🚧 WIP | Liquid Glass | Liquid Glass UI for MacOS 26 |
### Changelog

1
aec Submodule

@ -0,0 +1 @@
Subproject commit f00bb1fb948053c752b916adfee19f90644a0b2f

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "pickle-glass",
"version": "0.2.1",
"version": "0.2.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "pickle-glass",
"version": "0.2.1",
"version": "0.2.2",
"hasInstallScript": true,
"license": "GPL-3.0",
"dependencies": {

9
package.json
View File

@ -1,7 +1,9 @@
{
"name": "pickle-glass",
"productName": "Glass",
"version": "0.2.1",
"version": "0.2.2",
"description": "Cl*ely for Free",
"main": "src/index.js",
"scripts": {
@ -10,7 +12,7 @@
"package": "npm run build:renderer && electron-forge package",
"make": "npm run build:renderer && electron-forge make",
"build": "npm run build:all && electron-builder --config electron-builder.yml --publish never",
"build:win": "npm run build:all && electron-builder --win --x64 --publish never",
"build:win": "npm run build:all && electron-builder --win --x64 --publish never",
"publish": "npm run build:all && electron-builder --config electron-builder.yml --publish always",
"lint": "eslint --ext .ts,.tsx,.js .",
"postinstall": "electron-builder install-app-deps",
@ -39,7 +41,6 @@
"better-sqlite3": "^9.4.3",
"cors": "^2.8.5",
"dotenv": "^17.0.0",
"electron-squirrel-startup": "^1.0.1",
"electron-store": "^8.2.0",
"electron-updater": "^6.6.2",
@ -74,4 +75,4 @@
"optionalDependencies": {
"electron-liquid-glass": "^1.0.1"
}
}
}

10
pickleglass_web/package-lock.json generated
View File

@ -42,21 +42,27 @@
}
},
"node_modules/@emnapi/core": {
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.4.4.tgz",
"integrity": "sha512-A9CnAbC6ARNMKcIcrQwq6HeHCjpcBZ5wSx4U01WXCqEKlrzB9F9315WDNHkrs2xbx7YjjSxbUYxuN6EQzpcY2g==",
"dev": true,
"license": "MIT",
"optional": true,
"dependencies": {
"@emnapi/wasi-threads": "1.0.3",
"tslib": "^2.4.0"
}
},
"node_modules/@emnapi/runtime": {
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.4.tgz",
"integrity": "sha512-hHyapA4A3gPaDCNfiqyZUStTMqIkKRshqPIuDOXv1hcBnD4U3l8cP0T1HMCfGRxQ6V64TGCcoswChANyOAwbQg==",
"dev": true,
"license": "MIT",
"optional": true,
@ -65,9 +71,11 @@
}
},
"node_modules/@emnapi/wasi-threads": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.3.tgz",
"integrity": "sha512-8K5IFFsQqF9wQNJptGbS6FNKgUTsSRYnTqNCG1vPP8jFdjSv18n2mQfJpkt2Oibo9iBEzcDnDxNwKTzC7svlJw==",
"dev": true,
"license": "MIT",
"optional": true,
@ -2667,9 +2675,11 @@
"license": "MIT"
},
"node_modules/electron-to-chromium": {
"version": "1.5.180",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.180.tgz",
"integrity": "sha512-ED+GEyEh3kYMwt2faNmgMB0b8O5qtATGgR4RmRsIp4T6p7B8vdMbIedYndnvZfsaXvSzegtpfqRMDNCjjiSduA==",
"license": "ISC"
},
"node_modules/emoji-regex": {

20
src/assets/aec.js Normal file
View File

File diff suppressed because one or more lines are too long

9
src/common/ai/providers/gemini.js
View File

@ -17,7 +17,14 @@ async function createSTT({ apiKey, language = 'en-US', callbacks = {}, ...config
const session = await liveClient.live.connect({
model: 'gemini-live-2.5-flash-preview',
callbacks,
callbacks: {
...callbacks,
onMessage: (msg) => {
if (!msg || typeof msg !== 'object') return;
msg.provider = 'gemini';
callbacks.onmessage?.(msg);
}
},
config: {
inputAudioTranscription: {},
speechConfig: { languageCode: lang },

16
src/common/ai/providers/openai.js
View File

@ -72,6 +72,7 @@ async function createSTT({ apiKey, language = 'en', callbacks = {}, usePortkey =
close: () => {
if (ws.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({ type: 'session.close' }));
ws.onmessage = ws.onerror = () => {}; // 핸들러 제거
ws.close(1000, 'Client initiated close.');
}
}
@ -79,10 +80,17 @@ async function createSTT({ apiKey, language = 'en', callbacks = {}, usePortkey =
};
ws.onmessage = (event) => {
const message = JSON.parse(event.data);
if (callbacks && callbacks.onmessage) {
callbacks.onmessage(message);
}
// ── 종료·하트비트 패킷 필터링 ──────────────────────────────
if (!event.data || event.data === 'null' || event.data === '[DONE]') return;
let msg;
try { msg = JSON.parse(event.data); }
catch { return; } // JSON 파싱 실패 무시
if (!msg || typeof msg !== 'object') return;
msg.provider = 'openai'; // ← 항상 명시
callbacks.onmessage?.(msg);
};
ws.onerror = (error) => {

39
src/features/listen/listenService.js
View File

@ -1,4 +1,4 @@
const { BrowserWindow } = require('electron');
const { BrowserWindow, app } = require('electron');
const SttService = require('./stt/sttService');
const SummaryService = require('./summary/summaryService');
const authService = require('../../common/services/authService');
@ -117,8 +117,27 @@ class ListenService {
throw new Error('Failed to initialize database session');
}
// Initialize STT sessions
await this.sttService.initializeSttSessions(language);
/* ---------- STT Initialization Retry Logic ---------- */
const MAX_RETRY = 10;
const RETRY_DELAY_MS = 300; // 0.3 seconds
let sttReady = false;
for (let attempt = 1; attempt <= MAX_RETRY; attempt++) {
try {
await this.sttService.initializeSttSessions(language);
sttReady = true;
break; // Exit on success
} catch (err) {
console.warn(
`[ListenService] STT init attempt ${attempt} failed: ${err.message}`
);
if (attempt < MAX_RETRY) {
await new Promise(r => setTimeout(r, RETRY_DELAY_MS));
}
}
}
if (!sttReady) throw new Error('STT init failed after retries');
/* ------------------------------------------- */
console.log('✅ Listen service initialized successfully.');
@ -213,9 +232,9 @@ class ListenService {
try {
await this.sendAudioContent(data, mimeType);
return { success: true };
} catch (error) {
console.error('Error sending user audio:', error);
return { success: false, error: error.message };
} catch (e) {
console.error('Error sending user audio:', e);
return { success: false, error: e.message };
}
});
@ -237,9 +256,13 @@ class ListenService {
if (process.platform !== 'darwin') {
return { success: false, error: 'macOS audio capture only available on macOS' };
}
if (this.sttService.isMacOSAudioRunning?.()) {
return { success: false, error: 'already_running' };
}
try {
const success = await this.startMacOSAudioCapture();
return { success };
return { success, error: null };
} catch (error) {
console.error('Error starting macOS audio capture:', error);
return { success: false, error: error.message };
@ -274,4 +297,4 @@ class ListenService {
}
}
module.exports = ListenService;
module.exports = ListenService;

249
src/features/listen/renderer/listenCapture.js
View File

@ -1,5 +1,30 @@
const { ipcRenderer } = require('electron');
const createAecModule = require('../../../assets/aec.js');
let aecModPromise = null; // 한 번만 로드
let aecMod = null;
let aecPtr = 0; // Rust Aec* 1개만 재사용
/** WASM 모듈 가져오고 1회 초기화 */
async function getAec () {
if (aecModPromise) return aecModPromise; // 캐시
aecModPromise = createAecModule().then((M) => {
aecMod = M;
// C 심볼 → JS 래퍼 바인딩 (딱 1번)
M.newPtr = M.cwrap('AecNew', 'number',
['number','number','number','number']);
M.cancel = M.cwrap('AecCancelEcho', null,
['number','number','number','number','number']);
M.destroy = M.cwrap('AecDestroy', null, ['number']);
return M;
});
return aecModPromise;
}
// 바로 로드-실패 로그를 보기 위해
getAec().catch(console.error);
// ---------------------------
// Constants & Globals
// ---------------------------
@ -80,128 +105,49 @@ function arrayBufferToBase64(buffer) {
return btoa(binary);
}
// ---------------------------
// Complete SimpleAEC implementation (exact from renderer.js)
// ---------------------------
class SimpleAEC {
constructor() {
this.adaptiveFilter = new Float32Array(1024);
this.mu = 0.2;
this.echoDelay = 100;
this.sampleRate = 24000;
this.delaySamples = Math.floor((this.echoDelay / 1000) * this.sampleRate);
this.echoGain = 0.5;
this.noiseFloor = 0.01;
// 🔧 Adaptive-gain parameters (User-tuned, very aggressive)
this.targetErr = 0.002;
this.adaptRate = 0.1;
console.log('🎯 AEC initialized (hyper-aggressive)');
}
process(micData, systemData) {
if (!systemData || systemData.length === 0) {
return micData;
}
for (let i = 0; i < systemData.length; i++) {
if (systemData[i] > 0.98) systemData[i] = 0.98;
else if (systemData[i] < -0.98) systemData[i] = -0.98;
systemData[i] = Math.tanh(systemData[i] * 4);
}
let sum2 = 0;
for (let i = 0; i < systemData.length; i++) sum2 += systemData[i] * systemData[i];
const rms = Math.sqrt(sum2 / systemData.length);
const targetRms = 0.08; // 🔧 기준 RMS (기존 0.1)
const scale = targetRms / (rms + 1e-6); // 1e-6: 0-division 방지
const output = new Float32Array(micData.length);
const optimalDelay = this.findOptimalDelay(micData, systemData);
for (let i = 0; i < micData.length; i++) {
let echoEstimate = 0;
for (let d = -500; d <= 500; d += 100) {
const delayIndex = i - optimalDelay - d;
if (delayIndex >= 0 && delayIndex < systemData.length) {
const weight = Math.exp(-Math.abs(d) / 1000);
echoEstimate += systemData[delayIndex] * scale * this.echoGain * weight;
}
}
output[i] = micData[i] - echoEstimate * 0.9;
if (Math.abs(output[i]) < this.noiseFloor) {
output[i] *= 0.5;
}
if (this.isSimilarToSystem(output[i], systemData, i, optimalDelay)) {
output[i] *= 0.25;
}
output[i] = Math.max(-1, Math.min(1, output[i]));
}
let errSum = 0;
for (let i = 0; i < output.length; i++) errSum += output[i] * output[i];
const errRms = Math.sqrt(errSum / output.length);
const err = errRms - this.targetErr;
this.echoGain += this.adaptRate * err; // 비례 제어
this.echoGain = Math.max(0, Math.min(1, this.echoGain));
return output;
}
findOptimalDelay(micData, systemData) {
let maxCorr = 0;
let optimalDelay = this.delaySamples;
for (let delay = 0; delay < 5000 && delay < systemData.length; delay += 200) {
let corr = 0;
let count = 0;
for (let i = 0; i < Math.min(500, micData.length); i++) {
if (i + delay < systemData.length) {
corr += micData[i] * systemData[i + delay];
count++;
}
}
if (count > 0) {
corr = Math.abs(corr / count);
if (corr > maxCorr) {
maxCorr = corr;
optimalDelay = delay;
}
}
}
return optimalDelay;
}
isSimilarToSystem(sample, systemData, index, delay) {
const windowSize = 50;
let similarity = 0;
for (let i = -windowSize; i <= windowSize; i++) {
const sysIndex = index - delay + i;
if (sysIndex >= 0 && sysIndex < systemData.length) {
similarity += Math.abs(sample - systemData[sysIndex]);
}
}
return similarity / (2 * windowSize + 1) < 0.15;
}
/* ───────────────────────── JS ↔︎ WASM 헬퍼 ───────────────────────── */
function int16PtrFromFloat32(mod, f32) {
const len = f32.length;
const bytes = len * 2;
const ptr = mod._malloc(bytes);
// HEAP16이 없으면 HEAPU8.buffer로 직접 래핑
const heapBuf = (mod.HEAP16 ? mod.HEAP16.buffer : mod.HEAPU8.buffer);
const i16 = new Int16Array(heapBuf, ptr, len);
for (let i = 0; i < len; ++i) {
const s = Math.max(-1, Math.min(1, f32[i]));
i16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
}
return { ptr, view: i16 };
}
function float32FromInt16View(i16) {
const out = new Float32Array(i16.length);
for (let i = 0; i < i16.length; ++i) out[i] = i16[i] / 32768;
return out;
}
/* 필요하다면 종료 시 */
function disposeAec () {
getAec().then(mod => { if (aecPtr) mod.destroy(aecPtr); });
}
function runAecSync (micF32, sysF32) {
if (!aecMod || !aecPtr || !aecMod.HEAPU8) return micF32; // 아직 모듈 안 뜸 → 패스
const len = micF32.length;
const mic = int16PtrFromFloat32(aecMod, micF32);
const echo = int16PtrFromFloat32(aecMod, sysF32);
const out = aecMod._malloc(len * 2);
aecMod.cancel(aecPtr, mic.ptr, echo.ptr, out, len);
const heapBuf = (aecMod.HEAP16 ? aecMod.HEAP16.buffer : aecMod.HEAPU8.buffer);
const outF32 = float32FromInt16View(new Int16Array(heapBuf, out, len));
aecMod._free(mic.ptr); aecMod._free(echo.ptr); aecMod._free(out);
return outF32;
}
let aecProcessor = new SimpleAEC();
// System audio data handler
ipcRenderer.on('system-audio-data', (event, { data }) => {
@ -214,8 +160,6 @@ ipcRenderer.on('system-audio-data', (event, { data }) => {
if (systemAudioBuffer.length > MAX_SYSTEM_BUFFER_SIZE) {
systemAudioBuffer = systemAudioBuffer.slice(-MAX_SYSTEM_BUFFER_SIZE);
}
console.log('📥 Received system audio for AEC reference');
});
// ---------------------------
@ -305,39 +249,47 @@ setInterval(() => {
// ---------------------------
// Audio processing functions (exact from renderer.js)
// ---------------------------
function setupMicProcessing(micStream) {
async function setupMicProcessing(micStream) {
/* ── WASM 먼저 로드 ───────────────────────── */
const mod = await getAec();
if (!aecPtr) aecPtr = mod.newPtr(160, 1600, 24000, 1);
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
await micAudioContext.resume();
const micSource = micAudioContext.createMediaStreamSource(micStream);
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
let audioBuffer = [];
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
micProcessor.onaudioprocess = async e => {
micProcessor.onaudioprocess = (e) => {
const inputData = e.inputBuffer.getChannelData(0);
audioBuffer.push(...inputData);
console.log('🎤 micProcessor.onaudioprocess');
// samplesPerChunk(=2400) 만큼 모이면 전송
while (audioBuffer.length >= samplesPerChunk) {
let chunk = audioBuffer.splice(0, samplesPerChunk);
let processedChunk = new Float32Array(chunk);
let processedChunk = new Float32Array(chunk); // 기본값
// Check for system audio and apply AEC only if voice is active
if (aecProcessor && systemAudioBuffer.length > 0) {
const latestSystemAudio = systemAudioBuffer[systemAudioBuffer.length - 1];
const systemFloat32 = base64ToFloat32Array(latestSystemAudio.data);
// ───────────────── WASM AEC ─────────────────
if (systemAudioBuffer.length > 0) {
const latest = systemAudioBuffer[systemAudioBuffer.length - 1];
const sysF32 = base64ToFloat32Array(latest.data);
// Apply AEC only when system audio has active speech
if (isVoiceActive(systemFloat32)) {
processedChunk = aecProcessor.process(new Float32Array(chunk), systemFloat32);
console.log('🔊 Applied AEC because system audio is active');
}
// **음성 구간일 때만 런**
processedChunk = runAecSync(new Float32Array(chunk), sysF32);
console.log('🔊 Applied WASM-AEC (speex)');
} else {
console.log('🔊 No system audio for AEC reference');
}
const pcmData16 = convertFloat32ToInt16(processedChunk);
const base64Data = arrayBufferToBase64(pcmData16.buffer);
const pcm16 = convertFloat32ToInt16(processedChunk);
const b64 = arrayBufferToBase64(pcm16.buffer);
await ipcRenderer.invoke('send-audio-content', {
data: base64Data,
ipcRenderer.invoke('send-audio-content', {
data: b64,
mimeType: 'audio/pcm;rate=24000',
});
}
@ -520,7 +472,19 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
// Start macOS audio capture
const audioResult = await ipcRenderer.invoke('start-macos-audio');
if (!audioResult.success) {
throw new Error('Failed to start macOS audio capture: ' + audioResult.error);
console.warn('[listenCapture] macOS audio start failed:', audioResult.error);
// 이미 실행 중 → stop 후 재시도
if (audioResult.error === 'already_running') {
await ipcRenderer.invoke('stop-macos-audio');
await new Promise(r => setTimeout(r, 500));
const retry = await ipcRenderer.invoke('start-macos-audio');
if (!retry.success) {
throw new Error('Retry failed: ' + retry.error);
}
} else {
throw new Error('Failed to start macOS audio capture: ' + audioResult.error);
}
}
// Initialize screen capture in main process
@ -543,7 +507,7 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
});
console.log('macOS microphone capture started');
const { context, processor } = setupMicProcessing(micMediaStream);
const { context, processor } = await setupMicProcessing(micMediaStream);
audioContext = context;
audioProcessor = processor;
} catch (micErr) {
@ -616,7 +580,7 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
video: false,
});
console.log('Windows microphone capture started');
const { context, processor } = setupMicProcessing(micMediaStream);
const { context, processor } = await setupMicProcessing(micMediaStream);
audioContext = context;
audioProcessor = processor;
} catch (micErr) {
@ -719,6 +683,9 @@ function stopCapture() {
// Exports & global registration
// ---------------------------
module.exports = {
getAec, // 새로 만든 초기화 함수
runAecSync, // sync 버전
disposeAec, // 필요시 Rust 객체 파괴
startCapture,
stopCapture,
captureManualScreenshot,

30
src/features/listen/stt/sttService.js
View File

@ -74,7 +74,7 @@ class SttService {
}
flushMyCompletion() {
if (!this.myCompletionBuffer.trim()) return;
if (!this.modelInfo || !this.myCompletionBuffer.trim()) return;
const finalText = this.myCompletionBuffer.trim();
@ -102,7 +102,7 @@ class SttService {
}
flushTheirCompletion() {
if (!this.theirCompletionBuffer.trim()) return;
if (!this.modelInfo || !this.theirCompletionBuffer.trim()) return;
const finalText = this.theirCompletionBuffer.trim();
@ -176,6 +176,11 @@ class SttService {
// console.log(`[SttService] Initializing STT for provider: ${modelInfo.provider}`);
const handleMyMessage = message => {
if (!this.modelInfo) {
console.log('[SttService] Ignoring message - session already closed');
return;
}
if (this.modelInfo.provider === 'gemini') {
const text = message.serverContent?.inputTranscription?.text || '';
if (text && text.trim()) {
@ -217,6 +222,13 @@ class SttService {
};
const handleTheirMessage = message => {
if (!message || typeof message !== 'object') return;
if (!this.modelInfo) {
console.log('[SttService] Ignoring message - session already closed');
return;
}
if (this.modelInfo.provider === 'gemini') {
const text = message.serverContent?.inputTranscription?.text || '';
if (text && text.trim()) {
@ -320,14 +332,20 @@ class SttService {
}
async sendSystemAudioContent(data, mimeType) {
const provider = await this.getAiProvider();
const isGemini = provider === 'gemini';
if (!this.theirSttSession) {
throw new Error('Their STT session not active');
}
const payload = isGemini
let modelInfo = this.modelInfo;
if (!modelInfo) {
console.warn('[SttService] modelInfo not found, fetching on-the-fly as a fallback...');
modelInfo = await getCurrentModelInfo(null, { type: 'stt' });
}
if (!modelInfo) {
throw new Error('STT model info could not be retrieved.');
}
const payload = modelInfo.provider === 'gemini'
? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } }
: data;

2
src/features/listen/summary/summaryService.js
View File

@ -115,7 +115,7 @@ Please build upon this context while analyzing the new conversation segments.
await sessionRepository.touch(this.currentSessionId);
}
const modelInfo = await getCurrentModelInfo('llm');
const modelInfo = await getCurrentModelInfo(null, { type: 'llm' });
if (!modelInfo || !modelInfo.apiKey) {
throw new Error('AI model or API key is not configured.');
}

2
src/index.js
View File

@ -190,9 +190,11 @@ app.whenReady().then(async () => {
sessionRepository.endAllActiveSessions();
authService.initialize();
//////// after_modelStateService ////////
modelStateService.initialize();
//////// after_modelStateService ////////
listenService.setupIpcHandlers();
askService.initialize();
settingsService.initialize();