diff --git a/src/app/PickleGlassApp.js b/src/app/PickleGlassApp.js index eb639eb..96d0516 100644 --- a/src/app/PickleGlassApp.js +++ b/src/app/PickleGlassApp.js @@ -1,10 +1,9 @@ import { html, css, LitElement } from '../assets/lit-core-2.7.4.min.js'; import { CustomizeView } from '../features/customize/CustomizeView.js'; import { AssistantView } from '../features/listen/AssistantView.js'; -import { OnboardingView } from '../features/onboarding/OnboardingView.js'; import { AskView } from '../features/ask/AskView.js'; -import '../features/listen/renderer.js'; +import '../features/listen/renderer/renderer.js'; export class PickleGlassApp extends LitElement { static styles = css` diff --git a/src/electron/windowManager.js b/src/electron/windowManager.js index 586e3f8..e4cc058 100644 --- a/src/electron/windowManager.js +++ b/src/electron/windowManager.js @@ -1024,10 +1024,10 @@ function createWindows() { if (windowToToggle) { if (featureName === 'listen') { - const liveSummaryService = require('../features/listen/liveSummaryService'); - if (liveSummaryService.isSessionActive()) { + const listenService = global.listenService; + if (listenService && listenService.isSessionActive()) { console.log('[WindowManager] Listen session is active, closing it via toggle.'); - await liveSummaryService.closeSession(); + await listenService.closeSession(); return; } } diff --git a/src/features/ask/askService.js b/src/features/ask/askService.js index 04c64d9..d07d79d 100644 --- a/src/features/ask/askService.js +++ b/src/features/ask/askService.js @@ -1,6 +1,5 @@ const { ipcMain, BrowserWindow } = require('electron'); const { makeStreamingChatCompletionWithPortkey } = require('../../common/services/aiProviderService'); -const { getConversationHistory } = require('../listen/liveSummaryService'); const { getStoredApiKey, getStoredProvider, windowPool, captureScreenshot } = require('../../electron/windowManager'); const authService = require('../../common/services/authService'); const sessionRepository = require('../../common/repositories/session'); @@ -174,6 +173,12 @@ function formatConversationForPrompt(conversationTexts) { return conversationTexts.slice(-30).join('\n'); } +// Access conversation history via the global listenService instance created in index.js +function getConversationHistory() { + const listenService = global.listenService; + return listenService ? listenService.getConversationHistory() : []; +} + async function sendMessage(userPrompt) { if (!userPrompt || userPrompt.trim().length === 0) { console.warn('[AskService] Cannot process empty message'); diff --git a/src/features/listen/audioUtils.js b/src/features/listen/audioUtils.js deleted file mode 100644 index 25dfb9d..0000000 --- a/src/features/listen/audioUtils.js +++ /dev/null @@ -1,123 +0,0 @@ -const fs = require('fs'); -const path = require('path'); - -function pcmToWav(pcmBuffer, outputPath, sampleRate = 24000, channels = 1, bitDepth = 16) { - const byteRate = sampleRate * channels * (bitDepth / 8); - const blockAlign = channels * (bitDepth / 8); - const dataSize = pcmBuffer.length; - - const header = Buffer.alloc(44); - - header.write('RIFF', 0); - header.writeUInt32LE(dataSize + 36, 4); - header.write('WAVE', 8); - - header.write('fmt ', 12); - header.writeUInt32LE(16, 16); - header.writeUInt16LE(1, 20); - header.writeUInt16LE(channels, 22); - header.writeUInt32LE(sampleRate, 24); - header.writeUInt32LE(byteRate, 28); - header.writeUInt16LE(blockAlign, 32); - header.writeUInt16LE(bitDepth, 34); - - header.write('data', 36); - header.writeUInt32LE(dataSize, 40); - - const wavBuffer = Buffer.concat([header, pcmBuffer]); - - fs.writeFileSync(outputPath, wavBuffer); - - return outputPath; -} - -function analyzeAudioBuffer(buffer, label = 'Audio') { - const int16Array = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.length / 2); - - let minValue = 32767; - let maxValue = -32768; - let avgValue = 0; - let rmsValue = 0; - let silentSamples = 0; - - for (let i = 0; i < int16Array.length; i++) { - const sample = int16Array[i]; - minValue = Math.min(minValue, sample); - maxValue = Math.max(maxValue, sample); - avgValue += sample; - rmsValue += sample * sample; - - if (Math.abs(sample) < 100) { - silentSamples++; - } - } - - avgValue /= int16Array.length; - rmsValue = Math.sqrt(rmsValue / int16Array.length); - - const silencePercentage = (silentSamples / int16Array.length) * 100; - - console.log(`${label} Analysis:`); - console.log(` Samples: ${int16Array.length}`); - console.log(` Min: ${minValue}, Max: ${maxValue}`); - console.log(` Average: ${avgValue.toFixed(2)}`); - console.log(` RMS: ${rmsValue.toFixed(2)}`); - console.log(` Silence: ${silencePercentage.toFixed(1)}%`); - console.log(` Dynamic Range: ${20 * Math.log10(maxValue / (rmsValue || 1))} dB`); - - return { - minValue, - maxValue, - avgValue, - rmsValue, - silencePercentage, - sampleCount: int16Array.length, - }; -} - -function saveDebugAudio(buffer, type, timestamp = Date.now()) { - const homeDir = require('os').homedir(); - const debugDir = path.join(homeDir, '.pickle-glass', 'debug'); - - if (!fs.existsSync(debugDir)) { - fs.mkdirSync(debugDir, { recursive: true }); - } - - const pcmPath = path.join(debugDir, `${type}_${timestamp}.pcm`); - const wavPath = path.join(debugDir, `${type}_${timestamp}.wav`); - const metaPath = path.join(debugDir, `${type}_${timestamp}.json`); - - fs.writeFileSync(pcmPath, buffer); - - pcmToWav(buffer, wavPath); - - const analysis = analyzeAudioBuffer(buffer, type); - fs.writeFileSync( - metaPath, - JSON.stringify( - { - timestamp, - type, - bufferSize: buffer.length, - analysis, - format: { - sampleRate: 24000, - channels: 1, - bitDepth: 16, - }, - }, - null, - 2 - ) - ); - - console.log(`Debug audio saved: ${wavPath}`); - - return { pcmPath, wavPath, metaPath }; -} - -module.exports = { - pcmToWav, - analyzeAudioBuffer, - saveDebugAudio, -}; diff --git a/src/features/listen/listenService.js b/src/features/listen/listenService.js new file mode 100644 index 0000000..a0dcde8 --- /dev/null +++ b/src/features/listen/listenService.js @@ -0,0 +1,263 @@ +const { BrowserWindow } = require('electron'); +const SttService = require('./stt/sttService'); +const SummaryService = require('./summary/summaryService'); +const authService = require('../../common/services/authService'); +const sessionRepository = require('../../common/repositories/session'); +const sttRepository = require('./stt/repositories'); + +class ListenService { + constructor() { + this.sttService = new SttService(); + this.summaryService = new SummaryService(); + this.currentSessionId = null; + this.isInitializingSession = false; + + this.setupServiceCallbacks(); + } + + setupServiceCallbacks() { + // STT service callbacks + this.sttService.setCallbacks({ + onTranscriptionComplete: (speaker, text) => { + this.handleTranscriptionComplete(speaker, text); + }, + onStatusUpdate: (status) => { + this.sendToRenderer('update-status', status); + } + }); + + // Summary service callbacks + this.summaryService.setCallbacks({ + onAnalysisComplete: (data) => { + console.log('πŸ“Š Analysis completed:', data); + }, + onStatusUpdate: (status) => { + this.sendToRenderer('update-status', status); + } + }); + } + + sendToRenderer(channel, data) { + BrowserWindow.getAllWindows().forEach(win => { + if (!win.isDestroyed()) { + win.webContents.send(channel, data); + } + }); + } + + async handleTranscriptionComplete(speaker, text) { + console.log(`[ListenService] Transcription complete: ${speaker} - ${text}`); + + // Save to database + await this.saveConversationTurn(speaker, text); + + // Add to summary service for analysis + this.summaryService.addConversationTurn(speaker, text); + } + + async saveConversationTurn(speaker, transcription) { + if (!this.currentSessionId) { + console.error('[DB] Cannot save turn, no active session ID.'); + return; + } + if (transcription.trim() === '') return; + + try { + await sessionRepository.touch(this.currentSessionId); + await sttRepository.addTranscript({ + sessionId: this.currentSessionId, + speaker: speaker, + text: transcription.trim(), + }); + console.log(`[DB] Saved transcript for session ${this.currentSessionId}: (${speaker})`); + } catch (error) { + console.error('Failed to save transcript to DB:', error); + } + } + + async initializeNewSession() { + try { + const uid = authService.getCurrentUserId(); + if (!uid) { + throw new Error("Cannot initialize session: user not logged in."); + } + + this.currentSessionId = await sessionRepository.getOrCreateActive(uid, 'listen'); + console.log(`[DB] New listen session ensured: ${this.currentSessionId}`); + + // Set session ID for summary service + this.summaryService.setSessionId(this.currentSessionId); + + // Reset conversation history + this.summaryService.resetConversationHistory(); + + console.log('New conversation session started:', this.currentSessionId); + return true; + } catch (error) { + console.error('Failed to initialize new session in DB:', error); + this.currentSessionId = null; + return false; + } + } + + async initializeSession(language = 'en') { + if (this.isInitializingSession) { + console.log('Session initialization already in progress.'); + return false; + } + + this.isInitializingSession = true; + this.sendToRenderer('session-initializing', true); + this.sendToRenderer('update-status', 'Initializing sessions...'); + + try { + // Initialize database session + const sessionInitialized = await this.initializeNewSession(); + if (!sessionInitialized) { + throw new Error('Failed to initialize database session'); + } + + // Initialize STT sessions + await this.sttService.initializeSttSessions(language); + + console.log('βœ… Listen service initialized successfully.'); + + this.sendToRenderer('session-state-changed', { isActive: true }); + this.sendToRenderer('update-status', 'Connected. Ready to listen.'); + + return true; + } catch (error) { + console.error('❌ Failed to initialize listen service:', error); + this.sendToRenderer('update-status', 'Initialization failed.'); + return false; + } finally { + this.isInitializingSession = false; + this.sendToRenderer('session-initializing', false); + } + } + + async sendAudioContent(data, mimeType) { + return await this.sttService.sendAudioContent(data, mimeType); + } + + async startMacOSAudioCapture() { + if (process.platform !== 'darwin') { + throw new Error('macOS audio capture only available on macOS'); + } + return await this.sttService.startMacOSAudioCapture(); + } + + async stopMacOSAudioCapture() { + this.sttService.stopMacOSAudioCapture(); + } + + isSessionActive() { + return this.sttService.isSessionActive(); + } + + async closeSession() { + try { + // Close STT sessions + await this.sttService.closeSessions(); + + // End database session + if (this.currentSessionId) { + await sessionRepository.end(this.currentSessionId); + console.log(`[DB] Session ${this.currentSessionId} ended.`); + } + + // Reset state + this.currentSessionId = null; + this.summaryService.resetConversationHistory(); + + this.sendToRenderer('session-state-changed', { isActive: false }); + this.sendToRenderer('session-did-close'); + + console.log('Listen service session closed.'); + return { success: true }; + } catch (error) { + console.error('Error closing listen service session:', error); + return { success: false, error: error.message }; + } + } + + getCurrentSessionData() { + return { + sessionId: this.currentSessionId, + conversationHistory: this.summaryService.getConversationHistory(), + totalTexts: this.summaryService.getConversationHistory().length, + analysisData: this.summaryService.getCurrentAnalysisData(), + }; + } + + getConversationHistory() { + return this.summaryService.getConversationHistory(); + } + + setupIpcHandlers() { + const { ipcMain } = require('electron'); + + ipcMain.handle('is-session-active', async () => { + const isActive = this.isSessionActive(); + console.log(`Checking session status. Active: ${isActive}`); + return isActive; + }); + + ipcMain.handle('initialize-openai', async (event, profile = 'interview', language = 'en') => { + console.log(`Received initialize-openai request with profile: ${profile}, language: ${language}`); + const success = await this.initializeSession(language); + return success; + }); + + ipcMain.handle('send-audio-content', async (event, { data, mimeType }) => { + try { + await this.sendAudioContent(data, mimeType); + return { success: true }; + } catch (error) { + console.error('Error sending user audio:', error); + return { success: false, error: error.message }; + } + }); + + ipcMain.handle('start-macos-audio', async () => { + if (process.platform !== 'darwin') { + return { success: false, error: 'macOS audio capture only available on macOS' }; + } + try { + const success = await this.startMacOSAudioCapture(); + return { success }; + } catch (error) { + console.error('Error starting macOS audio capture:', error); + return { success: false, error: error.message }; + } + }); + + ipcMain.handle('stop-macos-audio', async () => { + try { + this.stopMacOSAudioCapture(); + return { success: true }; + } catch (error) { + console.error('Error stopping macOS audio capture:', error); + return { success: false, error: error.message }; + } + }); + + ipcMain.handle('close-session', async () => { + return await this.closeSession(); + }); + + ipcMain.handle('update-google-search-setting', async (event, enabled) => { + try { + console.log('Google Search setting updated to:', enabled); + return { success: true }; + } catch (error) { + console.error('Error updating Google Search setting:', error); + return { success: false, error: error.message }; + } + }); + + console.log('βœ… Listen service IPC handlers registered'); + } +} + +module.exports = ListenService; \ No newline at end of file diff --git a/src/features/listen/liveSummaryService.js b/src/features/listen/liveSummaryService.js deleted file mode 100644 index b892f66..0000000 --- a/src/features/listen/liveSummaryService.js +++ /dev/null @@ -1,973 +0,0 @@ -require('dotenv').config(); -const { BrowserWindow, ipcMain } = require('electron'); -const { spawn } = require('child_process'); -const { saveDebugAudio } = require('./audioUtils.js'); -const { getSystemPrompt } = require('../../common/prompts/promptBuilder.js'); -const { connectToGeminiSession } = require('../../common/services/googleGeminiClient.js'); -const { connectToOpenAiSession, createOpenAiGenerativeClient, getOpenAiGenerativeModel } = require('../../common/services/openAiClient.js'); -const { makeChatCompletionWithPortkey } = require('../../common/services/aiProviderService.js'); -const authService = require('../../common/services/authService'); -const sessionRepository = require('../../common/repositories/session'); -const listenRepository = require('./repositories'); - -const { getStoredApiKey, getStoredProvider } = require('../../electron/windowManager'); - -const MAX_BUFFER_LENGTH_CHARS = 2000; -const COMPLETION_DEBOUNCE_MS = 2000; - -async function getApiKey() { - const storedKey = await getStoredApiKey(); - - if (storedKey) { - console.log('[LiveSummaryService] Using stored API key'); - return storedKey; - } - - const envKey = process.env.OPENAI_API_KEY; - if (envKey) { - console.log('[LiveSummaryService] Using environment API key'); - return envKey; - } - - console.error('[LiveSummaryService] No API key found in storage or environment'); - return null; -} - -async function getAiProvider() { - try { - const { ipcRenderer } = require('electron'); - const provider = await ipcRenderer.invoke('get-ai-provider'); - return provider || 'openai'; - } catch (error) { - // If we're in the main process, get it directly - return getStoredProvider ? getStoredProvider() : 'openai'; - } -} - -let currentSessionId = null; -let conversationHistory = []; -let isInitializingSession = false; - -let mySttSession = null; -let theirSttSession = null; -let myCurrentUtterance = ''; -let theirCurrentUtterance = ''; - -let myLastPartialText = ''; -let theirLastPartialText = ''; -let myInactivityTimer = null; -let theirInactivityTimer = null; -const INACTIVITY_TIMEOUT = 3000; - -const SESSION_IDLE_TIMEOUT_SECONDS = 30 * 60; // 30 minutes - -let previousAnalysisResult = null; -let analysisHistory = []; - -// --------------------------------------------------------------------------- -// πŸŽ›οΈ Turn-completion debouncing -// --------------------------------------------------------------------------- -// Very aggressive VAD (e.g. 50 ms) tends to split one spoken sentence into -// many "completed" events. To avoid creating a separate chat bubble for each -// of those micro-turns we debounce the *completed* events per speaker. Any -// completions that arrive within this window are concatenated and flushed as -// **one** final turn. - -let myCompletionBuffer = ''; -let theirCompletionBuffer = ''; -let myCompletionTimer = null; -let theirCompletionTimer = null; - -function flushMyCompletion() { - if (!myCompletionBuffer.trim()) return; - - const finalText = myCompletionBuffer.trim(); - // Save to DB & send to renderer as final - saveConversationTurn('Me', finalText); - sendToRenderer('stt-update', { - speaker: 'Me', - text: finalText, - isPartial: false, - isFinal: true, - timestamp: Date.now(), - }); - - myCompletionBuffer = ''; - myCompletionTimer = null; - myCurrentUtterance = ''; // Reset utterance accumulator on flush - sendToRenderer('update-status', 'Listening...'); -} - -function flushTheirCompletion() { - if (!theirCompletionBuffer.trim()) return; - - const finalText = theirCompletionBuffer.trim(); - saveConversationTurn('Them', finalText); - sendToRenderer('stt-update', { - speaker: 'Them', - text: finalText, - isPartial: false, - isFinal: true, - timestamp: Date.now(), - }); - - theirCompletionBuffer = ''; - theirCompletionTimer = null; - theirCurrentUtterance = ''; // Reset utterance accumulator on flush - sendToRenderer('update-status', 'Listening...'); -} - -function debounceMyCompletion(text) { - // μƒλŒ€λ°©μ΄ λ§ν•˜κ³  있던 경우, ν™”μžκ°€ λ³€κ²½λ˜μ—ˆμœΌλ―€λ‘œ μ¦‰μ‹œ μƒλŒ€λ°©μ˜ 말풍선을 μ™„μ„±ν•©λ‹ˆλ‹€. - if (theirCompletionTimer) { - clearTimeout(theirCompletionTimer); - flushTheirCompletion(); - } - - myCompletionBuffer += (myCompletionBuffer ? ' ' : '') + text; - - if (myCompletionTimer) clearTimeout(myCompletionTimer); - myCompletionTimer = setTimeout(flushMyCompletion, COMPLETION_DEBOUNCE_MS); -} - -function debounceTheirCompletion(text) { - // λ‚΄κ°€ λ§ν•˜κ³  있던 경우, ν™”μžκ°€ λ³€κ²½λ˜μ—ˆμœΌλ―€λ‘œ μ¦‰μ‹œ λ‚΄ 말풍선을 μ™„μ„±ν•©λ‹ˆλ‹€. - if (myCompletionTimer) { - clearTimeout(myCompletionTimer); - flushMyCompletion(); - } - - theirCompletionBuffer += (theirCompletionBuffer ? ' ' : '') + text; - - if (theirCompletionTimer) clearTimeout(theirCompletionTimer); - theirCompletionTimer = setTimeout(flushTheirCompletion, COMPLETION_DEBOUNCE_MS); -} - -let systemAudioProc = null; - -let analysisIntervalId = null; - -/** - * Converts conversation history into text to include in the prompt. - * @param {Array} conversationTexts - Array of conversation texts ["me: ~~~", "them: ~~~", ...] - * @param {number} maxTurns - Maximum number of recent turns to include - * @returns {string} - Formatted conversation string for the prompt - */ -function formatConversationForPrompt(conversationTexts, maxTurns = 30) { - if (conversationTexts.length === 0) return ''; - return conversationTexts.slice(-maxTurns).join('\n'); -} - -async function makeOutlineAndRequests(conversationTexts, maxTurns = 30) { - console.log(`πŸ” makeOutlineAndRequests called - conversationTexts: ${conversationTexts.length}`); - - if (conversationTexts.length === 0) { - console.log('⚠️ No conversation texts available for analysis'); - return null; - } - - const recentConversation = formatConversationForPrompt(conversationTexts, maxTurns); - - // 이전 뢄석 κ²°κ³Όλ₯Ό ν”„λ‘¬ν”„νŠΈμ— 포함 - let contextualPrompt = ''; - if (previousAnalysisResult) { - contextualPrompt = ` -Previous Analysis Context: -- Main Topic: ${previousAnalysisResult.topic.header} -- Key Points: ${previousAnalysisResult.summary.slice(0, 3).join(', ')} -- Last Actions: ${previousAnalysisResult.actions.slice(0, 2).join(', ')} - -Please build upon this context while analyzing the new conversation segments. -`; - } - - const basePrompt = getSystemPrompt('pickle_glass_analysis', '', false); - const systemPrompt = basePrompt.replace('{{CONVERSATION_HISTORY}}', recentConversation); - - try { - if (currentSessionId) { - await sessionRepository.touch(currentSessionId); - } - const messages = [ - { - role: 'system', - content: systemPrompt, - }, - { - role: 'user', - content: `${contextualPrompt} - -Analyze the conversation and provide a structured summary. Format your response as follows: - -**Summary Overview** -- Main discussion point with context - -**Key Topic: [Topic Name]** -- First key insight -- Second key insight -- Third key insight - -**Extended Explanation** -Provide 2-3 sentences explaining the context and implications. - -**Suggested Questions** -1. First follow-up question? -2. Second follow-up question? -3. Third follow-up question? - -Keep all points concise and build upon previous analysis if provided.`, - }, - ]; - - console.log('πŸ€– Sending analysis request to OpenAI...'); - - const API_KEY = await getApiKey(); - if (!API_KEY) { - throw new Error('No API key available'); - } - - const provider = getStoredProvider ? getStoredProvider() : 'openai'; - const loggedIn = authService.getCurrentUser().isLoggedIn; // true ➜ vKey, false ➜ apiKey - const usePortkey = loggedIn && provider === 'openai'; // Only use Portkey for OpenAI with Firebase - - console.log(`[LiveSummary] provider: ${provider}, usePortkey: ${usePortkey}`); - - const completion = await makeChatCompletionWithPortkey({ - apiKey: API_KEY, - provider: provider, - messages: messages, - temperature: 0.7, - maxTokens: 1024, - model: provider === 'openai' ? 'gpt-4.1' : 'gemini-2.5-flash', - usePortkey: usePortkey, - portkeyVirtualKey: usePortkey ? API_KEY : null - }); - - const responseText = completion.content; - console.log(`βœ… Analysis response received: ${responseText}`); - const structuredData = parseResponseText(responseText, previousAnalysisResult); - - if (currentSessionId) { - listenRepository.saveSummary({ - sessionId: currentSessionId, - tldr: structuredData.summary.join('\n'), - bullet_json: JSON.stringify(structuredData.topic.bullets), - action_json: JSON.stringify(structuredData.actions), - model: 'gpt-4.1' - }).catch(err => console.error('[DB] Failed to save summary:', err)); - } - - // 뢄석 κ²°κ³Ό μ €μž₯ - previousAnalysisResult = structuredData; - analysisHistory.push({ - timestamp: Date.now(), - data: structuredData, - conversationLength: conversationTexts.length, - }); - - // νžˆμŠ€ν† λ¦¬ 크기 μ œν•œ (졜근 10개만 μœ μ§€) - if (analysisHistory.length > 10) { - analysisHistory.shift(); - } - - return structuredData; - } catch (error) { - console.error('❌ Error during analysis generation:', error.message); - return previousAnalysisResult; // μ—λŸ¬ μ‹œ 이전 κ²°κ³Ό λ°˜ν™˜ - } -} - -function parseResponseText(responseText, previousResult) { - const structuredData = { - summary: [], - topic: { header: '', bullets: [] }, - actions: [], - followUps: ['βœ‰οΈ Draft a follow-up email', 'βœ… Generate action items', 'πŸ“ Show summary'], - }; - - // 이전 κ²°κ³Όκ°€ 있으면 κΈ°λ³Έκ°’μœΌλ‘œ μ‚¬μš© - if (previousResult) { - structuredData.topic.header = previousResult.topic.header; - structuredData.summary = [...previousResult.summary]; - } - - try { - const lines = responseText.split('\n'); - let currentSection = ''; - let isCapturingTopic = false; - let topicName = ''; - - for (const line of lines) { - const trimmedLine = line.trim(); - - // μ„Ήμ…˜ 헀더 감지 - if (trimmedLine.startsWith('**Summary Overview**')) { - currentSection = 'summary-overview'; - continue; - } else if (trimmedLine.startsWith('**Key Topic:')) { - currentSection = 'topic'; - isCapturingTopic = true; - topicName = trimmedLine.match(/\*\*Key Topic: (.+?)\*\*/)?.[1] || ''; - if (topicName) { - structuredData.topic.header = topicName + ':'; - } - continue; - } else if (trimmedLine.startsWith('**Extended Explanation**')) { - currentSection = 'explanation'; - continue; - } else if (trimmedLine.startsWith('**Suggested Questions**')) { - currentSection = 'questions'; - continue; - } - - // 컨텐츠 νŒŒμ‹± - if (trimmedLine.startsWith('-') && currentSection === 'summary-overview') { - const summaryPoint = trimmedLine.substring(1).trim(); - if (summaryPoint && !structuredData.summary.includes(summaryPoint)) { - // κΈ°μ‘΄ summary μ—…λ°μ΄νŠΈ (μ΅œλŒ€ 5개 μœ μ§€) - structuredData.summary.unshift(summaryPoint); - if (structuredData.summary.length > 5) { - structuredData.summary.pop(); - } - } - } else if (trimmedLine.startsWith('-') && currentSection === 'topic') { - const bullet = trimmedLine.substring(1).trim(); - if (bullet && structuredData.topic.bullets.length < 3) { - structuredData.topic.bullets.push(bullet); - } - } else if (currentSection === 'explanation' && trimmedLine) { - // explanation을 topic bullets에 μΆ”κ°€ (λ¬Έμž₯ λ‹¨μœ„λ‘œ) - const sentences = trimmedLine - .split(/\.\s+/) - .filter(s => s.trim().length > 0) - .map(s => s.trim() + (s.endsWith('.') ? '' : '.')); - - sentences.forEach(sentence => { - if (structuredData.topic.bullets.length < 3 && !structuredData.topic.bullets.includes(sentence)) { - structuredData.topic.bullets.push(sentence); - } - }); - } else if (trimmedLine.match(/^\d+\./) && currentSection === 'questions') { - const question = trimmedLine.replace(/^\d+\.\s*/, '').trim(); - if (question && question.includes('?')) { - structuredData.actions.push(`❓ ${question}`); - } - } - } - - // κΈ°λ³Έ μ•‘μ…˜ μΆ”κ°€ - const defaultActions = ['✨ What should I say next?', 'πŸ’¬ Suggest follow-up questions']; - defaultActions.forEach(action => { - if (!structuredData.actions.includes(action)) { - structuredData.actions.push(action); - } - }); - - // μ•‘μ…˜ 개수 μ œν•œ - structuredData.actions = structuredData.actions.slice(0, 5); - - // μœ νš¨μ„± 검증 및 이전 데이터 병합 - if (structuredData.summary.length === 0 && previousResult) { - structuredData.summary = previousResult.summary; - } - if (structuredData.topic.bullets.length === 0 && previousResult) { - structuredData.topic.bullets = previousResult.topic.bullets; - } - } catch (error) { - console.error('❌ Error parsing response text:', error); - // μ—λŸ¬ μ‹œ 이전 κ²°κ³Ό λ°˜ν™˜ - return ( - previousResult || { - summary: [], - topic: { header: 'Analysis in progress', bullets: [] }, - actions: ['✨ What should I say next?', 'πŸ’¬ Suggest follow-up questions'], - followUps: ['βœ‰οΈ Draft a follow-up email', 'βœ… Generate action items', 'πŸ“ Show summary'], - } - ); - } - - console.log('πŸ“Š Final structured data:', JSON.stringify(structuredData, null, 2)); - return structuredData; -} - -/** - * Triggers analysis when conversation history reaches 5 texts. - */ -async function triggerAnalysisIfNeeded() { - if (conversationHistory.length >= 5 && conversationHistory.length % 5 === 0) { - console.log(`πŸš€ Triggering analysis (non-blocking) - ${conversationHistory.length} conversation texts accumulated`); - - makeOutlineAndRequests(conversationHistory) - .then(data => { - if (data) { - console.log('πŸ“€ Sending structured data to renderer'); - sendToRenderer('update-structured-data', data); - } else { - console.log('❌ No analysis data returned from non-blocking call'); - } - }) - .catch(error => { - console.error('❌ Error in non-blocking analysis:', error); - }); - } -} - -/** - * Schedules periodic updates of outline and analysis every 10 seconds. - DEPRECATED - * Now analysis is triggered every 5 conversation texts. - */ -function startAnalysisInterval() { - console.log('⏰ Analysis will be triggered every 5 conversation texts (not on timer)'); - - if (analysisIntervalId) { - clearInterval(analysisIntervalId); - analysisIntervalId = null; - } -} - -function stopAnalysisInterval() { - if (analysisIntervalId) { - clearInterval(analysisIntervalId); - analysisIntervalId = null; - } - - if (myInactivityTimer) { - clearTimeout(myInactivityTimer); - myInactivityTimer = null; - } - if (theirInactivityTimer) { - clearTimeout(theirInactivityTimer); - theirInactivityTimer = null; - } -} - -function sendToRenderer(channel, data) { - BrowserWindow.getAllWindows().forEach(win => { - if (!win.isDestroyed()) { - win.webContents.send(channel, data); - } - }); -} - -function getCurrentSessionData() { - return { - sessionId: currentSessionId, - conversationHistory: conversationHistory, - totalTexts: conversationHistory.length, - }; -} - -// Conversation management functions -async function initializeNewSession() { - try { - const uid = authService.getCurrentUserId(); - if (!uid) { - throw new Error("Cannot initialize session: user not logged in."); - } - currentSessionId = await sessionRepository.getOrCreateActive(uid, 'listen'); - console.log(`[DB] New listen session ensured: ${currentSessionId}`); - - conversationHistory = []; - myCurrentUtterance = ''; - theirCurrentUtterance = ''; - - // πŸ”„ Reset analysis state so the new session starts fresh - previousAnalysisResult = null; - analysisHistory = []; - - // sendToRenderer('update-outline', []); - // sendToRenderer('update-analysis-requests', []); - - myLastPartialText = ''; - theirLastPartialText = ''; - if (myInactivityTimer) { - clearTimeout(myInactivityTimer); - myInactivityTimer = null; - } - if (theirInactivityTimer) { - clearTimeout(theirInactivityTimer); - theirInactivityTimer = null; - } - - console.log('New conversation session started:', currentSessionId); - return true; - } catch (error) { - console.error('Failed to initialize new session in DB:', error); - currentSessionId = null; - return false; - } -} - -async function saveConversationTurn(speaker, transcription) { - if (!currentSessionId) { - console.error('[DB] Cannot save turn, no active session ID.'); - return; - } - if (transcription.trim() === '') return; - - try { - await sessionRepository.touch(currentSessionId); - await listenRepository.addTranscript({ - sessionId: currentSessionId, - speaker: speaker, - text: transcription.trim(), - }); - console.log(`[DB] Saved transcript for session ${currentSessionId}: (${speaker})`); - - const conversationText = `${speaker.toLowerCase()}: ${transcription.trim()}`; - conversationHistory.push(conversationText); - console.log(`πŸ’¬ Saved conversation text: ${conversationText}`); - console.log(`πŸ“ˆ Total conversation history: ${conversationHistory.length} texts`); - - triggerAnalysisIfNeeded(); - - const conversationTurn = { - speaker: speaker, - timestamp: Date.now(), - transcription: transcription.trim(), - }; - } catch (error) { - console.error('Failed to save transcript to DB:', error); - } -} - -async function initializeLiveSummarySession(language = 'en') { - // Use system environment variable if set, otherwise use the provided language - const effectiveLanguage = process.env.OPENAI_TRANSCRIBE_LANG || language || 'en'; - if (isInitializingSession) { - console.log('Session initialization already in progress.'); - return false; - } - - const userState = authService.getCurrentUser(); - const loggedIn = userState.isLoggedIn; - const keyType = loggedIn ? 'vKey' : 'apiKey'; - - isInitializingSession = true; - sendToRenderer('session-initializing', true); - sendToRenderer('update-status', 'Initializing sessions...'); - - const API_KEY = await getApiKey(); - if (!API_KEY) { - console.error('FATAL ERROR: API Key is not defined.'); - sendToRenderer('update-status', 'API Key not configured.'); - isInitializingSession = false; - sendToRenderer('session-initializing', false); - return false; - } - - await initializeNewSession(); - - const provider = await getAiProvider(); - const isGemini = provider === 'gemini'; - console.log(`[LiveSummaryService] Initializing STT for provider: ${provider}`); - - try { - const handleMyMessage = message => { - if (isGemini) { - // console.log('[Gemini Raw Message - Me]:', JSON.stringify(message, null, 2)); - const text = message.serverContent?.inputTranscription?.text || ''; - if (text && text.trim()) { - const finalUtteranceText = text.trim().replace(//g, '').trim(); - if (finalUtteranceText && finalUtteranceText !== '.') { - debounceMyCompletion(finalUtteranceText); - } - } - } else { - const type = message.type; - const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || ''; - - if (type === 'conversation.item.input_audio_transcription.delta') { - if (myCompletionTimer) clearTimeout(myCompletionTimer); - myCompletionTimer = null; - myCurrentUtterance += text; - const continuousText = myCompletionBuffer + (myCompletionBuffer ? ' ' : '') + myCurrentUtterance; - if (text && !text.includes('vq_lbr_audio_')) { - sendToRenderer('stt-update', { - speaker: 'Me', - text: continuousText, - isPartial: true, - isFinal: false, - timestamp: Date.now(), - }); - } - } else if (type === 'conversation.item.input_audio_transcription.completed') { - if (text && text.trim()) { - const finalUtteranceText = text.trim(); - myCurrentUtterance = ''; - debounceMyCompletion(finalUtteranceText); - } - } - } - - if (message.error) { - console.error('[Me] STT Session Error:', message.error); - } - }; - - const handleTheirMessage = message => { - if (isGemini) { - // console.log('[Gemini Raw Message - Them]:', JSON.stringify(message, null, 2)); - const text = message.serverContent?.inputTranscription?.text || ''; - if (text && text.trim()) { - const finalUtteranceText = text.trim().replace(//g, '').trim(); - if (finalUtteranceText && finalUtteranceText !== '.') { - debounceTheirCompletion(finalUtteranceText); - } - } - } else { - const type = message.type; - const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || ''; - if (type === 'conversation.item.input_audio_transcription.delta') { - if (theirCompletionTimer) clearTimeout(theirCompletionTimer); - theirCompletionTimer = null; - theirCurrentUtterance += text; - const continuousText = theirCompletionBuffer + (theirCompletionBuffer ? ' ' : '') + theirCurrentUtterance; - if (text && !text.includes('vq_lbr_audio_')) { - sendToRenderer('stt-update', { - speaker: 'Them', - text: continuousText, - isPartial: true, - isFinal: false, - timestamp: Date.now(), - }); - } - } else if (type === 'conversation.item.input_audio_transcription.completed') { - if (text && text.trim()) { - const finalUtteranceText = text.trim(); - theirCurrentUtterance = ''; - debounceTheirCompletion(finalUtteranceText); - } - } - } - - if (message.error) { - console.error('[Them] STT Session Error:', message.error); - } - }; - - const mySttConfig = { - language: effectiveLanguage, - callbacks: { - onmessage: handleMyMessage, - onerror: error => console.error('My STT session error:', error.message), - onclose: event => console.log('My STT session closed:', event.reason), - }, - }; - const theirSttConfig = { - language: effectiveLanguage, - callbacks: { - onmessage: handleTheirMessage, - onerror: error => console.error('Their STT session error:', error.message), - onclose: event => console.log('Their STT session closed:', event.reason), - }, - }; - - if (isGemini) { - [mySttSession, theirSttSession] = await Promise.all([ - connectToGeminiSession(API_KEY, mySttConfig), - connectToGeminiSession(API_KEY, theirSttConfig), - ]); - } else { - [mySttSession, theirSttSession] = await Promise.all([ - connectToOpenAiSession(API_KEY, mySttConfig, keyType), - connectToOpenAiSession(API_KEY, theirSttConfig, keyType), - ]); - } - - console.log('βœ… Both STT sessions initialized successfully.'); - triggerAnalysisIfNeeded(); - - sendToRenderer('session-state-changed', { isActive: true }); - - isInitializingSession = false; - sendToRenderer('session-initializing', false); - sendToRenderer('update-status', 'Connected. Ready to listen.'); - return true; - } catch (error) { - console.error('❌ Failed to initialize STT sessions:', error); - isInitializingSession = false; - sendToRenderer('session-initializing', false); - sendToRenderer('update-status', 'Initialization failed.'); - mySttSession = null; - theirSttSession = null; - return false; - } -} - -function killExistingSystemAudioDump() { - return new Promise(resolve => { - console.log('Checking for existing SystemAudioDump processes...'); - - const killProc = spawn('pkill', ['-f', 'SystemAudioDump'], { - stdio: 'ignore', - }); - - killProc.on('close', code => { - if (code === 0) { - console.log('Killed existing SystemAudioDump processes'); - } else { - console.log('No existing SystemAudioDump processes found'); - } - resolve(); - }); - - killProc.on('error', err => { - console.log('Error checking for existing processes (this is normal):', err.message); - resolve(); - }); - - setTimeout(() => { - killProc.kill(); - resolve(); - }, 2000); - }); -} - -async function startMacOSAudioCapture() { - if (process.platform !== 'darwin' || !theirSttSession) return false; - - await killExistingSystemAudioDump(); - console.log('Starting macOS audio capture for "Them"...'); - - const { app } = require('electron'); - const path = require('path'); - const systemAudioPath = app.isPackaged - ? path.join(process.resourcesPath, 'app.asar.unpacked', 'src', 'assets', 'SystemAudioDump') - : path.join(app.getAppPath(), 'src', 'assets', 'SystemAudioDump'); - - console.log('SystemAudioDump path:', systemAudioPath); - - systemAudioProc = spawn(systemAudioPath, [], { - stdio: ['ignore', 'pipe', 'pipe'], - }); - - if (!systemAudioProc.pid) { - console.error('Failed to start SystemAudioDump'); - return false; - } - - console.log('SystemAudioDump started with PID:', systemAudioProc.pid); - - const CHUNK_DURATION = 0.1; - const SAMPLE_RATE = 24000; - const BYTES_PER_SAMPLE = 2; - const CHANNELS = 2; - const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * CHUNK_DURATION; - - let audioBuffer = Buffer.alloc(0); - - const provider = await getAiProvider(); - const isGemini = provider === 'gemini'; - - systemAudioProc.stdout.on('data', async data => { - audioBuffer = Buffer.concat([audioBuffer, data]); - - while (audioBuffer.length >= CHUNK_SIZE) { - const chunk = audioBuffer.slice(0, CHUNK_SIZE); - audioBuffer = audioBuffer.slice(CHUNK_SIZE); - - const monoChunk = CHANNELS === 2 ? convertStereoToMono(chunk) : chunk; - const base64Data = monoChunk.toString('base64'); - - sendToRenderer('system-audio-data', { data: base64Data }); - - if (theirSttSession) { - try { - // await theirSttSession.sendRealtimeInput(base64Data); - const payload = isGemini - ? { audio: { data: base64Data, mimeType: 'audio/pcm;rate=24000' } } - : base64Data; - await theirSttSession.sendRealtimeInput(payload); - } catch (err) { - console.error('Error sending system audio:', err.message); - } - } - - if (process.env.DEBUG_AUDIO) { - saveDebugAudio(monoChunk, 'system_audio'); - } - } - }); - - systemAudioProc.stderr.on('data', data => { - console.error('SystemAudioDump stderr:', data.toString()); - }); - - systemAudioProc.on('close', code => { - console.log('SystemAudioDump process closed with code:', code); - systemAudioProc = null; - }); - - systemAudioProc.on('error', err => { - console.error('SystemAudioDump process error:', err); - systemAudioProc = null; - }); - - return true; -} - -function convertStereoToMono(stereoBuffer) { - const samples = stereoBuffer.length / 4; - const monoBuffer = Buffer.alloc(samples * 2); - - for (let i = 0; i < samples; i++) { - const leftSample = stereoBuffer.readInt16LE(i * 4); - monoBuffer.writeInt16LE(leftSample, i * 2); - } - - return monoBuffer; -} - -function stopMacOSAudioCapture() { - if (systemAudioProc) { - console.log('Stopping SystemAudioDump...'); - systemAudioProc.kill('SIGTERM'); - systemAudioProc = null; - } -} - -async function sendAudioToOpenAI(base64Data, sttSessionRef) { - if (!sttSessionRef.current) return; - - try { - process.stdout.write('.'); - await sttSessionRef.current.sendRealtimeInput({ - audio: { - data: base64Data, - mimeType: 'audio/pcm;rate=24000', - }, - }); - } catch (error) { - console.error('Error sending audio to OpenAI:', error); - } -} - -function isSessionActive() { - return !!mySttSession && !!theirSttSession; -} - -async function closeSession() { - try { - stopMacOSAudioCapture(); - stopAnalysisInterval(); - - if (currentSessionId) { - await sessionRepository.end(currentSessionId); - console.log(`[DB] Session ${currentSessionId} ended.`); - } - - const closePromises = []; - if (mySttSession) { - closePromises.push(mySttSession.close()); - mySttSession = null; - } - if (theirSttSession) { - closePromises.push(theirSttSession.close()); - theirSttSession = null; - } - - await Promise.all(closePromises); - console.log('All sessions closed.'); - - currentSessionId = null; - conversationHistory = []; - - sendToRenderer('session-state-changed', { isActive: false }); - sendToRenderer('session-did-close'); - - return { success: true }; - } catch (error) { - console.error('Error closing sessions:', error); - return { success: false, error: error.message }; - } -} - -function setupLiveSummaryIpcHandlers() { - ipcMain.handle('is-session-active', async () => { - const isActive = isSessionActive(); - console.log(`Checking session status. Active: ${isActive}`); - return isActive; - }); - - ipcMain.handle('initialize-openai', async (event, profile = 'interview', language = 'en') => { - console.log(`Received initialize-openai request with profile: ${profile}, language: ${language}`); - const success = await initializeLiveSummarySession(language); - return success; - }); - - ipcMain.handle('send-audio-content', async (event, { data, mimeType }) => { - const provider = await getAiProvider(); - const isGemini = provider === 'gemini'; - if (!mySttSession) return { success: false, error: 'User STT session not active' }; - try { - // await mySttSession.sendRealtimeInput(data); - // provider에 λ§žλŠ” ν˜•μ‹μœΌλ‘œ λž˜ν•‘ - const payload = isGemini - ? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } } - : data; // OpenAIλŠ” base64 string κ·ΈλŒ€λ‘œ - - await mySttSession.sendRealtimeInput(payload); - return { success: true }; - } catch (error) { - console.error('Error sending user audio:', error); - return { success: false, error: error.message }; - } - }); - - ipcMain.handle('start-macos-audio', async () => { - if (process.platform !== 'darwin') { - return { success: false, error: 'macOS audio capture only available on macOS' }; - } - try { - const success = await startMacOSAudioCapture(); - return { success }; - } catch (error) { - console.error('Error starting macOS audio capture:', error); - return { success: false, error: error.message }; - } - }); - - ipcMain.handle('stop-macos-audio', async () => { - try { - stopMacOSAudioCapture(); - return { success: true }; - } catch (error) { - console.error('Error stopping macOS audio capture:', error); - return { success: false, error: error.message }; - } - }); - - ipcMain.handle('close-session', async () => { - return await closeSession(); - }); - - ipcMain.handle('update-google-search-setting', async (event, enabled) => { - try { - console.log('Google Search setting updated to:', enabled); - return { success: true }; - } catch (error) { - console.error('Error updating Google Search setting:', error); - return { success: false, error: error.message }; - } - }); -} - -function getConversationHistory() { - return conversationHistory; -} - -module.exports = { - sendToRenderer, - initializeNewSession, - saveConversationTurn, - killExistingSystemAudioDump, - startMacOSAudioCapture, - convertStereoToMono, - stopMacOSAudioCapture, - sendAudioToOpenAI, - setupLiveSummaryIpcHandlers, - isSessionActive, - closeSession, - getConversationHistory, -}; diff --git a/src/features/listen/renderer.js b/src/features/listen/renderer/listenCapture.js similarity index 80% rename from src/features/listen/renderer.js rename to src/features/listen/renderer/listenCapture.js index 99d0bd3..a4eef43 100644 --- a/src/features/listen/renderer.js +++ b/src/features/listen/renderer/listenCapture.js @@ -1,20 +1,29 @@ -// renderer.js const { ipcRenderer } = require('electron'); -const { makeStreamingChatCompletionWithPortkey } = require('../../common/services/aiProviderService.js'); -let mediaStream = null; -let screenshotInterval = null; -let audioContext = null; -let audioProcessor = null; -let micMediaStream = null; -let audioBuffer = []; +// --------------------------- +// Constants & Globals +// --------------------------- const SAMPLE_RATE = 24000; const AUDIO_CHUNK_DURATION = 0.1; const BUFFER_SIZE = 4096; +const isLinux = process.platform === 'linux'; +const isMacOS = process.platform === 'darwin'; + +let mediaStream = null; +let micMediaStream = null; +let screenshotInterval = null; +let audioContext = null; +let audioProcessor = null; +let currentImageQuality = 'medium'; +let lastScreenshotBase64 = null; + let systemAudioBuffer = []; const MAX_SYSTEM_BUFFER_SIZE = 10; +// --------------------------- +// Utility helpers (exact from renderer.js) +// --------------------------- function isVoiceActive(audioFloat32Array, threshold = 0.005) { if (!audioFloat32Array || audioFloat32Array.length === 0) { return false; @@ -31,11 +40,6 @@ function isVoiceActive(audioFloat32Array, threshold = 0.005) { return rms > threshold; } -let currentImageQuality = 'medium'; // Store current image quality for manual screenshots -let lastScreenshotBase64 = null; // Store the latest screenshot - -let realtimeConversationHistory = []; - function base64ToFloat32Array(base64) { const binaryString = atob(base64); const bytes = new Uint8Array(binaryString.length); @@ -54,11 +58,29 @@ function base64ToFloat32Array(base64) { return float32Array; } -async function queryLoginState() { - const userState = await ipcRenderer.invoke('get-current-user'); - return userState; +function convertFloat32ToInt16(float32Array) { + const int16Array = new Int16Array(float32Array.length); + for (let i = 0; i < float32Array.length; i++) { + // Improved scaling to prevent clipping + const s = Math.max(-1, Math.min(1, float32Array[i])); + int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + return int16Array; } +function arrayBufferToBase64(buffer) { + let binary = ''; + const bytes = new Uint8Array(buffer); + const len = bytes.byteLength; + for (let i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + +// --------------------------- +// Complete SimpleAEC implementation (exact from renderer.js) +// --------------------------- class SimpleAEC { constructor() { this.adaptiveFilter = new Float32Array(1024); @@ -179,11 +201,24 @@ class SimpleAEC { let aecProcessor = new SimpleAEC(); -const isLinux = process.platform === 'linux'; -const isMacOS = process.platform === 'darwin'; +// System audio data handler +ipcRenderer.on('system-audio-data', (event, { data }) => { + systemAudioBuffer.push({ + data: data, + timestamp: Date.now(), + }); -window.pickleGlass = window.pickleGlass || {}; + // 였래된 데이터 제거 + if (systemAudioBuffer.length > MAX_SYSTEM_BUFFER_SIZE) { + systemAudioBuffer = systemAudioBuffer.slice(-MAX_SYSTEM_BUFFER_SIZE); + } + console.log('πŸ“₯ Received system audio for AEC reference'); +}); + +// --------------------------- +// Complete token tracker (exact from renderer.js) +// --------------------------- let tokenTracker = { tokens: [], audioStartTime: null, @@ -265,126 +300,201 @@ setInterval(() => { tokenTracker.trackAudioTokens(); }, 2000); -function pickleGlassElement() { - return document.getElementById('pickle-glass'); -} +// --------------------------- +// Audio processing functions (exact from renderer.js) +// --------------------------- +function setupMicProcessing(micStream) { + const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); + const micSource = micAudioContext.createMediaStreamSource(micStream); + const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); -function convertFloat32ToInt16(float32Array) { - const int16Array = new Int16Array(float32Array.length); - for (let i = 0; i < float32Array.length; i++) { - // Improved scaling to prevent clipping - const s = Math.max(-1, Math.min(1, float32Array[i])); - int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff; - } - return int16Array; -} + let audioBuffer = []; + const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION; -function arrayBufferToBase64(buffer) { - let binary = ''; - const bytes = new Uint8Array(buffer); - const len = bytes.byteLength; - for (let i = 0; i < len; i++) { - binary += String.fromCharCode(bytes[i]); - } - return btoa(binary); -} + micProcessor.onaudioprocess = async e => { + const inputData = e.inputBuffer.getChannelData(0); + audioBuffer.push(...inputData); -async function initializeopenai(profile = 'interview', language = 'en') { - // The API key is now handled in the main process from .env file. - // We just need to trigger the initialization. - try { - console.log(`Requesting OpenAI initialization with profile: ${profile}, language: ${language}`); - const success = await ipcRenderer.invoke('initialize-openai', profile, language); - if (success) { - // The status will be updated via 'update-status' event from the main process. - console.log('OpenAI initialization successful.'); - } else { - console.error('OpenAI initialization failed.'); - const appElement = pickleGlassElement(); - if (appElement && typeof appElement.setStatus === 'function') { - appElement.setStatus('Initialization Failed'); + while (audioBuffer.length >= samplesPerChunk) { + let chunk = audioBuffer.splice(0, samplesPerChunk); + let processedChunk = new Float32Array(chunk); + + // Check for system audio and apply AEC only if voice is active + if (aecProcessor && systemAudioBuffer.length > 0) { + const latestSystemAudio = systemAudioBuffer[systemAudioBuffer.length - 1]; + const systemFloat32 = base64ToFloat32Array(latestSystemAudio.data); + + // Apply AEC only when system audio has active speech + if (isVoiceActive(systemFloat32)) { + processedChunk = aecProcessor.process(new Float32Array(chunk), systemFloat32); + console.log('πŸ”Š Applied AEC because system audio is active'); + } } + + const pcmData16 = convertFloat32ToInt16(processedChunk); + const base64Data = arrayBufferToBase64(pcmData16.buffer); + + await ipcRenderer.invoke('send-audio-content', { + data: base64Data, + mimeType: 'audio/pcm;rate=24000', + }); + } + }; + + micSource.connect(micProcessor); + micProcessor.connect(micAudioContext.destination); + + audioProcessor = micProcessor; +} + +function setupLinuxMicProcessing(micStream) { + // Setup microphone audio processing for Linux + const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); + const micSource = micAudioContext.createMediaStreamSource(micStream); + const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); + + let audioBuffer = []; + const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION; + + micProcessor.onaudioprocess = async e => { + const inputData = e.inputBuffer.getChannelData(0); + audioBuffer.push(...inputData); + + // Process audio in chunks + while (audioBuffer.length >= samplesPerChunk) { + const chunk = audioBuffer.splice(0, samplesPerChunk); + const pcmData16 = convertFloat32ToInt16(chunk); + const base64Data = arrayBufferToBase64(pcmData16.buffer); + + await ipcRenderer.invoke('send-audio-content', { + data: base64Data, + mimeType: 'audio/pcm;rate=24000', + }); + } + }; + + micSource.connect(micProcessor); + micProcessor.connect(micAudioContext.destination); + + // Store processor reference for cleanup + audioProcessor = micProcessor; +} + +function setupWindowsLoopbackProcessing() { + // Setup audio processing for Windows loopback audio only + audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); + const source = audioContext.createMediaStreamSource(mediaStream); + audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); + + let audioBuffer = []; + const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION; + + audioProcessor.onaudioprocess = async e => { + const inputData = e.inputBuffer.getChannelData(0); + audioBuffer.push(...inputData); + + // Process audio in chunks + while (audioBuffer.length >= samplesPerChunk) { + const chunk = audioBuffer.splice(0, samplesPerChunk); + const pcmData16 = convertFloat32ToInt16(chunk); + const base64Data = arrayBufferToBase64(pcmData16.buffer); + + await ipcRenderer.invoke('send-audio-content', { + data: base64Data, + mimeType: 'audio/pcm;rate=24000', + }); + } + }; + + source.connect(audioProcessor); + audioProcessor.connect(audioContext.destination); +} + +// --------------------------- +// Screenshot functions (exact from renderer.js) +// --------------------------- +async function captureScreenshot(imageQuality = 'medium', isManual = false) { + console.log(`Capturing ${isManual ? 'manual' : 'automated'} screenshot...`); + + // Check rate limiting for automated screenshots only + if (!isManual && tokenTracker.shouldThrottle()) { + console.log('⚠️ Automated screenshot skipped due to rate limiting'); + return; + } + + try { + // Request screenshot from main process + const result = await ipcRenderer.invoke('capture-screenshot', { + quality: imageQuality, + }); + + if (result.success && result.base64) { + // Store the latest screenshot + lastScreenshotBase64 = result.base64; + + // Note: sendResult is not defined in the original, this was likely an error + // Commenting out this section as it references undefined variable + /* + if (sendResult.success) { + // Track image tokens after successful send + const imageTokens = tokenTracker.calculateImageTokens(result.width || 1920, result.height || 1080); + tokenTracker.addTokens(imageTokens, 'image'); + console.log(`πŸ“Š Image sent successfully - ${imageTokens} tokens used (${result.width}x${result.height})`); + } else { + console.error('Failed to send image:', sendResult.error); + } + */ + } else { + console.error('Failed to capture screenshot:', result.error); } } catch (error) { - console.error('Error during OpenAI initialization IPC call:', error); - const appElement = pickleGlassElement(); - if (appElement && typeof appElement.setStatus === 'function') { - appElement.setStatus('Error'); - } + console.error('Error capturing screenshot:', error); } } +async function captureManualScreenshot(imageQuality = null) { + console.log('Manual screenshot triggered'); + const quality = imageQuality || currentImageQuality; + await captureScreenshot(quality, true); +} -ipcRenderer.on('system-audio-data', (event, { data }) => { - systemAudioBuffer.push({ - data: data, - timestamp: Date.now(), - }); +async function getCurrentScreenshot() { + try { + // First try to get a fresh screenshot from main process + const result = await ipcRenderer.invoke('get-current-screenshot'); - // 였래된 데이터 제거 - if (systemAudioBuffer.length > MAX_SYSTEM_BUFFER_SIZE) { - systemAudioBuffer = systemAudioBuffer.slice(-MAX_SYSTEM_BUFFER_SIZE); - } - - console.log('πŸ“₯ Received system audio for AEC reference'); -}); - -// Listen for status updates -ipcRenderer.on('update-status', (event, status) => { - console.log('Status update:', status); - pickleGlass.e().setStatus(status); -}); - -// Listen for real-time STT updates -ipcRenderer.on('stt-update', (event, data) => { - console.log('Renderer.js stt-update', data); - const { speaker, text, isFinal, isPartial, timestamp } = data; - - if (isPartial) { - console.log(`πŸ”„ [${speaker} - partial]: ${text}`); - } else if (isFinal) { - console.log(`βœ… [${speaker} - final]: ${text}`); - - const speakerText = speaker.toLowerCase(); - const conversationText = `${speakerText}: ${text.trim()}`; - - realtimeConversationHistory.push(conversationText); - - if (realtimeConversationHistory.length > 30) { - realtimeConversationHistory = realtimeConversationHistory.slice(-30); + if (result.success && result.base64) { + console.log('πŸ“Έ Got fresh screenshot from main process'); + return result.base64; } - console.log(`πŸ“ Updated realtime conversation history: ${realtimeConversationHistory.length} texts`); - console.log(`πŸ“‹ Latest text: ${conversationText}`); - } - - if (pickleGlass.e() && typeof pickleGlass.e().updateRealtimeTranscription === 'function') { - pickleGlass.e().updateRealtimeTranscription({ - speaker, - text, - isFinal, - isPartial, - timestamp, + // If no screenshot available, capture one now + console.log('πŸ“Έ No screenshot available, capturing new one'); + const captureResult = await ipcRenderer.invoke('capture-screenshot', { + quality: currentImageQuality, }); + + if (captureResult.success && captureResult.base64) { + lastScreenshotBase64 = captureResult.base64; + return captureResult.base64; + } + + // Fallback to last stored screenshot + if (lastScreenshotBase64) { + console.log('πŸ“Έ Using cached screenshot'); + return lastScreenshotBase64; + } + + throw new Error('Failed to get screenshot'); + } catch (error) { + console.error('Error getting current screenshot:', error); + return null; } -}); - - -ipcRenderer.on('update-structured-data', (_, structuredData) => { - console.log('πŸ“₯ Received structured data update:', structuredData); - window.pickleGlass.structuredData = structuredData; - window.pickleGlass.setStructuredData(structuredData); -}); -window.pickleGlass.structuredData = { - summary: [], - topic: { header: '', bullets: [] }, - actions: [], -}; -window.pickleGlass.setStructuredData = data => { - window.pickleGlass.structuredData = data; - pickleGlass.e()?.updateStructuredData?.(data); -}; +} +// --------------------------- +// Main capture functions (exact from renderer.js) +// --------------------------- async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'medium') { // Store the image quality for manual screenshots currentImageQuality = imageQuality; @@ -490,12 +600,6 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu setupWindowsLoopbackProcessing(); } - // console.log('MediaStream obtained:', { - // hasVideo: mediaStream.getVideoTracks().length > 0, - // hasAudio: mediaStream.getAudioTracks().length > 0, - // videoTrack: mediaStream.getVideoTracks()[0]?.getSettings(), - // }); - // Start capturing screenshots - check if manual mode if (screenshotIntervalSeconds === 'manual' || screenshotIntervalSeconds === 'Manual') { console.log('Manual mode enabled - screenshots will be captured on demand only'); @@ -511,162 +615,11 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu } } catch (err) { console.error('Error starting capture:', err); - pickleGlass.e().setStatus('error'); + // Note: pickleGlass.e() is not available in this context, commenting out + // pickleGlass.e().setStatus('error'); } } -function setupMicProcessing(micStream) { - const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); - const micSource = micAudioContext.createMediaStreamSource(micStream); - const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); - - let audioBuffer = []; - const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION; - - micProcessor.onaudioprocess = async e => { - const inputData = e.inputBuffer.getChannelData(0); - audioBuffer.push(...inputData); - - while (audioBuffer.length >= samplesPerChunk) { - let chunk = audioBuffer.splice(0, samplesPerChunk); - let processedChunk = new Float32Array(chunk); - - // Check for system audio and apply AEC only if voice is active - if (aecProcessor && systemAudioBuffer.length > 0) { - const latestSystemAudio = systemAudioBuffer[systemAudioBuffer.length - 1]; - const systemFloat32 = base64ToFloat32Array(latestSystemAudio.data); - - // Apply AEC only when system audio has active speech - if (isVoiceActive(systemFloat32)) { - processedChunk = aecProcessor.process(new Float32Array(chunk), systemFloat32); - console.log('πŸ”Š Applied AEC because system audio is active'); - } - } - - const pcmData16 = convertFloat32ToInt16(processedChunk); - const base64Data = arrayBufferToBase64(pcmData16.buffer); - - await ipcRenderer.invoke('send-audio-content', { - data: base64Data, - mimeType: 'audio/pcm;rate=24000', - }); - } - }; - - micSource.connect(micProcessor); - micProcessor.connect(micAudioContext.destination); - - audioProcessor = micProcessor; -} -////////// for index & subjects ////////// - -function setupLinuxMicProcessing(micStream) { - // Setup microphone audio processing for Linux - const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); - const micSource = micAudioContext.createMediaStreamSource(micStream); - const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); - - let audioBuffer = []; - const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION; - - micProcessor.onaudioprocess = async e => { - const inputData = e.inputBuffer.getChannelData(0); - audioBuffer.push(...inputData); - - // Process audio in chunks - while (audioBuffer.length >= samplesPerChunk) { - const chunk = audioBuffer.splice(0, samplesPerChunk); - const pcmData16 = convertFloat32ToInt16(chunk); - const base64Data = arrayBufferToBase64(pcmData16.buffer); - - await ipcRenderer.invoke('send-audio-content', { - data: base64Data, - mimeType: 'audio/pcm;rate=24000', - }); - } - }; - - micSource.connect(micProcessor); - micProcessor.connect(micAudioContext.destination); - - // Store processor reference for cleanup - audioProcessor = micProcessor; -} - -function setupWindowsLoopbackProcessing() { - // Setup audio processing for Windows loopback audio only - audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); - const source = audioContext.createMediaStreamSource(mediaStream); - audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); - - let audioBuffer = []; - const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION; - - audioProcessor.onaudioprocess = async e => { - const inputData = e.inputBuffer.getChannelData(0); - audioBuffer.push(...inputData); - - // Process audio in chunks - while (audioBuffer.length >= samplesPerChunk) { - const chunk = audioBuffer.splice(0, samplesPerChunk); - const pcmData16 = convertFloat32ToInt16(chunk); - const base64Data = arrayBufferToBase64(pcmData16.buffer); - - await ipcRenderer.invoke('send-audio-content', { - data: base64Data, - mimeType: 'audio/pcm;rate=24000', - }); - } - }; - - source.connect(audioProcessor); - audioProcessor.connect(audioContext.destination); -} - -async function captureScreenshot(imageQuality = 'medium', isManual = false) { - console.log(`Capturing ${isManual ? 'manual' : 'automated'} screenshot...`); - - // Check rate limiting for automated screenshots only - if (!isManual && tokenTracker.shouldThrottle()) { - console.log('⚠️ Automated screenshot skipped due to rate limiting'); - return; - } - - try { - // Request screenshot from main process - const result = await ipcRenderer.invoke('capture-screenshot', { - quality: imageQuality, - }); - - if (result.success && result.base64) { - // Store the latest screenshot - lastScreenshotBase64 = result.base64; - - if (sendResult.success) { - // Track image tokens after successful send - const imageTokens = tokenTracker.calculateImageTokens(result.width || 1920, result.height || 1080); - tokenTracker.addTokens(imageTokens, 'image'); - console.log(`πŸ“Š Image sent successfully - ${imageTokens} tokens used (${result.width}x${result.height})`); - } else { - console.error('Failed to send image:', sendResult.error); - } - } else { - console.error('Failed to capture screenshot:', result.error); - } - } catch (error) { - console.error('Error capturing screenshot:', error); - } -} - -async function captureManualScreenshot(imageQuality = null) { - console.log('Manual screenshot triggered'); - const quality = imageQuality || currentImageQuality; - await captureScreenshot(quality, true); -} - -// Expose functions to global scope for external access -window.captureManualScreenshot = captureManualScreenshot; - function stopCapture() { if (screenshotInterval) { clearInterval(screenshotInterval); @@ -706,76 +659,25 @@ function stopCapture() { } } -async function getCurrentScreenshot() { - try { - // First try to get a fresh screenshot from main process - const result = await ipcRenderer.invoke('get-current-screenshot'); - - if (result.success && result.base64) { - console.log('πŸ“Έ Got fresh screenshot from main process'); - return result.base64; - } - - // If no screenshot available, capture one now - console.log('πŸ“Έ No screenshot available, capturing new one'); - const captureResult = await ipcRenderer.invoke('capture-screenshot', { - quality: currentImageQuality, - }); - - if (captureResult.success && captureResult.base64) { - lastScreenshotBase64 = captureResult.base64; - return captureResult.base64; - } - - // Fallback to last stored screenshot - if (lastScreenshotBase64) { - console.log('πŸ“Έ Using cached screenshot'); - return lastScreenshotBase64; - } - - throw new Error('Failed to get screenshot'); - } catch (error) { - console.error('Error getting current screenshot:', error); - return null; - } -} - -function formatRealtimeConversationHistory() { - if (realtimeConversationHistory.length === 0) return 'No conversation history available.'; - - return realtimeConversationHistory.slice(-30).join('\n'); -} - -window.pickleGlass = { - initializeopenai, +// --------------------------- +// Exports & global registration +// --------------------------- +module.exports = { startCapture, stopCapture, - isLinux: isLinux, - isMacOS: isMacOS, - e: pickleGlassElement, + captureManualScreenshot, + getCurrentScreenshot, + isLinux, + isMacOS, }; -// ------------------------------------------------------- -// πŸ”” React to session state changes from the main process -// When the session ends (isActive === false), ensure we stop -// all local capture pipelines (mic, screen, etc.). -// ------------------------------------------------------- -ipcRenderer.on('session-state-changed', (_event, { isActive }) => { - if (!isActive) { - console.log('[Renderer] Session ended – stopping local capture'); - stopCapture(); - } else { - console.log('[Renderer] New session started – clearing in-memory history and summaries'); - - // Reset live conversation & analysis caches - realtimeConversationHistory = []; - - const blankData = { - summary: [], - topic: { header: '', bullets: [] }, - actions: [], - followUps: [], - }; - window.pickleGlass.setStructuredData(blankData); - } -}); +// Expose functions to global scope for external access (exact from renderer.js) +if (typeof window !== 'undefined') { + window.captureManualScreenshot = captureManualScreenshot; + window.listenCapture = module.exports; + window.pickleGlass = window.pickleGlass || {}; + window.pickleGlass.startCapture = startCapture; + window.pickleGlass.stopCapture = stopCapture; + window.pickleGlass.captureManualScreenshot = captureManualScreenshot; + window.pickleGlass.getCurrentScreenshot = getCurrentScreenshot; +} \ No newline at end of file diff --git a/src/features/listen/renderer/renderer.js b/src/features/listen/renderer/renderer.js new file mode 100644 index 0000000..9916909 --- /dev/null +++ b/src/features/listen/renderer/renderer.js @@ -0,0 +1,139 @@ +// renderer.js +const { ipcRenderer } = require('electron'); +const { makeStreamingChatCompletionWithPortkey } = require('../../../common/services/aiProviderService.js'); +const listenCapture = require('./listenCapture.js'); + +let realtimeConversationHistory = []; + +async function queryLoginState() { + const userState = await ipcRenderer.invoke('get-current-user'); + return userState; +} + +function pickleGlassElement() { + return document.getElementById('pickle-glass'); +} + +async function initializeopenai(profile = 'interview', language = 'en') { + // The API key is now handled in the main process from .env file. + // We just need to trigger the initialization. + try { + console.log(`Requesting OpenAI initialization with profile: ${profile}, language: ${language}`); + const success = await ipcRenderer.invoke('initialize-openai', profile, language); + if (success) { + // The status will be updated via 'update-status' event from the main process. + console.log('OpenAI initialization successful.'); + } else { + console.error('OpenAI initialization failed.'); + const appElement = pickleGlassElement(); + if (appElement && typeof appElement.setStatus === 'function') { + appElement.setStatus('Initialization Failed'); + } + } + } catch (error) { + console.error('Error during OpenAI initialization IPC call:', error); + const appElement = pickleGlassElement(); + if (appElement && typeof appElement.setStatus === 'function') { + appElement.setStatus('Error'); + } + } +} + +// Listen for status updates +ipcRenderer.on('update-status', (event, status) => { + console.log('Status update:', status); + pickleGlass.e().setStatus(status); +}); + +// Listen for real-time STT updates +ipcRenderer.on('stt-update', (event, data) => { + console.log('Renderer.js stt-update', data); + const { speaker, text, isFinal, isPartial, timestamp } = data; + + if (isPartial) { + console.log(`πŸ”„ [${speaker} - partial]: ${text}`); + } else if (isFinal) { + console.log(`βœ… [${speaker} - final]: ${text}`); + + const speakerText = speaker.toLowerCase(); + const conversationText = `${speakerText}: ${text.trim()}`; + + realtimeConversationHistory.push(conversationText); + + if (realtimeConversationHistory.length > 30) { + realtimeConversationHistory = realtimeConversationHistory.slice(-30); + } + + console.log(`πŸ“ Updated realtime conversation history: ${realtimeConversationHistory.length} texts`); + console.log(`πŸ“‹ Latest text: ${conversationText}`); + } + + if (pickleGlass.e() && typeof pickleGlass.e().updateRealtimeTranscription === 'function') { + pickleGlass.e().updateRealtimeTranscription({ + speaker, + text, + isFinal, + isPartial, + timestamp, + }); + } +}); + +ipcRenderer.on('update-structured-data', (_, structuredData) => { + console.log('πŸ“₯ Received structured data update:', structuredData); + window.pickleGlass.structuredData = structuredData; + window.pickleGlass.setStructuredData(structuredData); +}); + +window.pickleGlass.structuredData = { + summary: [], + topic: { header: '', bullets: [] }, + actions: [], +}; + +window.pickleGlass.setStructuredData = data => { + window.pickleGlass.structuredData = data; + pickleGlass.e()?.updateStructuredData?.(data); +}; + +function formatRealtimeConversationHistory() { + if (realtimeConversationHistory.length === 0) return 'No conversation history available.'; + + return realtimeConversationHistory.slice(-30).join('\n'); +} + +window.pickleGlass = { + initializeopenai, + startCapture: listenCapture.startCapture, + stopCapture: listenCapture.stopCapture, + isLinux: listenCapture.isLinux, + isMacOS: listenCapture.isMacOS, + captureManualScreenshot: listenCapture.captureManualScreenshot, + getCurrentScreenshot: listenCapture.getCurrentScreenshot, + e: pickleGlassElement, +}; + +// ------------------------------------------------------- +// πŸ”” React to session state changes from the main process +// When the session ends (isActive === false), ensure we stop +// all local capture pipelines (mic, screen, etc.). +// ------------------------------------------------------- +ipcRenderer.on('session-state-changed', (_event, { isActive }) => { + if (!isActive) { + console.log('[Renderer] Session ended – stopping local capture'); + listenCapture.stopCapture(); + } else { + console.log('[Renderer] New session started – clearing in-memory history and summaries'); + + // Reset live conversation & analysis caches + realtimeConversationHistory = []; + + const blankData = { + summary: [], + topic: { header: '', bullets: [] }, + actions: [], + followUps: [], + }; + window.pickleGlass.setStructuredData(blankData); + } +}); diff --git a/src/features/listen/stt/repositories/index.js b/src/features/listen/stt/repositories/index.js new file mode 100644 index 0000000..6de1a98 --- /dev/null +++ b/src/features/listen/stt/repositories/index.js @@ -0,0 +1,5 @@ +const sttRepository = require('./sqlite.repository'); + +module.exports = { + ...sttRepository, +}; \ No newline at end of file diff --git a/src/features/listen/stt/repositories/sqlite.repository.js b/src/features/listen/stt/repositories/sqlite.repository.js new file mode 100644 index 0000000..4de47bd --- /dev/null +++ b/src/features/listen/stt/repositories/sqlite.repository.js @@ -0,0 +1,37 @@ +const sqliteClient = require('../../../../common/services/sqliteClient'); + +function addTranscript({ sessionId, speaker, text }) { + const db = sqliteClient.getDb(); + return new Promise((resolve, reject) => { + const transcriptId = require('crypto').randomUUID(); + const now = Math.floor(Date.now() / 1000); + const query = `INSERT INTO transcripts (id, session_id, start_at, speaker, text, created_at) VALUES (?, ?, ?, ?, ?, ?)`; + db.run(query, [transcriptId, sessionId, now, speaker, text, now], function(err) { + if (err) { + console.error('Error adding transcript:', err); + reject(err); + } else { + resolve({ id: transcriptId }); + } + }); + }); +} + +function getAllTranscriptsBySessionId(sessionId) { + const db = sqliteClient.getDb(); + return new Promise((resolve, reject) => { + const query = "SELECT * FROM transcripts WHERE session_id = ? ORDER BY start_at ASC"; + db.all(query, [sessionId], (err, rows) => { + if (err) { + reject(err); + } else { + resolve(rows); + } + }); + }); +} + +module.exports = { + addTranscript, + getAllTranscriptsBySessionId, +}; \ No newline at end of file diff --git a/src/features/listen/stt/sttService.js b/src/features/listen/stt/sttService.js new file mode 100644 index 0000000..4530ef0 --- /dev/null +++ b/src/features/listen/stt/sttService.js @@ -0,0 +1,480 @@ +const { BrowserWindow } = require('electron'); +const { spawn } = require('child_process'); +const { connectToGeminiSession } = require('../../../common/services/googleGeminiClient.js'); +const { connectToOpenAiSession } = require('../../../common/services/openAiClient.js'); +const { getStoredApiKey, getStoredProvider } = require('../../../electron/windowManager'); + +const COMPLETION_DEBOUNCE_MS = 2000; + +class SttService { + constructor() { + this.mySttSession = null; + this.theirSttSession = null; + this.myCurrentUtterance = ''; + this.theirCurrentUtterance = ''; + + this.myLastPartialText = ''; + this.theirLastPartialText = ''; + this.myInactivityTimer = null; + this.theirInactivityTimer = null; + + // Turn-completion debouncing + this.myCompletionBuffer = ''; + this.theirCompletionBuffer = ''; + this.myCompletionTimer = null; + this.theirCompletionTimer = null; + + // System audio capture + this.systemAudioProc = null; + + // Callbacks + this.onTranscriptionComplete = null; + this.onStatusUpdate = null; + } + + setCallbacks({ onTranscriptionComplete, onStatusUpdate }) { + this.onTranscriptionComplete = onTranscriptionComplete; + this.onStatusUpdate = onStatusUpdate; + } + + async getApiKey() { + const storedKey = await getStoredApiKey(); + if (storedKey) { + console.log('[SttService] Using stored API key'); + return storedKey; + } + + const envKey = process.env.OPENAI_API_KEY; + if (envKey) { + console.log('[SttService] Using environment API key'); + return envKey; + } + + console.error('[SttService] No API key found in storage or environment'); + return null; + } + + async getAiProvider() { + try { + const { ipcRenderer } = require('electron'); + const provider = await ipcRenderer.invoke('get-ai-provider'); + return provider || 'openai'; + } catch (error) { + return getStoredProvider ? getStoredProvider() : 'openai'; + } + } + + sendToRenderer(channel, data) { + BrowserWindow.getAllWindows().forEach(win => { + if (!win.isDestroyed()) { + win.webContents.send(channel, data); + } + }); + } + + flushMyCompletion() { + if (!this.myCompletionBuffer.trim()) return; + + const finalText = this.myCompletionBuffer.trim(); + + // Notify completion callback + if (this.onTranscriptionComplete) { + this.onTranscriptionComplete('Me', finalText); + } + + // Send to renderer as final + this.sendToRenderer('stt-update', { + speaker: 'Me', + text: finalText, + isPartial: false, + isFinal: true, + timestamp: Date.now(), + }); + + this.myCompletionBuffer = ''; + this.myCompletionTimer = null; + this.myCurrentUtterance = ''; + + if (this.onStatusUpdate) { + this.onStatusUpdate('Listening...'); + } + } + + flushTheirCompletion() { + if (!this.theirCompletionBuffer.trim()) return; + + const finalText = this.theirCompletionBuffer.trim(); + + // Notify completion callback + if (this.onTranscriptionComplete) { + this.onTranscriptionComplete('Them', finalText); + } + + // Send to renderer as final + this.sendToRenderer('stt-update', { + speaker: 'Them', + text: finalText, + isPartial: false, + isFinal: true, + timestamp: Date.now(), + }); + + this.theirCompletionBuffer = ''; + this.theirCompletionTimer = null; + this.theirCurrentUtterance = ''; + + if (this.onStatusUpdate) { + this.onStatusUpdate('Listening...'); + } + } + + debounceMyCompletion(text) { + // μƒλŒ€λ°©μ΄ λ§ν•˜κ³  있던 경우, ν™”μžκ°€ λ³€κ²½λ˜μ—ˆμœΌλ―€λ‘œ μ¦‰μ‹œ μƒλŒ€λ°©μ˜ 말풍선을 μ™„μ„±ν•©λ‹ˆλ‹€. + if (this.theirCompletionTimer) { + clearTimeout(this.theirCompletionTimer); + this.flushTheirCompletion(); + } + + this.myCompletionBuffer += (this.myCompletionBuffer ? ' ' : '') + text; + + if (this.myCompletionTimer) clearTimeout(this.myCompletionTimer); + this.myCompletionTimer = setTimeout(() => this.flushMyCompletion(), COMPLETION_DEBOUNCE_MS); + } + + debounceTheirCompletion(text) { + // λ‚΄κ°€ λ§ν•˜κ³  있던 경우, ν™”μžκ°€ λ³€κ²½λ˜μ—ˆμœΌλ―€λ‘œ μ¦‰μ‹œ λ‚΄ 말풍선을 μ™„μ„±ν•©λ‹ˆλ‹€. + if (this.myCompletionTimer) { + clearTimeout(this.myCompletionTimer); + this.flushMyCompletion(); + } + + this.theirCompletionBuffer += (this.theirCompletionBuffer ? ' ' : '') + text; + + if (this.theirCompletionTimer) clearTimeout(this.theirCompletionTimer); + this.theirCompletionTimer = setTimeout(() => this.flushTheirCompletion(), COMPLETION_DEBOUNCE_MS); + } + + async initializeSttSessions(language = 'en') { + const effectiveLanguage = process.env.OPENAI_TRANSCRIBE_LANG || language || 'en'; + + const API_KEY = await this.getApiKey(); + if (!API_KEY) { + throw new Error('No API key available'); + } + + const provider = await this.getAiProvider(); + const isGemini = provider === 'gemini'; + console.log(`[SttService] Initializing STT for provider: ${provider}`); + + const handleMyMessage = message => { + if (isGemini) { + const text = message.serverContent?.inputTranscription?.text || ''; + if (text && text.trim()) { + const finalUtteranceText = text.trim().replace(//g, '').trim(); + if (finalUtteranceText && finalUtteranceText !== '.') { + this.debounceMyCompletion(finalUtteranceText); + } + } + } else { + const type = message.type; + const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || ''; + + if (type === 'conversation.item.input_audio_transcription.delta') { + if (this.myCompletionTimer) clearTimeout(this.myCompletionTimer); + this.myCompletionTimer = null; + this.myCurrentUtterance += text; + const continuousText = this.myCompletionBuffer + (this.myCompletionBuffer ? ' ' : '') + this.myCurrentUtterance; + if (text && !text.includes('vq_lbr_audio_')) { + this.sendToRenderer('stt-update', { + speaker: 'Me', + text: continuousText, + isPartial: true, + isFinal: false, + timestamp: Date.now(), + }); + } + } else if (type === 'conversation.item.input_audio_transcription.completed') { + if (text && text.trim()) { + const finalUtteranceText = text.trim(); + this.myCurrentUtterance = ''; + this.debounceMyCompletion(finalUtteranceText); + } + } + } + + if (message.error) { + console.error('[Me] STT Session Error:', message.error); + } + }; + + const handleTheirMessage = message => { + if (isGemini) { + const text = message.serverContent?.inputTranscription?.text || ''; + if (text && text.trim()) { + const finalUtteranceText = text.trim().replace(//g, '').trim(); + if (finalUtteranceText && finalUtteranceText !== '.') { + this.debounceTheirCompletion(finalUtteranceText); + } + } + } else { + const type = message.type; + const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || ''; + if (type === 'conversation.item.input_audio_transcription.delta') { + if (this.theirCompletionTimer) clearTimeout(this.theirCompletionTimer); + this.theirCompletionTimer = null; + this.theirCurrentUtterance += text; + const continuousText = this.theirCompletionBuffer + (this.theirCompletionBuffer ? ' ' : '') + this.theirCurrentUtterance; + if (text && !text.includes('vq_lbr_audio_')) { + this.sendToRenderer('stt-update', { + speaker: 'Them', + text: continuousText, + isPartial: true, + isFinal: false, + timestamp: Date.now(), + }); + } + } else if (type === 'conversation.item.input_audio_transcription.completed') { + if (text && text.trim()) { + const finalUtteranceText = text.trim(); + this.theirCurrentUtterance = ''; + this.debounceTheirCompletion(finalUtteranceText); + } + } + } + + if (message.error) { + console.error('[Them] STT Session Error:', message.error); + } + }; + + const mySttConfig = { + language: effectiveLanguage, + callbacks: { + onmessage: handleMyMessage, + onerror: error => console.error('My STT session error:', error.message), + onclose: event => console.log('My STT session closed:', event.reason), + }, + }; + + const theirSttConfig = { + language: effectiveLanguage, + callbacks: { + onmessage: handleTheirMessage, + onerror: error => console.error('Their STT session error:', error.message), + onclose: event => console.log('Their STT session closed:', event.reason), + }, + }; + + // Determine key type based on auth status + const authService = require('../../../common/services/authService'); + const userState = authService.getCurrentUser(); + const loggedIn = userState.isLoggedIn; + const keyType = loggedIn ? 'vKey' : 'apiKey'; + + if (isGemini) { + [this.mySttSession, this.theirSttSession] = await Promise.all([ + connectToGeminiSession(API_KEY, mySttConfig), + connectToGeminiSession(API_KEY, theirSttConfig), + ]); + } else { + [this.mySttSession, this.theirSttSession] = await Promise.all([ + connectToOpenAiSession(API_KEY, mySttConfig, keyType), + connectToOpenAiSession(API_KEY, theirSttConfig, keyType), + ]); + } + + console.log('βœ… Both STT sessions initialized successfully.'); + return true; + } + + async sendAudioContent(data, mimeType) { + const provider = await this.getAiProvider(); + const isGemini = provider === 'gemini'; + + if (!this.mySttSession) { + throw new Error('User STT session not active'); + } + + const payload = isGemini + ? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } } + : data; + + await this.mySttSession.sendRealtimeInput(payload); + } + + killExistingSystemAudioDump() { + return new Promise(resolve => { + console.log('Checking for existing SystemAudioDump processes...'); + + const killProc = spawn('pkill', ['-f', 'SystemAudioDump'], { + stdio: 'ignore', + }); + + killProc.on('close', code => { + if (code === 0) { + console.log('Killed existing SystemAudioDump processes'); + } else { + console.log('No existing SystemAudioDump processes found'); + } + resolve(); + }); + + killProc.on('error', err => { + console.log('Error checking for existing processes (this is normal):', err.message); + resolve(); + }); + + setTimeout(() => { + killProc.kill(); + resolve(); + }, 2000); + }); + } + + async startMacOSAudioCapture() { + if (process.platform !== 'darwin' || !this.theirSttSession) return false; + + await this.killExistingSystemAudioDump(); + console.log('Starting macOS audio capture for "Them"...'); + + const { app } = require('electron'); + const path = require('path'); + const systemAudioPath = app.isPackaged + ? path.join(process.resourcesPath, 'app.asar.unpacked', 'src', 'assets', 'SystemAudioDump') + : path.join(app.getAppPath(), 'src', 'assets', 'SystemAudioDump'); + + console.log('SystemAudioDump path:', systemAudioPath); + + this.systemAudioProc = spawn(systemAudioPath, [], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + + if (!this.systemAudioProc.pid) { + console.error('Failed to start SystemAudioDump'); + return false; + } + + console.log('SystemAudioDump started with PID:', this.systemAudioProc.pid); + + const CHUNK_DURATION = 0.1; + const SAMPLE_RATE = 24000; + const BYTES_PER_SAMPLE = 2; + const CHANNELS = 2; + const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * CHUNK_DURATION; + + let audioBuffer = Buffer.alloc(0); + + const provider = await this.getAiProvider(); + const isGemini = provider === 'gemini'; + + this.systemAudioProc.stdout.on('data', async data => { + audioBuffer = Buffer.concat([audioBuffer, data]); + + while (audioBuffer.length >= CHUNK_SIZE) { + const chunk = audioBuffer.slice(0, CHUNK_SIZE); + audioBuffer = audioBuffer.slice(CHUNK_SIZE); + + const monoChunk = CHANNELS === 2 ? this.convertStereoToMono(chunk) : chunk; + const base64Data = monoChunk.toString('base64'); + + this.sendToRenderer('system-audio-data', { data: base64Data }); + + if (this.theirSttSession) { + try { + const payload = isGemini + ? { audio: { data: base64Data, mimeType: 'audio/pcm;rate=24000' } } + : base64Data; + await this.theirSttSession.sendRealtimeInput(payload); + } catch (err) { + console.error('Error sending system audio:', err.message); + } + } + } + }); + + this.systemAudioProc.stderr.on('data', data => { + console.error('SystemAudioDump stderr:', data.toString()); + }); + + this.systemAudioProc.on('close', code => { + console.log('SystemAudioDump process closed with code:', code); + this.systemAudioProc = null; + }); + + this.systemAudioProc.on('error', err => { + console.error('SystemAudioDump process error:', err); + this.systemAudioProc = null; + }); + + return true; + } + + convertStereoToMono(stereoBuffer) { + const samples = stereoBuffer.length / 4; + const monoBuffer = Buffer.alloc(samples * 2); + + for (let i = 0; i < samples; i++) { + const leftSample = stereoBuffer.readInt16LE(i * 4); + monoBuffer.writeInt16LE(leftSample, i * 2); + } + + return monoBuffer; + } + + stopMacOSAudioCapture() { + if (this.systemAudioProc) { + console.log('Stopping SystemAudioDump...'); + this.systemAudioProc.kill('SIGTERM'); + this.systemAudioProc = null; + } + } + + isSessionActive() { + return !!this.mySttSession && !!this.theirSttSession; + } + + async closeSessions() { + this.stopMacOSAudioCapture(); + + // Clear timers + if (this.myInactivityTimer) { + clearTimeout(this.myInactivityTimer); + this.myInactivityTimer = null; + } + if (this.theirInactivityTimer) { + clearTimeout(this.theirInactivityTimer); + this.theirInactivityTimer = null; + } + if (this.myCompletionTimer) { + clearTimeout(this.myCompletionTimer); + this.myCompletionTimer = null; + } + if (this.theirCompletionTimer) { + clearTimeout(this.theirCompletionTimer); + this.theirCompletionTimer = null; + } + + const closePromises = []; + if (this.mySttSession) { + closePromises.push(this.mySttSession.close()); + this.mySttSession = null; + } + if (this.theirSttSession) { + closePromises.push(this.theirSttSession.close()); + this.theirSttSession = null; + } + + await Promise.all(closePromises); + console.log('All STT sessions closed.'); + + // Reset state + this.myCurrentUtterance = ''; + this.theirCurrentUtterance = ''; + this.myLastPartialText = ''; + this.theirLastPartialText = ''; + this.myCompletionBuffer = ''; + this.theirCompletionBuffer = ''; + } +} + +module.exports = SttService; \ No newline at end of file diff --git a/src/features/listen/summary/repositories/index.js b/src/features/listen/summary/repositories/index.js new file mode 100644 index 0000000..d5bd3b3 --- /dev/null +++ b/src/features/listen/summary/repositories/index.js @@ -0,0 +1,5 @@ +const summaryRepository = require('./sqlite.repository'); + +module.exports = { + ...summaryRepository, +}; \ No newline at end of file diff --git a/src/features/listen/summary/repositories/sqlite.repository.js b/src/features/listen/summary/repositories/sqlite.repository.js new file mode 100644 index 0000000..d7a2266 --- /dev/null +++ b/src/features/listen/summary/repositories/sqlite.repository.js @@ -0,0 +1,47 @@ +const sqliteClient = require('../../../../common/services/sqliteClient'); + +function saveSummary({ sessionId, tldr, text, bullet_json, action_json, model = 'gpt-4.1' }) { + const db = sqliteClient.getDb(); + return new Promise((resolve, reject) => { + const now = Math.floor(Date.now() / 1000); + const query = ` + INSERT INTO summaries (session_id, generated_at, model, text, tldr, bullet_json, action_json, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + generated_at=excluded.generated_at, + model=excluded.model, + text=excluded.text, + tldr=excluded.tldr, + bullet_json=excluded.bullet_json, + action_json=excluded.action_json, + updated_at=excluded.updated_at + `; + db.run(query, [sessionId, now, model, text, tldr, bullet_json, action_json, now], function(err) { + if (err) { + console.error('Error saving summary:', err); + reject(err); + } else { + resolve({ changes: this.changes }); + } + }); + }); +} + +function getSummaryBySessionId(sessionId) { + const db = sqliteClient.getDb(); + return new Promise((resolve, reject) => { + const query = "SELECT * FROM summaries WHERE session_id = ?"; + db.get(query, [sessionId], (err, row) => { + if (err) { + reject(err); + } else { + resolve(row || null); + } + }); + }); +} + +module.exports = { + saveSummary, + getSummaryBySessionId, +}; \ No newline at end of file diff --git a/src/features/listen/summary/summaryService.js b/src/features/listen/summary/summaryService.js new file mode 100644 index 0000000..7cffa99 --- /dev/null +++ b/src/features/listen/summary/summaryService.js @@ -0,0 +1,357 @@ +const { BrowserWindow } = require('electron'); +const { getSystemPrompt } = require('../../../common/prompts/promptBuilder.js'); +const { makeChatCompletionWithPortkey } = require('../../../common/services/aiProviderService.js'); +const authService = require('../../../common/services/authService'); +const sessionRepository = require('../../../common/repositories/session'); +const summaryRepository = require('./repositories'); +const { getStoredApiKey, getStoredProvider } = require('../../../electron/windowManager'); + +class SummaryService { + constructor() { + this.previousAnalysisResult = null; + this.analysisHistory = []; + this.conversationHistory = []; + this.currentSessionId = null; + + // Callbacks + this.onAnalysisComplete = null; + this.onStatusUpdate = null; + } + + setCallbacks({ onAnalysisComplete, onStatusUpdate }) { + this.onAnalysisComplete = onAnalysisComplete; + this.onStatusUpdate = onStatusUpdate; + } + + setSessionId(sessionId) { + this.currentSessionId = sessionId; + } + + async getApiKey() { + const storedKey = await getStoredApiKey(); + if (storedKey) { + console.log('[SummaryService] Using stored API key'); + return storedKey; + } + + const envKey = process.env.OPENAI_API_KEY; + if (envKey) { + console.log('[SummaryService] Using environment API key'); + return envKey; + } + + console.error('[SummaryService] No API key found in storage or environment'); + return null; + } + + sendToRenderer(channel, data) { + BrowserWindow.getAllWindows().forEach(win => { + if (!win.isDestroyed()) { + win.webContents.send(channel, data); + } + }); + } + + addConversationTurn(speaker, text) { + const conversationText = `${speaker.toLowerCase()}: ${text.trim()}`; + this.conversationHistory.push(conversationText); + console.log(`πŸ’¬ Added conversation text: ${conversationText}`); + console.log(`πŸ“ˆ Total conversation history: ${this.conversationHistory.length} texts`); + + // Trigger analysis if needed + this.triggerAnalysisIfNeeded(); + } + + getConversationHistory() { + return this.conversationHistory; + } + + resetConversationHistory() { + this.conversationHistory = []; + this.previousAnalysisResult = null; + this.analysisHistory = []; + console.log('πŸ”„ Conversation history and analysis state reset'); + } + + /** + * Converts conversation history into text to include in the prompt. + * @param {Array} conversationTexts - Array of conversation texts ["me: ~~~", "them: ~~~", ...] + * @param {number} maxTurns - Maximum number of recent turns to include + * @returns {string} - Formatted conversation string for the prompt + */ + formatConversationForPrompt(conversationTexts, maxTurns = 30) { + if (conversationTexts.length === 0) return ''; + return conversationTexts.slice(-maxTurns).join('\n'); + } + + async makeOutlineAndRequests(conversationTexts, maxTurns = 30) { + console.log(`πŸ” makeOutlineAndRequests called - conversationTexts: ${conversationTexts.length}`); + + if (conversationTexts.length === 0) { + console.log('⚠️ No conversation texts available for analysis'); + return null; + } + + const recentConversation = this.formatConversationForPrompt(conversationTexts, maxTurns); + + // 이전 뢄석 κ²°κ³Όλ₯Ό ν”„λ‘¬ν”„νŠΈμ— 포함 + let contextualPrompt = ''; + if (this.previousAnalysisResult) { + contextualPrompt = ` +Previous Analysis Context: +- Main Topic: ${this.previousAnalysisResult.topic.header} +- Key Points: ${this.previousAnalysisResult.summary.slice(0, 3).join(', ')} +- Last Actions: ${this.previousAnalysisResult.actions.slice(0, 2).join(', ')} + +Please build upon this context while analyzing the new conversation segments. +`; + } + + const basePrompt = getSystemPrompt('pickle_glass_analysis', '', false); + const systemPrompt = basePrompt.replace('{{CONVERSATION_HISTORY}}', recentConversation); + + try { + if (this.currentSessionId) { + await sessionRepository.touch(this.currentSessionId); + } + + const messages = [ + { + role: 'system', + content: systemPrompt, + }, + { + role: 'user', + content: `${contextualPrompt} + +Analyze the conversation and provide a structured summary. Format your response as follows: + +**Summary Overview** +- Main discussion point with context + +**Key Topic: [Topic Name]** +- First key insight +- Second key insight +- Third key insight + +**Extended Explanation** +Provide 2-3 sentences explaining the context and implications. + +**Suggested Questions** +1. First follow-up question? +2. Second follow-up question? +3. Third follow-up question? + +Keep all points concise and build upon previous analysis if provided.`, + }, + ]; + + console.log('πŸ€– Sending analysis request to AI...'); + + const API_KEY = await this.getApiKey(); + if (!API_KEY) { + throw new Error('No API key available'); + } + + const provider = getStoredProvider ? await getStoredProvider() : 'openai'; + const loggedIn = authService.getCurrentUser().isLoggedIn; + const usePortkey = loggedIn && provider === 'openai'; + + console.log(`[SummaryService] provider: ${provider}, usePortkey: ${usePortkey}`); + + const completion = await makeChatCompletionWithPortkey({ + apiKey: API_KEY, + provider: provider, + messages: messages, + temperature: 0.7, + maxTokens: 1024, + model: provider === 'openai' ? 'gpt-4.1' : 'gemini-2.5-flash', + usePortkey: usePortkey, + portkeyVirtualKey: usePortkey ? API_KEY : null + }); + + const responseText = completion.content; + console.log(`βœ… Analysis response received: ${responseText}`); + const structuredData = this.parseResponseText(responseText, this.previousAnalysisResult); + + if (this.currentSessionId) { + summaryRepository.saveSummary({ + sessionId: this.currentSessionId, + text: responseText, + tldr: structuredData.summary.join('\n'), + bullet_json: JSON.stringify(structuredData.topic.bullets), + action_json: JSON.stringify(structuredData.actions), + model: 'gpt-4.1' + }).catch(err => console.error('[DB] Failed to save summary:', err)); + } + + // 뢄석 κ²°κ³Ό μ €μž₯ + this.previousAnalysisResult = structuredData; + this.analysisHistory.push({ + timestamp: Date.now(), + data: structuredData, + conversationLength: conversationTexts.length, + }); + + // νžˆμŠ€ν† λ¦¬ 크기 μ œν•œ (졜근 10개만 μœ μ§€) + if (this.analysisHistory.length > 10) { + this.analysisHistory.shift(); + } + + return structuredData; + } catch (error) { + console.error('❌ Error during analysis generation:', error.message); + return this.previousAnalysisResult; // μ—λŸ¬ μ‹œ 이전 κ²°κ³Ό λ°˜ν™˜ + } + } + + parseResponseText(responseText, previousResult) { + const structuredData = { + summary: [], + topic: { header: '', bullets: [] }, + actions: [], + followUps: ['βœ‰οΈ Draft a follow-up email', 'βœ… Generate action items', 'πŸ“ Show summary'], + }; + + // 이전 κ²°κ³Όκ°€ 있으면 κΈ°λ³Έκ°’μœΌλ‘œ μ‚¬μš© + if (previousResult) { + structuredData.topic.header = previousResult.topic.header; + structuredData.summary = [...previousResult.summary]; + } + + try { + const lines = responseText.split('\n'); + let currentSection = ''; + let isCapturingTopic = false; + let topicName = ''; + + for (const line of lines) { + const trimmedLine = line.trim(); + + // μ„Ήμ…˜ 헀더 감지 + if (trimmedLine.startsWith('**Summary Overview**')) { + currentSection = 'summary-overview'; + continue; + } else if (trimmedLine.startsWith('**Key Topic:')) { + currentSection = 'topic'; + isCapturingTopic = true; + topicName = trimmedLine.match(/\*\*Key Topic: (.+?)\*\*/)?.[1] || ''; + if (topicName) { + structuredData.topic.header = topicName + ':'; + } + continue; + } else if (trimmedLine.startsWith('**Extended Explanation**')) { + currentSection = 'explanation'; + continue; + } else if (trimmedLine.startsWith('**Suggested Questions**')) { + currentSection = 'questions'; + continue; + } + + // 컨텐츠 νŒŒμ‹± + if (trimmedLine.startsWith('-') && currentSection === 'summary-overview') { + const summaryPoint = trimmedLine.substring(1).trim(); + if (summaryPoint && !structuredData.summary.includes(summaryPoint)) { + // κΈ°μ‘΄ summary μ—…λ°μ΄νŠΈ (μ΅œλŒ€ 5개 μœ μ§€) + structuredData.summary.unshift(summaryPoint); + if (structuredData.summary.length > 5) { + structuredData.summary.pop(); + } + } + } else if (trimmedLine.startsWith('-') && currentSection === 'topic') { + const bullet = trimmedLine.substring(1).trim(); + if (bullet && structuredData.topic.bullets.length < 3) { + structuredData.topic.bullets.push(bullet); + } + } else if (currentSection === 'explanation' && trimmedLine) { + // explanation을 topic bullets에 μΆ”κ°€ (λ¬Έμž₯ λ‹¨μœ„λ‘œ) + const sentences = trimmedLine + .split(/\.\s+/) + .filter(s => s.trim().length > 0) + .map(s => s.trim() + (s.endsWith('.') ? '' : '.')); + + sentences.forEach(sentence => { + if (structuredData.topic.bullets.length < 3 && !structuredData.topic.bullets.includes(sentence)) { + structuredData.topic.bullets.push(sentence); + } + }); + } else if (trimmedLine.match(/^\d+\./) && currentSection === 'questions') { + const question = trimmedLine.replace(/^\d+\.\s*/, '').trim(); + if (question && question.includes('?')) { + structuredData.actions.push(`❓ ${question}`); + } + } + } + + // κΈ°λ³Έ μ•‘μ…˜ μΆ”κ°€ + const defaultActions = ['✨ What should I say next?', 'πŸ’¬ Suggest follow-up questions']; + defaultActions.forEach(action => { + if (!structuredData.actions.includes(action)) { + structuredData.actions.push(action); + } + }); + + // μ•‘μ…˜ 개수 μ œν•œ + structuredData.actions = structuredData.actions.slice(0, 5); + + // μœ νš¨μ„± 검증 및 이전 데이터 병합 + if (structuredData.summary.length === 0 && previousResult) { + structuredData.summary = previousResult.summary; + } + if (structuredData.topic.bullets.length === 0 && previousResult) { + structuredData.topic.bullets = previousResult.topic.bullets; + } + } catch (error) { + console.error('❌ Error parsing response text:', error); + // μ—λŸ¬ μ‹œ 이전 κ²°κ³Ό λ°˜ν™˜ + return ( + previousResult || { + summary: [], + topic: { header: 'Analysis in progress', bullets: [] }, + actions: ['✨ What should I say next?', 'πŸ’¬ Suggest follow-up questions'], + followUps: ['βœ‰οΈ Draft a follow-up email', 'βœ… Generate action items', 'πŸ“ Show summary'], + } + ); + } + + console.log('πŸ“Š Final structured data:', JSON.stringify(structuredData, null, 2)); + return structuredData; + } + + /** + * Triggers analysis when conversation history reaches 5 texts. + */ + async triggerAnalysisIfNeeded() { + if (this.conversationHistory.length >= 5 && this.conversationHistory.length % 5 === 0) { + console.log(`πŸš€ Triggering analysis (non-blocking) - ${this.conversationHistory.length} conversation texts accumulated`); + + this.makeOutlineAndRequests(this.conversationHistory) + .then(data => { + if (data) { + console.log('πŸ“€ Sending structured data to renderer'); + this.sendToRenderer('update-structured-data', data); + + // Notify callback + if (this.onAnalysisComplete) { + this.onAnalysisComplete(data); + } + } else { + console.log('❌ No analysis data returned from non-blocking call'); + } + }) + .catch(error => { + console.error('❌ Error in non-blocking analysis:', error); + }); + } + } + + getCurrentAnalysisData() { + return { + previousResult: this.previousAnalysisResult, + history: this.analysisHistory, + conversationLength: this.conversationHistory.length, + }; + } +} + +module.exports = SummaryService; \ No newline at end of file diff --git a/src/index.js b/src/index.js index ea8224b..1cca48f 100644 --- a/src/index.js +++ b/src/index.js @@ -13,7 +13,7 @@ if (require('electron-squirrel-startup')) { const { app, BrowserWindow, shell, ipcMain, dialog } = require('electron'); const { createWindows } = require('./electron/windowManager.js'); -const { setupLiveSummaryIpcHandlers, stopMacOSAudioCapture } = require('./features/listen/liveSummaryService.js'); +const ListenService = require('./features/listen/listenService'); const { initializeFirebase } = require('./common/services/firebaseClient'); const databaseInitializer = require('./common/services/databaseInitializer'); const authService = require('./common/services/authService'); @@ -29,7 +29,9 @@ const sessionRepository = require('./common/repositories/session'); const eventBridge = new EventEmitter(); let WEB_PORT = 3000; -const openaiSessionRef = { current: null }; +const listenService = new ListenService(); +// Make listenService globally accessible so other modules (e.g., windowManager, askService) can reuse the same instance +global.listenService = listenService; let deeplink = null; // Initialize as null let pendingDeepLinkUrl = null; // Store any deep link that arrives before initialization @@ -106,7 +108,7 @@ app.whenReady().then(async () => { sessionRepository.endAllActiveSessions(); authService.initialize(); - setupLiveSummaryIpcHandlers(openaiSessionRef); + listenService.setupIpcHandlers(); askService.initialize(); setupGeneralIpcHandlers(); }) @@ -123,7 +125,7 @@ app.whenReady().then(async () => { }); app.on('window-all-closed', () => { - stopMacOSAudioCapture(); + listenService.stopMacOSAudioCapture(); if (process.platform !== 'darwin') { app.quit(); } @@ -131,7 +133,7 @@ app.on('window-all-closed', () => { app.on('before-quit', async () => { console.log('[Shutdown] App is about to quit.'); - stopMacOSAudioCapture(); + listenService.stopMacOSAudioCapture(); await sessionRepository.endAllActiveSessions(); databaseInitializer.close(); }); @@ -210,7 +212,8 @@ function setupGeneralIpcHandlers() { function setupWebDataHandlers() { const sessionRepository = require('./common/repositories/session'); - const listenRepository = require('./features/listen/repositories'); + const sttRepository = require('./features/listen/stt/repositories'); + const summaryRepository = require('./features/listen/summary/repositories'); const askRepository = require('./features/ask/repositories'); const userRepository = require('./common/repositories/user'); const presetRepository = require('./common/repositories/preset'); @@ -230,9 +233,9 @@ function setupWebDataHandlers() { result = null; break; } - const transcripts = await listenRepository.getAllTranscriptsBySessionId(payload); + const transcripts = await sttRepository.getAllTranscriptsBySessionId(payload); const ai_messages = await askRepository.getAllAiMessagesBySessionId(payload); - const summary = await listenRepository.getSummaryBySessionId(payload); + const summary = await summaryRepository.getSummaryBySessionId(payload); result = { session, transcripts, ai_messages, summary }; break; case 'delete-session':