WIP: refactoring listen
This commit is contained in:
samtiz
parent
a18e93583f
commit
80a3c01656
@ -1,10 +1,9 @@
|
||||
import { html, css, LitElement } from '../assets/lit-core-2.7.4.min.js';
|
||||
import { CustomizeView } from '../features/customize/CustomizeView.js';
|
||||
import { AssistantView } from '../features/listen/AssistantView.js';
|
||||
import { OnboardingView } from '../features/onboarding/OnboardingView.js';
|
||||
import { AskView } from '../features/ask/AskView.js';
|
||||
|
||||
import '../features/listen/renderer.js';
|
||||
import '../features/listen/renderer/renderer.js';
|
||||
|
||||
export class PickleGlassApp extends LitElement {
|
||||
static styles = css`
|
||||
|
||||
@ -1024,10 +1024,10 @@ function createWindows() {
|
||||
|
||||
if (windowToToggle) {
|
||||
if (featureName === 'listen') {
|
||||
const liveSummaryService = require('../features/listen/liveSummaryService');
|
||||
if (liveSummaryService.isSessionActive()) {
|
||||
const listenService = global.listenService;
|
||||
if (listenService && listenService.isSessionActive()) {
|
||||
console.log('[WindowManager] Listen session is active, closing it via toggle.');
|
||||
await liveSummaryService.closeSession();
|
||||
await listenService.closeSession();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
const { ipcMain, BrowserWindow } = require('electron');
|
||||
const { makeStreamingChatCompletionWithPortkey } = require('../../common/services/aiProviderService');
|
||||
const { getConversationHistory } = require('../listen/liveSummaryService');
|
||||
const { getStoredApiKey, getStoredProvider, windowPool, captureScreenshot } = require('../../electron/windowManager');
|
||||
const authService = require('../../common/services/authService');
|
||||
const sessionRepository = require('../../common/repositories/session');
|
||||
@ -174,6 +173,12 @@ function formatConversationForPrompt(conversationTexts) {
|
||||
return conversationTexts.slice(-30).join('\n');
|
||||
}
|
||||
|
||||
// Access conversation history via the global listenService instance created in index.js
|
||||
function getConversationHistory() {
|
||||
const listenService = global.listenService;
|
||||
return listenService ? listenService.getConversationHistory() : [];
|
||||
}
|
||||
|
||||
async function sendMessage(userPrompt) {
|
||||
if (!userPrompt || userPrompt.trim().length === 0) {
|
||||
console.warn('[AskService] Cannot process empty message');
|
||||
|
||||
@ -1,123 +0,0 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
function pcmToWav(pcmBuffer, outputPath, sampleRate = 24000, channels = 1, bitDepth = 16) {
|
||||
const byteRate = sampleRate * channels * (bitDepth / 8);
|
||||
const blockAlign = channels * (bitDepth / 8);
|
||||
const dataSize = pcmBuffer.length;
|
||||
|
||||
const header = Buffer.alloc(44);
|
||||
|
||||
header.write('RIFF', 0);
|
||||
header.writeUInt32LE(dataSize + 36, 4);
|
||||
header.write('WAVE', 8);
|
||||
|
||||
header.write('fmt ', 12);
|
||||
header.writeUInt32LE(16, 16);
|
||||
header.writeUInt16LE(1, 20);
|
||||
header.writeUInt16LE(channels, 22);
|
||||
header.writeUInt32LE(sampleRate, 24);
|
||||
header.writeUInt32LE(byteRate, 28);
|
||||
header.writeUInt16LE(blockAlign, 32);
|
||||
header.writeUInt16LE(bitDepth, 34);
|
||||
|
||||
header.write('data', 36);
|
||||
header.writeUInt32LE(dataSize, 40);
|
||||
|
||||
const wavBuffer = Buffer.concat([header, pcmBuffer]);
|
||||
|
||||
fs.writeFileSync(outputPath, wavBuffer);
|
||||
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
function analyzeAudioBuffer(buffer, label = 'Audio') {
|
||||
const int16Array = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.length / 2);
|
||||
|
||||
let minValue = 32767;
|
||||
let maxValue = -32768;
|
||||
let avgValue = 0;
|
||||
let rmsValue = 0;
|
||||
let silentSamples = 0;
|
||||
|
||||
for (let i = 0; i < int16Array.length; i++) {
|
||||
const sample = int16Array[i];
|
||||
minValue = Math.min(minValue, sample);
|
||||
maxValue = Math.max(maxValue, sample);
|
||||
avgValue += sample;
|
||||
rmsValue += sample * sample;
|
||||
|
||||
if (Math.abs(sample) < 100) {
|
||||
silentSamples++;
|
||||
}
|
||||
}
|
||||
|
||||
avgValue /= int16Array.length;
|
||||
rmsValue = Math.sqrt(rmsValue / int16Array.length);
|
||||
|
||||
const silencePercentage = (silentSamples / int16Array.length) * 100;
|
||||
|
||||
console.log(`${label} Analysis:`);
|
||||
console.log(` Samples: ${int16Array.length}`);
|
||||
console.log(` Min: ${minValue}, Max: ${maxValue}`);
|
||||
console.log(` Average: ${avgValue.toFixed(2)}`);
|
||||
console.log(` RMS: ${rmsValue.toFixed(2)}`);
|
||||
console.log(` Silence: ${silencePercentage.toFixed(1)}%`);
|
||||
console.log(` Dynamic Range: ${20 * Math.log10(maxValue / (rmsValue || 1))} dB`);
|
||||
|
||||
return {
|
||||
minValue,
|
||||
maxValue,
|
||||
avgValue,
|
||||
rmsValue,
|
||||
silencePercentage,
|
||||
sampleCount: int16Array.length,
|
||||
};
|
||||
}
|
||||
|
||||
function saveDebugAudio(buffer, type, timestamp = Date.now()) {
|
||||
const homeDir = require('os').homedir();
|
||||
const debugDir = path.join(homeDir, '.pickle-glass', 'debug');
|
||||
|
||||
if (!fs.existsSync(debugDir)) {
|
||||
fs.mkdirSync(debugDir, { recursive: true });
|
||||
}
|
||||
|
||||
const pcmPath = path.join(debugDir, `${type}_${timestamp}.pcm`);
|
||||
const wavPath = path.join(debugDir, `${type}_${timestamp}.wav`);
|
||||
const metaPath = path.join(debugDir, `${type}_${timestamp}.json`);
|
||||
|
||||
fs.writeFileSync(pcmPath, buffer);
|
||||
|
||||
pcmToWav(buffer, wavPath);
|
||||
|
||||
const analysis = analyzeAudioBuffer(buffer, type);
|
||||
fs.writeFileSync(
|
||||
metaPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
timestamp,
|
||||
type,
|
||||
bufferSize: buffer.length,
|
||||
analysis,
|
||||
format: {
|
||||
sampleRate: 24000,
|
||||
channels: 1,
|
||||
bitDepth: 16,
|
||||
},
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
|
||||
console.log(`Debug audio saved: ${wavPath}`);
|
||||
|
||||
return { pcmPath, wavPath, metaPath };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
pcmToWav,
|
||||
analyzeAudioBuffer,
|
||||
saveDebugAudio,
|
||||
};
|
||||
263
src/features/listen/listenService.js
Normal file
263
src/features/listen/listenService.js
Normal file
@ -0,0 +1,263 @@
|
||||
const { BrowserWindow } = require('electron');
|
||||
const SttService = require('./stt/sttService');
|
||||
const SummaryService = require('./summary/summaryService');
|
||||
const authService = require('../../common/services/authService');
|
||||
const sessionRepository = require('../../common/repositories/session');
|
||||
const sttRepository = require('./stt/repositories');
|
||||
|
||||
class ListenService {
|
||||
constructor() {
|
||||
this.sttService = new SttService();
|
||||
this.summaryService = new SummaryService();
|
||||
this.currentSessionId = null;
|
||||
this.isInitializingSession = false;
|
||||
|
||||
this.setupServiceCallbacks();
|
||||
}
|
||||
|
||||
setupServiceCallbacks() {
|
||||
// STT service callbacks
|
||||
this.sttService.setCallbacks({
|
||||
onTranscriptionComplete: (speaker, text) => {
|
||||
this.handleTranscriptionComplete(speaker, text);
|
||||
},
|
||||
onStatusUpdate: (status) => {
|
||||
this.sendToRenderer('update-status', status);
|
||||
}
|
||||
});
|
||||
|
||||
// Summary service callbacks
|
||||
this.summaryService.setCallbacks({
|
||||
onAnalysisComplete: (data) => {
|
||||
console.log('📊 Analysis completed:', data);
|
||||
},
|
||||
onStatusUpdate: (status) => {
|
||||
this.sendToRenderer('update-status', status);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
sendToRenderer(channel, data) {
|
||||
BrowserWindow.getAllWindows().forEach(win => {
|
||||
if (!win.isDestroyed()) {
|
||||
win.webContents.send(channel, data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async handleTranscriptionComplete(speaker, text) {
|
||||
console.log(`[ListenService] Transcription complete: ${speaker} - ${text}`);
|
||||
|
||||
// Save to database
|
||||
await this.saveConversationTurn(speaker, text);
|
||||
|
||||
// Add to summary service for analysis
|
||||
this.summaryService.addConversationTurn(speaker, text);
|
||||
}
|
||||
|
||||
async saveConversationTurn(speaker, transcription) {
|
||||
if (!this.currentSessionId) {
|
||||
console.error('[DB] Cannot save turn, no active session ID.');
|
||||
return;
|
||||
}
|
||||
if (transcription.trim() === '') return;
|
||||
|
||||
try {
|
||||
await sessionRepository.touch(this.currentSessionId);
|
||||
await sttRepository.addTranscript({
|
||||
sessionId: this.currentSessionId,
|
||||
speaker: speaker,
|
||||
text: transcription.trim(),
|
||||
});
|
||||
console.log(`[DB] Saved transcript for session ${this.currentSessionId}: (${speaker})`);
|
||||
} catch (error) {
|
||||
console.error('Failed to save transcript to DB:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async initializeNewSession() {
|
||||
try {
|
||||
const uid = authService.getCurrentUserId();
|
||||
if (!uid) {
|
||||
throw new Error("Cannot initialize session: user not logged in.");
|
||||
}
|
||||
|
||||
this.currentSessionId = await sessionRepository.getOrCreateActive(uid, 'listen');
|
||||
console.log(`[DB] New listen session ensured: ${this.currentSessionId}`);
|
||||
|
||||
// Set session ID for summary service
|
||||
this.summaryService.setSessionId(this.currentSessionId);
|
||||
|
||||
// Reset conversation history
|
||||
this.summaryService.resetConversationHistory();
|
||||
|
||||
console.log('New conversation session started:', this.currentSessionId);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize new session in DB:', error);
|
||||
this.currentSessionId = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async initializeSession(language = 'en') {
|
||||
if (this.isInitializingSession) {
|
||||
console.log('Session initialization already in progress.');
|
||||
return false;
|
||||
}
|
||||
|
||||
this.isInitializingSession = true;
|
||||
this.sendToRenderer('session-initializing', true);
|
||||
this.sendToRenderer('update-status', 'Initializing sessions...');
|
||||
|
||||
try {
|
||||
// Initialize database session
|
||||
const sessionInitialized = await this.initializeNewSession();
|
||||
if (!sessionInitialized) {
|
||||
throw new Error('Failed to initialize database session');
|
||||
}
|
||||
|
||||
// Initialize STT sessions
|
||||
await this.sttService.initializeSttSessions(language);
|
||||
|
||||
console.log('✅ Listen service initialized successfully.');
|
||||
|
||||
this.sendToRenderer('session-state-changed', { isActive: true });
|
||||
this.sendToRenderer('update-status', 'Connected. Ready to listen.');
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to initialize listen service:', error);
|
||||
this.sendToRenderer('update-status', 'Initialization failed.');
|
||||
return false;
|
||||
} finally {
|
||||
this.isInitializingSession = false;
|
||||
this.sendToRenderer('session-initializing', false);
|
||||
}
|
||||
}
|
||||
|
||||
async sendAudioContent(data, mimeType) {
|
||||
return await this.sttService.sendAudioContent(data, mimeType);
|
||||
}
|
||||
|
||||
async startMacOSAudioCapture() {
|
||||
if (process.platform !== 'darwin') {
|
||||
throw new Error('macOS audio capture only available on macOS');
|
||||
}
|
||||
return await this.sttService.startMacOSAudioCapture();
|
||||
}
|
||||
|
||||
async stopMacOSAudioCapture() {
|
||||
this.sttService.stopMacOSAudioCapture();
|
||||
}
|
||||
|
||||
isSessionActive() {
|
||||
return this.sttService.isSessionActive();
|
||||
}
|
||||
|
||||
async closeSession() {
|
||||
try {
|
||||
// Close STT sessions
|
||||
await this.sttService.closeSessions();
|
||||
|
||||
// End database session
|
||||
if (this.currentSessionId) {
|
||||
await sessionRepository.end(this.currentSessionId);
|
||||
console.log(`[DB] Session ${this.currentSessionId} ended.`);
|
||||
}
|
||||
|
||||
// Reset state
|
||||
this.currentSessionId = null;
|
||||
this.summaryService.resetConversationHistory();
|
||||
|
||||
this.sendToRenderer('session-state-changed', { isActive: false });
|
||||
this.sendToRenderer('session-did-close');
|
||||
|
||||
console.log('Listen service session closed.');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
console.error('Error closing listen service session:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
getCurrentSessionData() {
|
||||
return {
|
||||
sessionId: this.currentSessionId,
|
||||
conversationHistory: this.summaryService.getConversationHistory(),
|
||||
totalTexts: this.summaryService.getConversationHistory().length,
|
||||
analysisData: this.summaryService.getCurrentAnalysisData(),
|
||||
};
|
||||
}
|
||||
|
||||
getConversationHistory() {
|
||||
return this.summaryService.getConversationHistory();
|
||||
}
|
||||
|
||||
setupIpcHandlers() {
|
||||
const { ipcMain } = require('electron');
|
||||
|
||||
ipcMain.handle('is-session-active', async () => {
|
||||
const isActive = this.isSessionActive();
|
||||
console.log(`Checking session status. Active: ${isActive}`);
|
||||
return isActive;
|
||||
});
|
||||
|
||||
ipcMain.handle('initialize-openai', async (event, profile = 'interview', language = 'en') => {
|
||||
console.log(`Received initialize-openai request with profile: ${profile}, language: ${language}`);
|
||||
const success = await this.initializeSession(language);
|
||||
return success;
|
||||
});
|
||||
|
||||
ipcMain.handle('send-audio-content', async (event, { data, mimeType }) => {
|
||||
try {
|
||||
await this.sendAudioContent(data, mimeType);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
console.error('Error sending user audio:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle('start-macos-audio', async () => {
|
||||
if (process.platform !== 'darwin') {
|
||||
return { success: false, error: 'macOS audio capture only available on macOS' };
|
||||
}
|
||||
try {
|
||||
const success = await this.startMacOSAudioCapture();
|
||||
return { success };
|
||||
} catch (error) {
|
||||
console.error('Error starting macOS audio capture:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle('stop-macos-audio', async () => {
|
||||
try {
|
||||
this.stopMacOSAudioCapture();
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
console.error('Error stopping macOS audio capture:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle('close-session', async () => {
|
||||
return await this.closeSession();
|
||||
});
|
||||
|
||||
ipcMain.handle('update-google-search-setting', async (event, enabled) => {
|
||||
try {
|
||||
console.log('Google Search setting updated to:', enabled);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
console.error('Error updating Google Search setting:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
|
||||
console.log('✅ Listen service IPC handlers registered');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = ListenService;
|
||||
@ -1,973 +0,0 @@
|
||||
require('dotenv').config();
|
||||
const { BrowserWindow, ipcMain } = require('electron');
|
||||
const { spawn } = require('child_process');
|
||||
const { saveDebugAudio } = require('./audioUtils.js');
|
||||
const { getSystemPrompt } = require('../../common/prompts/promptBuilder.js');
|
||||
const { connectToGeminiSession } = require('../../common/services/googleGeminiClient.js');
|
||||
const { connectToOpenAiSession, createOpenAiGenerativeClient, getOpenAiGenerativeModel } = require('../../common/services/openAiClient.js');
|
||||
const { makeChatCompletionWithPortkey } = require('../../common/services/aiProviderService.js');
|
||||
const authService = require('../../common/services/authService');
|
||||
const sessionRepository = require('../../common/repositories/session');
|
||||
const listenRepository = require('./repositories');
|
||||
|
||||
const { getStoredApiKey, getStoredProvider } = require('../../electron/windowManager');
|
||||
|
||||
const MAX_BUFFER_LENGTH_CHARS = 2000;
|
||||
const COMPLETION_DEBOUNCE_MS = 2000;
|
||||
|
||||
async function getApiKey() {
|
||||
const storedKey = await getStoredApiKey();
|
||||
|
||||
if (storedKey) {
|
||||
console.log('[LiveSummaryService] Using stored API key');
|
||||
return storedKey;
|
||||
}
|
||||
|
||||
const envKey = process.env.OPENAI_API_KEY;
|
||||
if (envKey) {
|
||||
console.log('[LiveSummaryService] Using environment API key');
|
||||
return envKey;
|
||||
}
|
||||
|
||||
console.error('[LiveSummaryService] No API key found in storage or environment');
|
||||
return null;
|
||||
}
|
||||
|
||||
async function getAiProvider() {
|
||||
try {
|
||||
const { ipcRenderer } = require('electron');
|
||||
const provider = await ipcRenderer.invoke('get-ai-provider');
|
||||
return provider || 'openai';
|
||||
} catch (error) {
|
||||
// If we're in the main process, get it directly
|
||||
return getStoredProvider ? getStoredProvider() : 'openai';
|
||||
}
|
||||
}
|
||||
|
||||
let currentSessionId = null;
|
||||
let conversationHistory = [];
|
||||
let isInitializingSession = false;
|
||||
|
||||
let mySttSession = null;
|
||||
let theirSttSession = null;
|
||||
let myCurrentUtterance = '';
|
||||
let theirCurrentUtterance = '';
|
||||
|
||||
let myLastPartialText = '';
|
||||
let theirLastPartialText = '';
|
||||
let myInactivityTimer = null;
|
||||
let theirInactivityTimer = null;
|
||||
const INACTIVITY_TIMEOUT = 3000;
|
||||
|
||||
const SESSION_IDLE_TIMEOUT_SECONDS = 30 * 60; // 30 minutes
|
||||
|
||||
let previousAnalysisResult = null;
|
||||
let analysisHistory = [];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 🎛️ Turn-completion debouncing
|
||||
// ---------------------------------------------------------------------------
|
||||
// Very aggressive VAD (e.g. 50 ms) tends to split one spoken sentence into
|
||||
// many "completed" events. To avoid creating a separate chat bubble for each
|
||||
// of those micro-turns we debounce the *completed* events per speaker. Any
|
||||
// completions that arrive within this window are concatenated and flushed as
|
||||
// **one** final turn.
|
||||
|
||||
let myCompletionBuffer = '';
|
||||
let theirCompletionBuffer = '';
|
||||
let myCompletionTimer = null;
|
||||
let theirCompletionTimer = null;
|
||||
|
||||
function flushMyCompletion() {
|
||||
if (!myCompletionBuffer.trim()) return;
|
||||
|
||||
const finalText = myCompletionBuffer.trim();
|
||||
// Save to DB & send to renderer as final
|
||||
saveConversationTurn('Me', finalText);
|
||||
sendToRenderer('stt-update', {
|
||||
speaker: 'Me',
|
||||
text: finalText,
|
||||
isPartial: false,
|
||||
isFinal: true,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
myCompletionBuffer = '';
|
||||
myCompletionTimer = null;
|
||||
myCurrentUtterance = ''; // Reset utterance accumulator on flush
|
||||
sendToRenderer('update-status', 'Listening...');
|
||||
}
|
||||
|
||||
function flushTheirCompletion() {
|
||||
if (!theirCompletionBuffer.trim()) return;
|
||||
|
||||
const finalText = theirCompletionBuffer.trim();
|
||||
saveConversationTurn('Them', finalText);
|
||||
sendToRenderer('stt-update', {
|
||||
speaker: 'Them',
|
||||
text: finalText,
|
||||
isPartial: false,
|
||||
isFinal: true,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
theirCompletionBuffer = '';
|
||||
theirCompletionTimer = null;
|
||||
theirCurrentUtterance = ''; // Reset utterance accumulator on flush
|
||||
sendToRenderer('update-status', 'Listening...');
|
||||
}
|
||||
|
||||
function debounceMyCompletion(text) {
|
||||
// 상대방이 말하고 있던 경우, 화자가 변경되었으므로 즉시 상대방의 말풍선을 완성합니다.
|
||||
if (theirCompletionTimer) {
|
||||
clearTimeout(theirCompletionTimer);
|
||||
flushTheirCompletion();
|
||||
}
|
||||
|
||||
myCompletionBuffer += (myCompletionBuffer ? ' ' : '') + text;
|
||||
|
||||
if (myCompletionTimer) clearTimeout(myCompletionTimer);
|
||||
myCompletionTimer = setTimeout(flushMyCompletion, COMPLETION_DEBOUNCE_MS);
|
||||
}
|
||||
|
||||
function debounceTheirCompletion(text) {
|
||||
// 내가 말하고 있던 경우, 화자가 변경되었으므로 즉시 내 말풍선을 완성합니다.
|
||||
if (myCompletionTimer) {
|
||||
clearTimeout(myCompletionTimer);
|
||||
flushMyCompletion();
|
||||
}
|
||||
|
||||
theirCompletionBuffer += (theirCompletionBuffer ? ' ' : '') + text;
|
||||
|
||||
if (theirCompletionTimer) clearTimeout(theirCompletionTimer);
|
||||
theirCompletionTimer = setTimeout(flushTheirCompletion, COMPLETION_DEBOUNCE_MS);
|
||||
}
|
||||
|
||||
let systemAudioProc = null;
|
||||
|
||||
let analysisIntervalId = null;
|
||||
|
||||
/**
|
||||
* Converts conversation history into text to include in the prompt.
|
||||
* @param {Array<string>} conversationTexts - Array of conversation texts ["me: ~~~", "them: ~~~", ...]
|
||||
* @param {number} maxTurns - Maximum number of recent turns to include
|
||||
* @returns {string} - Formatted conversation string for the prompt
|
||||
*/
|
||||
function formatConversationForPrompt(conversationTexts, maxTurns = 30) {
|
||||
if (conversationTexts.length === 0) return '';
|
||||
return conversationTexts.slice(-maxTurns).join('\n');
|
||||
}
|
||||
|
||||
async function makeOutlineAndRequests(conversationTexts, maxTurns = 30) {
|
||||
console.log(`🔍 makeOutlineAndRequests called - conversationTexts: ${conversationTexts.length}`);
|
||||
|
||||
if (conversationTexts.length === 0) {
|
||||
console.log('⚠️ No conversation texts available for analysis');
|
||||
return null;
|
||||
}
|
||||
|
||||
const recentConversation = formatConversationForPrompt(conversationTexts, maxTurns);
|
||||
|
||||
// 이전 분석 결과를 프롬프트에 포함
|
||||
let contextualPrompt = '';
|
||||
if (previousAnalysisResult) {
|
||||
contextualPrompt = `
|
||||
Previous Analysis Context:
|
||||
- Main Topic: ${previousAnalysisResult.topic.header}
|
||||
- Key Points: ${previousAnalysisResult.summary.slice(0, 3).join(', ')}
|
||||
- Last Actions: ${previousAnalysisResult.actions.slice(0, 2).join(', ')}
|
||||
|
||||
Please build upon this context while analyzing the new conversation segments.
|
||||
`;
|
||||
}
|
||||
|
||||
const basePrompt = getSystemPrompt('pickle_glass_analysis', '', false);
|
||||
const systemPrompt = basePrompt.replace('{{CONVERSATION_HISTORY}}', recentConversation);
|
||||
|
||||
try {
|
||||
if (currentSessionId) {
|
||||
await sessionRepository.touch(currentSessionId);
|
||||
}
|
||||
const messages = [
|
||||
{
|
||||
role: 'system',
|
||||
content: systemPrompt,
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `${contextualPrompt}
|
||||
|
||||
Analyze the conversation and provide a structured summary. Format your response as follows:
|
||||
|
||||
**Summary Overview**
|
||||
- Main discussion point with context
|
||||
|
||||
**Key Topic: [Topic Name]**
|
||||
- First key insight
|
||||
- Second key insight
|
||||
- Third key insight
|
||||
|
||||
**Extended Explanation**
|
||||
Provide 2-3 sentences explaining the context and implications.
|
||||
|
||||
**Suggested Questions**
|
||||
1. First follow-up question?
|
||||
2. Second follow-up question?
|
||||
3. Third follow-up question?
|
||||
|
||||
Keep all points concise and build upon previous analysis if provided.`,
|
||||
},
|
||||
];
|
||||
|
||||
console.log('🤖 Sending analysis request to OpenAI...');
|
||||
|
||||
const API_KEY = await getApiKey();
|
||||
if (!API_KEY) {
|
||||
throw new Error('No API key available');
|
||||
}
|
||||
|
||||
const provider = getStoredProvider ? getStoredProvider() : 'openai';
|
||||
const loggedIn = authService.getCurrentUser().isLoggedIn; // true ➜ vKey, false ➜ apiKey
|
||||
const usePortkey = loggedIn && provider === 'openai'; // Only use Portkey for OpenAI with Firebase
|
||||
|
||||
console.log(`[LiveSummary] provider: ${provider}, usePortkey: ${usePortkey}`);
|
||||
|
||||
const completion = await makeChatCompletionWithPortkey({
|
||||
apiKey: API_KEY,
|
||||
provider: provider,
|
||||
messages: messages,
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
model: provider === 'openai' ? 'gpt-4.1' : 'gemini-2.5-flash',
|
||||
usePortkey: usePortkey,
|
||||
portkeyVirtualKey: usePortkey ? API_KEY : null
|
||||
});
|
||||
|
||||
const responseText = completion.content;
|
||||
console.log(`✅ Analysis response received: ${responseText}`);
|
||||
const structuredData = parseResponseText(responseText, previousAnalysisResult);
|
||||
|
||||
if (currentSessionId) {
|
||||
listenRepository.saveSummary({
|
||||
sessionId: currentSessionId,
|
||||
tldr: structuredData.summary.join('\n'),
|
||||
bullet_json: JSON.stringify(structuredData.topic.bullets),
|
||||
action_json: JSON.stringify(structuredData.actions),
|
||||
model: 'gpt-4.1'
|
||||
}).catch(err => console.error('[DB] Failed to save summary:', err));
|
||||
}
|
||||
|
||||
// 분석 결과 저장
|
||||
previousAnalysisResult = structuredData;
|
||||
analysisHistory.push({
|
||||
timestamp: Date.now(),
|
||||
data: structuredData,
|
||||
conversationLength: conversationTexts.length,
|
||||
});
|
||||
|
||||
// 히스토리 크기 제한 (최근 10개만 유지)
|
||||
if (analysisHistory.length > 10) {
|
||||
analysisHistory.shift();
|
||||
}
|
||||
|
||||
return structuredData;
|
||||
} catch (error) {
|
||||
console.error('❌ Error during analysis generation:', error.message);
|
||||
return previousAnalysisResult; // 에러 시 이전 결과 반환
|
||||
}
|
||||
}
|
||||
|
||||
function parseResponseText(responseText, previousResult) {
|
||||
const structuredData = {
|
||||
summary: [],
|
||||
topic: { header: '', bullets: [] },
|
||||
actions: [],
|
||||
followUps: ['✉️ Draft a follow-up email', '✅ Generate action items', '📝 Show summary'],
|
||||
};
|
||||
|
||||
// 이전 결과가 있으면 기본값으로 사용
|
||||
if (previousResult) {
|
||||
structuredData.topic.header = previousResult.topic.header;
|
||||
structuredData.summary = [...previousResult.summary];
|
||||
}
|
||||
|
||||
try {
|
||||
const lines = responseText.split('\n');
|
||||
let currentSection = '';
|
||||
let isCapturingTopic = false;
|
||||
let topicName = '';
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmedLine = line.trim();
|
||||
|
||||
// 섹션 헤더 감지
|
||||
if (trimmedLine.startsWith('**Summary Overview**')) {
|
||||
currentSection = 'summary-overview';
|
||||
continue;
|
||||
} else if (trimmedLine.startsWith('**Key Topic:')) {
|
||||
currentSection = 'topic';
|
||||
isCapturingTopic = true;
|
||||
topicName = trimmedLine.match(/\*\*Key Topic: (.+?)\*\*/)?.[1] || '';
|
||||
if (topicName) {
|
||||
structuredData.topic.header = topicName + ':';
|
||||
}
|
||||
continue;
|
||||
} else if (trimmedLine.startsWith('**Extended Explanation**')) {
|
||||
currentSection = 'explanation';
|
||||
continue;
|
||||
} else if (trimmedLine.startsWith('**Suggested Questions**')) {
|
||||
currentSection = 'questions';
|
||||
continue;
|
||||
}
|
||||
|
||||
// 컨텐츠 파싱
|
||||
if (trimmedLine.startsWith('-') && currentSection === 'summary-overview') {
|
||||
const summaryPoint = trimmedLine.substring(1).trim();
|
||||
if (summaryPoint && !structuredData.summary.includes(summaryPoint)) {
|
||||
// 기존 summary 업데이트 (최대 5개 유지)
|
||||
structuredData.summary.unshift(summaryPoint);
|
||||
if (structuredData.summary.length > 5) {
|
||||
structuredData.summary.pop();
|
||||
}
|
||||
}
|
||||
} else if (trimmedLine.startsWith('-') && currentSection === 'topic') {
|
||||
const bullet = trimmedLine.substring(1).trim();
|
||||
if (bullet && structuredData.topic.bullets.length < 3) {
|
||||
structuredData.topic.bullets.push(bullet);
|
||||
}
|
||||
} else if (currentSection === 'explanation' && trimmedLine) {
|
||||
// explanation을 topic bullets에 추가 (문장 단위로)
|
||||
const sentences = trimmedLine
|
||||
.split(/\.\s+/)
|
||||
.filter(s => s.trim().length > 0)
|
||||
.map(s => s.trim() + (s.endsWith('.') ? '' : '.'));
|
||||
|
||||
sentences.forEach(sentence => {
|
||||
if (structuredData.topic.bullets.length < 3 && !structuredData.topic.bullets.includes(sentence)) {
|
||||
structuredData.topic.bullets.push(sentence);
|
||||
}
|
||||
});
|
||||
} else if (trimmedLine.match(/^\d+\./) && currentSection === 'questions') {
|
||||
const question = trimmedLine.replace(/^\d+\.\s*/, '').trim();
|
||||
if (question && question.includes('?')) {
|
||||
structuredData.actions.push(`❓ ${question}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 기본 액션 추가
|
||||
const defaultActions = ['✨ What should I say next?', '💬 Suggest follow-up questions'];
|
||||
defaultActions.forEach(action => {
|
||||
if (!structuredData.actions.includes(action)) {
|
||||
structuredData.actions.push(action);
|
||||
}
|
||||
});
|
||||
|
||||
// 액션 개수 제한
|
||||
structuredData.actions = structuredData.actions.slice(0, 5);
|
||||
|
||||
// 유효성 검증 및 이전 데이터 병합
|
||||
if (structuredData.summary.length === 0 && previousResult) {
|
||||
structuredData.summary = previousResult.summary;
|
||||
}
|
||||
if (structuredData.topic.bullets.length === 0 && previousResult) {
|
||||
structuredData.topic.bullets = previousResult.topic.bullets;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('❌ Error parsing response text:', error);
|
||||
// 에러 시 이전 결과 반환
|
||||
return (
|
||||
previousResult || {
|
||||
summary: [],
|
||||
topic: { header: 'Analysis in progress', bullets: [] },
|
||||
actions: ['✨ What should I say next?', '💬 Suggest follow-up questions'],
|
||||
followUps: ['✉️ Draft a follow-up email', '✅ Generate action items', '📝 Show summary'],
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
console.log('📊 Final structured data:', JSON.stringify(structuredData, null, 2));
|
||||
return structuredData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers analysis when conversation history reaches 5 texts.
|
||||
*/
|
||||
async function triggerAnalysisIfNeeded() {
|
||||
if (conversationHistory.length >= 5 && conversationHistory.length % 5 === 0) {
|
||||
console.log(`🚀 Triggering analysis (non-blocking) - ${conversationHistory.length} conversation texts accumulated`);
|
||||
|
||||
makeOutlineAndRequests(conversationHistory)
|
||||
.then(data => {
|
||||
if (data) {
|
||||
console.log('📤 Sending structured data to renderer');
|
||||
sendToRenderer('update-structured-data', data);
|
||||
} else {
|
||||
console.log('❌ No analysis data returned from non-blocking call');
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('❌ Error in non-blocking analysis:', error);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedules periodic updates of outline and analysis every 10 seconds. - DEPRECATED
|
||||
* Now analysis is triggered every 5 conversation texts.
|
||||
*/
|
||||
function startAnalysisInterval() {
|
||||
console.log('⏰ Analysis will be triggered every 5 conversation texts (not on timer)');
|
||||
|
||||
if (analysisIntervalId) {
|
||||
clearInterval(analysisIntervalId);
|
||||
analysisIntervalId = null;
|
||||
}
|
||||
}
|
||||
|
||||
function stopAnalysisInterval() {
|
||||
if (analysisIntervalId) {
|
||||
clearInterval(analysisIntervalId);
|
||||
analysisIntervalId = null;
|
||||
}
|
||||
|
||||
if (myInactivityTimer) {
|
||||
clearTimeout(myInactivityTimer);
|
||||
myInactivityTimer = null;
|
||||
}
|
||||
if (theirInactivityTimer) {
|
||||
clearTimeout(theirInactivityTimer);
|
||||
theirInactivityTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
function sendToRenderer(channel, data) {
|
||||
BrowserWindow.getAllWindows().forEach(win => {
|
||||
if (!win.isDestroyed()) {
|
||||
win.webContents.send(channel, data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getCurrentSessionData() {
|
||||
return {
|
||||
sessionId: currentSessionId,
|
||||
conversationHistory: conversationHistory,
|
||||
totalTexts: conversationHistory.length,
|
||||
};
|
||||
}
|
||||
|
||||
// Conversation management functions
|
||||
async function initializeNewSession() {
|
||||
try {
|
||||
const uid = authService.getCurrentUserId();
|
||||
if (!uid) {
|
||||
throw new Error("Cannot initialize session: user not logged in.");
|
||||
}
|
||||
currentSessionId = await sessionRepository.getOrCreateActive(uid, 'listen');
|
||||
console.log(`[DB] New listen session ensured: ${currentSessionId}`);
|
||||
|
||||
conversationHistory = [];
|
||||
myCurrentUtterance = '';
|
||||
theirCurrentUtterance = '';
|
||||
|
||||
// 🔄 Reset analysis state so the new session starts fresh
|
||||
previousAnalysisResult = null;
|
||||
analysisHistory = [];
|
||||
|
||||
// sendToRenderer('update-outline', []);
|
||||
// sendToRenderer('update-analysis-requests', []);
|
||||
|
||||
myLastPartialText = '';
|
||||
theirLastPartialText = '';
|
||||
if (myInactivityTimer) {
|
||||
clearTimeout(myInactivityTimer);
|
||||
myInactivityTimer = null;
|
||||
}
|
||||
if (theirInactivityTimer) {
|
||||
clearTimeout(theirInactivityTimer);
|
||||
theirInactivityTimer = null;
|
||||
}
|
||||
|
||||
console.log('New conversation session started:', currentSessionId);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize new session in DB:', error);
|
||||
currentSessionId = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function saveConversationTurn(speaker, transcription) {
|
||||
if (!currentSessionId) {
|
||||
console.error('[DB] Cannot save turn, no active session ID.');
|
||||
return;
|
||||
}
|
||||
if (transcription.trim() === '') return;
|
||||
|
||||
try {
|
||||
await sessionRepository.touch(currentSessionId);
|
||||
await listenRepository.addTranscript({
|
||||
sessionId: currentSessionId,
|
||||
speaker: speaker,
|
||||
text: transcription.trim(),
|
||||
});
|
||||
console.log(`[DB] Saved transcript for session ${currentSessionId}: (${speaker})`);
|
||||
|
||||
const conversationText = `${speaker.toLowerCase()}: ${transcription.trim()}`;
|
||||
conversationHistory.push(conversationText);
|
||||
console.log(`💬 Saved conversation text: ${conversationText}`);
|
||||
console.log(`📈 Total conversation history: ${conversationHistory.length} texts`);
|
||||
|
||||
triggerAnalysisIfNeeded();
|
||||
|
||||
const conversationTurn = {
|
||||
speaker: speaker,
|
||||
timestamp: Date.now(),
|
||||
transcription: transcription.trim(),
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Failed to save transcript to DB:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function initializeLiveSummarySession(language = 'en') {
|
||||
// Use system environment variable if set, otherwise use the provided language
|
||||
const effectiveLanguage = process.env.OPENAI_TRANSCRIBE_LANG || language || 'en';
|
||||
if (isInitializingSession) {
|
||||
console.log('Session initialization already in progress.');
|
||||
return false;
|
||||
}
|
||||
|
||||
const userState = authService.getCurrentUser();
|
||||
const loggedIn = userState.isLoggedIn;
|
||||
const keyType = loggedIn ? 'vKey' : 'apiKey';
|
||||
|
||||
isInitializingSession = true;
|
||||
sendToRenderer('session-initializing', true);
|
||||
sendToRenderer('update-status', 'Initializing sessions...');
|
||||
|
||||
const API_KEY = await getApiKey();
|
||||
if (!API_KEY) {
|
||||
console.error('FATAL ERROR: API Key is not defined.');
|
||||
sendToRenderer('update-status', 'API Key not configured.');
|
||||
isInitializingSession = false;
|
||||
sendToRenderer('session-initializing', false);
|
||||
return false;
|
||||
}
|
||||
|
||||
await initializeNewSession();
|
||||
|
||||
const provider = await getAiProvider();
|
||||
const isGemini = provider === 'gemini';
|
||||
console.log(`[LiveSummaryService] Initializing STT for provider: ${provider}`);
|
||||
|
||||
try {
|
||||
const handleMyMessage = message => {
|
||||
if (isGemini) {
|
||||
// console.log('[Gemini Raw Message - Me]:', JSON.stringify(message, null, 2));
|
||||
const text = message.serverContent?.inputTranscription?.text || '';
|
||||
if (text && text.trim()) {
|
||||
const finalUtteranceText = text.trim().replace(/<noise>/g, '').trim();
|
||||
if (finalUtteranceText && finalUtteranceText !== '.') {
|
||||
debounceMyCompletion(finalUtteranceText);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const type = message.type;
|
||||
const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || '';
|
||||
|
||||
if (type === 'conversation.item.input_audio_transcription.delta') {
|
||||
if (myCompletionTimer) clearTimeout(myCompletionTimer);
|
||||
myCompletionTimer = null;
|
||||
myCurrentUtterance += text;
|
||||
const continuousText = myCompletionBuffer + (myCompletionBuffer ? ' ' : '') + myCurrentUtterance;
|
||||
if (text && !text.includes('vq_lbr_audio_')) {
|
||||
sendToRenderer('stt-update', {
|
||||
speaker: 'Me',
|
||||
text: continuousText,
|
||||
isPartial: true,
|
||||
isFinal: false,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
}
|
||||
} else if (type === 'conversation.item.input_audio_transcription.completed') {
|
||||
if (text && text.trim()) {
|
||||
const finalUtteranceText = text.trim();
|
||||
myCurrentUtterance = '';
|
||||
debounceMyCompletion(finalUtteranceText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (message.error) {
|
||||
console.error('[Me] STT Session Error:', message.error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTheirMessage = message => {
|
||||
if (isGemini) {
|
||||
// console.log('[Gemini Raw Message - Them]:', JSON.stringify(message, null, 2));
|
||||
const text = message.serverContent?.inputTranscription?.text || '';
|
||||
if (text && text.trim()) {
|
||||
const finalUtteranceText = text.trim().replace(/<noise>/g, '').trim();
|
||||
if (finalUtteranceText && finalUtteranceText !== '.') {
|
||||
debounceTheirCompletion(finalUtteranceText);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const type = message.type;
|
||||
const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || '';
|
||||
if (type === 'conversation.item.input_audio_transcription.delta') {
|
||||
if (theirCompletionTimer) clearTimeout(theirCompletionTimer);
|
||||
theirCompletionTimer = null;
|
||||
theirCurrentUtterance += text;
|
||||
const continuousText = theirCompletionBuffer + (theirCompletionBuffer ? ' ' : '') + theirCurrentUtterance;
|
||||
if (text && !text.includes('vq_lbr_audio_')) {
|
||||
sendToRenderer('stt-update', {
|
||||
speaker: 'Them',
|
||||
text: continuousText,
|
||||
isPartial: true,
|
||||
isFinal: false,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
}
|
||||
} else if (type === 'conversation.item.input_audio_transcription.completed') {
|
||||
if (text && text.trim()) {
|
||||
const finalUtteranceText = text.trim();
|
||||
theirCurrentUtterance = '';
|
||||
debounceTheirCompletion(finalUtteranceText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (message.error) {
|
||||
console.error('[Them] STT Session Error:', message.error);
|
||||
}
|
||||
};
|
||||
|
||||
const mySttConfig = {
|
||||
language: effectiveLanguage,
|
||||
callbacks: {
|
||||
onmessage: handleMyMessage,
|
||||
onerror: error => console.error('My STT session error:', error.message),
|
||||
onclose: event => console.log('My STT session closed:', event.reason),
|
||||
},
|
||||
};
|
||||
const theirSttConfig = {
|
||||
language: effectiveLanguage,
|
||||
callbacks: {
|
||||
onmessage: handleTheirMessage,
|
||||
onerror: error => console.error('Their STT session error:', error.message),
|
||||
onclose: event => console.log('Their STT session closed:', event.reason),
|
||||
},
|
||||
};
|
||||
|
||||
if (isGemini) {
|
||||
[mySttSession, theirSttSession] = await Promise.all([
|
||||
connectToGeminiSession(API_KEY, mySttConfig),
|
||||
connectToGeminiSession(API_KEY, theirSttConfig),
|
||||
]);
|
||||
} else {
|
||||
[mySttSession, theirSttSession] = await Promise.all([
|
||||
connectToOpenAiSession(API_KEY, mySttConfig, keyType),
|
||||
connectToOpenAiSession(API_KEY, theirSttConfig, keyType),
|
||||
]);
|
||||
}
|
||||
|
||||
console.log('✅ Both STT sessions initialized successfully.');
|
||||
triggerAnalysisIfNeeded();
|
||||
|
||||
sendToRenderer('session-state-changed', { isActive: true });
|
||||
|
||||
isInitializingSession = false;
|
||||
sendToRenderer('session-initializing', false);
|
||||
sendToRenderer('update-status', 'Connected. Ready to listen.');
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to initialize STT sessions:', error);
|
||||
isInitializingSession = false;
|
||||
sendToRenderer('session-initializing', false);
|
||||
sendToRenderer('update-status', 'Initialization failed.');
|
||||
mySttSession = null;
|
||||
theirSttSession = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function killExistingSystemAudioDump() {
|
||||
return new Promise(resolve => {
|
||||
console.log('Checking for existing SystemAudioDump processes...');
|
||||
|
||||
const killProc = spawn('pkill', ['-f', 'SystemAudioDump'], {
|
||||
stdio: 'ignore',
|
||||
});
|
||||
|
||||
killProc.on('close', code => {
|
||||
if (code === 0) {
|
||||
console.log('Killed existing SystemAudioDump processes');
|
||||
} else {
|
||||
console.log('No existing SystemAudioDump processes found');
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
|
||||
killProc.on('error', err => {
|
||||
console.log('Error checking for existing processes (this is normal):', err.message);
|
||||
resolve();
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
killProc.kill();
|
||||
resolve();
|
||||
}, 2000);
|
||||
});
|
||||
}
|
||||
|
||||
async function startMacOSAudioCapture() {
|
||||
if (process.platform !== 'darwin' || !theirSttSession) return false;
|
||||
|
||||
await killExistingSystemAudioDump();
|
||||
console.log('Starting macOS audio capture for "Them"...');
|
||||
|
||||
const { app } = require('electron');
|
||||
const path = require('path');
|
||||
const systemAudioPath = app.isPackaged
|
||||
? path.join(process.resourcesPath, 'app.asar.unpacked', 'src', 'assets', 'SystemAudioDump')
|
||||
: path.join(app.getAppPath(), 'src', 'assets', 'SystemAudioDump');
|
||||
|
||||
console.log('SystemAudioDump path:', systemAudioPath);
|
||||
|
||||
systemAudioProc = spawn(systemAudioPath, [], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
if (!systemAudioProc.pid) {
|
||||
console.error('Failed to start SystemAudioDump');
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log('SystemAudioDump started with PID:', systemAudioProc.pid);
|
||||
|
||||
const CHUNK_DURATION = 0.1;
|
||||
const SAMPLE_RATE = 24000;
|
||||
const BYTES_PER_SAMPLE = 2;
|
||||
const CHANNELS = 2;
|
||||
const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * CHUNK_DURATION;
|
||||
|
||||
let audioBuffer = Buffer.alloc(0);
|
||||
|
||||
const provider = await getAiProvider();
|
||||
const isGemini = provider === 'gemini';
|
||||
|
||||
systemAudioProc.stdout.on('data', async data => {
|
||||
audioBuffer = Buffer.concat([audioBuffer, data]);
|
||||
|
||||
while (audioBuffer.length >= CHUNK_SIZE) {
|
||||
const chunk = audioBuffer.slice(0, CHUNK_SIZE);
|
||||
audioBuffer = audioBuffer.slice(CHUNK_SIZE);
|
||||
|
||||
const monoChunk = CHANNELS === 2 ? convertStereoToMono(chunk) : chunk;
|
||||
const base64Data = monoChunk.toString('base64');
|
||||
|
||||
sendToRenderer('system-audio-data', { data: base64Data });
|
||||
|
||||
if (theirSttSession) {
|
||||
try {
|
||||
// await theirSttSession.sendRealtimeInput(base64Data);
|
||||
const payload = isGemini
|
||||
? { audio: { data: base64Data, mimeType: 'audio/pcm;rate=24000' } }
|
||||
: base64Data;
|
||||
await theirSttSession.sendRealtimeInput(payload);
|
||||
} catch (err) {
|
||||
console.error('Error sending system audio:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env.DEBUG_AUDIO) {
|
||||
saveDebugAudio(monoChunk, 'system_audio');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
systemAudioProc.stderr.on('data', data => {
|
||||
console.error('SystemAudioDump stderr:', data.toString());
|
||||
});
|
||||
|
||||
systemAudioProc.on('close', code => {
|
||||
console.log('SystemAudioDump process closed with code:', code);
|
||||
systemAudioProc = null;
|
||||
});
|
||||
|
||||
systemAudioProc.on('error', err => {
|
||||
console.error('SystemAudioDump process error:', err);
|
||||
systemAudioProc = null;
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
function convertStereoToMono(stereoBuffer) {
|
||||
const samples = stereoBuffer.length / 4;
|
||||
const monoBuffer = Buffer.alloc(samples * 2);
|
||||
|
||||
for (let i = 0; i < samples; i++) {
|
||||
const leftSample = stereoBuffer.readInt16LE(i * 4);
|
||||
monoBuffer.writeInt16LE(leftSample, i * 2);
|
||||
}
|
||||
|
||||
return monoBuffer;
|
||||
}
|
||||
|
||||
function stopMacOSAudioCapture() {
|
||||
if (systemAudioProc) {
|
||||
console.log('Stopping SystemAudioDump...');
|
||||
systemAudioProc.kill('SIGTERM');
|
||||
systemAudioProc = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function sendAudioToOpenAI(base64Data, sttSessionRef) {
|
||||
if (!sttSessionRef.current) return;
|
||||
|
||||
try {
|
||||
process.stdout.write('.');
|
||||
await sttSessionRef.current.sendRealtimeInput({
|
||||
audio: {
|
||||
data: base64Data,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error sending audio to OpenAI:', error);
|
||||
}
|
||||
}
|
||||
|
||||
function isSessionActive() {
|
||||
return !!mySttSession && !!theirSttSession;
|
||||
}
|
||||
|
||||
async function closeSession() {
|
||||
try {
|
||||
stopMacOSAudioCapture();
|
||||
stopAnalysisInterval();
|
||||
|
||||
if (currentSessionId) {
|
||||
await sessionRepository.end(currentSessionId);
|
||||
console.log(`[DB] Session ${currentSessionId} ended.`);
|
||||
}
|
||||
|
||||
const closePromises = [];
|
||||
if (mySttSession) {
|
||||
closePromises.push(mySttSession.close());
|
||||
mySttSession = null;
|
||||
}
|
||||
if (theirSttSession) {
|
||||
closePromises.push(theirSttSession.close());
|
||||
theirSttSession = null;
|
||||
}
|
||||
|
||||
await Promise.all(closePromises);
|
||||
console.log('All sessions closed.');
|
||||
|
||||
currentSessionId = null;
|
||||
conversationHistory = [];
|
||||
|
||||
sendToRenderer('session-state-changed', { isActive: false });
|
||||
sendToRenderer('session-did-close');
|
||||
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
console.error('Error closing sessions:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
function setupLiveSummaryIpcHandlers() {
|
||||
ipcMain.handle('is-session-active', async () => {
|
||||
const isActive = isSessionActive();
|
||||
console.log(`Checking session status. Active: ${isActive}`);
|
||||
return isActive;
|
||||
});
|
||||
|
||||
ipcMain.handle('initialize-openai', async (event, profile = 'interview', language = 'en') => {
|
||||
console.log(`Received initialize-openai request with profile: ${profile}, language: ${language}`);
|
||||
const success = await initializeLiveSummarySession(language);
|
||||
return success;
|
||||
});
|
||||
|
||||
ipcMain.handle('send-audio-content', async (event, { data, mimeType }) => {
|
||||
const provider = await getAiProvider();
|
||||
const isGemini = provider === 'gemini';
|
||||
if (!mySttSession) return { success: false, error: 'User STT session not active' };
|
||||
try {
|
||||
// await mySttSession.sendRealtimeInput(data);
|
||||
// provider에 맞는 형식으로 래핑
|
||||
const payload = isGemini
|
||||
? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } }
|
||||
: data; // OpenAI는 base64 string 그대로
|
||||
|
||||
await mySttSession.sendRealtimeInput(payload);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
console.error('Error sending user audio:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle('start-macos-audio', async () => {
|
||||
if (process.platform !== 'darwin') {
|
||||
return { success: false, error: 'macOS audio capture only available on macOS' };
|
||||
}
|
||||
try {
|
||||
const success = await startMacOSAudioCapture();
|
||||
return { success };
|
||||
} catch (error) {
|
||||
console.error('Error starting macOS audio capture:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle('stop-macos-audio', async () => {
|
||||
try {
|
||||
stopMacOSAudioCapture();
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
console.error('Error stopping macOS audio capture:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle('close-session', async () => {
|
||||
return await closeSession();
|
||||
});
|
||||
|
||||
ipcMain.handle('update-google-search-setting', async (event, enabled) => {
|
||||
try {
|
||||
console.log('Google Search setting updated to:', enabled);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
console.error('Error updating Google Search setting:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getConversationHistory() {
|
||||
return conversationHistory;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
sendToRenderer,
|
||||
initializeNewSession,
|
||||
saveConversationTurn,
|
||||
killExistingSystemAudioDump,
|
||||
startMacOSAudioCapture,
|
||||
convertStereoToMono,
|
||||
stopMacOSAudioCapture,
|
||||
sendAudioToOpenAI,
|
||||
setupLiveSummaryIpcHandlers,
|
||||
isSessionActive,
|
||||
closeSession,
|
||||
getConversationHistory,
|
||||
};
|
||||
@ -1,20 +1,29 @@
|
||||
// renderer.js
|
||||
const { ipcRenderer } = require('electron');
|
||||
const { makeStreamingChatCompletionWithPortkey } = require('../../common/services/aiProviderService.js');
|
||||
|
||||
let mediaStream = null;
|
||||
let screenshotInterval = null;
|
||||
let audioContext = null;
|
||||
let audioProcessor = null;
|
||||
let micMediaStream = null;
|
||||
let audioBuffer = [];
|
||||
// ---------------------------
|
||||
// Constants & Globals
|
||||
// ---------------------------
|
||||
const SAMPLE_RATE = 24000;
|
||||
const AUDIO_CHUNK_DURATION = 0.1;
|
||||
const BUFFER_SIZE = 4096;
|
||||
|
||||
const isLinux = process.platform === 'linux';
|
||||
const isMacOS = process.platform === 'darwin';
|
||||
|
||||
let mediaStream = null;
|
||||
let micMediaStream = null;
|
||||
let screenshotInterval = null;
|
||||
let audioContext = null;
|
||||
let audioProcessor = null;
|
||||
let currentImageQuality = 'medium';
|
||||
let lastScreenshotBase64 = null;
|
||||
|
||||
let systemAudioBuffer = [];
|
||||
const MAX_SYSTEM_BUFFER_SIZE = 10;
|
||||
|
||||
// ---------------------------
|
||||
// Utility helpers (exact from renderer.js)
|
||||
// ---------------------------
|
||||
function isVoiceActive(audioFloat32Array, threshold = 0.005) {
|
||||
if (!audioFloat32Array || audioFloat32Array.length === 0) {
|
||||
return false;
|
||||
@ -31,11 +40,6 @@ function isVoiceActive(audioFloat32Array, threshold = 0.005) {
|
||||
return rms > threshold;
|
||||
}
|
||||
|
||||
let currentImageQuality = 'medium'; // Store current image quality for manual screenshots
|
||||
let lastScreenshotBase64 = null; // Store the latest screenshot
|
||||
|
||||
let realtimeConversationHistory = [];
|
||||
|
||||
function base64ToFloat32Array(base64) {
|
||||
const binaryString = atob(base64);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
@ -54,11 +58,29 @@ function base64ToFloat32Array(base64) {
|
||||
return float32Array;
|
||||
}
|
||||
|
||||
async function queryLoginState() {
|
||||
const userState = await ipcRenderer.invoke('get-current-user');
|
||||
return userState;
|
||||
function convertFloat32ToInt16(float32Array) {
|
||||
const int16Array = new Int16Array(float32Array.length);
|
||||
for (let i = 0; i < float32Array.length; i++) {
|
||||
// Improved scaling to prevent clipping
|
||||
const s = Math.max(-1, Math.min(1, float32Array[i]));
|
||||
int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
return int16Array;
|
||||
}
|
||||
|
||||
function arrayBufferToBase64(buffer) {
|
||||
let binary = '';
|
||||
const bytes = new Uint8Array(buffer);
|
||||
const len = bytes.byteLength;
|
||||
for (let i = 0; i < len; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
// ---------------------------
|
||||
// Complete SimpleAEC implementation (exact from renderer.js)
|
||||
// ---------------------------
|
||||
class SimpleAEC {
|
||||
constructor() {
|
||||
this.adaptiveFilter = new Float32Array(1024);
|
||||
@ -179,11 +201,24 @@ class SimpleAEC {
|
||||
|
||||
let aecProcessor = new SimpleAEC();
|
||||
|
||||
const isLinux = process.platform === 'linux';
|
||||
const isMacOS = process.platform === 'darwin';
|
||||
// System audio data handler
|
||||
ipcRenderer.on('system-audio-data', (event, { data }) => {
|
||||
systemAudioBuffer.push({
|
||||
data: data,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
window.pickleGlass = window.pickleGlass || {};
|
||||
// 오래된 데이터 제거
|
||||
if (systemAudioBuffer.length > MAX_SYSTEM_BUFFER_SIZE) {
|
||||
systemAudioBuffer = systemAudioBuffer.slice(-MAX_SYSTEM_BUFFER_SIZE);
|
||||
}
|
||||
|
||||
console.log('📥 Received system audio for AEC reference');
|
||||
});
|
||||
|
||||
// ---------------------------
|
||||
// Complete token tracker (exact from renderer.js)
|
||||
// ---------------------------
|
||||
let tokenTracker = {
|
||||
tokens: [],
|
||||
audioStartTime: null,
|
||||
@ -265,126 +300,201 @@ setInterval(() => {
|
||||
tokenTracker.trackAudioTokens();
|
||||
}, 2000);
|
||||
|
||||
function pickleGlassElement() {
|
||||
return document.getElementById('pickle-glass');
|
||||
}
|
||||
// ---------------------------
|
||||
// Audio processing functions (exact from renderer.js)
|
||||
// ---------------------------
|
||||
function setupMicProcessing(micStream) {
|
||||
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||
const micSource = micAudioContext.createMediaStreamSource(micStream);
|
||||
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
||||
|
||||
function convertFloat32ToInt16(float32Array) {
|
||||
const int16Array = new Int16Array(float32Array.length);
|
||||
for (let i = 0; i < float32Array.length; i++) {
|
||||
// Improved scaling to prevent clipping
|
||||
const s = Math.max(-1, Math.min(1, float32Array[i]));
|
||||
int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
return int16Array;
|
||||
}
|
||||
let audioBuffer = [];
|
||||
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
||||
|
||||
function arrayBufferToBase64(buffer) {
|
||||
let binary = '';
|
||||
const bytes = new Uint8Array(buffer);
|
||||
const len = bytes.byteLength;
|
||||
for (let i = 0; i < len; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
micProcessor.onaudioprocess = async e => {
|
||||
const inputData = e.inputBuffer.getChannelData(0);
|
||||
audioBuffer.push(...inputData);
|
||||
|
||||
async function initializeopenai(profile = 'interview', language = 'en') {
|
||||
// The API key is now handled in the main process from .env file.
|
||||
// We just need to trigger the initialization.
|
||||
try {
|
||||
console.log(`Requesting OpenAI initialization with profile: ${profile}, language: ${language}`);
|
||||
const success = await ipcRenderer.invoke('initialize-openai', profile, language);
|
||||
if (success) {
|
||||
// The status will be updated via 'update-status' event from the main process.
|
||||
console.log('OpenAI initialization successful.');
|
||||
} else {
|
||||
console.error('OpenAI initialization failed.');
|
||||
const appElement = pickleGlassElement();
|
||||
if (appElement && typeof appElement.setStatus === 'function') {
|
||||
appElement.setStatus('Initialization Failed');
|
||||
while (audioBuffer.length >= samplesPerChunk) {
|
||||
let chunk = audioBuffer.splice(0, samplesPerChunk);
|
||||
let processedChunk = new Float32Array(chunk);
|
||||
|
||||
// Check for system audio and apply AEC only if voice is active
|
||||
if (aecProcessor && systemAudioBuffer.length > 0) {
|
||||
const latestSystemAudio = systemAudioBuffer[systemAudioBuffer.length - 1];
|
||||
const systemFloat32 = base64ToFloat32Array(latestSystemAudio.data);
|
||||
|
||||
// Apply AEC only when system audio has active speech
|
||||
if (isVoiceActive(systemFloat32)) {
|
||||
processedChunk = aecProcessor.process(new Float32Array(chunk), systemFloat32);
|
||||
console.log('🔊 Applied AEC because system audio is active');
|
||||
}
|
||||
}
|
||||
|
||||
const pcmData16 = convertFloat32ToInt16(processedChunk);
|
||||
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
||||
|
||||
await ipcRenderer.invoke('send-audio-content', {
|
||||
data: base64Data,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
micSource.connect(micProcessor);
|
||||
micProcessor.connect(micAudioContext.destination);
|
||||
|
||||
audioProcessor = micProcessor;
|
||||
}
|
||||
|
||||
function setupLinuxMicProcessing(micStream) {
|
||||
// Setup microphone audio processing for Linux
|
||||
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||
const micSource = micAudioContext.createMediaStreamSource(micStream);
|
||||
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
||||
|
||||
let audioBuffer = [];
|
||||
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
||||
|
||||
micProcessor.onaudioprocess = async e => {
|
||||
const inputData = e.inputBuffer.getChannelData(0);
|
||||
audioBuffer.push(...inputData);
|
||||
|
||||
// Process audio in chunks
|
||||
while (audioBuffer.length >= samplesPerChunk) {
|
||||
const chunk = audioBuffer.splice(0, samplesPerChunk);
|
||||
const pcmData16 = convertFloat32ToInt16(chunk);
|
||||
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
||||
|
||||
await ipcRenderer.invoke('send-audio-content', {
|
||||
data: base64Data,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
micSource.connect(micProcessor);
|
||||
micProcessor.connect(micAudioContext.destination);
|
||||
|
||||
// Store processor reference for cleanup
|
||||
audioProcessor = micProcessor;
|
||||
}
|
||||
|
||||
function setupWindowsLoopbackProcessing() {
|
||||
// Setup audio processing for Windows loopback audio only
|
||||
audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||
const source = audioContext.createMediaStreamSource(mediaStream);
|
||||
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
||||
|
||||
let audioBuffer = [];
|
||||
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
||||
|
||||
audioProcessor.onaudioprocess = async e => {
|
||||
const inputData = e.inputBuffer.getChannelData(0);
|
||||
audioBuffer.push(...inputData);
|
||||
|
||||
// Process audio in chunks
|
||||
while (audioBuffer.length >= samplesPerChunk) {
|
||||
const chunk = audioBuffer.splice(0, samplesPerChunk);
|
||||
const pcmData16 = convertFloat32ToInt16(chunk);
|
||||
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
||||
|
||||
await ipcRenderer.invoke('send-audio-content', {
|
||||
data: base64Data,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(audioProcessor);
|
||||
audioProcessor.connect(audioContext.destination);
|
||||
}
|
||||
|
||||
// ---------------------------
|
||||
// Screenshot functions (exact from renderer.js)
|
||||
// ---------------------------
|
||||
async function captureScreenshot(imageQuality = 'medium', isManual = false) {
|
||||
console.log(`Capturing ${isManual ? 'manual' : 'automated'} screenshot...`);
|
||||
|
||||
// Check rate limiting for automated screenshots only
|
||||
if (!isManual && tokenTracker.shouldThrottle()) {
|
||||
console.log('⚠️ Automated screenshot skipped due to rate limiting');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Request screenshot from main process
|
||||
const result = await ipcRenderer.invoke('capture-screenshot', {
|
||||
quality: imageQuality,
|
||||
});
|
||||
|
||||
if (result.success && result.base64) {
|
||||
// Store the latest screenshot
|
||||
lastScreenshotBase64 = result.base64;
|
||||
|
||||
// Note: sendResult is not defined in the original, this was likely an error
|
||||
// Commenting out this section as it references undefined variable
|
||||
/*
|
||||
if (sendResult.success) {
|
||||
// Track image tokens after successful send
|
||||
const imageTokens = tokenTracker.calculateImageTokens(result.width || 1920, result.height || 1080);
|
||||
tokenTracker.addTokens(imageTokens, 'image');
|
||||
console.log(`📊 Image sent successfully - ${imageTokens} tokens used (${result.width}x${result.height})`);
|
||||
} else {
|
||||
console.error('Failed to send image:', sendResult.error);
|
||||
}
|
||||
*/
|
||||
} else {
|
||||
console.error('Failed to capture screenshot:', result.error);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during OpenAI initialization IPC call:', error);
|
||||
const appElement = pickleGlassElement();
|
||||
if (appElement && typeof appElement.setStatus === 'function') {
|
||||
appElement.setStatus('Error');
|
||||
}
|
||||
console.error('Error capturing screenshot:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureManualScreenshot(imageQuality = null) {
|
||||
console.log('Manual screenshot triggered');
|
||||
const quality = imageQuality || currentImageQuality;
|
||||
await captureScreenshot(quality, true);
|
||||
}
|
||||
|
||||
ipcRenderer.on('system-audio-data', (event, { data }) => {
|
||||
systemAudioBuffer.push({
|
||||
data: data,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
async function getCurrentScreenshot() {
|
||||
try {
|
||||
// First try to get a fresh screenshot from main process
|
||||
const result = await ipcRenderer.invoke('get-current-screenshot');
|
||||
|
||||
// 오래된 데이터 제거
|
||||
if (systemAudioBuffer.length > MAX_SYSTEM_BUFFER_SIZE) {
|
||||
systemAudioBuffer = systemAudioBuffer.slice(-MAX_SYSTEM_BUFFER_SIZE);
|
||||
}
|
||||
|
||||
console.log('📥 Received system audio for AEC reference');
|
||||
});
|
||||
|
||||
// Listen for status updates
|
||||
ipcRenderer.on('update-status', (event, status) => {
|
||||
console.log('Status update:', status);
|
||||
pickleGlass.e().setStatus(status);
|
||||
});
|
||||
|
||||
// Listen for real-time STT updates
|
||||
ipcRenderer.on('stt-update', (event, data) => {
|
||||
console.log('Renderer.js stt-update', data);
|
||||
const { speaker, text, isFinal, isPartial, timestamp } = data;
|
||||
|
||||
if (isPartial) {
|
||||
console.log(`🔄 [${speaker} - partial]: ${text}`);
|
||||
} else if (isFinal) {
|
||||
console.log(`✅ [${speaker} - final]: ${text}`);
|
||||
|
||||
const speakerText = speaker.toLowerCase();
|
||||
const conversationText = `${speakerText}: ${text.trim()}`;
|
||||
|
||||
realtimeConversationHistory.push(conversationText);
|
||||
|
||||
if (realtimeConversationHistory.length > 30) {
|
||||
realtimeConversationHistory = realtimeConversationHistory.slice(-30);
|
||||
if (result.success && result.base64) {
|
||||
console.log('📸 Got fresh screenshot from main process');
|
||||
return result.base64;
|
||||
}
|
||||
|
||||
console.log(`📝 Updated realtime conversation history: ${realtimeConversationHistory.length} texts`);
|
||||
console.log(`📋 Latest text: ${conversationText}`);
|
||||
}
|
||||
|
||||
if (pickleGlass.e() && typeof pickleGlass.e().updateRealtimeTranscription === 'function') {
|
||||
pickleGlass.e().updateRealtimeTranscription({
|
||||
speaker,
|
||||
text,
|
||||
isFinal,
|
||||
isPartial,
|
||||
timestamp,
|
||||
// If no screenshot available, capture one now
|
||||
console.log('📸 No screenshot available, capturing new one');
|
||||
const captureResult = await ipcRenderer.invoke('capture-screenshot', {
|
||||
quality: currentImageQuality,
|
||||
});
|
||||
|
||||
if (captureResult.success && captureResult.base64) {
|
||||
lastScreenshotBase64 = captureResult.base64;
|
||||
return captureResult.base64;
|
||||
}
|
||||
|
||||
// Fallback to last stored screenshot
|
||||
if (lastScreenshotBase64) {
|
||||
console.log('📸 Using cached screenshot');
|
||||
return lastScreenshotBase64;
|
||||
}
|
||||
|
||||
throw new Error('Failed to get screenshot');
|
||||
} catch (error) {
|
||||
console.error('Error getting current screenshot:', error);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
ipcRenderer.on('update-structured-data', (_, structuredData) => {
|
||||
console.log('📥 Received structured data update:', structuredData);
|
||||
window.pickleGlass.structuredData = structuredData;
|
||||
window.pickleGlass.setStructuredData(structuredData);
|
||||
});
|
||||
window.pickleGlass.structuredData = {
|
||||
summary: [],
|
||||
topic: { header: '', bullets: [] },
|
||||
actions: [],
|
||||
};
|
||||
window.pickleGlass.setStructuredData = data => {
|
||||
window.pickleGlass.structuredData = data;
|
||||
pickleGlass.e()?.updateStructuredData?.(data);
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------
|
||||
// Main capture functions (exact from renderer.js)
|
||||
// ---------------------------
|
||||
async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'medium') {
|
||||
// Store the image quality for manual screenshots
|
||||
currentImageQuality = imageQuality;
|
||||
@ -490,12 +600,6 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
|
||||
setupWindowsLoopbackProcessing();
|
||||
}
|
||||
|
||||
// console.log('MediaStream obtained:', {
|
||||
// hasVideo: mediaStream.getVideoTracks().length > 0,
|
||||
// hasAudio: mediaStream.getAudioTracks().length > 0,
|
||||
// videoTrack: mediaStream.getVideoTracks()[0]?.getSettings(),
|
||||
// });
|
||||
|
||||
// Start capturing screenshots - check if manual mode
|
||||
if (screenshotIntervalSeconds === 'manual' || screenshotIntervalSeconds === 'Manual') {
|
||||
console.log('Manual mode enabled - screenshots will be captured on demand only');
|
||||
@ -511,162 +615,11 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error starting capture:', err);
|
||||
pickleGlass.e().setStatus('error');
|
||||
// Note: pickleGlass.e() is not available in this context, commenting out
|
||||
// pickleGlass.e().setStatus('error');
|
||||
}
|
||||
}
|
||||
|
||||
function setupMicProcessing(micStream) {
|
||||
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||
const micSource = micAudioContext.createMediaStreamSource(micStream);
|
||||
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
||||
|
||||
let audioBuffer = [];
|
||||
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
||||
|
||||
micProcessor.onaudioprocess = async e => {
|
||||
const inputData = e.inputBuffer.getChannelData(0);
|
||||
audioBuffer.push(...inputData);
|
||||
|
||||
while (audioBuffer.length >= samplesPerChunk) {
|
||||
let chunk = audioBuffer.splice(0, samplesPerChunk);
|
||||
let processedChunk = new Float32Array(chunk);
|
||||
|
||||
// Check for system audio and apply AEC only if voice is active
|
||||
if (aecProcessor && systemAudioBuffer.length > 0) {
|
||||
const latestSystemAudio = systemAudioBuffer[systemAudioBuffer.length - 1];
|
||||
const systemFloat32 = base64ToFloat32Array(latestSystemAudio.data);
|
||||
|
||||
// Apply AEC only when system audio has active speech
|
||||
if (isVoiceActive(systemFloat32)) {
|
||||
processedChunk = aecProcessor.process(new Float32Array(chunk), systemFloat32);
|
||||
console.log('🔊 Applied AEC because system audio is active');
|
||||
}
|
||||
}
|
||||
|
||||
const pcmData16 = convertFloat32ToInt16(processedChunk);
|
||||
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
||||
|
||||
await ipcRenderer.invoke('send-audio-content', {
|
||||
data: base64Data,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
micSource.connect(micProcessor);
|
||||
micProcessor.connect(micAudioContext.destination);
|
||||
|
||||
audioProcessor = micProcessor;
|
||||
}
|
||||
////////// for index & subjects //////////
|
||||
|
||||
function setupLinuxMicProcessing(micStream) {
|
||||
// Setup microphone audio processing for Linux
|
||||
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||
const micSource = micAudioContext.createMediaStreamSource(micStream);
|
||||
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
||||
|
||||
let audioBuffer = [];
|
||||
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
||||
|
||||
micProcessor.onaudioprocess = async e => {
|
||||
const inputData = e.inputBuffer.getChannelData(0);
|
||||
audioBuffer.push(...inputData);
|
||||
|
||||
// Process audio in chunks
|
||||
while (audioBuffer.length >= samplesPerChunk) {
|
||||
const chunk = audioBuffer.splice(0, samplesPerChunk);
|
||||
const pcmData16 = convertFloat32ToInt16(chunk);
|
||||
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
||||
|
||||
await ipcRenderer.invoke('send-audio-content', {
|
||||
data: base64Data,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
micSource.connect(micProcessor);
|
||||
micProcessor.connect(micAudioContext.destination);
|
||||
|
||||
// Store processor reference for cleanup
|
||||
audioProcessor = micProcessor;
|
||||
}
|
||||
|
||||
function setupWindowsLoopbackProcessing() {
|
||||
// Setup audio processing for Windows loopback audio only
|
||||
audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||
const source = audioContext.createMediaStreamSource(mediaStream);
|
||||
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
||||
|
||||
let audioBuffer = [];
|
||||
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
||||
|
||||
audioProcessor.onaudioprocess = async e => {
|
||||
const inputData = e.inputBuffer.getChannelData(0);
|
||||
audioBuffer.push(...inputData);
|
||||
|
||||
// Process audio in chunks
|
||||
while (audioBuffer.length >= samplesPerChunk) {
|
||||
const chunk = audioBuffer.splice(0, samplesPerChunk);
|
||||
const pcmData16 = convertFloat32ToInt16(chunk);
|
||||
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
||||
|
||||
await ipcRenderer.invoke('send-audio-content', {
|
||||
data: base64Data,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(audioProcessor);
|
||||
audioProcessor.connect(audioContext.destination);
|
||||
}
|
||||
|
||||
async function captureScreenshot(imageQuality = 'medium', isManual = false) {
|
||||
console.log(`Capturing ${isManual ? 'manual' : 'automated'} screenshot...`);
|
||||
|
||||
// Check rate limiting for automated screenshots only
|
||||
if (!isManual && tokenTracker.shouldThrottle()) {
|
||||
console.log('⚠️ Automated screenshot skipped due to rate limiting');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Request screenshot from main process
|
||||
const result = await ipcRenderer.invoke('capture-screenshot', {
|
||||
quality: imageQuality,
|
||||
});
|
||||
|
||||
if (result.success && result.base64) {
|
||||
// Store the latest screenshot
|
||||
lastScreenshotBase64 = result.base64;
|
||||
|
||||
if (sendResult.success) {
|
||||
// Track image tokens after successful send
|
||||
const imageTokens = tokenTracker.calculateImageTokens(result.width || 1920, result.height || 1080);
|
||||
tokenTracker.addTokens(imageTokens, 'image');
|
||||
console.log(`📊 Image sent successfully - ${imageTokens} tokens used (${result.width}x${result.height})`);
|
||||
} else {
|
||||
console.error('Failed to send image:', sendResult.error);
|
||||
}
|
||||
} else {
|
||||
console.error('Failed to capture screenshot:', result.error);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error capturing screenshot:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureManualScreenshot(imageQuality = null) {
|
||||
console.log('Manual screenshot triggered');
|
||||
const quality = imageQuality || currentImageQuality;
|
||||
await captureScreenshot(quality, true);
|
||||
}
|
||||
|
||||
// Expose functions to global scope for external access
|
||||
window.captureManualScreenshot = captureManualScreenshot;
|
||||
|
||||
function stopCapture() {
|
||||
if (screenshotInterval) {
|
||||
clearInterval(screenshotInterval);
|
||||
@ -706,76 +659,25 @@ function stopCapture() {
|
||||
}
|
||||
}
|
||||
|
||||
async function getCurrentScreenshot() {
|
||||
try {
|
||||
// First try to get a fresh screenshot from main process
|
||||
const result = await ipcRenderer.invoke('get-current-screenshot');
|
||||
|
||||
if (result.success && result.base64) {
|
||||
console.log('📸 Got fresh screenshot from main process');
|
||||
return result.base64;
|
||||
}
|
||||
|
||||
// If no screenshot available, capture one now
|
||||
console.log('📸 No screenshot available, capturing new one');
|
||||
const captureResult = await ipcRenderer.invoke('capture-screenshot', {
|
||||
quality: currentImageQuality,
|
||||
});
|
||||
|
||||
if (captureResult.success && captureResult.base64) {
|
||||
lastScreenshotBase64 = captureResult.base64;
|
||||
return captureResult.base64;
|
||||
}
|
||||
|
||||
// Fallback to last stored screenshot
|
||||
if (lastScreenshotBase64) {
|
||||
console.log('📸 Using cached screenshot');
|
||||
return lastScreenshotBase64;
|
||||
}
|
||||
|
||||
throw new Error('Failed to get screenshot');
|
||||
} catch (error) {
|
||||
console.error('Error getting current screenshot:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function formatRealtimeConversationHistory() {
|
||||
if (realtimeConversationHistory.length === 0) return 'No conversation history available.';
|
||||
|
||||
return realtimeConversationHistory.slice(-30).join('\n');
|
||||
}
|
||||
|
||||
window.pickleGlass = {
|
||||
initializeopenai,
|
||||
// ---------------------------
|
||||
// Exports & global registration
|
||||
// ---------------------------
|
||||
module.exports = {
|
||||
startCapture,
|
||||
stopCapture,
|
||||
isLinux: isLinux,
|
||||
isMacOS: isMacOS,
|
||||
e: pickleGlassElement,
|
||||
captureManualScreenshot,
|
||||
getCurrentScreenshot,
|
||||
isLinux,
|
||||
isMacOS,
|
||||
};
|
||||
|
||||
// -------------------------------------------------------
|
||||
// 🔔 React to session state changes from the main process
|
||||
// When the session ends (isActive === false), ensure we stop
|
||||
// all local capture pipelines (mic, screen, etc.).
|
||||
// -------------------------------------------------------
|
||||
ipcRenderer.on('session-state-changed', (_event, { isActive }) => {
|
||||
if (!isActive) {
|
||||
console.log('[Renderer] Session ended – stopping local capture');
|
||||
stopCapture();
|
||||
} else {
|
||||
console.log('[Renderer] New session started – clearing in-memory history and summaries');
|
||||
|
||||
// Reset live conversation & analysis caches
|
||||
realtimeConversationHistory = [];
|
||||
|
||||
const blankData = {
|
||||
summary: [],
|
||||
topic: { header: '', bullets: [] },
|
||||
actions: [],
|
||||
followUps: [],
|
||||
};
|
||||
window.pickleGlass.setStructuredData(blankData);
|
||||
}
|
||||
});
|
||||
// Expose functions to global scope for external access (exact from renderer.js)
|
||||
if (typeof window !== 'undefined') {
|
||||
window.captureManualScreenshot = captureManualScreenshot;
|
||||
window.listenCapture = module.exports;
|
||||
window.pickleGlass = window.pickleGlass || {};
|
||||
window.pickleGlass.startCapture = startCapture;
|
||||
window.pickleGlass.stopCapture = stopCapture;
|
||||
window.pickleGlass.captureManualScreenshot = captureManualScreenshot;
|
||||
window.pickleGlass.getCurrentScreenshot = getCurrentScreenshot;
|
||||
}
|
||||
139
src/features/listen/renderer/renderer.js
Normal file
139
src/features/listen/renderer/renderer.js
Normal file
@ -0,0 +1,139 @@
|
||||
// renderer.js
|
||||
const { ipcRenderer } = require('electron');
|
||||
const { makeStreamingChatCompletionWithPortkey } = require('../../../common/services/aiProviderService.js');
|
||||
const listenCapture = require('./listenCapture.js');
|
||||
|
||||
let realtimeConversationHistory = [];
|
||||
|
||||
async function queryLoginState() {
|
||||
const userState = await ipcRenderer.invoke('get-current-user');
|
||||
return userState;
|
||||
}
|
||||
|
||||
function pickleGlassElement() {
|
||||
return document.getElementById('pickle-glass');
|
||||
}
|
||||
|
||||
async function initializeopenai(profile = 'interview', language = 'en') {
|
||||
// The API key is now handled in the main process from .env file.
|
||||
// We just need to trigger the initialization.
|
||||
try {
|
||||
console.log(`Requesting OpenAI initialization with profile: ${profile}, language: ${language}`);
|
||||
const success = await ipcRenderer.invoke('initialize-openai', profile, language);
|
||||
if (success) {
|
||||
// The status will be updated via 'update-status' event from the main process.
|
||||
console.log('OpenAI initialization successful.');
|
||||
} else {
|
||||
console.error('OpenAI initialization failed.');
|
||||
const appElement = pickleGlassElement();
|
||||
if (appElement && typeof appElement.setStatus === 'function') {
|
||||
appElement.setStatus('Initialization Failed');
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during OpenAI initialization IPC call:', error);
|
||||
const appElement = pickleGlassElement();
|
||||
if (appElement && typeof appElement.setStatus === 'function') {
|
||||
appElement.setStatus('Error');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Listen for status updates
|
||||
ipcRenderer.on('update-status', (event, status) => {
|
||||
console.log('Status update:', status);
|
||||
pickleGlass.e().setStatus(status);
|
||||
});
|
||||
|
||||
// Listen for real-time STT updates
|
||||
ipcRenderer.on('stt-update', (event, data) => {
|
||||
console.log('Renderer.js stt-update', data);
|
||||
const { speaker, text, isFinal, isPartial, timestamp } = data;
|
||||
|
||||
if (isPartial) {
|
||||
console.log(`🔄 [${speaker} - partial]: ${text}`);
|
||||
} else if (isFinal) {
|
||||
console.log(`✅ [${speaker} - final]: ${text}`);
|
||||
|
||||
const speakerText = speaker.toLowerCase();
|
||||
const conversationText = `${speakerText}: ${text.trim()}`;
|
||||
|
||||
realtimeConversationHistory.push(conversationText);
|
||||
|
||||
if (realtimeConversationHistory.length > 30) {
|
||||
realtimeConversationHistory = realtimeConversationHistory.slice(-30);
|
||||
}
|
||||
|
||||
console.log(`📝 Updated realtime conversation history: ${realtimeConversationHistory.length} texts`);
|
||||
console.log(`📋 Latest text: ${conversationText}`);
|
||||
}
|
||||
|
||||
if (pickleGlass.e() && typeof pickleGlass.e().updateRealtimeTranscription === 'function') {
|
||||
pickleGlass.e().updateRealtimeTranscription({
|
||||
speaker,
|
||||
text,
|
||||
isFinal,
|
||||
isPartial,
|
||||
timestamp,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
ipcRenderer.on('update-structured-data', (_, structuredData) => {
|
||||
console.log('📥 Received structured data update:', structuredData);
|
||||
window.pickleGlass.structuredData = structuredData;
|
||||
window.pickleGlass.setStructuredData(structuredData);
|
||||
});
|
||||
|
||||
window.pickleGlass.structuredData = {
|
||||
summary: [],
|
||||
topic: { header: '', bullets: [] },
|
||||
actions: [],
|
||||
};
|
||||
|
||||
window.pickleGlass.setStructuredData = data => {
|
||||
window.pickleGlass.structuredData = data;
|
||||
pickleGlass.e()?.updateStructuredData?.(data);
|
||||
};
|
||||
|
||||
function formatRealtimeConversationHistory() {
|
||||
if (realtimeConversationHistory.length === 0) return 'No conversation history available.';
|
||||
|
||||
return realtimeConversationHistory.slice(-30).join('\n');
|
||||
}
|
||||
|
||||
window.pickleGlass = {
|
||||
initializeopenai,
|
||||
startCapture: listenCapture.startCapture,
|
||||
stopCapture: listenCapture.stopCapture,
|
||||
isLinux: listenCapture.isLinux,
|
||||
isMacOS: listenCapture.isMacOS,
|
||||
captureManualScreenshot: listenCapture.captureManualScreenshot,
|
||||
getCurrentScreenshot: listenCapture.getCurrentScreenshot,
|
||||
e: pickleGlassElement,
|
||||
};
|
||||
|
||||
// -------------------------------------------------------
|
||||
// 🔔 React to session state changes from the main process
|
||||
// When the session ends (isActive === false), ensure we stop
|
||||
// all local capture pipelines (mic, screen, etc.).
|
||||
// -------------------------------------------------------
|
||||
ipcRenderer.on('session-state-changed', (_event, { isActive }) => {
|
||||
if (!isActive) {
|
||||
console.log('[Renderer] Session ended – stopping local capture');
|
||||
listenCapture.stopCapture();
|
||||
} else {
|
||||
console.log('[Renderer] New session started – clearing in-memory history and summaries');
|
||||
|
||||
// Reset live conversation & analysis caches
|
||||
realtimeConversationHistory = [];
|
||||
|
||||
const blankData = {
|
||||
summary: [],
|
||||
topic: { header: '', bullets: [] },
|
||||
actions: [],
|
||||
followUps: [],
|
||||
};
|
||||
window.pickleGlass.setStructuredData(blankData);
|
||||
}
|
||||
});
|
||||
5
src/features/listen/stt/repositories/index.js
Normal file
5
src/features/listen/stt/repositories/index.js
Normal file
@ -0,0 +1,5 @@
|
||||
const sttRepository = require('./sqlite.repository');
|
||||
|
||||
module.exports = {
|
||||
...sttRepository,
|
||||
};
|
||||
37
src/features/listen/stt/repositories/sqlite.repository.js
Normal file
37
src/features/listen/stt/repositories/sqlite.repository.js
Normal file
@ -0,0 +1,37 @@
|
||||
const sqliteClient = require('../../../../common/services/sqliteClient');
|
||||
|
||||
function addTranscript({ sessionId, speaker, text }) {
|
||||
const db = sqliteClient.getDb();
|
||||
return new Promise((resolve, reject) => {
|
||||
const transcriptId = require('crypto').randomUUID();
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const query = `INSERT INTO transcripts (id, session_id, start_at, speaker, text, created_at) VALUES (?, ?, ?, ?, ?, ?)`;
|
||||
db.run(query, [transcriptId, sessionId, now, speaker, text, now], function(err) {
|
||||
if (err) {
|
||||
console.error('Error adding transcript:', err);
|
||||
reject(err);
|
||||
} else {
|
||||
resolve({ id: transcriptId });
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getAllTranscriptsBySessionId(sessionId) {
|
||||
const db = sqliteClient.getDb();
|
||||
return new Promise((resolve, reject) => {
|
||||
const query = "SELECT * FROM transcripts WHERE session_id = ? ORDER BY start_at ASC";
|
||||
db.all(query, [sessionId], (err, rows) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(rows);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
addTranscript,
|
||||
getAllTranscriptsBySessionId,
|
||||
};
|
||||
480
src/features/listen/stt/sttService.js
Normal file
480
src/features/listen/stt/sttService.js
Normal file
@ -0,0 +1,480 @@
|
||||
const { BrowserWindow } = require('electron');
|
||||
const { spawn } = require('child_process');
|
||||
const { connectToGeminiSession } = require('../../../common/services/googleGeminiClient.js');
|
||||
const { connectToOpenAiSession } = require('../../../common/services/openAiClient.js');
|
||||
const { getStoredApiKey, getStoredProvider } = require('../../../electron/windowManager');
|
||||
|
||||
const COMPLETION_DEBOUNCE_MS = 2000;
|
||||
|
||||
class SttService {
|
||||
constructor() {
|
||||
this.mySttSession = null;
|
||||
this.theirSttSession = null;
|
||||
this.myCurrentUtterance = '';
|
||||
this.theirCurrentUtterance = '';
|
||||
|
||||
this.myLastPartialText = '';
|
||||
this.theirLastPartialText = '';
|
||||
this.myInactivityTimer = null;
|
||||
this.theirInactivityTimer = null;
|
||||
|
||||
// Turn-completion debouncing
|
||||
this.myCompletionBuffer = '';
|
||||
this.theirCompletionBuffer = '';
|
||||
this.myCompletionTimer = null;
|
||||
this.theirCompletionTimer = null;
|
||||
|
||||
// System audio capture
|
||||
this.systemAudioProc = null;
|
||||
|
||||
// Callbacks
|
||||
this.onTranscriptionComplete = null;
|
||||
this.onStatusUpdate = null;
|
||||
}
|
||||
|
||||
setCallbacks({ onTranscriptionComplete, onStatusUpdate }) {
|
||||
this.onTranscriptionComplete = onTranscriptionComplete;
|
||||
this.onStatusUpdate = onStatusUpdate;
|
||||
}
|
||||
|
||||
async getApiKey() {
|
||||
const storedKey = await getStoredApiKey();
|
||||
if (storedKey) {
|
||||
console.log('[SttService] Using stored API key');
|
||||
return storedKey;
|
||||
}
|
||||
|
||||
const envKey = process.env.OPENAI_API_KEY;
|
||||
if (envKey) {
|
||||
console.log('[SttService] Using environment API key');
|
||||
return envKey;
|
||||
}
|
||||
|
||||
console.error('[SttService] No API key found in storage or environment');
|
||||
return null;
|
||||
}
|
||||
|
||||
async getAiProvider() {
|
||||
try {
|
||||
const { ipcRenderer } = require('electron');
|
||||
const provider = await ipcRenderer.invoke('get-ai-provider');
|
||||
return provider || 'openai';
|
||||
} catch (error) {
|
||||
return getStoredProvider ? getStoredProvider() : 'openai';
|
||||
}
|
||||
}
|
||||
|
||||
sendToRenderer(channel, data) {
|
||||
BrowserWindow.getAllWindows().forEach(win => {
|
||||
if (!win.isDestroyed()) {
|
||||
win.webContents.send(channel, data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
flushMyCompletion() {
|
||||
if (!this.myCompletionBuffer.trim()) return;
|
||||
|
||||
const finalText = this.myCompletionBuffer.trim();
|
||||
|
||||
// Notify completion callback
|
||||
if (this.onTranscriptionComplete) {
|
||||
this.onTranscriptionComplete('Me', finalText);
|
||||
}
|
||||
|
||||
// Send to renderer as final
|
||||
this.sendToRenderer('stt-update', {
|
||||
speaker: 'Me',
|
||||
text: finalText,
|
||||
isPartial: false,
|
||||
isFinal: true,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
this.myCompletionBuffer = '';
|
||||
this.myCompletionTimer = null;
|
||||
this.myCurrentUtterance = '';
|
||||
|
||||
if (this.onStatusUpdate) {
|
||||
this.onStatusUpdate('Listening...');
|
||||
}
|
||||
}
|
||||
|
||||
flushTheirCompletion() {
|
||||
if (!this.theirCompletionBuffer.trim()) return;
|
||||
|
||||
const finalText = this.theirCompletionBuffer.trim();
|
||||
|
||||
// Notify completion callback
|
||||
if (this.onTranscriptionComplete) {
|
||||
this.onTranscriptionComplete('Them', finalText);
|
||||
}
|
||||
|
||||
// Send to renderer as final
|
||||
this.sendToRenderer('stt-update', {
|
||||
speaker: 'Them',
|
||||
text: finalText,
|
||||
isPartial: false,
|
||||
isFinal: true,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
this.theirCompletionBuffer = '';
|
||||
this.theirCompletionTimer = null;
|
||||
this.theirCurrentUtterance = '';
|
||||
|
||||
if (this.onStatusUpdate) {
|
||||
this.onStatusUpdate('Listening...');
|
||||
}
|
||||
}
|
||||
|
||||
debounceMyCompletion(text) {
|
||||
// 상대방이 말하고 있던 경우, 화자가 변경되었으므로 즉시 상대방의 말풍선을 완성합니다.
|
||||
if (this.theirCompletionTimer) {
|
||||
clearTimeout(this.theirCompletionTimer);
|
||||
this.flushTheirCompletion();
|
||||
}
|
||||
|
||||
this.myCompletionBuffer += (this.myCompletionBuffer ? ' ' : '') + text;
|
||||
|
||||
if (this.myCompletionTimer) clearTimeout(this.myCompletionTimer);
|
||||
this.myCompletionTimer = setTimeout(() => this.flushMyCompletion(), COMPLETION_DEBOUNCE_MS);
|
||||
}
|
||||
|
||||
debounceTheirCompletion(text) {
|
||||
// 내가 말하고 있던 경우, 화자가 변경되었으므로 즉시 내 말풍선을 완성합니다.
|
||||
if (this.myCompletionTimer) {
|
||||
clearTimeout(this.myCompletionTimer);
|
||||
this.flushMyCompletion();
|
||||
}
|
||||
|
||||
this.theirCompletionBuffer += (this.theirCompletionBuffer ? ' ' : '') + text;
|
||||
|
||||
if (this.theirCompletionTimer) clearTimeout(this.theirCompletionTimer);
|
||||
this.theirCompletionTimer = setTimeout(() => this.flushTheirCompletion(), COMPLETION_DEBOUNCE_MS);
|
||||
}
|
||||
|
||||
async initializeSttSessions(language = 'en') {
|
||||
const effectiveLanguage = process.env.OPENAI_TRANSCRIBE_LANG || language || 'en';
|
||||
|
||||
const API_KEY = await this.getApiKey();
|
||||
if (!API_KEY) {
|
||||
throw new Error('No API key available');
|
||||
}
|
||||
|
||||
const provider = await this.getAiProvider();
|
||||
const isGemini = provider === 'gemini';
|
||||
console.log(`[SttService] Initializing STT for provider: ${provider}`);
|
||||
|
||||
const handleMyMessage = message => {
|
||||
if (isGemini) {
|
||||
const text = message.serverContent?.inputTranscription?.text || '';
|
||||
if (text && text.trim()) {
|
||||
const finalUtteranceText = text.trim().replace(/<noise>/g, '').trim();
|
||||
if (finalUtteranceText && finalUtteranceText !== '.') {
|
||||
this.debounceMyCompletion(finalUtteranceText);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const type = message.type;
|
||||
const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || '';
|
||||
|
||||
if (type === 'conversation.item.input_audio_transcription.delta') {
|
||||
if (this.myCompletionTimer) clearTimeout(this.myCompletionTimer);
|
||||
this.myCompletionTimer = null;
|
||||
this.myCurrentUtterance += text;
|
||||
const continuousText = this.myCompletionBuffer + (this.myCompletionBuffer ? ' ' : '') + this.myCurrentUtterance;
|
||||
if (text && !text.includes('vq_lbr_audio_')) {
|
||||
this.sendToRenderer('stt-update', {
|
||||
speaker: 'Me',
|
||||
text: continuousText,
|
||||
isPartial: true,
|
||||
isFinal: false,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
}
|
||||
} else if (type === 'conversation.item.input_audio_transcription.completed') {
|
||||
if (text && text.trim()) {
|
||||
const finalUtteranceText = text.trim();
|
||||
this.myCurrentUtterance = '';
|
||||
this.debounceMyCompletion(finalUtteranceText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (message.error) {
|
||||
console.error('[Me] STT Session Error:', message.error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTheirMessage = message => {
|
||||
if (isGemini) {
|
||||
const text = message.serverContent?.inputTranscription?.text || '';
|
||||
if (text && text.trim()) {
|
||||
const finalUtteranceText = text.trim().replace(/<noise>/g, '').trim();
|
||||
if (finalUtteranceText && finalUtteranceText !== '.') {
|
||||
this.debounceTheirCompletion(finalUtteranceText);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const type = message.type;
|
||||
const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || '';
|
||||
if (type === 'conversation.item.input_audio_transcription.delta') {
|
||||
if (this.theirCompletionTimer) clearTimeout(this.theirCompletionTimer);
|
||||
this.theirCompletionTimer = null;
|
||||
this.theirCurrentUtterance += text;
|
||||
const continuousText = this.theirCompletionBuffer + (this.theirCompletionBuffer ? ' ' : '') + this.theirCurrentUtterance;
|
||||
if (text && !text.includes('vq_lbr_audio_')) {
|
||||
this.sendToRenderer('stt-update', {
|
||||
speaker: 'Them',
|
||||
text: continuousText,
|
||||
isPartial: true,
|
||||
isFinal: false,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
}
|
||||
} else if (type === 'conversation.item.input_audio_transcription.completed') {
|
||||
if (text && text.trim()) {
|
||||
const finalUtteranceText = text.trim();
|
||||
this.theirCurrentUtterance = '';
|
||||
this.debounceTheirCompletion(finalUtteranceText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (message.error) {
|
||||
console.error('[Them] STT Session Error:', message.error);
|
||||
}
|
||||
};
|
||||
|
||||
const mySttConfig = {
|
||||
language: effectiveLanguage,
|
||||
callbacks: {
|
||||
onmessage: handleMyMessage,
|
||||
onerror: error => console.error('My STT session error:', error.message),
|
||||
onclose: event => console.log('My STT session closed:', event.reason),
|
||||
},
|
||||
};
|
||||
|
||||
const theirSttConfig = {
|
||||
language: effectiveLanguage,
|
||||
callbacks: {
|
||||
onmessage: handleTheirMessage,
|
||||
onerror: error => console.error('Their STT session error:', error.message),
|
||||
onclose: event => console.log('Their STT session closed:', event.reason),
|
||||
},
|
||||
};
|
||||
|
||||
// Determine key type based on auth status
|
||||
const authService = require('../../../common/services/authService');
|
||||
const userState = authService.getCurrentUser();
|
||||
const loggedIn = userState.isLoggedIn;
|
||||
const keyType = loggedIn ? 'vKey' : 'apiKey';
|
||||
|
||||
if (isGemini) {
|
||||
[this.mySttSession, this.theirSttSession] = await Promise.all([
|
||||
connectToGeminiSession(API_KEY, mySttConfig),
|
||||
connectToGeminiSession(API_KEY, theirSttConfig),
|
||||
]);
|
||||
} else {
|
||||
[this.mySttSession, this.theirSttSession] = await Promise.all([
|
||||
connectToOpenAiSession(API_KEY, mySttConfig, keyType),
|
||||
connectToOpenAiSession(API_KEY, theirSttConfig, keyType),
|
||||
]);
|
||||
}
|
||||
|
||||
console.log('✅ Both STT sessions initialized successfully.');
|
||||
return true;
|
||||
}
|
||||
|
||||
async sendAudioContent(data, mimeType) {
|
||||
const provider = await this.getAiProvider();
|
||||
const isGemini = provider === 'gemini';
|
||||
|
||||
if (!this.mySttSession) {
|
||||
throw new Error('User STT session not active');
|
||||
}
|
||||
|
||||
const payload = isGemini
|
||||
? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } }
|
||||
: data;
|
||||
|
||||
await this.mySttSession.sendRealtimeInput(payload);
|
||||
}
|
||||
|
||||
killExistingSystemAudioDump() {
|
||||
return new Promise(resolve => {
|
||||
console.log('Checking for existing SystemAudioDump processes...');
|
||||
|
||||
const killProc = spawn('pkill', ['-f', 'SystemAudioDump'], {
|
||||
stdio: 'ignore',
|
||||
});
|
||||
|
||||
killProc.on('close', code => {
|
||||
if (code === 0) {
|
||||
console.log('Killed existing SystemAudioDump processes');
|
||||
} else {
|
||||
console.log('No existing SystemAudioDump processes found');
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
|
||||
killProc.on('error', err => {
|
||||
console.log('Error checking for existing processes (this is normal):', err.message);
|
||||
resolve();
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
killProc.kill();
|
||||
resolve();
|
||||
}, 2000);
|
||||
});
|
||||
}
|
||||
|
||||
async startMacOSAudioCapture() {
|
||||
if (process.platform !== 'darwin' || !this.theirSttSession) return false;
|
||||
|
||||
await this.killExistingSystemAudioDump();
|
||||
console.log('Starting macOS audio capture for "Them"...');
|
||||
|
||||
const { app } = require('electron');
|
||||
const path = require('path');
|
||||
const systemAudioPath = app.isPackaged
|
||||
? path.join(process.resourcesPath, 'app.asar.unpacked', 'src', 'assets', 'SystemAudioDump')
|
||||
: path.join(app.getAppPath(), 'src', 'assets', 'SystemAudioDump');
|
||||
|
||||
console.log('SystemAudioDump path:', systemAudioPath);
|
||||
|
||||
this.systemAudioProc = spawn(systemAudioPath, [], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
if (!this.systemAudioProc.pid) {
|
||||
console.error('Failed to start SystemAudioDump');
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log('SystemAudioDump started with PID:', this.systemAudioProc.pid);
|
||||
|
||||
const CHUNK_DURATION = 0.1;
|
||||
const SAMPLE_RATE = 24000;
|
||||
const BYTES_PER_SAMPLE = 2;
|
||||
const CHANNELS = 2;
|
||||
const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * CHUNK_DURATION;
|
||||
|
||||
let audioBuffer = Buffer.alloc(0);
|
||||
|
||||
const provider = await this.getAiProvider();
|
||||
const isGemini = provider === 'gemini';
|
||||
|
||||
this.systemAudioProc.stdout.on('data', async data => {
|
||||
audioBuffer = Buffer.concat([audioBuffer, data]);
|
||||
|
||||
while (audioBuffer.length >= CHUNK_SIZE) {
|
||||
const chunk = audioBuffer.slice(0, CHUNK_SIZE);
|
||||
audioBuffer = audioBuffer.slice(CHUNK_SIZE);
|
||||
|
||||
const monoChunk = CHANNELS === 2 ? this.convertStereoToMono(chunk) : chunk;
|
||||
const base64Data = monoChunk.toString('base64');
|
||||
|
||||
this.sendToRenderer('system-audio-data', { data: base64Data });
|
||||
|
||||
if (this.theirSttSession) {
|
||||
try {
|
||||
const payload = isGemini
|
||||
? { audio: { data: base64Data, mimeType: 'audio/pcm;rate=24000' } }
|
||||
: base64Data;
|
||||
await this.theirSttSession.sendRealtimeInput(payload);
|
||||
} catch (err) {
|
||||
console.error('Error sending system audio:', err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
this.systemAudioProc.stderr.on('data', data => {
|
||||
console.error('SystemAudioDump stderr:', data.toString());
|
||||
});
|
||||
|
||||
this.systemAudioProc.on('close', code => {
|
||||
console.log('SystemAudioDump process closed with code:', code);
|
||||
this.systemAudioProc = null;
|
||||
});
|
||||
|
||||
this.systemAudioProc.on('error', err => {
|
||||
console.error('SystemAudioDump process error:', err);
|
||||
this.systemAudioProc = null;
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
convertStereoToMono(stereoBuffer) {
|
||||
const samples = stereoBuffer.length / 4;
|
||||
const monoBuffer = Buffer.alloc(samples * 2);
|
||||
|
||||
for (let i = 0; i < samples; i++) {
|
||||
const leftSample = stereoBuffer.readInt16LE(i * 4);
|
||||
monoBuffer.writeInt16LE(leftSample, i * 2);
|
||||
}
|
||||
|
||||
return monoBuffer;
|
||||
}
|
||||
|
||||
stopMacOSAudioCapture() {
|
||||
if (this.systemAudioProc) {
|
||||
console.log('Stopping SystemAudioDump...');
|
||||
this.systemAudioProc.kill('SIGTERM');
|
||||
this.systemAudioProc = null;
|
||||
}
|
||||
}
|
||||
|
||||
isSessionActive() {
|
||||
return !!this.mySttSession && !!this.theirSttSession;
|
||||
}
|
||||
|
||||
async closeSessions() {
|
||||
this.stopMacOSAudioCapture();
|
||||
|
||||
// Clear timers
|
||||
if (this.myInactivityTimer) {
|
||||
clearTimeout(this.myInactivityTimer);
|
||||
this.myInactivityTimer = null;
|
||||
}
|
||||
if (this.theirInactivityTimer) {
|
||||
clearTimeout(this.theirInactivityTimer);
|
||||
this.theirInactivityTimer = null;
|
||||
}
|
||||
if (this.myCompletionTimer) {
|
||||
clearTimeout(this.myCompletionTimer);
|
||||
this.myCompletionTimer = null;
|
||||
}
|
||||
if (this.theirCompletionTimer) {
|
||||
clearTimeout(this.theirCompletionTimer);
|
||||
this.theirCompletionTimer = null;
|
||||
}
|
||||
|
||||
const closePromises = [];
|
||||
if (this.mySttSession) {
|
||||
closePromises.push(this.mySttSession.close());
|
||||
this.mySttSession = null;
|
||||
}
|
||||
if (this.theirSttSession) {
|
||||
closePromises.push(this.theirSttSession.close());
|
||||
this.theirSttSession = null;
|
||||
}
|
||||
|
||||
await Promise.all(closePromises);
|
||||
console.log('All STT sessions closed.');
|
||||
|
||||
// Reset state
|
||||
this.myCurrentUtterance = '';
|
||||
this.theirCurrentUtterance = '';
|
||||
this.myLastPartialText = '';
|
||||
this.theirLastPartialText = '';
|
||||
this.myCompletionBuffer = '';
|
||||
this.theirCompletionBuffer = '';
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = SttService;
|
||||
5
src/features/listen/summary/repositories/index.js
Normal file
5
src/features/listen/summary/repositories/index.js
Normal file
@ -0,0 +1,5 @@
|
||||
const summaryRepository = require('./sqlite.repository');
|
||||
|
||||
module.exports = {
|
||||
...summaryRepository,
|
||||
};
|
||||
@ -0,0 +1,47 @@
|
||||
const sqliteClient = require('../../../../common/services/sqliteClient');
|
||||
|
||||
function saveSummary({ sessionId, tldr, text, bullet_json, action_json, model = 'gpt-4.1' }) {
|
||||
const db = sqliteClient.getDb();
|
||||
return new Promise((resolve, reject) => {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const query = `
|
||||
INSERT INTO summaries (session_id, generated_at, model, text, tldr, bullet_json, action_json, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(session_id) DO UPDATE SET
|
||||
generated_at=excluded.generated_at,
|
||||
model=excluded.model,
|
||||
text=excluded.text,
|
||||
tldr=excluded.tldr,
|
||||
bullet_json=excluded.bullet_json,
|
||||
action_json=excluded.action_json,
|
||||
updated_at=excluded.updated_at
|
||||
`;
|
||||
db.run(query, [sessionId, now, model, text, tldr, bullet_json, action_json, now], function(err) {
|
||||
if (err) {
|
||||
console.error('Error saving summary:', err);
|
||||
reject(err);
|
||||
} else {
|
||||
resolve({ changes: this.changes });
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getSummaryBySessionId(sessionId) {
|
||||
const db = sqliteClient.getDb();
|
||||
return new Promise((resolve, reject) => {
|
||||
const query = "SELECT * FROM summaries WHERE session_id = ?";
|
||||
db.get(query, [sessionId], (err, row) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(row || null);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
saveSummary,
|
||||
getSummaryBySessionId,
|
||||
};
|
||||
357
src/features/listen/summary/summaryService.js
Normal file
357
src/features/listen/summary/summaryService.js
Normal file
@ -0,0 +1,357 @@
|
||||
const { BrowserWindow } = require('electron');
|
||||
const { getSystemPrompt } = require('../../../common/prompts/promptBuilder.js');
|
||||
const { makeChatCompletionWithPortkey } = require('../../../common/services/aiProviderService.js');
|
||||
const authService = require('../../../common/services/authService');
|
||||
const sessionRepository = require('../../../common/repositories/session');
|
||||
const summaryRepository = require('./repositories');
|
||||
const { getStoredApiKey, getStoredProvider } = require('../../../electron/windowManager');
|
||||
|
||||
class SummaryService {
|
||||
constructor() {
|
||||
this.previousAnalysisResult = null;
|
||||
this.analysisHistory = [];
|
||||
this.conversationHistory = [];
|
||||
this.currentSessionId = null;
|
||||
|
||||
// Callbacks
|
||||
this.onAnalysisComplete = null;
|
||||
this.onStatusUpdate = null;
|
||||
}
|
||||
|
||||
setCallbacks({ onAnalysisComplete, onStatusUpdate }) {
|
||||
this.onAnalysisComplete = onAnalysisComplete;
|
||||
this.onStatusUpdate = onStatusUpdate;
|
||||
}
|
||||
|
||||
setSessionId(sessionId) {
|
||||
this.currentSessionId = sessionId;
|
||||
}
|
||||
|
||||
async getApiKey() {
|
||||
const storedKey = await getStoredApiKey();
|
||||
if (storedKey) {
|
||||
console.log('[SummaryService] Using stored API key');
|
||||
return storedKey;
|
||||
}
|
||||
|
||||
const envKey = process.env.OPENAI_API_KEY;
|
||||
if (envKey) {
|
||||
console.log('[SummaryService] Using environment API key');
|
||||
return envKey;
|
||||
}
|
||||
|
||||
console.error('[SummaryService] No API key found in storage or environment');
|
||||
return null;
|
||||
}
|
||||
|
||||
sendToRenderer(channel, data) {
|
||||
BrowserWindow.getAllWindows().forEach(win => {
|
||||
if (!win.isDestroyed()) {
|
||||
win.webContents.send(channel, data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
addConversationTurn(speaker, text) {
|
||||
const conversationText = `${speaker.toLowerCase()}: ${text.trim()}`;
|
||||
this.conversationHistory.push(conversationText);
|
||||
console.log(`💬 Added conversation text: ${conversationText}`);
|
||||
console.log(`📈 Total conversation history: ${this.conversationHistory.length} texts`);
|
||||
|
||||
// Trigger analysis if needed
|
||||
this.triggerAnalysisIfNeeded();
|
||||
}
|
||||
|
||||
getConversationHistory() {
|
||||
return this.conversationHistory;
|
||||
}
|
||||
|
||||
resetConversationHistory() {
|
||||
this.conversationHistory = [];
|
||||
this.previousAnalysisResult = null;
|
||||
this.analysisHistory = [];
|
||||
console.log('🔄 Conversation history and analysis state reset');
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts conversation history into text to include in the prompt.
|
||||
* @param {Array<string>} conversationTexts - Array of conversation texts ["me: ~~~", "them: ~~~", ...]
|
||||
* @param {number} maxTurns - Maximum number of recent turns to include
|
||||
* @returns {string} - Formatted conversation string for the prompt
|
||||
*/
|
||||
formatConversationForPrompt(conversationTexts, maxTurns = 30) {
|
||||
if (conversationTexts.length === 0) return '';
|
||||
return conversationTexts.slice(-maxTurns).join('\n');
|
||||
}
|
||||
|
||||
async makeOutlineAndRequests(conversationTexts, maxTurns = 30) {
|
||||
console.log(`🔍 makeOutlineAndRequests called - conversationTexts: ${conversationTexts.length}`);
|
||||
|
||||
if (conversationTexts.length === 0) {
|
||||
console.log('⚠️ No conversation texts available for analysis');
|
||||
return null;
|
||||
}
|
||||
|
||||
const recentConversation = this.formatConversationForPrompt(conversationTexts, maxTurns);
|
||||
|
||||
// 이전 분석 결과를 프롬프트에 포함
|
||||
let contextualPrompt = '';
|
||||
if (this.previousAnalysisResult) {
|
||||
contextualPrompt = `
|
||||
Previous Analysis Context:
|
||||
- Main Topic: ${this.previousAnalysisResult.topic.header}
|
||||
- Key Points: ${this.previousAnalysisResult.summary.slice(0, 3).join(', ')}
|
||||
- Last Actions: ${this.previousAnalysisResult.actions.slice(0, 2).join(', ')}
|
||||
|
||||
Please build upon this context while analyzing the new conversation segments.
|
||||
`;
|
||||
}
|
||||
|
||||
const basePrompt = getSystemPrompt('pickle_glass_analysis', '', false);
|
||||
const systemPrompt = basePrompt.replace('{{CONVERSATION_HISTORY}}', recentConversation);
|
||||
|
||||
try {
|
||||
if (this.currentSessionId) {
|
||||
await sessionRepository.touch(this.currentSessionId);
|
||||
}
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: 'system',
|
||||
content: systemPrompt,
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `${contextualPrompt}
|
||||
|
||||
Analyze the conversation and provide a structured summary. Format your response as follows:
|
||||
|
||||
**Summary Overview**
|
||||
- Main discussion point with context
|
||||
|
||||
**Key Topic: [Topic Name]**
|
||||
- First key insight
|
||||
- Second key insight
|
||||
- Third key insight
|
||||
|
||||
**Extended Explanation**
|
||||
Provide 2-3 sentences explaining the context and implications.
|
||||
|
||||
**Suggested Questions**
|
||||
1. First follow-up question?
|
||||
2. Second follow-up question?
|
||||
3. Third follow-up question?
|
||||
|
||||
Keep all points concise and build upon previous analysis if provided.`,
|
||||
},
|
||||
];
|
||||
|
||||
console.log('🤖 Sending analysis request to AI...');
|
||||
|
||||
const API_KEY = await this.getApiKey();
|
||||
if (!API_KEY) {
|
||||
throw new Error('No API key available');
|
||||
}
|
||||
|
||||
const provider = getStoredProvider ? await getStoredProvider() : 'openai';
|
||||
const loggedIn = authService.getCurrentUser().isLoggedIn;
|
||||
const usePortkey = loggedIn && provider === 'openai';
|
||||
|
||||
console.log(`[SummaryService] provider: ${provider}, usePortkey: ${usePortkey}`);
|
||||
|
||||
const completion = await makeChatCompletionWithPortkey({
|
||||
apiKey: API_KEY,
|
||||
provider: provider,
|
||||
messages: messages,
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
model: provider === 'openai' ? 'gpt-4.1' : 'gemini-2.5-flash',
|
||||
usePortkey: usePortkey,
|
||||
portkeyVirtualKey: usePortkey ? API_KEY : null
|
||||
});
|
||||
|
||||
const responseText = completion.content;
|
||||
console.log(`✅ Analysis response received: ${responseText}`);
|
||||
const structuredData = this.parseResponseText(responseText, this.previousAnalysisResult);
|
||||
|
||||
if (this.currentSessionId) {
|
||||
summaryRepository.saveSummary({
|
||||
sessionId: this.currentSessionId,
|
||||
text: responseText,
|
||||
tldr: structuredData.summary.join('\n'),
|
||||
bullet_json: JSON.stringify(structuredData.topic.bullets),
|
||||
action_json: JSON.stringify(structuredData.actions),
|
||||
model: 'gpt-4.1'
|
||||
}).catch(err => console.error('[DB] Failed to save summary:', err));
|
||||
}
|
||||
|
||||
// 분석 결과 저장
|
||||
this.previousAnalysisResult = structuredData;
|
||||
this.analysisHistory.push({
|
||||
timestamp: Date.now(),
|
||||
data: structuredData,
|
||||
conversationLength: conversationTexts.length,
|
||||
});
|
||||
|
||||
// 히스토리 크기 제한 (최근 10개만 유지)
|
||||
if (this.analysisHistory.length > 10) {
|
||||
this.analysisHistory.shift();
|
||||
}
|
||||
|
||||
return structuredData;
|
||||
} catch (error) {
|
||||
console.error('❌ Error during analysis generation:', error.message);
|
||||
return this.previousAnalysisResult; // 에러 시 이전 결과 반환
|
||||
}
|
||||
}
|
||||
|
||||
parseResponseText(responseText, previousResult) {
|
||||
const structuredData = {
|
||||
summary: [],
|
||||
topic: { header: '', bullets: [] },
|
||||
actions: [],
|
||||
followUps: ['✉️ Draft a follow-up email', '✅ Generate action items', '📝 Show summary'],
|
||||
};
|
||||
|
||||
// 이전 결과가 있으면 기본값으로 사용
|
||||
if (previousResult) {
|
||||
structuredData.topic.header = previousResult.topic.header;
|
||||
structuredData.summary = [...previousResult.summary];
|
||||
}
|
||||
|
||||
try {
|
||||
const lines = responseText.split('\n');
|
||||
let currentSection = '';
|
||||
let isCapturingTopic = false;
|
||||
let topicName = '';
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmedLine = line.trim();
|
||||
|
||||
// 섹션 헤더 감지
|
||||
if (trimmedLine.startsWith('**Summary Overview**')) {
|
||||
currentSection = 'summary-overview';
|
||||
continue;
|
||||
} else if (trimmedLine.startsWith('**Key Topic:')) {
|
||||
currentSection = 'topic';
|
||||
isCapturingTopic = true;
|
||||
topicName = trimmedLine.match(/\*\*Key Topic: (.+?)\*\*/)?.[1] || '';
|
||||
if (topicName) {
|
||||
structuredData.topic.header = topicName + ':';
|
||||
}
|
||||
continue;
|
||||
} else if (trimmedLine.startsWith('**Extended Explanation**')) {
|
||||
currentSection = 'explanation';
|
||||
continue;
|
||||
} else if (trimmedLine.startsWith('**Suggested Questions**')) {
|
||||
currentSection = 'questions';
|
||||
continue;
|
||||
}
|
||||
|
||||
// 컨텐츠 파싱
|
||||
if (trimmedLine.startsWith('-') && currentSection === 'summary-overview') {
|
||||
const summaryPoint = trimmedLine.substring(1).trim();
|
||||
if (summaryPoint && !structuredData.summary.includes(summaryPoint)) {
|
||||
// 기존 summary 업데이트 (최대 5개 유지)
|
||||
structuredData.summary.unshift(summaryPoint);
|
||||
if (structuredData.summary.length > 5) {
|
||||
structuredData.summary.pop();
|
||||
}
|
||||
}
|
||||
} else if (trimmedLine.startsWith('-') && currentSection === 'topic') {
|
||||
const bullet = trimmedLine.substring(1).trim();
|
||||
if (bullet && structuredData.topic.bullets.length < 3) {
|
||||
structuredData.topic.bullets.push(bullet);
|
||||
}
|
||||
} else if (currentSection === 'explanation' && trimmedLine) {
|
||||
// explanation을 topic bullets에 추가 (문장 단위로)
|
||||
const sentences = trimmedLine
|
||||
.split(/\.\s+/)
|
||||
.filter(s => s.trim().length > 0)
|
||||
.map(s => s.trim() + (s.endsWith('.') ? '' : '.'));
|
||||
|
||||
sentences.forEach(sentence => {
|
||||
if (structuredData.topic.bullets.length < 3 && !structuredData.topic.bullets.includes(sentence)) {
|
||||
structuredData.topic.bullets.push(sentence);
|
||||
}
|
||||
});
|
||||
} else if (trimmedLine.match(/^\d+\./) && currentSection === 'questions') {
|
||||
const question = trimmedLine.replace(/^\d+\.\s*/, '').trim();
|
||||
if (question && question.includes('?')) {
|
||||
structuredData.actions.push(`❓ ${question}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 기본 액션 추가
|
||||
const defaultActions = ['✨ What should I say next?', '💬 Suggest follow-up questions'];
|
||||
defaultActions.forEach(action => {
|
||||
if (!structuredData.actions.includes(action)) {
|
||||
structuredData.actions.push(action);
|
||||
}
|
||||
});
|
||||
|
||||
// 액션 개수 제한
|
||||
structuredData.actions = structuredData.actions.slice(0, 5);
|
||||
|
||||
// 유효성 검증 및 이전 데이터 병합
|
||||
if (structuredData.summary.length === 0 && previousResult) {
|
||||
structuredData.summary = previousResult.summary;
|
||||
}
|
||||
if (structuredData.topic.bullets.length === 0 && previousResult) {
|
||||
structuredData.topic.bullets = previousResult.topic.bullets;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('❌ Error parsing response text:', error);
|
||||
// 에러 시 이전 결과 반환
|
||||
return (
|
||||
previousResult || {
|
||||
summary: [],
|
||||
topic: { header: 'Analysis in progress', bullets: [] },
|
||||
actions: ['✨ What should I say next?', '💬 Suggest follow-up questions'],
|
||||
followUps: ['✉️ Draft a follow-up email', '✅ Generate action items', '📝 Show summary'],
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
console.log('📊 Final structured data:', JSON.stringify(structuredData, null, 2));
|
||||
return structuredData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers analysis when conversation history reaches 5 texts.
|
||||
*/
|
||||
async triggerAnalysisIfNeeded() {
|
||||
if (this.conversationHistory.length >= 5 && this.conversationHistory.length % 5 === 0) {
|
||||
console.log(`🚀 Triggering analysis (non-blocking) - ${this.conversationHistory.length} conversation texts accumulated`);
|
||||
|
||||
this.makeOutlineAndRequests(this.conversationHistory)
|
||||
.then(data => {
|
||||
if (data) {
|
||||
console.log('📤 Sending structured data to renderer');
|
||||
this.sendToRenderer('update-structured-data', data);
|
||||
|
||||
// Notify callback
|
||||
if (this.onAnalysisComplete) {
|
||||
this.onAnalysisComplete(data);
|
||||
}
|
||||
} else {
|
||||
console.log('❌ No analysis data returned from non-blocking call');
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('❌ Error in non-blocking analysis:', error);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
getCurrentAnalysisData() {
|
||||
return {
|
||||
previousResult: this.previousAnalysisResult,
|
||||
history: this.analysisHistory,
|
||||
conversationLength: this.conversationHistory.length,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = SummaryService;
|
||||
19
src/index.js
19
src/index.js
@ -13,7 +13,7 @@ if (require('electron-squirrel-startup')) {
|
||||
|
||||
const { app, BrowserWindow, shell, ipcMain, dialog } = require('electron');
|
||||
const { createWindows } = require('./electron/windowManager.js');
|
||||
const { setupLiveSummaryIpcHandlers, stopMacOSAudioCapture } = require('./features/listen/liveSummaryService.js');
|
||||
const ListenService = require('./features/listen/listenService');
|
||||
const { initializeFirebase } = require('./common/services/firebaseClient');
|
||||
const databaseInitializer = require('./common/services/databaseInitializer');
|
||||
const authService = require('./common/services/authService');
|
||||
@ -29,7 +29,9 @@ const sessionRepository = require('./common/repositories/session');
|
||||
const eventBridge = new EventEmitter();
|
||||
let WEB_PORT = 3000;
|
||||
|
||||
const openaiSessionRef = { current: null };
|
||||
const listenService = new ListenService();
|
||||
// Make listenService globally accessible so other modules (e.g., windowManager, askService) can reuse the same instance
|
||||
global.listenService = listenService;
|
||||
let deeplink = null; // Initialize as null
|
||||
let pendingDeepLinkUrl = null; // Store any deep link that arrives before initialization
|
||||
|
||||
@ -106,7 +108,7 @@ app.whenReady().then(async () => {
|
||||
sessionRepository.endAllActiveSessions();
|
||||
|
||||
authService.initialize();
|
||||
setupLiveSummaryIpcHandlers(openaiSessionRef);
|
||||
listenService.setupIpcHandlers();
|
||||
askService.initialize();
|
||||
setupGeneralIpcHandlers();
|
||||
})
|
||||
@ -123,7 +125,7 @@ app.whenReady().then(async () => {
|
||||
});
|
||||
|
||||
app.on('window-all-closed', () => {
|
||||
stopMacOSAudioCapture();
|
||||
listenService.stopMacOSAudioCapture();
|
||||
if (process.platform !== 'darwin') {
|
||||
app.quit();
|
||||
}
|
||||
@ -131,7 +133,7 @@ app.on('window-all-closed', () => {
|
||||
|
||||
app.on('before-quit', async () => {
|
||||
console.log('[Shutdown] App is about to quit.');
|
||||
stopMacOSAudioCapture();
|
||||
listenService.stopMacOSAudioCapture();
|
||||
await sessionRepository.endAllActiveSessions();
|
||||
databaseInitializer.close();
|
||||
});
|
||||
@ -210,7 +212,8 @@ function setupGeneralIpcHandlers() {
|
||||
|
||||
function setupWebDataHandlers() {
|
||||
const sessionRepository = require('./common/repositories/session');
|
||||
const listenRepository = require('./features/listen/repositories');
|
||||
const sttRepository = require('./features/listen/stt/repositories');
|
||||
const summaryRepository = require('./features/listen/summary/repositories');
|
||||
const askRepository = require('./features/ask/repositories');
|
||||
const userRepository = require('./common/repositories/user');
|
||||
const presetRepository = require('./common/repositories/preset');
|
||||
@ -230,9 +233,9 @@ function setupWebDataHandlers() {
|
||||
result = null;
|
||||
break;
|
||||
}
|
||||
const transcripts = await listenRepository.getAllTranscriptsBySessionId(payload);
|
||||
const transcripts = await sttRepository.getAllTranscriptsBySessionId(payload);
|
||||
const ai_messages = await askRepository.getAllAiMessagesBySessionId(payload);
|
||||
const summary = await listenRepository.getSummaryBySessionId(payload);
|
||||
const summary = await summaryRepository.getSummaryBySessionId(payload);
|
||||
result = { session, transcripts, ai_messages, summary };
|
||||
break;
|
||||
case 'delete-session':
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user