WIP: refactoring listen
This commit is contained in:
samtiz
parent
a18e93583f
commit
80a3c01656
@ -1,10 +1,9 @@
|
|||||||
import { html, css, LitElement } from '../assets/lit-core-2.7.4.min.js';
|
import { html, css, LitElement } from '../assets/lit-core-2.7.4.min.js';
|
||||||
import { CustomizeView } from '../features/customize/CustomizeView.js';
|
import { CustomizeView } from '../features/customize/CustomizeView.js';
|
||||||
import { AssistantView } from '../features/listen/AssistantView.js';
|
import { AssistantView } from '../features/listen/AssistantView.js';
|
||||||
import { OnboardingView } from '../features/onboarding/OnboardingView.js';
|
|
||||||
import { AskView } from '../features/ask/AskView.js';
|
import { AskView } from '../features/ask/AskView.js';
|
||||||
|
|
||||||
import '../features/listen/renderer.js';
|
import '../features/listen/renderer/renderer.js';
|
||||||
|
|
||||||
export class PickleGlassApp extends LitElement {
|
export class PickleGlassApp extends LitElement {
|
||||||
static styles = css`
|
static styles = css`
|
||||||
|
|||||||
@ -1024,10 +1024,10 @@ function createWindows() {
|
|||||||
|
|
||||||
if (windowToToggle) {
|
if (windowToToggle) {
|
||||||
if (featureName === 'listen') {
|
if (featureName === 'listen') {
|
||||||
const liveSummaryService = require('../features/listen/liveSummaryService');
|
const listenService = global.listenService;
|
||||||
if (liveSummaryService.isSessionActive()) {
|
if (listenService && listenService.isSessionActive()) {
|
||||||
console.log('[WindowManager] Listen session is active, closing it via toggle.');
|
console.log('[WindowManager] Listen session is active, closing it via toggle.');
|
||||||
await liveSummaryService.closeSession();
|
await listenService.closeSession();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,6 +1,5 @@
|
|||||||
const { ipcMain, BrowserWindow } = require('electron');
|
const { ipcMain, BrowserWindow } = require('electron');
|
||||||
const { makeStreamingChatCompletionWithPortkey } = require('../../common/services/aiProviderService');
|
const { makeStreamingChatCompletionWithPortkey } = require('../../common/services/aiProviderService');
|
||||||
const { getConversationHistory } = require('../listen/liveSummaryService');
|
|
||||||
const { getStoredApiKey, getStoredProvider, windowPool, captureScreenshot } = require('../../electron/windowManager');
|
const { getStoredApiKey, getStoredProvider, windowPool, captureScreenshot } = require('../../electron/windowManager');
|
||||||
const authService = require('../../common/services/authService');
|
const authService = require('../../common/services/authService');
|
||||||
const sessionRepository = require('../../common/repositories/session');
|
const sessionRepository = require('../../common/repositories/session');
|
||||||
@ -174,6 +173,12 @@ function formatConversationForPrompt(conversationTexts) {
|
|||||||
return conversationTexts.slice(-30).join('\n');
|
return conversationTexts.slice(-30).join('\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Access conversation history via the global listenService instance created in index.js
|
||||||
|
function getConversationHistory() {
|
||||||
|
const listenService = global.listenService;
|
||||||
|
return listenService ? listenService.getConversationHistory() : [];
|
||||||
|
}
|
||||||
|
|
||||||
async function sendMessage(userPrompt) {
|
async function sendMessage(userPrompt) {
|
||||||
if (!userPrompt || userPrompt.trim().length === 0) {
|
if (!userPrompt || userPrompt.trim().length === 0) {
|
||||||
console.warn('[AskService] Cannot process empty message');
|
console.warn('[AskService] Cannot process empty message');
|
||||||
|
|||||||
@ -1,123 +0,0 @@
|
|||||||
const fs = require('fs');
|
|
||||||
const path = require('path');
|
|
||||||
|
|
||||||
function pcmToWav(pcmBuffer, outputPath, sampleRate = 24000, channels = 1, bitDepth = 16) {
|
|
||||||
const byteRate = sampleRate * channels * (bitDepth / 8);
|
|
||||||
const blockAlign = channels * (bitDepth / 8);
|
|
||||||
const dataSize = pcmBuffer.length;
|
|
||||||
|
|
||||||
const header = Buffer.alloc(44);
|
|
||||||
|
|
||||||
header.write('RIFF', 0);
|
|
||||||
header.writeUInt32LE(dataSize + 36, 4);
|
|
||||||
header.write('WAVE', 8);
|
|
||||||
|
|
||||||
header.write('fmt ', 12);
|
|
||||||
header.writeUInt32LE(16, 16);
|
|
||||||
header.writeUInt16LE(1, 20);
|
|
||||||
header.writeUInt16LE(channels, 22);
|
|
||||||
header.writeUInt32LE(sampleRate, 24);
|
|
||||||
header.writeUInt32LE(byteRate, 28);
|
|
||||||
header.writeUInt16LE(blockAlign, 32);
|
|
||||||
header.writeUInt16LE(bitDepth, 34);
|
|
||||||
|
|
||||||
header.write('data', 36);
|
|
||||||
header.writeUInt32LE(dataSize, 40);
|
|
||||||
|
|
||||||
const wavBuffer = Buffer.concat([header, pcmBuffer]);
|
|
||||||
|
|
||||||
fs.writeFileSync(outputPath, wavBuffer);
|
|
||||||
|
|
||||||
return outputPath;
|
|
||||||
}
|
|
||||||
|
|
||||||
function analyzeAudioBuffer(buffer, label = 'Audio') {
|
|
||||||
const int16Array = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.length / 2);
|
|
||||||
|
|
||||||
let minValue = 32767;
|
|
||||||
let maxValue = -32768;
|
|
||||||
let avgValue = 0;
|
|
||||||
let rmsValue = 0;
|
|
||||||
let silentSamples = 0;
|
|
||||||
|
|
||||||
for (let i = 0; i < int16Array.length; i++) {
|
|
||||||
const sample = int16Array[i];
|
|
||||||
minValue = Math.min(minValue, sample);
|
|
||||||
maxValue = Math.max(maxValue, sample);
|
|
||||||
avgValue += sample;
|
|
||||||
rmsValue += sample * sample;
|
|
||||||
|
|
||||||
if (Math.abs(sample) < 100) {
|
|
||||||
silentSamples++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
avgValue /= int16Array.length;
|
|
||||||
rmsValue = Math.sqrt(rmsValue / int16Array.length);
|
|
||||||
|
|
||||||
const silencePercentage = (silentSamples / int16Array.length) * 100;
|
|
||||||
|
|
||||||
console.log(`${label} Analysis:`);
|
|
||||||
console.log(` Samples: ${int16Array.length}`);
|
|
||||||
console.log(` Min: ${minValue}, Max: ${maxValue}`);
|
|
||||||
console.log(` Average: ${avgValue.toFixed(2)}`);
|
|
||||||
console.log(` RMS: ${rmsValue.toFixed(2)}`);
|
|
||||||
console.log(` Silence: ${silencePercentage.toFixed(1)}%`);
|
|
||||||
console.log(` Dynamic Range: ${20 * Math.log10(maxValue / (rmsValue || 1))} dB`);
|
|
||||||
|
|
||||||
return {
|
|
||||||
minValue,
|
|
||||||
maxValue,
|
|
||||||
avgValue,
|
|
||||||
rmsValue,
|
|
||||||
silencePercentage,
|
|
||||||
sampleCount: int16Array.length,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function saveDebugAudio(buffer, type, timestamp = Date.now()) {
|
|
||||||
const homeDir = require('os').homedir();
|
|
||||||
const debugDir = path.join(homeDir, '.pickle-glass', 'debug');
|
|
||||||
|
|
||||||
if (!fs.existsSync(debugDir)) {
|
|
||||||
fs.mkdirSync(debugDir, { recursive: true });
|
|
||||||
}
|
|
||||||
|
|
||||||
const pcmPath = path.join(debugDir, `${type}_${timestamp}.pcm`);
|
|
||||||
const wavPath = path.join(debugDir, `${type}_${timestamp}.wav`);
|
|
||||||
const metaPath = path.join(debugDir, `${type}_${timestamp}.json`);
|
|
||||||
|
|
||||||
fs.writeFileSync(pcmPath, buffer);
|
|
||||||
|
|
||||||
pcmToWav(buffer, wavPath);
|
|
||||||
|
|
||||||
const analysis = analyzeAudioBuffer(buffer, type);
|
|
||||||
fs.writeFileSync(
|
|
||||||
metaPath,
|
|
||||||
JSON.stringify(
|
|
||||||
{
|
|
||||||
timestamp,
|
|
||||||
type,
|
|
||||||
bufferSize: buffer.length,
|
|
||||||
analysis,
|
|
||||||
format: {
|
|
||||||
sampleRate: 24000,
|
|
||||||
channels: 1,
|
|
||||||
bitDepth: 16,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
null,
|
|
||||||
2
|
|
||||||
)
|
|
||||||
);
|
|
||||||
|
|
||||||
console.log(`Debug audio saved: ${wavPath}`);
|
|
||||||
|
|
||||||
return { pcmPath, wavPath, metaPath };
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
pcmToWav,
|
|
||||||
analyzeAudioBuffer,
|
|
||||||
saveDebugAudio,
|
|
||||||
};
|
|
||||||
263
src/features/listen/listenService.js
Normal file
263
src/features/listen/listenService.js
Normal file
@ -0,0 +1,263 @@
|
|||||||
|
const { BrowserWindow } = require('electron');
|
||||||
|
const SttService = require('./stt/sttService');
|
||||||
|
const SummaryService = require('./summary/summaryService');
|
||||||
|
const authService = require('../../common/services/authService');
|
||||||
|
const sessionRepository = require('../../common/repositories/session');
|
||||||
|
const sttRepository = require('./stt/repositories');
|
||||||
|
|
||||||
|
class ListenService {
|
||||||
|
constructor() {
|
||||||
|
this.sttService = new SttService();
|
||||||
|
this.summaryService = new SummaryService();
|
||||||
|
this.currentSessionId = null;
|
||||||
|
this.isInitializingSession = false;
|
||||||
|
|
||||||
|
this.setupServiceCallbacks();
|
||||||
|
}
|
||||||
|
|
||||||
|
setupServiceCallbacks() {
|
||||||
|
// STT service callbacks
|
||||||
|
this.sttService.setCallbacks({
|
||||||
|
onTranscriptionComplete: (speaker, text) => {
|
||||||
|
this.handleTranscriptionComplete(speaker, text);
|
||||||
|
},
|
||||||
|
onStatusUpdate: (status) => {
|
||||||
|
this.sendToRenderer('update-status', status);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Summary service callbacks
|
||||||
|
this.summaryService.setCallbacks({
|
||||||
|
onAnalysisComplete: (data) => {
|
||||||
|
console.log('📊 Analysis completed:', data);
|
||||||
|
},
|
||||||
|
onStatusUpdate: (status) => {
|
||||||
|
this.sendToRenderer('update-status', status);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
sendToRenderer(channel, data) {
|
||||||
|
BrowserWindow.getAllWindows().forEach(win => {
|
||||||
|
if (!win.isDestroyed()) {
|
||||||
|
win.webContents.send(channel, data);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async handleTranscriptionComplete(speaker, text) {
|
||||||
|
console.log(`[ListenService] Transcription complete: ${speaker} - ${text}`);
|
||||||
|
|
||||||
|
// Save to database
|
||||||
|
await this.saveConversationTurn(speaker, text);
|
||||||
|
|
||||||
|
// Add to summary service for analysis
|
||||||
|
this.summaryService.addConversationTurn(speaker, text);
|
||||||
|
}
|
||||||
|
|
||||||
|
async saveConversationTurn(speaker, transcription) {
|
||||||
|
if (!this.currentSessionId) {
|
||||||
|
console.error('[DB] Cannot save turn, no active session ID.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (transcription.trim() === '') return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await sessionRepository.touch(this.currentSessionId);
|
||||||
|
await sttRepository.addTranscript({
|
||||||
|
sessionId: this.currentSessionId,
|
||||||
|
speaker: speaker,
|
||||||
|
text: transcription.trim(),
|
||||||
|
});
|
||||||
|
console.log(`[DB] Saved transcript for session ${this.currentSessionId}: (${speaker})`);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to save transcript to DB:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async initializeNewSession() {
|
||||||
|
try {
|
||||||
|
const uid = authService.getCurrentUserId();
|
||||||
|
if (!uid) {
|
||||||
|
throw new Error("Cannot initialize session: user not logged in.");
|
||||||
|
}
|
||||||
|
|
||||||
|
this.currentSessionId = await sessionRepository.getOrCreateActive(uid, 'listen');
|
||||||
|
console.log(`[DB] New listen session ensured: ${this.currentSessionId}`);
|
||||||
|
|
||||||
|
// Set session ID for summary service
|
||||||
|
this.summaryService.setSessionId(this.currentSessionId);
|
||||||
|
|
||||||
|
// Reset conversation history
|
||||||
|
this.summaryService.resetConversationHistory();
|
||||||
|
|
||||||
|
console.log('New conversation session started:', this.currentSessionId);
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to initialize new session in DB:', error);
|
||||||
|
this.currentSessionId = null;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async initializeSession(language = 'en') {
|
||||||
|
if (this.isInitializingSession) {
|
||||||
|
console.log('Session initialization already in progress.');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.isInitializingSession = true;
|
||||||
|
this.sendToRenderer('session-initializing', true);
|
||||||
|
this.sendToRenderer('update-status', 'Initializing sessions...');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Initialize database session
|
||||||
|
const sessionInitialized = await this.initializeNewSession();
|
||||||
|
if (!sessionInitialized) {
|
||||||
|
throw new Error('Failed to initialize database session');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize STT sessions
|
||||||
|
await this.sttService.initializeSttSessions(language);
|
||||||
|
|
||||||
|
console.log('✅ Listen service initialized successfully.');
|
||||||
|
|
||||||
|
this.sendToRenderer('session-state-changed', { isActive: true });
|
||||||
|
this.sendToRenderer('update-status', 'Connected. Ready to listen.');
|
||||||
|
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Failed to initialize listen service:', error);
|
||||||
|
this.sendToRenderer('update-status', 'Initialization failed.');
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
this.isInitializingSession = false;
|
||||||
|
this.sendToRenderer('session-initializing', false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async sendAudioContent(data, mimeType) {
|
||||||
|
return await this.sttService.sendAudioContent(data, mimeType);
|
||||||
|
}
|
||||||
|
|
||||||
|
async startMacOSAudioCapture() {
|
||||||
|
if (process.platform !== 'darwin') {
|
||||||
|
throw new Error('macOS audio capture only available on macOS');
|
||||||
|
}
|
||||||
|
return await this.sttService.startMacOSAudioCapture();
|
||||||
|
}
|
||||||
|
|
||||||
|
async stopMacOSAudioCapture() {
|
||||||
|
this.sttService.stopMacOSAudioCapture();
|
||||||
|
}
|
||||||
|
|
||||||
|
isSessionActive() {
|
||||||
|
return this.sttService.isSessionActive();
|
||||||
|
}
|
||||||
|
|
||||||
|
async closeSession() {
|
||||||
|
try {
|
||||||
|
// Close STT sessions
|
||||||
|
await this.sttService.closeSessions();
|
||||||
|
|
||||||
|
// End database session
|
||||||
|
if (this.currentSessionId) {
|
||||||
|
await sessionRepository.end(this.currentSessionId);
|
||||||
|
console.log(`[DB] Session ${this.currentSessionId} ended.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset state
|
||||||
|
this.currentSessionId = null;
|
||||||
|
this.summaryService.resetConversationHistory();
|
||||||
|
|
||||||
|
this.sendToRenderer('session-state-changed', { isActive: false });
|
||||||
|
this.sendToRenderer('session-did-close');
|
||||||
|
|
||||||
|
console.log('Listen service session closed.');
|
||||||
|
return { success: true };
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error closing listen service session:', error);
|
||||||
|
return { success: false, error: error.message };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
getCurrentSessionData() {
|
||||||
|
return {
|
||||||
|
sessionId: this.currentSessionId,
|
||||||
|
conversationHistory: this.summaryService.getConversationHistory(),
|
||||||
|
totalTexts: this.summaryService.getConversationHistory().length,
|
||||||
|
analysisData: this.summaryService.getCurrentAnalysisData(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
getConversationHistory() {
|
||||||
|
return this.summaryService.getConversationHistory();
|
||||||
|
}
|
||||||
|
|
||||||
|
setupIpcHandlers() {
|
||||||
|
const { ipcMain } = require('electron');
|
||||||
|
|
||||||
|
ipcMain.handle('is-session-active', async () => {
|
||||||
|
const isActive = this.isSessionActive();
|
||||||
|
console.log(`Checking session status. Active: ${isActive}`);
|
||||||
|
return isActive;
|
||||||
|
});
|
||||||
|
|
||||||
|
ipcMain.handle('initialize-openai', async (event, profile = 'interview', language = 'en') => {
|
||||||
|
console.log(`Received initialize-openai request with profile: ${profile}, language: ${language}`);
|
||||||
|
const success = await this.initializeSession(language);
|
||||||
|
return success;
|
||||||
|
});
|
||||||
|
|
||||||
|
ipcMain.handle('send-audio-content', async (event, { data, mimeType }) => {
|
||||||
|
try {
|
||||||
|
await this.sendAudioContent(data, mimeType);
|
||||||
|
return { success: true };
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error sending user audio:', error);
|
||||||
|
return { success: false, error: error.message };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
ipcMain.handle('start-macos-audio', async () => {
|
||||||
|
if (process.platform !== 'darwin') {
|
||||||
|
return { success: false, error: 'macOS audio capture only available on macOS' };
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const success = await this.startMacOSAudioCapture();
|
||||||
|
return { success };
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error starting macOS audio capture:', error);
|
||||||
|
return { success: false, error: error.message };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
ipcMain.handle('stop-macos-audio', async () => {
|
||||||
|
try {
|
||||||
|
this.stopMacOSAudioCapture();
|
||||||
|
return { success: true };
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error stopping macOS audio capture:', error);
|
||||||
|
return { success: false, error: error.message };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
ipcMain.handle('close-session', async () => {
|
||||||
|
return await this.closeSession();
|
||||||
|
});
|
||||||
|
|
||||||
|
ipcMain.handle('update-google-search-setting', async (event, enabled) => {
|
||||||
|
try {
|
||||||
|
console.log('Google Search setting updated to:', enabled);
|
||||||
|
return { success: true };
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error updating Google Search setting:', error);
|
||||||
|
return { success: false, error: error.message };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('✅ Listen service IPC handlers registered');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = ListenService;
|
||||||
@ -1,973 +0,0 @@
|
|||||||
require('dotenv').config();
|
|
||||||
const { BrowserWindow, ipcMain } = require('electron');
|
|
||||||
const { spawn } = require('child_process');
|
|
||||||
const { saveDebugAudio } = require('./audioUtils.js');
|
|
||||||
const { getSystemPrompt } = require('../../common/prompts/promptBuilder.js');
|
|
||||||
const { connectToGeminiSession } = require('../../common/services/googleGeminiClient.js');
|
|
||||||
const { connectToOpenAiSession, createOpenAiGenerativeClient, getOpenAiGenerativeModel } = require('../../common/services/openAiClient.js');
|
|
||||||
const { makeChatCompletionWithPortkey } = require('../../common/services/aiProviderService.js');
|
|
||||||
const authService = require('../../common/services/authService');
|
|
||||||
const sessionRepository = require('../../common/repositories/session');
|
|
||||||
const listenRepository = require('./repositories');
|
|
||||||
|
|
||||||
const { getStoredApiKey, getStoredProvider } = require('../../electron/windowManager');
|
|
||||||
|
|
||||||
const MAX_BUFFER_LENGTH_CHARS = 2000;
|
|
||||||
const COMPLETION_DEBOUNCE_MS = 2000;
|
|
||||||
|
|
||||||
async function getApiKey() {
|
|
||||||
const storedKey = await getStoredApiKey();
|
|
||||||
|
|
||||||
if (storedKey) {
|
|
||||||
console.log('[LiveSummaryService] Using stored API key');
|
|
||||||
return storedKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
const envKey = process.env.OPENAI_API_KEY;
|
|
||||||
if (envKey) {
|
|
||||||
console.log('[LiveSummaryService] Using environment API key');
|
|
||||||
return envKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.error('[LiveSummaryService] No API key found in storage or environment');
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getAiProvider() {
|
|
||||||
try {
|
|
||||||
const { ipcRenderer } = require('electron');
|
|
||||||
const provider = await ipcRenderer.invoke('get-ai-provider');
|
|
||||||
return provider || 'openai';
|
|
||||||
} catch (error) {
|
|
||||||
// If we're in the main process, get it directly
|
|
||||||
return getStoredProvider ? getStoredProvider() : 'openai';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let currentSessionId = null;
|
|
||||||
let conversationHistory = [];
|
|
||||||
let isInitializingSession = false;
|
|
||||||
|
|
||||||
let mySttSession = null;
|
|
||||||
let theirSttSession = null;
|
|
||||||
let myCurrentUtterance = '';
|
|
||||||
let theirCurrentUtterance = '';
|
|
||||||
|
|
||||||
let myLastPartialText = '';
|
|
||||||
let theirLastPartialText = '';
|
|
||||||
let myInactivityTimer = null;
|
|
||||||
let theirInactivityTimer = null;
|
|
||||||
const INACTIVITY_TIMEOUT = 3000;
|
|
||||||
|
|
||||||
const SESSION_IDLE_TIMEOUT_SECONDS = 30 * 60; // 30 minutes
|
|
||||||
|
|
||||||
let previousAnalysisResult = null;
|
|
||||||
let analysisHistory = [];
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// 🎛️ Turn-completion debouncing
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Very aggressive VAD (e.g. 50 ms) tends to split one spoken sentence into
|
|
||||||
// many "completed" events. To avoid creating a separate chat bubble for each
|
|
||||||
// of those micro-turns we debounce the *completed* events per speaker. Any
|
|
||||||
// completions that arrive within this window are concatenated and flushed as
|
|
||||||
// **one** final turn.
|
|
||||||
|
|
||||||
let myCompletionBuffer = '';
|
|
||||||
let theirCompletionBuffer = '';
|
|
||||||
let myCompletionTimer = null;
|
|
||||||
let theirCompletionTimer = null;
|
|
||||||
|
|
||||||
function flushMyCompletion() {
|
|
||||||
if (!myCompletionBuffer.trim()) return;
|
|
||||||
|
|
||||||
const finalText = myCompletionBuffer.trim();
|
|
||||||
// Save to DB & send to renderer as final
|
|
||||||
saveConversationTurn('Me', finalText);
|
|
||||||
sendToRenderer('stt-update', {
|
|
||||||
speaker: 'Me',
|
|
||||||
text: finalText,
|
|
||||||
isPartial: false,
|
|
||||||
isFinal: true,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
});
|
|
||||||
|
|
||||||
myCompletionBuffer = '';
|
|
||||||
myCompletionTimer = null;
|
|
||||||
myCurrentUtterance = ''; // Reset utterance accumulator on flush
|
|
||||||
sendToRenderer('update-status', 'Listening...');
|
|
||||||
}
|
|
||||||
|
|
||||||
function flushTheirCompletion() {
|
|
||||||
if (!theirCompletionBuffer.trim()) return;
|
|
||||||
|
|
||||||
const finalText = theirCompletionBuffer.trim();
|
|
||||||
saveConversationTurn('Them', finalText);
|
|
||||||
sendToRenderer('stt-update', {
|
|
||||||
speaker: 'Them',
|
|
||||||
text: finalText,
|
|
||||||
isPartial: false,
|
|
||||||
isFinal: true,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
});
|
|
||||||
|
|
||||||
theirCompletionBuffer = '';
|
|
||||||
theirCompletionTimer = null;
|
|
||||||
theirCurrentUtterance = ''; // Reset utterance accumulator on flush
|
|
||||||
sendToRenderer('update-status', 'Listening...');
|
|
||||||
}
|
|
||||||
|
|
||||||
function debounceMyCompletion(text) {
|
|
||||||
// 상대방이 말하고 있던 경우, 화자가 변경되었으므로 즉시 상대방의 말풍선을 완성합니다.
|
|
||||||
if (theirCompletionTimer) {
|
|
||||||
clearTimeout(theirCompletionTimer);
|
|
||||||
flushTheirCompletion();
|
|
||||||
}
|
|
||||||
|
|
||||||
myCompletionBuffer += (myCompletionBuffer ? ' ' : '') + text;
|
|
||||||
|
|
||||||
if (myCompletionTimer) clearTimeout(myCompletionTimer);
|
|
||||||
myCompletionTimer = setTimeout(flushMyCompletion, COMPLETION_DEBOUNCE_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
function debounceTheirCompletion(text) {
|
|
||||||
// 내가 말하고 있던 경우, 화자가 변경되었으므로 즉시 내 말풍선을 완성합니다.
|
|
||||||
if (myCompletionTimer) {
|
|
||||||
clearTimeout(myCompletionTimer);
|
|
||||||
flushMyCompletion();
|
|
||||||
}
|
|
||||||
|
|
||||||
theirCompletionBuffer += (theirCompletionBuffer ? ' ' : '') + text;
|
|
||||||
|
|
||||||
if (theirCompletionTimer) clearTimeout(theirCompletionTimer);
|
|
||||||
theirCompletionTimer = setTimeout(flushTheirCompletion, COMPLETION_DEBOUNCE_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
let systemAudioProc = null;
|
|
||||||
|
|
||||||
let analysisIntervalId = null;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Converts conversation history into text to include in the prompt.
|
|
||||||
* @param {Array<string>} conversationTexts - Array of conversation texts ["me: ~~~", "them: ~~~", ...]
|
|
||||||
* @param {number} maxTurns - Maximum number of recent turns to include
|
|
||||||
* @returns {string} - Formatted conversation string for the prompt
|
|
||||||
*/
|
|
||||||
function formatConversationForPrompt(conversationTexts, maxTurns = 30) {
|
|
||||||
if (conversationTexts.length === 0) return '';
|
|
||||||
return conversationTexts.slice(-maxTurns).join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
async function makeOutlineAndRequests(conversationTexts, maxTurns = 30) {
|
|
||||||
console.log(`🔍 makeOutlineAndRequests called - conversationTexts: ${conversationTexts.length}`);
|
|
||||||
|
|
||||||
if (conversationTexts.length === 0) {
|
|
||||||
console.log('⚠️ No conversation texts available for analysis');
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const recentConversation = formatConversationForPrompt(conversationTexts, maxTurns);
|
|
||||||
|
|
||||||
// 이전 분석 결과를 프롬프트에 포함
|
|
||||||
let contextualPrompt = '';
|
|
||||||
if (previousAnalysisResult) {
|
|
||||||
contextualPrompt = `
|
|
||||||
Previous Analysis Context:
|
|
||||||
- Main Topic: ${previousAnalysisResult.topic.header}
|
|
||||||
- Key Points: ${previousAnalysisResult.summary.slice(0, 3).join(', ')}
|
|
||||||
- Last Actions: ${previousAnalysisResult.actions.slice(0, 2).join(', ')}
|
|
||||||
|
|
||||||
Please build upon this context while analyzing the new conversation segments.
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const basePrompt = getSystemPrompt('pickle_glass_analysis', '', false);
|
|
||||||
const systemPrompt = basePrompt.replace('{{CONVERSATION_HISTORY}}', recentConversation);
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (currentSessionId) {
|
|
||||||
await sessionRepository.touch(currentSessionId);
|
|
||||||
}
|
|
||||||
const messages = [
|
|
||||||
{
|
|
||||||
role: 'system',
|
|
||||||
content: systemPrompt,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
role: 'user',
|
|
||||||
content: `${contextualPrompt}
|
|
||||||
|
|
||||||
Analyze the conversation and provide a structured summary. Format your response as follows:
|
|
||||||
|
|
||||||
**Summary Overview**
|
|
||||||
- Main discussion point with context
|
|
||||||
|
|
||||||
**Key Topic: [Topic Name]**
|
|
||||||
- First key insight
|
|
||||||
- Second key insight
|
|
||||||
- Third key insight
|
|
||||||
|
|
||||||
**Extended Explanation**
|
|
||||||
Provide 2-3 sentences explaining the context and implications.
|
|
||||||
|
|
||||||
**Suggested Questions**
|
|
||||||
1. First follow-up question?
|
|
||||||
2. Second follow-up question?
|
|
||||||
3. Third follow-up question?
|
|
||||||
|
|
||||||
Keep all points concise and build upon previous analysis if provided.`,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
console.log('🤖 Sending analysis request to OpenAI...');
|
|
||||||
|
|
||||||
const API_KEY = await getApiKey();
|
|
||||||
if (!API_KEY) {
|
|
||||||
throw new Error('No API key available');
|
|
||||||
}
|
|
||||||
|
|
||||||
const provider = getStoredProvider ? getStoredProvider() : 'openai';
|
|
||||||
const loggedIn = authService.getCurrentUser().isLoggedIn; // true ➜ vKey, false ➜ apiKey
|
|
||||||
const usePortkey = loggedIn && provider === 'openai'; // Only use Portkey for OpenAI with Firebase
|
|
||||||
|
|
||||||
console.log(`[LiveSummary] provider: ${provider}, usePortkey: ${usePortkey}`);
|
|
||||||
|
|
||||||
const completion = await makeChatCompletionWithPortkey({
|
|
||||||
apiKey: API_KEY,
|
|
||||||
provider: provider,
|
|
||||||
messages: messages,
|
|
||||||
temperature: 0.7,
|
|
||||||
maxTokens: 1024,
|
|
||||||
model: provider === 'openai' ? 'gpt-4.1' : 'gemini-2.5-flash',
|
|
||||||
usePortkey: usePortkey,
|
|
||||||
portkeyVirtualKey: usePortkey ? API_KEY : null
|
|
||||||
});
|
|
||||||
|
|
||||||
const responseText = completion.content;
|
|
||||||
console.log(`✅ Analysis response received: ${responseText}`);
|
|
||||||
const structuredData = parseResponseText(responseText, previousAnalysisResult);
|
|
||||||
|
|
||||||
if (currentSessionId) {
|
|
||||||
listenRepository.saveSummary({
|
|
||||||
sessionId: currentSessionId,
|
|
||||||
tldr: structuredData.summary.join('\n'),
|
|
||||||
bullet_json: JSON.stringify(structuredData.topic.bullets),
|
|
||||||
action_json: JSON.stringify(structuredData.actions),
|
|
||||||
model: 'gpt-4.1'
|
|
||||||
}).catch(err => console.error('[DB] Failed to save summary:', err));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 분석 결과 저장
|
|
||||||
previousAnalysisResult = structuredData;
|
|
||||||
analysisHistory.push({
|
|
||||||
timestamp: Date.now(),
|
|
||||||
data: structuredData,
|
|
||||||
conversationLength: conversationTexts.length,
|
|
||||||
});
|
|
||||||
|
|
||||||
// 히스토리 크기 제한 (최근 10개만 유지)
|
|
||||||
if (analysisHistory.length > 10) {
|
|
||||||
analysisHistory.shift();
|
|
||||||
}
|
|
||||||
|
|
||||||
return structuredData;
|
|
||||||
} catch (error) {
|
|
||||||
console.error('❌ Error during analysis generation:', error.message);
|
|
||||||
return previousAnalysisResult; // 에러 시 이전 결과 반환
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseResponseText(responseText, previousResult) {
|
|
||||||
const structuredData = {
|
|
||||||
summary: [],
|
|
||||||
topic: { header: '', bullets: [] },
|
|
||||||
actions: [],
|
|
||||||
followUps: ['✉️ Draft a follow-up email', '✅ Generate action items', '📝 Show summary'],
|
|
||||||
};
|
|
||||||
|
|
||||||
// 이전 결과가 있으면 기본값으로 사용
|
|
||||||
if (previousResult) {
|
|
||||||
structuredData.topic.header = previousResult.topic.header;
|
|
||||||
structuredData.summary = [...previousResult.summary];
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const lines = responseText.split('\n');
|
|
||||||
let currentSection = '';
|
|
||||||
let isCapturingTopic = false;
|
|
||||||
let topicName = '';
|
|
||||||
|
|
||||||
for (const line of lines) {
|
|
||||||
const trimmedLine = line.trim();
|
|
||||||
|
|
||||||
// 섹션 헤더 감지
|
|
||||||
if (trimmedLine.startsWith('**Summary Overview**')) {
|
|
||||||
currentSection = 'summary-overview';
|
|
||||||
continue;
|
|
||||||
} else if (trimmedLine.startsWith('**Key Topic:')) {
|
|
||||||
currentSection = 'topic';
|
|
||||||
isCapturingTopic = true;
|
|
||||||
topicName = trimmedLine.match(/\*\*Key Topic: (.+?)\*\*/)?.[1] || '';
|
|
||||||
if (topicName) {
|
|
||||||
structuredData.topic.header = topicName + ':';
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
} else if (trimmedLine.startsWith('**Extended Explanation**')) {
|
|
||||||
currentSection = 'explanation';
|
|
||||||
continue;
|
|
||||||
} else if (trimmedLine.startsWith('**Suggested Questions**')) {
|
|
||||||
currentSection = 'questions';
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 컨텐츠 파싱
|
|
||||||
if (trimmedLine.startsWith('-') && currentSection === 'summary-overview') {
|
|
||||||
const summaryPoint = trimmedLine.substring(1).trim();
|
|
||||||
if (summaryPoint && !structuredData.summary.includes(summaryPoint)) {
|
|
||||||
// 기존 summary 업데이트 (최대 5개 유지)
|
|
||||||
structuredData.summary.unshift(summaryPoint);
|
|
||||||
if (structuredData.summary.length > 5) {
|
|
||||||
structuredData.summary.pop();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (trimmedLine.startsWith('-') && currentSection === 'topic') {
|
|
||||||
const bullet = trimmedLine.substring(1).trim();
|
|
||||||
if (bullet && structuredData.topic.bullets.length < 3) {
|
|
||||||
structuredData.topic.bullets.push(bullet);
|
|
||||||
}
|
|
||||||
} else if (currentSection === 'explanation' && trimmedLine) {
|
|
||||||
// explanation을 topic bullets에 추가 (문장 단위로)
|
|
||||||
const sentences = trimmedLine
|
|
||||||
.split(/\.\s+/)
|
|
||||||
.filter(s => s.trim().length > 0)
|
|
||||||
.map(s => s.trim() + (s.endsWith('.') ? '' : '.'));
|
|
||||||
|
|
||||||
sentences.forEach(sentence => {
|
|
||||||
if (structuredData.topic.bullets.length < 3 && !structuredData.topic.bullets.includes(sentence)) {
|
|
||||||
structuredData.topic.bullets.push(sentence);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} else if (trimmedLine.match(/^\d+\./) && currentSection === 'questions') {
|
|
||||||
const question = trimmedLine.replace(/^\d+\.\s*/, '').trim();
|
|
||||||
if (question && question.includes('?')) {
|
|
||||||
structuredData.actions.push(`❓ ${question}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 기본 액션 추가
|
|
||||||
const defaultActions = ['✨ What should I say next?', '💬 Suggest follow-up questions'];
|
|
||||||
defaultActions.forEach(action => {
|
|
||||||
if (!structuredData.actions.includes(action)) {
|
|
||||||
structuredData.actions.push(action);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// 액션 개수 제한
|
|
||||||
structuredData.actions = structuredData.actions.slice(0, 5);
|
|
||||||
|
|
||||||
// 유효성 검증 및 이전 데이터 병합
|
|
||||||
if (structuredData.summary.length === 0 && previousResult) {
|
|
||||||
structuredData.summary = previousResult.summary;
|
|
||||||
}
|
|
||||||
if (structuredData.topic.bullets.length === 0 && previousResult) {
|
|
||||||
structuredData.topic.bullets = previousResult.topic.bullets;
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error('❌ Error parsing response text:', error);
|
|
||||||
// 에러 시 이전 결과 반환
|
|
||||||
return (
|
|
||||||
previousResult || {
|
|
||||||
summary: [],
|
|
||||||
topic: { header: 'Analysis in progress', bullets: [] },
|
|
||||||
actions: ['✨ What should I say next?', '💬 Suggest follow-up questions'],
|
|
||||||
followUps: ['✉️ Draft a follow-up email', '✅ Generate action items', '📝 Show summary'],
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('📊 Final structured data:', JSON.stringify(structuredData, null, 2));
|
|
||||||
return structuredData;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Triggers analysis when conversation history reaches 5 texts.
|
|
||||||
*/
|
|
||||||
async function triggerAnalysisIfNeeded() {
|
|
||||||
if (conversationHistory.length >= 5 && conversationHistory.length % 5 === 0) {
|
|
||||||
console.log(`🚀 Triggering analysis (non-blocking) - ${conversationHistory.length} conversation texts accumulated`);
|
|
||||||
|
|
||||||
makeOutlineAndRequests(conversationHistory)
|
|
||||||
.then(data => {
|
|
||||||
if (data) {
|
|
||||||
console.log('📤 Sending structured data to renderer');
|
|
||||||
sendToRenderer('update-structured-data', data);
|
|
||||||
} else {
|
|
||||||
console.log('❌ No analysis data returned from non-blocking call');
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.catch(error => {
|
|
||||||
console.error('❌ Error in non-blocking analysis:', error);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Schedules periodic updates of outline and analysis every 10 seconds. - DEPRECATED
|
|
||||||
* Now analysis is triggered every 5 conversation texts.
|
|
||||||
*/
|
|
||||||
function startAnalysisInterval() {
|
|
||||||
console.log('⏰ Analysis will be triggered every 5 conversation texts (not on timer)');
|
|
||||||
|
|
||||||
if (analysisIntervalId) {
|
|
||||||
clearInterval(analysisIntervalId);
|
|
||||||
analysisIntervalId = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function stopAnalysisInterval() {
|
|
||||||
if (analysisIntervalId) {
|
|
||||||
clearInterval(analysisIntervalId);
|
|
||||||
analysisIntervalId = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (myInactivityTimer) {
|
|
||||||
clearTimeout(myInactivityTimer);
|
|
||||||
myInactivityTimer = null;
|
|
||||||
}
|
|
||||||
if (theirInactivityTimer) {
|
|
||||||
clearTimeout(theirInactivityTimer);
|
|
||||||
theirInactivityTimer = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function sendToRenderer(channel, data) {
|
|
||||||
BrowserWindow.getAllWindows().forEach(win => {
|
|
||||||
if (!win.isDestroyed()) {
|
|
||||||
win.webContents.send(channel, data);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function getCurrentSessionData() {
|
|
||||||
return {
|
|
||||||
sessionId: currentSessionId,
|
|
||||||
conversationHistory: conversationHistory,
|
|
||||||
totalTexts: conversationHistory.length,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Conversation management functions
|
|
||||||
async function initializeNewSession() {
|
|
||||||
try {
|
|
||||||
const uid = authService.getCurrentUserId();
|
|
||||||
if (!uid) {
|
|
||||||
throw new Error("Cannot initialize session: user not logged in.");
|
|
||||||
}
|
|
||||||
currentSessionId = await sessionRepository.getOrCreateActive(uid, 'listen');
|
|
||||||
console.log(`[DB] New listen session ensured: ${currentSessionId}`);
|
|
||||||
|
|
||||||
conversationHistory = [];
|
|
||||||
myCurrentUtterance = '';
|
|
||||||
theirCurrentUtterance = '';
|
|
||||||
|
|
||||||
// 🔄 Reset analysis state so the new session starts fresh
|
|
||||||
previousAnalysisResult = null;
|
|
||||||
analysisHistory = [];
|
|
||||||
|
|
||||||
// sendToRenderer('update-outline', []);
|
|
||||||
// sendToRenderer('update-analysis-requests', []);
|
|
||||||
|
|
||||||
myLastPartialText = '';
|
|
||||||
theirLastPartialText = '';
|
|
||||||
if (myInactivityTimer) {
|
|
||||||
clearTimeout(myInactivityTimer);
|
|
||||||
myInactivityTimer = null;
|
|
||||||
}
|
|
||||||
if (theirInactivityTimer) {
|
|
||||||
clearTimeout(theirInactivityTimer);
|
|
||||||
theirInactivityTimer = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('New conversation session started:', currentSessionId);
|
|
||||||
return true;
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Failed to initialize new session in DB:', error);
|
|
||||||
currentSessionId = null;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function saveConversationTurn(speaker, transcription) {
|
|
||||||
if (!currentSessionId) {
|
|
||||||
console.error('[DB] Cannot save turn, no active session ID.');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (transcription.trim() === '') return;
|
|
||||||
|
|
||||||
try {
|
|
||||||
await sessionRepository.touch(currentSessionId);
|
|
||||||
await listenRepository.addTranscript({
|
|
||||||
sessionId: currentSessionId,
|
|
||||||
speaker: speaker,
|
|
||||||
text: transcription.trim(),
|
|
||||||
});
|
|
||||||
console.log(`[DB] Saved transcript for session ${currentSessionId}: (${speaker})`);
|
|
||||||
|
|
||||||
const conversationText = `${speaker.toLowerCase()}: ${transcription.trim()}`;
|
|
||||||
conversationHistory.push(conversationText);
|
|
||||||
console.log(`💬 Saved conversation text: ${conversationText}`);
|
|
||||||
console.log(`📈 Total conversation history: ${conversationHistory.length} texts`);
|
|
||||||
|
|
||||||
triggerAnalysisIfNeeded();
|
|
||||||
|
|
||||||
const conversationTurn = {
|
|
||||||
speaker: speaker,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
transcription: transcription.trim(),
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Failed to save transcript to DB:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function initializeLiveSummarySession(language = 'en') {
|
|
||||||
// Use system environment variable if set, otherwise use the provided language
|
|
||||||
const effectiveLanguage = process.env.OPENAI_TRANSCRIBE_LANG || language || 'en';
|
|
||||||
if (isInitializingSession) {
|
|
||||||
console.log('Session initialization already in progress.');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const userState = authService.getCurrentUser();
|
|
||||||
const loggedIn = userState.isLoggedIn;
|
|
||||||
const keyType = loggedIn ? 'vKey' : 'apiKey';
|
|
||||||
|
|
||||||
isInitializingSession = true;
|
|
||||||
sendToRenderer('session-initializing', true);
|
|
||||||
sendToRenderer('update-status', 'Initializing sessions...');
|
|
||||||
|
|
||||||
const API_KEY = await getApiKey();
|
|
||||||
if (!API_KEY) {
|
|
||||||
console.error('FATAL ERROR: API Key is not defined.');
|
|
||||||
sendToRenderer('update-status', 'API Key not configured.');
|
|
||||||
isInitializingSession = false;
|
|
||||||
sendToRenderer('session-initializing', false);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
await initializeNewSession();
|
|
||||||
|
|
||||||
const provider = await getAiProvider();
|
|
||||||
const isGemini = provider === 'gemini';
|
|
||||||
console.log(`[LiveSummaryService] Initializing STT for provider: ${provider}`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const handleMyMessage = message => {
|
|
||||||
if (isGemini) {
|
|
||||||
// console.log('[Gemini Raw Message - Me]:', JSON.stringify(message, null, 2));
|
|
||||||
const text = message.serverContent?.inputTranscription?.text || '';
|
|
||||||
if (text && text.trim()) {
|
|
||||||
const finalUtteranceText = text.trim().replace(/<noise>/g, '').trim();
|
|
||||||
if (finalUtteranceText && finalUtteranceText !== '.') {
|
|
||||||
debounceMyCompletion(finalUtteranceText);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
const type = message.type;
|
|
||||||
const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || '';
|
|
||||||
|
|
||||||
if (type === 'conversation.item.input_audio_transcription.delta') {
|
|
||||||
if (myCompletionTimer) clearTimeout(myCompletionTimer);
|
|
||||||
myCompletionTimer = null;
|
|
||||||
myCurrentUtterance += text;
|
|
||||||
const continuousText = myCompletionBuffer + (myCompletionBuffer ? ' ' : '') + myCurrentUtterance;
|
|
||||||
if (text && !text.includes('vq_lbr_audio_')) {
|
|
||||||
sendToRenderer('stt-update', {
|
|
||||||
speaker: 'Me',
|
|
||||||
text: continuousText,
|
|
||||||
isPartial: true,
|
|
||||||
isFinal: false,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else if (type === 'conversation.item.input_audio_transcription.completed') {
|
|
||||||
if (text && text.trim()) {
|
|
||||||
const finalUtteranceText = text.trim();
|
|
||||||
myCurrentUtterance = '';
|
|
||||||
debounceMyCompletion(finalUtteranceText);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (message.error) {
|
|
||||||
console.error('[Me] STT Session Error:', message.error);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleTheirMessage = message => {
|
|
||||||
if (isGemini) {
|
|
||||||
// console.log('[Gemini Raw Message - Them]:', JSON.stringify(message, null, 2));
|
|
||||||
const text = message.serverContent?.inputTranscription?.text || '';
|
|
||||||
if (text && text.trim()) {
|
|
||||||
const finalUtteranceText = text.trim().replace(/<noise>/g, '').trim();
|
|
||||||
if (finalUtteranceText && finalUtteranceText !== '.') {
|
|
||||||
debounceTheirCompletion(finalUtteranceText);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
const type = message.type;
|
|
||||||
const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || '';
|
|
||||||
if (type === 'conversation.item.input_audio_transcription.delta') {
|
|
||||||
if (theirCompletionTimer) clearTimeout(theirCompletionTimer);
|
|
||||||
theirCompletionTimer = null;
|
|
||||||
theirCurrentUtterance += text;
|
|
||||||
const continuousText = theirCompletionBuffer + (theirCompletionBuffer ? ' ' : '') + theirCurrentUtterance;
|
|
||||||
if (text && !text.includes('vq_lbr_audio_')) {
|
|
||||||
sendToRenderer('stt-update', {
|
|
||||||
speaker: 'Them',
|
|
||||||
text: continuousText,
|
|
||||||
isPartial: true,
|
|
||||||
isFinal: false,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else if (type === 'conversation.item.input_audio_transcription.completed') {
|
|
||||||
if (text && text.trim()) {
|
|
||||||
const finalUtteranceText = text.trim();
|
|
||||||
theirCurrentUtterance = '';
|
|
||||||
debounceTheirCompletion(finalUtteranceText);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (message.error) {
|
|
||||||
console.error('[Them] STT Session Error:', message.error);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const mySttConfig = {
|
|
||||||
language: effectiveLanguage,
|
|
||||||
callbacks: {
|
|
||||||
onmessage: handleMyMessage,
|
|
||||||
onerror: error => console.error('My STT session error:', error.message),
|
|
||||||
onclose: event => console.log('My STT session closed:', event.reason),
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const theirSttConfig = {
|
|
||||||
language: effectiveLanguage,
|
|
||||||
callbacks: {
|
|
||||||
onmessage: handleTheirMessage,
|
|
||||||
onerror: error => console.error('Their STT session error:', error.message),
|
|
||||||
onclose: event => console.log('Their STT session closed:', event.reason),
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
if (isGemini) {
|
|
||||||
[mySttSession, theirSttSession] = await Promise.all([
|
|
||||||
connectToGeminiSession(API_KEY, mySttConfig),
|
|
||||||
connectToGeminiSession(API_KEY, theirSttConfig),
|
|
||||||
]);
|
|
||||||
} else {
|
|
||||||
[mySttSession, theirSttSession] = await Promise.all([
|
|
||||||
connectToOpenAiSession(API_KEY, mySttConfig, keyType),
|
|
||||||
connectToOpenAiSession(API_KEY, theirSttConfig, keyType),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('✅ Both STT sessions initialized successfully.');
|
|
||||||
triggerAnalysisIfNeeded();
|
|
||||||
|
|
||||||
sendToRenderer('session-state-changed', { isActive: true });
|
|
||||||
|
|
||||||
isInitializingSession = false;
|
|
||||||
sendToRenderer('session-initializing', false);
|
|
||||||
sendToRenderer('update-status', 'Connected. Ready to listen.');
|
|
||||||
return true;
|
|
||||||
} catch (error) {
|
|
||||||
console.error('❌ Failed to initialize STT sessions:', error);
|
|
||||||
isInitializingSession = false;
|
|
||||||
sendToRenderer('session-initializing', false);
|
|
||||||
sendToRenderer('update-status', 'Initialization failed.');
|
|
||||||
mySttSession = null;
|
|
||||||
theirSttSession = null;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function killExistingSystemAudioDump() {
|
|
||||||
return new Promise(resolve => {
|
|
||||||
console.log('Checking for existing SystemAudioDump processes...');
|
|
||||||
|
|
||||||
const killProc = spawn('pkill', ['-f', 'SystemAudioDump'], {
|
|
||||||
stdio: 'ignore',
|
|
||||||
});
|
|
||||||
|
|
||||||
killProc.on('close', code => {
|
|
||||||
if (code === 0) {
|
|
||||||
console.log('Killed existing SystemAudioDump processes');
|
|
||||||
} else {
|
|
||||||
console.log('No existing SystemAudioDump processes found');
|
|
||||||
}
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
|
|
||||||
killProc.on('error', err => {
|
|
||||||
console.log('Error checking for existing processes (this is normal):', err.message);
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
|
|
||||||
setTimeout(() => {
|
|
||||||
killProc.kill();
|
|
||||||
resolve();
|
|
||||||
}, 2000);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function startMacOSAudioCapture() {
|
|
||||||
if (process.platform !== 'darwin' || !theirSttSession) return false;
|
|
||||||
|
|
||||||
await killExistingSystemAudioDump();
|
|
||||||
console.log('Starting macOS audio capture for "Them"...');
|
|
||||||
|
|
||||||
const { app } = require('electron');
|
|
||||||
const path = require('path');
|
|
||||||
const systemAudioPath = app.isPackaged
|
|
||||||
? path.join(process.resourcesPath, 'app.asar.unpacked', 'src', 'assets', 'SystemAudioDump')
|
|
||||||
: path.join(app.getAppPath(), 'src', 'assets', 'SystemAudioDump');
|
|
||||||
|
|
||||||
console.log('SystemAudioDump path:', systemAudioPath);
|
|
||||||
|
|
||||||
systemAudioProc = spawn(systemAudioPath, [], {
|
|
||||||
stdio: ['ignore', 'pipe', 'pipe'],
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!systemAudioProc.pid) {
|
|
||||||
console.error('Failed to start SystemAudioDump');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('SystemAudioDump started with PID:', systemAudioProc.pid);
|
|
||||||
|
|
||||||
const CHUNK_DURATION = 0.1;
|
|
||||||
const SAMPLE_RATE = 24000;
|
|
||||||
const BYTES_PER_SAMPLE = 2;
|
|
||||||
const CHANNELS = 2;
|
|
||||||
const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * CHUNK_DURATION;
|
|
||||||
|
|
||||||
let audioBuffer = Buffer.alloc(0);
|
|
||||||
|
|
||||||
const provider = await getAiProvider();
|
|
||||||
const isGemini = provider === 'gemini';
|
|
||||||
|
|
||||||
systemAudioProc.stdout.on('data', async data => {
|
|
||||||
audioBuffer = Buffer.concat([audioBuffer, data]);
|
|
||||||
|
|
||||||
while (audioBuffer.length >= CHUNK_SIZE) {
|
|
||||||
const chunk = audioBuffer.slice(0, CHUNK_SIZE);
|
|
||||||
audioBuffer = audioBuffer.slice(CHUNK_SIZE);
|
|
||||||
|
|
||||||
const monoChunk = CHANNELS === 2 ? convertStereoToMono(chunk) : chunk;
|
|
||||||
const base64Data = monoChunk.toString('base64');
|
|
||||||
|
|
||||||
sendToRenderer('system-audio-data', { data: base64Data });
|
|
||||||
|
|
||||||
if (theirSttSession) {
|
|
||||||
try {
|
|
||||||
// await theirSttSession.sendRealtimeInput(base64Data);
|
|
||||||
const payload = isGemini
|
|
||||||
? { audio: { data: base64Data, mimeType: 'audio/pcm;rate=24000' } }
|
|
||||||
: base64Data;
|
|
||||||
await theirSttSession.sendRealtimeInput(payload);
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Error sending system audio:', err.message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (process.env.DEBUG_AUDIO) {
|
|
||||||
saveDebugAudio(monoChunk, 'system_audio');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
systemAudioProc.stderr.on('data', data => {
|
|
||||||
console.error('SystemAudioDump stderr:', data.toString());
|
|
||||||
});
|
|
||||||
|
|
||||||
systemAudioProc.on('close', code => {
|
|
||||||
console.log('SystemAudioDump process closed with code:', code);
|
|
||||||
systemAudioProc = null;
|
|
||||||
});
|
|
||||||
|
|
||||||
systemAudioProc.on('error', err => {
|
|
||||||
console.error('SystemAudioDump process error:', err);
|
|
||||||
systemAudioProc = null;
|
|
||||||
});
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
function convertStereoToMono(stereoBuffer) {
|
|
||||||
const samples = stereoBuffer.length / 4;
|
|
||||||
const monoBuffer = Buffer.alloc(samples * 2);
|
|
||||||
|
|
||||||
for (let i = 0; i < samples; i++) {
|
|
||||||
const leftSample = stereoBuffer.readInt16LE(i * 4);
|
|
||||||
monoBuffer.writeInt16LE(leftSample, i * 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
return monoBuffer;
|
|
||||||
}
|
|
||||||
|
|
||||||
function stopMacOSAudioCapture() {
|
|
||||||
if (systemAudioProc) {
|
|
||||||
console.log('Stopping SystemAudioDump...');
|
|
||||||
systemAudioProc.kill('SIGTERM');
|
|
||||||
systemAudioProc = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function sendAudioToOpenAI(base64Data, sttSessionRef) {
|
|
||||||
if (!sttSessionRef.current) return;
|
|
||||||
|
|
||||||
try {
|
|
||||||
process.stdout.write('.');
|
|
||||||
await sttSessionRef.current.sendRealtimeInput({
|
|
||||||
audio: {
|
|
||||||
data: base64Data,
|
|
||||||
mimeType: 'audio/pcm;rate=24000',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error sending audio to OpenAI:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function isSessionActive() {
|
|
||||||
return !!mySttSession && !!theirSttSession;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function closeSession() {
|
|
||||||
try {
|
|
||||||
stopMacOSAudioCapture();
|
|
||||||
stopAnalysisInterval();
|
|
||||||
|
|
||||||
if (currentSessionId) {
|
|
||||||
await sessionRepository.end(currentSessionId);
|
|
||||||
console.log(`[DB] Session ${currentSessionId} ended.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const closePromises = [];
|
|
||||||
if (mySttSession) {
|
|
||||||
closePromises.push(mySttSession.close());
|
|
||||||
mySttSession = null;
|
|
||||||
}
|
|
||||||
if (theirSttSession) {
|
|
||||||
closePromises.push(theirSttSession.close());
|
|
||||||
theirSttSession = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
await Promise.all(closePromises);
|
|
||||||
console.log('All sessions closed.');
|
|
||||||
|
|
||||||
currentSessionId = null;
|
|
||||||
conversationHistory = [];
|
|
||||||
|
|
||||||
sendToRenderer('session-state-changed', { isActive: false });
|
|
||||||
sendToRenderer('session-did-close');
|
|
||||||
|
|
||||||
return { success: true };
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error closing sessions:', error);
|
|
||||||
return { success: false, error: error.message };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function setupLiveSummaryIpcHandlers() {
|
|
||||||
ipcMain.handle('is-session-active', async () => {
|
|
||||||
const isActive = isSessionActive();
|
|
||||||
console.log(`Checking session status. Active: ${isActive}`);
|
|
||||||
return isActive;
|
|
||||||
});
|
|
||||||
|
|
||||||
ipcMain.handle('initialize-openai', async (event, profile = 'interview', language = 'en') => {
|
|
||||||
console.log(`Received initialize-openai request with profile: ${profile}, language: ${language}`);
|
|
||||||
const success = await initializeLiveSummarySession(language);
|
|
||||||
return success;
|
|
||||||
});
|
|
||||||
|
|
||||||
ipcMain.handle('send-audio-content', async (event, { data, mimeType }) => {
|
|
||||||
const provider = await getAiProvider();
|
|
||||||
const isGemini = provider === 'gemini';
|
|
||||||
if (!mySttSession) return { success: false, error: 'User STT session not active' };
|
|
||||||
try {
|
|
||||||
// await mySttSession.sendRealtimeInput(data);
|
|
||||||
// provider에 맞는 형식으로 래핑
|
|
||||||
const payload = isGemini
|
|
||||||
? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } }
|
|
||||||
: data; // OpenAI는 base64 string 그대로
|
|
||||||
|
|
||||||
await mySttSession.sendRealtimeInput(payload);
|
|
||||||
return { success: true };
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error sending user audio:', error);
|
|
||||||
return { success: false, error: error.message };
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
ipcMain.handle('start-macos-audio', async () => {
|
|
||||||
if (process.platform !== 'darwin') {
|
|
||||||
return { success: false, error: 'macOS audio capture only available on macOS' };
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const success = await startMacOSAudioCapture();
|
|
||||||
return { success };
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error starting macOS audio capture:', error);
|
|
||||||
return { success: false, error: error.message };
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
ipcMain.handle('stop-macos-audio', async () => {
|
|
||||||
try {
|
|
||||||
stopMacOSAudioCapture();
|
|
||||||
return { success: true };
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error stopping macOS audio capture:', error);
|
|
||||||
return { success: false, error: error.message };
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
ipcMain.handle('close-session', async () => {
|
|
||||||
return await closeSession();
|
|
||||||
});
|
|
||||||
|
|
||||||
ipcMain.handle('update-google-search-setting', async (event, enabled) => {
|
|
||||||
try {
|
|
||||||
console.log('Google Search setting updated to:', enabled);
|
|
||||||
return { success: true };
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error updating Google Search setting:', error);
|
|
||||||
return { success: false, error: error.message };
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function getConversationHistory() {
|
|
||||||
return conversationHistory;
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
sendToRenderer,
|
|
||||||
initializeNewSession,
|
|
||||||
saveConversationTurn,
|
|
||||||
killExistingSystemAudioDump,
|
|
||||||
startMacOSAudioCapture,
|
|
||||||
convertStereoToMono,
|
|
||||||
stopMacOSAudioCapture,
|
|
||||||
sendAudioToOpenAI,
|
|
||||||
setupLiveSummaryIpcHandlers,
|
|
||||||
isSessionActive,
|
|
||||||
closeSession,
|
|
||||||
getConversationHistory,
|
|
||||||
};
|
|
||||||
@ -1,20 +1,29 @@
|
|||||||
// renderer.js
|
|
||||||
const { ipcRenderer } = require('electron');
|
const { ipcRenderer } = require('electron');
|
||||||
const { makeStreamingChatCompletionWithPortkey } = require('../../common/services/aiProviderService.js');
|
|
||||||
|
|
||||||
let mediaStream = null;
|
// ---------------------------
|
||||||
let screenshotInterval = null;
|
// Constants & Globals
|
||||||
let audioContext = null;
|
// ---------------------------
|
||||||
let audioProcessor = null;
|
|
||||||
let micMediaStream = null;
|
|
||||||
let audioBuffer = [];
|
|
||||||
const SAMPLE_RATE = 24000;
|
const SAMPLE_RATE = 24000;
|
||||||
const AUDIO_CHUNK_DURATION = 0.1;
|
const AUDIO_CHUNK_DURATION = 0.1;
|
||||||
const BUFFER_SIZE = 4096;
|
const BUFFER_SIZE = 4096;
|
||||||
|
|
||||||
|
const isLinux = process.platform === 'linux';
|
||||||
|
const isMacOS = process.platform === 'darwin';
|
||||||
|
|
||||||
|
let mediaStream = null;
|
||||||
|
let micMediaStream = null;
|
||||||
|
let screenshotInterval = null;
|
||||||
|
let audioContext = null;
|
||||||
|
let audioProcessor = null;
|
||||||
|
let currentImageQuality = 'medium';
|
||||||
|
let lastScreenshotBase64 = null;
|
||||||
|
|
||||||
let systemAudioBuffer = [];
|
let systemAudioBuffer = [];
|
||||||
const MAX_SYSTEM_BUFFER_SIZE = 10;
|
const MAX_SYSTEM_BUFFER_SIZE = 10;
|
||||||
|
|
||||||
|
// ---------------------------
|
||||||
|
// Utility helpers (exact from renderer.js)
|
||||||
|
// ---------------------------
|
||||||
function isVoiceActive(audioFloat32Array, threshold = 0.005) {
|
function isVoiceActive(audioFloat32Array, threshold = 0.005) {
|
||||||
if (!audioFloat32Array || audioFloat32Array.length === 0) {
|
if (!audioFloat32Array || audioFloat32Array.length === 0) {
|
||||||
return false;
|
return false;
|
||||||
@ -31,11 +40,6 @@ function isVoiceActive(audioFloat32Array, threshold = 0.005) {
|
|||||||
return rms > threshold;
|
return rms > threshold;
|
||||||
}
|
}
|
||||||
|
|
||||||
let currentImageQuality = 'medium'; // Store current image quality for manual screenshots
|
|
||||||
let lastScreenshotBase64 = null; // Store the latest screenshot
|
|
||||||
|
|
||||||
let realtimeConversationHistory = [];
|
|
||||||
|
|
||||||
function base64ToFloat32Array(base64) {
|
function base64ToFloat32Array(base64) {
|
||||||
const binaryString = atob(base64);
|
const binaryString = atob(base64);
|
||||||
const bytes = new Uint8Array(binaryString.length);
|
const bytes = new Uint8Array(binaryString.length);
|
||||||
@ -54,11 +58,29 @@ function base64ToFloat32Array(base64) {
|
|||||||
return float32Array;
|
return float32Array;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function queryLoginState() {
|
function convertFloat32ToInt16(float32Array) {
|
||||||
const userState = await ipcRenderer.invoke('get-current-user');
|
const int16Array = new Int16Array(float32Array.length);
|
||||||
return userState;
|
for (let i = 0; i < float32Array.length; i++) {
|
||||||
|
// Improved scaling to prevent clipping
|
||||||
|
const s = Math.max(-1, Math.min(1, float32Array[i]));
|
||||||
|
int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||||
|
}
|
||||||
|
return int16Array;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function arrayBufferToBase64(buffer) {
|
||||||
|
let binary = '';
|
||||||
|
const bytes = new Uint8Array(buffer);
|
||||||
|
const len = bytes.byteLength;
|
||||||
|
for (let i = 0; i < len; i++) {
|
||||||
|
binary += String.fromCharCode(bytes[i]);
|
||||||
|
}
|
||||||
|
return btoa(binary);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------
|
||||||
|
// Complete SimpleAEC implementation (exact from renderer.js)
|
||||||
|
// ---------------------------
|
||||||
class SimpleAEC {
|
class SimpleAEC {
|
||||||
constructor() {
|
constructor() {
|
||||||
this.adaptiveFilter = new Float32Array(1024);
|
this.adaptiveFilter = new Float32Array(1024);
|
||||||
@ -179,11 +201,24 @@ class SimpleAEC {
|
|||||||
|
|
||||||
let aecProcessor = new SimpleAEC();
|
let aecProcessor = new SimpleAEC();
|
||||||
|
|
||||||
const isLinux = process.platform === 'linux';
|
// System audio data handler
|
||||||
const isMacOS = process.platform === 'darwin';
|
ipcRenderer.on('system-audio-data', (event, { data }) => {
|
||||||
|
systemAudioBuffer.push({
|
||||||
|
data: data,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
});
|
||||||
|
|
||||||
window.pickleGlass = window.pickleGlass || {};
|
// 오래된 데이터 제거
|
||||||
|
if (systemAudioBuffer.length > MAX_SYSTEM_BUFFER_SIZE) {
|
||||||
|
systemAudioBuffer = systemAudioBuffer.slice(-MAX_SYSTEM_BUFFER_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('📥 Received system audio for AEC reference');
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---------------------------
|
||||||
|
// Complete token tracker (exact from renderer.js)
|
||||||
|
// ---------------------------
|
||||||
let tokenTracker = {
|
let tokenTracker = {
|
||||||
tokens: [],
|
tokens: [],
|
||||||
audioStartTime: null,
|
audioStartTime: null,
|
||||||
@ -265,126 +300,201 @@ setInterval(() => {
|
|||||||
tokenTracker.trackAudioTokens();
|
tokenTracker.trackAudioTokens();
|
||||||
}, 2000);
|
}, 2000);
|
||||||
|
|
||||||
function pickleGlassElement() {
|
// ---------------------------
|
||||||
return document.getElementById('pickle-glass');
|
// Audio processing functions (exact from renderer.js)
|
||||||
}
|
// ---------------------------
|
||||||
|
function setupMicProcessing(micStream) {
|
||||||
|
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||||
|
const micSource = micAudioContext.createMediaStreamSource(micStream);
|
||||||
|
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
||||||
|
|
||||||
function convertFloat32ToInt16(float32Array) {
|
let audioBuffer = [];
|
||||||
const int16Array = new Int16Array(float32Array.length);
|
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
||||||
for (let i = 0; i < float32Array.length; i++) {
|
|
||||||
// Improved scaling to prevent clipping
|
|
||||||
const s = Math.max(-1, Math.min(1, float32Array[i]));
|
|
||||||
int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
||||||
}
|
|
||||||
return int16Array;
|
|
||||||
}
|
|
||||||
|
|
||||||
function arrayBufferToBase64(buffer) {
|
micProcessor.onaudioprocess = async e => {
|
||||||
let binary = '';
|
const inputData = e.inputBuffer.getChannelData(0);
|
||||||
const bytes = new Uint8Array(buffer);
|
audioBuffer.push(...inputData);
|
||||||
const len = bytes.byteLength;
|
|
||||||
for (let i = 0; i < len; i++) {
|
|
||||||
binary += String.fromCharCode(bytes[i]);
|
|
||||||
}
|
|
||||||
return btoa(binary);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function initializeopenai(profile = 'interview', language = 'en') {
|
while (audioBuffer.length >= samplesPerChunk) {
|
||||||
// The API key is now handled in the main process from .env file.
|
let chunk = audioBuffer.splice(0, samplesPerChunk);
|
||||||
// We just need to trigger the initialization.
|
let processedChunk = new Float32Array(chunk);
|
||||||
try {
|
|
||||||
console.log(`Requesting OpenAI initialization with profile: ${profile}, language: ${language}`);
|
// Check for system audio and apply AEC only if voice is active
|
||||||
const success = await ipcRenderer.invoke('initialize-openai', profile, language);
|
if (aecProcessor && systemAudioBuffer.length > 0) {
|
||||||
if (success) {
|
const latestSystemAudio = systemAudioBuffer[systemAudioBuffer.length - 1];
|
||||||
// The status will be updated via 'update-status' event from the main process.
|
const systemFloat32 = base64ToFloat32Array(latestSystemAudio.data);
|
||||||
console.log('OpenAI initialization successful.');
|
|
||||||
} else {
|
// Apply AEC only when system audio has active speech
|
||||||
console.error('OpenAI initialization failed.');
|
if (isVoiceActive(systemFloat32)) {
|
||||||
const appElement = pickleGlassElement();
|
processedChunk = aecProcessor.process(new Float32Array(chunk), systemFloat32);
|
||||||
if (appElement && typeof appElement.setStatus === 'function') {
|
console.log('🔊 Applied AEC because system audio is active');
|
||||||
appElement.setStatus('Initialization Failed');
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const pcmData16 = convertFloat32ToInt16(processedChunk);
|
||||||
|
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
||||||
|
|
||||||
|
await ipcRenderer.invoke('send-audio-content', {
|
||||||
|
data: base64Data,
|
||||||
|
mimeType: 'audio/pcm;rate=24000',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
micSource.connect(micProcessor);
|
||||||
|
micProcessor.connect(micAudioContext.destination);
|
||||||
|
|
||||||
|
audioProcessor = micProcessor;
|
||||||
|
}
|
||||||
|
|
||||||
|
function setupLinuxMicProcessing(micStream) {
|
||||||
|
// Setup microphone audio processing for Linux
|
||||||
|
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||||
|
const micSource = micAudioContext.createMediaStreamSource(micStream);
|
||||||
|
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
||||||
|
|
||||||
|
let audioBuffer = [];
|
||||||
|
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
||||||
|
|
||||||
|
micProcessor.onaudioprocess = async e => {
|
||||||
|
const inputData = e.inputBuffer.getChannelData(0);
|
||||||
|
audioBuffer.push(...inputData);
|
||||||
|
|
||||||
|
// Process audio in chunks
|
||||||
|
while (audioBuffer.length >= samplesPerChunk) {
|
||||||
|
const chunk = audioBuffer.splice(0, samplesPerChunk);
|
||||||
|
const pcmData16 = convertFloat32ToInt16(chunk);
|
||||||
|
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
||||||
|
|
||||||
|
await ipcRenderer.invoke('send-audio-content', {
|
||||||
|
data: base64Data,
|
||||||
|
mimeType: 'audio/pcm;rate=24000',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
micSource.connect(micProcessor);
|
||||||
|
micProcessor.connect(micAudioContext.destination);
|
||||||
|
|
||||||
|
// Store processor reference for cleanup
|
||||||
|
audioProcessor = micProcessor;
|
||||||
|
}
|
||||||
|
|
||||||
|
function setupWindowsLoopbackProcessing() {
|
||||||
|
// Setup audio processing for Windows loopback audio only
|
||||||
|
audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||||
|
const source = audioContext.createMediaStreamSource(mediaStream);
|
||||||
|
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
||||||
|
|
||||||
|
let audioBuffer = [];
|
||||||
|
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
||||||
|
|
||||||
|
audioProcessor.onaudioprocess = async e => {
|
||||||
|
const inputData = e.inputBuffer.getChannelData(0);
|
||||||
|
audioBuffer.push(...inputData);
|
||||||
|
|
||||||
|
// Process audio in chunks
|
||||||
|
while (audioBuffer.length >= samplesPerChunk) {
|
||||||
|
const chunk = audioBuffer.splice(0, samplesPerChunk);
|
||||||
|
const pcmData16 = convertFloat32ToInt16(chunk);
|
||||||
|
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
||||||
|
|
||||||
|
await ipcRenderer.invoke('send-audio-content', {
|
||||||
|
data: base64Data,
|
||||||
|
mimeType: 'audio/pcm;rate=24000',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
source.connect(audioProcessor);
|
||||||
|
audioProcessor.connect(audioContext.destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------
|
||||||
|
// Screenshot functions (exact from renderer.js)
|
||||||
|
// ---------------------------
|
||||||
|
async function captureScreenshot(imageQuality = 'medium', isManual = false) {
|
||||||
|
console.log(`Capturing ${isManual ? 'manual' : 'automated'} screenshot...`);
|
||||||
|
|
||||||
|
// Check rate limiting for automated screenshots only
|
||||||
|
if (!isManual && tokenTracker.shouldThrottle()) {
|
||||||
|
console.log('⚠️ Automated screenshot skipped due to rate limiting');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Request screenshot from main process
|
||||||
|
const result = await ipcRenderer.invoke('capture-screenshot', {
|
||||||
|
quality: imageQuality,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (result.success && result.base64) {
|
||||||
|
// Store the latest screenshot
|
||||||
|
lastScreenshotBase64 = result.base64;
|
||||||
|
|
||||||
|
// Note: sendResult is not defined in the original, this was likely an error
|
||||||
|
// Commenting out this section as it references undefined variable
|
||||||
|
/*
|
||||||
|
if (sendResult.success) {
|
||||||
|
// Track image tokens after successful send
|
||||||
|
const imageTokens = tokenTracker.calculateImageTokens(result.width || 1920, result.height || 1080);
|
||||||
|
tokenTracker.addTokens(imageTokens, 'image');
|
||||||
|
console.log(`📊 Image sent successfully - ${imageTokens} tokens used (${result.width}x${result.height})`);
|
||||||
|
} else {
|
||||||
|
console.error('Failed to send image:', sendResult.error);
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
} else {
|
||||||
|
console.error('Failed to capture screenshot:', result.error);
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error during OpenAI initialization IPC call:', error);
|
console.error('Error capturing screenshot:', error);
|
||||||
const appElement = pickleGlassElement();
|
|
||||||
if (appElement && typeof appElement.setStatus === 'function') {
|
|
||||||
appElement.setStatus('Error');
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function captureManualScreenshot(imageQuality = null) {
|
||||||
|
console.log('Manual screenshot triggered');
|
||||||
|
const quality = imageQuality || currentImageQuality;
|
||||||
|
await captureScreenshot(quality, true);
|
||||||
|
}
|
||||||
|
|
||||||
ipcRenderer.on('system-audio-data', (event, { data }) => {
|
async function getCurrentScreenshot() {
|
||||||
systemAudioBuffer.push({
|
try {
|
||||||
data: data,
|
// First try to get a fresh screenshot from main process
|
||||||
timestamp: Date.now(),
|
const result = await ipcRenderer.invoke('get-current-screenshot');
|
||||||
});
|
|
||||||
|
|
||||||
// 오래된 데이터 제거
|
if (result.success && result.base64) {
|
||||||
if (systemAudioBuffer.length > MAX_SYSTEM_BUFFER_SIZE) {
|
console.log('📸 Got fresh screenshot from main process');
|
||||||
systemAudioBuffer = systemAudioBuffer.slice(-MAX_SYSTEM_BUFFER_SIZE);
|
return result.base64;
|
||||||
}
|
|
||||||
|
|
||||||
console.log('📥 Received system audio for AEC reference');
|
|
||||||
});
|
|
||||||
|
|
||||||
// Listen for status updates
|
|
||||||
ipcRenderer.on('update-status', (event, status) => {
|
|
||||||
console.log('Status update:', status);
|
|
||||||
pickleGlass.e().setStatus(status);
|
|
||||||
});
|
|
||||||
|
|
||||||
// Listen for real-time STT updates
|
|
||||||
ipcRenderer.on('stt-update', (event, data) => {
|
|
||||||
console.log('Renderer.js stt-update', data);
|
|
||||||
const { speaker, text, isFinal, isPartial, timestamp } = data;
|
|
||||||
|
|
||||||
if (isPartial) {
|
|
||||||
console.log(`🔄 [${speaker} - partial]: ${text}`);
|
|
||||||
} else if (isFinal) {
|
|
||||||
console.log(`✅ [${speaker} - final]: ${text}`);
|
|
||||||
|
|
||||||
const speakerText = speaker.toLowerCase();
|
|
||||||
const conversationText = `${speakerText}: ${text.trim()}`;
|
|
||||||
|
|
||||||
realtimeConversationHistory.push(conversationText);
|
|
||||||
|
|
||||||
if (realtimeConversationHistory.length > 30) {
|
|
||||||
realtimeConversationHistory = realtimeConversationHistory.slice(-30);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`📝 Updated realtime conversation history: ${realtimeConversationHistory.length} texts`);
|
// If no screenshot available, capture one now
|
||||||
console.log(`📋 Latest text: ${conversationText}`);
|
console.log('📸 No screenshot available, capturing new one');
|
||||||
}
|
const captureResult = await ipcRenderer.invoke('capture-screenshot', {
|
||||||
|
quality: currentImageQuality,
|
||||||
if (pickleGlass.e() && typeof pickleGlass.e().updateRealtimeTranscription === 'function') {
|
|
||||||
pickleGlass.e().updateRealtimeTranscription({
|
|
||||||
speaker,
|
|
||||||
text,
|
|
||||||
isFinal,
|
|
||||||
isPartial,
|
|
||||||
timestamp,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (captureResult.success && captureResult.base64) {
|
||||||
|
lastScreenshotBase64 = captureResult.base64;
|
||||||
|
return captureResult.base64;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to last stored screenshot
|
||||||
|
if (lastScreenshotBase64) {
|
||||||
|
console.log('📸 Using cached screenshot');
|
||||||
|
return lastScreenshotBase64;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error('Failed to get screenshot');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error getting current screenshot:', error);
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
|
|
||||||
|
|
||||||
ipcRenderer.on('update-structured-data', (_, structuredData) => {
|
|
||||||
console.log('📥 Received structured data update:', structuredData);
|
|
||||||
window.pickleGlass.structuredData = structuredData;
|
|
||||||
window.pickleGlass.setStructuredData(structuredData);
|
|
||||||
});
|
|
||||||
window.pickleGlass.structuredData = {
|
|
||||||
summary: [],
|
|
||||||
topic: { header: '', bullets: [] },
|
|
||||||
actions: [],
|
|
||||||
};
|
|
||||||
window.pickleGlass.setStructuredData = data => {
|
|
||||||
window.pickleGlass.structuredData = data;
|
|
||||||
pickleGlass.e()?.updateStructuredData?.(data);
|
|
||||||
};
|
|
||||||
|
|
||||||
|
// ---------------------------
|
||||||
|
// Main capture functions (exact from renderer.js)
|
||||||
|
// ---------------------------
|
||||||
async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'medium') {
|
async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'medium') {
|
||||||
// Store the image quality for manual screenshots
|
// Store the image quality for manual screenshots
|
||||||
currentImageQuality = imageQuality;
|
currentImageQuality = imageQuality;
|
||||||
@ -490,12 +600,6 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
|
|||||||
setupWindowsLoopbackProcessing();
|
setupWindowsLoopbackProcessing();
|
||||||
}
|
}
|
||||||
|
|
||||||
// console.log('MediaStream obtained:', {
|
|
||||||
// hasVideo: mediaStream.getVideoTracks().length > 0,
|
|
||||||
// hasAudio: mediaStream.getAudioTracks().length > 0,
|
|
||||||
// videoTrack: mediaStream.getVideoTracks()[0]?.getSettings(),
|
|
||||||
// });
|
|
||||||
|
|
||||||
// Start capturing screenshots - check if manual mode
|
// Start capturing screenshots - check if manual mode
|
||||||
if (screenshotIntervalSeconds === 'manual' || screenshotIntervalSeconds === 'Manual') {
|
if (screenshotIntervalSeconds === 'manual' || screenshotIntervalSeconds === 'Manual') {
|
||||||
console.log('Manual mode enabled - screenshots will be captured on demand only');
|
console.log('Manual mode enabled - screenshots will be captured on demand only');
|
||||||
@ -511,162 +615,11 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
|
|||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Error starting capture:', err);
|
console.error('Error starting capture:', err);
|
||||||
pickleGlass.e().setStatus('error');
|
// Note: pickleGlass.e() is not available in this context, commenting out
|
||||||
|
// pickleGlass.e().setStatus('error');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function setupMicProcessing(micStream) {
|
|
||||||
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
|
||||||
const micSource = micAudioContext.createMediaStreamSource(micStream);
|
|
||||||
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
|
||||||
|
|
||||||
let audioBuffer = [];
|
|
||||||
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
|
||||||
|
|
||||||
micProcessor.onaudioprocess = async e => {
|
|
||||||
const inputData = e.inputBuffer.getChannelData(0);
|
|
||||||
audioBuffer.push(...inputData);
|
|
||||||
|
|
||||||
while (audioBuffer.length >= samplesPerChunk) {
|
|
||||||
let chunk = audioBuffer.splice(0, samplesPerChunk);
|
|
||||||
let processedChunk = new Float32Array(chunk);
|
|
||||||
|
|
||||||
// Check for system audio and apply AEC only if voice is active
|
|
||||||
if (aecProcessor && systemAudioBuffer.length > 0) {
|
|
||||||
const latestSystemAudio = systemAudioBuffer[systemAudioBuffer.length - 1];
|
|
||||||
const systemFloat32 = base64ToFloat32Array(latestSystemAudio.data);
|
|
||||||
|
|
||||||
// Apply AEC only when system audio has active speech
|
|
||||||
if (isVoiceActive(systemFloat32)) {
|
|
||||||
processedChunk = aecProcessor.process(new Float32Array(chunk), systemFloat32);
|
|
||||||
console.log('🔊 Applied AEC because system audio is active');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const pcmData16 = convertFloat32ToInt16(processedChunk);
|
|
||||||
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
|
||||||
|
|
||||||
await ipcRenderer.invoke('send-audio-content', {
|
|
||||||
data: base64Data,
|
|
||||||
mimeType: 'audio/pcm;rate=24000',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
micSource.connect(micProcessor);
|
|
||||||
micProcessor.connect(micAudioContext.destination);
|
|
||||||
|
|
||||||
audioProcessor = micProcessor;
|
|
||||||
}
|
|
||||||
////////// for index & subjects //////////
|
|
||||||
|
|
||||||
function setupLinuxMicProcessing(micStream) {
|
|
||||||
// Setup microphone audio processing for Linux
|
|
||||||
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
|
||||||
const micSource = micAudioContext.createMediaStreamSource(micStream);
|
|
||||||
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
|
||||||
|
|
||||||
let audioBuffer = [];
|
|
||||||
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
|
||||||
|
|
||||||
micProcessor.onaudioprocess = async e => {
|
|
||||||
const inputData = e.inputBuffer.getChannelData(0);
|
|
||||||
audioBuffer.push(...inputData);
|
|
||||||
|
|
||||||
// Process audio in chunks
|
|
||||||
while (audioBuffer.length >= samplesPerChunk) {
|
|
||||||
const chunk = audioBuffer.splice(0, samplesPerChunk);
|
|
||||||
const pcmData16 = convertFloat32ToInt16(chunk);
|
|
||||||
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
|
||||||
|
|
||||||
await ipcRenderer.invoke('send-audio-content', {
|
|
||||||
data: base64Data,
|
|
||||||
mimeType: 'audio/pcm;rate=24000',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
micSource.connect(micProcessor);
|
|
||||||
micProcessor.connect(micAudioContext.destination);
|
|
||||||
|
|
||||||
// Store processor reference for cleanup
|
|
||||||
audioProcessor = micProcessor;
|
|
||||||
}
|
|
||||||
|
|
||||||
function setupWindowsLoopbackProcessing() {
|
|
||||||
// Setup audio processing for Windows loopback audio only
|
|
||||||
audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
|
|
||||||
const source = audioContext.createMediaStreamSource(mediaStream);
|
|
||||||
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
|
|
||||||
|
|
||||||
let audioBuffer = [];
|
|
||||||
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
|
|
||||||
|
|
||||||
audioProcessor.onaudioprocess = async e => {
|
|
||||||
const inputData = e.inputBuffer.getChannelData(0);
|
|
||||||
audioBuffer.push(...inputData);
|
|
||||||
|
|
||||||
// Process audio in chunks
|
|
||||||
while (audioBuffer.length >= samplesPerChunk) {
|
|
||||||
const chunk = audioBuffer.splice(0, samplesPerChunk);
|
|
||||||
const pcmData16 = convertFloat32ToInt16(chunk);
|
|
||||||
const base64Data = arrayBufferToBase64(pcmData16.buffer);
|
|
||||||
|
|
||||||
await ipcRenderer.invoke('send-audio-content', {
|
|
||||||
data: base64Data,
|
|
||||||
mimeType: 'audio/pcm;rate=24000',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
source.connect(audioProcessor);
|
|
||||||
audioProcessor.connect(audioContext.destination);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function captureScreenshot(imageQuality = 'medium', isManual = false) {
|
|
||||||
console.log(`Capturing ${isManual ? 'manual' : 'automated'} screenshot...`);
|
|
||||||
|
|
||||||
// Check rate limiting for automated screenshots only
|
|
||||||
if (!isManual && tokenTracker.shouldThrottle()) {
|
|
||||||
console.log('⚠️ Automated screenshot skipped due to rate limiting');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Request screenshot from main process
|
|
||||||
const result = await ipcRenderer.invoke('capture-screenshot', {
|
|
||||||
quality: imageQuality,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (result.success && result.base64) {
|
|
||||||
// Store the latest screenshot
|
|
||||||
lastScreenshotBase64 = result.base64;
|
|
||||||
|
|
||||||
if (sendResult.success) {
|
|
||||||
// Track image tokens after successful send
|
|
||||||
const imageTokens = tokenTracker.calculateImageTokens(result.width || 1920, result.height || 1080);
|
|
||||||
tokenTracker.addTokens(imageTokens, 'image');
|
|
||||||
console.log(`📊 Image sent successfully - ${imageTokens} tokens used (${result.width}x${result.height})`);
|
|
||||||
} else {
|
|
||||||
console.error('Failed to send image:', sendResult.error);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
console.error('Failed to capture screenshot:', result.error);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error capturing screenshot:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function captureManualScreenshot(imageQuality = null) {
|
|
||||||
console.log('Manual screenshot triggered');
|
|
||||||
const quality = imageQuality || currentImageQuality;
|
|
||||||
await captureScreenshot(quality, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Expose functions to global scope for external access
|
|
||||||
window.captureManualScreenshot = captureManualScreenshot;
|
|
||||||
|
|
||||||
function stopCapture() {
|
function stopCapture() {
|
||||||
if (screenshotInterval) {
|
if (screenshotInterval) {
|
||||||
clearInterval(screenshotInterval);
|
clearInterval(screenshotInterval);
|
||||||
@ -706,76 +659,25 @@ function stopCapture() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getCurrentScreenshot() {
|
// ---------------------------
|
||||||
try {
|
// Exports & global registration
|
||||||
// First try to get a fresh screenshot from main process
|
// ---------------------------
|
||||||
const result = await ipcRenderer.invoke('get-current-screenshot');
|
module.exports = {
|
||||||
|
|
||||||
if (result.success && result.base64) {
|
|
||||||
console.log('📸 Got fresh screenshot from main process');
|
|
||||||
return result.base64;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no screenshot available, capture one now
|
|
||||||
console.log('📸 No screenshot available, capturing new one');
|
|
||||||
const captureResult = await ipcRenderer.invoke('capture-screenshot', {
|
|
||||||
quality: currentImageQuality,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (captureResult.success && captureResult.base64) {
|
|
||||||
lastScreenshotBase64 = captureResult.base64;
|
|
||||||
return captureResult.base64;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to last stored screenshot
|
|
||||||
if (lastScreenshotBase64) {
|
|
||||||
console.log('📸 Using cached screenshot');
|
|
||||||
return lastScreenshotBase64;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error('Failed to get screenshot');
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error getting current screenshot:', error);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatRealtimeConversationHistory() {
|
|
||||||
if (realtimeConversationHistory.length === 0) return 'No conversation history available.';
|
|
||||||
|
|
||||||
return realtimeConversationHistory.slice(-30).join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
window.pickleGlass = {
|
|
||||||
initializeopenai,
|
|
||||||
startCapture,
|
startCapture,
|
||||||
stopCapture,
|
stopCapture,
|
||||||
isLinux: isLinux,
|
captureManualScreenshot,
|
||||||
isMacOS: isMacOS,
|
getCurrentScreenshot,
|
||||||
e: pickleGlassElement,
|
isLinux,
|
||||||
|
isMacOS,
|
||||||
};
|
};
|
||||||
|
|
||||||
// -------------------------------------------------------
|
// Expose functions to global scope for external access (exact from renderer.js)
|
||||||
// 🔔 React to session state changes from the main process
|
if (typeof window !== 'undefined') {
|
||||||
// When the session ends (isActive === false), ensure we stop
|
window.captureManualScreenshot = captureManualScreenshot;
|
||||||
// all local capture pipelines (mic, screen, etc.).
|
window.listenCapture = module.exports;
|
||||||
// -------------------------------------------------------
|
window.pickleGlass = window.pickleGlass || {};
|
||||||
ipcRenderer.on('session-state-changed', (_event, { isActive }) => {
|
window.pickleGlass.startCapture = startCapture;
|
||||||
if (!isActive) {
|
window.pickleGlass.stopCapture = stopCapture;
|
||||||
console.log('[Renderer] Session ended – stopping local capture');
|
window.pickleGlass.captureManualScreenshot = captureManualScreenshot;
|
||||||
stopCapture();
|
window.pickleGlass.getCurrentScreenshot = getCurrentScreenshot;
|
||||||
} else {
|
}
|
||||||
console.log('[Renderer] New session started – clearing in-memory history and summaries');
|
|
||||||
|
|
||||||
// Reset live conversation & analysis caches
|
|
||||||
realtimeConversationHistory = [];
|
|
||||||
|
|
||||||
const blankData = {
|
|
||||||
summary: [],
|
|
||||||
topic: { header: '', bullets: [] },
|
|
||||||
actions: [],
|
|
||||||
followUps: [],
|
|
||||||
};
|
|
||||||
window.pickleGlass.setStructuredData(blankData);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
139
src/features/listen/renderer/renderer.js
Normal file
139
src/features/listen/renderer/renderer.js
Normal file
@ -0,0 +1,139 @@
|
|||||||
|
// renderer.js
|
||||||
|
const { ipcRenderer } = require('electron');
|
||||||
|
const { makeStreamingChatCompletionWithPortkey } = require('../../../common/services/aiProviderService.js');
|
||||||
|
const listenCapture = require('./listenCapture.js');
|
||||||
|
|
||||||
|
let realtimeConversationHistory = [];
|
||||||
|
|
||||||
|
async function queryLoginState() {
|
||||||
|
const userState = await ipcRenderer.invoke('get-current-user');
|
||||||
|
return userState;
|
||||||
|
}
|
||||||
|
|
||||||
|
function pickleGlassElement() {
|
||||||
|
return document.getElementById('pickle-glass');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function initializeopenai(profile = 'interview', language = 'en') {
|
||||||
|
// The API key is now handled in the main process from .env file.
|
||||||
|
// We just need to trigger the initialization.
|
||||||
|
try {
|
||||||
|
console.log(`Requesting OpenAI initialization with profile: ${profile}, language: ${language}`);
|
||||||
|
const success = await ipcRenderer.invoke('initialize-openai', profile, language);
|
||||||
|
if (success) {
|
||||||
|
// The status will be updated via 'update-status' event from the main process.
|
||||||
|
console.log('OpenAI initialization successful.');
|
||||||
|
} else {
|
||||||
|
console.error('OpenAI initialization failed.');
|
||||||
|
const appElement = pickleGlassElement();
|
||||||
|
if (appElement && typeof appElement.setStatus === 'function') {
|
||||||
|
appElement.setStatus('Initialization Failed');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error during OpenAI initialization IPC call:', error);
|
||||||
|
const appElement = pickleGlassElement();
|
||||||
|
if (appElement && typeof appElement.setStatus === 'function') {
|
||||||
|
appElement.setStatus('Error');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Listen for status updates
|
||||||
|
ipcRenderer.on('update-status', (event, status) => {
|
||||||
|
console.log('Status update:', status);
|
||||||
|
pickleGlass.e().setStatus(status);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Listen for real-time STT updates
|
||||||
|
ipcRenderer.on('stt-update', (event, data) => {
|
||||||
|
console.log('Renderer.js stt-update', data);
|
||||||
|
const { speaker, text, isFinal, isPartial, timestamp } = data;
|
||||||
|
|
||||||
|
if (isPartial) {
|
||||||
|
console.log(`🔄 [${speaker} - partial]: ${text}`);
|
||||||
|
} else if (isFinal) {
|
||||||
|
console.log(`✅ [${speaker} - final]: ${text}`);
|
||||||
|
|
||||||
|
const speakerText = speaker.toLowerCase();
|
||||||
|
const conversationText = `${speakerText}: ${text.trim()}`;
|
||||||
|
|
||||||
|
realtimeConversationHistory.push(conversationText);
|
||||||
|
|
||||||
|
if (realtimeConversationHistory.length > 30) {
|
||||||
|
realtimeConversationHistory = realtimeConversationHistory.slice(-30);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`📝 Updated realtime conversation history: ${realtimeConversationHistory.length} texts`);
|
||||||
|
console.log(`📋 Latest text: ${conversationText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pickleGlass.e() && typeof pickleGlass.e().updateRealtimeTranscription === 'function') {
|
||||||
|
pickleGlass.e().updateRealtimeTranscription({
|
||||||
|
speaker,
|
||||||
|
text,
|
||||||
|
isFinal,
|
||||||
|
isPartial,
|
||||||
|
timestamp,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
ipcRenderer.on('update-structured-data', (_, structuredData) => {
|
||||||
|
console.log('📥 Received structured data update:', structuredData);
|
||||||
|
window.pickleGlass.structuredData = structuredData;
|
||||||
|
window.pickleGlass.setStructuredData(structuredData);
|
||||||
|
});
|
||||||
|
|
||||||
|
window.pickleGlass.structuredData = {
|
||||||
|
summary: [],
|
||||||
|
topic: { header: '', bullets: [] },
|
||||||
|
actions: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
window.pickleGlass.setStructuredData = data => {
|
||||||
|
window.pickleGlass.structuredData = data;
|
||||||
|
pickleGlass.e()?.updateStructuredData?.(data);
|
||||||
|
};
|
||||||
|
|
||||||
|
function formatRealtimeConversationHistory() {
|
||||||
|
if (realtimeConversationHistory.length === 0) return 'No conversation history available.';
|
||||||
|
|
||||||
|
return realtimeConversationHistory.slice(-30).join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
window.pickleGlass = {
|
||||||
|
initializeopenai,
|
||||||
|
startCapture: listenCapture.startCapture,
|
||||||
|
stopCapture: listenCapture.stopCapture,
|
||||||
|
isLinux: listenCapture.isLinux,
|
||||||
|
isMacOS: listenCapture.isMacOS,
|
||||||
|
captureManualScreenshot: listenCapture.captureManualScreenshot,
|
||||||
|
getCurrentScreenshot: listenCapture.getCurrentScreenshot,
|
||||||
|
e: pickleGlassElement,
|
||||||
|
};
|
||||||
|
|
||||||
|
// -------------------------------------------------------
|
||||||
|
// 🔔 React to session state changes from the main process
|
||||||
|
// When the session ends (isActive === false), ensure we stop
|
||||||
|
// all local capture pipelines (mic, screen, etc.).
|
||||||
|
// -------------------------------------------------------
|
||||||
|
ipcRenderer.on('session-state-changed', (_event, { isActive }) => {
|
||||||
|
if (!isActive) {
|
||||||
|
console.log('[Renderer] Session ended – stopping local capture');
|
||||||
|
listenCapture.stopCapture();
|
||||||
|
} else {
|
||||||
|
console.log('[Renderer] New session started – clearing in-memory history and summaries');
|
||||||
|
|
||||||
|
// Reset live conversation & analysis caches
|
||||||
|
realtimeConversationHistory = [];
|
||||||
|
|
||||||
|
const blankData = {
|
||||||
|
summary: [],
|
||||||
|
topic: { header: '', bullets: [] },
|
||||||
|
actions: [],
|
||||||
|
followUps: [],
|
||||||
|
};
|
||||||
|
window.pickleGlass.setStructuredData(blankData);
|
||||||
|
}
|
||||||
|
});
|
||||||
5
src/features/listen/stt/repositories/index.js
Normal file
5
src/features/listen/stt/repositories/index.js
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
const sttRepository = require('./sqlite.repository');
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
...sttRepository,
|
||||||
|
};
|
||||||
37
src/features/listen/stt/repositories/sqlite.repository.js
Normal file
37
src/features/listen/stt/repositories/sqlite.repository.js
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
const sqliteClient = require('../../../../common/services/sqliteClient');
|
||||||
|
|
||||||
|
function addTranscript({ sessionId, speaker, text }) {
|
||||||
|
const db = sqliteClient.getDb();
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const transcriptId = require('crypto').randomUUID();
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const query = `INSERT INTO transcripts (id, session_id, start_at, speaker, text, created_at) VALUES (?, ?, ?, ?, ?, ?)`;
|
||||||
|
db.run(query, [transcriptId, sessionId, now, speaker, text, now], function(err) {
|
||||||
|
if (err) {
|
||||||
|
console.error('Error adding transcript:', err);
|
||||||
|
reject(err);
|
||||||
|
} else {
|
||||||
|
resolve({ id: transcriptId });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAllTranscriptsBySessionId(sessionId) {
|
||||||
|
const db = sqliteClient.getDb();
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const query = "SELECT * FROM transcripts WHERE session_id = ? ORDER BY start_at ASC";
|
||||||
|
db.all(query, [sessionId], (err, rows) => {
|
||||||
|
if (err) {
|
||||||
|
reject(err);
|
||||||
|
} else {
|
||||||
|
resolve(rows);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
addTranscript,
|
||||||
|
getAllTranscriptsBySessionId,
|
||||||
|
};
|
||||||
480
src/features/listen/stt/sttService.js
Normal file
480
src/features/listen/stt/sttService.js
Normal file
@ -0,0 +1,480 @@
|
|||||||
|
const { BrowserWindow } = require('electron');
|
||||||
|
const { spawn } = require('child_process');
|
||||||
|
const { connectToGeminiSession } = require('../../../common/services/googleGeminiClient.js');
|
||||||
|
const { connectToOpenAiSession } = require('../../../common/services/openAiClient.js');
|
||||||
|
const { getStoredApiKey, getStoredProvider } = require('../../../electron/windowManager');
|
||||||
|
|
||||||
|
const COMPLETION_DEBOUNCE_MS = 2000;
|
||||||
|
|
||||||
|
class SttService {
|
||||||
|
constructor() {
|
||||||
|
this.mySttSession = null;
|
||||||
|
this.theirSttSession = null;
|
||||||
|
this.myCurrentUtterance = '';
|
||||||
|
this.theirCurrentUtterance = '';
|
||||||
|
|
||||||
|
this.myLastPartialText = '';
|
||||||
|
this.theirLastPartialText = '';
|
||||||
|
this.myInactivityTimer = null;
|
||||||
|
this.theirInactivityTimer = null;
|
||||||
|
|
||||||
|
// Turn-completion debouncing
|
||||||
|
this.myCompletionBuffer = '';
|
||||||
|
this.theirCompletionBuffer = '';
|
||||||
|
this.myCompletionTimer = null;
|
||||||
|
this.theirCompletionTimer = null;
|
||||||
|
|
||||||
|
// System audio capture
|
||||||
|
this.systemAudioProc = null;
|
||||||
|
|
||||||
|
// Callbacks
|
||||||
|
this.onTranscriptionComplete = null;
|
||||||
|
this.onStatusUpdate = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
setCallbacks({ onTranscriptionComplete, onStatusUpdate }) {
|
||||||
|
this.onTranscriptionComplete = onTranscriptionComplete;
|
||||||
|
this.onStatusUpdate = onStatusUpdate;
|
||||||
|
}
|
||||||
|
|
||||||
|
async getApiKey() {
|
||||||
|
const storedKey = await getStoredApiKey();
|
||||||
|
if (storedKey) {
|
||||||
|
console.log('[SttService] Using stored API key');
|
||||||
|
return storedKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
const envKey = process.env.OPENAI_API_KEY;
|
||||||
|
if (envKey) {
|
||||||
|
console.log('[SttService] Using environment API key');
|
||||||
|
return envKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error('[SttService] No API key found in storage or environment');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async getAiProvider() {
|
||||||
|
try {
|
||||||
|
const { ipcRenderer } = require('electron');
|
||||||
|
const provider = await ipcRenderer.invoke('get-ai-provider');
|
||||||
|
return provider || 'openai';
|
||||||
|
} catch (error) {
|
||||||
|
return getStoredProvider ? getStoredProvider() : 'openai';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sendToRenderer(channel, data) {
|
||||||
|
BrowserWindow.getAllWindows().forEach(win => {
|
||||||
|
if (!win.isDestroyed()) {
|
||||||
|
win.webContents.send(channel, data);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
flushMyCompletion() {
|
||||||
|
if (!this.myCompletionBuffer.trim()) return;
|
||||||
|
|
||||||
|
const finalText = this.myCompletionBuffer.trim();
|
||||||
|
|
||||||
|
// Notify completion callback
|
||||||
|
if (this.onTranscriptionComplete) {
|
||||||
|
this.onTranscriptionComplete('Me', finalText);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send to renderer as final
|
||||||
|
this.sendToRenderer('stt-update', {
|
||||||
|
speaker: 'Me',
|
||||||
|
text: finalText,
|
||||||
|
isPartial: false,
|
||||||
|
isFinal: true,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
});
|
||||||
|
|
||||||
|
this.myCompletionBuffer = '';
|
||||||
|
this.myCompletionTimer = null;
|
||||||
|
this.myCurrentUtterance = '';
|
||||||
|
|
||||||
|
if (this.onStatusUpdate) {
|
||||||
|
this.onStatusUpdate('Listening...');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
flushTheirCompletion() {
|
||||||
|
if (!this.theirCompletionBuffer.trim()) return;
|
||||||
|
|
||||||
|
const finalText = this.theirCompletionBuffer.trim();
|
||||||
|
|
||||||
|
// Notify completion callback
|
||||||
|
if (this.onTranscriptionComplete) {
|
||||||
|
this.onTranscriptionComplete('Them', finalText);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send to renderer as final
|
||||||
|
this.sendToRenderer('stt-update', {
|
||||||
|
speaker: 'Them',
|
||||||
|
text: finalText,
|
||||||
|
isPartial: false,
|
||||||
|
isFinal: true,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
});
|
||||||
|
|
||||||
|
this.theirCompletionBuffer = '';
|
||||||
|
this.theirCompletionTimer = null;
|
||||||
|
this.theirCurrentUtterance = '';
|
||||||
|
|
||||||
|
if (this.onStatusUpdate) {
|
||||||
|
this.onStatusUpdate('Listening...');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
debounceMyCompletion(text) {
|
||||||
|
// 상대방이 말하고 있던 경우, 화자가 변경되었으므로 즉시 상대방의 말풍선을 완성합니다.
|
||||||
|
if (this.theirCompletionTimer) {
|
||||||
|
clearTimeout(this.theirCompletionTimer);
|
||||||
|
this.flushTheirCompletion();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.myCompletionBuffer += (this.myCompletionBuffer ? ' ' : '') + text;
|
||||||
|
|
||||||
|
if (this.myCompletionTimer) clearTimeout(this.myCompletionTimer);
|
||||||
|
this.myCompletionTimer = setTimeout(() => this.flushMyCompletion(), COMPLETION_DEBOUNCE_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
debounceTheirCompletion(text) {
|
||||||
|
// 내가 말하고 있던 경우, 화자가 변경되었으므로 즉시 내 말풍선을 완성합니다.
|
||||||
|
if (this.myCompletionTimer) {
|
||||||
|
clearTimeout(this.myCompletionTimer);
|
||||||
|
this.flushMyCompletion();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.theirCompletionBuffer += (this.theirCompletionBuffer ? ' ' : '') + text;
|
||||||
|
|
||||||
|
if (this.theirCompletionTimer) clearTimeout(this.theirCompletionTimer);
|
||||||
|
this.theirCompletionTimer = setTimeout(() => this.flushTheirCompletion(), COMPLETION_DEBOUNCE_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
async initializeSttSessions(language = 'en') {
|
||||||
|
const effectiveLanguage = process.env.OPENAI_TRANSCRIBE_LANG || language || 'en';
|
||||||
|
|
||||||
|
const API_KEY = await this.getApiKey();
|
||||||
|
if (!API_KEY) {
|
||||||
|
throw new Error('No API key available');
|
||||||
|
}
|
||||||
|
|
||||||
|
const provider = await this.getAiProvider();
|
||||||
|
const isGemini = provider === 'gemini';
|
||||||
|
console.log(`[SttService] Initializing STT for provider: ${provider}`);
|
||||||
|
|
||||||
|
const handleMyMessage = message => {
|
||||||
|
if (isGemini) {
|
||||||
|
const text = message.serverContent?.inputTranscription?.text || '';
|
||||||
|
if (text && text.trim()) {
|
||||||
|
const finalUtteranceText = text.trim().replace(/<noise>/g, '').trim();
|
||||||
|
if (finalUtteranceText && finalUtteranceText !== '.') {
|
||||||
|
this.debounceMyCompletion(finalUtteranceText);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const type = message.type;
|
||||||
|
const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || '';
|
||||||
|
|
||||||
|
if (type === 'conversation.item.input_audio_transcription.delta') {
|
||||||
|
if (this.myCompletionTimer) clearTimeout(this.myCompletionTimer);
|
||||||
|
this.myCompletionTimer = null;
|
||||||
|
this.myCurrentUtterance += text;
|
||||||
|
const continuousText = this.myCompletionBuffer + (this.myCompletionBuffer ? ' ' : '') + this.myCurrentUtterance;
|
||||||
|
if (text && !text.includes('vq_lbr_audio_')) {
|
||||||
|
this.sendToRenderer('stt-update', {
|
||||||
|
speaker: 'Me',
|
||||||
|
text: continuousText,
|
||||||
|
isPartial: true,
|
||||||
|
isFinal: false,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else if (type === 'conversation.item.input_audio_transcription.completed') {
|
||||||
|
if (text && text.trim()) {
|
||||||
|
const finalUtteranceText = text.trim();
|
||||||
|
this.myCurrentUtterance = '';
|
||||||
|
this.debounceMyCompletion(finalUtteranceText);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (message.error) {
|
||||||
|
console.error('[Me] STT Session Error:', message.error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleTheirMessage = message => {
|
||||||
|
if (isGemini) {
|
||||||
|
const text = message.serverContent?.inputTranscription?.text || '';
|
||||||
|
if (text && text.trim()) {
|
||||||
|
const finalUtteranceText = text.trim().replace(/<noise>/g, '').trim();
|
||||||
|
if (finalUtteranceText && finalUtteranceText !== '.') {
|
||||||
|
this.debounceTheirCompletion(finalUtteranceText);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const type = message.type;
|
||||||
|
const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || '';
|
||||||
|
if (type === 'conversation.item.input_audio_transcription.delta') {
|
||||||
|
if (this.theirCompletionTimer) clearTimeout(this.theirCompletionTimer);
|
||||||
|
this.theirCompletionTimer = null;
|
||||||
|
this.theirCurrentUtterance += text;
|
||||||
|
const continuousText = this.theirCompletionBuffer + (this.theirCompletionBuffer ? ' ' : '') + this.theirCurrentUtterance;
|
||||||
|
if (text && !text.includes('vq_lbr_audio_')) {
|
||||||
|
this.sendToRenderer('stt-update', {
|
||||||
|
speaker: 'Them',
|
||||||
|
text: continuousText,
|
||||||
|
isPartial: true,
|
||||||
|
isFinal: false,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else if (type === 'conversation.item.input_audio_transcription.completed') {
|
||||||
|
if (text && text.trim()) {
|
||||||
|
const finalUtteranceText = text.trim();
|
||||||
|
this.theirCurrentUtterance = '';
|
||||||
|
this.debounceTheirCompletion(finalUtteranceText);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (message.error) {
|
||||||
|
console.error('[Them] STT Session Error:', message.error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const mySttConfig = {
|
||||||
|
language: effectiveLanguage,
|
||||||
|
callbacks: {
|
||||||
|
onmessage: handleMyMessage,
|
||||||
|
onerror: error => console.error('My STT session error:', error.message),
|
||||||
|
onclose: event => console.log('My STT session closed:', event.reason),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const theirSttConfig = {
|
||||||
|
language: effectiveLanguage,
|
||||||
|
callbacks: {
|
||||||
|
onmessage: handleTheirMessage,
|
||||||
|
onerror: error => console.error('Their STT session error:', error.message),
|
||||||
|
onclose: event => console.log('Their STT session closed:', event.reason),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// Determine key type based on auth status
|
||||||
|
const authService = require('../../../common/services/authService');
|
||||||
|
const userState = authService.getCurrentUser();
|
||||||
|
const loggedIn = userState.isLoggedIn;
|
||||||
|
const keyType = loggedIn ? 'vKey' : 'apiKey';
|
||||||
|
|
||||||
|
if (isGemini) {
|
||||||
|
[this.mySttSession, this.theirSttSession] = await Promise.all([
|
||||||
|
connectToGeminiSession(API_KEY, mySttConfig),
|
||||||
|
connectToGeminiSession(API_KEY, theirSttConfig),
|
||||||
|
]);
|
||||||
|
} else {
|
||||||
|
[this.mySttSession, this.theirSttSession] = await Promise.all([
|
||||||
|
connectToOpenAiSession(API_KEY, mySttConfig, keyType),
|
||||||
|
connectToOpenAiSession(API_KEY, theirSttConfig, keyType),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('✅ Both STT sessions initialized successfully.');
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async sendAudioContent(data, mimeType) {
|
||||||
|
const provider = await this.getAiProvider();
|
||||||
|
const isGemini = provider === 'gemini';
|
||||||
|
|
||||||
|
if (!this.mySttSession) {
|
||||||
|
throw new Error('User STT session not active');
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload = isGemini
|
||||||
|
? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } }
|
||||||
|
: data;
|
||||||
|
|
||||||
|
await this.mySttSession.sendRealtimeInput(payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
killExistingSystemAudioDump() {
|
||||||
|
return new Promise(resolve => {
|
||||||
|
console.log('Checking for existing SystemAudioDump processes...');
|
||||||
|
|
||||||
|
const killProc = spawn('pkill', ['-f', 'SystemAudioDump'], {
|
||||||
|
stdio: 'ignore',
|
||||||
|
});
|
||||||
|
|
||||||
|
killProc.on('close', code => {
|
||||||
|
if (code === 0) {
|
||||||
|
console.log('Killed existing SystemAudioDump processes');
|
||||||
|
} else {
|
||||||
|
console.log('No existing SystemAudioDump processes found');
|
||||||
|
}
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
|
||||||
|
killProc.on('error', err => {
|
||||||
|
console.log('Error checking for existing processes (this is normal):', err.message);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
|
||||||
|
setTimeout(() => {
|
||||||
|
killProc.kill();
|
||||||
|
resolve();
|
||||||
|
}, 2000);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async startMacOSAudioCapture() {
|
||||||
|
if (process.platform !== 'darwin' || !this.theirSttSession) return false;
|
||||||
|
|
||||||
|
await this.killExistingSystemAudioDump();
|
||||||
|
console.log('Starting macOS audio capture for "Them"...');
|
||||||
|
|
||||||
|
const { app } = require('electron');
|
||||||
|
const path = require('path');
|
||||||
|
const systemAudioPath = app.isPackaged
|
||||||
|
? path.join(process.resourcesPath, 'app.asar.unpacked', 'src', 'assets', 'SystemAudioDump')
|
||||||
|
: path.join(app.getAppPath(), 'src', 'assets', 'SystemAudioDump');
|
||||||
|
|
||||||
|
console.log('SystemAudioDump path:', systemAudioPath);
|
||||||
|
|
||||||
|
this.systemAudioProc = spawn(systemAudioPath, [], {
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!this.systemAudioProc.pid) {
|
||||||
|
console.error('Failed to start SystemAudioDump');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('SystemAudioDump started with PID:', this.systemAudioProc.pid);
|
||||||
|
|
||||||
|
const CHUNK_DURATION = 0.1;
|
||||||
|
const SAMPLE_RATE = 24000;
|
||||||
|
const BYTES_PER_SAMPLE = 2;
|
||||||
|
const CHANNELS = 2;
|
||||||
|
const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * CHUNK_DURATION;
|
||||||
|
|
||||||
|
let audioBuffer = Buffer.alloc(0);
|
||||||
|
|
||||||
|
const provider = await this.getAiProvider();
|
||||||
|
const isGemini = provider === 'gemini';
|
||||||
|
|
||||||
|
this.systemAudioProc.stdout.on('data', async data => {
|
||||||
|
audioBuffer = Buffer.concat([audioBuffer, data]);
|
||||||
|
|
||||||
|
while (audioBuffer.length >= CHUNK_SIZE) {
|
||||||
|
const chunk = audioBuffer.slice(0, CHUNK_SIZE);
|
||||||
|
audioBuffer = audioBuffer.slice(CHUNK_SIZE);
|
||||||
|
|
||||||
|
const monoChunk = CHANNELS === 2 ? this.convertStereoToMono(chunk) : chunk;
|
||||||
|
const base64Data = monoChunk.toString('base64');
|
||||||
|
|
||||||
|
this.sendToRenderer('system-audio-data', { data: base64Data });
|
||||||
|
|
||||||
|
if (this.theirSttSession) {
|
||||||
|
try {
|
||||||
|
const payload = isGemini
|
||||||
|
? { audio: { data: base64Data, mimeType: 'audio/pcm;rate=24000' } }
|
||||||
|
: base64Data;
|
||||||
|
await this.theirSttSession.sendRealtimeInput(payload);
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Error sending system audio:', err.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
this.systemAudioProc.stderr.on('data', data => {
|
||||||
|
console.error('SystemAudioDump stderr:', data.toString());
|
||||||
|
});
|
||||||
|
|
||||||
|
this.systemAudioProc.on('close', code => {
|
||||||
|
console.log('SystemAudioDump process closed with code:', code);
|
||||||
|
this.systemAudioProc = null;
|
||||||
|
});
|
||||||
|
|
||||||
|
this.systemAudioProc.on('error', err => {
|
||||||
|
console.error('SystemAudioDump process error:', err);
|
||||||
|
this.systemAudioProc = null;
|
||||||
|
});
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
convertStereoToMono(stereoBuffer) {
|
||||||
|
const samples = stereoBuffer.length / 4;
|
||||||
|
const monoBuffer = Buffer.alloc(samples * 2);
|
||||||
|
|
||||||
|
for (let i = 0; i < samples; i++) {
|
||||||
|
const leftSample = stereoBuffer.readInt16LE(i * 4);
|
||||||
|
monoBuffer.writeInt16LE(leftSample, i * 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
return monoBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
stopMacOSAudioCapture() {
|
||||||
|
if (this.systemAudioProc) {
|
||||||
|
console.log('Stopping SystemAudioDump...');
|
||||||
|
this.systemAudioProc.kill('SIGTERM');
|
||||||
|
this.systemAudioProc = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
isSessionActive() {
|
||||||
|
return !!this.mySttSession && !!this.theirSttSession;
|
||||||
|
}
|
||||||
|
|
||||||
|
async closeSessions() {
|
||||||
|
this.stopMacOSAudioCapture();
|
||||||
|
|
||||||
|
// Clear timers
|
||||||
|
if (this.myInactivityTimer) {
|
||||||
|
clearTimeout(this.myInactivityTimer);
|
||||||
|
this.myInactivityTimer = null;
|
||||||
|
}
|
||||||
|
if (this.theirInactivityTimer) {
|
||||||
|
clearTimeout(this.theirInactivityTimer);
|
||||||
|
this.theirInactivityTimer = null;
|
||||||
|
}
|
||||||
|
if (this.myCompletionTimer) {
|
||||||
|
clearTimeout(this.myCompletionTimer);
|
||||||
|
this.myCompletionTimer = null;
|
||||||
|
}
|
||||||
|
if (this.theirCompletionTimer) {
|
||||||
|
clearTimeout(this.theirCompletionTimer);
|
||||||
|
this.theirCompletionTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const closePromises = [];
|
||||||
|
if (this.mySttSession) {
|
||||||
|
closePromises.push(this.mySttSession.close());
|
||||||
|
this.mySttSession = null;
|
||||||
|
}
|
||||||
|
if (this.theirSttSession) {
|
||||||
|
closePromises.push(this.theirSttSession.close());
|
||||||
|
this.theirSttSession = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
await Promise.all(closePromises);
|
||||||
|
console.log('All STT sessions closed.');
|
||||||
|
|
||||||
|
// Reset state
|
||||||
|
this.myCurrentUtterance = '';
|
||||||
|
this.theirCurrentUtterance = '';
|
||||||
|
this.myLastPartialText = '';
|
||||||
|
this.theirLastPartialText = '';
|
||||||
|
this.myCompletionBuffer = '';
|
||||||
|
this.theirCompletionBuffer = '';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = SttService;
|
||||||
5
src/features/listen/summary/repositories/index.js
Normal file
5
src/features/listen/summary/repositories/index.js
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
const summaryRepository = require('./sqlite.repository');
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
...summaryRepository,
|
||||||
|
};
|
||||||
@ -0,0 +1,47 @@
|
|||||||
|
const sqliteClient = require('../../../../common/services/sqliteClient');
|
||||||
|
|
||||||
|
function saveSummary({ sessionId, tldr, text, bullet_json, action_json, model = 'gpt-4.1' }) {
|
||||||
|
const db = sqliteClient.getDb();
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const query = `
|
||||||
|
INSERT INTO summaries (session_id, generated_at, model, text, tldr, bullet_json, action_json, updated_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(session_id) DO UPDATE SET
|
||||||
|
generated_at=excluded.generated_at,
|
||||||
|
model=excluded.model,
|
||||||
|
text=excluded.text,
|
||||||
|
tldr=excluded.tldr,
|
||||||
|
bullet_json=excluded.bullet_json,
|
||||||
|
action_json=excluded.action_json,
|
||||||
|
updated_at=excluded.updated_at
|
||||||
|
`;
|
||||||
|
db.run(query, [sessionId, now, model, text, tldr, bullet_json, action_json, now], function(err) {
|
||||||
|
if (err) {
|
||||||
|
console.error('Error saving summary:', err);
|
||||||
|
reject(err);
|
||||||
|
} else {
|
||||||
|
resolve({ changes: this.changes });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function getSummaryBySessionId(sessionId) {
|
||||||
|
const db = sqliteClient.getDb();
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const query = "SELECT * FROM summaries WHERE session_id = ?";
|
||||||
|
db.get(query, [sessionId], (err, row) => {
|
||||||
|
if (err) {
|
||||||
|
reject(err);
|
||||||
|
} else {
|
||||||
|
resolve(row || null);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
saveSummary,
|
||||||
|
getSummaryBySessionId,
|
||||||
|
};
|
||||||
357
src/features/listen/summary/summaryService.js
Normal file
357
src/features/listen/summary/summaryService.js
Normal file
@ -0,0 +1,357 @@
|
|||||||
|
const { BrowserWindow } = require('electron');
|
||||||
|
const { getSystemPrompt } = require('../../../common/prompts/promptBuilder.js');
|
||||||
|
const { makeChatCompletionWithPortkey } = require('../../../common/services/aiProviderService.js');
|
||||||
|
const authService = require('../../../common/services/authService');
|
||||||
|
const sessionRepository = require('../../../common/repositories/session');
|
||||||
|
const summaryRepository = require('./repositories');
|
||||||
|
const { getStoredApiKey, getStoredProvider } = require('../../../electron/windowManager');
|
||||||
|
|
||||||
|
class SummaryService {
|
||||||
|
constructor() {
|
||||||
|
this.previousAnalysisResult = null;
|
||||||
|
this.analysisHistory = [];
|
||||||
|
this.conversationHistory = [];
|
||||||
|
this.currentSessionId = null;
|
||||||
|
|
||||||
|
// Callbacks
|
||||||
|
this.onAnalysisComplete = null;
|
||||||
|
this.onStatusUpdate = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
setCallbacks({ onAnalysisComplete, onStatusUpdate }) {
|
||||||
|
this.onAnalysisComplete = onAnalysisComplete;
|
||||||
|
this.onStatusUpdate = onStatusUpdate;
|
||||||
|
}
|
||||||
|
|
||||||
|
setSessionId(sessionId) {
|
||||||
|
this.currentSessionId = sessionId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async getApiKey() {
|
||||||
|
const storedKey = await getStoredApiKey();
|
||||||
|
if (storedKey) {
|
||||||
|
console.log('[SummaryService] Using stored API key');
|
||||||
|
return storedKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
const envKey = process.env.OPENAI_API_KEY;
|
||||||
|
if (envKey) {
|
||||||
|
console.log('[SummaryService] Using environment API key');
|
||||||
|
return envKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error('[SummaryService] No API key found in storage or environment');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
sendToRenderer(channel, data) {
|
||||||
|
BrowserWindow.getAllWindows().forEach(win => {
|
||||||
|
if (!win.isDestroyed()) {
|
||||||
|
win.webContents.send(channel, data);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
addConversationTurn(speaker, text) {
|
||||||
|
const conversationText = `${speaker.toLowerCase()}: ${text.trim()}`;
|
||||||
|
this.conversationHistory.push(conversationText);
|
||||||
|
console.log(`💬 Added conversation text: ${conversationText}`);
|
||||||
|
console.log(`📈 Total conversation history: ${this.conversationHistory.length} texts`);
|
||||||
|
|
||||||
|
// Trigger analysis if needed
|
||||||
|
this.triggerAnalysisIfNeeded();
|
||||||
|
}
|
||||||
|
|
||||||
|
getConversationHistory() {
|
||||||
|
return this.conversationHistory;
|
||||||
|
}
|
||||||
|
|
||||||
|
resetConversationHistory() {
|
||||||
|
this.conversationHistory = [];
|
||||||
|
this.previousAnalysisResult = null;
|
||||||
|
this.analysisHistory = [];
|
||||||
|
console.log('🔄 Conversation history and analysis state reset');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Converts conversation history into text to include in the prompt.
|
||||||
|
* @param {Array<string>} conversationTexts - Array of conversation texts ["me: ~~~", "them: ~~~", ...]
|
||||||
|
* @param {number} maxTurns - Maximum number of recent turns to include
|
||||||
|
* @returns {string} - Formatted conversation string for the prompt
|
||||||
|
*/
|
||||||
|
formatConversationForPrompt(conversationTexts, maxTurns = 30) {
|
||||||
|
if (conversationTexts.length === 0) return '';
|
||||||
|
return conversationTexts.slice(-maxTurns).join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
async makeOutlineAndRequests(conversationTexts, maxTurns = 30) {
|
||||||
|
console.log(`🔍 makeOutlineAndRequests called - conversationTexts: ${conversationTexts.length}`);
|
||||||
|
|
||||||
|
if (conversationTexts.length === 0) {
|
||||||
|
console.log('⚠️ No conversation texts available for analysis');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const recentConversation = this.formatConversationForPrompt(conversationTexts, maxTurns);
|
||||||
|
|
||||||
|
// 이전 분석 결과를 프롬프트에 포함
|
||||||
|
let contextualPrompt = '';
|
||||||
|
if (this.previousAnalysisResult) {
|
||||||
|
contextualPrompt = `
|
||||||
|
Previous Analysis Context:
|
||||||
|
- Main Topic: ${this.previousAnalysisResult.topic.header}
|
||||||
|
- Key Points: ${this.previousAnalysisResult.summary.slice(0, 3).join(', ')}
|
||||||
|
- Last Actions: ${this.previousAnalysisResult.actions.slice(0, 2).join(', ')}
|
||||||
|
|
||||||
|
Please build upon this context while analyzing the new conversation segments.
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const basePrompt = getSystemPrompt('pickle_glass_analysis', '', false);
|
||||||
|
const systemPrompt = basePrompt.replace('{{CONVERSATION_HISTORY}}', recentConversation);
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (this.currentSessionId) {
|
||||||
|
await sessionRepository.touch(this.currentSessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
const messages = [
|
||||||
|
{
|
||||||
|
role: 'system',
|
||||||
|
content: systemPrompt,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: 'user',
|
||||||
|
content: `${contextualPrompt}
|
||||||
|
|
||||||
|
Analyze the conversation and provide a structured summary. Format your response as follows:
|
||||||
|
|
||||||
|
**Summary Overview**
|
||||||
|
- Main discussion point with context
|
||||||
|
|
||||||
|
**Key Topic: [Topic Name]**
|
||||||
|
- First key insight
|
||||||
|
- Second key insight
|
||||||
|
- Third key insight
|
||||||
|
|
||||||
|
**Extended Explanation**
|
||||||
|
Provide 2-3 sentences explaining the context and implications.
|
||||||
|
|
||||||
|
**Suggested Questions**
|
||||||
|
1. First follow-up question?
|
||||||
|
2. Second follow-up question?
|
||||||
|
3. Third follow-up question?
|
||||||
|
|
||||||
|
Keep all points concise and build upon previous analysis if provided.`,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
console.log('🤖 Sending analysis request to AI...');
|
||||||
|
|
||||||
|
const API_KEY = await this.getApiKey();
|
||||||
|
if (!API_KEY) {
|
||||||
|
throw new Error('No API key available');
|
||||||
|
}
|
||||||
|
|
||||||
|
const provider = getStoredProvider ? await getStoredProvider() : 'openai';
|
||||||
|
const loggedIn = authService.getCurrentUser().isLoggedIn;
|
||||||
|
const usePortkey = loggedIn && provider === 'openai';
|
||||||
|
|
||||||
|
console.log(`[SummaryService] provider: ${provider}, usePortkey: ${usePortkey}`);
|
||||||
|
|
||||||
|
const completion = await makeChatCompletionWithPortkey({
|
||||||
|
apiKey: API_KEY,
|
||||||
|
provider: provider,
|
||||||
|
messages: messages,
|
||||||
|
temperature: 0.7,
|
||||||
|
maxTokens: 1024,
|
||||||
|
model: provider === 'openai' ? 'gpt-4.1' : 'gemini-2.5-flash',
|
||||||
|
usePortkey: usePortkey,
|
||||||
|
portkeyVirtualKey: usePortkey ? API_KEY : null
|
||||||
|
});
|
||||||
|
|
||||||
|
const responseText = completion.content;
|
||||||
|
console.log(`✅ Analysis response received: ${responseText}`);
|
||||||
|
const structuredData = this.parseResponseText(responseText, this.previousAnalysisResult);
|
||||||
|
|
||||||
|
if (this.currentSessionId) {
|
||||||
|
summaryRepository.saveSummary({
|
||||||
|
sessionId: this.currentSessionId,
|
||||||
|
text: responseText,
|
||||||
|
tldr: structuredData.summary.join('\n'),
|
||||||
|
bullet_json: JSON.stringify(structuredData.topic.bullets),
|
||||||
|
action_json: JSON.stringify(structuredData.actions),
|
||||||
|
model: 'gpt-4.1'
|
||||||
|
}).catch(err => console.error('[DB] Failed to save summary:', err));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 분석 결과 저장
|
||||||
|
this.previousAnalysisResult = structuredData;
|
||||||
|
this.analysisHistory.push({
|
||||||
|
timestamp: Date.now(),
|
||||||
|
data: structuredData,
|
||||||
|
conversationLength: conversationTexts.length,
|
||||||
|
});
|
||||||
|
|
||||||
|
// 히스토리 크기 제한 (최근 10개만 유지)
|
||||||
|
if (this.analysisHistory.length > 10) {
|
||||||
|
this.analysisHistory.shift();
|
||||||
|
}
|
||||||
|
|
||||||
|
return structuredData;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Error during analysis generation:', error.message);
|
||||||
|
return this.previousAnalysisResult; // 에러 시 이전 결과 반환
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
parseResponseText(responseText, previousResult) {
|
||||||
|
const structuredData = {
|
||||||
|
summary: [],
|
||||||
|
topic: { header: '', bullets: [] },
|
||||||
|
actions: [],
|
||||||
|
followUps: ['✉️ Draft a follow-up email', '✅ Generate action items', '📝 Show summary'],
|
||||||
|
};
|
||||||
|
|
||||||
|
// 이전 결과가 있으면 기본값으로 사용
|
||||||
|
if (previousResult) {
|
||||||
|
structuredData.topic.header = previousResult.topic.header;
|
||||||
|
structuredData.summary = [...previousResult.summary];
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const lines = responseText.split('\n');
|
||||||
|
let currentSection = '';
|
||||||
|
let isCapturingTopic = false;
|
||||||
|
let topicName = '';
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
const trimmedLine = line.trim();
|
||||||
|
|
||||||
|
// 섹션 헤더 감지
|
||||||
|
if (trimmedLine.startsWith('**Summary Overview**')) {
|
||||||
|
currentSection = 'summary-overview';
|
||||||
|
continue;
|
||||||
|
} else if (trimmedLine.startsWith('**Key Topic:')) {
|
||||||
|
currentSection = 'topic';
|
||||||
|
isCapturingTopic = true;
|
||||||
|
topicName = trimmedLine.match(/\*\*Key Topic: (.+?)\*\*/)?.[1] || '';
|
||||||
|
if (topicName) {
|
||||||
|
structuredData.topic.header = topicName + ':';
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
} else if (trimmedLine.startsWith('**Extended Explanation**')) {
|
||||||
|
currentSection = 'explanation';
|
||||||
|
continue;
|
||||||
|
} else if (trimmedLine.startsWith('**Suggested Questions**')) {
|
||||||
|
currentSection = 'questions';
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 컨텐츠 파싱
|
||||||
|
if (trimmedLine.startsWith('-') && currentSection === 'summary-overview') {
|
||||||
|
const summaryPoint = trimmedLine.substring(1).trim();
|
||||||
|
if (summaryPoint && !structuredData.summary.includes(summaryPoint)) {
|
||||||
|
// 기존 summary 업데이트 (최대 5개 유지)
|
||||||
|
structuredData.summary.unshift(summaryPoint);
|
||||||
|
if (structuredData.summary.length > 5) {
|
||||||
|
structuredData.summary.pop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (trimmedLine.startsWith('-') && currentSection === 'topic') {
|
||||||
|
const bullet = trimmedLine.substring(1).trim();
|
||||||
|
if (bullet && structuredData.topic.bullets.length < 3) {
|
||||||
|
structuredData.topic.bullets.push(bullet);
|
||||||
|
}
|
||||||
|
} else if (currentSection === 'explanation' && trimmedLine) {
|
||||||
|
// explanation을 topic bullets에 추가 (문장 단위로)
|
||||||
|
const sentences = trimmedLine
|
||||||
|
.split(/\.\s+/)
|
||||||
|
.filter(s => s.trim().length > 0)
|
||||||
|
.map(s => s.trim() + (s.endsWith('.') ? '' : '.'));
|
||||||
|
|
||||||
|
sentences.forEach(sentence => {
|
||||||
|
if (structuredData.topic.bullets.length < 3 && !structuredData.topic.bullets.includes(sentence)) {
|
||||||
|
structuredData.topic.bullets.push(sentence);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} else if (trimmedLine.match(/^\d+\./) && currentSection === 'questions') {
|
||||||
|
const question = trimmedLine.replace(/^\d+\.\s*/, '').trim();
|
||||||
|
if (question && question.includes('?')) {
|
||||||
|
structuredData.actions.push(`❓ ${question}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 기본 액션 추가
|
||||||
|
const defaultActions = ['✨ What should I say next?', '💬 Suggest follow-up questions'];
|
||||||
|
defaultActions.forEach(action => {
|
||||||
|
if (!structuredData.actions.includes(action)) {
|
||||||
|
structuredData.actions.push(action);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// 액션 개수 제한
|
||||||
|
structuredData.actions = structuredData.actions.slice(0, 5);
|
||||||
|
|
||||||
|
// 유효성 검증 및 이전 데이터 병합
|
||||||
|
if (structuredData.summary.length === 0 && previousResult) {
|
||||||
|
structuredData.summary = previousResult.summary;
|
||||||
|
}
|
||||||
|
if (structuredData.topic.bullets.length === 0 && previousResult) {
|
||||||
|
structuredData.topic.bullets = previousResult.topic.bullets;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Error parsing response text:', error);
|
||||||
|
// 에러 시 이전 결과 반환
|
||||||
|
return (
|
||||||
|
previousResult || {
|
||||||
|
summary: [],
|
||||||
|
topic: { header: 'Analysis in progress', bullets: [] },
|
||||||
|
actions: ['✨ What should I say next?', '💬 Suggest follow-up questions'],
|
||||||
|
followUps: ['✉️ Draft a follow-up email', '✅ Generate action items', '📝 Show summary'],
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('📊 Final structured data:', JSON.stringify(structuredData, null, 2));
|
||||||
|
return structuredData;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Triggers analysis when conversation history reaches 5 texts.
|
||||||
|
*/
|
||||||
|
async triggerAnalysisIfNeeded() {
|
||||||
|
if (this.conversationHistory.length >= 5 && this.conversationHistory.length % 5 === 0) {
|
||||||
|
console.log(`🚀 Triggering analysis (non-blocking) - ${this.conversationHistory.length} conversation texts accumulated`);
|
||||||
|
|
||||||
|
this.makeOutlineAndRequests(this.conversationHistory)
|
||||||
|
.then(data => {
|
||||||
|
if (data) {
|
||||||
|
console.log('📤 Sending structured data to renderer');
|
||||||
|
this.sendToRenderer('update-structured-data', data);
|
||||||
|
|
||||||
|
// Notify callback
|
||||||
|
if (this.onAnalysisComplete) {
|
||||||
|
this.onAnalysisComplete(data);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log('❌ No analysis data returned from non-blocking call');
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('❌ Error in non-blocking analysis:', error);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
getCurrentAnalysisData() {
|
||||||
|
return {
|
||||||
|
previousResult: this.previousAnalysisResult,
|
||||||
|
history: this.analysisHistory,
|
||||||
|
conversationLength: this.conversationHistory.length,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = SummaryService;
|
||||||
19
src/index.js
19
src/index.js
@ -13,7 +13,7 @@ if (require('electron-squirrel-startup')) {
|
|||||||
|
|
||||||
const { app, BrowserWindow, shell, ipcMain, dialog } = require('electron');
|
const { app, BrowserWindow, shell, ipcMain, dialog } = require('electron');
|
||||||
const { createWindows } = require('./electron/windowManager.js');
|
const { createWindows } = require('./electron/windowManager.js');
|
||||||
const { setupLiveSummaryIpcHandlers, stopMacOSAudioCapture } = require('./features/listen/liveSummaryService.js');
|
const ListenService = require('./features/listen/listenService');
|
||||||
const { initializeFirebase } = require('./common/services/firebaseClient');
|
const { initializeFirebase } = require('./common/services/firebaseClient');
|
||||||
const databaseInitializer = require('./common/services/databaseInitializer');
|
const databaseInitializer = require('./common/services/databaseInitializer');
|
||||||
const authService = require('./common/services/authService');
|
const authService = require('./common/services/authService');
|
||||||
@ -29,7 +29,9 @@ const sessionRepository = require('./common/repositories/session');
|
|||||||
const eventBridge = new EventEmitter();
|
const eventBridge = new EventEmitter();
|
||||||
let WEB_PORT = 3000;
|
let WEB_PORT = 3000;
|
||||||
|
|
||||||
const openaiSessionRef = { current: null };
|
const listenService = new ListenService();
|
||||||
|
// Make listenService globally accessible so other modules (e.g., windowManager, askService) can reuse the same instance
|
||||||
|
global.listenService = listenService;
|
||||||
let deeplink = null; // Initialize as null
|
let deeplink = null; // Initialize as null
|
||||||
let pendingDeepLinkUrl = null; // Store any deep link that arrives before initialization
|
let pendingDeepLinkUrl = null; // Store any deep link that arrives before initialization
|
||||||
|
|
||||||
@ -106,7 +108,7 @@ app.whenReady().then(async () => {
|
|||||||
sessionRepository.endAllActiveSessions();
|
sessionRepository.endAllActiveSessions();
|
||||||
|
|
||||||
authService.initialize();
|
authService.initialize();
|
||||||
setupLiveSummaryIpcHandlers(openaiSessionRef);
|
listenService.setupIpcHandlers();
|
||||||
askService.initialize();
|
askService.initialize();
|
||||||
setupGeneralIpcHandlers();
|
setupGeneralIpcHandlers();
|
||||||
})
|
})
|
||||||
@ -123,7 +125,7 @@ app.whenReady().then(async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
app.on('window-all-closed', () => {
|
app.on('window-all-closed', () => {
|
||||||
stopMacOSAudioCapture();
|
listenService.stopMacOSAudioCapture();
|
||||||
if (process.platform !== 'darwin') {
|
if (process.platform !== 'darwin') {
|
||||||
app.quit();
|
app.quit();
|
||||||
}
|
}
|
||||||
@ -131,7 +133,7 @@ app.on('window-all-closed', () => {
|
|||||||
|
|
||||||
app.on('before-quit', async () => {
|
app.on('before-quit', async () => {
|
||||||
console.log('[Shutdown] App is about to quit.');
|
console.log('[Shutdown] App is about to quit.');
|
||||||
stopMacOSAudioCapture();
|
listenService.stopMacOSAudioCapture();
|
||||||
await sessionRepository.endAllActiveSessions();
|
await sessionRepository.endAllActiveSessions();
|
||||||
databaseInitializer.close();
|
databaseInitializer.close();
|
||||||
});
|
});
|
||||||
@ -210,7 +212,8 @@ function setupGeneralIpcHandlers() {
|
|||||||
|
|
||||||
function setupWebDataHandlers() {
|
function setupWebDataHandlers() {
|
||||||
const sessionRepository = require('./common/repositories/session');
|
const sessionRepository = require('./common/repositories/session');
|
||||||
const listenRepository = require('./features/listen/repositories');
|
const sttRepository = require('./features/listen/stt/repositories');
|
||||||
|
const summaryRepository = require('./features/listen/summary/repositories');
|
||||||
const askRepository = require('./features/ask/repositories');
|
const askRepository = require('./features/ask/repositories');
|
||||||
const userRepository = require('./common/repositories/user');
|
const userRepository = require('./common/repositories/user');
|
||||||
const presetRepository = require('./common/repositories/preset');
|
const presetRepository = require('./common/repositories/preset');
|
||||||
@ -230,9 +233,9 @@ function setupWebDataHandlers() {
|
|||||||
result = null;
|
result = null;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
const transcripts = await listenRepository.getAllTranscriptsBySessionId(payload);
|
const transcripts = await sttRepository.getAllTranscriptsBySessionId(payload);
|
||||||
const ai_messages = await askRepository.getAllAiMessagesBySessionId(payload);
|
const ai_messages = await askRepository.getAllAiMessagesBySessionId(payload);
|
||||||
const summary = await listenRepository.getSummaryBySessionId(payload);
|
const summary = await summaryRepository.getSummaryBySessionId(payload);
|
||||||
result = { session, transcripts, ai_messages, summary };
|
result = { session, transcripts, ai_messages, summary };
|
||||||
break;
|
break;
|
||||||
case 'delete-session':
|
case 'delete-session':
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user