diff --git a/package.json b/package.json index 9365e5d..68cb6e8 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,8 @@ }, "license": "GPL-3.0", "dependencies": { + "@google/genai": "^1.8.0", + "@google/generative-ai": "^0.24.1", "axios": "^1.10.0", "better-sqlite3": "^9.4.3", "cors": "^2.8.5", diff --git a/src/app/ApiKeyHeader.js b/src/app/ApiKeyHeader.js index f9540f4..2e885f0 100644 --- a/src/app/ApiKeyHeader.js +++ b/src/app/ApiKeyHeader.js @@ -5,6 +5,7 @@ export class ApiKeyHeader extends LitElement { apiKey: { type: String }, isLoading: { type: Boolean }, errorMessage: { type: String }, + selectedProvider: { type: String }, }; static styles = css` @@ -45,11 +46,11 @@ export class ApiKeyHeader extends LitElement { .container { width: 285px; - height: 220px; + min-height: 260px; padding: 18px 20px; background: rgba(0, 0, 0, 0.3); border-radius: 16px; - overflow: hidden; + overflow: visible; position: relative; display: flex; flex-direction: column; @@ -108,7 +109,7 @@ export class ApiKeyHeader extends LitElement { font-weight: 500; /* Medium */ margin: 0; text-align: center; - flex-shrink: 0; /* 제목이 줄어들지 않도록 고정 */ + flex-shrink: 0; } .form-content { @@ -116,14 +117,14 @@ export class ApiKeyHeader extends LitElement { flex-direction: column; align-items: center; width: 100%; - margin-top: auto; /* 이 속성이 제목과 폼 사이의 공간을 만듭니다. */ + margin-top: auto; } .error-message { color: rgba(239, 68, 68, 0.9); font-weight: 500; font-size: 11px; - height: 14px; /* Reserve space to prevent layout shift */ + height: 14px; text-align: center; margin-bottom: 4px; } @@ -152,6 +153,46 @@ export class ApiKeyHeader extends LitElement { outline: none; } + .provider-select { + width: 100%; + height: 34px; + background: rgba(255, 255, 255, 0.1); + border-radius: 10px; + border: 1px solid rgba(255, 255, 255, 0.2); + padding: 0 10px; + color: white; + font-size: 12px; + font-weight: 400; + margin-bottom: 6px; + text-align: center; + cursor: pointer; + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; + background-image: url('data:image/svg+xml;charset=US-ASCII,%3Csvg%20width%3D%2714%27%20height%3D%278%27%20viewBox%3D%270%200%2014%208%27%20xmlns%3D%27http%3A//www.w3.org/2000/svg%27%3E%3Cpath%20d%3D%27M1%201l6%206%206-6%27%20stroke%3D%27%23ffffff%27%20stroke-width%3D%271.5%27%20fill%3D%27none%27%20fill-rule%3D%27evenodd%27/%3E%3C/svg%3E'); + background-repeat: no-repeat; + background-position: right 10px center; + background-size: 12px; + padding-right: 30px; + } + + .provider-select:hover { + background-color: rgba(255, 255, 255, 0.15); + border-color: rgba(255, 255, 255, 0.3); + } + + .provider-select:focus { + outline: none; + background-color: rgba(255, 255, 255, 0.15); + border-color: rgba(255, 255, 255, 0.4); + } + + .provider-select option { + background: #1a1a1a; + color: white; + padding: 5px; + } + .action-button { width: 100%; height: 34px; @@ -164,7 +205,7 @@ export class ApiKeyHeader extends LitElement { cursor: pointer; transition: background 0.15s ease; position: relative; - overflow: hidden; + overflow: visible; } .action-button::after { @@ -198,6 +239,15 @@ export class ApiKeyHeader extends LitElement { font-weight: 500; /* Medium */ margin: 10px 0; } + + .provider-label { + color: rgba(255, 255, 255, 0.7); + font-size: 11px; + font-weight: 400; + margin-bottom: 4px; + width: 100%; + text-align: left; + } `; constructor() { @@ -208,6 +258,7 @@ export class ApiKeyHeader extends LitElement { this.isLoading = false; this.errorMessage = ''; this.validatedApiKey = null; + this.selectedProvider = 'openai'; this.handleMouseMove = this.handleMouseMove.bind(this); this.handleMouseUp = this.handleMouseUp.bind(this); @@ -216,6 +267,8 @@ export class ApiKeyHeader extends LitElement { this.handleInput = this.handleInput.bind(this); this.handleAnimationEnd = this.handleAnimationEnd.bind(this); this.handleUsePicklesKey = this.handleUsePicklesKey.bind(this); + this.handleProviderChange = this.handleProviderChange.bind(this); + this.checkAndRequestPermissions = this.checkAndRequestPermissions.bind(this); } reset() { @@ -223,11 +276,12 @@ export class ApiKeyHeader extends LitElement { this.isLoading = false; this.errorMessage = ''; this.validatedApiKey = null; + this.selectedProvider = 'openai'; this.requestUpdate(); } async handleMouseDown(e) { - if (e.target.tagName === 'INPUT' || e.target.tagName === 'BUTTON') { + if (e.target.tagName === 'INPUT' || e.target.tagName === 'BUTTON' || e.target.tagName === 'SELECT') { return; } @@ -295,6 +349,13 @@ export class ApiKeyHeader extends LitElement { }); } + handleProviderChange(e) { + this.selectedProvider = e.target.value; + this.errorMessage = ''; + console.log('Provider changed to:', this.selectedProvider); + this.requestUpdate(); + } + handlePaste(e) { e.preventDefault(); this.errorMessage = ''; @@ -343,17 +404,17 @@ export class ApiKeyHeader extends LitElement { const apiKey = this.apiKey.trim(); let isValid = false; try { - const isValid = await this.validateApiKey(this.apiKey.trim()); - + const isValid = await this.validateApiKey(this.apiKey.trim(), this.selectedProvider); + if (isValid) { - console.log('API key valid - checking system permissions...'); - + console.log('API key valid – checking system permissions…'); const permissionResult = await this.checkAndRequestPermissions(); - + if (permissionResult.success) { - console.log('All permissions granted - starting slide out animation'); + console.log('All permissions granted – starting slide-out animation'); this.startSlideOutAnimation(); this.validatedApiKey = this.apiKey.trim(); + this.validatedProvider = this.selectedProvider; } else { this.errorMessage = permissionResult.error || 'Permission setup required'; console.log('Permission setup incomplete:', permissionResult); @@ -371,91 +432,107 @@ export class ApiKeyHeader extends LitElement { } } - async validateApiKey(apiKey) { + async validateApiKey(apiKey, provider = 'openai') { if (!apiKey || apiKey.length < 15) return false; - if (!apiKey.match(/^[A-Za-z0-9_-]+$/)) return false; + + if (provider === 'openai') { + if (!apiKey.match(/^[A-Za-z0-9_-]+$/)) return false; + + try { + console.log('Validating OpenAI API key...'); - try { - console.log('Validating API key with openai models endpoint...'); + const response = await fetch('https://api.openai.com/v1/models', { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + }); - const response = await fetch('https://api.openai.com/v1/models', { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${apiKey}`, - }, - }); + if (response.ok) { + const data = await response.json(); - if (response.ok) { - const data = await response.json(); - - const hasGPTModels = data.data && data.data.some(m => m.id.startsWith('gpt-')); - if (hasGPTModels) { - console.log('API key validation successful - GPT models available'); - return true; + const hasGPTModels = data.data && data.data.some(m => m.id.startsWith('gpt-')); + if (hasGPTModels) { + console.log('OpenAI API key validation successful'); + return true; + } else { + console.log('API key valid but no GPT models available'); + return false; + } } else { - console.log('API key valid but no GPT models available'); + const errorData = await response.json().catch(() => ({})); + console.log('API key validation failed:', response.status, errorData.error?.message || 'Unknown error'); return false; } - } else { - const errorData = await response.json().catch(() => ({})); - console.log('API key validation failed:', response.status, errorData.error?.message || 'Unknown error'); - return false; + } catch (error) { + console.error('API key validation network error:', error); + return apiKey.length >= 20; // Fallback for network issues + } + } else if (provider === 'gemini') { + // Gemini API keys typically start with 'AIza' + if (!apiKey.match(/^[A-Za-z0-9_-]+$/)) return false; + + try { + console.log('Validating Gemini API key...'); + + // Test the API key with a simple models list request + const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models?key=${apiKey}`); + + if (response.ok) { + const data = await response.json(); + if (data.models && data.models.length > 0) { + console.log('Gemini API key validation successful'); + return true; + } + } + + console.log('Gemini API key validation failed'); + return false; + } catch (error) { + console.error('Gemini API key validation network error:', error); + return apiKey.length >= 20; // Fallback } - } catch (error) { - console.error('API key validation network error:', error); - return apiKey.length >= 20; // Fallback for network issues } + + return false; } async checkAndRequestPermissions() { - if (!window.require) { - return { success: true }; - } - + if (!window.require) return { success: true }; + const { ipcRenderer } = window.require('electron'); - + try { const permissions = await ipcRenderer.invoke('check-system-permissions'); console.log('[Permissions] Current status:', permissions); - - if (!permissions.needsSetup) { - return { success: true }; - } - + + if (!permissions.needsSetup) return { success: true }; + if (!permissions.microphone) { - console.log('[Permissions] Requesting microphone permission...'); + console.log('[Permissions] Requesting microphone permission…'); const micResult = await ipcRenderer.invoke('request-microphone-permission'); - if (!micResult.success) { - console.log('[Permissions] Microphone permission denied'); await ipcRenderer.invoke('open-system-preferences', 'microphone'); - return { - success: false, - error: 'Please grant microphone access in System Preferences' + return { + success: false, + error: 'Please grant microphone access in System Preferences', }; } } - + if (!permissions.screen) { - console.log('[Permissions] Screen recording permission needed'); + console.log('[Permissions] Screen-recording permission needed'); await ipcRenderer.invoke('open-system-preferences', 'screen-recording'); - - this.errorMessage = 'Please grant screen recording permission and try again'; - this.requestUpdate(); - - return { - success: false, - error: 'Please grant screen recording access in System Preferences' + return { + success: false, + error: 'Please grant screen recording access in System Preferences', }; } - + return { success: true }; - } catch (error) { - console.error('[Permissions] Error checking/requesting permissions:', error); - return { - success: false, - error: 'Failed to check permissions' - }; + } catch (err) { + console.error('[Permissions] Error checking/requesting permissions:', err); + return { success: false, error: 'Failed to check permissions' }; } } @@ -489,9 +566,13 @@ export class ApiKeyHeader extends LitElement { if (this.validatedApiKey) { if (window.require) { - window.require('electron').ipcRenderer.invoke('api-key-validated', this.validatedApiKey); + window.require('electron').ipcRenderer.invoke('api-key-validated', { + apiKey: this.validatedApiKey, + provider: this.validatedProvider || 'openai' + }); } this.validatedApiKey = null; + this.validatedProvider = null; } } } @@ -510,6 +591,7 @@ export class ApiKeyHeader extends LitElement { render() { const isButtonDisabled = this.isLoading || !this.apiKey || !this.apiKey.trim(); + console.log('Rendering with provider:', this.selectedProvider); return html`
@@ -522,10 +604,21 @@ export class ApiKeyHeader extends LitElement {
${this.errorMessage}
+
Select AI Provider:
+ { console.log('[HeaderController] Received request to sign out.'); try { + this.hasApiKey = false; await signOut(auth); } catch (error) { console.error('[HeaderController] Sign out failed', error); @@ -316,7 +317,7 @@ class HeaderTransitionManager { if (!window.require) return; return window .require('electron') - .ipcRenderer.invoke('resize-header-window', { width: 285, height: 220 }) + .ipcRenderer.invoke('resize-header-window', { width: 285, height: 300 }) .catch(() => {}); } diff --git a/src/common/config/schema.js b/src/common/config/schema.js index 7e50323..19e0d8c 100644 --- a/src/common/config/schema.js +++ b/src/common/config/schema.js @@ -5,7 +5,8 @@ const LATEST_SCHEMA = { { name: 'display_name', type: 'TEXT NOT NULL' }, { name: 'email', type: 'TEXT NOT NULL' }, { name: 'created_at', type: 'INTEGER' }, - { name: 'api_key', type: 'TEXT' } + { name: 'api_key', type: 'TEXT' }, + { name: 'provider', type: 'TEXT DEFAULT \'openai\'' } ] }, sessions: { diff --git a/src/common/services/aiProviderService.js b/src/common/services/aiProviderService.js new file mode 100644 index 0000000..fe24e47 --- /dev/null +++ b/src/common/services/aiProviderService.js @@ -0,0 +1,377 @@ +const { createOpenAiGenerativeClient, getOpenAiGenerativeModel } = require('./openAiClient.js'); +const { createGeminiClient, getGeminiGenerativeModel, createGeminiChat } = require('./googleGeminiClient.js'); + +/** + * Creates an AI client based on the provider + * @param {string} apiKey - The API key + * @param {string} provider - The provider ('openai' or 'gemini') + * @returns {object} The AI client + */ +function createAIClient(apiKey, provider = 'openai') { + switch (provider) { + case 'openai': + return createOpenAiGenerativeClient(apiKey); + case 'gemini': + return createGeminiClient(apiKey); + default: + throw new Error(`Unsupported AI provider: ${provider}`); + } +} + +/** + * Gets a generative model based on the provider + * @param {object} client - The AI client + * @param {string} provider - The provider ('openai' or 'gemini') + * @param {string} model - The model name (optional) + * @returns {object} The model object + */ +function getGenerativeModel(client, provider = 'openai', model) { + switch (provider) { + case 'openai': + return getOpenAiGenerativeModel(client, model || 'gpt-4.1'); + case 'gemini': + return getGeminiGenerativeModel(client, model || 'gemini-2.5-flash'); + default: + throw new Error(`Unsupported AI provider: ${provider}`); + } +} + +/** + * Makes a chat completion request based on the provider + * @param {object} params - Request parameters + * @returns {Promise} The completion response + */ +async function makeChatCompletion({ apiKey, provider = 'openai', messages, temperature = 0.7, maxTokens = 1024, model, stream = false }) { + if (provider === 'openai') { + const fetchUrl = 'https://api.openai.com/v1/chat/completions'; + const response = await fetch(fetchUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: model || 'gpt-4.1', + messages, + temperature, + max_tokens: maxTokens, + stream, + }), + }); + + if (!response.ok) { + throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`); + } + + if (stream) { + return response; + } + + const result = await response.json(); + return { + content: result.choices[0].message.content.trim(), + raw: result + }; + } else if (provider === 'gemini') { + const client = createGeminiClient(apiKey); + const genModel = getGeminiGenerativeModel(client, model || 'gemini-2.5-flash'); + + // Convert OpenAI format messages to Gemini format + const parts = []; + for (const message of messages) { + if (message.role === 'system') { + parts.push(message.content); + } else if (message.role === 'user') { + if (typeof message.content === 'string') { + parts.push(message.content); + } else if (Array.isArray(message.content)) { + // Handle multimodal content + for (const part of message.content) { + if (part.type === 'text') { + parts.push(part.text); + } else if (part.type === 'image_url' && part.image_url?.url) { + // Extract base64 data from data URL + const base64Match = part.image_url.url.match(/^data:(.+);base64,(.+)$/); + if (base64Match) { + parts.push({ + inlineData: { + mimeType: base64Match[1], + data: base64Match[2] + } + }); + } + } + } + } + } + } + + const result = await genModel.generateContent(parts); + return { + content: result.response.text(), + raw: result + }; + } else { + throw new Error(`Unsupported AI provider: ${provider}`); + } +} + +/** + * Makes a chat completion request with Portkey support + * @param {object} params - Request parameters including Portkey options + * @returns {Promise} The completion response + */ +async function makeChatCompletionWithPortkey({ + apiKey, + provider = 'openai', + messages, + temperature = 0.7, + maxTokens = 1024, + model, + usePortkey = false, + portkeyVirtualKey = null +}) { + if (!usePortkey) { + return makeChatCompletion({ apiKey, provider, messages, temperature, maxTokens, model }); + } + + // Portkey is only supported for OpenAI currently + if (provider !== 'openai') { + console.warn('Portkey is only supported for OpenAI provider, falling back to direct API'); + return makeChatCompletion({ apiKey, provider, messages, temperature, maxTokens, model }); + } + + const fetchUrl = 'https://api.portkey.ai/v1/chat/completions'; + const response = await fetch(fetchUrl, { + method: 'POST', + headers: { + 'x-portkey-api-key': 'gRv2UGRMq6GGLJ8aVEB4e7adIewu', + 'x-portkey-virtual-key': portkeyVirtualKey || apiKey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: model || 'gpt-4.1', + messages, + temperature, + max_tokens: maxTokens, + }), + }); + + if (!response.ok) { + throw new Error(`Portkey API error: ${response.status} ${response.statusText}`); + } + + const result = await response.json(); + return { + content: result.choices[0].message.content.trim(), + raw: result + }; +} + +/** + * Makes a streaming chat completion request + * @param {object} params - Request parameters + * @returns {Promise} The streaming response + */ +async function makeStreamingChatCompletion({ apiKey, provider = 'openai', messages, temperature = 0.7, maxTokens = 1024, model }) { + if (provider === 'openai') { + const fetchUrl = 'https://api.openai.com/v1/chat/completions'; + const response = await fetch(fetchUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: model || 'gpt-4.1', + messages, + temperature, + max_tokens: maxTokens, + stream: true, + }), + }); + + if (!response.ok) { + throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`); + } + + return response; + } else if (provider === 'gemini') { + console.log('[AIProviderService] Starting Gemini streaming request'); + // Gemini streaming requires a different approach + // We'll create a ReadableStream that mimics OpenAI's SSE format + const geminiClient = createGeminiClient(apiKey); + + // Extract system instruction if present + let systemInstruction = ''; + const nonSystemMessages = []; + + for (const msg of messages) { + if (msg.role === 'system') { + systemInstruction = msg.content; + } else { + nonSystemMessages.push(msg); + } + } + + const chat = createGeminiChat(geminiClient, model || 'gemini-2.0-flash-exp', { + temperature, + maxOutputTokens: maxTokens || 8192, + systemInstruction: systemInstruction || undefined + }); + + // Create a ReadableStream to handle Gemini's streaming + const stream = new ReadableStream({ + async start(controller) { + try { + console.log('[AIProviderService] Processing messages for Gemini:', nonSystemMessages.length, 'messages (excluding system)'); + + // Get the last user message + const lastMessage = nonSystemMessages[nonSystemMessages.length - 1]; + let lastUserMessage = lastMessage.content; + + // Handle case where content might be an array (multimodal) + if (Array.isArray(lastUserMessage)) { + // Extract text content from array + const textParts = lastUserMessage.filter(part => + typeof part === 'string' || (part && part.type === 'text') + ); + lastUserMessage = textParts.map(part => + typeof part === 'string' ? part : part.text + ).join(' '); + } + + console.log('[AIProviderService] Sending message to Gemini:', + typeof lastUserMessage === 'string' ? lastUserMessage.substring(0, 100) + '...' : 'multimodal content'); + + // Prepare the message content for Gemini + let geminiContent = []; + + // Handle multimodal content properly + if (Array.isArray(lastMessage.content)) { + for (const part of lastMessage.content) { + if (typeof part === 'string') { + geminiContent.push(part); + } else if (part.type === 'text') { + geminiContent.push(part.text); + } else if (part.type === 'image_url' && part.image_url) { + // Convert base64 image to Gemini format + const base64Data = part.image_url.url.split(',')[1]; + geminiContent.push({ + inlineData: { + mimeType: 'image/png', + data: base64Data + } + }); + } + } + } else { + geminiContent = [lastUserMessage]; + } + + console.log('[AIProviderService] Prepared Gemini content:', + geminiContent.length, 'parts'); + + // Stream the response + let chunkCount = 0; + let totalContent = ''; + + for await (const chunk of chat.sendMessageStream(geminiContent)) { + chunkCount++; + const chunkText = chunk.text || ''; + totalContent += chunkText; + + // Format as SSE data + const data = JSON.stringify({ + choices: [{ + delta: { + content: chunkText + } + }] + }); + controller.enqueue(new TextEncoder().encode(`data: ${data}\n\n`)); + } + + console.log(`[AIProviderService] Streamed ${chunkCount} chunks, total length: ${totalContent.length} chars`); + + // Send the final done message + controller.enqueue(new TextEncoder().encode('data: [DONE]\n\n')); + controller.close(); + console.log('[AIProviderService] Gemini streaming completed successfully'); + } catch (error) { + console.error('[AIProviderService] Gemini streaming error:', error); + controller.error(error); + } + } + }); + + // Create a Response object with the stream + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive' + } + }); + } else { + throw new Error(`Unsupported AI provider: ${provider}`); + } +} + +/** + * Makes a streaming chat completion request with Portkey support + * @param {object} params - Request parameters + * @returns {Promise} The streaming response + */ +async function makeStreamingChatCompletionWithPortkey({ + apiKey, + provider = 'openai', + messages, + temperature = 0.7, + maxTokens = 1024, + model, + usePortkey = false, + portkeyVirtualKey = null +}) { + if (!usePortkey) { + return makeStreamingChatCompletion({ apiKey, provider, messages, temperature, maxTokens, model }); + } + + // Portkey is only supported for OpenAI currently + if (provider !== 'openai') { + console.warn('Portkey is only supported for OpenAI provider, falling back to direct API'); + return makeStreamingChatCompletion({ apiKey, provider, messages, temperature, maxTokens, model }); + } + + const fetchUrl = 'https://api.portkey.ai/v1/chat/completions'; + const response = await fetch(fetchUrl, { + method: 'POST', + headers: { + 'x-portkey-api-key': 'gRv2UGRMq6GGLJ8aVEB4e7adIewu', + 'x-portkey-virtual-key': portkeyVirtualKey || apiKey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: model || 'gpt-4.1', + messages, + temperature, + max_tokens: maxTokens, + stream: true, + }), + }); + + if (!response.ok) { + throw new Error(`Portkey API error: ${response.status} ${response.statusText}`); + } + + return response; +} + +module.exports = { + createAIClient, + getGenerativeModel, + makeChatCompletion, + makeChatCompletionWithPortkey, + makeStreamingChatCompletion, + makeStreamingChatCompletionWithPortkey +}; \ No newline at end of file diff --git a/src/common/services/googleGeminiClient.js b/src/common/services/googleGeminiClient.js new file mode 100644 index 0000000..877c82e --- /dev/null +++ b/src/common/services/googleGeminiClient.js @@ -0,0 +1,171 @@ +const { GoogleGenerativeAI } = require('@google/generative-ai'); +const { GoogleGenAI } = require('@google/genai'); + +/** + * Creates and returns a Google Gemini client instance for generative AI. + * @param {string} apiKey - The API key for authentication. + * @returns {GoogleGenerativeAI} The initialized Gemini client. + */ +function createGeminiClient(apiKey) { + return new GoogleGenerativeAI(apiKey); +} + +/** + * Gets a Gemini model for text/image generation. + * @param {GoogleGenerativeAI} client - The Gemini client instance. + * @param {string} [model='gemini-2.5-flash'] - The name for the text/vision model. + * @returns {object} Model object with generateContent method + */ +function getGeminiGenerativeModel(client, model = 'gemini-2.5-flash') { + const genAI = client; + const geminiModel = genAI.getGenerativeModel({ model: model }); + + return { + generateContent: async (parts) => { + let systemPrompt = ''; + let userContent = []; + + for (const part of parts) { + if (typeof part === 'string') { + if (systemPrompt === '' && part.includes('You are')) { + systemPrompt = part; + } else { + userContent.push(part); + } + } else if (part.inlineData) { + // Convert base64 image data to Gemini format + userContent.push({ + inlineData: { + mimeType: part.inlineData.mimeType, + data: part.inlineData.data + } + }); + } + } + + // Prepare content array + const content = []; + + // Add system instruction if present + if (systemPrompt) { + // For Gemini, we'll prepend system prompt to user content + content.push(systemPrompt + '\n\n' + userContent[0]); + content.push(...userContent.slice(1)); + } else { + content.push(...userContent); + } + + try { + const result = await geminiModel.generateContent(content); + const response = await result.response; + + return { + response: { + text: () => response.text() + } + }; + } catch (error) { + console.error('Gemini API error:', error); + throw error; + } + } + }; +} + +/** + * Creates a Gemini chat session for multi-turn conversations. + * @param {GoogleGenerativeAI} client - The Gemini client instance. + * @param {string} [model='gemini-2.5-flash'] - The model to use. + * @param {object} [config={}] - Configuration options. + * @returns {object} Chat session object + */ +function createGeminiChat(client, model = 'gemini-2.5-flash', config = {}) { + const genAI = client; + const geminiModel = genAI.getGenerativeModel({ + model: model, + systemInstruction: config.systemInstruction + }); + + const chat = geminiModel.startChat({ + history: config.history || [], + generationConfig: { + temperature: config.temperature || 0.7, + maxOutputTokens: config.maxOutputTokens || 8192, + } + }); + + return { + sendMessage: async (message) => { + const result = await chat.sendMessage(message); + const response = await result.response; + return { + text: response.text() + }; + }, + sendMessageStream: async function* (message) { + const result = await chat.sendMessageStream(message); + for await (const chunk of result.stream) { + yield { + text: chunk.text() + }; + } + }, + getHistory: () => chat.getHistory() + }; +} + +// async function connectToGeminiSession(apiKey, { language = 'en-US', callbacks = {} } = {}) { +// const liveClient = new GoogleGenAI({ +// vertexai: false, // Vertex AI 사용 안함 +// apiKey, +// }); + +// // 라이브 STT 세션 열기 +// const session = await liveClient.live.connect({ +// model: 'gemini-live-2.5-flash-preview', +// callbacks, +// config: { +// inputAudioTranscription: {}, // 실시간 STT 필수 +// speechConfig: { languageCode: language }, +// }, +// }); + +// return { +// sendRealtimeInput: async data => session.send({ +// audio: { data, mimeType: 'audio/pcm;rate=24000' } +// }), +// close: async () => session.close(), +// }; +// } + +async function connectToGeminiSession(apiKey, { language = 'en-US', callbacks = {} } = {}) { + // ① 옛날 스타일 helper 재사용 + const liveClient = new GoogleGenAI({ vertexai: false, apiKey }); + + // ② 언어 코드 강제 BCP-47 변환 + const lang = language.includes('-') ? language : `${language}-US`; + + const session = await liveClient.live.connect({ + model: 'gemini-live-2.5-flash-preview', + callbacks, + config: { + inputAudioTranscription: {}, + speechConfig: { languageCode: lang }, + }, + }); + + // ③ SDK 0.5+ : sendRealtimeInput 가 정식 이름 + return { + sendRealtimeInput: async payload => session.sendRealtimeInput(payload), + close: async () => session.close(), + }; + } + + + +module.exports = { + createGeminiClient, + getGeminiGenerativeModel, + createGeminiChat, + connectToGeminiSession, +}; \ No newline at end of file diff --git a/src/common/services/sqliteClient.js b/src/common/services/sqliteClient.js index e1f4c86..08ce9db 100644 --- a/src/common/services/sqliteClient.js +++ b/src/common/services/sqliteClient.js @@ -226,17 +226,17 @@ class SQLiteClient { }); } - async saveApiKey(apiKey, uid = this.defaultUserId) { + async saveApiKey(apiKey, uid = this.defaultUserId, provider = 'openai') { return new Promise((resolve, reject) => { this.db.run( - 'UPDATE users SET api_key = ? WHERE uid = ?', - [apiKey, uid], + 'UPDATE users SET api_key = ?, provider = ? WHERE uid = ?', + [apiKey, provider, uid], function(err) { if (err) { console.error('SQLite: Failed to save API key:', err); reject(err); } else { - console.log(`SQLite: API key saved for user ${uid}.`); + console.log(`SQLite: API key saved for user ${uid} with provider ${provider}.`); resolve({ changes: this.changes }); } } diff --git a/src/electron/windowManager.js b/src/electron/windowManager.js index 5f217b1..9dc8b5f 100644 --- a/src/electron/windowManager.js +++ b/src/electron/windowManager.js @@ -90,6 +90,11 @@ function createFeatureWindows(header) { ask.setVisibleOnAllWorkspaces(true,{visibleOnFullScreen:true}); ask.loadFile(path.join(__dirname,'../app/content.html'),{query:{view:'ask'}}); ask.on('blur',()=>ask.webContents.send('window-blur')); + + // Open DevTools in development + if (!app.isPackaged) { + ask.webContents.openDevTools({ mode: 'detach' }); + } windowPool.set('ask', ask); // settings @@ -102,6 +107,10 @@ function createFeatureWindows(header) { } function destroyFeatureWindows() { + if (settingsHideTimer) { + clearTimeout(settingsHideTimer); + settingsHideTimer = null; + } featureWindows.forEach(name=>{ const win = windowPool.get(name); if (win && !win.isDestroyed()) win.destroy(); @@ -224,7 +233,7 @@ class WindowLayoutManager { const PAD = 8; - /* ① 헤더 중심 X를 "디스플레이 기준 상대좌표"로 변환 */ + /* ① 헤더 중심 X를 “디스플레이 기준 상대좌표”로 변환 */ const headerCenterXRel = headerBounds.x - workAreaX + headerBounds.width / 2; let askBounds = askVisible ? ask.getBounds() : null; @@ -418,49 +427,6 @@ class SmoothMovementManager { this.hiddenPosition = null; this.lastVisiblePosition = null; this.currentDisplayId = null; - this.currentAnimationTimer = null; - this.animationAbortController = null; - this.animationFrameRate = 16; // ~60fps - } - - safeSetPosition(window, x, y) { - if (!window || window.isDestroyed()) { - return false; - } - - let safeX = Number.isFinite(x) ? Math.round(x) : 0; - let safeY = Number.isFinite(y) ? Math.round(y) : 0; - - if (Object.is(safeX, -0)) safeX = 0; - if (Object.is(safeY, -0)) safeY = 0; - - safeX = parseInt(safeX, 10); - safeY = parseInt(safeY, 10); - - if (!Number.isInteger(safeX) || !Number.isInteger(safeY)) { - console.error('[Movement] Invalid position after conversion:', { x: safeX, y: safeY, originalX: x, originalY: y }); - return false; - } - - try { - window.setPosition(safeX, safeY); - return true; - } catch (err) { - console.error('[Movement] setPosition failed with values:', { x: safeX, y: safeY }, err); - return false; - } - } - - cancelCurrentAnimation() { - if (this.currentAnimationTimer) { - clearTimeout(this.currentAnimationTimer); - this.currentAnimationTimer = null; - } - if (this.animationAbortController) { - this.animationAbortController.abort(); - this.animationAbortController = null; - } - this.isAnimating = false; } moveToDisplay(displayId) { @@ -499,83 +465,50 @@ class SmoothMovementManager { this.currentDisplayId = targetDisplay.id; } - hideToEdge(edge, callback, errorCallback) { + hideToEdge(edge, callback) { const header = windowPool.get('header'); - if (!header || !header.isVisible()) { - if (errorCallback) errorCallback(new Error('Header not available or not visible')); - return; - } - // cancel current animation - this.cancelCurrentAnimation(); + if (!header || !header.isVisible() || this.isAnimating) return; console.log(`[Movement] Hiding to ${edge} edge`); - let currentBounds; - try { - currentBounds = header.getBounds(); - } catch (err) { - console.error('[Movement] Failed to get header bounds:', err); - if (errorCallback) errorCallback(err); - return; - } - + const currentBounds = header.getBounds(); this.lastVisiblePosition = { x: currentBounds.x, y: currentBounds.y }; this.headerPosition = { x: currentBounds.x, y: currentBounds.y }; const display = getCurrentDisplay(header); const { width: screenWidth, height: screenHeight } = display.workAreaSize; const { x: workAreaX, y: workAreaY } = display.workArea; + const headerBounds = header.getBounds(); let targetX = this.headerPosition.x; let targetY = this.headerPosition.y; switch (edge) { case 'top': - targetY = workAreaY - currentBounds.height - 20; + targetY = workAreaY - headerBounds.height - 20; break; case 'bottom': targetY = workAreaY + screenHeight + 20; break; case 'left': - targetX = workAreaX - currentBounds.width - 20; + targetX = workAreaX - headerBounds.width - 20; break; case 'right': targetX = workAreaX + screenWidth + 20; break; } - // 대상 위치 유효성 검사 - if (!Number.isFinite(targetX) || !Number.isFinite(targetY)) { - console.error('[Movement] Invalid target position:', { targetX, targetY }); - if (errorCallback) errorCallback(new Error('Invalid target position')); - return; - } - this.hiddenPosition = { x: targetX, y: targetY, edge }; - // create AbortController - this.animationAbortController = new AbortController(); - const signal = this.animationAbortController.signal; - this.isAnimating = true; const startX = this.headerPosition.x; const startY = this.headerPosition.y; - const duration = 300; + const duration = 400; const startTime = Date.now(); const animate = () => { - // check aborted - if (signal.aborted) { + if (!header || typeof header.setPosition !== 'function' || header.isDestroyed()) { this.isAnimating = false; - if (errorCallback) errorCallback(new Error('Animation aborted')); - return; - } - - // check destroyed - if (!header || header.isDestroyed()) { - this.isAnimating = false; - this.currentAnimationTimer = null; - if (errorCallback) errorCallback(new Error('Window destroyed during animation')); return; } @@ -586,33 +519,44 @@ class SmoothMovementManager { const currentX = startX + (targetX - startX) * eased; const currentY = startY + (targetY - startY) * eased; - // set position safe - const success = this.safeSetPosition(header, currentX, currentY); - if (!success) { + // Validate computed positions before using + if (!Number.isFinite(currentX) || !Number.isFinite(currentY)) { + console.error('[Movement] Invalid animation values for hide:', { + currentX, currentY, progress, eased, startX, startY, targetX, targetY + }); + this.isAnimating = false; + return; + } + + // Safely call setPosition + try { + header.setPosition(Math.round(currentX), Math.round(currentY)); + } catch (err) { + console.error('[Movement] Failed to set position:', err); this.isAnimating = false; - this.currentAnimationTimer = null; - if (errorCallback) errorCallback(new Error('Failed to set position')); return; } if (progress < 1) { - this.currentAnimationTimer = setTimeout(animate, this.animationFrameRate); + setTimeout(animate, 8); } else { this.headerPosition = { x: targetX, y: targetY }; - - // set final position - this.safeSetPosition(header, targetX, targetY); + + if (Number.isFinite(targetX) && Number.isFinite(targetY)) { + try { + header.setPosition(Math.round(targetX), Math.round(targetY)); + } catch (err) { + console.error('[Movement] Failed to set final position:', err); + } + } this.isAnimating = false; - this.currentAnimationTimer = null; - this.animationAbortController = null; - if (typeof callback === 'function' && !signal.aborted) { + if (typeof callback === 'function') { try { callback(); } catch (err) { console.error('[Movement] Callback error:', err); - if (errorCallback) errorCallback(err); } } @@ -620,62 +564,30 @@ class SmoothMovementManager { } }; - try { - animate(); - } catch (err) { - console.error('[Movement] Animation start error:', err); - this.isAnimating = false; - if (errorCallback) errorCallback(err); - } + animate(); } - showFromEdge(callback, errorCallback) { + showFromEdge(callback) { const header = windowPool.get('header'); - if (!header || !this.hiddenPosition || !this.lastVisiblePosition) { - if (errorCallback) errorCallback(new Error('Cannot show - missing required data')); - return; - } - - this.cancelCurrentAnimation(); + if (!header || this.isAnimating || !this.hiddenPosition || !this.lastVisiblePosition) return; console.log(`[Movement] Showing from ${this.hiddenPosition.edge} edge`); - if (!this.safeSetPosition(header, this.hiddenPosition.x, this.hiddenPosition.y)) { - if (errorCallback) errorCallback(new Error('Failed to set initial position')); - return; - } - + header.setPosition(this.hiddenPosition.x, this.hiddenPosition.y); this.headerPosition = { x: this.hiddenPosition.x, y: this.hiddenPosition.y }; const targetX = this.lastVisiblePosition.x; const targetY = this.lastVisiblePosition.y; - if (!Number.isFinite(targetX) || !Number.isFinite(targetY)) { - console.error('[Movement] Invalid target position for show:', { targetX, targetY }); - if (errorCallback) errorCallback(new Error('Invalid target position for show')); - return; - } - - this.animationAbortController = new AbortController(); - const signal = this.animationAbortController.signal; - this.isAnimating = true; const startX = this.headerPosition.x; const startY = this.headerPosition.y; - const duration = 400; + const duration = 500; const startTime = Date.now(); const animate = () => { - if (signal.aborted) { - this.isAnimating = false; - if (errorCallback) errorCallback(new Error('Animation aborted')); - return; - } - if (!header || header.isDestroyed()) { this.isAnimating = false; - this.currentAnimationTimer = null; - if (errorCallback) errorCallback(new Error('Window destroyed during animation')); return; } @@ -689,47 +601,34 @@ class SmoothMovementManager { const currentX = startX + (targetX - startX) * eased; const currentY = startY + (targetY - startY) * eased; - const success = this.safeSetPosition(header, currentX, currentY); - if (!success) { + if (!Number.isFinite(currentX) || !Number.isFinite(currentY)) { + console.error('[Movement] Invalid animation values for show:', { currentX, currentY, progress, eased }); this.isAnimating = false; - this.currentAnimationTimer = null; - if (errorCallback) errorCallback(new Error('Failed to set position')); return; } + header.setPosition(Math.round(currentX), Math.round(currentY)); + if (progress < 1) { - this.currentAnimationTimer = setTimeout(animate, this.animationFrameRate); + setTimeout(animate, 8); } else { this.headerPosition = { x: targetX, y: targetY }; - this.safeSetPosition(header, targetX, targetY); - + this.headerPosition = { x: targetX, y: targetY }; + if (Number.isFinite(targetX) && Number.isFinite(targetY)) { + header.setPosition(Math.round(targetX), Math.round(targetY)); + } this.isAnimating = false; - this.currentAnimationTimer = null; - this.animationAbortController = null; this.hiddenPosition = null; this.lastVisiblePosition = null; - if (typeof callback === 'function' && !signal.aborted) { - try { - callback(); - } catch (err) { - console.error('[Movement] Show callback error:', err); - if (errorCallback) errorCallback(err); - } - } + if (callback) callback(); console.log(`[Movement] Show from edge completed`); } }; - try { - animate(); - } catch (err) { - console.error('[Movement] Animation start error:', err); - this.isAnimating = false; - if (errorCallback) errorCallback(err); - } + animate(); } moveStep(direction) { @@ -792,9 +691,6 @@ class SmoothMovementManager { } animateToPosition(header, targetX, targetY) { - // cancel animation - this.cancelCurrentAnimation(); - this.isAnimating = true; const startX = this.headerPosition.x; @@ -807,14 +703,9 @@ class SmoothMovementManager { return; } - - this.animationAbortController = new AbortController(); - const signal = this.animationAbortController.signal; - const animate = () => { - if (signal.aborted || !header || header.isDestroyed()) { + if (!header || header.isDestroyed()) { this.isAnimating = false; - this.currentAnimationTimer = null; return; } @@ -826,24 +717,24 @@ class SmoothMovementManager { const currentX = startX + (targetX - startX) * eased; const currentY = startY + (targetY - startY) * eased; - const success = this.safeSetPosition(header, currentX, currentY); - if (!success) { + if (!Number.isFinite(currentX) || !Number.isFinite(currentY)) { + console.error('[Movement] Invalid animation values:', { currentX, currentY, progress, eased }); this.isAnimating = false; - this.currentAnimationTimer = null; return; } + header.setPosition(Math.round(currentX), Math.round(currentY)); + if (progress < 1) { - this.currentAnimationTimer = setTimeout(animate, this.animationFrameRate); + setTimeout(animate, 8); } else { this.headerPosition = { x: targetX, y: targetY }; - - - this.safeSetPosition(header, targetX, targetY); - + if (Number.isFinite(targetX) && Number.isFinite(targetY)) { + header.setPosition(Math.round(targetX), Math.round(targetY)); + } else { + console.warn('[Movement] Final position invalid, skip setPosition:', { targetX, targetY }); + } this.isAnimating = false; - this.currentAnimationTimer = null; - this.animationAbortController = null; updateLayout(); @@ -856,23 +747,16 @@ class SmoothMovementManager { moveToEdge(direction) { const header = windowPool.get('header'); - if (!header || !header.isVisible()) return; - this.cancelCurrentAnimation(); + if (!header || !header.isVisible() || this.isAnimating) return; console.log(`[Movement] Move to edge: ${direction}`); const display = getCurrentDisplay(header); const { width, height } = display.workAreaSize; const { x: workAreaX, y: workAreaY } = display.workArea; - - let currentBounds; - try { - currentBounds = header.getBounds(); - } catch (err) { - console.error('[Movement] Failed to get header bounds:', err); - return; - } + const headerBounds = header.getBounds(); + const currentBounds = header.getBounds(); let targetX = currentBounds.x; let targetY = currentBounds.y; @@ -881,26 +765,23 @@ class SmoothMovementManager { targetX = workAreaX; break; case 'right': - targetX = workAreaX + width - currentBounds.width; + targetX = workAreaX + width - headerBounds.width; break; case 'up': targetY = workAreaY; break; case 'down': - targetY = workAreaY + height - currentBounds.height; + targetY = workAreaY + height - headerBounds.height; break; } this.headerPosition = { x: currentBounds.x, y: currentBounds.y }; - this.animationAbortController = new AbortController(); - const signal = this.animationAbortController.signal; - this.isAnimating = true; const startX = this.headerPosition.x; const startY = this.headerPosition.y; - const duration = 350; - const startTime = Date.now(); + const duration = 400; + const startTime = Date.now(); // 이 줄을 animate 함수 정의 전으로 이동 if (!Number.isFinite(targetX) || !Number.isFinite(targetY) || !Number.isFinite(startX) || !Number.isFinite(startY)) { console.error('[Movement] Invalid edge position values:', { startX, startY, targetX, targetY }); @@ -909,9 +790,8 @@ class SmoothMovementManager { } const animate = () => { - if (signal.aborted || !header || header.isDestroyed()) { + if (!header || header.isDestroyed()) { this.isAnimating = false; - this.currentAnimationTimer = null; return; } @@ -923,24 +803,22 @@ class SmoothMovementManager { const currentX = startX + (targetX - startX) * eased; const currentY = startY + (targetY - startY) * eased; - - const success = this.safeSetPosition(header, currentX, currentY); - if (!success) { + if (!Number.isFinite(currentX) || !Number.isFinite(currentY)) { + console.error('[Movement] Invalid edge animation values:', { currentX, currentY, progress, eased }); this.isAnimating = false; - this.currentAnimationTimer = null; return; } - if (progress < 1) { - this.currentAnimationTimer = setTimeout(animate, this.animationFrameRate); - } else { + header.setPosition(Math.round(currentX), Math.round(currentY)); - this.safeSetPosition(header, targetX, targetY); - + if (progress < 1) { + setTimeout(animate, 8); + } else { + if (Number.isFinite(targetX) && Number.isFinite(targetY)) { + header.setPosition(Math.round(targetX), Math.round(targetY)); + } this.headerPosition = { x: targetX, y: targetY }; this.isAnimating = false; - this.currentAnimationTimer = null; - this.animationAbortController = null; updateLayout(); @@ -960,7 +838,6 @@ class SmoothMovementManager { } destroy() { - this.cancelCurrentAnimation(); this.isAnimating = false; console.log('[Movement] Destroyed'); } @@ -969,219 +846,75 @@ class SmoothMovementManager { const layoutManager = new WindowLayoutManager(); let movementManager = null; -function isWindowSafe(window) { - return window && !window.isDestroyed() && typeof window.getBounds === 'function'; -} - -function safeWindowOperation(window, operation, fallback = null) { - if (!isWindowSafe(window)) { - console.warn('[WindowManager] Window not safe for operation'); - return fallback; - } - - try { - return operation(window); - } catch (error) { - console.error('[WindowManager] Window operation failed:', error); - return fallback; - } -} - -function safeSetPosition(window, x, y) { - return safeWindowOperation(window, (win) => { - win.setPosition(Math.round(x), Math.round(y)); - return true; - }, false); -} - -function safeGetBounds(window) { - return safeWindowOperation(window, (win) => win.getBounds(), null); -} - -function safeShow(window) { - return safeWindowOperation(window, (win) => { - win.show(); - return true; - }, false); -} - -function safeHide(window) { - return safeWindowOperation(window, (win) => { - win.hide(); - return true; - }, false); -} - -let toggleState = { - isToggling: false, - lastToggleTime: 0, - pendingToggle: null, - toggleDebounceTimer: null, - failsafeTimer: null -}; - function toggleAllWindowsVisibility() { - const now = Date.now(); - const timeSinceLastToggle = now - toggleState.lastToggleTime; - - if (timeSinceLastToggle < 200) { - console.log('[Visibility] Toggle ignored - too fast (debounced)'); - return; - } - if (toggleState.isToggling) { - console.log('[Visibility] Toggle in progress, queueing request'); - - if (toggleState.toggleDebounceTimer) { - clearTimeout(toggleState.toggleDebounceTimer); - } - - toggleState.toggleDebounceTimer = setTimeout(() => { - toggleState.toggleDebounceTimer = null; - if (!toggleState.isToggling) { - toggleAllWindowsVisibility(); - } - }, 300); - - return; - } - const header = windowPool.get('header'); - if (!header || header.isDestroyed()) { - console.error('[Visibility] Header window not found or destroyed'); - return; - } + if (!header) return; - toggleState.isToggling = true; - toggleState.lastToggleTime = now; - const resetToggleState = () => { - toggleState.isToggling = false; - if (toggleState.toggleDebounceTimer) { - clearTimeout(toggleState.toggleDebounceTimer); - toggleState.toggleDebounceTimer = null; - } - if (toggleState.failsafeTimer) { - clearTimeout(toggleState.failsafeTimer); - toggleState.failsafeTimer = null; - } - }; - toggleState.failsafeTimer = setTimeout(() => { - console.warn('[Visibility] Toggle operation timed out, resetting state'); - resetToggleState(); - }, 2000); + if (header.isVisible()) { + console.log('[Visibility] Smart hiding - calculating nearest edge'); - try { - if (header.isVisible()) { - console.log('[Visibility] Smart hiding - calculating nearest edge'); + const headerBounds = header.getBounds(); + const display = screen.getPrimaryDisplay(); + const { width: screenWidth, height: screenHeight } = display.workAreaSize; - const headerBounds = header.getBounds(); - const display = getCurrentDisplay(header); - const { width: screenWidth, height: screenHeight } = display.workAreaSize; - const { x: workAreaX, y: workAreaY } = display.workArea; + const centerX = headerBounds.x + headerBounds.width / 2; + const centerY = headerBounds.y + headerBounds.height / 2; - const centerX = headerBounds.x + headerBounds.width / 2 - workAreaX; - const centerY = headerBounds.y + headerBounds.height / 2 - workAreaY; + const distances = { + top: centerY, + bottom: screenHeight - centerY, + left: centerX, + right: screenWidth - centerX, + }; - const distances = { - top: centerY, - bottom: screenHeight - centerY, - left: centerX, - right: screenWidth - centerX, - }; + const nearestEdge = Object.keys(distances).reduce((nearest, edge) => (distances[edge] < distances[nearest] ? edge : nearest)); - const nearestEdge = Object.keys(distances).reduce((nearest, edge) => - (distances[edge] < distances[nearest] ? edge : nearest) - ); + console.log(`[Visibility] Nearest edge: ${nearestEdge} (distance: ${distances[nearestEdge].toFixed(1)}px)`); - console.log(`[Visibility] Nearest edge: ${nearestEdge} (distance: ${distances[nearestEdge].toFixed(1)}px)`); + lastVisibleWindows.clear(); + lastVisibleWindows.add('header'); - lastVisibleWindows.clear(); - lastVisibleWindows.add('header'); - - const hidePromises = []; - windowPool.forEach((win, name) => { - if (win && !win.isDestroyed() && win.isVisible() && name !== 'header') { - lastVisibleWindows.add(name); - + windowPool.forEach((win, name) => { + if (win.isVisible()) { + lastVisibleWindows.add(name); + if (name !== 'header') { win.webContents.send('window-hide-animation'); - - hidePromises.push(new Promise(resolve => { - setTimeout(() => { - if (!win.isDestroyed()) { - win.hide(); - } - resolve(); - }, 180); // 200ms ->180ms - })); + setTimeout(() => { + if (!win.isDestroyed()) { + win.hide(); + } + }, 200); } - }); - - console.log('[Visibility] Visible windows before hide:', Array.from(lastVisibleWindows)); - - Promise.all(hidePromises).then(() => { - if (!movementManager || header.isDestroyed()) { - resetToggleState(); - return; - } - - movementManager.hideToEdge(nearestEdge, () => { - if (!header.isDestroyed()) { - header.hide(); - } - resetToggleState(); - console.log('[Visibility] Smart hide completed'); - }, (error) => { - console.error('[Visibility] Error in hideToEdge:', error); - resetToggleState(); - }); - }).catch(err => { - console.error('[Visibility] Error during hide:', err); - resetToggleState(); - }); - - } else { - console.log('[Visibility] Smart showing from hidden position'); - console.log('[Visibility] Restoring windows:', Array.from(lastVisibleWindows)); - header.show(); - - if (!movementManager) { - console.error('[Visibility] Movement manager not initialized'); - resetToggleState(); - return; } + }); - movementManager.showFromEdge(() => { - const showPromises = []; - lastVisibleWindows.forEach(name => { - if (name === 'header') return; - - const win = windowPool.get(name); - if (win && !win.isDestroyed()) { - showPromises.push(new Promise(resolve => { - win.show(); - win.webContents.send('window-show-animation'); - setTimeout(resolve, 100); - })); - } - }); + console.log('[Visibility] Visible windows before hide:', Array.from(lastVisibleWindows)); - Promise.all(showPromises).then(() => { - setImmediate(updateLayout); - setTimeout(updateLayout, 100); - - resetToggleState(); - console.log('[Visibility] Smart show completed'); - }).catch(err => { - console.error('[Visibility] Error during show:', err); - resetToggleState(); - }); - }, (error) => { - console.error('[Visibility] Error in showFromEdge:', error); - resetToggleState(); + movementManager.hideToEdge(nearestEdge, () => { + header.hide(); + console.log('[Visibility] Smart hide completed'); + }); + } else { + console.log('[Visibility] Smart showing from hidden position'); + console.log('[Visibility] Restoring windows:', Array.from(lastVisibleWindows)); + + header.show(); + + movementManager.showFromEdge(() => { + lastVisibleWindows.forEach(name => { + if (name === 'header') return; + const win = windowPool.get(name); + if (win && !win.isDestroyed()) { + win.show(); + win.webContents.send('window-show-animation'); + } }); - } - } catch (error) { - console.error('[Visibility] Unexpected error in toggle:', error); - resetToggleState(); + + setImmediate(updateLayout); + setTimeout(updateLayout, 120); + + console.log('[Visibility] Smart show completed'); + }); } } @@ -1202,21 +935,6 @@ function ensureDataDirectories() { } function createWindows() { - if (movementManager) { - movementManager.destroy(); - movementManager = null; - } - - toggleState.isToggling = false; - if (toggleState.toggleDebounceTimer) { - clearTimeout(toggleState.toggleDebounceTimer); - toggleState.toggleDebounceTimer = null; - } - if (toggleState.failsafeTimer) { - clearTimeout(toggleState.failsafeTimer); - toggleState.failsafeTimer = null; - } - const primaryDisplay = screen.getPrimaryDisplay(); const { y: workAreaY, width: screenWidth } = primaryDisplay.workArea; @@ -1261,6 +979,11 @@ function createWindows() { header.setContentProtection(isContentProtectionOn); header.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }); header.loadFile(path.join(__dirname, '../app/header.html')); + + // Open DevTools in development + if (!app.isPackaged) { + header.webContents.openDevTools({ mode: 'detach' }); + } header.on('focus', () => { console.log('[WindowManager] Header gained focus'); @@ -1285,171 +1008,153 @@ function createWindows() { loadAndRegisterShortcuts(); }); - ipcMain.handle('toggle-all-windows-visibility', () => { - try { - toggleAllWindowsVisibility(); - } catch (error) { - console.error('[WindowManager] Error in toggle-all-windows-visibility:', error); - toggleState.isToggling = false; - } - }); + ipcMain.handle('toggle-all-windows-visibility', toggleAllWindowsVisibility); ipcMain.handle('toggle-feature', async (event, featureName) => { - try { - const header = windowPool.get('header'); - if (!header || header.isDestroyed()) { - console.error('[WindowManager] Header window not available'); - return; - } - - if (!windowPool.get(featureName) && currentHeaderState === 'app') { - createFeatureWindows(header); - } + if (!windowPool.get(featureName) && currentHeaderState === 'app') { + createFeatureWindows(windowPool.get('header')); + } - if (!windowPool.get(featureName) && currentHeaderState === 'app') { - createFeatureWindows(windowPool.get('header')); - } + if (!windowPool.get(featureName) && currentHeaderState === 'app') { + createFeatureWindows(windowPool.get('header')); + } - const windowToToggle = windowPool.get(featureName); + const windowToToggle = windowPool.get(featureName); - if (windowToToggle) { - if (featureName === 'listen') { - const liveSummaryService = require('../features/listen/liveSummaryService'); - if (liveSummaryService.isSessionActive()) { - console.log('[WindowManager] Listen session is active, closing it via toggle.'); - await liveSummaryService.closeSession(); - return; - } - } - console.log(`[WindowManager] Toggling feature: ${featureName}`); - } - - if (featureName === 'ask') { - let askWindow = windowPool.get('ask'); - - if (!askWindow || askWindow.isDestroyed()) { - console.log('[WindowManager] Ask window not found, creating new one'); + if (windowToToggle) { + if (featureName === 'listen') { + const liveSummaryService = require('../features/listen/liveSummaryService'); + if (liveSummaryService.isSessionActive()) { + console.log('[WindowManager] Listen session is active, closing it via toggle.'); + await liveSummaryService.closeSession(); return; } + } + console.log(`[WindowManager] Toggling feature: ${featureName}`); + } - if (askWindow.isVisible()) { - try { - const hasResponse = await askWindow.webContents.executeJavaScript(` - (() => { - try { - // PickleGlassApp의 Shadow DOM 내부로 접근 - const pickleApp = document.querySelector('pickle-glass-app'); - if (!pickleApp || !pickleApp.shadowRoot) { - console.log('PickleGlassApp not found'); - return false; - } - - // PickleGlassApp의 shadowRoot 내부에서 ask-view 찾기 - const askView = pickleApp.shadowRoot.querySelector('ask-view'); - if (!askView) { - console.log('AskView not found in PickleGlassApp shadow DOM'); - return false; - } - - console.log('AskView found, checking state...'); - console.log('currentResponse:', askView.currentResponse); - console.log('isLoading:', askView.isLoading); - console.log('isStreaming:', askView.isStreaming); - - const hasContent = !!(askView.currentResponse || askView.isLoading || askView.isStreaming); - - if (!hasContent && askView.shadowRoot) { - const responseContainer = askView.shadowRoot.querySelector('.response-container'); - if (responseContainer && !responseContainer.classList.contains('hidden')) { - const textContent = responseContainer.textContent.trim(); - const hasActualContent = textContent && - !textContent.includes('Ask a question to see the response here') && - textContent.length > 0; - console.log('Response container content check:', hasActualContent); - return hasActualContent; - } - } - - return hasContent; - } catch (error) { - console.error('Error checking AskView state:', error); + if (featureName === 'ask') { + let askWindow = windowPool.get('ask'); + + if (!askWindow || askWindow.isDestroyed()) { + console.log('[WindowManager] Ask window not found, creating new one'); + return; + } + + if (askWindow.isVisible()) { + try { + const hasResponse = await askWindow.webContents.executeJavaScript(` + (() => { + try { + // PickleGlassApp의 Shadow DOM 내부로 접근 + const pickleApp = document.querySelector('pickle-glass-app'); + if (!pickleApp || !pickleApp.shadowRoot) { + console.log('PickleGlassApp not found'); return false; } - })() - `); - - console.log(`[WindowManager] Ask window visible, hasResponse: ${hasResponse}`); - - if (hasResponse) { - askWindow.webContents.send('toggle-text-input'); - console.log('[WindowManager] Sent toggle-text-input command'); - } else { - console.log('[WindowManager] No response found, closing window'); - askWindow.webContents.send('window-hide-animation'); - - setTimeout(() => { - if (!askWindow.isDestroyed()) { - askWindow.hide(); - updateLayout(); + + // PickleGlassApp의 shadowRoot 내부에서 ask-view 찾기 + const askView = pickleApp.shadowRoot.querySelector('ask-view'); + if (!askView) { + console.log('AskView not found in PickleGlassApp shadow DOM'); + return false; } - }, 250); - } - } catch (error) { - console.error('[WindowManager] Error checking Ask window state:', error); - console.log('[WindowManager] Falling back to toggle text input'); + + console.log('AskView found, checking state...'); + console.log('currentResponse:', askView.currentResponse); + console.log('isLoading:', askView.isLoading); + console.log('isStreaming:', askView.isStreaming); + + const hasContent = !!(askView.currentResponse || askView.isLoading || askView.isStreaming); + + if (!hasContent && askView.shadowRoot) { + const responseContainer = askView.shadowRoot.querySelector('.response-container'); + if (responseContainer && !responseContainer.classList.contains('hidden')) { + const textContent = responseContainer.textContent.trim(); + const hasActualContent = textContent && + !textContent.includes('Ask a question to see the response here') && + textContent.length > 0; + console.log('Response container content check:', hasActualContent); + return hasActualContent; + } + } + + return hasContent; + } catch (error) { + console.error('Error checking AskView state:', error); + return false; + } + })() + `); + + console.log(`[WindowManager] Ask window visible, hasResponse: ${hasResponse}`); + + if (hasResponse) { askWindow.webContents.send('toggle-text-input'); - } - } else { - console.log('[WindowManager] Showing hidden Ask window'); - askWindow.show(); - updateLayout(); - askWindow.webContents.send('window-show-animation'); - askWindow.webContents.send('window-did-show'); - } - } else { - const windowToToggle = windowPool.get(featureName); - - if (windowToToggle) { - if (windowToToggle.isDestroyed()) { - console.error(`Window ${featureName} is destroyed, cannot toggle`); - return; - } - - if (windowToToggle.isVisible()) { - if (featureName === 'settings') { - windowToToggle.webContents.send('settings-window-hide-animation'); - } else { - windowToToggle.webContents.send('window-hide-animation'); - } + console.log('[WindowManager] Sent toggle-text-input command'); + } else { + console.log('[WindowManager] No response found, closing window'); + askWindow.webContents.send('window-hide-animation'); setTimeout(() => { - if (!windowToToggle.isDestroyed()) { - windowToToggle.hide(); + if (!askWindow.isDestroyed()) { + askWindow.hide(); updateLayout(); } }, 250); - } else { - try { - windowToToggle.show(); - updateLayout(); - - if (featureName === 'listen') { - windowToToggle.webContents.send('start-listening-session'); - } - - windowToToggle.webContents.send('window-show-animation'); - } catch (e) { - console.error('Error showing window:', e); - } } - } else { - console.error(`Window not found for feature: ${featureName}`); - console.error('Available windows:', Array.from(windowPool.keys())); + } catch (error) { + console.error('[WindowManager] Error checking Ask window state:', error); + console.log('[WindowManager] Falling back to toggle text input'); + askWindow.webContents.send('toggle-text-input'); } + } else { + console.log('[WindowManager] Showing hidden Ask window'); + askWindow.show(); + updateLayout(); + askWindow.webContents.send('window-show-animation'); + askWindow.webContents.send('window-did-show'); + } + } else { + const windowToToggle = windowPool.get(featureName); + + if (windowToToggle) { + if (windowToToggle.isDestroyed()) { + console.error(`Window ${featureName} is destroyed, cannot toggle`); + return; + } + + if (windowToToggle.isVisible()) { + if (featureName === 'settings') { + windowToToggle.webContents.send('settings-window-hide-animation'); + } else { + windowToToggle.webContents.send('window-hide-animation'); + } + + setTimeout(() => { + if (!windowToToggle.isDestroyed()) { + windowToToggle.hide(); + updateLayout(); + } + }, 250); + } else { + try { + windowToToggle.show(); + updateLayout(); + + if (featureName === 'listen') { + windowToToggle.webContents.send('start-listening-session'); + } + + windowToToggle.webContents.send('window-show-animation'); + } catch (e) { + console.error('Error showing window:', e); + } + } + } else { + console.error(`Window not found for feature: ${featureName}`); + console.error('Available windows:', Array.from(windowPool.keys())); } - } catch (error) { - console.error('[WindowManager] Error in toggle-feature:', error); - toggleState.isToggling = false; } }); @@ -1537,50 +1242,12 @@ function loadAndRegisterShortcuts() { } function updateLayout() { - if (layoutManager._updateTimer) { - clearTimeout(layoutManager._updateTimer); - } - - layoutManager._updateTimer = setTimeout(() => { - layoutManager._updateTimer = null; - layoutManager.updateLayout(); - }, 16); + layoutManager.updateLayout(); } function setupIpcHandlers(openaiSessionRef) { const layoutManager = new WindowLayoutManager(); // const movementManager = new SmoothMovementManager(); - - //cleanup - app.on('before-quit', () => { - console.log('[WindowManager] App is quitting, cleaning up...'); - - if (movementManager) { - movementManager.destroy(); - } - - if (toggleState.toggleDebounceTimer) { - clearTimeout(toggleState.toggleDebounceTimer); - toggleState.toggleDebounceTimer = null; - } - - if (toggleState.failsafeTimer) { - clearTimeout(toggleState.failsafeTimer); - toggleState.failsafeTimer = null; - } - - if (settingsHideTimer) { - clearTimeout(settingsHideTimer); - settingsHideTimer = null; - } - - windowPool.forEach((win, name) => { - if (win && !win.isDestroyed()) { - win.destroy(); - } - }); - windowPool.clear(); - }); screen.on('display-added', (event, newDisplay) => { console.log('[Display] New display added:', newDisplay.id); @@ -1665,8 +1332,12 @@ function setupIpcHandlers(openaiSessionRef) { clearTimeout(settingsHideTimer); } settingsHideTimer = setTimeout(() => { - window.setAlwaysOnTop(false); - window.hide(); + // window.setAlwaysOnTop(false); + // window.hide(); + if (window && !window.isDestroyed()) { + window.setAlwaysOnTop(false); + window.hide(); + } settingsHideTimer = null; }, 200); } else { @@ -2146,23 +1817,23 @@ function setupIpcHandlers(openaiSessionRef) { ipcMain.handle('firebase-logout', () => { console.log('[WindowManager] Received request to log out.'); - // setApiKey(null) - // .then(() => { - // console.log('[WindowManager] API key cleared successfully after logout'); - // windowPool.forEach(win => { - // if (win && !win.isDestroyed()) { - // win.webContents.send('api-key-removed'); - // } - // }); - // }) - // .catch(err => { - // console.error('[WindowManager] setApiKey error:', err); - // windowPool.forEach(win => { - // if (win && !win.isDestroyed()) { - // win.webContents.send('api-key-removed'); - // } - // }); - // }); + setApiKey(null) + .then(() => { + console.log('[WindowManager] API key cleared successfully after logout'); + windowPool.forEach(win => { + if (win && !win.isDestroyed()) { + win.webContents.send('api-key-removed'); + } + }); + }) + .catch(err => { + console.error('[WindowManager] setApiKey error:', err); + windowPool.forEach(win => { + if (win && !win.isDestroyed()) { + win.webContents.send('api-key-removed'); + } + }); + }); const header = windowPool.get('header'); if (header && !header.isDestroyed()) { @@ -2265,22 +1936,32 @@ function setupIpcHandlers(openaiSessionRef) { }); } -let storedApiKey = null; -async function setApiKey(apiKey) { + +let storedApiKey = null; +let storedProvider = 'openai'; + +async function setApiKey(apiKey, provider = 'openai') { storedApiKey = apiKey; - console.log('[WindowManager] API key stored (and will be persisted to DB)'); + storedProvider = provider; + console.log('[WindowManager] API key and provider stored (and will be persisted to DB)'); try { - await sqliteClient.saveApiKey(apiKey); - console.log('[WindowManager] API key saved to SQLite'); + await sqliteClient.saveApiKey(apiKey, sqliteClient.defaultUserId, provider); + console.log('[WindowManager] API key and provider saved to SQLite'); } catch (err) { console.error('[WindowManager] Failed to save API key to SQLite:', err); } windowPool.forEach(win => { if (win && !win.isDestroyed()) { - const js = apiKey ? `localStorage.setItem('openai_api_key', ${JSON.stringify(apiKey)});` : `localStorage.removeItem('openai_api_key');`; + const js = apiKey ? ` + localStorage.setItem('openai_api_key', ${JSON.stringify(apiKey)}); + localStorage.setItem('ai_provider', ${JSON.stringify(provider)}); + ` : ` + localStorage.removeItem('openai_api_key'); + localStorage.removeItem('ai_provider'); + `; win.webContents.executeJavaScript(js).catch(() => {}); } }); @@ -2290,7 +1971,9 @@ async function loadApiKeyFromDb() { try { const user = await sqliteClient.getUser(sqliteClient.defaultUserId); if (user && user.api_key) { - console.log('[WindowManager] API key loaded from SQLite for default user.'); + console.log('[WindowManager] API key and provider loaded from SQLite for default user.'); + storedApiKey = user.api_key; + storedProvider = user.provider || 'openai'; return user.api_key; } return null; @@ -2317,6 +2000,10 @@ function getStoredApiKey() { return storedApiKey; } +function getStoredProvider() { + return storedProvider || 'openai'; +} + function setupApiKeyIPC() { const { ipcMain } = require('electron'); @@ -2324,19 +2011,24 @@ function setupApiKeyIPC() { if (storedApiKey === null) { const dbKey = await loadApiKeyFromDb(); if (dbKey) { - await setApiKey(dbKey); + await setApiKey(dbKey, storedProvider); } } return storedApiKey; }); - ipcMain.handle('api-key-validated', async (event, apiKey) => { + ipcMain.handle('api-key-validated', async (event, data) => { console.log('[WindowManager] API key validation completed, saving...'); - await setApiKey(apiKey); + + // Support both old format (string) and new format (object) + const apiKey = typeof data === 'string' ? data : data.apiKey; + const provider = typeof data === 'string' ? 'openai' : (data.provider || 'openai'); + + await setApiKey(apiKey, provider); windowPool.forEach((win, name) => { if (win && !win.isDestroyed()) { - win.webContents.send('api-key-validated', apiKey); + win.webContents.send('api-key-validated', { apiKey, provider }); } }); @@ -2366,11 +2058,16 @@ function setupApiKeyIPC() { if (storedApiKey === null) { const dbKey = await loadApiKeyFromDb(); if (dbKey) { - await setApiKey(dbKey); + await setApiKey(dbKey, storedProvider); } } return storedApiKey; }); + + ipcMain.handle('get-ai-provider', async () => { + console.log('[WindowManager] AI provider requested from renderer'); + return storedProvider || 'openai'; + }); console.log('[WindowManager] API key related IPC handlers registered (SQLite-backed)'); } @@ -2823,6 +2520,7 @@ module.exports = { fixedYPosition, setApiKey, getStoredApiKey, + getStoredProvider, clearApiKey, getCurrentFirebaseUser, isFirebaseLoggedIn, diff --git a/src/features/listen/liveSummaryService.js b/src/features/listen/liveSummaryService.js index b999793..bb1c845 100644 --- a/src/features/listen/liveSummaryService.js +++ b/src/features/listen/liveSummaryService.js @@ -3,11 +3,13 @@ const { BrowserWindow, ipcMain } = require('electron'); const { spawn } = require('child_process'); const { saveDebugAudio } = require('./audioUtils.js'); const { getSystemPrompt } = require('../../common/prompts/promptBuilder.js'); +const { connectToGeminiSession } = require('../../common/services/googleGeminiClient.js'); const { connectToOpenAiSession, createOpenAiGenerativeClient, getOpenAiGenerativeModel } = require('../../common/services/openAiClient.js'); +const { makeChatCompletionWithPortkey } = require('../../common/services/aiProviderService.js'); const sqliteClient = require('../../common/services/sqliteClient'); const dataService = require('../../common/services/dataService'); -const { isFirebaseLoggedIn, getCurrentFirebaseUser } = require('../../electron/windowManager.js'); +const { isFirebaseLoggedIn, getCurrentFirebaseUser, getStoredProvider } = require('../../electron/windowManager.js'); function getApiKey() { const { getStoredApiKey } = require('../../electron/windowManager.js'); @@ -28,6 +30,18 @@ function getApiKey() { return null; } +async function getAiProvider() { + try { + const { ipcRenderer } = require('electron'); + const provider = await ipcRenderer.invoke('get-ai-provider'); + return provider || 'openai'; + } catch (error) { + // If we're in the main process, get it directly + const { getStoredProvider } = require('../../electron/windowManager.js'); + return getStoredProvider ? getStoredProvider() : 'openai'; + } +} + let currentSessionId = null; let conversationHistory = []; let isInitializingSession = false; @@ -208,41 +222,25 @@ Keep all points concise and build upon previous analysis if provided.`, if (!API_KEY) { throw new Error('No API key available'); } + + const provider = getStoredProvider ? getStoredProvider() : 'openai'; const loggedIn = isFirebaseLoggedIn(); // true ➜ vKey, false ➜ apiKey - const keyType = loggedIn ? 'vKey' : 'apiKey'; - console.log(`[LiveSummary] keyType: ${keyType}`); + const usePortkey = loggedIn && provider === 'openai'; // Only use Portkey for OpenAI with Firebase + + console.log(`[LiveSummary] provider: ${provider}, usePortkey: ${usePortkey}`); - const fetchUrl = keyType === 'apiKey' ? 'https://api.openai.com/v1/chat/completions' : 'https://api.portkey.ai/v1/chat/completions'; - - const headers = - keyType === 'apiKey' - ? { - Authorization: `Bearer ${API_KEY}`, - 'Content-Type': 'application/json', - } - : { - 'x-portkey-api-key': 'gRv2UGRMq6GGLJ8aVEB4e7adIewu', - 'x-portkey-virtual-key': API_KEY, - 'Content-Type': 'application/json', - }; - - const response = await fetch(fetchUrl, { - method: 'POST', - headers, - body: JSON.stringify({ - model: 'gpt-4.1', - messages, - temperature: 0.7, - max_tokens: 1024, - }), + const completion = await makeChatCompletionWithPortkey({ + apiKey: API_KEY, + provider: provider, + messages: messages, + temperature: 0.7, + maxTokens: 1024, + model: provider === 'openai' ? 'gpt-4.1' : 'gemini-2.5-flash', + usePortkey: usePortkey, + portkeyVirtualKey: usePortkey ? API_KEY : null }); - if (!response.ok) { - throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`); - } - - const result = await response.json(); - const responseText = result.choices[0].message.content.trim(); + const responseText = completion.content; console.log(`✅ Analysis response received: ${responseText}`); const structuredData = parseResponseText(responseText, previousAnalysisResult); @@ -582,7 +580,6 @@ async function initializeLiveSummarySession(language = 'en') { sendToRenderer('session-initializing', true); sendToRenderer('update-status', 'Initializing sessions...'); - // Merged block const API_KEY = getApiKey(); if (!API_KEY) { console.error('FATAL ERROR: API Key is not defined.'); @@ -594,73 +591,90 @@ async function initializeLiveSummarySession(language = 'en') { await initializeNewSession(); + const provider = await getAiProvider(); + const isGemini = provider === 'gemini'; + console.log(`[LiveSummaryService] Initializing STT for provider: ${provider}`); + try { const handleMyMessage = message => { - const type = message.type; - const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || ''; - - if (type === 'conversation.item.input_audio_transcription.delta') { - if (myCompletionTimer) { - clearTimeout(myCompletionTimer); - myCompletionTimer = null; - } - - myCurrentUtterance += text; - - const continuousText = myCompletionBuffer + (myCompletionBuffer ? ' ' : '') + myCurrentUtterance; - - if (text && !text.includes('vq_lbr_audio_')) { - sendToRenderer('stt-update', { - speaker: 'Me', - text: continuousText, - isPartial: true, - isFinal: false, - timestamp: Date.now(), - }); - } - } else if (type === 'conversation.item.input_audio_transcription.completed') { + if (isGemini) { + // console.log('[Gemini Raw Message - Me]:', JSON.stringify(message, null, 2)); + const text = message.serverContent?.inputTranscription?.text || ''; if (text && text.trim()) { - const finalUtteranceText = text.trim(); - myCurrentUtterance = ''; - - debounceMyCompletion(finalUtteranceText); + const finalUtteranceText = text.trim().replace(//g, '').trim(); + if (finalUtteranceText && finalUtteranceText !== '.') { + debounceMyCompletion(finalUtteranceText); + } } - } else if (message.error) { + } else { + const type = message.type; + const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || ''; + + if (type === 'conversation.item.input_audio_transcription.delta') { + if (myCompletionTimer) clearTimeout(myCompletionTimer); + myCompletionTimer = null; + myCurrentUtterance += text; + const continuousText = myCompletionBuffer + (myCompletionBuffer ? ' ' : '') + myCurrentUtterance; + if (text && !text.includes('vq_lbr_audio_')) { + sendToRenderer('stt-update', { + speaker: 'Me', + text: continuousText, + isPartial: true, + isFinal: false, + timestamp: Date.now(), + }); + } + } else if (type === 'conversation.item.input_audio_transcription.completed') { + if (text && text.trim()) { + const finalUtteranceText = text.trim(); + myCurrentUtterance = ''; + debounceMyCompletion(finalUtteranceText); + } + } + } + + if (message.error) { console.error('[Me] STT Session Error:', message.error); } }; const handleTheirMessage = message => { - const type = message.type; - const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || ''; - - if (type === 'conversation.item.input_audio_transcription.delta') { - if (theirCompletionTimer) { - clearTimeout(theirCompletionTimer); - theirCompletionTimer = null; - } - - theirCurrentUtterance += text; - - const continuousText = theirCompletionBuffer + (theirCompletionBuffer ? ' ' : '') + theirCurrentUtterance; - - if (text && !text.includes('vq_lbr_audio_')) { - sendToRenderer('stt-update', { - speaker: 'Them', - text: continuousText, - isPartial: true, - isFinal: false, - timestamp: Date.now(), - }); - } - } else if (type === 'conversation.item.input_audio_transcription.completed') { + if (isGemini) { + // console.log('[Gemini Raw Message - Them]:', JSON.stringify(message, null, 2)); + const text = message.serverContent?.inputTranscription?.text || ''; if (text && text.trim()) { - const finalUtteranceText = text.trim(); - theirCurrentUtterance = ''; - - debounceTheirCompletion(finalUtteranceText); + const finalUtteranceText = text.trim().replace(//g, '').trim(); + if (finalUtteranceText && finalUtteranceText !== '.') { + debounceTheirCompletion(finalUtteranceText); + } } - } else if (message.error) { + } else { + const type = message.type; + const text = message.transcript || message.delta || (message.alternatives && message.alternatives[0]?.transcript) || ''; + if (type === 'conversation.item.input_audio_transcription.delta') { + if (theirCompletionTimer) clearTimeout(theirCompletionTimer); + theirCompletionTimer = null; + theirCurrentUtterance += text; + const continuousText = theirCompletionBuffer + (theirCompletionBuffer ? ' ' : '') + theirCurrentUtterance; + if (text && !text.includes('vq_lbr_audio_')) { + sendToRenderer('stt-update', { + speaker: 'Them', + text: continuousText, + isPartial: true, + isFinal: false, + timestamp: Date.now(), + }); + } + } else if (type === 'conversation.item.input_audio_transcription.completed') { + if (text && text.trim()) { + const finalUtteranceText = text.trim(); + theirCurrentUtterance = ''; + debounceTheirCompletion(finalUtteranceText); + } + } + } + + if (message.error) { console.error('[Them] STT Session Error:', message.error); } }; @@ -682,10 +696,17 @@ async function initializeLiveSummarySession(language = 'en') { }, }; - [mySttSession, theirSttSession] = await Promise.all([ - connectToOpenAiSession(API_KEY, mySttConfig, keyType), - connectToOpenAiSession(API_KEY, theirSttConfig, keyType), - ]); + if (isGemini) { + [mySttSession, theirSttSession] = await Promise.all([ + connectToGeminiSession(API_KEY, mySttConfig), + connectToGeminiSession(API_KEY, theirSttConfig), + ]); + } else { + [mySttSession, theirSttSession] = await Promise.all([ + connectToOpenAiSession(API_KEY, mySttConfig, keyType), + connectToOpenAiSession(API_KEY, theirSttConfig, keyType), + ]); + } console.log('✅ Both STT sessions initialized successfully.'); triggerAnalysisIfNeeded(); @@ -697,7 +718,7 @@ async function initializeLiveSummarySession(language = 'en') { sendToRenderer('update-status', 'Connected. Ready to listen.'); return true; } catch (error) { - console.error('❌ Failed to initialize OpenAI STT sessions:', error); + console.error('❌ Failed to initialize STT sessions:', error); isInitializingSession = false; sendToRenderer('session-initializing', false); sendToRenderer('update-status', 'Initialization failed.'); @@ -769,6 +790,9 @@ async function startMacOSAudioCapture() { let audioBuffer = Buffer.alloc(0); + const provider = await getAiProvider(); + const isGemini = provider === 'gemini'; + systemAudioProc.stdout.on('data', async data => { audioBuffer = Buffer.concat([audioBuffer, data]); @@ -783,10 +807,11 @@ async function startMacOSAudioCapture() { if (theirSttSession) { try { - // await theirSttSession.sendRealtimeInput({ - // audio: { data: base64Data, mimeType: 'audio/pcm;rate=24000' }, - // }); - await theirSttSession.sendRealtimeInput(base64Data); + // await theirSttSession.sendRealtimeInput(base64Data); + const payload = isGemini + ? { audio: { data: base64Data, mimeType: 'audio/pcm;rate=24000' } } + : base64Data; + await theirSttSession.sendRealtimeInput(payload); } catch (err) { console.error('Error sending system audio:', err.message); } @@ -905,9 +930,17 @@ function setupLiveSummaryIpcHandlers() { }); ipcMain.handle('send-audio-content', async (event, { data, mimeType }) => { + const provider = await getAiProvider(); + const isGemini = provider === 'gemini'; if (!mySttSession) return { success: false, error: 'User STT session not active' }; try { - await mySttSession.sendRealtimeInput(data); + // await mySttSession.sendRealtimeInput(data); + // provider에 맞는 형식으로 래핑 + const payload = isGemini + ? { audio: { data, mimeType: mimeType || 'audio/pcm;rate=24000' } } + : data; // OpenAI는 base64 string 그대로 + + await mySttSession.sendRealtimeInput(payload); return { success: true }; } catch (error) { console.error('Error sending user audio:', error); diff --git a/src/features/listen/renderer.js b/src/features/listen/renderer.js index 38571d5..1701d77 100644 --- a/src/features/listen/renderer.js +++ b/src/features/listen/renderer.js @@ -1,5 +1,6 @@ // renderer.js const { ipcRenderer } = require('electron'); +const { makeStreamingChatCompletionWithPortkey } = require('../../common/services/aiProviderService.js'); let mediaStream = null; let screenshotInterval = null; @@ -229,7 +230,7 @@ class SimpleAEC { this.sampleRate = 24000; this.delaySamples = Math.floor((this.echoDelay / 1000) * this.sampleRate); - this.echoGain = 0.9; + this.echoGain = 0.5; this.noiseFloor = 0.01; // 🔧 Adaptive-gain parameters (User-tuned, very aggressive) @@ -998,40 +999,22 @@ async function sendMessage(userPrompt, options = {}) { } const { isLoggedIn } = await queryLoginState(); - const keyType = isLoggedIn ? 'vKey' : 'apiKey'; + const provider = await ipcRenderer.invoke('get-ai-provider'); + const usePortkey = isLoggedIn && provider === 'openai'; - console.log('🚀 Sending request to OpenAI...'); - const { url, headers } = - keyType === 'apiKey' - ? { - url: 'https://api.openai.com/v1/chat/completions', - headers: { Authorization: `Bearer ${API_KEY}`, 'Content-Type': 'application/json' }, - } - : { - url: 'https://api.portkey.ai/v1/chat/completions', - headers: { - 'x-portkey-api-key': 'gRv2UGRMq6GGLJ8aVEB4e7adIewu', - 'x-portkey-virtual-key': API_KEY, - 'Content-Type': 'application/json', - }, - }; - - const response = await fetch(url, { - method: 'POST', - headers, - body: JSON.stringify({ - model: 'gpt-4.1', - messages, - temperature: 0.7, - max_tokens: 2048, - stream: true, - }), + console.log(`🚀 Sending request to ${provider} AI...`); + + const response = await makeStreamingChatCompletionWithPortkey({ + apiKey: API_KEY, + provider: provider, + messages: messages, + temperature: 0.7, + maxTokens: 2048, + model: provider === 'openai' ? 'gpt-4.1' : 'gemini-2.5-flash', + usePortkey: usePortkey, + portkeyVirtualKey: usePortkey ? API_KEY : null }); - if (!response.ok) { - throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`); - } - // --- 스트리밍 응답 처리 --- const reader = response.body.getReader(); const decoder = new TextDecoder();