From e6976616cf1031f29097ded535b77911afe0d0d2 Mon Sep 17 00:00:00 2001 From: gimmeThePillow <71290334+yanheChen@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:08:58 -0800 Subject: [PATCH 1/4] Add a side panel / update the google meeting tag --- background/background.ts | 34 ++++++++++++- content/client.ts | 99 ++++++++++++++++++++++++++++++------ content/interactive_image.ts | 79 ++++++++++++++++++++++------ manifest.json | 3 ++ rollup.config.js | 12 ++++- sidepanel.html | 37 ++++++++++++++ sidepanel.ts | 46 +++++++++++++++++ tsconfig.json | 3 +- 8 files changed, 277 insertions(+), 36 deletions(-) create mode 100644 sidepanel.html create mode 100644 sidepanel.ts diff --git a/background/background.ts b/background/background.ts index 3d42667..d84a08c 100644 --- a/background/background.ts +++ b/background/background.ts @@ -13,6 +13,36 @@ See the License for the specific language governing permissions and limitations under the License. */ -chrome.action.onClicked.addListener(() => { - chrome.runtime.openOptionsPage(); + chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + if (message.type === 'SIDE_PANEL_TO_CONTENT') { + // Forward to active tab + chrome.tabs.query({active: true, currentWindow: true}, function(tabs) { + if (!tabs[0]?.id) return; + + chrome.tabs.sendMessage(tabs[0].id, { + incoming: true, + type: 'SIDE_PANEL_COMMAND', + command: message.command, + data: message.data + }); + }); + } else if (message.type === 'UPDATE_SIDE_PANEL') { + // Make sure we have a valid tab ID from the sender + if (!sender.tab?.id) return; + + // Get side panel options for the specific tab + chrome.sidePanel.getOptions({tabId: sender.tab.id}).then(options => { + if (options.enabled) { + chrome.runtime.sendMessage({ + type: 'CONTENT_TO_SIDE_PANEL', + state: message.state + }); + } + }).catch(error => { + console.error('Error getting side panel options:', error); + }); + } }); + +// Setup side panel behavior +chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true }); diff --git a/content/client.ts b/content/client.ts index 1cb4ca3..9a9db98 100644 --- a/content/client.ts +++ b/content/client.ts @@ -102,38 +102,73 @@ function fetchGoogleMeetCaptions() { let captions = ''; let selfCaptions = ''; let allCaptions = ''; - - const gMeetCaptionsView = - document.querySelector('div[jscontroller="D1tHje"]'); - - if (gMeetCaptionsView) { - const divs = document.querySelectorAll('div[class="TBMuR bj4p3b"]'); - for (const div of divs) { - let name = div.querySelector('div[class="zs7s8d jxFHg"]')!.textContent; - let wordSpans = Array.from(div.querySelectorAll('span')); - captions += name + ': '; - const sentence = wordSpans.map(span => span.textContent.trim()).join(' '); - if (name === 'You' || name.indexOf('Presentation') !== -1) { + const captionContainer = document.querySelector('div[jsname="dsyhDe"]'); + + if (captionContainer) { + const captionBlocks = document.querySelectorAll('div[jsname="tgaKEf"][class*="bh44bd"]'); + + for (const block of captionBlocks) { + const spans = Array.from(block.querySelectorAll('span')); + if (spans.length === 0) continue; + + const sentence = spans.map(span => span.textContent.trim()).join(' '); + + const parentBlock = block.closest('div[jsname="YSxPC"]'); + const isUserCaption = determineIfUserCaption(parentBlock); + + if (isUserCaption) { if (selfCaptions.length > 0) { selfCaptions += ' '; } selfCaptions += sentence; } + captions += sentence + '\n'; - allCaptions += sentence; + allCaptions += sentence + ' '; + + console.log('Caption found:', sentence); } } renderer.setCaptions( - Boolean(gMeetCaptionsView), + Boolean(captionContainer && allCaptions.length > 0), captions, selfCaptions, - allCaptions, + allCaptions ); setTimeout(fetchGoogleMeetCaptions, APP_SETTINGS.fetchMeetCaptionFPSInMs); } +// Helper function to determine if a caption block belongs to the current user +function determineIfUserCaption(element: Element) { + if (!element) return false; + + const hasUserIndicators = element.classList.contains('wY1pdd'); + + const siblingElements: any[] | HTMLCollection = element.parentElement?.children || []; + for (const sibling of siblingElements) { + if (sibling.textContent?.includes('You')) { + return true; + } + } + let current = element; + for (let i = 0; i < 3; i++) { + if (!current.parentElement) break; + current = current.parentElement; + + // Check if any parent element contains "You" text + if (current.textContent?.includes('You')) { + return true; + } + } + + // Default to true for the first caption if we can't determine + // This is a fallback assuming the first caption is often the user's + const isFirstCaptionBlock = !element.previousElementSibling; + return isFirstCaptionBlock || hasUserIndicators; +} + function makeStreamFromTrack(track: MediaStreamTrack) { const stream = new MediaStream(); stream.addTrack(track); @@ -158,7 +193,6 @@ function patchNativeAPI() { if (videoDeviceId !== 'virtual') { restoreVideoMirrorMode(); - console.log('Current this', this); return MediaDevices.prototype.getUserMedia.call(this, constraints); } else { removeVideoMirrorMode(); @@ -216,5 +250,38 @@ function restoreVideoMirrorMode() { } } +window.addEventListener('message', (event) => { + const message = event.data; + if (!message.incoming) return; + + if (message.type === 'SETTINGS') { + console.log('Received settings', message.settings); + applySettings(message.settings); + } else if (message.type === 'SIDE_PANEL_COMMAND') { + // Handle side panel specific commands + handleSidePanelCommand(message.command, message.data); + } +}); + +// Function to handle side panel commands +function handleSidePanelCommand(command: string, data: any) { + switch(command) { + case 'SET_SCENE': + renderer.setScene(data.scene); + break; + case 'TOGGLE_FEATURE': + // Handle feature toggle + break; + } + + window.postMessage({ + type: 'UPDATE_SIDE_PANEL', + outgoing: true, + state: { + currentScene: renderer.getCurrentScene(), + } + }, '*'); +} + patchNativeAPI(); fetchGoogleMeetCaptions(); diff --git a/content/interactive_image.ts b/content/interactive_image.ts index 8e80cf3..28162d0 100644 --- a/content/interactive_image.ts +++ b/content/interactive_image.ts @@ -761,31 +761,78 @@ declare global { return scheduleNextFetch(); } - + function fetchCaptionsFromDOM() { - const gMeetCaptionsView = document.querySelector(`div[jscontroller="${gMeetControllerName}"]`); - if (!gMeetCaptionsView) return null; - + const captionContainer = document.querySelector('div[jsname="dsyhDe"]'); + + if (!captionContainer) return null; + let selfCaptions = ''; let allCaptions = ''; - - const divs = document.querySelectorAll('div[class="TBMuR bj4p3b"]'); - for (const div of divs) { - const name = div.querySelector('div[class="zs7s8d jxFHg"]').textContent; - const sentence = Array.from(div.querySelectorAll('span')) - .map(span => span.textContent.trim()) - .join(' '); - - if (name === 'You' || name === 'Your Presentation') { + + const captionBlocks = document.querySelectorAll('div[jsname="tgaKEf"][class*="bh44bd"]'); + + if (captionBlocks.length === 0) return null; + + for (const block of captionBlocks) { + // Extract text from all spans in this block + const spans = Array.from(block.querySelectorAll('span')); + if (spans.length === 0) continue; + + const sentence = spans.map(span => span.textContent.trim()).join(' '); + + // Try to determine if this is the user's caption + const parentBlock = block.closest('div[jsname="YSxPC"]'); + const isUserCaption = determineIfUserCaption(parentBlock); + + // Add to appropriate captions collection + if (isUserCaption) { + if (selfCaptions.length > 0) { + selfCaptions += ' '; + } selfCaptions += sentence; } - allCaptions += sentence; + + // Add to all captions + allCaptions += sentence + ' '; + + console.log('VC - Caption found:', sentence.substring(0, 50) + (sentence.length > 50 ? '...' : '')); } - - return { selfCaptions, allCaptions }; + + return allCaptions || selfCaptions ? { selfCaptions, allCaptions } : null; + } + + function determineIfUserCaption(element: Element) { + if (!element) return false; + + // Method 1: Check for visual indicators in parent elements + const hasUserIndicators = element.classList.contains('wY1pdd'); + + // Method 2: Look for "You" text nearby + const siblingElements: any[] | HTMLCollection = element.parentElement?.children || []; + for (const sibling of siblingElements) { + if (sibling.textContent?.includes('You')) { + return true; + } + } + + // Check parent elements up to 3 levels + let current = element; + for (let i = 0; i < 3; i++) { + if (!current.parentElement) break; + current = current.parentElement; + + if (current.textContent?.includes('You')) { + return true; + } + } + + const isFirstCaptionBlock = !element.previousElementSibling; + return isFirstCaptionBlock || hasUserIndicators; } async function processAndGenerateVisuals(captions: string) { + console.log('VC - processAndGenerateVisuals', captions); const text = extractTextForProcessing(captions); if (text === lastText) { console.log('VC - same text will not generate new content: ', text); diff --git a/manifest.json b/manifest.json index 96c71f0..ae58dc3 100644 --- a/manifest.json +++ b/manifest.json @@ -9,6 +9,9 @@ "service_worker": "background.js", "type": "module" }, + "side_panel": { + "default_path": "sidepanel.html" + }, "icons": { "16": "images/icon16.png", "32": "images/icon32.png", diff --git a/rollup.config.js b/rollup.config.js index ade6608..4afd6d0 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -46,6 +46,15 @@ export default [ }, plugins: [typescript(), nodeResolve()] }, + { + input: 'sidepanel.ts', + output: { + dir: 'output', + format: 'es', + generatedCode: 'es2015', + }, + plugins: [typescript(), nodeResolve()] + }, { input: 'background/background.ts', output: { @@ -62,6 +71,7 @@ export default [ src: [ 'background/background.html', 'options/options.html', + 'sidepanel.html', 'manifest.json', 'content/content.js' ], dest: 'output' @@ -72,4 +82,4 @@ export default [ }) ] } -]; +]; \ No newline at end of file diff --git a/sidepanel.html b/sidepanel.html new file mode 100644 index 0000000..0bba1c9 --- /dev/null +++ b/sidepanel.html @@ -0,0 +1,37 @@ + + + + ARChat Controls + + + +

ARChat Controls

+ +
+ + +
+ +
+

Interactive Image Settings

+ +
+ +
+

Current Status

+
+
+ + + + \ No newline at end of file diff --git a/sidepanel.ts b/sidepanel.ts new file mode 100644 index 0000000..673a5d6 --- /dev/null +++ b/sidepanel.ts @@ -0,0 +1,46 @@ +// In sidepanel.ts +import { SCENE_NAMES } from './scenes/scene_names'; + +// Initialize UI elements +document.addEventListener('DOMContentLoaded', () => { + // Get references to UI elements + const sceneSelector = document.getElementById('scene-selector') as HTMLSelectElement; + + // Populate scene selector + Object.values(SCENE_NAMES).forEach(sceneName => { + const option = document.createElement('option'); + option.value = sceneName.toString(); + option.textContent = sceneName.toString(); + sceneSelector.appendChild(option); + }); + // Add event listeners + sceneSelector.addEventListener('change', () => { + chrome.runtime.sendMessage({ + type: 'SIDE_PANEL_TO_CONTENT', + command: 'SET_SCENE', + data: { scene: sceneSelector.value } + }); + }); + + // Request initial state + chrome.tabs.query({active: true, currentWindow: true}, function(tabs) { + if (!tabs[0]?.id) return; + + chrome.tabs.sendMessage(tabs[0].id, { + type: 'GET_STATE' + }); + }); +}); + +// Listen for updates from content script +chrome.runtime.onMessage.addListener((message) => { + if (message.type === 'CONTENT_TO_SIDE_PANEL') { + updateUI(message.state); + } +}); + +function updateUI(state: any) { + // Update UI based on current state + const sceneSelector = document.getElementById('scene-selector') as HTMLSelectElement; + sceneSelector.value = state.currentScene; +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 8bc681a..5e6bbe2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,6 +14,7 @@ "options/options.ts", "background/background.ts", "content/client.ts", - "content/interactive_image.ts" + "content/interactive_image.ts", + "sidepanel.ts" ] } From bd9404ce770d4e51f3c8cd012fca94d7e3a72b64 Mon Sep 17 00:00:00 2001 From: gimmeThePillow <71290334+yanheChen@users.noreply.github.com> Date: Tue, 25 Feb 2025 18:51:11 -0800 Subject: [PATCH 2/4] improve the prompt with newly gemini api endpoint --- models/modelClient.ts | 140 +++++++++++++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 22 deletions(-) diff --git a/models/modelClient.ts b/models/modelClient.ts index 45a2766..a02bac5 100644 --- a/models/modelClient.ts +++ b/models/modelClient.ts @@ -79,41 +79,137 @@ async function callGPT3(prompt: string, modelType: ModelType): Promise { // Gemini functions export async function getGeminiRawResult(text: string): Promise { - if (text.length === 0) { + if (!text || text.trim().length === 0) { + console.log('VC - Empty text provided to getGeminiRawResult'); + return undefined; + } + + try { + return await callGemini(text); + } catch (error) { + console.error('VC - Error in getGeminiRawResult:', error); return undefined; } - return callGemini(text); } async function callGemini(text: string): Promise { try { - const prompt = `Analyze the following text and identify concrete nouns or noun phrases that could be visualized with an image. For each identified noun or phrase, provide a suggested image search query. Format your response as a semicolon-separated list of "noun/phrase from suggested search query". + // Improved prompt with clearer instructions and better examples + const prompt = `Analyze the following speech text and identify concrete nouns or phrases that would make good visuals in a video call. + + For each identified noun/phrase, provide a suggested image search query. + Format your response as a semicolon-separated list of "noun/phrase from suggested search query". Rules: - 1. Focus on specific, tangible nouns or noun phrases that would make good image subjects. - 2. Prioritize proper nouns, locations, objects, and well-defined concepts. - 3. Avoid abstract concepts, general categories, or vague terms. - 4. For compound nouns or phrases, use the full phrase as both the noun and the search query. - 5. If a noun is part of a larger context, include relevant context in the search query. - 6. Limit your response to the most relevant and important nouns/phrases (max 5). - 7. Do not include any explanations or additional text in your response. - - Example input: "I visited Palo Alto Central Park yesterday and saw a red-tailed hawk perched on an old oak tree." + 1. Focus on SPECIFIC, tangible items that would enhance a video call with relevant visuals + 2. Prioritize proper nouns, locations, objects, and well-defined concepts mentioned in the speech + 3. Avoid abstract concepts or vague terms that wouldn't make clear images + 4. Include "emoji" in search query if the concept would work well as an emoji + 5. For maps or locations, include "map" in the search query + 6. Limit to 2-3 most relevant visual items from the text + 7. IMPORTANT: Respond ONLY with the semicolon-separated list, no explanations or additional text + + Examples: + + Input: "I visited Golden Gate Bridge yesterday and saw a beautiful sunset." + Output: Golden Gate Bridge from Golden Gate Bridge San Francisco;sunset from beautiful sunset over ocean + + Input: "I'm feeling happy today because I got a new puppy!" + Output: happy face from happy face emoji;puppy from cute puppy + + Input: "Let me show you how to get to the restaurant. It's on Main Street near the library." + Output: directions from map of Main Street;restaurant from restaurant exterior + + Now, analyze this speech text:`; + + // Call Gemini model with the improved prompt + const result = await geminiModel.generateContent(prompt + text); + const response = result.response.candidates[0]; + + console.log('VC - Gemini raw response:', response); - Example output: - Palo Alto Central Park from Palo Alto Central Park;red-tailed hawk from red-tailed hawk perched on tree;oak tree from old oak tree in park + // Validate response format and clean it if needed + return extractSuggestionsFromResponse(response.content.parts[0].text.trim()); + } catch (error) { + console.error('VC - Error calling Gemini:', error); + // Return a fallback empty result instead of throwing + return ''; + } +} - Example input: "Did you want to go to san francisco zoo today?" +function extractSuggestionsFromResponse(response: string): string { + try { + let jsonData: any; - Example output: - San Francisco Zoo from San Francisco + try { + jsonData = JSON.parse(response); + } catch (e) { + const jsonMatch = response.match(/\{.*("suggested_search_queries"|"visuals").*\}/s); + if (jsonMatch) { + jsonData = JSON.parse(jsonMatch[0]); + } else { + throw new Error("No valid JSON found in response"); + } + } - Now, analyze the following text:` - const result = await geminiModel.generateContent(prompt + text); - return result.response.text(); + // Handle Format 1: {"suggested_search_queries": ["noun from query", "noun from query"]} + if (jsonData.suggested_search_queries && Array.isArray(jsonData.suggested_search_queries)) { + return jsonData.suggested_search_queries.join(';'); + } + + // Handle Format 2: {"visuals": [{"noun": "X", "search_query": "Y"}, ...]} + if (jsonData.visuals && Array.isArray(jsonData.visuals) && + jsonData.visuals[0] && typeof jsonData.visuals[0] === 'object') { + return jsonData.visuals.map((item: any) => + `${item.noun} from ${item.search_query}` + ).join(';'); + } + + // Handle Format 3: {"visuals": ["noun from query", "noun from query"]} + if (jsonData.visuals && Array.isArray(jsonData.visuals) && + typeof jsonData.visuals[0] === 'string') { + return jsonData.visuals.join(';'); + } + + // Last resort: check for any field that contains an array of strings with "from" in them + for (const key in jsonData) { + if (Array.isArray(jsonData[key])) { + if (typeof jsonData[key][0] === 'string' && jsonData[key][0].includes(' from ')) { + return jsonData[key].join(';'); + } else if (typeof jsonData[key][0] === 'object') { + // Try to extract noun/search_query pairs from any object array + try { + const items = jsonData[key].map((item: any) => { + const noun = item.noun || item.name || item.item || item.visual || ''; + const query = item.search_query || item.query || item.search || ''; + if (noun && query) { + return `${noun} from ${query}`; + } + return null; + }).filter(Boolean); + + if (items.length > 0) { + return items.join(';'); + } + } catch (e) { + console.warn('VC - Failed to extract from object array:', e); + } + } + } + } + + console.warn('VC - Could not find suggestions in response:', response); + return ''; } catch (error) { - console.error('Error calling Gemini:', error); - throw error; + console.error('VC - Error parsing Gemini response:', error); + + // Last resort: try to extract anything that looks like "X from Y" using regex + const fromMatches = response.match(/[^;]+? from [^;]+/g); + if (fromMatches && fromMatches.length > 0) { + return fromMatches.join(';'); + } + + return ''; } } From 0b20a4caa9548bd442a6c4c70b2393d7734ac713 Mon Sep 17 00:00:00 2001 From: gimmeThePillow <71290334+yanheChen@users.noreply.github.com> Date: Tue, 25 Feb 2025 18:51:22 -0800 Subject: [PATCH 3/4] update the DOMTREE selector --- content/interactive_image.ts | 116 +++++++++++++++-------------------- 1 file changed, 48 insertions(+), 68 deletions(-) diff --git a/content/interactive_image.ts b/content/interactive_image.ts index 28162d0..a8ba0c6 100644 --- a/content/interactive_image.ts +++ b/content/interactive_image.ts @@ -720,11 +720,7 @@ declare global { function createWidget(entity: Entity, isEmoji: boolean) { const videos = document.querySelectorAll('video'); let video; - if (videos.length === 1) { - video = videos[0]; - } else { - video = Array.from(videos).find((video) => getName(video).includes('You')); - } + video = videos[0]; const container = video.parentElement; if (isEmoji) { @@ -741,7 +737,7 @@ declare global { ////////////////////////////////////////////////////////////////////////////// // MAIN ////////////////////////////////////////////////////////////////////////////// - let fetchCaptionsHandle: number|NodeJS.Timeout; + let fetchCaptionsHandle: number|NodeJS.Timeout; async function fetchGoogleMeetCaptions() { options.update(); if (!options.enabled) { @@ -764,73 +760,57 @@ declare global { function fetchCaptionsFromDOM() { const captionContainer = document.querySelector('div[jsname="dsyhDe"]'); - - if (!captionContainer) return null; + if (!captionContainer) { + console.log('VC - No caption container found'); + return null; + } let selfCaptions = ''; let allCaptions = ''; - - const captionBlocks = document.querySelectorAll('div[jsname="tgaKEf"][class*="bh44bd"]'); + + const childDivs = Array.from(captionContainer.children[0].children); + if (childDivs.length <= 1) { + console.log('VC - Not enough caption content divs found'); + return null; + } - if (captionBlocks.length === 0) return null; + const contentDivs = childDivs.slice(0, -1); - for (const block of captionBlocks) { - // Extract text from all spans in this block - const spans = Array.from(block.querySelectorAll('span')); - if (spans.length === 0) continue; - - const sentence = spans.map(span => span.textContent.trim()).join(' '); + for (const contentDiv of contentDivs) { + const nameElement = contentDiv.querySelector('div[class="KcIKyf jxFHg"]'); + const speakerName = nameElement ? nameElement.textContent.trim() : ''; + const isUserCaption = speakerName === 'You'; - // Try to determine if this is the user's caption - const parentBlock = block.closest('div[jsname="YSxPC"]'); - const isUserCaption = determineIfUserCaption(parentBlock); - - // Add to appropriate captions collection - if (isUserCaption) { - if (selfCaptions.length > 0) { - selfCaptions += ' '; + // Find all caption text under the YSxPC divs in this content section + const captionSections = contentDiv.querySelectorAll('div[jsname="YSxPC"]'); + for (const section of captionSections) { + // Find the caption text divs with jsname="tgaKEf" + const captionDivs = section.querySelectorAll('div[jsname="tgaKEf"]'); + for (const captionDiv of captionDivs) { + const text = captionDiv.textContent.trim(); + if (!text) continue; + + // Add to self captions if this is the user + if (isUserCaption) { + if (selfCaptions) selfCaptions += ' '; + selfCaptions += text; + } + + // Add to all captions with speaker name + if (allCaptions) allCaptions += ' '; + if (speakerName && !allCaptions.endsWith(`${speakerName}: `)) { + allCaptions += `${speakerName}: `; + } + allCaptions += text; + + console.log(`VC - Caption found from ${speakerName || 'unknown'}: ${text.substring(0, 50)}${text.length > 50 ? '...' : ''}`); } - selfCaptions += sentence; } - - // Add to all captions - allCaptions += sentence + ' '; - - console.log('VC - Caption found:', sentence.substring(0, 50) + (sentence.length > 50 ? '...' : '')); } - return allCaptions || selfCaptions ? { selfCaptions, allCaptions } : null; + return (allCaptions || selfCaptions) ? { selfCaptions, allCaptions } : null; } - function determineIfUserCaption(element: Element) { - if (!element) return false; - - // Method 1: Check for visual indicators in parent elements - const hasUserIndicators = element.classList.contains('wY1pdd'); - - // Method 2: Look for "You" text nearby - const siblingElements: any[] | HTMLCollection = element.parentElement?.children || []; - for (const sibling of siblingElements) { - if (sibling.textContent?.includes('You')) { - return true; - } - } - - // Check parent elements up to 3 levels - let current = element; - for (let i = 0; i < 3; i++) { - if (!current.parentElement) break; - current = current.parentElement; - - if (current.textContent?.includes('You')) { - return true; - } - } - - const isFirstCaptionBlock = !element.previousElementSibling; - return isFirstCaptionBlock || hasUserIndicators; - } - async function processAndGenerateVisuals(captions: string) { console.log('VC - processAndGenerateVisuals', captions); const text = extractTextForProcessing(captions); @@ -950,19 +930,19 @@ declare global { window.addEventListener('message', (event) => { const message = event.data; - - if (message.type !== 'CURRENT_SCENE') return; + if (message.type !== 'CURRENT_SCENE') return; toggleEffect(message.scene === 'Interactive Images (beta)'); }); toggleEffect(true); window.postMessage( - { - type: 'GET_CURRENT_SCENE', - outgoing: true, - }, - '*'); + { + type: 'GET_CURRENT_SCENE', + outgoing: true, + }, + '*' + ); ////////////////////////////////////////////////////////////////////////////// // HELPERS From 7f3f65facdafecacea19bd825cf7ddca87056dd2 Mon Sep 17 00:00:00 2001 From: gimmeThePillow <71290334+yanheChen@users.noreply.github.com> Date: Tue, 25 Feb 2025 20:29:50 -0800 Subject: [PATCH 4/4] updating the sidepanel UI --- sidepanel.html | 580 ++++++++++++++++++++++++++++++++++++-- sidepanel.ts | 735 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 1273 insertions(+), 42 deletions(-) diff --git a/sidepanel.html b/sidepanel.html index 0bba1c9..78785eb 100644 --- a/sidepanel.html +++ b/sidepanel.html @@ -1,37 +1,575 @@ - + - ARChat Controls + + + AR Chat Side Panel -

ARChat Controls

- -
- - -
- -
-

Interactive Image Settings

- -
- -
-

Current Status

-
+
+
+

AR Chat Settings

+
+ +
+ Scene: None | Captions: Off +
+ +
+
+ + +
+ + +
+

Please select a scene from the dropdown above.

+
+ + + + + + +
- + \ No newline at end of file diff --git a/sidepanel.ts b/sidepanel.ts index 673a5d6..905782f 100644 --- a/sidepanel.ts +++ b/sidepanel.ts @@ -1,46 +1,739 @@ // In sidepanel.ts import { SCENE_NAMES } from './scenes/scene_names'; +import { + DEFAULT_SOUND_OPTIONS, + SoundOptions, + LANGUAGES, + TranscriptionLayout, + LambdaLayout, + SummaryMode, + CaptionMode, + BackgroundImage, + EmphasizingMode, + AutoCapitalization, + Colors +} from './scenes/Sound/sound_options'; + +import { + DEFAULT_INTERACTIVEIMAGE_OPTIONS, + InteractiveImageOptions +} from './scenes/Sound/InteractiveImage_options'; + +// Define the state interface +interface ARChatState { + currentScene: string; + captionsEnabled: boolean; + aiProactiveness: string; + allParticipants: boolean; + suggestEmojis: boolean; + suggestPersonal: boolean; + modelCapability: string; + minWords: number; + lastNSentences: number; + maxVisuals: number; + maxEmojis: number; + visualSize: number; + loggingEnabled: boolean; + interactiveImage?: { + url: string; + name: string; + }; +} + +// Initial state +let currentState: ARChatState = { + currentScene: '', + captionsEnabled: true, + aiProactiveness: 'suggestion', + allParticipants: false, + suggestEmojis: true, + suggestPersonal: true, + modelCapability: 'most-capable', + minWords: 4, + lastNSentences: 1, + maxVisuals: 5, + maxEmojis: 4, + visualSize: 1, + loggingEnabled: false +}; -// Initialize UI elements document.addEventListener('DOMContentLoaded', () => { - // Get references to UI elements + // Get references to tab elements + const tabButtons = document.querySelectorAll('.tab-button'); + const tabPanels = document.querySelectorAll('.tab-panel'); + + // Tab switching logic + tabButtons.forEach(button => { + button.addEventListener('click', () => { + const targetId = button.getAttribute('data-target'); + + tabButtons.forEach(btn => btn.classList.remove('active')); + tabPanels.forEach(panel => panel.classList.remove('active')); + + button.classList.add('active'); + document.getElementById(targetId).classList.add('active'); + }); + }); + + // Scene selector setup const sceneSelector = document.getElementById('scene-selector') as HTMLSelectElement; + const blankPage = document.getElementById('blank-page'); + const soundSettings = document.getElementById('sound-settings'); + const visualCaptions = document.getElementById('visual-captions'); + const toggleAdvanced = document.getElementById('toggle-advanced'); + const advancedContent = document.getElementById('advanced-content'); - // Populate scene selector - Object.values(SCENE_NAMES).forEach(sceneName => { + // Clear any existing options + while (sceneSelector.firstChild) { + sceneSelector.removeChild(sceneSelector.firstChild); + } + + // Add blank option first + const blankOption = document.createElement('option'); + blankOption.value = ''; + blankOption.textContent = 'Select a scene'; + sceneSelector.appendChild(blankOption); + + console.log("SCENE_NAMES:", SCENE_NAMES); // Debug what's being imported + + Object.entries(SCENE_NAMES).forEach(([key, sceneName]) => { + console.log(`Adding option: key=${key}, value=${sceneName}`); // Debug each iteration const option = document.createElement('option'); - option.value = sceneName.toString(); - option.textContent = sceneName.toString(); + option.value = key; // Use the key as the value + option.textContent = sceneName; // Use the name as display text sceneSelector.appendChild(option); }); - // Add event listeners + + // Setup different panels + setupInteractiveImageOptions(); + setupSoundOptions(); + setupCaptionSettings(); + setupAdvancedSettings(); + setupLoggingSettings(); + setupInteractiveImageSettings(); + + const sliders = document.querySelectorAll('input[type="range"]'); + sliders.forEach(function(slider) { + const valueDisplay = document.getElementById(`${slider.id}-value`); + if (valueDisplay) { + valueDisplay.textContent = (slider as HTMLInputElement).value; + slider.addEventListener('input', function(this: HTMLInputElement) { + valueDisplay.textContent = this.value; + }); + } + }); + sceneSelector.addEventListener('change', () => { + const selectedScene = sceneSelector.value; + currentState.currentScene = selectedScene; + chrome.runtime.sendMessage({ type: 'SIDE_PANEL_TO_CONTENT', command: 'SET_SCENE', - data: { scene: sceneSelector.value } + data: { scene: selectedScene } + }); + + // Hide all pages + blankPage.style.display = 'none'; + soundSettings.style.display = 'none'; + visualCaptions.style.display = 'none'; + + // Show the selected page + if (selectedScene === 'Sound') { + soundSettings.style.display = 'block'; + } else if (selectedScene === 'InteractiveImage') { + visualCaptions.style.display = 'block'; + toggleInteractiveImageSection(true); + } else if (selectedScene === 'PassThrough') { + blankPage.style.display = 'block'; + } else { + blankPage.style.display = 'block'; + } + + updateStatusDisplay(); + }); + + // Toggle advanced settings + if (toggleAdvanced && advancedContent) { + toggleAdvanced.addEventListener('click', function() { + const isHidden = advancedContent.style.display === 'none' || !advancedContent.style.display; + + if (isHidden) { + advancedContent.style.display = 'block'; + toggleAdvanced.textContent = 'Hide'; + } else { + advancedContent.style.display = 'none'; + toggleAdvanced.textContent = 'Show'; + } }); + } +}); + +function setupInteractiveImageOptions() { + // Get references to UI elements + const enableButton = document.getElementById('enable-button') as HTMLInputElement; + const proactivenessSelect = document.getElementById('proactiveness') as HTMLSelectElement; + const enableAllCaptions = document.getElementById('enable-all-captions') as HTMLInputElement; + const enableEmoji = document.getElementById('enable-emoji') as HTMLInputElement; + const enablePersonal = document.getElementById('enable-personal') as HTMLInputElement; + const visualSize = document.getElementById('visual-size') as HTMLInputElement; + const numVisuals = document.getElementById('num-visuals') as HTMLInputElement; + const numEmojis = document.getElementById('num-emojis') as HTMLInputElement; + const numWords = document.getElementById('num-words') as HTMLInputElement; + const lastNSentences = document.getElementById('last-n-sentences') as HTMLInputElement; + + // Update UI value displays + const visualSizeValue = document.getElementById('visual-size-value') as HTMLSpanElement; + const numVisualsValue = document.getElementById('num-visuals-value') as HTMLSpanElement; + const numEmojisValue = document.getElementById('num-emojis-value') as HTMLSpanElement; + const numWordsValue = document.getElementById('num-words-value') as HTMLSpanElement; + const lastNSentencesValue = document.getElementById('last-n-sentences-value') as HTMLSpanElement; + + function updateSettings() { + const options: InteractiveImageOptions = { + enableButton: enableButton?.checked || false, + proactiveness: proactivenessSelect?.value || 'suggestion', + enableAllCaptions: enableAllCaptions?.checked || false, + enableEmoji: enableEmoji?.checked || true, + enablePersonal: enablePersonal?.checked || true, + model: 'davinci', // Fixed for now + visualSize: Number(visualSize?.value || 1), + numVisuals: Number(numVisuals?.value || 5), + numEmojis: Number(numEmojis?.value || 4), + numWords: Number(numWords?.value || 4), + lastNSentences: Number(lastNSentences?.value || 1), + enableLogging: false // Fixed for now + }; + + // Update slider value displays + if (visualSizeValue) visualSizeValue.textContent = visualSize?.value || '1'; + if (numVisualsValue) numVisualsValue.textContent = numVisuals?.value || '5'; + if (numEmojisValue) numEmojisValue.textContent = numEmojis?.value || '4'; + if (numWordsValue) numWordsValue.textContent = numWords?.value || '4'; + if (lastNSentencesValue) lastNSentencesValue.textContent = lastNSentences?.value || '1'; + + chrome.storage.local.set({ interactiveImageOptions: options }); + + // Update current state with relevant values + currentState.suggestEmojis = options.enableEmoji; + currentState.suggestPersonal = options.enablePersonal; + currentState.visualSize = options.visualSize; + currentState.maxVisuals = options.numVisuals; + currentState.maxEmojis = options.numEmojis; + currentState.minWords = options.numWords; + currentState.lastNSentences = options.lastNSentences; + + sendStateUpdate(); + } + + // Add event listeners + [enableButton, enableAllCaptions, enableEmoji, enablePersonal].forEach(checkbox => { + if (checkbox) checkbox.addEventListener('change', updateSettings); }); + + if (proactivenessSelect) proactivenessSelect.addEventListener('change', updateSettings); - // Request initial state - chrome.tabs.query({active: true, currentWindow: true}, function(tabs) { - if (!tabs[0]?.id) return; + [visualSize, numVisuals, numEmojis, numWords, lastNSentences].forEach(input => { + if (input) input.addEventListener('input', updateSettings); + }); + + // Load initial settings + chrome.storage.local.get({ interactiveImageOptions: DEFAULT_INTERACTIVEIMAGE_OPTIONS }, (response) => { + const options = response.interactiveImageOptions; + if (enableButton) enableButton.checked = options.enableButton; + if (proactivenessSelect) proactivenessSelect.value = options.proactiveness; + if (enableAllCaptions) enableAllCaptions.checked = options.enableAllCaptions; + if (enableEmoji) enableEmoji.checked = options.enableEmoji; + if (enablePersonal) enablePersonal.checked = options.enablePersonal; + if (visualSize) visualSize.value = String(options.visualSize); + if (numVisuals) numVisuals.value = String(options.numVisuals); + if (numEmojis) numEmojis.value = String(options.numEmojis); + if (numWords) numWords.value = String(options.numWords); + if (lastNSentences) lastNSentences.value = String(options.lastNSentences); - chrome.tabs.sendMessage(tabs[0].id, { - type: 'GET_STATE' + // Update value displays + if (visualSizeValue) visualSizeValue.textContent = String(options.visualSize); + if (numVisualsValue) numVisualsValue.textContent = String(options.numVisuals); + if (numEmojisValue) numEmojisValue.textContent = String(options.numEmojis); + if (numWordsValue) numWordsValue.textContent = String(options.numWords); + if (lastNSentencesValue) lastNSentencesValue.textContent = String(options.lastNSentences); + }); +} + +function setupSoundOptions() { + // Get references to all sound option elements + const sourceLanguage = document.getElementById('source-language') as HTMLSelectElement; + const summarizationMode = document.getElementById('summarization-mode') as HTMLSelectElement; + const fontFamily = document.getElementById('font-family') as HTMLSelectElement; + const fontSize = document.getElementById('font-size') as HTMLSelectElement; + const backgroundColor = document.getElementById('background-color') as HTMLSelectElement; + const summaryDelay = document.getElementById('summary-delay') as HTMLSelectElement; + const summaryShowTime = document.getElementById('summary-show-time') as HTMLSelectElement; + const summaryMaxWords = document.getElementById('summary-max-words') as HTMLSelectElement; + const summaryMinWords = document.getElementById('summary-min-words') as HTMLSelectElement; + const summaryColor = document.getElementById('summary-color') as HTMLSelectElement; + const captionColor = document.getElementById('caption-color') as HTMLSelectElement; + const rangeScreenX = document.getElementById('range-screenX') as HTMLInputElement; + const rangeScreenY = document.getElementById('range-screenY') as HTMLInputElement; + const rangeScreenSize = document.getElementById('range-screenSize') as HTMLInputElement; + const useMeetCaptions = document.getElementById('use-meet-captions') as HTMLSelectElement; + const autoCapitalization = document.getElementById('auto-capitalization') as HTMLSelectElement; + const emphasizingMode = document.getElementById('emphasizing-mode') as HTMLSelectElement; + const backgroundImage = document.getElementById('background-image') as HTMLSelectElement; + const textLogMeet = document.getElementById('text-log-meet') as HTMLTextAreaElement; + const textLayout = document.getElementById('text-layout') as HTMLSelectElement; + const summaryLayout = document.getElementById('summary-layout') as HTMLSelectElement; + + // Populate language dropdown if it exists + if (sourceLanguage) { + // Use the LANGUAGES object from the imported sound options + const option = document.createElement('option'); + option.value = LANGUAGES.EN_US; + option.textContent = 'English (United States)'; + sourceLanguage.appendChild(option); + } + + // Populate other dropdowns with enum values + if (summarizationMode) { + Object.values(SummaryMode).forEach(mode => { + const option = document.createElement('option'); + option.value = mode; + option.textContent = mode; + summarizationMode.appendChild(option); + }); + } + + if (useMeetCaptions) { + Object.values(CaptionMode).forEach(mode => { + const option = document.createElement('option'); + option.value = mode; + option.textContent = mode; + useMeetCaptions.appendChild(option); + }); + } + + if (autoCapitalization) { + Object.values(AutoCapitalization).forEach(value => { + const option = document.createElement('option'); + option.value = value; + option.textContent = value; + autoCapitalization.appendChild(option); + }); + } + + if (emphasizingMode) { + Object.values(EmphasizingMode).forEach(value => { + const option = document.createElement('option'); + option.value = value; + option.textContent = value; + emphasizingMode.appendChild(option); }); + } + + if (backgroundImage) { + Object.values(BackgroundImage).forEach(value => { + const option = document.createElement('option'); + option.value = value; + option.textContent = value; + backgroundImage.appendChild(option); + }); + } + + function updateSoundSettings() { + // Create a partial options object with only the elements that exist + const options: Partial = { ...DEFAULT_SOUND_OPTIONS }; + + if (sourceLanguage) options.sourceLanguage = sourceLanguage.value; + if (summarizationMode) options.summarizationMode = summarizationMode.value; + if (fontFamily) options.fontFamily = fontFamily.value; + if (fontSize) options.fontSize = Number(fontSize.value); + if (backgroundColor) options.backgroundColor = backgroundColor.value; + if (summaryDelay) options.summaryDelay = Number(summaryDelay.value); + if (summaryShowTime) options.lamdaShowTimeMs = Number(summaryShowTime.value); + if (summaryMaxWords) options.summaryMaxWords = Number(summaryMaxWords.value); + if (summaryMinWords) options.summaryMinWords = Number(summaryMinWords.value); + if (summaryColor) options.summaryColor = summaryColor.value; + if (captionColor) options.captionColor = captionColor.value; + if (rangeScreenX) options.screenX = Number(rangeScreenX.value); + if (rangeScreenY) options.screenY = Number(rangeScreenY.value); + if (rangeScreenSize) options.zoomRatio = Number(rangeScreenSize.value); + if (useMeetCaptions) options.useMeetCaptions = useMeetCaptions.value; + if (autoCapitalization) options.autoCapitalization = autoCapitalization.value; + if (emphasizingMode) options.emphasizingMode = emphasizingMode.value; + if (backgroundImage) options.backgroundImage = backgroundImage.value; + if (textLayout) options.textLayout = textLayout.value; + if (summaryLayout) options.summaryLayout = summaryLayout.value; + + chrome.storage.local.set({ soundOptions: options as SoundOptions }); + } + + // Add event listeners to elements that exist + [sourceLanguage, summarizationMode, fontFamily, fontSize, backgroundColor, + summaryDelay, summaryShowTime, summaryMaxWords, summaryMinWords, + summaryColor, captionColor, useMeetCaptions, autoCapitalization, + emphasizingMode, backgroundImage, textLayout, summaryLayout].forEach(element => { + if (element) { + element.addEventListener('change', updateSoundSettings); + } }); -}); + + [rangeScreenX, rangeScreenY, rangeScreenSize].forEach(element => { + if (element) { + element.addEventListener('input', updateSoundSettings); + } + }); + + if (textLogMeet) { + textLogMeet.addEventListener('input', updateSoundSettings); + } + + // Load initial settings + chrome.storage.local.get({ soundOptions: DEFAULT_SOUND_OPTIONS }, (response) => { + const options = response.soundOptions; + if (sourceLanguage) sourceLanguage.value = options.sourceLanguage; + if (summarizationMode && options.summarizationMode) summarizationMode.value = options.summarizationMode; + if (fontFamily) fontFamily.value = options.fontFamily; + if (fontSize) fontSize.value = String(options.fontSize); + if (backgroundColor) backgroundColor.value = options.backgroundColor; + if (summaryDelay) summaryDelay.value = String(options.summaryDelay / 1000); // Convert to seconds for display + if (summaryShowTime) summaryShowTime.value = String(options.lamdaShowTimeMs / 1000); // Convert to seconds for display + if (summaryMaxWords) summaryMaxWords.value = String(options.summaryMaxWords); + if (summaryMinWords) summaryMinWords.value = String(options.summaryMinWords); + if (summaryColor) summaryColor.value = options.summaryColor; + if (captionColor) captionColor.value = options.captionColor; + if (rangeScreenX) rangeScreenX.value = String(options.screenX); + if (rangeScreenY) rangeScreenY.value = String(options.screenY); + if (rangeScreenSize) rangeScreenSize.value = String(options.zoomRatio * 100); // Convert to percentage for display + if (useMeetCaptions && options.useMeetCaptions) useMeetCaptions.value = options.useMeetCaptions; + if (autoCapitalization && options.autoCapitalization) autoCapitalization.value = options.autoCapitalization; + if (emphasizingMode && options.emphasizingMode) emphasizingMode.value = options.emphasizingMode; + if (backgroundImage && options.backgroundImage) backgroundImage.value = options.backgroundImage; + if (textLogMeet) textLogMeet.value = options.textLogMeet || ''; + if (textLayout && options.textLayout) textLayout.value = options.textLayout; + if (summaryLayout && options.summaryLayout) summaryLayout.value = options.summaryLayout; + }); +} + +// Setup caption settings event listeners +function setupCaptionSettings() { + const enableCaptions = document.getElementById('enable-captions') as HTMLInputElement; + const aiProactiveness = document.getElementById('ai-proactiveness') as HTMLSelectElement; + + if (enableCaptions) { + enableCaptions.addEventListener('change', () => { + currentState.captionsEnabled = enableCaptions.checked; + sendStateUpdate(); + }); + } + + if (aiProactiveness) { + aiProactiveness.addEventListener('change', () => { + currentState.aiProactiveness = aiProactiveness.value; + sendStateUpdate(); + }); + } +} + +// Setup advanced settings event listeners +function setupAdvancedSettings() { + const allParticipants = document.getElementById('all-participants') as HTMLInputElement; + const suggestEmojis = document.getElementById('suggest-emojis') as HTMLInputElement; + const suggestPersonal = document.getElementById('suggest-personal') as HTMLInputElement; + const modelCapability = document.getElementById('model-capability') as HTMLSelectElement; + + const minWords = document.getElementById('min-words') as HTMLInputElement; + const minWordsValue = document.getElementById('min-words-value') as HTMLSpanElement; + + const lastNSentences = document.getElementById('last-n-sentences') as HTMLInputElement; + const lastNSentencesValue = document.getElementById('last-n-sentences-value') as HTMLSpanElement; + + const maxVisuals = document.getElementById('max-visuals') as HTMLInputElement; + const maxVisualsValue = document.getElementById('max-visuals-value') as HTMLSpanElement; + + const maxEmojis = document.getElementById('max-emojis') as HTMLInputElement; + const maxEmojisValue = document.getElementById('max-emojis-value') as HTMLSpanElement; + + const visualSize = document.getElementById('visual-size') as HTMLInputElement; + const visualSizeValue = document.getElementById('visual-size-value') as HTMLSpanElement; + + // Checkbox event listeners + if (allParticipants) { + allParticipants.addEventListener('change', () => { + currentState.allParticipants = allParticipants.checked; + sendStateUpdate(); + }); + } + + if (suggestEmojis) { + suggestEmojis.addEventListener('change', () => { + currentState.suggestEmojis = suggestEmojis.checked; + sendStateUpdate(); + }); + } + + if (suggestPersonal) { + suggestPersonal.addEventListener('change', () => { + currentState.suggestPersonal = suggestPersonal.checked; + sendStateUpdate(); + }); + } + + // Dropdown event listeners + if (modelCapability) { + modelCapability.addEventListener('change', () => { + currentState.modelCapability = modelCapability.value; + sendStateUpdate(); + }); + } + + // Slider event listeners + if (minWords && minWordsValue) { + minWords.addEventListener('input', () => { + currentState.minWords = parseInt(minWords.value); + minWordsValue.textContent = minWords.value; + sendStateUpdate(); + }); + } + + if (lastNSentences && lastNSentencesValue) { + lastNSentences.addEventListener('input', () => { + currentState.lastNSentences = parseInt(lastNSentences.value); + lastNSentencesValue.textContent = lastNSentences.value; + sendStateUpdate(); + }); + } + + if (maxVisuals && maxVisualsValue) { + maxVisuals.addEventListener('input', () => { + currentState.maxVisuals = parseInt(maxVisuals.value); + maxVisualsValue.textContent = maxVisuals.value; + sendStateUpdate(); + }); + } + + if (maxEmojis && maxEmojisValue) { + maxEmojis.addEventListener('input', () => { + currentState.maxEmojis = parseInt(maxEmojis.value); + maxEmojisValue.textContent = maxEmojis.value; + sendStateUpdate(); + }); + } + + if (visualSize && visualSizeValue) { + visualSize.addEventListener('input', () => { + currentState.visualSize = parseFloat(visualSize.value); + visualSizeValue.textContent = visualSize.value; + sendStateUpdate(); + }); + } +} + +// Setup logging settings event listeners +function setupLoggingSettings() { + const enableLogging = document.getElementById('enable-logging') as HTMLInputElement; + const downloadLog = document.getElementById('download-log') as HTMLButtonElement; + + if (enableLogging) { + enableLogging.addEventListener('change', () => { + currentState.loggingEnabled = enableLogging.checked; + sendStateUpdate(); + }); + } + + if (downloadLog) { + downloadLog.addEventListener('click', () => { + sendCommand('DOWNLOAD_LOG', {}); + }); + } +} + +// Setup interactive image settings +function setupInteractiveImageSettings() { + const imageUrl = document.getElementById('image-url') as HTMLInputElement; + const imageName = document.getElementById('image-name') as HTMLInputElement; + const applyImage = document.getElementById('apply-image') as HTMLButtonElement; + + // Add position and size controls + const imageX = document.getElementById('image-x') as HTMLInputElement; + const imageY = document.getElementById('image-y') as HTMLInputElement; + const imageWidth = document.getElementById('image-width') as HTMLInputElement; + const imageHeight = document.getElementById('image-height') as HTMLInputElement; + + // Value displays + const imageXValue = document.getElementById('image-x-value') as HTMLSpanElement; + const imageYValue = document.getElementById('image-y-value') as HTMLSpanElement; + const imageWidthValue = document.getElementById('image-width-value') as HTMLSpanElement; + const imageHeightValue = document.getElementById('image-height-value') as HTMLSpanElement; + + // Update value displays when sliders change + if (imageX && imageXValue) { + imageX.addEventListener('input', () => { + imageXValue.textContent = imageX.value; + }); + } + + if (imageY && imageYValue) { + imageY.addEventListener('input', () => { + imageYValue.textContent = imageY.value; + }); + } + + if (imageWidth && imageWidthValue) { + imageWidth.addEventListener('input', () => { + imageWidthValue.textContent = imageWidth.value; + }); + } + + if (imageHeight && imageHeightValue) { + imageHeight.addEventListener('input', () => { + imageHeightValue.textContent = imageHeight.value; + }); + } + + if (applyImage && imageUrl) { + applyImage.addEventListener('click', () => { + if (imageUrl.value) { + // Default values if sliders don't exist + const x = imageX ? parseFloat(imageX.value) : 0.1; + const y = imageY ? parseFloat(imageY.value) : 0.1; + const width = imageWidth ? parseFloat(imageWidth.value) : 0.3; + const height = imageHeight ? parseFloat(imageHeight.value) : 0.3; + + currentState.interactiveImage = { + url: imageUrl.value, + name: imageName ? imageName.value || 'Image' : 'Image' + }; + + sendCommand('SET_IMAGE', { + url: imageUrl.value, + name: imageName ? imageName.value || 'Image' : 'Image', + x: x, + y: y, + width: width, + height: height + }); + } + }); + } +} + +function toggleInteractiveImageSection(show: boolean) { + const section = document.getElementById('interactive-image-section'); + if (section) { + section.style.display = show ? 'block' : 'none'; + } +} + +// Send state update to content script +function sendStateUpdate() { + chrome.runtime.sendMessage({ + type: 'SIDE_PANEL_TO_CONTENT', + command: 'UPDATE_STATE', + data: currentState + }); + + updateStatusDisplay(); +} + +// Send specific command to content script +function sendCommand(command: string, data: any) { + chrome.runtime.sendMessage({ + type: 'SIDE_PANEL_TO_CONTENT', + command: command, + data: data + }); +} + +// Update status display +function updateStatusDisplay() { + const statusDisplay = document.getElementById('status-display'); + if (statusDisplay) { + statusDisplay.textContent = `Scene: ${currentState.currentScene || 'None'} | Captions: ${currentState.captionsEnabled ? 'On' : 'Off'}`; + } +} + +// Update UI based on current state +function updateUI(state: ARChatState) { + // Update the state object + currentState = {...currentState, ...state}; + + // Update scene selector + const sceneSelector = document.getElementById('scene-selector') as HTMLSelectElement; + if (sceneSelector && state.currentScene) { + sceneSelector.value = state.currentScene; + + // Show/hide Interactive Image settings if appropriate + toggleInteractiveImageSection(state.currentScene === 'InteractiveImage'); + } + + // Update caption settings + const enableCaptions = document.getElementById('enable-captions') as HTMLInputElement; + const aiProactiveness = document.getElementById('ai-proactiveness') as HTMLSelectElement; + + if (enableCaptions && state.captionsEnabled !== undefined) { + enableCaptions.checked = state.captionsEnabled; + } + + if (aiProactiveness && state.aiProactiveness) { + aiProactiveness.value = state.aiProactiveness; + } + + + // Update advanced settings + const allParticipants = document.getElementById('all-participants') as HTMLInputElement; + const suggestEmojis = document.getElementById('suggest-emojis') as HTMLInputElement; + const suggestPersonal = document.getElementById('suggest-personal') as HTMLInputElement; + const modelCapability = document.getElementById('model-capability') as HTMLSelectElement; + + if (allParticipants && state.allParticipants !== undefined) { + allParticipants.checked = state.allParticipants; + } + + if (suggestEmojis && state.suggestEmojis !== undefined) { + suggestEmojis.checked = state.suggestEmojis; + } + + if (suggestPersonal && state.suggestPersonal !== undefined) { + suggestPersonal.checked = state.suggestPersonal; + } + + if (modelCapability && state.modelCapability) { + modelCapability.value = state.modelCapability; + } + + // Update slider values + updateSlider('min-words', 'min-words-value', state.minWords); + updateSlider('last-n-sentences', 'last-n-sentences-value', state.lastNSentences); + updateSlider('max-visuals', 'max-visuals-value', state.maxVisuals); + updateSlider('max-emojis', 'max-emojis-value', state.maxEmojis); + updateSlider('visual-size', 'visual-size-value', state.visualSize); + + // Update logging settings + const enableLogging = document.getElementById('enable-logging') as HTMLInputElement; + if (enableLogging && state.loggingEnabled !== undefined) { + enableLogging.checked = state.loggingEnabled; + } + + // Update status display + updateStatusDisplay(); +} + +// Helper to update slider and its value display +function updateSlider(sliderId: string, valueId: string, value: number | undefined) { + if (value !== undefined) { + const slider = document.getElementById(sliderId) as HTMLInputElement; + const valueDisplay = document.getElementById(valueId) as HTMLSpanElement; + + if (slider && valueDisplay) { + slider.value = value.toString(); + valueDisplay.textContent = value.toString(); + } + } +} // Listen for updates from content script chrome.runtime.onMessage.addListener((message) => { if (message.type === 'CONTENT_TO_SIDE_PANEL') { updateUI(message.state); } -}); - -function updateUI(state: any) { - // Update UI based on current state - const sceneSelector = document.getElementById('scene-selector') as HTMLSelectElement; - sceneSelector.value = state.currentScene; -} \ No newline at end of file +}); \ No newline at end of file