From 26f78dd9529c772cc273c00dca17e9f95759c96c Mon Sep 17 00:00:00 2001 From: Philippe Date: Mon, 22 Jun 2026 10:07:18 +0200 Subject: [PATCH 1/3] feat(keystrokes): record keystrokes during capture and expose via extension API Wire up uiohook-napi keyboard events alongside existing mouse capture to record keystrokes with playback timestamps. Save to a .keys.json sidecar on recording stop (mirrors .cursor.json pattern). Expose via IPC + extensionHost.setKeystrokeEvents so getKeystrokesInRange() works in extensions. Co-Authored-By: Claude Sonnet 4.6 --- electron/ipc/constants.ts | 1 + electron/ipc/cursor/interaction.ts | 59 ++++++++++++++++++++- electron/ipc/cursor/telemetry.ts | 57 ++++++++++++++++++++ electron/ipc/recording/mac.ts | 8 +++ electron/ipc/register/recording.ts | 23 ++++++++ electron/ipc/state.ts | 8 +++ electron/ipc/types.ts | 22 ++++++-- electron/ipc/utils.ts | 4 ++ electron/preload.ts | 3 ++ src/components/video-editor/VideoEditor.tsx | 13 +++++ 10 files changed, 194 insertions(+), 4 deletions(-) diff --git a/electron/ipc/constants.ts b/electron/ipc/constants.ts index 2c5cf8f17..edc501f46 100644 --- a/electron/ipc/constants.ts +++ b/electron/ipc/constants.ts @@ -29,3 +29,4 @@ export const COMPANION_AUDIO_LAYOUTS = [ export const CURSOR_TELEMETRY_VERSION = 2; export const CURSOR_SAMPLE_INTERVAL_MS = 33; export const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz +export const KEYSTROKE_TELEMETRY_VERSION = 1; diff --git a/electron/ipc/cursor/interaction.ts b/electron/ipc/cursor/interaction.ts index 9b6f3ac92..acff32125 100644 --- a/electron/ipc/cursor/interaction.ts +++ b/electron/ipc/cursor/interaction.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; -import type { HookMouseEvent, UiohookLike, UiohookModuleNamespace, CursorInteractionType } from "../types"; +import type { HookKeyboardEvent, HookMouseEvent, UiohookLike, UiohookModuleNamespace, CursorInteractionType } from "../types"; import { isCursorCaptureActive, interactionCaptureCleanup, @@ -18,8 +18,41 @@ import { getHookCursorScreenPoint, isCursorCapturePaused, pushCursorSample, + pushKeystroke, } from "./telemetry"; +// Maps uiohook-napi keycodes to Web KeyboardEvent.key values for common keys. +// Full keycode table: https://github.com/kwhat/uiohook/blob/master/include/uiohook.h +const KEYCODE_MAP: Record = { + 1: "Escape", 2: "1", 3: "2", 4: "3", 5: "4", 6: "5", 7: "6", 8: "7", 9: "8", 10: "9", + 11: "0", 12: "-", 13: "=", 14: "Backspace", 15: "Tab", + 16: "q", 17: "w", 18: "e", 19: "r", 20: "t", 21: "y", 22: "u", 23: "i", 24: "o", 25: "p", + 26: "[", 27: "]", 28: "Enter", + 30: "a", 31: "s", 32: "d", 33: "f", 34: "g", 35: "h", 36: "j", 37: "k", 38: "l", + 39: ";", 40: "'", 41: "`", 43: "\\", + 44: "z", 45: "x", 46: "c", 47: "v", 48: "b", 49: "n", 50: "m", + 51: ",", 52: ".", 53: "/", 57: " ", + 58: "CapsLock", + 59: "F1", 60: "F2", 61: "F3", 62: "F4", 63: "F5", 64: "F6", + 65: "F7", 66: "F8", 67: "F9", 68: "F10", 87: "F11", 88: "F12", + 71: "7", 72: "8", 73: "9", 75: "4", 76: "5", 77: "6", 79: "1", 80: "2", 81: "3", 82: "0", + 3639: "Meta", 3640: "Shift", 3641: "Control", 3642: "Alt", + 3675: "Meta", 3676: "Meta", + 57416: "ArrowUp", 57419: "ArrowLeft", 57421: "ArrowRight", 57424: "ArrowDown", + 57426: "Insert", 57427: "Delete", 57418: "PageUp", 57422: "PageDown", + 57423: "End", 57415: "Home", +}; + +const MODIFIER_KEYCODES = new Set([3640, 3641, 3642, 3643, 3675, 3676, 3639]); + +function keycodeToKey(keycode: number | undefined): string | null { + if (keycode === undefined) return null; + return KEYCODE_MAP[keycode] ?? null; +} + +// Track active modifiers during recording +const _activeModifiers = new Set(); + const nodeRequire = createRequire(import.meta.url); export function normalizeHookMouseButton(rawButton: unknown): 1 | 2 | 3 { @@ -271,23 +304,47 @@ export async function startInteractionCapture() { setLinuxCursorScreenPoint({ x: point.x, y: point.y, updatedAt: Date.now() }); }; + const onKeyDown = (event: HookKeyboardEvent) => { + if (!isCursorCaptureActive || isCursorCapturePaused()) return; + const key = keycodeToKey(event.keycode); + if (!key) return; + if (MODIFIER_KEYCODES.has(event.keycode ?? -1)) { + _activeModifiers.add(key); + return; // don't emit bare modifier events + } + const timeMs = getCursorCaptureElapsedMs(); + pushKeystroke({ timeMs, key, modifiers: Array.from(_activeModifiers) }); + }; + + const onKeyUp = (event: HookKeyboardEvent) => { + const key = keycodeToKey(event.keycode); + if (key) _activeModifiers.delete(key); + }; + hook.on("mousedown", onMouseDown); hook.on("mouseup", onMouseUp); + hook.on("keydown", onKeyDown); + hook.on("keyup", onKeyUp); if (process.platform === "linux") { hook.on("mousemove", onMouseMove); } setInteractionCaptureCleanup(() => { + _activeModifiers.clear(); try { if (typeof hook.off === "function") { hook.off("mousedown", onMouseDown); hook.off("mouseup", onMouseUp); + hook.off("keydown", onKeyDown); + hook.off("keyup", onKeyUp); if (process.platform === "linux") { hook.off("mousemove", onMouseMove); } } else if (typeof hook.removeListener === "function") { hook.removeListener("mousedown", onMouseDown); hook.removeListener("mouseup", onMouseUp); + hook.removeListener("keydown", onKeyDown); + hook.removeListener("keyup", onKeyUp); if (process.platform === "linux") { hook.removeListener("mousemove", onMouseMove); } diff --git a/electron/ipc/cursor/telemetry.ts b/electron/ipc/cursor/telemetry.ts index ebedfe72a..64cf6c821 100644 --- a/electron/ipc/cursor/telemetry.ts +++ b/electron/ipc/cursor/telemetry.ts @@ -326,3 +326,60 @@ export function startCursorSampling() { export { CURSOR_SAMPLE_INTERVAL_MS } from "../constants"; // Re-export for consumers that use it from this module export { getTelemetryPathForVideo } from "../utils"; + +// ── Keystroke telemetry ─────────────────────────────────────────────────────── + +import { KEYSTROKE_TELEMETRY_VERSION } from "../constants"; +import { activeKeystrokeEvents, setActiveKeystrokeEvents } from "../state"; +import type { KeystrokeEvent } from "../types"; +import { getKeystrokePathForVideo } from "../utils"; + +export function pushKeystroke(event: KeystrokeEvent): void { + setActiveKeystrokeEvents([...activeKeystrokeEvents, event]); +} + +export function getActiveKeystrokes(): KeystrokeEvent[] { + return activeKeystrokeEvents; +} + +export async function saveKeystrokeTelemetry(videoPath: string): Promise { + const events = activeKeystrokeEvents; + const sidecarPath = getKeystrokePathForVideo(videoPath); + + if (events.length === 0) { + await fs.rm(sidecarPath, { force: true }); + return; + } + + await fs.writeFile( + sidecarPath, + JSON.stringify({ version: KEYSTROKE_TELEMETRY_VERSION, events }, null, 2), + "utf-8", + ); +} + +export async function loadKeystrokeTelemetry(videoPath: string): Promise { + const sidecarPath = getKeystrokePathForVideo(videoPath); + + try { + const raw = await fs.readFile(sidecarPath, "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || parsed.version !== KEYSTROKE_TELEMETRY_VERSION || !Array.isArray(parsed.events)) { + return []; + } + return parsed.events.filter( + (e: unknown): e is KeystrokeEvent => + e !== null && + typeof e === "object" && + typeof (e as KeystrokeEvent).timeMs === "number" && + typeof (e as KeystrokeEvent).key === "string" && + Array.isArray((e as KeystrokeEvent).modifiers), + ); + } catch { + return []; + } +} + +export function resetKeystrokeTelemetry(): void { + setActiveKeystrokeEvents([]); +} diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index 01951345d..66576b0d9 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -3,6 +3,8 @@ import fs from "node:fs/promises"; import { BrowserWindow } from "electron"; import { persistPendingCursorTelemetry, + resetKeystrokeTelemetry, + saveKeystrokeTelemetry, snapshotCursorTelemetryForPersistence, } from "../cursor/telemetry"; import { @@ -227,6 +229,12 @@ export async function finalizeStoredVideo(videoPath: string) { } catch (error) { console.warn("[mac-stop] Failed to persist cursor telemetry:", error); } + try { + await saveKeystrokeTelemetry(videoPath); + } catch (error) { + console.warn("[mac-stop] Failed to persist keystroke telemetry:", error); + } + resetKeystrokeTelemetry(); if (isAutoRecordingPath(videoPath)) { await pruneAutoRecordings([videoPath]); } diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index b13453c74..8ef928db3 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -29,6 +29,9 @@ import { startCursorSampling, stopCursorCapture, writeCursorTelemetry, + loadKeystrokeTelemetry, + saveKeystrokeTelemetry, + resetKeystrokeTelemetry, } from "../cursor/telemetry"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { @@ -997,6 +1000,12 @@ export function registerRecordingHandlers( } catch (error) { console.warn("Failed to persist cursor telemetry during native stop:", error); } + try { + await saveKeystrokeTelemetry(finalVideoPath); + } catch (error) { + console.warn("Failed to persist keystroke telemetry during native stop:", error); + } + resetKeystrokeTelemetry(); return { success: true, path: finalVideoPath }; } catch (error) { @@ -1893,6 +1902,20 @@ export function registerRecordingHandlers( } }); + ipcMain.handle("get-keystroke-telemetry", async (_, videoPath?: string) => { + const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath); + if (!targetVideoPath) { + return { success: true, events: [] }; + } + try { + const events = await loadKeystrokeTelemetry(targetVideoPath); + return { success: true, events }; + } catch (error) { + console.error("Failed to load keystroke telemetry:", error); + return { success: false, events: [], error: String(error) }; + } + }); + ipcMain.handle( "set-cursor-telemetry", async (_, videoPath: string | undefined, samples: CursorTelemetryPoint[]) => { diff --git a/electron/ipc/state.ts b/electron/ipc/state.ts index a0a41744e..71bb6bd04 100644 --- a/electron/ipc/state.ts +++ b/electron/ipc/state.ts @@ -3,6 +3,7 @@ import type { CursorInteractionType, CursorTelemetryPoint, CursorVisualType, + KeystrokeEvent, NativeCaptureDiagnostics, RecordingSessionData, SelectedSource, @@ -291,3 +292,10 @@ export function setCachedNativeVideoEncoder( export function setNativeHelperMigrationPromise(v: Promise | null) { nativeHelperMigrationPromise = v; } + +// ── Keystroke telemetry ─────────────────────────────────────────────────────── +export let activeKeystrokeEvents: KeystrokeEvent[] = []; + +export function setActiveKeystrokeEvents(v: KeystrokeEvent[]) { + activeKeystrokeEvents = v; +} diff --git a/electron/ipc/types.ts b/electron/ipc/types.ts index 58f5425bd..6e2e6f63c 100644 --- a/electron/ipc/types.ts +++ b/electron/ipc/types.ts @@ -141,10 +141,26 @@ export type HookMouseEvent = { export type HookEventListener = (event: HookMouseEvent) => void; +export type HookKeyboardEvent = { + keycode?: number; + rawcode?: number; + type?: "keydown" | "keyup"; +}; + +export type HookKeyboardEventName = "keydown" | "keyup"; + +export type HookKeyboardListener = (event: HookKeyboardEvent) => void; + +export type KeystrokeEvent = { + timeMs: number; + key: string; + modifiers: string[]; +}; + export type UiohookLike = { - on: (eventName: HookEventName, listener: HookEventListener) => void; - off?: (eventName: HookEventName, listener: HookEventListener) => void; - removeListener?: (eventName: HookEventName, listener: HookEventListener) => void; + on: (eventName: HookEventName | HookKeyboardEventName, listener: HookEventListener | HookKeyboardListener) => void; + off?: (eventName: HookEventName | HookKeyboardEventName, listener: HookEventListener | HookKeyboardListener) => void; + removeListener?: (eventName: HookEventName | HookKeyboardEventName, listener: HookEventListener | HookKeyboardListener) => void; start: () => void; stop?: () => void; }; diff --git a/electron/ipc/utils.ts b/electron/ipc/utils.ts index 3f2efb065..020f72c9c 100644 --- a/electron/ipc/utils.ts +++ b/electron/ipc/utils.ts @@ -67,6 +67,10 @@ export function getTelemetryPathForVideo(videoPath: string) { return `${videoPath}.cursor.json`; } +export function getKeystrokePathForVideo(videoPath: string) { + return `${videoPath}.keys.json`; +} + export function isAutoRecordingPath(filePath: string) { return path.basename(filePath).startsWith(AUTO_RECORDING_PREFIX); } diff --git a/electron/preload.ts b/electron/preload.ts index c339ad3ce..d3781c31a 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -584,6 +584,9 @@ contextBridge.exposeInMainWorld("electronAPI", { setCursorTelemetry: (videoPath: string | undefined, samples: CursorTelemetryPoint[]) => { return ipcRenderer.invoke("set-cursor-telemetry", videoPath, samples); }, + getKeystrokeTelemetry: (videoPath?: string) => { + return ipcRenderer.invoke("get-keystroke-telemetry", videoPath); + }, getSystemCursorAssets: () => { return ipcRenderer.invoke("get-system-cursor-assets"); }, diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index e26fe4231..7c714cd1c 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -3338,6 +3338,19 @@ export default function VideoEditor() { }; }, [videoPath, videoSourcePath]); + useEffect(() => { + if (!videoSourcePath) { + extensionHost.setKeystrokeEvents([]); + return; + } + window.electronAPI + .getKeystrokeTelemetry(videoSourcePath) + .then((result: { success: boolean; events: Array<{ timeMs: number; key: string; modifiers: string[] }> }) => { + extensionHost.setKeystrokeEvents(result.success ? result.events : []); + }) + .catch(() => extensionHost.setKeystrokeEvents([])); + }, [videoSourcePath]); + const normalizedCursorTelemetry = useMemo(() => { if (cursorTelemetry.length === 0) { return [] as CursorTelemetryPoint[]; From 18ab1f6d27354532c3e22e2aba232bb39fd0214f Mon Sep 17 00:00:00 2001 From: Philippe Date: Mon, 22 Jun 2026 10:32:01 +0200 Subject: [PATCH 2/3] feat(keystrokes): ship as built-in extension, fix modifiers, polish animation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Read modifier state from uiohook event booleans (shiftKey/ctrlKey/altKey/ metaKey) instead of hand-tracking keycodes — fixes Shift not showing in chords like Shift+Cmd+V, and deletes the manual modifier set + keyup handler - Correct the extended nav keycodes (Home/End/PageUp/PageDown/Insert/Delete) to match uiohook-napi's UiohookKey constants - Bundle the overlay as a built-in extension (public/builtin-extensions/ keystrokes) so it auto-activates and ships with the app; settings nest under the cursor section via parentSection - keyviz-style entrance/exit animation (pop + slide + fade) and configurable horizontal/vertical margins; drop shadow on badges Co-Authored-By: Claude Sonnet 4.6 --- electron-builder.json5 | 4 + electron/electron-env.d.ts | 5 + electron/ipc/cursor/interaction.ts | 38 ++--- electron/ipc/types.ts | 4 + public/builtin-extensions/keystrokes/index.js | 153 ++++++++++++++++++ .../keystrokes/recordly-extension.json | 10 ++ 6 files changed, 188 insertions(+), 26 deletions(-) create mode 100644 public/builtin-extensions/keystrokes/index.js create mode 100644 public/builtin-extensions/keystrokes/recordly-extension.json diff --git a/electron-builder.json5 b/electron-builder.json5 index a333a8723..22d6898b9 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -36,6 +36,10 @@ { "from": "public/wallpapers", "to": "assets/wallpapers" + }, + { + "from": "public/builtin-extensions", + "to": "builtin-extensions" } ], "publish": [ diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index fdc19ccbc..ed1399f47 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -581,6 +581,11 @@ interface Window { message?: string; error?: string; }>; + getKeystrokeTelemetry: (videoPath?: string) => Promise<{ + success: boolean; + events: Array<{ timeMs: number; key: string; modifiers: string[] }>; + error?: string; + }>; getSystemCursorAssets: () => Promise<{ success: boolean; cursors: Record; diff --git a/electron/ipc/cursor/interaction.ts b/electron/ipc/cursor/interaction.ts index acff32125..931dfa520 100644 --- a/electron/ipc/cursor/interaction.ts +++ b/electron/ipc/cursor/interaction.ts @@ -21,8 +21,10 @@ import { pushKeystroke, } from "./telemetry"; -// Maps uiohook-napi keycodes to Web KeyboardEvent.key values for common keys. -// Full keycode table: https://github.com/kwhat/uiohook/blob/master/include/uiohook.h +// Maps uiohook-napi keycodes to Web KeyboardEvent.key values. Values match +// uiohook-napi's UiohookKey constants. Modifier keys are intentionally absent — +// modifiers come from the event's shiftKey/ctrlKey/altKey/metaKey booleans, and +// bare modifier presses map to null here so they aren't emitted on their own. const KEYCODE_MAP: Record = { 1: "Escape", 2: "1", 3: "2", 4: "3", 5: "4", 6: "5", 7: "6", 8: "7", 9: "8", 10: "9", 11: "0", 12: "-", 13: "=", 14: "Backspace", 15: "Tab", @@ -36,23 +38,15 @@ const KEYCODE_MAP: Record = { 59: "F1", 60: "F2", 61: "F3", 62: "F4", 63: "F5", 64: "F6", 65: "F7", 66: "F8", 67: "F9", 68: "F10", 87: "F11", 88: "F12", 71: "7", 72: "8", 73: "9", 75: "4", 76: "5", 77: "6", 79: "1", 80: "2", 81: "3", 82: "0", - 3639: "Meta", 3640: "Shift", 3641: "Control", 3642: "Alt", - 3675: "Meta", 3676: "Meta", + 3655: "Home", 3657: "PageUp", 3663: "End", 3665: "PageDown", 3666: "Insert", 3667: "Delete", 57416: "ArrowUp", 57419: "ArrowLeft", 57421: "ArrowRight", 57424: "ArrowDown", - 57426: "Insert", 57427: "Delete", 57418: "PageUp", 57422: "PageDown", - 57423: "End", 57415: "Home", }; -const MODIFIER_KEYCODES = new Set([3640, 3641, 3642, 3643, 3675, 3676, 3639]); - function keycodeToKey(keycode: number | undefined): string | null { if (keycode === undefined) return null; return KEYCODE_MAP[keycode] ?? null; } -// Track active modifiers during recording -const _activeModifiers = new Set(); - const nodeRequire = createRequire(import.meta.url); export function normalizeHookMouseButton(rawButton: unknown): 1 | 2 | 3 { @@ -307,36 +301,29 @@ export async function startInteractionCapture() { const onKeyDown = (event: HookKeyboardEvent) => { if (!isCursorCaptureActive || isCursorCapturePaused()) return; const key = keycodeToKey(event.keycode); - if (!key) return; - if (MODIFIER_KEYCODES.has(event.keycode ?? -1)) { - _activeModifiers.add(key); - return; // don't emit bare modifier events - } + if (!key) return; // bare modifiers map to null and are skipped + const modifiers: string[] = []; + if (event.ctrlKey) modifiers.push("Control"); + if (event.altKey) modifiers.push("Alt"); + if (event.shiftKey) modifiers.push("Shift"); + if (event.metaKey) modifiers.push("Meta"); const timeMs = getCursorCaptureElapsedMs(); - pushKeystroke({ timeMs, key, modifiers: Array.from(_activeModifiers) }); - }; - - const onKeyUp = (event: HookKeyboardEvent) => { - const key = keycodeToKey(event.keycode); - if (key) _activeModifiers.delete(key); + pushKeystroke({ timeMs, key, modifiers }); }; hook.on("mousedown", onMouseDown); hook.on("mouseup", onMouseUp); hook.on("keydown", onKeyDown); - hook.on("keyup", onKeyUp); if (process.platform === "linux") { hook.on("mousemove", onMouseMove); } setInteractionCaptureCleanup(() => { - _activeModifiers.clear(); try { if (typeof hook.off === "function") { hook.off("mousedown", onMouseDown); hook.off("mouseup", onMouseUp); hook.off("keydown", onKeyDown); - hook.off("keyup", onKeyUp); if (process.platform === "linux") { hook.off("mousemove", onMouseMove); } @@ -344,7 +331,6 @@ export async function startInteractionCapture() { hook.removeListener("mousedown", onMouseDown); hook.removeListener("mouseup", onMouseUp); hook.removeListener("keydown", onKeyDown); - hook.removeListener("keyup", onKeyUp); if (process.platform === "linux") { hook.removeListener("mousemove", onMouseMove); } diff --git a/electron/ipc/types.ts b/electron/ipc/types.ts index 6e2e6f63c..d29236b19 100644 --- a/electron/ipc/types.ts +++ b/electron/ipc/types.ts @@ -145,6 +145,10 @@ export type HookKeyboardEvent = { keycode?: number; rawcode?: number; type?: "keydown" | "keyup"; + shiftKey?: boolean; + ctrlKey?: boolean; + altKey?: boolean; + metaKey?: boolean; }; export type HookKeyboardEventName = "keydown" | "keyup"; diff --git a/public/builtin-extensions/keystrokes/index.js b/public/builtin-extensions/keystrokes/index.js new file mode 100644 index 000000000..642799207 --- /dev/null +++ b/public/builtin-extensions/keystrokes/index.js @@ -0,0 +1,153 @@ +// Built-in keystroke overlay. Settings live under the cursor section. + +export function activate(api) { + api.registerSettingsPanel({ + id: "keystrokes", + label: "Keystrokes", + parentSection: "cursor", // nest under the cursor (mouse) settings + fields: [ + { id: "enabled", label: "Show keystrokes", type: "toggle", defaultValue: true }, + { + id: "position", + label: "Position", + type: "select", + defaultValue: "bottom-center", + options: [ + { label: "Bottom Left", value: "bottom-left" }, + { label: "Bottom Center", value: "bottom-center" }, + { label: "Bottom Right", value: "bottom-right" }, + { label: "Top Left", value: "top-left" }, + { label: "Top Center", value: "top-center" }, + { label: "Top Right", value: "top-right" }, + ], + }, + { id: "marginX", label: "Horizontal margin", type: "slider", defaultValue: 40, min: 0, max: 300, step: 4 }, + { id: "marginY", label: "Vertical margin", type: "slider", defaultValue: 48, min: 0, max: 300, step: 4 }, + { id: "fadeMs", label: "Display duration (ms)", type: "slider", defaultValue: 1500, min: 500, max: 4000, step: 100 }, + ], + }); + + api.registerRenderHook("final", (hookCtx) => { + if (api.getSetting("enabled") === false) return; + + const fadeMs = Number(api.getSetting("fadeMs") ?? 1500); + const position = String(api.getSetting("position") ?? "bottom-center"); + const marginX = Number(api.getSetting("marginX") ?? 40); + const marginY = Number(api.getSetting("marginY") ?? 48); + + const events = api.getKeystrokesInRange(hookCtx.timeMs - fadeMs, hookCtx.timeMs + 50); + if (!events.length) return; + + const last = events[events.length - 1]; + const life = hookCtx.timeMs - last.timeMs; + const t = life / fadeMs; + if (t < 0 || t >= 1) return; + + // keyviz-style: pop+slide-up on entrance, hold, fade+drift on exit + const ENTER = 0.1, EXIT = 0.72; + let alpha, slide, scale; + if (t < ENTER) { + const e = 1 - Math.pow(1 - t / ENTER, 3); // ease-out cubic + alpha = e; + slide = (1 - e) * 16; + scale = 0.86 + 0.14 * e; + } else if (t > EXIT) { + const p = (t - EXIT) / (1 - EXIT); + alpha = 1 - p; + slide = -p * 10; + scale = 1; + } else { + alpha = 1; slide = 0; scale = 1; + } + if (alpha <= 0) return; + + const { ctx, width, height } = hookCtx; + + const badges = [ + ...last.modifiers.map((m) => ({ text: fmtMod(m), isMod: true })), + { text: fmtKey(last.key), isMod: false }, + ]; + + const FONT = 18, PX = 14, PY = 8, GAP = 6, R = 8; + ctx.save(); + ctx.font = `600 ${FONT}px -apple-system, system-ui, sans-serif`; + + const bw = badges.map((b) => ctx.measureText(b.text).width + PX * 2); + const totalW = bw.reduce((a, v) => a + v, 0) + GAP * (badges.length - 1); + const rowH = FONT + PY * 2; + + const isRight = position.endsWith("right"); + const isCenter = position.endsWith("center"); + const x0 = isRight ? width - totalW - marginX : isCenter ? (width - totalW) / 2 : marginX; + const y0 = position.startsWith("top") ? marginY : height - marginY - rowH; + + // animate around the row center + const cx = x0 + totalW / 2, cy = y0 + rowH / 2; + ctx.translate(cx, cy + slide); + ctx.scale(scale, scale); + ctx.translate(-cx, -cy); + ctx.globalAlpha = alpha; + + let x = x0; + for (let i = 0; i < badges.length; i++) { + const b = badges[i]; + const w = bw[i]; + + ctx.globalAlpha = alpha * (b.isMod ? 0.75 : 1); + ctx.fillStyle = b.isMod ? "#374151" : "#1e293b"; + ctx.shadowColor = "rgba(0,0,0,0.35)"; + ctx.shadowBlur = 12; + ctx.shadowOffsetY = 3; + rrect(ctx, x, y0, w, rowH, R); + ctx.fill(); + ctx.shadowColor = "transparent"; + ctx.shadowBlur = 0; + ctx.shadowOffsetY = 0; + + if (!b.isMod) { + ctx.globalAlpha = alpha * 0.3; + ctx.strokeStyle = "rgba(255,255,255,0.6)"; + ctx.lineWidth = 1; + rrect(ctx, x, y0, w, rowH, R); + ctx.stroke(); + } + + ctx.globalAlpha = alpha; + ctx.fillStyle = b.isMod ? "#d1d5db" : "#ffffff"; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + ctx.fillText(b.text, x + w / 2, y0 + rowH / 2); + x += w + GAP; + } + + ctx.restore(); + }); +} + +export function deactivate() {} + +function rrect(ctx, x, y, w, h, r) { + ctx.beginPath(); + ctx.moveTo(x + r, y); + ctx.lineTo(x + w - r, y); + ctx.arcTo(x + w, y, x + w, y + r, r); + ctx.lineTo(x + w, y + h - r); + ctx.arcTo(x + w, y + h, x + w - r, y + h, r); + ctx.lineTo(x + r, y + h); + ctx.arcTo(x, y + h, x, y + h - r, r); + ctx.lineTo(x, y + r); + ctx.arcTo(x, y, x + r, y, r); + ctx.closePath(); +} + +function fmtMod(m) { + return { Meta: "⌘", Command: "⌘", Control: "⌃", Ctrl: "⌃", Alt: "⌥", Option: "⌥", Shift: "⇧" }[m] ?? m; +} + +function fmtKey(k) { + return ( + { " ": "Space", Enter: "↩", Escape: "Esc", Backspace: "⌫", Delete: "⌦", + Tab: "⇥", ArrowUp: "↑", ArrowDown: "↓", ArrowLeft: "←", ArrowRight: "→", + CapsLock: "⇪" }[k] ?? (k.length === 1 ? k.toUpperCase() : k) + ); +} diff --git a/public/builtin-extensions/keystrokes/recordly-extension.json b/public/builtin-extensions/keystrokes/recordly-extension.json new file mode 100644 index 000000000..9d8353fac --- /dev/null +++ b/public/builtin-extensions/keystrokes/recordly-extension.json @@ -0,0 +1,10 @@ +{ + "id": "recordly.keystrokes", + "name": "Keystrokes", + "version": "1.0.0", + "description": "Show pressed keys as on-screen badges during playback and export", + "author": "Recordly", + "license": "MIT", + "main": "index.js", + "permissions": ["render", "ui"] +} From 501eb7ab017b4ec13dba36ba7849fc07b4198afe Mon Sep 17 00:00:00 2001 From: Philippe Date: Mon, 22 Jun 2026 10:40:03 +0200 Subject: [PATCH 3/3] feat(keystrokes): keyviz-style keycaps + keyboard layout remap - Redesign overlay as white 3D keycaps (extruded dark base, soft shadow, black glyphs) sized to the canvas; modifiers render glyph + label stacked - Add Keyboard layout setting (QWERTY/AZERTY/QWERTZ): uiohook reports physical key positions, so AZERTY 'A' (physical Q slot) was showing as Q. Remap fixes the common non-US layouts. Co-Authored-By: Claude Sonnet 4.6 --- public/builtin-extensions/keystrokes/index.js | 180 ++++++++++++------ 1 file changed, 121 insertions(+), 59 deletions(-) diff --git a/public/builtin-extensions/keystrokes/index.js b/public/builtin-extensions/keystrokes/index.js index 642799207..585292b9d 100644 --- a/public/builtin-extensions/keystrokes/index.js +++ b/public/builtin-extensions/keystrokes/index.js @@ -1,12 +1,37 @@ -// Built-in keystroke overlay. Settings live under the cursor section. +// Built-in keystroke overlay, keyviz-style keycaps. Settings under cursor section. +// +// uiohook reports PHYSICAL key positions, not characters. The layout setting +// remaps them to the user's keyboard. ponytail: AZERTY/QWERTZ cover the common +// non-US layouts; add more rows here if someone needs Dvorak etc. + +const AZERTY = { q: "a", a: "q", w: "z", z: "w", ";": "m", m: "," }; +const QWERTZ = { y: "z", z: "y" }; + +function remapKey(key, layout) { + if (key.length !== 1) return key; + if (layout === "azerty") return AZERTY[key] ?? key; + if (layout === "qwertz") return QWERTZ[key] ?? key; + return key; +} export function activate(api) { api.registerSettingsPanel({ id: "keystrokes", label: "Keystrokes", - parentSection: "cursor", // nest under the cursor (mouse) settings + parentSection: "cursor", fields: [ { id: "enabled", label: "Show keystrokes", type: "toggle", defaultValue: true }, + { + id: "layout", + label: "Keyboard layout", + type: "select", + defaultValue: "qwerty", + options: [ + { label: "QWERTY", value: "qwerty" }, + { label: "AZERTY", value: "azerty" }, + { label: "QWERTZ", value: "qwertz" }, + ], + }, { id: "position", label: "Position", @@ -21,8 +46,8 @@ export function activate(api) { { label: "Top Right", value: "top-right" }, ], }, - { id: "marginX", label: "Horizontal margin", type: "slider", defaultValue: 40, min: 0, max: 300, step: 4 }, - { id: "marginY", label: "Vertical margin", type: "slider", defaultValue: 48, min: 0, max: 300, step: 4 }, + { id: "marginX", label: "Horizontal margin", type: "slider", defaultValue: 48, min: 0, max: 300, step: 4 }, + { id: "marginY", label: "Vertical margin", type: "slider", defaultValue: 56, min: 0, max: 300, step: 4 }, { id: "fadeMs", label: "Display duration (ms)", type: "slider", defaultValue: 1500, min: 500, max: 4000, step: 100 }, ], }); @@ -32,30 +57,26 @@ export function activate(api) { const fadeMs = Number(api.getSetting("fadeMs") ?? 1500); const position = String(api.getSetting("position") ?? "bottom-center"); - const marginX = Number(api.getSetting("marginX") ?? 40); - const marginY = Number(api.getSetting("marginY") ?? 48); + const layout = String(api.getSetting("layout") ?? "qwerty"); + const marginX = Number(api.getSetting("marginX") ?? 48); + const marginY = Number(api.getSetting("marginY") ?? 56); const events = api.getKeystrokesInRange(hookCtx.timeMs - fadeMs, hookCtx.timeMs + 50); if (!events.length) return; const last = events[events.length - 1]; - const life = hookCtx.timeMs - last.timeMs; - const t = life / fadeMs; + const t = (hookCtx.timeMs - last.timeMs) / fadeMs; if (t < 0 || t >= 1) return; // keyviz-style: pop+slide-up on entrance, hold, fade+drift on exit const ENTER = 0.1, EXIT = 0.72; let alpha, slide, scale; if (t < ENTER) { - const e = 1 - Math.pow(1 - t / ENTER, 3); // ease-out cubic - alpha = e; - slide = (1 - e) * 16; - scale = 0.86 + 0.14 * e; + const e = 1 - Math.pow(1 - t / ENTER, 3); + alpha = e; slide = (1 - e) * 18; scale = 0.85 + 0.15 * e; } else if (t > EXIT) { const p = (t - EXIT) / (1 - EXIT); - alpha = 1 - p; - slide = -p * 10; - scale = 1; + alpha = 1 - p; slide = -p * 10; scale = 1; } else { alpha = 1; slide = 0; scale = 1; } @@ -63,61 +84,71 @@ export function activate(api) { const { ctx, width, height } = hookCtx; - const badges = [ - ...last.modifiers.map((m) => ({ text: fmtMod(m), isMod: true })), - { text: fmtKey(last.key), isMod: false }, - ]; - - const FONT = 18, PX = 14, PY = 8, GAP = 6, R = 8; - ctx.save(); - ctx.font = `600 ${FONT}px -apple-system, system-ui, sans-serif`; - - const bw = badges.map((b) => ctx.measureText(b.text).width + PX * 2); - const totalW = bw.reduce((a, v) => a + v, 0) + GAP * (badges.length - 1); - const rowH = FONT + PY * 2; + // scale keycaps to the canvas so they read at any resolution + const u = Math.max(44, Math.min(90, Math.round(height * 0.058))); + const faceH = u; + const lip = Math.round(u * 0.13); + const radius = Math.round(u * 0.18); + const gap = Math.round(u * 0.18); + const padH = Math.round(u * 0.34); + const minW = u; + const capH = faceH + lip; + + const fRegBig = `700 ${Math.round(u * 0.46)}px -apple-system, system-ui, sans-serif`; + const fRegSmall = `600 ${Math.round(u * 0.26)}px -apple-system, system-ui, sans-serif`; + const fGlyph = `600 ${Math.round(u * 0.32)}px -apple-system, system-ui, sans-serif`; + const fLabel = `500 ${Math.round(u * 0.2)}px -apple-system, system-ui, sans-serif`; + + // Build keycap descriptors: modifiers first (⌃⌥⇧⌘ order), then the key + const caps = last.modifiers.map(modInfo); + const keyText = fmtKey(remapKey(last.key, layout)); + caps.push({ glyph: keyText, label: null, big: keyText.length === 1 }); + + // Measure widths + for (const c of caps) { + if (c.label) { + ctx.font = fGlyph; + const gw = ctx.measureText(c.glyph).width; + ctx.font = fLabel; + const lw = ctx.measureText(c.label).width; + c.w = Math.max(minW, Math.max(gw, lw) + padH * 2); + } else { + ctx.font = c.big ? fRegBig : fRegSmall; + c.w = Math.max(minW, ctx.measureText(c.glyph).width + padH * 2); + } + } + const totalW = caps.reduce((a, c) => a + c.w, 0) + gap * (caps.length - 1); const isRight = position.endsWith("right"); const isCenter = position.endsWith("center"); const x0 = isRight ? width - totalW - marginX : isCenter ? (width - totalW) / 2 : marginX; - const y0 = position.startsWith("top") ? marginY : height - marginY - rowH; + const y0 = position.startsWith("top") ? marginY : height - marginY - capH; - // animate around the row center - const cx = x0 + totalW / 2, cy = y0 + rowH / 2; + ctx.save(); + const cx = x0 + totalW / 2, cy = y0 + capH / 2; ctx.translate(cx, cy + slide); ctx.scale(scale, scale); ctx.translate(-cx, -cy); ctx.globalAlpha = alpha; let x = x0; - for (let i = 0; i < badges.length; i++) { - const b = badges[i]; - const w = bw[i]; - - ctx.globalAlpha = alpha * (b.isMod ? 0.75 : 1); - ctx.fillStyle = b.isMod ? "#374151" : "#1e293b"; - ctx.shadowColor = "rgba(0,0,0,0.35)"; - ctx.shadowBlur = 12; - ctx.shadowOffsetY = 3; - rrect(ctx, x, y0, w, rowH, R); - ctx.fill(); - ctx.shadowColor = "transparent"; - ctx.shadowBlur = 0; - ctx.shadowOffsetY = 0; - - if (!b.isMod) { - ctx.globalAlpha = alpha * 0.3; - ctx.strokeStyle = "rgba(255,255,255,0.6)"; - ctx.lineWidth = 1; - rrect(ctx, x, y0, w, rowH, R); - ctx.stroke(); - } + for (const c of caps) { + drawKeycap(ctx, x, y0, c.w, faceH, lip, radius, u); - ctx.globalAlpha = alpha; - ctx.fillStyle = b.isMod ? "#d1d5db" : "#ffffff"; + ctx.fillStyle = "#1a1a1a"; ctx.textAlign = "center"; - ctx.textBaseline = "middle"; - ctx.fillText(b.text, x + w / 2, y0 + rowH / 2); - x += w + GAP; + if (c.label) { + ctx.textBaseline = "middle"; + ctx.font = fGlyph; + ctx.fillText(c.glyph, x + c.w / 2, y0 + faceH * 0.36); + ctx.font = fLabel; + ctx.fillText(c.label, x + c.w / 2, y0 + faceH * 0.7); + } else { + ctx.textBaseline = "middle"; + ctx.font = c.big ? fRegBig : fRegSmall; + ctx.fillText(c.glyph, x + c.w / 2, y0 + faceH / 2); + } + x += c.w + gap; } ctx.restore(); @@ -126,6 +157,27 @@ export function activate(api) { export function deactivate() {} +function drawKeycap(ctx, x, y, w, faceH, lip, r, u) { + // extruded dark base (the black bottom edge) + ctx.save(); + ctx.shadowColor = "rgba(0,0,0,0.28)"; + ctx.shadowBlur = Math.round(u * 0.22); + ctx.shadowOffsetY = Math.round(u * 0.08); + ctx.fillStyle = "#0f1115"; + rrect(ctx, x, y, w, faceH + lip, r); + ctx.fill(); + ctx.restore(); + + // white face + ctx.fillStyle = "#ffffff"; + rrect(ctx, x, y, w, faceH, r); + ctx.fill(); + ctx.strokeStyle = "rgba(0,0,0,0.1)"; + ctx.lineWidth = Math.max(1, u * 0.02); + rrect(ctx, x, y, w, faceH, r); + ctx.stroke(); +} + function rrect(ctx, x, y, w, h, r) { ctx.beginPath(); ctx.moveTo(x + r, y); @@ -140,8 +192,18 @@ function rrect(ctx, x, y, w, h, r) { ctx.closePath(); } -function fmtMod(m) { - return { Meta: "⌘", Command: "⌘", Control: "⌃", Ctrl: "⌃", Alt: "⌥", Option: "⌥", Shift: "⇧" }[m] ?? m; +function modInfo(m) { + return ( + { + Meta: { glyph: "⌘", label: "command" }, + Command: { glyph: "⌘", label: "command" }, + Control: { glyph: "⌃", label: "control" }, + Ctrl: { glyph: "⌃", label: "control" }, + Alt: { glyph: "⌥", label: "option" }, + Option: { glyph: "⌥", label: "option" }, + Shift: { glyph: "⇧", label: "shift" }, + }[m] ?? { glyph: m, label: null, big: m.length === 1 } + ); } function fmtKey(k) {