From 1ad158d2f052cb9432eca7aeb8b852de70638eca Mon Sep 17 00:00:00 2001 From: ukimsanov Date: Tue, 16 Jun 2026 09:34:22 -0700 Subject: [PATCH 1/3] feat(runtime): apply color grading in preview and render --- .../src/inline-scripts/runtimeContract.ts | 2 + packages/core/src/runtime/bridge.test.ts | 26 + packages/core/src/runtime/bridge.ts | 43 + .../core/src/runtime/colorGrading.test.ts | 308 +++++ packages/core/src/runtime/colorGrading.ts | 1231 +++++++++++++++++ packages/core/src/runtime/init.ts | 26 +- packages/core/src/runtime/picker.ts | 8 +- packages/core/src/runtime/types.ts | 5 + packages/core/src/runtime/window.d.ts | 5 + .../core/src/studio-api/routes/preview.ts | 2 +- .../engine/src/services/videoFrameInjector.ts | 20 + packages/player/src/hyperframes-player.ts | 35 + .../producer/src/services/fileServer.test.ts | 74 +- packages/producer/src/services/fileServer.ts | 1 + packages/studio/vite.browser.ts | 10 +- 15 files changed, 1762 insertions(+), 34 deletions(-) create mode 100644 packages/core/src/runtime/colorGrading.test.ts create mode 100644 packages/core/src/runtime/colorGrading.ts diff --git a/packages/core/src/inline-scripts/runtimeContract.ts b/packages/core/src/inline-scripts/runtimeContract.ts index 55d0934905..0ccf38cf49 100644 --- a/packages/core/src/inline-scripts/runtimeContract.ts +++ b/packages/core/src/inline-scripts/runtimeContract.ts @@ -17,6 +17,8 @@ export const HYPERFRAME_CONTROL_ACTIONS = [ "seek", "set-muted", "set-playback-rate", + "set-color-grading", + "set-color-grading-compare", "enable-pick-mode", "disable-pick-mode", ] as const; diff --git a/packages/core/src/runtime/bridge.test.ts b/packages/core/src/runtime/bridge.test.ts index 732b033f57..560d7c19dc 100644 --- a/packages/core/src/runtime/bridge.test.ts +++ b/packages/core/src/runtime/bridge.test.ts @@ -11,6 +11,8 @@ function createMockDeps() { onSetVolume: vi.fn(), onSetMediaOutputMuted: vi.fn(), onSetPlaybackRate: vi.fn(), + onSetColorGrading: vi.fn(), + onSetColorGradingCompare: vi.fn(), onEnablePickMode: vi.fn(), onDisablePickMode: vi.fn(), }; @@ -111,6 +113,30 @@ describe("installRuntimeControlBridge", () => { expect(deps.onSetPlaybackRate).toHaveBeenCalledWith(1); }); + it("dispatches set-color-grading command with target and grading payload", () => { + const deps = createMockDeps(); + const handler = installRuntimeControlBridge(deps); + const grading = { preset: "warm-clean", intensity: 0.7 }; + const target = { id: "hero-video", selectorIndex: 0 }; + handler(makeControlMessage("set-color-grading", { target, grading })); + expect(deps.onSetColorGrading).toHaveBeenCalledWith( + { id: "hero-video", hfId: null, selector: null, selectorIndex: 0 }, + grading, + ); + }); + + it("dispatches set-color-grading-compare command with target and compare payload", () => { + const deps = createMockDeps(); + const handler = installRuntimeControlBridge(deps); + const compare = { enabled: true, position: 0.42 }; + const target = { id: "hero-video", selectorIndex: 0 }; + handler(makeControlMessage("set-color-grading-compare", { target, compare })); + expect(deps.onSetColorGradingCompare).toHaveBeenCalledWith( + { id: "hero-video", hfId: null, selector: null, selectorIndex: 0 }, + compare, + ); + }); + it("dispatches tick command", () => { const deps = createMockDeps(); const handler = installRuntimeControlBridge(deps); diff --git a/packages/core/src/runtime/bridge.ts b/packages/core/src/runtime/bridge.ts index b767bce30d..11b51aa5bb 100644 --- a/packages/core/src/runtime/bridge.ts +++ b/packages/core/src/runtime/bridge.ts @@ -1,4 +1,5 @@ import { swallow } from "./diagnostics"; +import type { HfColorGradingTarget } from "../colorGrading"; import type { RuntimeBridgeControlMessage, RuntimeOutboundMessage } from "./types"; type BridgeDeps = { @@ -10,10 +11,39 @@ type BridgeDeps = { onSetVolume: (volume: number) => void; onSetMediaOutputMuted: (muted: boolean) => void; onSetPlaybackRate: (rate: number) => void; + onSetColorGrading: (target: HfColorGradingTarget | string | null, grading: unknown) => void; + onSetColorGradingCompare: ( + target: HfColorGradingTarget | string | null, + compare: unknown, + ) => void; onEnablePickMode: () => void; onDisablePickMode: () => void; }; +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function readOptionalString(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value : null; +} + +function readOptionalIndex(value: unknown): number | null { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : null; +} + +function readColorGradingTarget(value: unknown): HfColorGradingTarget | string | null { + if (typeof value === "string") return value; + if (!isRecord(value)) return null; + return { + id: readOptionalString(value.id), + hfId: readOptionalString(value.hfId), + selector: readOptionalString(value.selector), + selectorIndex: readOptionalIndex(value.selectorIndex), + }; +} + export function postRuntimeMessage(payload: RuntimeOutboundMessage): void { try { window.parent.postMessage(payload, "*"); @@ -60,6 +90,19 @@ export function installRuntimeControlBridge(deps: BridgeDeps): (event: MessageEv deps.onSetPlaybackRate(Number(data.playbackRate ?? 1)); return; } + if (action === "set-color-grading") { + const payload = isRecord(data) ? data : {}; + deps.onSetColorGrading(readColorGradingTarget(payload.target), payload.grading ?? null); + return; + } + if (action === "set-color-grading-compare") { + const payload = isRecord(data) ? data : {}; + deps.onSetColorGradingCompare( + readColorGradingTarget(payload.target), + payload.compare ?? null, + ); + return; + } if (action === "enable-pick-mode") { deps.onEnablePickMode(); return; diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.ts new file mode 100644 index 0000000000..f5d68e9cd2 --- /dev/null +++ b/packages/core/src/runtime/colorGrading.test.ts @@ -0,0 +1,308 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { HF_COLOR_GRADING_ATTR, serializeHfColorGrading } from "../colorGrading"; +import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading"; + +let lastUniform1f: ReturnType | null = null; +let lastUniform3f: ReturnType | null = null; + +const IDENTITY_2 = ` +LUT_3D_SIZE 2 +0 0 0 +1 0 0 +0 1 0 +1 1 0 +0 0 1 +1 0 1 +0 1 1 +1 1 1 +`; + +function createMockWebGl(): WebGLRenderingContext { + const shader = {}; + const program = {}; + const texture = {}; + const buffer = {}; + const uniform1f = vi.fn(); + const uniform3f = vi.fn(); + lastUniform1f = uniform1f; + lastUniform3f = uniform3f; + return { + VERTEX_SHADER: 0x8b31, + FRAGMENT_SHADER: 0x8b30, + COMPILE_STATUS: 0x8b81, + LINK_STATUS: 0x8b82, + TEXTURE_2D: 0x0de1, + TEXTURE_WRAP_S: 0x2802, + TEXTURE_WRAP_T: 0x2803, + TEXTURE_MIN_FILTER: 0x2801, + TEXTURE_MAG_FILTER: 0x2800, + CLAMP_TO_EDGE: 0x812f, + LINEAR: 0x2601, + NEAREST: 0x2600, + RGBA: 0x1908, + UNSIGNED_BYTE: 0x1401, + ARRAY_BUFFER: 0x8892, + STATIC_DRAW: 0x88e4, + TEXTURE0: 0x84c0, + TEXTURE1: 0x84c1, + FLOAT: 0x1406, + TRIANGLE_STRIP: 0x0005, + UNPACK_FLIP_Y_WEBGL: 0x9240, + createShader: vi.fn(() => shader), + shaderSource: vi.fn(), + compileShader: vi.fn(), + getShaderParameter: vi.fn(() => true), + getShaderInfoLog: vi.fn(() => ""), + deleteShader: vi.fn(), + createProgram: vi.fn(() => program), + attachShader: vi.fn(), + linkProgram: vi.fn(), + getProgramParameter: vi.fn(() => true), + getProgramInfoLog: vi.fn(() => ""), + deleteProgram: vi.fn(), + createTexture: vi.fn(() => texture), + bindTexture: vi.fn(), + texParameteri: vi.fn(), + texImage2D: vi.fn(), + createBuffer: vi.fn(() => buffer), + bindBuffer: vi.fn(), + bufferData: vi.fn(), + getAttribLocation: vi.fn(() => 0), + getUniformLocation: vi.fn((_program, name: string) => name), + viewport: vi.fn(), + useProgram: vi.fn(), + activeTexture: vi.fn(), + pixelStorei: vi.fn(), + uniform1i: vi.fn(), + uniform2f: vi.fn(), + uniform1f, + uniform3f, + enableVertexAttribArray: vi.fn(), + vertexAttribPointer: vi.fn(), + drawArrays: vi.fn(), + deleteTexture: vi.fn(), + } as unknown as WebGLRenderingContext; +} + +function makeDrawableVideo(): HTMLVideoElement { + const video = document.createElement("video"); + video.id = "hero-video"; + video.setAttribute(HF_COLOR_GRADING_ATTR, serializeHfColorGrading({ adjust: { exposure: 0.5 } })); + Object.defineProperty(video, "readyState", { + value: HTMLMediaElement.HAVE_CURRENT_DATA, + configurable: true, + }); + Object.defineProperty(video, "videoWidth", { value: 640, configurable: true }); + Object.defineProperty(video, "videoHeight", { value: 360, configurable: true }); + Object.defineProperty(video, "offsetWidth", { value: 640, configurable: true }); + Object.defineProperty(video, "offsetHeight", { value: 360, configurable: true }); + Object.defineProperty(video, "offsetLeft", { value: 0, configurable: true }); + Object.defineProperty(video, "offsetTop", { value: 0, configurable: true }); + video.getBoundingClientRect = () => + ({ + x: 0, + y: 0, + left: 0, + top: 0, + right: 640, + bottom: 360, + width: 640, + height: 360, + toJSON: () => ({}), + }) as DOMRect; + return video; +} + +function stubCubeLutFetch(): ReturnType { + const fetchMock = vi.fn(() => + Promise.resolve({ + ok: true, + status: 200, + text: () => Promise.resolve(IDENTITY_2), + }), + ); + vi.stubGlobal("fetch", fetchMock); + return fetchMock; +} + +describe("createColorGradingRuntime", () => { + let getContextSpy: ReturnType; + let runtime: RuntimeColorGradingApi | null = null; + + beforeEach(() => { + document.body.innerHTML = ""; + lastUniform1f = null; + lastUniform3f = null; + getContextSpy = vi + .spyOn(HTMLCanvasElement.prototype, "getContext") + .mockImplementation((type: string) => + type === "webgl" ? createMockWebGl() : null, + ) as ReturnType; + }); + + afterEach(() => { + runtime?.destroy(); + runtime = null; + vi.unstubAllGlobals(); + getContextSpy.mockRestore(); + delete window.__hfVariables; + delete window.__hfVariablesByComp; + document.head.innerHTML = ""; + document.body.innerHTML = ""; + }); + + function startRuntimeWithVideo(video = makeDrawableVideo()): { + video: HTMLVideoElement; + canvas: HTMLCanvasElement; + } { + document.body.appendChild(video); + runtime = createColorGradingRuntime(); + const canvas = document.querySelector("[data-hf-color-grading-canvas]"); + if (!canvas) throw new Error("Expected color grading canvas"); + return { video, canvas }; + } + + async function flushLutLoad(): Promise { + await Promise.resolve(); + await Promise.resolve(); + await new Promise((resolve) => window.setTimeout(resolve, 0)); + runtime?.redraw(); + } + + it("re-hides source media after timeline visibility sync", () => { + const { video, canvas } = startRuntimeWithVideo(); + + expect(video.style.getPropertyValue("visibility")).toBe(""); + expect(video.style.getPropertyValue("opacity")).toBe("0"); + expect(video.style.getPropertyPriority("opacity")).toBe("important"); + expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(true); + expect(canvas?.style.visibility).toBe("visible"); + expect(canvas?.style.opacity).toBe("1"); + + video.style.visibility = "visible"; + runtime.setSourceVisibility(video, true); + runtime.redraw(); + + expect(video.style.getPropertyValue("visibility")).toBe("visible"); + expect(video.style.getPropertyValue("opacity")).toBe("0"); + expect(video.style.getPropertyPriority("opacity")).toBe("important"); + expect(canvas?.style.visibility).toBe("visible"); + + video.style.visibility = "hidden"; + runtime.setSourceVisibility(video, false); + runtime.redraw(); + + expect(video.style.getPropertyValue("visibility")).toBe("hidden"); + expect(video.style.getPropertyValue("opacity")).toBe("0"); + expect(video.style.getPropertyPriority("opacity")).toBe("important"); + expect(canvas?.style.visibility).toBe("hidden"); + }); + + it("resolves grading values from the nearest sub-composition variable scope", () => { + window.__hfVariables = { + exposure: -0.25, + }; + window.__hfVariablesByComp = { + card__hf1: { + exposure: 0.75, + }, + }; + const host = document.createElement("div"); + host.setAttribute("data-composition-id", "card__hf1"); + const video = makeDrawableVideo(); + video.id = "first-video"; + video.setAttribute( + HF_COLOR_GRADING_ATTR, + JSON.stringify({ adjust: { exposure: "$exposure" } }), + ); + host.appendChild(video); + document.body.appendChild(host); + + runtime = createColorGradingRuntime(); + + if (!lastUniform1f) throw new Error("Expected WebGL uniform calls"); + expect(lastUniform1f).toHaveBeenCalledWith("u_exposure", 0.75); + }); + + it("falls back to top-level variables for root media color grading", () => { + window.__hfVariables = { + exposure: 0.35, + }; + const video = makeDrawableVideo(); + video.setAttribute( + HF_COLOR_GRADING_ATTR, + JSON.stringify({ adjust: { exposure: "${exposure}" } }), + ); + document.body.appendChild(video); + + runtime = createColorGradingRuntime(); + + if (!lastUniform1f) throw new Error("Expected WebGL uniform calls"); + expect(lastUniform1f).toHaveBeenCalledWith("u_exposure", 0.35); + }); + + it("keeps the last shader frame visible while a video seek is waiting for a drawable frame", () => { + const { video, canvas } = startRuntimeWithVideo(); + + expect(canvas.style.display).toBe("block"); + + Object.defineProperty(video, "readyState", { + value: HTMLMediaElement.HAVE_METADATA, + configurable: true, + }); + + runtime.redraw(); + + expect(canvas.style.display).toBe("block"); + expect(video.style.getPropertyValue("opacity")).toBe("0"); + expect(video.style.getPropertyPriority("opacity")).toBe("important"); + }); + + it("updates before-after compare uniforms without changing the source grading", () => { + const video = makeDrawableVideo(); + document.body.appendChild(video); + + runtime = createColorGradingRuntime(); + const updated = runtime.setCompare("#hero-video", { + enabled: true, + position: 0.25, + lineWidth: 4, + }); + + if (!lastUniform1f) throw new Error("Expected WebGL uniform calls"); + expect(updated).toBe(true); + expect(lastUniform1f).toHaveBeenCalledWith("u_compareEnabled", 1); + expect(lastUniform1f).toHaveBeenCalledWith("u_comparePosition", 0.25); + expect(lastUniform1f).toHaveBeenCalledWith("u_compareLineWidth", 4); + expect(video.getAttribute(HF_COLOR_GRADING_ATTR)).toBe( + serializeHfColorGrading({ adjust: { exposure: 0.5 } }), + ); + }); + + it("loads cube LUTs and enables LUT uniforms", async () => { + const fetchMock = stubCubeLutFetch(); + const origin = window.location.origin; + document.head.innerHTML = ``; + const video = makeDrawableVideo(); + video.setAttribute( + HF_COLOR_GRADING_ATTR, + serializeHfColorGrading({ lut: { src: "assets/luts/identity.cube", intensity: 0.4 } }), + ); + document.body.appendChild(video); + + runtime = createColorGradingRuntime(); + await flushLutLoad(); + + expect(fetchMock).toHaveBeenCalledWith( + `${origin}/api/projects/demo/preview/assets/luts/identity.cube`, + { credentials: "same-origin" }, + ); + if (!lastUniform1f || !lastUniform3f) throw new Error("Expected WebGL uniform calls"); + expect(lastUniform1f).toHaveBeenCalledWith("u_lutEnabled", 1); + expect(lastUniform1f).toHaveBeenCalledWith("u_lutSize", 2); + expect(lastUniform1f).toHaveBeenCalledWith("u_lutIntensity", 0.4); + expect(lastUniform3f).toHaveBeenCalledWith("u_lutDomainMin", 0, 0, 0); + expect(lastUniform3f).toHaveBeenCalledWith("u_lutDomainMax", 1, 1, 1); + expect(runtime.getStatus("#hero-video").message).toBe("Shader + LUT active"); + }); +}); diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.ts new file mode 100644 index 0000000000..3abcbe8b13 --- /dev/null +++ b/packages/core/src/runtime/colorGrading.ts @@ -0,0 +1,1231 @@ +import { + HF_COLOR_GRADING_ATTR, + isHfColorGradingActive, + normalizeHfColorGrading, + normalizeHfColorGradingWithVariables, + type HfColorGradingVariableMap, + type HfColorGradingTarget, + type NormalizedHfColorGrading, +} from "../colorGrading"; +import { packCubeLutToRgba8, parseCubeLut, type CubeLut3D, type CubeLutVec3 } from "../colorLuts"; +import { copyMediaVisualStyles } from "../inline-scripts/parityContract"; +import { swallow } from "./diagnostics"; + +type ColorGradingMediaElement = HTMLVideoElement | HTMLImageElement; + +type EntrySource = "attribute" | "live"; + +interface VideoFrameCallbackMetadata { + mediaTime: number; + presentedFrames: number; + expectedDisplayTime: number; + width: number; + height: number; +} + +type VideoFrameCallback = (now: number, metadata: VideoFrameCallbackMetadata) => void; + +interface VideoFrameCallbackHost { + requestVideoFrameCallback?: (callback: VideoFrameCallback) => number; + cancelVideoFrameCallback?: (handle: number) => void; +} + +interface ProgramInfo { + program: WebGLProgram; + texture: WebGLTexture; + lutTexture: WebGLTexture; + position: number; + source: WebGLUniformLocation | null; + lut: WebGLUniformLocation | null; + resolution: WebGLUniformLocation | null; + uvScale: WebGLUniformLocation | null; + uvOffset: WebGLUniformLocation | null; + lutEnabled: WebGLUniformLocation | null; + lutSize: WebGLUniformLocation | null; + lutTextureSize: WebGLUniformLocation | null; + lutDomainMin: WebGLUniformLocation | null; + lutDomainMax: WebGLUniformLocation | null; + lutIntensity: WebGLUniformLocation | null; + exposure: WebGLUniformLocation | null; + contrast: WebGLUniformLocation | null; + highlights: WebGLUniformLocation | null; + shadows: WebGLUniformLocation | null; + whites: WebGLUniformLocation | null; + blacks: WebGLUniformLocation | null; + temperature: WebGLUniformLocation | null; + tint: WebGLUniformLocation | null; + saturation: WebGLUniformLocation | null; + intensity: WebGLUniformLocation | null; + compareEnabled: WebGLUniformLocation | null; + comparePosition: WebGLUniformLocation | null; + compareSoftness: WebGLUniformLocation | null; + compareLineWidth: WebGLUniformLocation | null; +} + +interface RuntimeColorGradingCompareState { + enabled: boolean; + position: number; + softness: number; + lineWidth: number; +} + +interface ColorGradingEntry { + element: ColorGradingMediaElement; + canvas: HTMLCanvasElement; + gl: WebGLRenderingContext; + program: ProgramInfo; + grading: NormalizedHfColorGrading; + compare: RuntimeColorGradingCompareState; + lut: RuntimeLutTexture | null; + lutLoadingSrc: string | null; + lutError: string | null; + source: EntrySource; + animationFrame: number | null; + videoFrameHandle: number | null; + resizeObserver: ResizeObserver | null; + cleanup: Array<() => void>; + touchedParent: HTMLElement | null; + parentInlinePosition: string | null; + sourceHidden: boolean; + sourceInlineOpacity: string | null; + sourceInlineOpacityPriority: string; + sourceOpacityForCanvas: string; + sourceVisibleForCanvas: boolean; + hasDrawn: boolean; + destroyed: boolean; +} + +export interface RuntimeColorGradingApi { + refresh: () => number; + redraw: () => number; + setGrading: ( + target: HfColorGradingTarget | string | null | undefined, + rawGrading: unknown, + ) => boolean; + clearGrading: (target: HfColorGradingTarget | string | null | undefined) => boolean; + setCompare: ( + target: HfColorGradingTarget | string | null | undefined, + rawCompare: unknown, + ) => boolean; + setSourceVisibility: (target: Element, visible: boolean) => boolean; + getStatus: ( + target: HfColorGradingTarget | string | null | undefined, + ) => RuntimeColorGradingStatus; + destroy: () => void; +} + +export type RuntimeColorGradingStatus = + | { state: "missing"; message: string } + | { state: "inactive"; message: string } + | { state: "pending"; message: string } + | { state: "active"; message: string } + | { state: "unavailable"; message: string }; + +interface HfGlobalWithColorGrading { + colorGrading?: RuntimeColorGradingApi; +} + +type WindowWithColorGrading = Window & { + __hf?: HfGlobalWithColorGrading; + __hyperframes?: { + getVariables?: () => Partial>; + }; + __hfVariables?: Record; + __hfVariablesByComp?: Record>; +}; + +interface RuntimeLutTexture { + src: string; + title: string | null; + size: number; + domainMin: CubeLutVec3; + domainMax: CubeLutVec3; + textureWidth: number; + textureHeight: number; +} + +type LutCacheEntry = + | { state: "pending"; promise: Promise } + | { state: "ready"; lut: CubeLut3D } + | { state: "error"; message: string }; + +const LUT_CACHE = new Map(); +const COLOR_GRADING_CANVAS_ATTR = "data-hf-color-grading-canvas"; +const COLOR_GRADING_SOURCE_HIDDEN_ATTR = "data-hf-color-grading-source-hidden"; +const COLOR_GRADING_CANVAS_CLASS = "__hf_color_grading_canvas__"; +const MAX_LUT_SIZE = 64; +const DEFAULT_COMPARE: RuntimeColorGradingCompareState = { + enabled: false, + position: 0.5, + softness: 0, + lineWidth: 2, +}; + +function readColorGradingRawAttribute(element: Element): string | null { + return element.getAttribute(HF_COLOR_GRADING_ATTR); +} + +function readVariablesForElement(element: Element): HfColorGradingVariableMap { + const win = window as WindowWithColorGrading; + const scope = element.closest("[data-composition-id]"); + const compositionId = scope?.getAttribute("data-composition-id")?.trim() ?? ""; + const scoped = compositionId ? win.__hfVariablesByComp?.[compositionId] : undefined; + if (scoped) return scoped; + + const fromHelper = win.__hyperframes?.getVariables?.(); + if (fromHelper && typeof fromHelper === "object") { + return fromHelper; + } + return win.__hfVariables ?? {}; +} + +function readColorGradingAttribute(element: Element): NormalizedHfColorGrading | null { + const raw = readColorGradingRawAttribute(element); + if (raw == null) return null; + return normalizeHfColorGradingWithVariables(raw, readVariablesForElement(element)); +} + +const VERTEX_SHADER = [ + "attribute vec2 a_pos;", + "varying vec2 v_uv;", + "void main(){", + " v_uv = a_pos * 0.5 + 0.5;", + " gl_Position = vec4(a_pos, 0.0, 1.0);", + "}", +].join("\n"); + +const FRAGMENT_SHADER = [ + "#ifdef GL_FRAGMENT_PRECISION_HIGH", + "precision highp float;", + "#else", + "precision mediump float;", + "#endif", + "varying vec2 v_uv;", + "uniform sampler2D u_source;", + "uniform sampler2D u_lut;", + "uniform vec2 u_resolution;", + "uniform vec2 u_uvScale;", + "uniform vec2 u_uvOffset;", + "uniform float u_lutEnabled;", + "uniform float u_lutSize;", + "uniform vec2 u_lutTextureSize;", + "uniform vec3 u_lutDomainMin;", + "uniform vec3 u_lutDomainMax;", + "uniform float u_lutIntensity;", + "uniform float u_exposure;", + "uniform float u_contrast;", + "uniform float u_highlights;", + "uniform float u_shadows;", + "uniform float u_whites;", + "uniform float u_blacks;", + "uniform float u_temperature;", + "uniform float u_tint;", + "uniform float u_saturation;", + "uniform float u_intensity;", + "uniform float u_compareEnabled;", + "uniform float u_comparePosition;", + "uniform float u_compareSoftness;", + "uniform float u_compareLineWidth;", + "float lumaOf(vec3 c){ return dot(c, vec3(0.2126, 0.7152, 0.0722)); }", + "vec3 sampleLut(float r, float g, float b){", + " float size = max(u_lutSize, 2.0);", + " float x = (r + b * size + 0.5) / max(u_lutTextureSize.x, 1.0);", + " float y = (g + 0.5) / max(u_lutTextureSize.y, 1.0);", + " return texture2D(u_lut, vec2(x, y)).rgb;", + "}", + "vec3 applyLut(vec3 color){", + " if (u_lutEnabled < 0.5) return color;", + " float size = max(u_lutSize, 2.0);", + " vec3 span = max(u_lutDomainMax - u_lutDomainMin, vec3(0.00001));", + " vec3 scaled = clamp((color - u_lutDomainMin) / span, 0.0, 1.0) * (size - 1.0);", + " vec3 lo = floor(scaled);", + " vec3 hi = min(lo + 1.0, vec3(size - 1.0));", + " vec3 f = scaled - lo;", + " vec3 c000 = sampleLut(lo.r, lo.g, lo.b);", + " vec3 c100 = sampleLut(hi.r, lo.g, lo.b);", + " vec3 c010 = sampleLut(lo.r, hi.g, lo.b);", + " vec3 c110 = sampleLut(hi.r, hi.g, lo.b);", + " vec3 c001 = sampleLut(lo.r, lo.g, hi.b);", + " vec3 c101 = sampleLut(hi.r, lo.g, hi.b);", + " vec3 c011 = sampleLut(lo.r, hi.g, hi.b);", + " vec3 c111 = sampleLut(hi.r, hi.g, hi.b);", + " vec3 c00 = mix(c000, c100, f.r);", + " vec3 c10 = mix(c010, c110, f.r);", + " vec3 c01 = mix(c001, c101, f.r);", + " vec3 c11 = mix(c011, c111, f.r);", + " vec3 c0 = mix(c00, c10, f.g);", + " vec3 c1 = mix(c01, c11, f.g);", + " vec3 lutColor = mix(c0, c1, f.b);", + " return mix(color, lutColor, clamp(u_lutIntensity, 0.0, 1.0));", + "}", + "void main(){", + " vec2 uv = (v_uv - u_uvOffset) / u_uvScale;", + " if (uv.x < 0.0 || uv.y < 0.0 || uv.x > 1.0 || uv.y > 1.0) {", + " gl_FragColor = vec4(0.0);", + " return;", + " }", + " vec4 sampleColor = texture2D(u_source, uv);", + " vec3 original = sampleColor.rgb;", + " vec3 color = original * pow(2.0, u_exposure);", + " float y = lumaOf(color);", + " float shadowMask = 1.0 - smoothstep(0.0, 0.65, y);", + " float highlightMask = smoothstep(0.35, 1.0, y);", + " color += u_shadows * 0.35 * shadowMask;", + " color += u_highlights * 0.35 * highlightMask;", + " color += u_blacks * 0.25 * (1.0 - smoothstep(0.0, 0.35, y));", + " color += u_whites * 0.25 * smoothstep(0.65, 1.0, y);", + " color.r += u_temperature * 0.08 + u_tint * 0.04;", + " color.b -= u_temperature * 0.08 - u_tint * 0.04;", + " color.g -= u_tint * 0.08;", + " color = (color - 0.5) * max(0.0, 1.0 + u_contrast) + 0.5;", + " float satLuma = lumaOf(color);", + " color = mix(vec3(satLuma), color, max(0.0, 1.0 + u_saturation));", + " color = clamp(color, 0.0, 1.0);", + " color = clamp(applyLut(color), 0.0, 1.0);", + " vec3 graded = mix(original, color, u_intensity);", + " if (u_compareEnabled > 0.5) {", + " float pos = clamp(u_comparePosition, 0.0, 1.0);", + " float softness = max(u_compareSoftness, 0.00001);", + " float afterMask = smoothstep(pos - softness, pos + softness, v_uv.x);", + " vec3 splitColor = mix(original, graded, afterMask);", + " float lineMask = 0.0;", + " if (u_compareLineWidth > 0.0) {", + " float lineWidth = max(u_compareLineWidth / max(u_resolution.x, 1.0), 0.00001);", + " lineMask = 1.0 - smoothstep(lineWidth, lineWidth * 1.8, abs(v_uv.x - pos));", + " }", + " gl_FragColor = vec4(mix(splitColor, vec3(1.0), lineMask * 0.82), sampleColor.a);", + " return;", + " }", + " gl_FragColor = vec4(graded, sampleColor.a);", + "}", +].join("\n"); + +function isColorGradingMediaElement(value: Element): value is ColorGradingMediaElement { + return value instanceof HTMLVideoElement || value instanceof HTMLImageElement; +} + +function compileShader( + gl: WebGLRenderingContext, + source: string, + type: number, +): WebGLShader | null { + const shader = gl.createShader(type); + if (!shader) return null; + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + swallow("runtime.colorGrading.compileShader", gl.getShaderInfoLog(shader)); + gl.deleteShader(shader); + return null; + } + return shader; +} + +function createProgram(gl: WebGLRenderingContext): WebGLProgram | null { + const vertex = compileShader(gl, VERTEX_SHADER, gl.VERTEX_SHADER); + const fragment = compileShader(gl, FRAGMENT_SHADER, gl.FRAGMENT_SHADER); + if (!vertex || !fragment) { + if (vertex) gl.deleteShader(vertex); + if (fragment) gl.deleteShader(fragment); + return null; + } + const program = gl.createProgram(); + if (!program) return null; + gl.attachShader(program, vertex); + gl.attachShader(program, fragment); + gl.linkProgram(program); + gl.deleteShader(vertex); + gl.deleteShader(fragment); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + swallow("runtime.colorGrading.linkProgram", gl.getProgramInfoLog(program)); + gl.deleteProgram(program); + return null; + } + return program; +} + +function createTexture(gl: WebGLRenderingContext, filter = gl.LINEAR): WebGLTexture | null { + const texture = gl.createTexture(); + if (!texture) return null; + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, filter); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, filter); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + return texture; +} + +function createProgramInfo(canvas: HTMLCanvasElement): { + gl: WebGLRenderingContext; + program: ProgramInfo; +} | null { + const gl = canvas.getContext("webgl", { + alpha: true, + premultipliedAlpha: false, + preserveDrawingBuffer: true, + }); + if (!gl) return null; + const program = createProgram(gl); + const texture = createTexture(gl); + const lutTexture = createTexture(gl, gl.NEAREST); + if (!program || !texture || !lutTexture) { + if (program) gl.deleteProgram(program); + if (texture) gl.deleteTexture(texture); + if (lutTexture) gl.deleteTexture(lutTexture); + return null; + } + const quad = gl.createBuffer(); + if (!quad) { + gl.deleteProgram(program); + gl.deleteTexture(texture); + gl.deleteTexture(lutTexture); + return null; + } + gl.bindBuffer(gl.ARRAY_BUFFER, quad); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]), gl.STATIC_DRAW); + + return { + gl, + program: { + program, + texture, + lutTexture, + position: gl.getAttribLocation(program, "a_pos"), + source: gl.getUniformLocation(program, "u_source"), + lut: gl.getUniformLocation(program, "u_lut"), + resolution: gl.getUniformLocation(program, "u_resolution"), + uvScale: gl.getUniformLocation(program, "u_uvScale"), + uvOffset: gl.getUniformLocation(program, "u_uvOffset"), + lutEnabled: gl.getUniformLocation(program, "u_lutEnabled"), + lutSize: gl.getUniformLocation(program, "u_lutSize"), + lutTextureSize: gl.getUniformLocation(program, "u_lutTextureSize"), + lutDomainMin: gl.getUniformLocation(program, "u_lutDomainMin"), + lutDomainMax: gl.getUniformLocation(program, "u_lutDomainMax"), + lutIntensity: gl.getUniformLocation(program, "u_lutIntensity"), + exposure: gl.getUniformLocation(program, "u_exposure"), + contrast: gl.getUniformLocation(program, "u_contrast"), + highlights: gl.getUniformLocation(program, "u_highlights"), + shadows: gl.getUniformLocation(program, "u_shadows"), + whites: gl.getUniformLocation(program, "u_whites"), + blacks: gl.getUniformLocation(program, "u_blacks"), + temperature: gl.getUniformLocation(program, "u_temperature"), + tint: gl.getUniformLocation(program, "u_tint"), + saturation: gl.getUniformLocation(program, "u_saturation"), + intensity: gl.getUniformLocation(program, "u_intensity"), + compareEnabled: gl.getUniformLocation(program, "u_compareEnabled"), + comparePosition: gl.getUniformLocation(program, "u_comparePosition"), + compareSoftness: gl.getUniformLocation(program, "u_compareSoftness"), + compareLineWidth: gl.getUniformLocation(program, "u_compareLineWidth"), + }, + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function readNumber(value: unknown, fallback: number): number { + const parsed = typeof value === "number" ? value : Number(value); + return Number.isFinite(parsed) ? parsed : fallback; +} + +function clamp(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)); +} + +function normalizeCompare(raw: unknown): RuntimeColorGradingCompareState { + if (!isRecord(raw)) return { ...DEFAULT_COMPARE }; + return { + enabled: raw.enabled === true, + position: clamp(readNumber(raw.position, DEFAULT_COMPARE.position), 0, 1), + softness: clamp(readNumber(raw.softness, DEFAULT_COMPARE.softness), 0, 0.25), + lineWidth: clamp(readNumber(raw.lineWidth, DEFAULT_COMPARE.lineWidth), 0, 12), + }; +} + +function resolveLutUrl(src: string): { href: string } | { error: string } { + try { + const url = new URL(src, document.baseURI); + if (url.protocol === "data:") return { href: url.href }; + if (url.protocol !== "http:" && url.protocol !== "https:") { + return { error: "LUT must be project-local or a data URL" }; + } + if (url.origin !== window.location.origin) { + return { error: "Remote LUT URLs are not supported" }; + } + return { href: url.href }; + } catch { + return { error: "Invalid LUT URL" }; + } +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : "LUT failed to load"; +} + +function getCubeLut(src: string): LutCacheEntry { + const resolved = resolveLutUrl(src); + if ("error" in resolved) return { state: "error", message: resolved.error }; + const cached = LUT_CACHE.get(resolved.href); + if (cached) return cached; + + const promise = fetch(resolved.href, { credentials: "same-origin" }) + .then((response) => { + if (!response.ok) throw new Error(`Failed to load LUT (${response.status})`); + return response.text(); + }) + .then((text) => parseCubeLut(text, { maxSize: MAX_LUT_SIZE })); + + const pending: LutCacheEntry = { state: "pending", promise }; + LUT_CACHE.set(resolved.href, pending); + promise.then( + (lut) => LUT_CACHE.set(resolved.href, { state: "ready", lut }), + (err) => LUT_CACHE.set(resolved.href, { state: "error", message: errorMessage(err) }), + ); + return pending; +} + +function uploadEntryLut( + entry: ColorGradingEntry, + src: string, + lut: CubeLut3D, +): RuntimeLutTexture | null { + if (entry.lut?.src === src) return entry.lut; + const packed = packCubeLutToRgba8(lut); + const { gl, program } = entry; + try { + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, program.lutTexture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false); + gl.texImage2D( + gl.TEXTURE_2D, + 0, + gl.RGBA, + packed.width, + packed.height, + 0, + gl.RGBA, + gl.UNSIGNED_BYTE, + packed.data, + ); + entry.lut = { + src, + title: lut.title, + size: lut.size, + domainMin: lut.domainMin, + domainMax: lut.domainMax, + textureWidth: packed.width, + textureHeight: packed.height, + }; + entry.lutError = null; + entry.lutLoadingSrc = null; + return entry.lut; + } catch (err) { + entry.lut = null; + entry.lutError = errorMessage(err); + entry.lutLoadingSrc = null; + swallow("runtime.colorGrading.uploadLut", err); + return null; + } +} + +// fallow-ignore-next-line complexity +function ensureEntryLut(entry: ColorGradingEntry): RuntimeLutTexture | null { + const src = entry.grading.lut?.src.trim() ?? ""; + const intensity = entry.grading.lut?.intensity ?? 1; + if (!src || intensity <= 0) { + entry.lut = null; + entry.lutLoadingSrc = null; + entry.lutError = null; + return null; + } + + const resolved = resolveLutUrl(src); + if ("error" in resolved) { + entry.lut = null; + entry.lutLoadingSrc = null; + entry.lutError = resolved.error; + return null; + } + if (entry.lut?.src === resolved.href) return entry.lut; + entry.lut = null; + + const cached = getCubeLut(src); + if (cached.state === "ready") return uploadEntryLut(entry, resolved.href, cached.lut); + if (cached.state === "error") { + entry.lutError = cached.message; + entry.lutLoadingSrc = null; + return null; + } + + if (entry.lutLoadingSrc !== resolved.href) { + entry.lutLoadingSrc = resolved.href; + entry.lutError = null; + cached.promise.then( + (lut) => { + if (entry.destroyed || entry.grading.lut?.src.trim() !== src) return; + uploadEntryLut(entry, resolved.href, lut); + drawEntry(entry); + }, + (err) => { + if (entry.destroyed || entry.grading.lut?.src.trim() !== src) return; + entry.lut = null; + entry.lutError = errorMessage(err); + entry.lutLoadingSrc = null; + drawEntry(entry); + }, + ); + } + return null; +} + +// fallow-ignore-next-line complexity +function resolveTarget( + target: HfColorGradingTarget | string | null | undefined, +): ColorGradingMediaElement | null { + if (!target) return null; + if (typeof target === "string") { + const trimmed = target.trim(); + if (!trimmed) return null; + const byId = document.getElementById(trimmed.replace(/^#/, "")); + if (byId && isColorGradingMediaElement(byId)) return byId; + try { + const bySelector = document.querySelector(trimmed); + return bySelector && isColorGradingMediaElement(bySelector) ? bySelector : null; + } catch { + return null; + } + } + if (target.hfId) { + const byHfId = document.querySelector(`[data-hf-id="${CSS.escape(target.hfId)}"]`); + if (byHfId && isColorGradingMediaElement(byHfId)) return byHfId; + } + if (target.id) { + const byId = document.getElementById(target.id); + if (byId && isColorGradingMediaElement(byId)) return byId; + } + if (!target.selector) return null; + try { + const matches = Array.from(document.querySelectorAll(target.selector)); + const index = Math.max(0, Math.floor(Number(target.selectorIndex ?? 0) || 0)); + const match = matches[index] ?? null; + return match && isColorGradingMediaElement(match) ? match : null; + } catch { + return null; + } +} + +function readSourceSize(source: TexImageSource): { width: number; height: number } | null { + if (source instanceof HTMLVideoElement) { + return source.videoWidth > 0 && source.videoHeight > 0 + ? { width: source.videoWidth, height: source.videoHeight } + : null; + } + if (source instanceof HTMLImageElement) { + return source.naturalWidth > 0 && source.naturalHeight > 0 + ? { width: source.naturalWidth, height: source.naturalHeight } + : null; + } + return null; +} + +function isDrawableSource(source: TexImageSource): boolean { + if (source instanceof HTMLVideoElement) { + return ( + source.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && + source.videoWidth > 0 && + source.videoHeight > 0 + ); + } + if (source instanceof HTMLImageElement) { + return source.complete && source.naturalWidth > 0 && source.naturalHeight > 0; + } + return false; +} + +function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null { + if (!video.id) return null; + const frame = document.getElementById(`__render_frame_${video.id}__`); + return frame instanceof HTMLImageElement && isDrawableSource(frame) ? frame : null; +} + +function getDrawableSource(element: ColorGradingMediaElement): TexImageSource | null { + if (element instanceof HTMLVideoElement) { + const renderFrame = findRenderFrameImage(element); + if (renderFrame) return renderFrame; + } + return isDrawableSource(element) ? element : null; +} + +function parseObjectPositionPart(value: string, axis: "x" | "y"): number | null { + const lower = value.toLowerCase(); + if (lower === "center") return 0.5; + if (axis === "x") { + if (lower === "left") return 0; + if (lower === "right") return 1; + } else { + if (lower === "top") return 0; + if (lower === "bottom") return 1; + } + if (lower.endsWith("%")) { + const parsed = Number.parseFloat(lower); + return Number.isFinite(parsed) ? parsed / 100 : null; + } + return null; +} + +// fallow-ignore-next-line complexity +function parseObjectPosition(value: string): { x: number; y: number } { + const tokens = value.trim().split(/\s+/).filter(Boolean); + let x = 0.5; + let y = 0.5; + for (const token of tokens) { + const xValue = parseObjectPositionPart(token, "x"); + const yValue = parseObjectPositionPart(token, "y"); + if (xValue !== null && (token === "left" || token === "right" || token.endsWith("%"))) { + x = xValue; + continue; + } + if (yValue !== null && (token === "top" || token === "bottom")) { + y = yValue; + continue; + } + if (token === "center") { + if (x === 0.5) x = 0.5; + else y = 0.5; + } + } + return { x, y }; +} + +// fallow-ignore-next-line complexity +function calculateObjectFitUv( + boxWidth: number, + boxHeight: number, + sourceWidth: number, + sourceHeight: number, + objectFit: string, + objectPosition: string, +): { scaleX: number; scaleY: number; offsetX: number; offsetY: number } { + if (boxWidth <= 0 || boxHeight <= 0 || sourceWidth <= 0 || sourceHeight <= 0) { + return { scaleX: 1, scaleY: 1, offsetX: 0, offsetY: 0 }; + } + const fit = objectFit || "fill"; + let drawWidth = boxWidth; + let drawHeight = boxHeight; + if (fit === "contain" || fit === "cover" || fit === "scale-down") { + const scale = + fit === "cover" + ? Math.max(boxWidth / sourceWidth, boxHeight / sourceHeight) + : Math.min(boxWidth / sourceWidth, boxHeight / sourceHeight); + drawWidth = sourceWidth * scale; + drawHeight = sourceHeight * scale; + if (fit === "scale-down" && drawWidth > sourceWidth && drawHeight > sourceHeight) { + drawWidth = sourceWidth; + drawHeight = sourceHeight; + } + } else if (fit === "none") { + drawWidth = sourceWidth; + drawHeight = sourceHeight; + } + const position = parseObjectPosition(objectPosition || "center"); + const offsetX = ((boxWidth - drawWidth) * position.x) / boxWidth; + const offsetY = ((boxHeight - drawHeight) * position.y) / boxHeight; + return { + scaleX: drawWidth / boxWidth, + scaleY: drawHeight / boxHeight, + offsetX, + offsetY, + }; +} + +function ensureParentPosition(entry: ColorGradingEntry, parent: HTMLElement): void { + const computed = window.getComputedStyle(parent); + if (computed.position !== "static") return; + if (!entry.touchedParent) { + entry.touchedParent = parent; + entry.parentInlinePosition = parent.style.position || null; + } + parent.style.position = "relative"; +} + +function updateCanvasLayout( + entry: ColorGradingEntry, + styleSource: HTMLElement, +): { width: number; height: number } | null { + const { element, canvas } = entry; + const parent = element.parentElement; + if (parent) ensureParentPosition(entry, parent); + + const computed = window.getComputedStyle(styleSource); + copyMediaVisualStyles(canvas.style, computed); + canvas.style.pointerEvents = "none"; + canvas.style.position = "absolute"; + canvas.style.inset = "auto"; + canvas.style.left = `${element.offsetLeft}px`; + canvas.style.top = `${element.offsetTop}px`; + canvas.style.right = "auto"; + canvas.style.bottom = "auto"; + canvas.style.width = `${element.offsetWidth}px`; + canvas.style.height = `${element.offsetHeight}px`; + canvas.style.display = "block"; + canvas.style.opacity = entry.sourceOpacityForCanvas; + canvas.style.visibility = entry.sourceVisibleForCanvas ? "visible" : "hidden"; + + const rect = element.getBoundingClientRect(); + const width = Math.max(0, Math.round(element.offsetWidth || rect.width)); + const height = Math.max(0, Math.round(element.offsetHeight || rect.height)); + if (width <= 0 || height <= 0) { + canvas.style.display = "none"; + return null; + } + if (canvas.width !== width) canvas.width = width; + if (canvas.height !== height) canvas.height = height; + return { width, height }; +} + +// fallow-ignore-next-line complexity +function applyUniforms( + gl: WebGLRenderingContext, + program: ProgramInfo, + grading: NormalizedHfColorGrading, + lut: RuntimeLutTexture | null, + compare: RuntimeColorGradingCompareState, + layout: { width: number; height: number }, + uv: { scaleX: number; scaleY: number; offsetX: number; offsetY: number }, +): void { + gl.uniform1i(program.source, 0); + gl.uniform1i(program.lut, 1); + gl.uniform2f(program.resolution, layout.width, layout.height); + gl.uniform2f(program.uvScale, uv.scaleX, uv.scaleY); + gl.uniform2f(program.uvOffset, uv.offsetX, uv.offsetY); + gl.uniform1f(program.lutEnabled, lut ? 1 : 0); + gl.uniform1f(program.lutSize, lut?.size ?? 2); + gl.uniform2f(program.lutTextureSize, lut?.textureWidth ?? 1, lut?.textureHeight ?? 1); + gl.uniform3f( + program.lutDomainMin, + lut?.domainMin[0] ?? 0, + lut?.domainMin[1] ?? 0, + lut?.domainMin[2] ?? 0, + ); + gl.uniform3f( + program.lutDomainMax, + lut?.domainMax[0] ?? 1, + lut?.domainMax[1] ?? 1, + lut?.domainMax[2] ?? 1, + ); + gl.uniform1f(program.lutIntensity, grading.lut?.intensity ?? 0); + gl.uniform1f(program.exposure, grading.adjust.exposure); + gl.uniform1f(program.contrast, grading.adjust.contrast); + gl.uniform1f(program.highlights, grading.adjust.highlights); + gl.uniform1f(program.shadows, grading.adjust.shadows); + gl.uniform1f(program.whites, grading.adjust.whites); + gl.uniform1f(program.blacks, grading.adjust.blacks); + gl.uniform1f(program.temperature, grading.adjust.temperature); + gl.uniform1f(program.tint, grading.adjust.tint); + gl.uniform1f(program.saturation, grading.adjust.saturation); + gl.uniform1f(program.intensity, grading.intensity); + gl.uniform1f(program.compareEnabled, compare.enabled ? 1 : 0); + gl.uniform1f(program.comparePosition, compare.position); + gl.uniform1f(program.compareSoftness, compare.softness); + gl.uniform1f(program.compareLineWidth, compare.lineWidth); +} + +function hideSourceElement(entry: ColorGradingEntry): void { + if (!entry.sourceHidden) { + entry.sourceInlineOpacity = entry.element.style.getPropertyValue("opacity") || null; + entry.sourceInlineOpacityPriority = entry.element.style.getPropertyPriority("opacity"); + } + entry.element.setAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR, "true"); + entry.element.style.setProperty("opacity", "0", "important"); + entry.sourceHidden = true; +} + +// fallow-ignore-next-line complexity +function drawEntry(entry: ColorGradingEntry): boolean { + if (entry.destroyed) return false; + const source = getDrawableSource(entry.element); + if (!source) { + if (!entry.hasDrawn) entry.canvas.style.display = "none"; + return false; + } + const sourceSize = readSourceSize(source); + if (!sourceSize) return false; + const styleSource = source instanceof HTMLElement ? source : entry.element; + const sourceOpacity = entry.element.style.getPropertyValue("opacity"); + const sourceOpacityPriority = entry.element.style.getPropertyPriority("opacity"); + const hiddenByColorGrading = + entry.sourceHidden && sourceOpacity === "0" && sourceOpacityPriority === "important"; + const sourceVisibility = entry.element.style.getPropertyValue("visibility"); + if (!hiddenByColorGrading) { + const computed = window.getComputedStyle(entry.element); + entry.sourceOpacityForCanvas = computed.opacity || "1"; + entry.sourceVisibleForCanvas = + sourceVisibility !== "hidden" && computed.visibility !== "hidden"; + } + const layout = updateCanvasLayout(entry, styleSource); + if (!layout) return false; + + const style = window.getComputedStyle(styleSource); + const uv = calculateObjectFitUv( + layout.width, + layout.height, + sourceSize.width, + sourceSize.height, + style.objectFit, + style.objectPosition, + ); + const { gl, program } = entry; + try { + const lut = ensureEntryLut(entry); + gl.viewport(0, 0, layout.width, layout.height); + gl.useProgram(program.program); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, program.texture); + // Browser media elements are top-left oriented; WebGL texture coordinates + // are bottom-left oriented unless the upload is flipped. + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source); + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, program.lutTexture); + applyUniforms(gl, program, entry.grading, lut, entry.compare, layout, uv); + gl.enableVertexAttribArray(program.position); + gl.vertexAttribPointer(program.position, 2, gl.FLOAT, false, 0, 0); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + hideSourceElement(entry); + entry.hasDrawn = true; + return true; + } catch (err) { + swallow("runtime.colorGrading.drawEntry", err); + return false; + } +} + +function addListener( + entry: ColorGradingEntry, + target: EventTarget, + type: string, + listener: EventListener, +): void { + target.addEventListener(type, listener); + entry.cleanup.push(() => target.removeEventListener(type, listener)); +} + +function cancelScheduledFrame(entry: ColorGradingEntry): void { + if (entry.animationFrame !== null) { + window.cancelAnimationFrame(entry.animationFrame); + entry.animationFrame = null; + } + if (entry.videoFrameHandle !== null && entry.element instanceof HTMLVideoElement) { + const videoFrameHost: VideoFrameCallbackHost = entry.element; + videoFrameHost.cancelVideoFrameCallback?.(entry.videoFrameHandle); + entry.videoFrameHandle = null; + } +} + +function scheduleVideoDraw(entry: ColorGradingEntry): void { + if (entry.destroyed || !(entry.element instanceof HTMLVideoElement)) return; + if (entry.videoFrameHandle !== null || entry.animationFrame !== null) return; + const video = entry.element; + const videoFrameHost: VideoFrameCallbackHost = video; + if (typeof videoFrameHost.requestVideoFrameCallback === "function") { + entry.videoFrameHandle = videoFrameHost.requestVideoFrameCallback(() => { + entry.videoFrameHandle = null; + drawEntry(entry); + if (!entry.destroyed && !video.paused && !video.ended) scheduleVideoDraw(entry); + }); + return; + } + entry.animationFrame = window.requestAnimationFrame(() => { + entry.animationFrame = null; + drawEntry(entry); + if (!entry.destroyed && !video.paused && !video.ended) scheduleVideoDraw(entry); + }); +} + +function installEntryListeners(entry: ColorGradingEntry): void { + const redraw = () => { + drawEntry(entry); + }; + addListener(entry, entry.element, "load", redraw); + addListener(entry, entry.element, "loadedmetadata", redraw); + addListener(entry, entry.element, "loadeddata", redraw); + addListener(entry, entry.element, "seeked", redraw); + addListener(entry, entry.element, "timeupdate", redraw); + addListener(entry, window, "resize", redraw); + if (entry.element instanceof HTMLVideoElement) { + addListener(entry, entry.element, "play", () => scheduleVideoDraw(entry)); + addListener(entry, entry.element, "pause", redraw); + } + if (typeof ResizeObserver !== "undefined") { + entry.resizeObserver = new ResizeObserver(redraw); + entry.resizeObserver.observe(entry.element); + } +} + +function destroyEntry(entry: ColorGradingEntry): void { + if (entry.destroyed) return; + entry.destroyed = true; + cancelScheduledFrame(entry); + entry.resizeObserver?.disconnect(); + for (const cleanup of entry.cleanup) cleanup(); + entry.cleanup.length = 0; + entry.canvas.remove(); + entry.gl.deleteTexture(entry.program.texture); + entry.gl.deleteTexture(entry.program.lutTexture); + entry.gl.deleteProgram(entry.program.program); + if (entry.sourceHidden) { + entry.element.removeAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR); + const opacity = entry.element.style.getPropertyValue("opacity"); + const priority = entry.element.style.getPropertyPriority("opacity"); + if (opacity === "0" && priority === "important") { + if (entry.sourceInlineOpacity === null) { + entry.element.style.removeProperty("opacity"); + } else { + entry.element.style.setProperty( + "opacity", + entry.sourceInlineOpacity, + entry.sourceInlineOpacityPriority, + ); + } + } + } + if (entry.touchedParent) { + if (entry.parentInlinePosition === null) { + entry.touchedParent.style.removeProperty("position"); + } else { + entry.touchedParent.style.position = entry.parentInlinePosition; + } + } +} + +function makeCanvas(element: ColorGradingMediaElement): HTMLCanvasElement { + const canvas = document.createElement("canvas"); + canvas.className = COLOR_GRADING_CANVAS_CLASS; + canvas.setAttribute(COLOR_GRADING_CANVAS_ATTR, "true"); + canvas.setAttribute("data-hyperframes-ignore", ""); + canvas.setAttribute("data-hyperframes-picker-ignore", ""); + canvas.setAttribute("data-hf-ignore", ""); + canvas.setAttribute("aria-hidden", "true"); + canvas.style.pointerEvents = "none"; + canvas.style.display = "none"; + element.parentNode?.insertBefore(canvas, element.nextSibling); + return canvas; +} + +export function createColorGradingRuntime(): RuntimeColorGradingApi { + const entries = new WeakMap(); + const trackedElements = new Set(); + let observer: MutationObserver | null = null; + let destroyed = false; + + const upsert = ( + element: ColorGradingMediaElement, + grading: NormalizedHfColorGrading, + source: EntrySource, + ): boolean => { + const existing = entries.get(element); + if (existing) { + existing.grading = grading; + existing.source = source; + drawEntry(existing); + if (element instanceof HTMLVideoElement && !element.paused) scheduleVideoDraw(existing); + return true; + } + const canvas = makeCanvas(element); + const created = createProgramInfo(canvas); + if (!created) { + canvas.remove(); + return false; + } + const entry: ColorGradingEntry = { + element, + canvas, + gl: created.gl, + program: created.program, + grading, + compare: { ...DEFAULT_COMPARE }, + lut: null, + lutLoadingSrc: null, + lutError: null, + source, + animationFrame: null, + videoFrameHandle: null, + resizeObserver: null, + cleanup: [], + touchedParent: null, + parentInlinePosition: null, + sourceHidden: false, + sourceInlineOpacity: null, + sourceInlineOpacityPriority: "", + sourceOpacityForCanvas: window.getComputedStyle(element).opacity || "1", + sourceVisibleForCanvas: window.getComputedStyle(element).visibility !== "hidden", + hasDrawn: false, + destroyed: false, + }; + entries.set(element, entry); + trackedElements.add(element); + installEntryListeners(entry); + drawEntry(entry); + if (element instanceof HTMLVideoElement && !element.paused) scheduleVideoDraw(entry); + return true; + }; + + const setCompare = ( + target: HfColorGradingTarget | string | null | undefined, + rawCompare: unknown, + ): boolean => { + if (destroyed) return false; + const element = resolveTarget(target); + if (!element) return false; + let entry = entries.get(element); + if (!entry) { + const grading = readColorGradingAttribute(element); + if (!isHfColorGradingActive(grading) || !upsert(element, grading, "attribute")) return false; + entry = entries.get(element); + } + if (!entry) return false; + entry.compare = normalizeCompare(rawCompare); + drawEntry(entry); + return true; + }; + + const removeElement = (element: ColorGradingMediaElement): void => { + const entry = entries.get(element); + if (!entry) return; + destroyEntry(entry); + entries.delete(element); + trackedElements.delete(element); + }; + + const refresh = (): number => { + if (destroyed) return 0; + const attributeElements = new Set(); + const nodes = document.querySelectorAll( + `video[${HF_COLOR_GRADING_ATTR}], img[${HF_COLOR_GRADING_ATTR}]`, + ); + nodes.forEach((node) => { + if (!isColorGradingMediaElement(node)) return; + attributeElements.add(node); + const grading = readColorGradingAttribute(node); + if (isHfColorGradingActive(grading)) { + upsert(node, grading, "attribute"); + } else { + removeElement(node); + } + }); + for (const element of Array.from(trackedElements)) { + const entry = entries.get(element); + if (!entry) continue; + if ( + !element.isConnected || + (entry.source === "attribute" && !attributeElements.has(element)) + ) { + removeElement(element); + } + } + return trackedElements.size; + }; + + const redraw = (): number => { + if (destroyed) return 0; + let drawn = 0; + for (const entry of Array.from(trackedElements, (element) => entries.get(element))) { + if (entry && drawEntry(entry)) drawn += 1; + } + return drawn; + }; + + const setGrading = ( + target: HfColorGradingTarget | string | null | undefined, + rawGrading: unknown, + ): boolean => { + if (destroyed) return false; + const element = resolveTarget(target); + if (!element) return false; + const grading = normalizeHfColorGrading(rawGrading); + if (!isHfColorGradingActive(grading)) { + removeElement(element); + return true; + } + return upsert(element, grading, "live"); + }; + + const clearGrading = (target: HfColorGradingTarget | string | null | undefined): boolean => { + const element = resolveTarget(target); + if (!element) return false; + removeElement(element); + return true; + }; + + const setSourceVisibility = (target: Element, visible: boolean): boolean => { + if (!isColorGradingMediaElement(target)) return false; + const entry = entries.get(target); + if (!entry) return false; + entry.sourceVisibleForCanvas = visible; + return true; + }; + + const getStatus = ( + target: HfColorGradingTarget | string | null | undefined, + ): RuntimeColorGradingStatus => { + const element = resolveTarget(target); + if (!element) return { state: "missing", message: "Media not found" }; + const entry = entries.get(element); + if (entry) { + if (entry.lutError) { + return { state: "unavailable", message: entry.lutError }; + } + if (entry.grading.lut && entry.lutLoadingSrc) { + return { state: "pending", message: "Loading LUT" }; + } + if (entry.canvas.style.display === "none") { + return { state: "pending", message: "Waiting for media frame" }; + } + return { + state: "active", + message: entry.lut ? "Shader + LUT active" : "Shader active", + }; + } + const grading = readColorGradingAttribute(element); + if (isHfColorGradingActive(grading)) { + return { state: "unavailable", message: "WebGL unavailable" }; + } + return { state: "inactive", message: "No grading applied" }; + }; + + const destroy = (): void => { + if (destroyed) return; + destroyed = true; + observer?.disconnect(); + observer = null; + for (const element of Array.from(trackedElements)) removeElement(element); + }; + + if (document.body) { + observer = new MutationObserver(() => refresh()); + observer.observe(document.body, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: [HF_COLOR_GRADING_ATTR], + }); + } + + const api: RuntimeColorGradingApi = { + refresh, + redraw, + setGrading, + clearGrading, + setCompare, + setSourceVisibility, + getStatus, + destroy, + }; + const win = window as WindowWithColorGrading; + win.__hf = win.__hf || {}; + win.__hf.colorGrading = api; + refresh(); + return api; +} diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 8ee49183c1..4d0133a92d 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -23,6 +23,7 @@ import { createRuntimeStartTimeResolver } from "./startResolver"; import { createClipTree } from "./clipTree"; import { loadExternalCompositions, loadInlineTemplateCompositions } from "./compositionLoader"; import { applyCaptionOverrides } from "./captionOverrides"; +import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading"; import { TransportClock } from "./clock"; import { WebAudioTransport } from "./webAudioTransport"; import { quantizeTimeToFrame } from "../inline-scripts/parityContract"; @@ -36,6 +37,7 @@ const AUTHORED_END_ATTR = "data-hf-authored-end"; export function initSandboxRuntimeModular(): void { const state = createRuntimeState(); + let colorGradingRuntime: RuntimeColorGradingApi | null = null; let runtimeErrorListener: ((event: ErrorEvent) => void) | null = null; let runtimeUnhandledRejectionListener: ((event: PromiseRejectionEvent) => void) | null = null; const runtimeCleanupCallbacks: Array<() => void> = []; @@ -1534,6 +1536,9 @@ export function initSandboxRuntimeModular(): void { } } rawNode.style.visibility = isVisibleNow ? "visible" : "hidden"; + if (rawNode instanceof HTMLVideoElement || rawNode instanceof HTMLImageElement) { + colorGradingRuntime?.setSourceVisibility(rawNode, isVisibleNow); + } } }; @@ -1678,6 +1683,13 @@ export function initSandboxRuntimeModular(): void { }); picker.installPickerApi(); + const colorGrading = createColorGradingRuntime(); + colorGradingRuntime = colorGrading; + registerRuntimeCleanup(() => { + colorGrading.destroy(); + colorGradingRuntime = null; + }); + const applyPlaybackRate = (nextRate: number) => { const parsed = Number(nextRate); if (!Number.isFinite(parsed) || parsed <= 0) { @@ -1735,7 +1747,9 @@ export function initSandboxRuntimeModular(): void { }, onDeterministicPause: () => runAdapters("pause"), onDeterministicPlay: () => runAdapters("play"), - onRenderFrameSeek: () => {}, + onRenderFrameSeek: () => { + colorGrading.redraw(); + }, onShowNativeVideos: () => {}, getSafeDuration: () => getSafeTimelineDurationSeconds(state.capturedTimeline, 0), }); @@ -1801,6 +1815,12 @@ export function initSandboxRuntimeModular(): void { if (state.transportClock) state.transportClock.setRate(state.playbackRate); applyWebAudioRate(); }, + onSetColorGrading: (target, grading) => { + colorGrading.setGrading(target, grading); + }, + onSetColorGradingCompare: (target, compare) => { + colorGrading.setCompare(target, compare); + }, onTick: () => { if (state.tornDown || !clock.isPlaying()) return; const t = clock.now(); @@ -2288,6 +2308,7 @@ export function initSandboxRuntimeModular(): void { if (webAudioReady) scheduleWebAudioForActiveClips(); runAdapters("play"); syncMediaForCurrentState(); + colorGrading.redraw(); postState(true); }; @@ -2304,6 +2325,7 @@ export function initSandboxRuntimeModular(): void { if (tl) tl.pause(); runAdapters("pause"); syncMediaForCurrentState(); + colorGrading.redraw(); postState(true); }; @@ -2325,6 +2347,7 @@ export function initSandboxRuntimeModular(): void { seekTimelineAndAdapters(state.currentTime); runAdapters("pause"); syncMediaForCurrentState(); + colorGrading.redraw(); postState(true); }; @@ -2340,6 +2363,7 @@ export function initSandboxRuntimeModular(): void { state.mediaForceSyncNextTick = true; seekTimelineAndAdapters(state.currentTime, { activateChildren: true }); syncMediaForCurrentState(); + colorGrading.redraw(); postState(true); }; diff --git a/packages/core/src/runtime/picker.ts b/packages/core/src/runtime/picker.ts index 6f12cd1183..2cf4d4c347 100644 --- a/packages/core/src/runtime/picker.ts +++ b/packages/core/src/runtime/picker.ts @@ -17,6 +17,7 @@ const PICKER_BLOCK_SELECTOR = [ "[data-hyperframes-picker-block]", "[data-hyper-shader-loading]", ].join(","); +const COLOR_GRADING_SOURCE_HIDDEN_ATTR = "data-hf-color-grading-source-hidden"; export type PickerModule = { enablePickMode: () => void; @@ -67,7 +68,12 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule { if (computed.display === "none" || computed.visibility === "hidden") return true; if (computed.pointerEvents === "none") return true; const opacity = Number.parseFloat(computed.opacity); - if (Number.isFinite(opacity) && opacity <= 0.01) return true; + if ( + Number.isFinite(opacity) && + opacity <= 0.01 && + !current.hasAttribute(COLOR_GRADING_SOURCE_HIDDEN_ATTR) + ) + return true; current = current.parentElement; } return false; diff --git a/packages/core/src/runtime/types.ts b/packages/core/src/runtime/types.ts index e20ccbcf65..6d6acc8990 100644 --- a/packages/core/src/runtime/types.ts +++ b/packages/core/src/runtime/types.ts @@ -1,3 +1,5 @@ +import type { HfColorGradingTarget } from "../colorGrading"; + export type RuntimeJson = | string | number @@ -24,6 +26,9 @@ export type RuntimeBridgeControlMessage = { muted?: boolean; volume?: number; playbackRate?: number; + target?: HfColorGradingTarget | string | null; + grading?: RuntimeJson; + compare?: RuntimeJson; seekMode?: "drag" | "commit"; }; diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index c6a306068d..5d74675755 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -1,4 +1,5 @@ import type { RuntimeTimelineMessage, RuntimeTimelineLike } from "./types"; +import type { RuntimeColorGradingApi } from "./colorGrading"; import type { HyperframePickerApi } from "../inline-scripts/pickerApi"; import type { PlayerAPI } from "../core.types"; import type { ClipTree } from "./clipTree"; @@ -31,6 +32,10 @@ declare global { __player?: PlayerAPI; __clipManifest?: RuntimeTimelineMessage; __clipTree?: ClipTree; + __hf?: { + colorGrading?: RuntimeColorGradingApi; + onSwallowed?: (label: string, err: unknown) => void; + }; __playerReady?: boolean; __renderReady?: boolean; __hfRuntimeTeardown?: (() => void) | null; diff --git a/packages/core/src/studio-api/routes/preview.ts b/packages/core/src/studio-api/routes/preview.ts index 34846c7ea1..551ddf477f 100644 --- a/packages/core/src/studio-api/routes/preview.ts +++ b/packages/core/src/studio-api/routes/preview.ts @@ -326,7 +326,7 @@ export function registerPreviewRoutes(api: Hono, adapter: StudioApiAdapter): voi return c.text("not found", 404); } const contentType = getMimeType(subPath); - const isText = /\.(html|css|js|json|svg|txt|md)$/i.test(subPath); + const isText = /\.(html|css|js|json|svg|txt|md|cube)$/i.test(subPath); const etag = `"${stat.mtimeMs.toString(36)}-${stat.size.toString(36)}"`; const cacheHeaders: Record = isText diff --git a/packages/engine/src/services/videoFrameInjector.ts b/packages/engine/src/services/videoFrameInjector.ts index d13c1b6b2e..4fd2548086 100644 --- a/packages/engine/src/services/videoFrameInjector.ts +++ b/packages/engine/src/services/videoFrameInjector.ts @@ -146,6 +146,25 @@ function createFrameSourceCache( export const __testing = { createFrameSourceCache }; +async function redrawRuntimeColorGrading(page: Page): Promise { + await page.evaluate(async () => { + const hf = ( + window as Window & { + __hf?: { + colorGrading?: { redraw?: () => unknown }; + }; + } + ).__hf; + const redraw = hf?.colorGrading?.redraw; + if (typeof redraw !== "function") return; + try { + await Promise.resolve(redraw()); + } catch { + // Optional page-side shader layer. + } + }); +} + /** * Creates a BeforeCaptureHook that injects pre-extracted video frames * into the page, replacing native