diff --git a/desktop/package.json b/desktop/package.json index c57d4a12d..45989b065 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -26,6 +26,7 @@ "@dnd-kit/utilities": "^3.2.2", "@emoji-mart/data": "^1.2.1", "@emoji-mart/react": "^1.1.1", + "@mediapipe/tasks-vision": "^0.10.35", "@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-avatar": "^1.1.11", "@radix-ui/react-checkbox": "^1.3.3", @@ -70,6 +71,7 @@ "sonner": "^2.0.7", "tailwind-merge": "^3.5.0", "tiptap-markdown": "^0.9.0", + "upng-js": "^2.1.0", "yaml": "^2.8.3", "zod": "^4.4.3" }, diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index d9b0ee127..bd918140b 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -40,6 +40,7 @@ export default defineConfig({ "**/identity-archive.spec.ts", "**/identity-archive-hide.spec.ts", "**/relay-connectivity-screenshots.spec.ts", + "**/animated-avatar-screenshots.spec.ts", ], use: { ...devices["Desktop Chrome"], diff --git a/desktop/src-tauri/Entitlements.plist b/desktop/src-tauri/Entitlements.plist index d459cb2ca..73bf832c7 100644 --- a/desktop/src-tauri/Entitlements.plist +++ b/desktop/src-tauri/Entitlements.plist @@ -4,5 +4,7 @@ com.apple.security.device.audio-input + com.apple.security.device.camera + diff --git a/desktop/src-tauri/Info.plist b/desktop/src-tauri/Info.plist index 61604f062..c1b7331eb 100644 --- a/desktop/src-tauri/Info.plist +++ b/desktop/src-tauri/Info.plist @@ -8,5 +8,7 @@ Buzz NSMicrophoneUsageDescription Buzz needs microphone access for voice huddles. + NSCameraUsageDescription + Buzz needs camera access to record animated avatars. diff --git a/desktop/src/features/profile/lib/animatedAvatarCapture.ts b/desktop/src/features/profile/lib/animatedAvatarCapture.ts new file mode 100644 index 000000000..5365efa28 --- /dev/null +++ b/desktop/src/features/profile/lib/animatedAvatarCapture.ts @@ -0,0 +1,718 @@ +/** + * Animated avatar capture pipeline. + * + * Records a short clip from the user's camera, removes the background per + * frame with MediaPipe selfie segmentation (so only the person remains, with + * soft alpha edges), and composes each frame as a "sticker": the person pops + * out of a colored backdrop disc that sits inside the avatar circle. The + * result is encoded as a ping-pong looping animated PNG (APNG) — full 24-bit + * color and 8-bit alpha, unlike GIF — plus a static poster frame. + * + * Segmentation assets (wasm + model) are fetched lazily from public CDNs the + * first time the feature is used. If they can't be loaded (e.g. offline), + * recording still works — the background just isn't removed. + */ + +import type { ImageSegmenter } from "@mediapipe/tasks-vision"; +import UPNG from "upng-js"; + +export const ANIMATED_AVATAR_SIZE = 256; +const ANIMATED_AVATAR_CAPTURE_SIZE = 512; +export const ANIMATED_AVATAR_FPS = 12; +export const ANIMATED_AVATAR_DURATION_MS = 3000; +export const ANIMATED_AVATAR_FRAME_COUNT = Math.round( + (ANIMATED_AVATAR_DURATION_MS / 1000) * ANIMATED_AVATAR_FPS, +); +export const ANIMATED_AVATAR_FRAME_DELAY_MS = Math.round( + 1000 / ANIMATED_AVATAR_FPS, +); + +// Preserve truecolor pixels in the final APNG. Palette quantization made +// high-detail dark areas, like hair, shimmer between frames. +const APNG_COLOR_COUNT = 0; + +// Soft alpha ramp for the segmentation confidence: fully transparent below +// the low bound, fully opaque above the high bound, feathered in between. +const PERSON_CONFIDENCE_LOW = 0.32; +const PERSON_CONFIDENCE_HIGH = 0.7; + +// Dark camera pixels tend to carry sensor noise, which reads as shimmer once +// looped. Blend only low-luma pixels that barely changed from the previous +// frame, so hair noise settles without smearing real motion. +const DARK_DETAIL_DENOISE_LUMA_LOW = 24; +const DARK_DETAIL_DENOISE_LUMA_HIGH = 150; +const DARK_DETAIL_DENOISE_DIFF_LOW = 4; +const DARK_DETAIL_DENOISE_DIFF_HIGH = 32; +const DARK_DETAIL_DENOISE_MAX_BLEND = 0.52; + +// Stabilize tiny frame-to-frame mask confidence changes around fine edges +// (hair especially), while letting larger movement update immediately. +const MASK_TEMPORAL_MAX_BLEND = 0.64; +const MASK_TEMPORAL_MOTION_THRESHOLD = 0.26; + +/** + * How the recorded person and the backdrop circle are placed in the frame. + * Offsets are in 256x256 frame coordinates. The person's scale multiplies + * the frame size (the default draws them slightly oversized and + * bottom-anchored so their head pops above the backdrop); the circle's + * scale multiplies its base geometry around its own center. + */ +export type AvatarComposition = { + backdropColor: string | null; + offsetX: number; + offsetY: number; + personOutline: boolean; + scale: number; + shapeOffsetX: number; + shapeOffsetY: number; + shapeScale: number; +}; + +// Default framing tuned by hand against real recordings. +export const DEFAULT_PERSON_SCALE = 1.26; +export const DEFAULT_PERSON_OFFSET_X = 1; +export const DEFAULT_PERSON_OFFSET_Y = 7; +export const DEFAULT_PERSON_OUTLINE = true; +export const MIN_PERSON_SCALE = 0.7; +export const MAX_PERSON_SCALE = 2; + +export const DEFAULT_SHAPE_SCALE = 1.12; +export const DEFAULT_SHAPE_OFFSET_X = 0; +export const DEFAULT_SHAPE_OFFSET_Y = -7; +export const MIN_SHAPE_SCALE = 0; +export const MAX_SHAPE_SCALE = 1.5; + +// Backdrop circle geometry (in 256x256 frame coordinates). The circle sits +// low in the frame and stays inside the inscribed circle that avatar +// components crop to; the pop-out clip extends a column from the circle's +// midline to the frame top so the person can rise out of the circle's top +// without spilling past its sides. +const CIRCLE_GEOMETRY = { + centerX: 128, + centerY: 152, + radius: 100, +}; +const PERSON_OUTLINE_ALPHA = 0.92; +const PERSON_OUTLINE_RADIUS = 2.75; +const PERSON_OUTLINE_OFFSETS = [ + [0, -PERSON_OUTLINE_RADIUS], + [PERSON_OUTLINE_RADIUS, 0], + [0, PERSON_OUTLINE_RADIUS], + [-PERSON_OUTLINE_RADIUS, 0], + [PERSON_OUTLINE_RADIUS * 0.72, -PERSON_OUTLINE_RADIUS * 0.72], + [PERSON_OUTLINE_RADIUS * 0.72, PERSON_OUTLINE_RADIUS * 0.72], + [-PERSON_OUTLINE_RADIUS * 0.72, PERSON_OUTLINE_RADIUS * 0.72], + [-PERSON_OUTLINE_RADIUS * 0.72, -PERSON_OUTLINE_RADIUS * 0.72], +] as const; + +// Pinned to the installed @mediapipe/tasks-vision version so the wasm loader +// always matches the JS API. +const MEDIAPIPE_WASM_BASE = + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.35/wasm"; +const SELFIE_SEGMENTER_MODEL_URL = + "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite"; + +export type AnimatedAvatarRecording = { + /** Square RGBA cut-out frames, mirrored like a selfie preview. */ + frames: ImageData[]; + /** False when the segmentation model couldn't be loaded. */ + backgroundRemoved: boolean; +}; + +export type AvatarCameraDevice = { + deviceId: string; + label: string; +}; + +export async function listAvatarCameras(): Promise { + if (!navigator.mediaDevices?.enumerateDevices) { + return []; + } + + const devices = await navigator.mediaDevices.enumerateDevices(); + return devices + .filter((device) => device.kind === "videoinput" && device.deviceId) + .map((device, index) => ({ + deviceId: device.deviceId, + label: device.label || `Camera ${index + 1}`, + })); +} + +export async function openAvatarCamera( + deviceId?: string | null, +): Promise { + const video: MediaTrackConstraints = { + facingMode: { ideal: "user" }, + frameRate: { ideal: 30 }, + height: { ideal: 1080 }, + width: { ideal: 1080 }, + }; + if (deviceId) { + video.deviceId = { exact: deviceId }; + } + + return navigator.mediaDevices.getUserMedia({ + audio: false, + video, + }); +} + +export function stopAvatarCamera(stream: MediaStream): void { + for (const track of stream.getTracks()) { + track.stop(); + } +} + +type SegmenterHandle = { + segmenter: ImageSegmenter; + /** Which confidence-mask channel holds the person. */ + personChannel: number; + /** True when the channel is a background mask and must be inverted. */ + invert: boolean; +}; + +let segmenterPromise: Promise | null = null; + +/** + * Lazily create the selfie segmenter. Resolves to null when the CDN assets + * are unreachable; a failed load is retried on the next call. + */ +function loadSegmenter(): Promise { + if (!segmenterPromise) { + segmenterPromise = createSegmenter().catch(() => { + segmenterPromise = null; + return null; + }); + } + return segmenterPromise; +} + +async function createSegmenter(): Promise { + const vision = await import("@mediapipe/tasks-vision"); + const fileset = + await vision.FilesetResolver.forVisionTasks(MEDIAPIPE_WASM_BASE); + const segmenter = await vision.ImageSegmenter.createFromOptions(fileset, { + baseOptions: { modelAssetPath: SELFIE_SEGMENTER_MODEL_URL }, + outputCategoryMask: false, + outputConfidenceMasks: true, + runningMode: "VIDEO", + }); + + // The selfie model labels vary across releases ("background"/"person" vs a + // single foreground channel) — resolve which channel is the person once. + const labels = segmenter.getLabels().map((label) => label.toLowerCase()); + let personChannel = labels.findIndex( + (label) => + label.includes("person") || + label.includes("selfie") || + label.includes("foreground"), + ); + let invert = false; + if (personChannel < 0) { + personChannel = 0; + invert = + labels.length === 1 && (labels[0]?.includes("background") ?? false); + } + + return { invert, personChannel, segmenter }; +} + +/** Warm the segmentation model while the user lines up their shot. */ +export function preloadAvatarSegmenter(): void { + void loadSegmenter(); +} + +export async function recordAnimatedAvatarFrames( + video: HTMLVideoElement, + options: { + signal?: AbortSignal; + onProgress?: (fraction: number) => void; + } = {}, +): Promise { + const { onProgress, signal } = options; + const handle = await loadSegmenter(); + + const canvas = document.createElement("canvas"); + canvas.width = ANIMATED_AVATAR_CAPTURE_SIZE; + canvas.height = ANIMATED_AVATAR_CAPTURE_SIZE; + const context = canvas.getContext("2d", { willReadFrequently: true }); + if (!context) { + throw new Error("Could not create a drawing context for recording."); + } + + const frames: ImageData[] = []; + let lastTimestamp = 0; + let previousDenoisedFrame: Uint8ClampedArray | null = null; + let previousMask: Float32Array | null = null; + + for (let i = 0; i < ANIMATED_AVATAR_FRAME_COUNT; i++) { + if (signal?.aborted) { + throw new DOMException("Recording cancelled", "AbortError"); + } + + const frameStart = performance.now(); + drawMirroredCenterCrop(context, video); + const frame = context.getImageData( + 0, + 0, + ANIMATED_AVATAR_CAPTURE_SIZE, + ANIMATED_AVATAR_CAPTURE_SIZE, + ); + previousDenoisedFrame = stabilizeDarkDetailNoise( + frame, + previousDenoisedFrame, + ); + + if (handle) { + // Video-mode segmentation requires strictly increasing timestamps. + let timestamp = performance.now(); + if (timestamp <= lastTimestamp) { + timestamp = lastTimestamp + 1; + } + lastTimestamp = timestamp; + const result = handle.segmenter.segmentForVideo(canvas, timestamp); + try { + previousMask = applyPersonMask( + frame, + result.confidenceMasks, + handle, + previousMask, + ); + } finally { + result.close(); + } + } + + frames.push(frame); + onProgress?.((i + 1) / ANIMATED_AVATAR_FRAME_COUNT); + + const elapsed = performance.now() - frameStart; + await sleep(Math.max(0, ANIMATED_AVATAR_FRAME_DELAY_MS - elapsed)); + } + + return { backgroundRemoved: handle !== null, frames }; +} + +type ConfidenceMasks = + | readonly { + getAsFloat32Array(): Float32Array; + width: number; + height: number; + }[] + | undefined; + +function applyPersonMask( + frame: ImageData, + masks: ConfidenceMasks, + handle: SegmenterHandle, + previousMask: Float32Array | null, +): Float32Array | null { + const mask = masks?.[handle.personChannel] ?? masks?.[0]; + if (!mask) { + return previousMask; + } + + const values = mask.getAsFloat32Array(); + const pixels = frame.data; + const scaleX = mask.width / frame.width; + const scaleY = mask.height / frame.height; + const rampRange = PERSON_CONFIDENCE_HIGH - PERSON_CONFIDENCE_LOW; + const nextMask = new Float32Array(frame.width * frame.height); + const canSmoothMask = previousMask?.length === nextMask.length; + + for (let y = 0; y < frame.height; y++) { + for (let x = 0; x < frame.width; x++) { + const confidence = sampleMaskConfidence( + values, + mask.width, + mask.height, + x * scaleX, + y * scaleY, + ); + let person = handle.invert ? 1 - confidence : confidence; + const pixelIndex = y * frame.width + x; + if (canSmoothMask) { + person = stabilizeMaskConfidence( + person, + previousMask[pixelIndex] ?? person, + ); + } + nextMask[pixelIndex] = person; + const offset = (y * frame.width + x) * 4; + if (person <= PERSON_CONFIDENCE_LOW) { + pixels[offset] = 0; + pixels[offset + 1] = 0; + pixels[offset + 2] = 0; + pixels[offset + 3] = 0; + } else if (person < PERSON_CONFIDENCE_HIGH) { + // Feathered edge — APNG keeps the 8-bit alpha, so the cut-out blends + // smoothly instead of GIF-style jagged edges. + const alpha = (person - PERSON_CONFIDENCE_LOW) / rampRange; + pixels[offset + 3] = Math.round((pixels[offset + 3] ?? 255) * alpha); + } + } + } + + return nextMask; +} + +function stabilizeDarkDetailNoise( + frame: ImageData, + previousFrame: Uint8ClampedArray | null, +): Uint8ClampedArray { + const pixels = frame.data; + const nextFrame = new Uint8ClampedArray(pixels); + if (!previousFrame || previousFrame.length !== pixels.length) { + return nextFrame; + } + + for (let offset = 0; offset < pixels.length; offset += 4) { + const red = pixels[offset] ?? 0; + const green = pixels[offset + 1] ?? 0; + const blue = pixels[offset + 2] ?? 0; + const luma = 0.2126 * red + 0.7152 * green + 0.0722 * blue; + const darkWeight = clamp01( + (DARK_DETAIL_DENOISE_LUMA_HIGH - luma) / + (DARK_DETAIL_DENOISE_LUMA_HIGH - DARK_DETAIL_DENOISE_LUMA_LOW), + ); + if (darkWeight <= 0) { + continue; + } + + const previousRed = previousFrame[offset] ?? red; + const previousGreen = previousFrame[offset + 1] ?? green; + const previousBlue = previousFrame[offset + 2] ?? blue; + const colorDiff = + (Math.abs(red - previousRed) + + Math.abs(green - previousGreen) + + Math.abs(blue - previousBlue)) / + 3; + const stabilityWeight = clamp01( + (DARK_DETAIL_DENOISE_DIFF_HIGH - colorDiff) / + (DARK_DETAIL_DENOISE_DIFF_HIGH - DARK_DETAIL_DENOISE_DIFF_LOW), + ); + if (stabilityWeight <= 0) { + continue; + } + + const blend = DARK_DETAIL_DENOISE_MAX_BLEND * darkWeight * stabilityWeight; + pixels[offset] = Math.round(red * (1 - blend) + previousRed * blend); + pixels[offset + 1] = Math.round( + green * (1 - blend) + previousGreen * blend, + ); + pixels[offset + 2] = Math.round(blue * (1 - blend) + previousBlue * blend); + nextFrame[offset] = pixels[offset] ?? red; + nextFrame[offset + 1] = pixels[offset + 1] ?? green; + nextFrame[offset + 2] = pixels[offset + 2] ?? blue; + } + + return nextFrame; +} + +function stabilizeMaskConfidence(current: number, previous: number): number { + const delta = Math.abs(current - previous); + const blend = + MASK_TEMPORAL_MAX_BLEND * + Math.max(0, 1 - delta / MASK_TEMPORAL_MOTION_THRESHOLD); + return previous * blend + current * (1 - blend); +} + +function clamp01(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +function drawMirroredCenterCrop( + context: CanvasRenderingContext2D, + video: HTMLVideoElement, +): void { + const sourceSize = Math.min(video.videoWidth, video.videoHeight) || 1; + const sourceX = (video.videoWidth - sourceSize) / 2; + const sourceY = (video.videoHeight - sourceSize) / 2; + + context.save(); + context.clearRect( + 0, + 0, + ANIMATED_AVATAR_CAPTURE_SIZE, + ANIMATED_AVATAR_CAPTURE_SIZE, + ); + // Mirror horizontally so the capture matches the selfie-style live preview. + context.translate(ANIMATED_AVATAR_CAPTURE_SIZE, 0); + context.scale(-1, 1); + context.drawImage( + video, + sourceX, + sourceY, + sourceSize, + sourceSize, + 0, + 0, + ANIMATED_AVATAR_CAPTURE_SIZE, + ANIMATED_AVATAR_CAPTURE_SIZE, + ); + context.restore(); +} + +/** Convert recorded frames into bitmaps for fast repeated composition. */ +export function createAvatarFrameBitmaps( + frames: ImageData[], +): Promise { + return Promise.all(frames.map((frame) => createImageBitmap(frame))); +} + +/** + * Draw one composed avatar frame: the cut-out person, placed per the + * composition's offset/scale, popping out of the backdrop circle. With no + * backdrop color the placed cut-out renders on its own. The context must + * belong to a 256x256 canvas. + */ +export function composeAvatarFrame( + context: CanvasRenderingContext2D, + person: CanvasImageSource, + composition: AvatarComposition, +): void { + const size = ANIMATED_AVATAR_SIZE; + context.clearRect(0, 0, size, size); + + // Person placement: centered horizontally and bottom-anchored at the + // default, then shifted/scaled by the user's framing choices. + const personSize = size * composition.scale; + const personX = (size - personSize) / 2 + composition.offsetX; + const personY = size - personSize + composition.offsetY; + + if (!composition.backdropColor || composition.shapeScale <= 0) { + drawPersonWithOutline( + context, + person, + composition, + personX, + personY, + personSize, + ); + return; + } + + // The circle scales around its own center, then shifts by the user's + // offsets. + const geometry = CIRCLE_GEOMETRY; + const shapeScale = composition.shapeScale; + const circleX = geometry.centerX + composition.shapeOffsetX; + const circleY = geometry.centerY + composition.shapeOffsetY; + const circleRadius = geometry.radius * shapeScale; + + context.fillStyle = composition.backdropColor; + context.beginPath(); + context.arc(circleX, circleY, circleRadius, 0, Math.PI * 2); + context.fill(); + + // Clip to the circle plus the column above it, so the person rises out of + // the circle's top without spilling past its sides. + context.save(); + context.beginPath(); + context.arc(circleX, circleY, circleRadius, 0, Math.PI * 2); + context.rect( + circleX - circleRadius, + 0, + circleRadius * 2, + Math.max(0, circleY), + ); + context.clip(); + drawPersonWithOutline( + context, + person, + composition, + personX, + personY, + personSize, + ); + context.restore(); +} + +function drawPersonWithOutline( + context: CanvasRenderingContext2D, + person: CanvasImageSource, + composition: AvatarComposition, + personX: number, + personY: number, + personSize: number, +): void { + if (composition.personOutline) { + drawPersonOutline( + context, + person, + composition.backdropColor, + personX, + personY, + personSize, + ); + } + + context.drawImage(person, personX, personY, personSize, personSize); +} + +function drawPersonOutline( + context: CanvasRenderingContext2D, + person: CanvasImageSource, + backdropColor: string | null, + personX: number, + personY: number, + personSize: number, +): void { + const outline = document.createElement("canvas"); + outline.width = ANIMATED_AVATAR_SIZE; + outline.height = ANIMATED_AVATAR_SIZE; + const outlineContext = outline.getContext("2d"); + if (!outlineContext) { + return; + } + + outlineContext.drawImage(person, personX, personY, personSize, personSize); + outlineContext.globalCompositeOperation = "source-in"; + outlineContext.fillStyle = personOutlineColor(backdropColor); + outlineContext.fillRect(0, 0, ANIMATED_AVATAR_SIZE, ANIMATED_AVATAR_SIZE); + + context.save(); + context.globalAlpha = PERSON_OUTLINE_ALPHA; + context.imageSmoothingEnabled = false; + for (const [offsetX, offsetY] of PERSON_OUTLINE_OFFSETS) { + context.drawImage(outline, offsetX, offsetY); + } + context.restore(); +} + +function personOutlineColor(backdropColor: string | null): string { + const color = backdropColor ? parseHexColor(backdropColor) : null; + if (!color) { + return "#ffffff"; + } + + const luma = + (0.2126 * color.red + 0.7152 * color.green + 0.0722 * color.blue) / 255; + return luma > 0.74 ? "#111111" : "#ffffff"; +} + +function parseHexColor( + value: string, +): { red: number; green: number; blue: number } | null { + const match = /^#?([0-9a-f]{6})$/i.exec(value); + if (!match) { + return null; + } + + const hex = match[1]; + if (!hex) { + return null; + } + + return { + blue: Number.parseInt(hex.slice(4, 6), 16), + green: Number.parseInt(hex.slice(2, 4), 16), + red: Number.parseInt(hex.slice(0, 2), 16), + }; +} + +/** Compose every recorded frame with the chosen backdrop and framing. */ +export function composeAvatarFrames( + bitmaps: ImageBitmap[], + composition: AvatarComposition, +): ImageData[] { + const canvas = document.createElement("canvas"); + canvas.width = ANIMATED_AVATAR_SIZE; + canvas.height = ANIMATED_AVATAR_SIZE; + const context = canvas.getContext("2d", { willReadFrequently: true }); + if (!context) { + throw new Error("Could not create a drawing context."); + } + + return bitmaps.map((bitmap) => { + composeAvatarFrame(context, bitmap, composition); + return context.getImageData( + 0, + 0, + ANIMATED_AVATAR_SIZE, + ANIMATED_AVATAR_SIZE, + ); + }); +} + +/** Mirror the sequence so the animation plays forward, then backward. */ +export function buildPingPongAvatarFrames(frames: T[]): T[] { + if (frames.length <= 2) { + return frames; + } + + return frames.concat(frames.slice(1, -1).reverse()); +} + +/** Encode composed frames as an infinitely looping ping-pong animated PNG. */ +export function encodeAvatarAnimation(frames: ImageData[]): Uint8Array { + const animationFrames = buildPingPongAvatarFrames(frames); + const buffers = animationFrames.map( + (frame) => + frame.data.buffer.slice( + frame.data.byteOffset, + frame.data.byteOffset + frame.data.byteLength, + ) as ArrayBuffer, + ); + const delays = animationFrames.map(() => ANIMATED_AVATAR_FRAME_DELAY_MS); + const encoded = UPNG.encode( + buffers, + ANIMATED_AVATAR_SIZE, + ANIMATED_AVATAR_SIZE, + APNG_COLOR_COUNT, + delays, + ); + return new Uint8Array(encoded); +} + +function sampleMaskConfidence( + values: Float32Array, + width: number, + height: number, + x: number, + y: number, +): number { + const sampleX = Math.min(Math.max(x, 0), Math.max(0, width - 1)); + const sampleY = Math.min(Math.max(y, 0), Math.max(0, height - 1)); + const x0 = Math.floor(sampleX); + const y0 = Math.floor(sampleY); + const x1 = Math.min(width - 1, x0 + 1); + const y1 = Math.min(height - 1, y0 + 1); + const tx = sampleX - x0; + const ty = sampleY - y0; + const topLeft = values[y0 * width + x0] ?? 0; + const topRight = values[y0 * width + x1] ?? topLeft; + const bottomLeft = values[y1 * width + x0] ?? topLeft; + const bottomRight = values[y1 * width + x1] ?? bottomLeft; + const top = topLeft + (topRight - topLeft) * tx; + const bottom = bottomLeft + (bottomRight - bottomLeft) * tx; + return top + (bottom - top) * ty; +} + +export async function renderAvatarPosterPng( + frame: ImageData, +): Promise { + const blob = await new Promise((resolve) => { + frameToCanvas(frame).toBlob(resolve, "image/png"); + }); + if (!blob) { + throw new Error("Could not render the poster frame."); + } + return new Uint8Array(await blob.arrayBuffer()); +} + +function frameToCanvas(frame: ImageData): HTMLCanvasElement { + const canvas = document.createElement("canvas"); + canvas.width = frame.width; + canvas.height = frame.height; + const context = canvas.getContext("2d"); + if (!context) { + throw new Error("Could not create a drawing context."); + } + context.putImageData(frame, 0, 0); + return canvas; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/desktop/src/features/profile/ui/AnimatedAvatarBackdropPanel.tsx b/desktop/src/features/profile/ui/AnimatedAvatarBackdropPanel.tsx new file mode 100644 index 000000000..d167e5523 --- /dev/null +++ b/desktop/src/features/profile/ui/AnimatedAvatarBackdropPanel.tsx @@ -0,0 +1,90 @@ +import { + AVATAR_COLOR_SWATCHES, + CUSTOM_AVATAR_COLOR_SWATCH, + contrastColorForBackground, +} from "@/features/profile/ui/ProfileAvatarEditor.utils"; + +type AnimatedAvatarBackdropPanelProps = { + backdropColor: string | null; + disabled?: boolean; + isCustomBackdropSelected: boolean; + isSaving: boolean; + onOpenCustomPicker: () => void; + onSelectColor: (color: string) => void; + testIdPrefix: string; +}; + +export function AnimatedAvatarBackdropPanel({ + backdropColor, + disabled = false, + isCustomBackdropSelected, + isSaving, + onOpenCustomPicker, + onSelectColor, + testIdPrefix, +}: AnimatedAvatarBackdropPanelProps) { + return ( +
+
+ {AVATAR_COLOR_SWATCHES.map((swatch) => { + const isCustomSwatch = swatch === CUSTOM_AVATAR_COLOR_SWATCH; + const isSelected = isCustomSwatch + ? isCustomBackdropSelected + : backdropColor !== null && + swatch.toUpperCase() === backdropColor.toUpperCase(); + + return ( + + ); + })} +
+
+ ); +} diff --git a/desktop/src/features/profile/ui/AnimatedAvatarCameraPicker.tsx b/desktop/src/features/profile/ui/AnimatedAvatarCameraPicker.tsx new file mode 100644 index 000000000..1525f0312 --- /dev/null +++ b/desktop/src/features/profile/ui/AnimatedAvatarCameraPicker.tsx @@ -0,0 +1,77 @@ +import { Smartphone, Webcam } from "lucide-react"; + +import type { CameraSource } from "@/features/profile/ui/AnimatedAvatarCapture.helpers"; +import { cn } from "@/shared/lib/cn"; + +type AnimatedAvatarCameraPickerProps = { + activeCameraSource: CameraSource | null; + computerDisabled: boolean; + disabled?: boolean; + iphoneDisabled: boolean; + onSelectSource: (source: CameraSource) => void; + testIdPrefix: string; +}; + +export function AnimatedAvatarCameraPicker({ + activeCameraSource, + computerDisabled, + disabled = false, + iphoneDisabled, + onSelectSource, + testIdPrefix, +}: AnimatedAvatarCameraPickerProps) { + return ( +
+ {[ + { + disabled: iphoneDisabled, + icon: Smartphone, + label: "Use iPhone", + source: "iphone" as const, + }, + { + disabled: computerDisabled, + icon: Webcam, + label: "Use this computer", + source: "computer" as const, + }, + ].map((option) => { + const Icon = option.icon; + const isSelected = activeCameraSource === option.source; + const isDisabled = disabled || option.disabled; + return ( + + ); + })} +
+ ); +} diff --git a/desktop/src/features/profile/ui/AnimatedAvatarCapture.helpers.ts b/desktop/src/features/profile/ui/AnimatedAvatarCapture.helpers.ts new file mode 100644 index 000000000..711803b27 --- /dev/null +++ b/desktop/src/features/profile/ui/AnimatedAvatarCapture.helpers.ts @@ -0,0 +1,125 @@ +import { + ANIMATED_AVATAR_DURATION_MS, + ANIMATED_AVATAR_SIZE, + type AvatarCameraDevice, + type AvatarComposition, + composeAvatarFrame, + DEFAULT_PERSON_SCALE, +} from "@/features/profile/lib/animatedAvatarCapture"; +import { AVATAR_COLORS } from "@/features/profile/ui/ProfileAvatarEditor.utils"; + +export type CapturePhase = + | "idle" + | "starting" + | "live" + | "recording" + | "processing" + | "review"; +export type CameraSource = "computer" | "iphone"; + +const FILMSTRIP_FRAME_SIZE = 48; +const SLIDER_TICK_STEP = 10; +const PHONE_DEFAULT_PERSON_SCALE = + (Math.round(DEFAULT_PERSON_SCALE * 100) - SLIDER_TICK_STEP) / 100; + +export const RECORD_SECONDS = ANIMATED_AVATAR_DURATION_MS / 1000; +export const PERSON_SIZE_TIP = + "Scale just past the color circle for a pop-out look."; +export const ENTRANCE_TRANSITION = { + duration: 0.18, + ease: [0.23, 1, 0.32, 1], +} as const; + +/** Keep things draggable but never fully lost off-frame. */ +const MAX_OFFSET = 192; + +export function clampOffset(value: number): number { + return Math.min(MAX_OFFSET, Math.max(-MAX_OFFSET, value)); +} + +export function randomBackdropColor(): string { + return ( + AVATAR_COLORS[Math.floor(Math.random() * AVATAR_COLORS.length)] ?? + AVATAR_COLORS[0] ?? + "#FFFFFF" + ); +} + +function isIPhoneCamera(device: AvatarCameraDevice): boolean { + return /\b(continuity|ios|iphone|mobile|phone)\b/i.test(device.label); +} + +function isFrontCamera(device: AvatarCameraDevice): boolean { + return /\b(front|facetime|user)\b/i.test(device.label); +} + +export function cameraLabelsAreAvailable( + devices: AvatarCameraDevice[], +): boolean { + return devices.some((device) => device.label.trim().length > 0); +} + +export function preferredCameraDevice( + devices: AvatarCameraDevice[], + source: CameraSource, +): AvatarCameraDevice | null { + const candidates = + source === "iphone" + ? devices.filter(isIPhoneCamera) + : devices.filter((device) => !isIPhoneCamera(device)); + return ( + candidates.find(isFrontCamera) ?? + candidates[0] ?? + (source === "computer" ? devices[0] : null) ?? + null + ); +} + +export function defaultPersonScaleForSource( + source: CameraSource | null, +): number { + return source === "iphone" + ? PHONE_DEFAULT_PERSON_SCALE + : DEFAULT_PERSON_SCALE; +} + +export function clampFrameIndex(value: number, frameCount: number): number { + return Math.min(Math.max(0, frameCount - 1), Math.max(0, value)); +} + +export function buildFilmstripFrames( + bitmaps: ImageBitmap[], + composition: AvatarComposition, +): string[] { + const composed = document.createElement("canvas"); + composed.width = ANIMATED_AVATAR_SIZE; + composed.height = ANIMATED_AVATAR_SIZE; + const composedContext = composed.getContext("2d"); + const thumbnail = document.createElement("canvas"); + thumbnail.width = FILMSTRIP_FRAME_SIZE; + thumbnail.height = FILMSTRIP_FRAME_SIZE; + const thumbnailContext = thumbnail.getContext("2d"); + if (!composedContext || !thumbnailContext) { + return []; + } + + const urls: string[] = []; + for (const bitmap of bitmaps) { + composeAvatarFrame(composedContext, bitmap, composition); + thumbnailContext.clearRect( + 0, + 0, + FILMSTRIP_FRAME_SIZE, + FILMSTRIP_FRAME_SIZE, + ); + thumbnailContext.drawImage( + composed, + 0, + 0, + FILMSTRIP_FRAME_SIZE, + FILMSTRIP_FRAME_SIZE, + ); + urls.push(thumbnail.toDataURL("image/png")); + } + return urls; +} diff --git a/desktop/src/features/profile/ui/AnimatedAvatarCapture.tsx b/desktop/src/features/profile/ui/AnimatedAvatarCapture.tsx new file mode 100644 index 000000000..f19fe9199 --- /dev/null +++ b/desktop/src/features/profile/ui/AnimatedAvatarCapture.tsx @@ -0,0 +1,996 @@ +import { Camera, Video } from "lucide-react"; +import { motion } from "motion/react"; +import * as React from "react"; +import { createPortal } from "react-dom"; + +import { + type AnimatedAvatarRecording, + ANIMATED_AVATAR_FRAME_DELAY_MS, + ANIMATED_AVATAR_SIZE, + type AvatarCameraDevice, + type AvatarComposition, + buildPingPongAvatarFrames, + composeAvatarFrame, + composeAvatarFrames, + createAvatarFrameBitmaps, + DEFAULT_PERSON_OFFSET_X, + DEFAULT_PERSON_OFFSET_Y, + DEFAULT_PERSON_OUTLINE, + DEFAULT_PERSON_SCALE, + DEFAULT_SHAPE_OFFSET_X, + DEFAULT_SHAPE_OFFSET_Y, + DEFAULT_SHAPE_SCALE, + encodeAvatarAnimation, + MAX_PERSON_SCALE, + MAX_SHAPE_SCALE, + MIN_PERSON_SCALE, + MIN_SHAPE_SCALE, + listAvatarCameras, + openAvatarCamera, + preloadAvatarSegmenter, + recordAnimatedAvatarFrames, + renderAvatarPosterPng, + stopAvatarCamera, +} from "@/features/profile/lib/animatedAvatarCapture"; +import { AnimatedAvatarBackdropPanel } from "@/features/profile/ui/AnimatedAvatarBackdropPanel"; +import { AnimatedAvatarCameraPicker } from "@/features/profile/ui/AnimatedAvatarCameraPicker"; +import { + AvatarFilmstripPicker, + AvatarFramingSlider, + AvatarOutlineToggle, +} from "@/features/profile/ui/AnimatedAvatarControls"; +import { + buildFilmstripFrames, + cameraLabelsAreAvailable, + type CapturePhase, + type CameraSource, + clampFrameIndex, + clampOffset, + defaultPersonScaleForSource, + ENTRANCE_TRANSITION, + PERSON_SIZE_TIP, + preferredCameraDevice, + randomBackdropColor, + RECORD_SECONDS, +} from "@/features/profile/ui/AnimatedAvatarCapture.helpers"; +import { + AnimatedAvatarReviewNav, + type ReviewSection, +} from "@/features/profile/ui/AnimatedAvatarReviewNav"; +import { AvatarCustomColorPanel } from "@/features/profile/ui/AvatarCustomColorPanel"; +import { + AVATAR_COLORS, + hexToHsv, + hsvToHex, + normalizeHue, +} from "@/features/profile/ui/ProfileAvatarEditor.utils"; +import { uploadMediaBytes } from "@/shared/api/tauri"; +import { buildAnimatedAvatarUrl } from "@/shared/lib/animatedAvatar"; +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; +import { Spinner } from "@/shared/ui/spinner"; + +type AnimatedAvatarCaptureProps = { + disabled?: boolean; + testIdPrefix: string; + /** Receives the composed animated avatar URL after upload. */ + onApply: (avatarUrl: string) => void; + /** + * Host element inside the page's main avatar preview. When provided, the + * camera feed, recording ring, and composed preview render there (via a + * portal) instead of inside the tab — so edits show exactly where the + * avatar will live. Pair with `onPreviewActiveChange` so the host can + * hide its regular preview while the capture content is showing. + */ + previewContainer?: HTMLElement | null; + onPreviewActiveChange?: (active: boolean) => void; + onPreviewCaptionChange?: (caption: string | null) => void; + onCustomColorPickerOpenChange?: (isOpen: boolean) => void; + /** + * Receives the current apply function (or null when there is nothing to + * apply) so the host's Done button can upload-and-apply in one step. + */ + registerApply?: (apply: (() => Promise) | null) => void; + /** Show the in-tab "Use as avatar" button (hosts without a Done button). */ + showApplyButton?: boolean; +}; + +export function AnimatedAvatarCapture({ + disabled = false, + testIdPrefix, + onApply, + onCustomColorPickerOpenChange, + previewContainer = null, + onPreviewActiveChange, + onPreviewCaptionChange, + registerApply, + showApplyButton = true, +}: AnimatedAvatarCaptureProps) { + const [phase, setPhase] = React.useState("idle"); + const [errorMessage, setErrorMessage] = React.useState(null); + const [cameraDevices, setCameraDevices] = React.useState< + AvatarCameraDevice[] + >([]); + const [selectedCameraSource, setSelectedCameraSource] = + React.useState(null); + const [selectedCameraId, setSelectedCameraId] = React.useState( + null, + ); + const [recordProgress, setRecordProgress] = React.useState(0); + const [recording, setRecording] = + React.useState(null); + const [bitmaps, setBitmaps] = React.useState([]); + const [filmstripFrames, setFilmstripFrames] = React.useState([]); + const [posterIndex, setPosterIndex] = React.useState(0); + const [backdropColor, setBackdropColor] = React.useState( + randomBackdropColor, + ); + const [personOffset, setPersonOffset] = React.useState({ + x: DEFAULT_PERSON_OFFSET_X, + y: DEFAULT_PERSON_OFFSET_Y, + }); + const [personScale, setPersonScale] = React.useState(DEFAULT_PERSON_SCALE); + const [personOutline, setPersonOutline] = React.useState( + DEFAULT_PERSON_OUTLINE, + ); + const [shapeOffset, setShapeOffset] = React.useState({ + x: DEFAULT_SHAPE_OFFSET_X, + y: DEFAULT_SHAPE_OFFSET_Y, + }); + const [shapeScale, setShapeScale] = React.useState(DEFAULT_SHAPE_SCALE); + const [activeSection, setActiveSection] = + React.useState("person"); + const [isPreviewPlaying, setIsPreviewPlaying] = React.useState(false); + const [isDraggingPerson, setIsDraggingPerson] = React.useState(false); + const [isSaving, setIsSaving] = React.useState(false); + + // Drag, arrow keys, and the size slider adjust whichever framing section + // is open; the color/poster sections leave the person active. + const editTarget = activeSection === "shape" ? "shape" : "person"; + const activeOffset = editTarget === "shape" ? shapeOffset : personOffset; + const setActiveOffset = + editTarget === "shape" ? setShapeOffset : setPersonOffset; + const activeScale = editTarget === "shape" ? shapeScale : personScale; + const setActiveScale = + editTarget === "shape" ? setShapeScale : setPersonScale; + const activeScaleMin = + editTarget === "shape" ? MIN_SHAPE_SCALE : MIN_PERSON_SCALE; + const activeScaleMax = + editTarget === "shape" ? MAX_SHAPE_SCALE : MAX_PERSON_SCALE; + const activeScaleReset = + editTarget === "shape" ? DEFAULT_SHAPE_SCALE : DEFAULT_PERSON_SCALE; + + const resetActiveFraming = React.useCallback(() => { + if (activeSection === "shape") { + setShapeOffset({ x: DEFAULT_SHAPE_OFFSET_X, y: DEFAULT_SHAPE_OFFSET_Y }); + setShapeScale(DEFAULT_SHAPE_SCALE); + return; + } + setPersonOffset({ + x: DEFAULT_PERSON_OFFSET_X, + y: DEFAULT_PERSON_OFFSET_Y, + }); + setPersonScale(DEFAULT_PERSON_SCALE); + }, [activeSection]); + + const resetAllFraming = React.useCallback(() => { + setPersonOffset({ + x: DEFAULT_PERSON_OFFSET_X, + y: DEFAULT_PERSON_OFFSET_Y, + }); + setPersonScale(DEFAULT_PERSON_SCALE); + setShapeOffset({ x: DEFAULT_SHAPE_OFFSET_X, y: DEFAULT_SHAPE_OFFSET_Y }); + setShapeScale(DEFAULT_SHAPE_SCALE); + }, []); + + // Custom backdrop color picker (shared HSV panel). + const [isCustomPickerOpen, setIsCustomPickerOpen] = React.useState(false); + const [customHue, setCustomHue] = React.useState(210); + const [customSaturation, setCustomSaturation] = React.useState(80); + const [customValue, setCustomValue] = React.useState(90); + const customColorDraft = React.useMemo( + () => hsvToHex(customHue, customSaturation, customValue), + [customHue, customSaturation, customValue], + ); + const isCustomPickerVisible = isCustomPickerOpen && phase === "review"; + const visibleBackdropColor = isCustomPickerVisible + ? customColorDraft + : backdropColor; + + const composition = React.useMemo( + () => ({ + backdropColor: visibleBackdropColor, + offsetX: personOffset.x, + offsetY: personOffset.y, + personOutline, + scale: personScale, + shapeOffsetX: shapeOffset.x, + shapeOffsetY: shapeOffset.y, + shapeScale, + }), + [ + visibleBackdropColor, + personOffset, + personOutline, + personScale, + shapeOffset, + shapeScale, + ], + ); + + const videoRef = React.useRef(null); + const previewCanvasRef = React.useRef(null); + const streamRef = React.useRef(null); + const activeStreamSourceRef = React.useRef(null); + const recordAbortRef = React.useRef(null); + const dragStateRef = React.useRef<{ + pointerId: number; + startClientX: number; + startClientY: number; + baseOffsetX: number; + baseOffsetY: number; + } | null>(null); + + const releaseCamera = React.useCallback(() => { + if (streamRef.current) { + stopAvatarCamera(streamRef.current); + streamRef.current = null; + } + activeStreamSourceRef.current = null; + if (videoRef.current) { + videoRef.current.srcObject = null; + } + }, []); + + const releaseBitmaps = React.useCallback(() => { + setBitmaps((previous) => { + for (const bitmap of previous) { + bitmap.close(); + } + return []; + }); + }, []); + + const refreshCameraDevices = React.useCallback(async () => { + try { + const devices = await listAvatarCameras(); + setCameraDevices(devices); + setSelectedCameraId((current) => { + if (current && devices.some((device) => device.deviceId === current)) { + return current; + } + return selectedCameraSource + ? (preferredCameraDevice(devices, selectedCameraSource)?.deviceId ?? + null) + : null; + }); + } catch { + setCameraDevices([]); + setSelectedCameraId(null); + } + }, [selectedCameraSource]); + + React.useEffect(() => { + return () => { + recordAbortRef.current?.abort(); + releaseCamera(); + }; + }, [releaseCamera]); + + // Close bitmaps on unmount only — phase changes manage them explicitly. + React.useEffect(() => releaseBitmaps, [releaseBitmaps]); + + React.useEffect(() => { + void refreshCameraDevices(); + const mediaDevices = navigator.mediaDevices; + if (!mediaDevices?.addEventListener) { + return; + } + const handleDeviceChange = () => { + void refreshCameraDevices(); + }; + mediaDevices.addEventListener("devicechange", handleDeviceChange); + return () => { + mediaDevices.removeEventListener("devicechange", handleDeviceChange); + }; + }, [refreshCameraDevices]); + + // Review preview mirrors the final avatar: the selected poster frame is + // shown as a still, and hovering plays the animation. + React.useEffect(() => { + if (phase !== "review" || bitmaps.length === 0) { + return; + } + const canvas = previewCanvasRef.current; + const context = canvas?.getContext("2d"); + if (!context) { + return; + } + + if (!isPreviewPlaying) { + const poster = bitmaps[Math.min(posterIndex, bitmaps.length - 1)]; + if (poster) { + composeAvatarFrame(context, poster, composition); + } + return; + } + + const previewFrames = buildPingPongAvatarFrames(bitmaps); + let frameIndex = 0; + let lastDrawn = 0; + let animationFrame = 0; + const tick = (now: number) => { + if (now - lastDrawn >= ANIMATED_AVATAR_FRAME_DELAY_MS) { + const bitmap = previewFrames[frameIndex]; + if (bitmap) { + composeAvatarFrame(context, bitmap, composition); + } + frameIndex = (frameIndex + 1) % previewFrames.length; + lastDrawn = now; + } + animationFrame = requestAnimationFrame(tick); + }; + animationFrame = requestAnimationFrame(tick); + + return () => cancelAnimationFrame(animationFrame); + }, [phase, bitmaps, composition, isPreviewPlaying, posterIndex]); + + // Filmstrip frames track the composition. Debounced so dragging or + // scrubbing the size slider doesn't re-render the strip per tick. + React.useEffect(() => { + if (phase !== "review" || bitmaps.length === 0) { + setFilmstripFrames([]); + return; + } + const handle = setTimeout(() => { + setFilmstripFrames(buildFilmstripFrames(bitmaps, composition)); + }, 200); + return () => clearTimeout(handle); + }, [phase, bitmaps, composition]); + + const startCamera = React.useCallback( + async (cameraId = selectedCameraId, source = selectedCameraSource) => { + setErrorMessage(null); + setPhase("starting"); + releaseCamera(); + // Warm the segmentation model while the user lines up their shot. + preloadAvatarSegmenter(); + try { + let stream = await openAvatarCamera(cameraId || null); + let resolvedCameraId = cameraId ?? null; + if (source === "iphone" && !cameraId) { + const devicesAfterPermission = await listAvatarCameras().catch( + () => [], + ); + setCameraDevices(devicesAfterPermission); + const preferredIphone = preferredCameraDevice( + devicesAfterPermission, + "iphone", + ); + if (!preferredIphone) { + stopAvatarCamera(stream); + setSelectedCameraSource(null); + setSelectedCameraId(null); + setErrorMessage( + "Could not find an iPhone camera. Make sure Continuity Camera is available, then try again.", + ); + setPhase("idle"); + return; + } + + resolvedCameraId = preferredIphone.deviceId; + const activeDeviceId = + stream.getVideoTracks()[0]?.getSettings().deviceId ?? null; + if (activeDeviceId !== preferredIphone.deviceId) { + stopAvatarCamera(stream); + stream = await openAvatarCamera(preferredIphone.deviceId); + } + } + streamRef.current = stream; + activeStreamSourceRef.current = source; + setSelectedCameraId(resolvedCameraId); + const video = videoRef.current; + if (!video) { + stopAvatarCamera(stream); + activeStreamSourceRef.current = null; + setPhase("idle"); + return; + } + video.srcObject = stream; + await video.play(); + setPhase("live"); + void refreshCameraDevices(); + } catch { + releaseCamera(); + setErrorMessage( + "Could not access the camera. Check Buzz's camera permission and try again.", + ); + setPhase("idle"); + } + }, + [ + refreshCameraDevices, + releaseCamera, + selectedCameraId, + selectedCameraSource, + ], + ); + + const selectCameraSource = React.useCallback( + (source: CameraSource) => { + const cameraId = + preferredCameraDevice(cameraDevices, source)?.deviceId ?? null; + const hasDeviceLabels = cameraLabelsAreAvailable(cameraDevices); + if (source === "iphone" && !cameraId && hasDeviceLabels) { + return; + } + setSelectedCameraSource(source); + setSelectedCameraId(cameraId); + if (phase === "idle" || phase === "live") { + void startCamera(cameraId, source); + } + }, + [cameraDevices, phase, startCamera], + ); + + const record = React.useCallback(async () => { + const video = videoRef.current; + if (!video || phase !== "live") { + return; + } + + setErrorMessage(null); + setRecordProgress(0); + setPhase("recording"); + const abort = new AbortController(); + recordAbortRef.current = abort; + + try { + const captured = await recordAnimatedAvatarFrames(video, { + onProgress: setRecordProgress, + signal: abort.signal, + }); + const recordingSource = + activeStreamSourceRef.current ?? selectedCameraSource; + setPhase("processing"); + releaseCamera(); + const nextBitmaps = await createAvatarFrameBitmaps(captured.frames); + releaseBitmaps(); + setBitmaps(nextBitmaps); + setPosterIndex(0); + setRecording(captured); + setPersonOffset({ + x: DEFAULT_PERSON_OFFSET_X, + y: DEFAULT_PERSON_OFFSET_Y, + }); + setPersonScale(defaultPersonScaleForSource(recordingSource)); + setPhase("review"); + } catch (error) { + if (error instanceof DOMException && error.name === "AbortError") { + return; + } + setErrorMessage( + error instanceof Error ? error.message : "Recording failed. Try again.", + ); + releaseCamera(); + setPhase("idle"); + } finally { + recordAbortRef.current = null; + } + }, [phase, releaseBitmaps, releaseCamera, selectedCameraSource]); + + const retake = React.useCallback(() => { + setRecording(null); + setFilmstripFrames([]); + setIsCustomPickerOpen(false); + setIsPreviewPlaying(false); + setActiveSection("person"); + resetAllFraming(); + releaseBitmaps(); + void startCamera(selectedCameraId, selectedCameraSource); + }, [ + releaseBitmaps, + resetAllFraming, + selectedCameraId, + selectedCameraSource, + startCamera, + ]); + + const apply = React.useCallback(async (): Promise => { + if (bitmaps.length === 0 || isSaving) { + return false; + } + + setIsSaving(true); + setErrorMessage(null); + try { + const composed = composeAvatarFrames(bitmaps, composition); + const posterFrame = composed[posterIndex] ?? composed[0]; + if (!posterFrame) { + throw new Error("No frames were recorded."); + } + const animationBytes = encodeAvatarAnimation(composed); + const posterBytes = await renderAvatarPosterPng(posterFrame); + const [animationUpload, posterUpload] = await Promise.all([ + uploadMediaBytes([...animationBytes], "animated-avatar.png"), + uploadMediaBytes([...posterBytes], "animated-avatar-poster.png"), + ]); + if ( + !animationUpload.type.startsWith("image/") || + !posterUpload.type.startsWith("image/") + ) { + setErrorMessage("The relay rejected the recording. Try again."); + return false; + } + onApply(buildAnimatedAvatarUrl(posterUpload.url, animationUpload.url)); + return true; + } catch (error) { + setErrorMessage( + error instanceof Error + ? error.message + : "Could not upload the animated avatar.", + ); + return false; + } finally { + setIsSaving(false); + } + }, [bitmaps, composition, isSaving, onApply, posterIndex]); + + // Hand the host's Done button the current apply function whenever a + // recording is ready to upload. + React.useEffect(() => { + if (!registerApply) { + return; + } + registerApply(phase === "review" && bitmaps.length > 0 ? apply : null); + return () => registerApply(null); + }, [apply, bitmaps.length, phase, registerApply]); + + const openCustomPicker = React.useCallback(() => { + const baseColor = hexToHsv(backdropColor ?? customColorDraft); + setCustomHue(normalizeHue(baseColor.hue)); + setCustomSaturation(baseColor.saturation); + setCustomValue(baseColor.value); + setIsCustomPickerOpen(true); + }, [backdropColor, customColorDraft]); + + const isCameraVisible = phase === "live" || phase === "recording"; + const isCameraStageVisible = + phase === "starting" || phase === "live" || phase === "recording"; + const computerCamera = React.useMemo( + () => preferredCameraDevice(cameraDevices, "computer"), + [cameraDevices], + ); + const iphoneCamera = React.useMemo( + () => preferredCameraDevice(cameraDevices, "iphone"), + [cameraDevices], + ); + const hasCameraLabels = cameraLabelsAreAvailable(cameraDevices); + const activeCameraSource = selectedCameraSource; + const isCustomBackdropSelected = + backdropColor !== null && + !AVATAR_COLORS.some( + (color) => color.toUpperCase() === backdropColor.toUpperCase(), + ); + const isFramingSection = + activeSection === "person" || activeSection === "shape"; + + // With a host preview container, the camera feed / recording ring / + // composed preview render inside the page's main avatar preview (via a + // portal) so edits show exactly where the avatar will live. + const usePortal = previewContainer !== null; + const isPreviewActive = usePortal && phase !== "idle" && phase !== "starting"; + const reviewWarning = + phase === "review" && recording && !recording.backgroundRemoved + ? "Background removal model couldn't be loaded, so the background was kept. Retake while online to remove it." + : null; + const captureHelpText = + phase === "idle" + ? null + : phase === "starting" + ? null + : phase === "live" + ? "Line up your shot." + : phase === "recording" + ? "Recording... hold still-ish." + : phase === "processing" + ? "Cutting you out of the background..." + : null; + const previewCaption = + usePortal && (phase === "live" || phase === "recording") + ? captureHelpText + : usePortal && phase === "review" + ? "Hover to play" + : null; + const inlineCaptureHelpText = + usePortal && (phase === "live" || phase === "recording") + ? null + : captureHelpText; + const showCaptureCard = !usePortal && phase !== "review"; + const showCameraPicker = + phase === "idle" || phase === "starting" || phase === "live"; + + React.useEffect(() => { + onPreviewActiveChange?.(isPreviewActive); + }, [isPreviewActive, onPreviewActiveChange]); + + React.useEffect(() => { + return () => onPreviewActiveChange?.(false); + }, [onPreviewActiveChange]); + + React.useEffect(() => { + onPreviewCaptionChange?.(previewCaption); + + return () => onPreviewCaptionChange?.(null); + }, [onPreviewCaptionChange, previewCaption]); + + React.useLayoutEffect(() => { + onCustomColorPickerOpenChange?.(isCustomPickerVisible); + + return () => { + onCustomColorPickerOpenChange?.(false); + }; + }, [isCustomPickerVisible, onCustomColorPickerOpenChange]); + + const stageContent = ( +
+ {/* Live camera preview — kept mounted so the stream can attach. */} +
+
+ + {/* Recording timer: a stroke that sweeps around the capture circle. */} + {phase === "recording" ? ( + + ) : null} + + {/* Review preview: circle-cropped like the real avatar, shows the + selected poster frame as a still, and plays the composed + animation on hover — exactly how the avatar behaves in the app. + Dragging repositions the active framing target. */} +
{ + if (phase !== "review" || disabled || isSaving) { + return; + } + const step = event.shiftKey ? 16 : 4; + const moves: Record = { + ArrowDown: [0, step], + ArrowLeft: [-step, 0], + ArrowRight: [step, 0], + ArrowUp: [0, -step], + }; + const move = moves[event.key]; + if (!move) { + return; + } + event.preventDefault(); + setActiveOffset((previous) => ({ + x: clampOffset(previous.x + move[0]), + y: clampOffset(previous.y + move[1]), + })); + }} + onMouseEnter={() => setIsPreviewPlaying(true)} + onMouseLeave={() => setIsPreviewPlaying(false)} + onPointerCancel={() => { + dragStateRef.current = null; + setIsDraggingPerson(false); + }} + onPointerDown={(event) => { + if (phase !== "review" || disabled || isSaving) { + return; + } + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + dragStateRef.current = { + baseOffsetX: activeOffset.x, + baseOffsetY: activeOffset.y, + pointerId: event.pointerId, + startClientX: event.clientX, + startClientY: event.clientY, + }; + setIsDraggingPerson(true); + }} + onPointerMove={(event) => { + const drag = dragStateRef.current; + if (!drag || drag.pointerId !== event.pointerId) { + return; + } + const rect = event.currentTarget.getBoundingClientRect(); + const toFrame = ANIMATED_AVATAR_SIZE / Math.max(rect.width, 1); + setActiveOffset({ + x: clampOffset( + drag.baseOffsetX + (event.clientX - drag.startClientX) * toFrame, + ), + y: clampOffset( + drag.baseOffsetY + (event.clientY - drag.startClientY) * toFrame, + ), + }); + }} + onPointerUp={() => { + dragStateRef.current = null; + setIsDraggingPerson(false); + }} + role="application" + tabIndex={phase === "review" ? 0 : -1} + > + +
+ + {phase === "idle" && !usePortal ? ( +
+ +
+ ) : phase === "starting" ? ( +
+
+ + + Starting camera + +
+
+ ) : phase === "processing" ? ( +
+ +
+ ) : null} +
+ ); + + return ( +
+ {usePortal && previewContainer + ? createPortal(stageContent, previewContainer) + : null} + + {showCaptureCard ? ( +
+ {usePortal ? null : stageContent} + + {inlineCaptureHelpText ? ( +

+ {inlineCaptureHelpText} +

+ ) : null} + + {reviewWarning ? ( +

+ {reviewWarning} +

+ ) : null} +
+ ) : null} + + {!showCaptureCard && reviewWarning ? ( +

+ {reviewWarning} +

+ ) : null} + + {phase === "review" ? ( + + ) : null} + + {phase === "review" && isFramingSection ? ( +
+ { + if (activeSection === "shape" && value > 0) { + setBackdropColor((current) => current ?? randomBackdropColor()); + } + setActiveScale(value / 100); + }} + onReset={resetActiveFraming} + resetValue={Math.round(activeScaleReset * 100)} + resetTestId={`${testIdPrefix}-animated-reset-framing`} + testId={`${testIdPrefix}-animated-size`} + tipText={activeSection === "person" ? PERSON_SIZE_TIP : null} + value={Math.round(activeScale * 100)} + /> + {activeSection === "person" ? ( + + ) : null} +
+ ) : null} + + {phase === "review" && activeSection === "color" ? ( + + ) : null} + + {phase === "review" && activeSection === "poster" ? ( + + setPosterIndex(clampFrameIndex(index, bitmaps.length)) + } + selectedFrame={posterIndex} + testIdPrefix={testIdPrefix} + /> + ) : null} + + {showCameraPicker ? ( +
+ 0 && !computerCamera} + disabled={disabled || phase === "starting"} + iphoneDisabled={ + cameraDevices.length > 0 && !iphoneCamera && hasCameraLabels + } + onSelectSource={selectCameraSource} + testIdPrefix={testIdPrefix} + /> + {usePortal && inlineCaptureHelpText ? ( +

+ {inlineCaptureHelpText} +

+ ) : null} +
+ {phase === "live" ? ( + + ) : null} +
+
+ ) : usePortal && inlineCaptureHelpText ? ( +

+ {inlineCaptureHelpText} +

+ ) : null} + + {phase === "review" && showApplyButton ? ( + + ) : null} + + {errorMessage ? ( +

+ {errorMessage} +

+ ) : null} + + { + setBackdropColor(customColorDraft); + setIsCustomPickerOpen(false); + }} + onHueChange={setCustomHue} + onSaturationValueChange={(nextSaturation, nextValue) => { + setCustomSaturation(nextSaturation); + setCustomValue(nextValue); + }} + saturation={customSaturation} + className="h-[504px]" + testIdPrefix={`${testIdPrefix}-animated`} + value={customValue} + visible={isCustomPickerVisible} + /> +
+ ); +} diff --git a/desktop/src/features/profile/ui/AnimatedAvatarControls.tsx b/desktop/src/features/profile/ui/AnimatedAvatarControls.tsx new file mode 100644 index 000000000..d7d41bdae --- /dev/null +++ b/desktop/src/features/profile/ui/AnimatedAvatarControls.tsx @@ -0,0 +1,470 @@ +import { Circle, CircleDashed } from "lucide-react"; +import * as React from "react"; + +import { clampFrameIndex } from "@/features/profile/ui/AnimatedAvatarCapture.helpers"; +import { cn } from "@/shared/lib/cn"; +import { performDefaultHaptic } from "@/shared/lib/haptics"; +import { Spinner } from "@/shared/ui/spinner"; + +const FILMSTRIP_SELECTOR_SIZE = 48; +const SLIDER_TICK_STEP = 10; + +function percentFromSliderValue( + value: number, + min: number, + max: number, +): number { + if (max === min) { + return 0; + } + return ((value - min) / (max - min)) * 100; +} + +function buildAnchoredSliderTicks( + min: number, + max: number, + resetValue: number, +): number[] { + const ticks = new Set(); + for (let tick = resetValue; tick >= min; tick -= SLIDER_TICK_STEP) { + ticks.add(Math.round(tick)); + } + for ( + let tick = resetValue + SLIDER_TICK_STEP; + tick <= max; + tick += SLIDER_TICK_STEP + ) { + ticks.add(Math.round(tick)); + } + return [...ticks].sort((first, second) => first - second); +} + +function findCrossedSliderTick( + previousValue: number, + nextValue: number, + ticks: number[], +): number | null { + if (previousValue === nextValue) { + return null; + } + + if (nextValue > previousValue) { + return ( + ticks.find((tick) => tick > previousValue && tick <= nextValue) ?? null + ); + } + + return ( + [...ticks] + .reverse() + .find((tick) => tick < previousValue && tick >= nextValue) ?? null + ); +} + +type AvatarFramingSliderProps = { + disabled?: boolean; + helpText?: string | null; + helpTestId?: string; + max: number; + min: number; + onChange: (value: number) => void; + onReset: () => void; + resetValue: number; + resetTestId: string; + testId: string; + tipText?: string | null; + value: number; +}; + +export function AvatarFramingSlider({ + disabled = false, + helpText = null, + helpTestId, + max, + min, + onChange, + onReset, + resetValue, + resetTestId, + testId, + tipText = null, + value, +}: AvatarFramingSliderProps) { + const sliderRef = React.useRef(null); + const activePointerRef = React.useRef(null); + const valueRef = React.useRef(value); + const lastHapticTickRef = React.useRef(null); + const [isHovered, setIsHovered] = React.useState(false); + const [isFocused, setIsFocused] = React.useState(false); + const [isInteracting, setIsInteracting] = React.useState(false); + const fill = percentFromSliderValue(value, min, max); + const isActive = isHovered || isFocused || isInteracting; + const tipId = React.useId(); + const ticks = React.useMemo( + () => buildAnchoredSliderTicks(min, max, resetValue), + [max, min, resetValue], + ); + + React.useEffect(() => { + valueRef.current = value; + }, [value]); + + const commitValue = React.useCallback( + (nextValue: number) => { + const clampedValue = Math.min(max, Math.max(min, Math.round(nextValue))); + const crossedTick = findCrossedSliderTick( + valueRef.current, + clampedValue, + ticks, + ); + valueRef.current = clampedValue; + onChange(clampedValue); + if (crossedTick !== null && crossedTick !== lastHapticTickRef.current) { + lastHapticTickRef.current = crossedTick; + performDefaultHaptic(); + } else if (crossedTick === null) { + lastHapticTickRef.current = null; + } + }, + [max, min, onChange, ticks], + ); + + const commitPointerValue = React.useCallback( + (clientX: number) => { + const slider = sliderRef.current; + if (!slider) { + return; + } + const rect = slider.getBoundingClientRect(); + const progress = Math.min( + 1, + Math.max(0, (clientX - rect.left) / Math.max(rect.width, 1)), + ); + commitValue(min + progress * (max - min)); + }, + [commitValue, max, min], + ); + + const nudgeValue = React.useCallback( + (delta: number) => { + commitValue(value + delta); + }, + [commitValue, value], + ); + + const resetTickStyle = { + left: `${percentFromSliderValue(resetValue, min, max)}%`, + }; + const sliderControl = ( +
+
{ + if (disabled) { + return; + } + const step = event.shiftKey ? 10 : 1; + if (event.key === "ArrowLeft" || event.key === "ArrowDown") { + event.preventDefault(); + nudgeValue(-step); + } else if (event.key === "ArrowRight" || event.key === "ArrowUp") { + event.preventDefault(); + nudgeValue(step); + } else if (event.key === "Home") { + event.preventDefault(); + commitValue(min); + } else if (event.key === "End") { + event.preventDefault(); + commitValue(max); + } + }} + onBlur={() => setIsFocused(false)} + onFocus={() => setIsFocused(true)} + onMouseEnter={() => setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} + onPointerCancel={(event) => { + if (activePointerRef.current === event.pointerId) { + activePointerRef.current = null; + setIsInteracting(false); + } + }} + onPointerDown={(event) => { + if (disabled) { + return; + } + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + activePointerRef.current = event.pointerId; + setIsInteracting(true); + commitPointerValue(event.clientX); + }} + onPointerMove={(event) => { + if (activePointerRef.current !== event.pointerId) { + return; + } + commitPointerValue(event.clientX); + }} + onPointerUp={(event) => { + if (activePointerRef.current === event.pointerId) { + activePointerRef.current = null; + setIsInteracting(false); + } + }} + ref={sliderRef} + role="slider" + style={ + { + "--buzz-avatar-framing-slider-fill": `${fill}%`, + } as React.CSSProperties + } + tabIndex={disabled ? -1 : 0} + > +
+ {ticks.map((tick) => ( +
+ + ); + + return helpText ? ( +
+ {sliderControl} +

+ {helpText} +

+
+ ) : ( + sliderControl + ); +} + +type AvatarOutlineToggleProps = { + disabled?: boolean; + enabled: boolean; + onChange: (enabled: boolean) => void; + testIdPrefix: string; +}; + +export function AvatarOutlineToggle({ + disabled = false, + enabled, + onChange, + testIdPrefix, +}: AvatarOutlineToggleProps) { + const Icon = enabled ? Circle : CircleDashed; + return ( + + ); +} + +type AvatarFilmstripPickerProps = { + disabled?: boolean; + frameCount: number; + frames: string[]; + helpText?: string; + helpTestId?: string; + onSelectFrame: (index: number) => void; + selectedFrame: number; + testIdPrefix: string; +}; + +export function AvatarFilmstripPicker({ + disabled = false, + frameCount, + frames, + helpText, + helpTestId, + onSelectFrame, + selectedFrame, + testIdPrefix, +}: AvatarFilmstripPickerProps) { + const stripRef = React.useRef(null); + const maxFrameIndex = Math.max(0, frameCount - 1); + const safeSelectedFrame = clampFrameIndex(selectedFrame, frameCount); + const selectedFrameProgress = + maxFrameIndex === 0 ? 0 : safeSelectedFrame / maxFrameIndex; + + const selectFromClientX = React.useCallback( + (clientX: number) => { + if (disabled) { + return; + } + + const strip = stripRef.current; + if (!strip) { + return; + } + + const rect = strip.getBoundingClientRect(); + const nextProgress = Math.min( + 1, + Math.max(0, (clientX - rect.left) / Math.max(rect.width, 1)), + ); + onSelectFrame(Math.round(nextProgress * maxFrameIndex)); + }, + [disabled, maxFrameIndex, onSelectFrame], + ); + + const nudge = React.useCallback( + (delta: number) => { + if (disabled) { + return; + } + onSelectFrame(clampFrameIndex(safeSelectedFrame + delta, frameCount)); + }, + [disabled, frameCount, onSelectFrame, safeSelectedFrame], + ); + + return ( +
+
{ + if (event.key === "ArrowLeft") { + event.preventDefault(); + nudge(event.shiftKey ? -3 : -1); + } else if (event.key === "ArrowRight") { + event.preventDefault(); + nudge(event.shiftKey ? 3 : 1); + } else if (event.key === "Home") { + event.preventDefault(); + onSelectFrame(0); + } else if (event.key === "End") { + event.preventDefault(); + onSelectFrame(maxFrameIndex); + } + }} + onPointerDown={(event) => { + if (disabled) { + return; + } + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + selectFromClientX(event.clientX); + }} + onPointerMove={(event) => { + if (event.buttons !== 1) { + return; + } + selectFromClientX(event.clientX); + }} + ref={stripRef} + role="slider" + tabIndex={disabled ? -1 : 0} + > +
+ {frames.length === 0 ? ( +
+ +
+ ) : ( + + )} +
+ {frames.length > 0 ? ( + + {helpText ? ( +

+ {helpText} +

+ ) : null} +
+ ); +} diff --git a/desktop/src/features/profile/ui/AnimatedAvatarReviewNav.tsx b/desktop/src/features/profile/ui/AnimatedAvatarReviewNav.tsx new file mode 100644 index 000000000..39d5642d2 --- /dev/null +++ b/desktop/src/features/profile/ui/AnimatedAvatarReviewNav.tsx @@ -0,0 +1,130 @@ +import { + Camera, + Circle, + GalleryThumbnails, + Palette, + UserRound, +} from "lucide-react"; + +import { cn } from "@/shared/lib/cn"; + +export type ReviewSection = "person" | "shape" | "color" | "poster"; + +const REVIEW_SECTIONS: { + key: ReviewSection; + label: string; + caption: string; + hidden?: boolean; + icon: typeof UserRound; +}[] = [ + { + caption: "You", + icon: UserRound, + key: "person", + label: "Position yourself", + }, + { + caption: "Circle", + hidden: true, + icon: Circle, + key: "shape", + label: "Adjust the circle", + }, + { caption: "Background", icon: Palette, key: "color", label: "Background" }, + { + caption: "Frame", + icon: GalleryThumbnails, + key: "poster", + label: "Still frame", + }, +]; + +type AnimatedAvatarReviewNavProps = { + activeSection: ReviewSection; + disabled?: boolean; + isSaving: boolean; + onRetake: () => void; + onSectionChange: (section: ReviewSection) => void; + testIdPrefix: string; +}; + +export function AnimatedAvatarReviewNav({ + activeSection, + disabled = false, + isSaving, + onRetake, + onSectionChange, + testIdPrefix, +}: AnimatedAvatarReviewNavProps) { + const controlsDisabled = disabled || isSaving; + + return ( +
+ {REVIEW_SECTIONS.filter((section) => !section.hidden).map((section) => { + const Icon = section.icon; + return ( + + ); + })} +
+ ); +} diff --git a/desktop/src/features/profile/ui/AvatarCustomColorPanel.tsx b/desktop/src/features/profile/ui/AvatarCustomColorPanel.tsx new file mode 100644 index 000000000..534ecb648 --- /dev/null +++ b/desktop/src/features/profile/ui/AvatarCustomColorPanel.tsx @@ -0,0 +1,289 @@ +import { motion } from "motion/react"; +import * as React from "react"; + +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; +import { + CUSTOM_COLOR_GRID_COLUMNS, + CUSTOM_COLOR_GRID_HORIZONTAL_INSET, + CUSTOM_COLOR_GRID_ROWS, + CUSTOM_COLOR_GRID_VERTICAL_INSET, + CUSTOM_HUE_SCRUBBER_INSET, + clampPercent, + gridInsetPosition, + hueScrubberPosition, + normalizeHue, + snapToGrid, +} from "./ProfileAvatarEditor.utils"; + +const PANEL_MOTION_TRANSITION = { + duration: 0.25, + ease: "easeOut", +} as const; + +type AvatarCustomColorPanelProps = { + visible: boolean; + hue: number; + /** 0-100 */ + saturation: number; + /** 0-100 */ + value: number; + /** Hex color derived from hue/saturation/value. */ + colorDraft: string; + onHueChange: (hue: number) => void; + onSaturationValueChange: (saturation: number, value: number) => void; + onCommit: () => void; + testIdPrefix: string; + className?: string; +}; + +/** + * The HSV custom color picker overlay shared by the emoji and animated + * avatar tabs: a saturation/value spectrum grid, a hue scrubber, and a + * commit button. The parent owns the HSV state and positions the panel + * (it fills its nearest relative ancestor). + */ +export function AvatarCustomColorPanel({ + visible, + hue, + saturation, + value, + colorDraft, + onHueChange, + onSaturationValueChange, + onCommit, + testIdPrefix, + className, +}: AvatarCustomColorPanelProps) { + const hueDragUserSelectRef = React.useRef(null); + + const unlockHueDragSelection = React.useCallback(() => { + if (hueDragUserSelectRef.current === null) { + return; + } + + document.body.style.userSelect = hueDragUserSelectRef.current; + hueDragUserSelectRef.current = null; + }, []); + + const lockHueDragSelection = React.useCallback(() => { + if (hueDragUserSelectRef.current !== null) { + return; + } + + hueDragUserSelectRef.current = document.body.style.userSelect; + document.body.style.userSelect = "none"; + }, []); + + const updateColorFromPointer = React.useCallback( + (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + const width = Math.max( + rect.width - CUSTOM_COLOR_GRID_HORIZONTAL_INSET * 2, + 1, + ); + const height = Math.max( + rect.height - CUSTOM_COLOR_GRID_VERTICAL_INSET * 2, + 1, + ); + const rawSaturation = clampPercent( + ((event.clientX - rect.left - CUSTOM_COLOR_GRID_HORIZONTAL_INSET) / + width) * + 100, + ); + const rawValue = clampPercent( + (1 - + (event.clientY - rect.top - CUSTOM_COLOR_GRID_VERTICAL_INSET) / + height) * + 100, + ); + const nextSaturation = Math.round( + snapToGrid(rawSaturation, CUSTOM_COLOR_GRID_COLUMNS), + ); + const nextValue = Math.round( + snapToGrid(rawValue, CUSTOM_COLOR_GRID_ROWS), + ); + + onSaturationValueChange(nextSaturation, nextValue); + }, + [onSaturationValueChange], + ); + + const updateHueFromPointer = React.useCallback( + (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + const trackWidth = Math.max( + rect.width - CUSTOM_HUE_SCRUBBER_INSET * 2, + 1, + ); + const nextPercent = clampPercent( + ((event.clientX - rect.left - CUSTOM_HUE_SCRUBBER_INSET) / trackWidth) * + 100, + ); + onHueChange(Math.round((nextPercent / 100) * 360)); + }, + [onHueChange], + ); + + const adjustHue = React.useCallback( + (delta: number) => { + onHueChange(normalizeHue(hue + delta)); + }, + [hue, onHueChange], + ); + + return ( + +
{ + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + updateColorFromPointer(event); + }} + onPointerMove={(event) => { + if (event.buttons === 1) { + event.preventDefault(); + updateColorFromPointer(event); + } + }} + onPointerUp={(event) => { + event.preventDefault(); + updateColorFromPointer(event); + }} + style={{ + backgroundColor: `hsl(${hue}, 100%, 50%)`, + backgroundImage: + "linear-gradient(to bottom, transparent 0%, #000000 100%), linear-gradient(to right, #ffffff 0%, rgba(255,255,255,0) 100%)", + }} + > + +
+
+ +
{ + if (event.key === "ArrowLeft" || event.key === "ArrowDown") { + event.preventDefault(); + adjustHue(-6); + } else if (event.key === "ArrowRight" || event.key === "ArrowUp") { + event.preventDefault(); + adjustHue(6); + } else if (event.key === "Home") { + event.preventDefault(); + onHueChange(0); + } else if (event.key === "End") { + event.preventDefault(); + onHueChange(360); + } + }} + onPointerDown={(event) => { + event.preventDefault(); + lockHueDragSelection(); + event.currentTarget.setPointerCapture(event.pointerId); + updateHueFromPointer(event); + }} + onPointerMove={(event) => { + if (event.buttons === 1) { + event.preventDefault(); + updateHueFromPointer(event); + } + }} + onPointerCancel={unlockHueDragSelection} + onPointerUp={unlockHueDragSelection} + onLostPointerCapture={unlockHueDragSelection} + role="slider" + tabIndex={visible ? 0 : -1} + > + + + + + ); +} diff --git a/desktop/src/features/profile/ui/ProfileAvatar.tsx b/desktop/src/features/profile/ui/ProfileAvatar.tsx index a26690625..c5d6cb66a 100644 --- a/desktop/src/features/profile/ui/ProfileAvatar.tsx +++ b/desktop/src/features/profile/ui/ProfileAvatar.tsx @@ -1,10 +1,12 @@ import * as React from "react"; import { UserRound } from "lucide-react"; +import { parseAnimatedAvatarUrl } from "@/shared/lib/animatedAvatar"; import { cn } from "@/shared/lib/cn"; import { getInitials } from "@/shared/lib/initials"; import { rewriteRelayUrl } from "@/shared/lib/mediaUrl"; import { Avatar, AvatarFallback, AvatarImage } from "@/shared/ui/avatar"; +import { Spinner } from "@/shared/ui/spinner"; type ProfileAvatarProps = { avatarUrl: string | null; @@ -13,6 +15,7 @@ type ProfileAvatarProps = { className?: string; iconClassName?: string; plain?: boolean; + showAnimationLoader?: boolean; testId?: string; }; @@ -23,57 +26,125 @@ export function ProfileAvatar({ className, iconClassName, plain = false, + showAnimationLoader = false, testId, }: ProfileAvatarProps) { const initials = getInitials(label); - // Compute the live (proxied) source and reset failure state when the URL changes. - const liveSrc = avatarUrl ? rewriteRelayUrl(avatarUrl) : null; - const [liveFailed, setLiveFailed] = React.useState(false); - // biome-ignore lint/correctness/useExhaustiveDependencies: avatarUrl is the trigger — reset liveFailed when the URL changes even though the effect body doesn't reference it directly. - React.useEffect(() => { - setLiveFailed(false); - }, [avatarUrl]); + // Animated avatars show their static poster frame until hovered, then play + // the animation. + const animated = parseAnimatedAvatarUrl(avatarUrl); + const [isHovered, setIsHovered] = React.useState(false); + const [loadedAnimationSrc, setLoadedAnimationSrc] = React.useState< + string | null + >(null); + const baseUrl = animated + ? isHovered + ? animated.animationUrl + : animated.posterUrl + : avatarUrl; + + // Compute the live (proxied) source. Failures are tracked per resolved URL so + // the poster and hover animation can recover independently. + const liveSrc = baseUrl ? rewriteRelayUrl(baseUrl) : null; + const animationSrc = animated ? rewriteRelayUrl(animated.animationUrl) : null; + const posterSrc = animated ? rewriteRelayUrl(animated.posterUrl) : null; + const [failedSrc, setFailedSrc] = React.useState(null); + const liveFailed = liveSrc !== null && failedSrc === liveSrc; + const posterFailed = posterSrc !== null && failedSrc === posterSrc; // When the relay is unreachable the proxied avatar URL 404s/times out; fall // back to the locally cached data URL instead of dropping to initials. const src = liveFailed ? (avatarDataUrl ?? undefined) : (liveSrc ?? avatarDataUrl ?? undefined); + const shouldShowFallback = src === undefined || (!animated && liveFailed); + const shouldUseAnimationLoader = showAnimationLoader && animated !== null; + const shouldShowAnimationLoader = + shouldUseAnimationLoader && + isHovered && + animationSrc !== null && + liveSrc === animationSrc && + failedSrc !== animationSrc && + loadedAnimationSrc !== animationSrc; + const posterUnderlaySrc = shouldShowAnimationLoader + ? posterFailed + ? (avatarDataUrl ?? undefined) + : (posterSrc ?? avatarDataUrl ?? undefined) + : undefined; return ( setIsHovered(true) : undefined} + onMouseLeave={animated ? () => setIsHovered(false) : undefined} > + {posterUnderlaySrc ? ( + + ) : null} {src !== undefined ? ( { - if (status === "error") setLiveFailed(true); + if (status === "error") setFailedSrc(liveSrc); + if (status === "loaded" && src === liveSrc) { + setFailedSrc(null); + if (liveSrc === animationSrc) { + setLoadedAnimationSrc(liveSrc); + } + } }} referrerPolicy="no-referrer" src={src} /> ) : null} - - {initials.length > 0 ? ( - initials - ) : ( - - )} - + {shouldShowAnimationLoader ? ( + + + + ) : null} + {shouldShowFallback ? ( + + {initials.length > 0 ? ( + initials + ) : ( + + )} + + ) : null} ); } diff --git a/desktop/src/features/profile/ui/ProfileAvatarEditor.tsx b/desktop/src/features/profile/ui/ProfileAvatarEditor.tsx index 5ae567faf..53ba36bc1 100644 --- a/desktop/src/features/profile/ui/ProfileAvatarEditor.tsx +++ b/desktop/src/features/profile/ui/ProfileAvatarEditor.tsx @@ -1,9 +1,12 @@ import emojiData from "@emoji-mart/data"; import Picker from "@emoji-mart/react"; import { Link2, Loader2, UploadCloud } from "lucide-react"; -import { motion } from "motion/react"; +import { AnimatePresence, motion, useReducedMotion } from "motion/react"; import * as React from "react"; +import { createPortal, flushSync } from "react-dom"; +import { AnimatedAvatarCapture } from "@/features/profile/ui/AnimatedAvatarCapture"; +import { AvatarCustomColorPanel } from "@/features/profile/ui/AvatarCustomColorPanel"; import { useAvatarUpload } from "@/features/profile/useAvatarUpload"; import { cn } from "@/shared/lib/cn"; import { Button } from "@/shared/ui/button"; @@ -14,36 +17,54 @@ import { AVATAR_COLORS, AVATAR_COLOR_SWATCHES, CUSTOM_AVATAR_COLOR_SWATCH, - CUSTOM_COLOR_GRID_COLUMNS, - CUSTOM_COLOR_GRID_HORIZONTAL_INSET, - CUSTOM_COLOR_GRID_ROWS, - CUSTOM_COLOR_GRID_VERTICAL_INSET, - CUSTOM_HUE_SCRUBBER_INSET, DEFAULT_CUSTOM_HUE, DEFAULT_CUSTOM_SATURATION, DEFAULT_CUSTOM_VALUE, DEFAULT_EMOJI_AVATAR_COLOR, EMOJI_MART_CATEGORIES, type AvatarColorSwatch, - clampPercent, contrastColorForBackground, dataTransferHasImage, emojiAvatarDataUrl, - gridInsetPosition, hexToHsv, hsvToHex, - hueScrubberPosition, normalizeHue, parseEmojiAvatarDataUrl, - snapToGrid, useEmojiMartStyles, useEmojiMartThemeVars, - visibleUrlDraft, } from "./ProfileAvatarEditor.utils"; export { parseEmojiAvatarDataUrl } from "./ProfileAvatarEditor.utils"; -export type AvatarMode = "image" | "emoji"; +export type AvatarMode = "image" | "emoji" | "animated"; + +const MODE_TAB_ORDER: AvatarMode[] = ["image", "emoji", "animated"]; +const DONE_BUTTON_CONTENT_TRANSITION = { + duration: 0.14, + ease: [0.23, 1, 0.32, 1], +} as const; +const DONE_BUTTON_SHELL_TRANSITION = { + duration: 0.18, + ease: [0.23, 1, 0.32, 1], +} as const; + +function waitForPendingButtonPaint() { + return new Promise((resolve) => { + if ( + typeof window === "undefined" || + typeof window.requestAnimationFrame !== "function" + ) { + setTimeout(resolve, 0); + return; + } + + window.requestAnimationFrame(() => { + window.requestAnimationFrame(() => { + setTimeout(resolve, 0); + }); + }); + }); +} type ProfileAvatarEditorProps = { avatarUrl: string; @@ -58,20 +79,23 @@ type ProfileAvatarEditorProps = { onUploadingChange?: (isUploading: boolean) => void; onDone?: () => void; donePending?: boolean; - hiddenAvatarUrl?: string | null; showEmojiColorControlsWhenEmpty?: boolean; disabled?: boolean; testIdPrefix?: string; + /** Host element the animated tab renders its live preview into. */ + animatedPreviewContainer?: HTMLElement | null; + /** Optional host element for the mode tabs; undefined keeps them inline. */ + modeTabsContainer?: HTMLElement | null; + /** Fires when the animated tab starts/stops occupying the host preview. */ + onAnimatedPreviewActiveChange?: (active: boolean) => void; + /** Caption shown under the host preview while animated capture is active. */ + onAnimatedPreviewCaptionChange?: (caption: string | null) => void; }; type EmojiMartEmoji = { native?: string; }; -const AVATAR_EDITOR_MOTION_TRANSITION = { - duration: 0.25, - ease: "easeOut", -} as const; const INITIAL_EMOJI_AVATAR_COLORS = AVATAR_COLORS.filter( (color) => color !== DEFAULT_EMOJI_AVATAR_COLOR, ); @@ -92,7 +116,6 @@ export function ProfileAvatarEditor({ donePending = false, emojiPickerTheme = "dark", emojiPickerThemeVars, - hiddenAvatarUrl, onCustomColorPickerOpenChange, onEmojiAvatarChange, onModeChange, @@ -103,17 +126,20 @@ export function ProfileAvatarEditor({ showEmojiColorControlsWhenEmpty = false, disabled, testIdPrefix = "profile-avatar", + animatedPreviewContainer = null, + modeTabsContainer, + onAnimatedPreviewActiveChange, + onAnimatedPreviewCaptionChange, }: ProfileAvatarEditorProps) { const { burstEmoji } = useEmojiBurst(); + const shouldReduceMotion = useReducedMotion(); const initialEmojiAvatar = React.useMemo( () => parseEmojiAvatarDataUrl(avatarUrl), [avatarUrl], ); const [mode, setMode] = React.useState("image"); const [isDragging, setIsDragging] = React.useState(false); - const [urlDraft, setUrlDraft] = React.useState(() => - visibleUrlDraft(avatarUrl, hiddenAvatarUrl), - ); + const [urlDraft, setUrlDraft] = React.useState(""); const [selectedEmoji, setSelectedEmoji] = React.useState( () => initialEmojiAvatar?.emoji ?? null, ); @@ -127,9 +153,10 @@ export function ProfileAvatarEditor({ const [customValue, setCustomValue] = React.useState(DEFAULT_CUSTOM_VALUE); const [isCustomColorPickerOpen, setIsCustomColorPickerOpen] = React.useState(false); + const [isAnimatedCustomColorPickerOpen, setIsAnimatedCustomColorPickerOpen] = + React.useState(false); const dragDepthRef = React.useRef(0); const emojiPickerContainerRef = React.useRef(null); - const hueDragUserSelectRef = React.useRef(null); const modeContentRef = React.useRef(null); const isUrlInputFocusedRef = React.useRef(false); const hasUserEditedUrlDraftRef = React.useRef(false); @@ -147,6 +174,8 @@ export function ProfileAvatarEditor({ (selectedEmoji !== null || showEmojiColorControlsWhenEmpty); const isCustomColorPickerVisible = isCustomColorPickerOpen && shouldShowColorControls; + const isAnyCustomColorPickerVisible = + isCustomColorPickerVisible || isAnimatedCustomColorPickerOpen; const updateMode = React.useCallback( (nextMode: AvatarMode) => { if (mode === nextMode) { @@ -177,6 +206,66 @@ export function ProfileAvatarEditor({ uploadFile, } = useAvatarUpload({ onUploadSuccess: handleUploadSuccess }); const isInputDisabled = disabled || isUploading; + const handleAnimatedApply = React.useCallback( + (animatedUrl: string) => { + clearUploadError(); + setUrlDraft(""); + onUploadedAvatarChange?.(animatedUrl); + onUrlChange(animatedUrl); + }, + [clearUploadError, onUploadedAvatarChange, onUrlChange], + ); + // Done on the animated tab uploads the pending recording first, then + // saves. The save is queued through state so it runs on the next render, + // after the freshly applied avatar URL has propagated into the host's + // drafts (calling onDone directly would read stale state). + const animatedApplyRef = React.useRef<(() => Promise) | null>(null); + const [hasAnimatedApply, setHasAnimatedApply] = React.useState(false); + const registerAnimatedApply = React.useCallback( + (apply: (() => Promise) | null) => { + animatedApplyRef.current = apply; + setHasAnimatedApply(apply !== null); + }, + [], + ); + const [isAnimatedApplyPending, setIsAnimatedApplyPending] = + React.useState(false); + const [isAnimatedDoneQueued, setIsAnimatedDoneQueued] = React.useState(false); + const isDoneButtonPending = + donePending || + isUploading || + isAnimatedApplyPending || + isAnimatedDoneQueued; + const handleDoneClick = React.useCallback(() => { + const applyAnimated = mode === "animated" ? animatedApplyRef.current : null; + if (applyAnimated) { + flushSync(() => { + setIsAnimatedApplyPending(true); + }); + void waitForPendingButtonPaint() + .then(() => applyAnimated()) + .then((applied) => { + if (applied) { + setIsAnimatedDoneQueued(true); + return; + } + }) + .catch(() => {}) + .finally(() => { + setIsAnimatedApplyPending(false); + }); + return; + } + onDone?.(); + }, [mode, onDone]); + + React.useEffect(() => { + if (!isAnimatedDoneQueued) { + return; + } + setIsAnimatedDoneQueued(false); + onDone?.(); + }, [isAnimatedDoneQueued, onDone]); useEmojiMartStyles(emojiPickerContainerRef, mode === "emoji"); @@ -202,13 +291,6 @@ export function ProfileAvatarEditor({ onUploadingChange?.(isUploading); }, [isUploading, onUploadingChange]); - React.useLayoutEffect(() => { - if (isUrlInputFocusedRef.current || hasUserEditedUrlDraftRef.current) { - return; - } - setUrlDraft(visibleUrlDraft(avatarUrl, hiddenAvatarUrl)); - }, [avatarUrl, hiddenAvatarUrl]); - React.useEffect(() => { const emojiAvatar = parseEmojiAvatarDataUrl(avatarUrl); if (emojiAvatar) { @@ -229,12 +311,12 @@ export function ProfileAvatarEditor({ }, [shouldShowColorControls]); React.useLayoutEffect(() => { - onCustomColorPickerOpenChange?.(isCustomColorPickerVisible); + onCustomColorPickerOpenChange?.(isAnyCustomColorPickerVisible); return () => { onCustomColorPickerOpenChange?.(false); }; - }, [isCustomColorPickerVisible, onCustomColorPickerOpenChange]); + }, [isAnyCustomColorPickerVisible, onCustomColorPickerOpenChange]); React.useEffect(() => { if (!isCustomColorPickerOpen || !selectedEmoji) { @@ -257,24 +339,6 @@ export function ProfileAvatarEditor({ selectedEmoji, ]); - const unlockHueDragSelection = React.useCallback(() => { - if (hueDragUserSelectRef.current === null) { - return; - } - - document.body.style.userSelect = hueDragUserSelectRef.current; - hueDragUserSelectRef.current = null; - }, []); - - const lockHueDragSelection = React.useCallback(() => { - if (hueDragUserSelectRef.current !== null) { - return; - } - - hueDragUserSelectRef.current = document.body.style.userSelect; - document.body.style.userSelect = "none"; - }, []); - const handleFiles = React.useCallback( (files: FileList | null) => { const file = files?.[0]; @@ -311,6 +375,8 @@ export function ProfileAvatarEditor({ const applyEmojiAvatar = React.useCallback( (emoji: string, color = selectedColor) => { + setUrlDraft(""); + hasUserEditedUrlDraftRef.current = false; onUploadedAvatarChange?.(null); onUrlChange(emojiAvatarDataUrl(emoji, color)); onEmojiAvatarChange?.(); @@ -326,61 +392,6 @@ export function ProfileAvatarEditor({ setIsCustomColorPickerOpen(true); }, [selectedColor]); - const updateCustomColorFromPointer = React.useCallback( - (event: React.PointerEvent) => { - const rect = event.currentTarget.getBoundingClientRect(); - const width = Math.max( - rect.width - CUSTOM_COLOR_GRID_HORIZONTAL_INSET * 2, - 1, - ); - const height = Math.max( - rect.height - CUSTOM_COLOR_GRID_VERTICAL_INSET * 2, - 1, - ); - const rawSaturation = clampPercent( - ((event.clientX - rect.left - CUSTOM_COLOR_GRID_HORIZONTAL_INSET) / - width) * - 100, - ); - const rawValue = clampPercent( - (1 - - (event.clientY - rect.top - CUSTOM_COLOR_GRID_VERTICAL_INSET) / - height) * - 100, - ); - const nextSaturation = Math.round( - snapToGrid(rawSaturation, CUSTOM_COLOR_GRID_COLUMNS), - ); - const nextValue = Math.round( - snapToGrid(rawValue, CUSTOM_COLOR_GRID_ROWS), - ); - - setCustomSaturation(nextSaturation); - setCustomValue(nextValue); - }, - [], - ); - - const updateCustomHueFromPointer = React.useCallback( - (event: React.PointerEvent) => { - const rect = event.currentTarget.getBoundingClientRect(); - const trackWidth = Math.max( - rect.width - CUSTOM_HUE_SCRUBBER_INSET * 2, - 1, - ); - const nextPercent = clampPercent( - ((event.clientX - rect.left - CUSTOM_HUE_SCRUBBER_INSET) / trackWidth) * - 100, - ); - setCustomHue(Math.round((nextPercent / 100) * 360)); - }, - [], - ); - - const adjustCustomHue = React.useCallback((delta: number) => { - setCustomHue((current) => normalizeHue(current + delta)); - }, []); - const commitCustomColor = React.useCallback(() => { setSelectedColor(customColorDraft); if (selectedEmoji) { @@ -449,6 +460,63 @@ export function ProfileAvatarEditor({ }, [isDragging, resetDragState]); const isImageDropActive = mode === "image" && isDragging; + const shouldShowDoneButton = + onDone && + !isAnyCustomColorPickerVisible && + (mode !== "animated" || hasAnimatedApply || isDoneButtonPending); + const modeTabs = ( + { + if (isInputDisabled) { + return; + } + updateMode(nextMode as AvatarMode); + }} + value={mode} + > + +