From f7e2242e5f450002e016e8d3ff8c6f759cfea0d1 Mon Sep 17 00:00:00 2001 From: JakeBreath Date: Thu, 17 Sep 2026 20:10:31 -0500 Subject: [PATCH] Fix the client optimizer: PNG init, GIF compositing, streaming video - PNG failed because wasm-bindgen's init only accepts a URL string (or a module/buffer): passing { module_or_path } broke every PNG optimization. - GIF/APNG showed flashing colors because patches were written with putImageData, which ignores the transparency that means "keep the previous frame". Patches now blend through drawImage with correct disposal handling, and quantization keeps a transparent palette entry. - Video no longer downloads the whole file into memory: Mediabunny reads it with range requests (UrlSource) and the muxer streams into Blob chunks (StreamTarget) that are assembled with last-write-wins range merging. BufferTarget held the entire output in memory, which crashed the tab on large files. --- frontend/src/features/optimize/animation.ts | 70 ++++++++-- frontend/src/features/optimize/image.ts | 5 +- .../src/features/optimize/optimize.worker.ts | 26 ++-- frontend/src/features/optimize/video.ts | 125 +++++++++++++++--- 4 files changed, 185 insertions(+), 41 deletions(-) diff --git a/frontend/src/features/optimize/animation.ts b/frontend/src/features/optimize/animation.ts index 8f78811..69f9db5 100644 --- a/frontend/src/features/optimize/animation.ts +++ b/frontend/src/features/optimize/animation.ts @@ -79,24 +79,64 @@ async function decodeGif( const height = parsed.lsd.height; const canvas = new OffscreenCanvas(width, height); const context = context2d(canvas); + const patchCanvas = new OffscreenCanvas(1, 1); const frames: AnimationFrame[] = []; - let restoreNeeded = false; + let previous: { + disposalType: number; + left: number; + top: number; + width: number; + height: number; + } | null = null; + let snapshot: ImageData | null = null; + for (const frame of decoded) { - if (restoreNeeded) { - context.clearRect(0, 0, width, height); - restoreNeeded = false; + // Apply the previous frame's disposal before drawing this one. + if (previous) { + if (previous.disposalType === 2) { + context.clearRect( + previous.left, + previous.top, + previous.width, + previous.height, + ); + } else if (previous.disposalType === 3 && snapshot) { + context.putImageData(snapshot, 0, 0); + } + snapshot = null; } - const patch = new ImageData( - new Uint8ClampedArray(frame.patch), - frame.dims.width, - frame.dims.height, + if (frame.disposalType === 3) { + snapshot = context.getImageData(0, 0, width, height); + } + + // GIF patches use transparency to mean "keep what is underneath", so the + // patch must be drawn (alpha blending), not written with putImageData. + patchCanvas.width = frame.dims.width; + patchCanvas.height = frame.dims.height; + const patchContext = context2d(patchCanvas); + patchContext.clearRect(0, 0, frame.dims.width, frame.dims.height); + patchContext.putImageData( + new ImageData( + new Uint8ClampedArray(frame.patch), + frame.dims.width, + frame.dims.height, + ), + 0, + 0, ); - context.putImageData(patch, frame.dims.left, frame.dims.top); + context.drawImage(patchCanvas, frame.dims.left, frame.dims.top); + frames.push({ imageData: context.getImageData(0, 0, width, height), delayMs: frame.delay || 100, }); - if (frame.disposalType === 2) restoreNeeded = true; + previous = { + disposalType: frame.disposalType, + left: frame.dims.left, + top: frame.dims.top, + width: frame.dims.width, + height: frame.dims.height, + }; } return { frames, width, height }; } @@ -130,14 +170,20 @@ function encodeGif( const encoder = GIFEncoder(); frames.forEach((frame, index) => { const palette = quantize(frame.imageData.data, colors, { - format: "rgb444", + format: "rgba4444", + oneBitAlpha: true, }); - const indexed = applyPalette(frame.imageData.data, palette, "rgb444"); + const indexed = applyPalette(frame.imageData.data, palette, "rgba4444"); + const transparentIndex = palette.findIndex( + (entry) => entry.length >= 4 && entry[3] === 0, + ); encoder.writeFrame(indexed, width, height, { palette, delay: Math.max(20, Math.round(frame.delayMs)), repeat: 0, first: index === 0, + transparent: transparentIndex >= 0, + transparentIndex: transparentIndex >= 0 ? transparentIndex : 0, }); if (index % 8 === 0) { report(0.85 + 0.1 * (index / frames.length), "Encoding the GIF…"); diff --git a/frontend/src/features/optimize/image.ts b/frontend/src/features/optimize/image.ts index 395b18f..7a78407 100644 --- a/frontend/src/features/optimize/image.ts +++ b/frontend/src/features/optimize/image.ts @@ -30,9 +30,8 @@ async function ensureOxipng(): Promise { if (!oxipngReady) { const cores = globalThis.navigator?.hardwareConcurrency ?? 1; const useThreads = cores > 1 && (await threads()); - oxipngReady = initOxipng({ - module_or_path: useThreads ? oxipngMtWasmUrl : oxipngStWasmUrl, - }); + // wasm-bindgen's init only understands a URL string (or module/buffer). + oxipngReady = initOxipng(useThreads ? oxipngMtWasmUrl : oxipngStWasmUrl); } await oxipngReady; } diff --git a/frontend/src/features/optimize/optimize.worker.ts b/frontend/src/features/optimize/optimize.worker.ts index 37922ab..2fc117a 100644 --- a/frontend/src/features/optimize/optimize.worker.ts +++ b/frontend/src/features/optimize/optimize.worker.ts @@ -16,19 +16,23 @@ scope.onmessage = async (event: MessageEvent) => { scope.postMessage({ type: "progress", progress, stage }); try { - report(0.01, "Downloading the original…"); - const response = await fetch(url); - if (!response.ok) { - throw new Error(`Could not load the file (HTTP ${response.status}).`); - } - const blob = await response.blob(); - - const result = - kind === "video" - ? await optimizeVideo(blob, filename, options, report) - : kind === "animation" + let result; + if (kind === "video") { + // Mediabunny streams the source with range requests — never download a + // whole video into memory. + result = await optimizeVideo(url, filename, options, report); + } else { + report(0.01, "Downloading the original…"); + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Could not load the file (HTTP ${response.status}).`); + } + const blob = await response.blob(); + result = + kind === "animation" ? await optimizeAnimation(blob, filename, options, report) : await optimizeImage(blob, filename, options, report); + } scope.postMessage({ type: "done", diff --git a/frontend/src/features/optimize/video.ts b/frontend/src/features/optimize/video.ts index 478ad1d..3906eca 100644 --- a/frontend/src/features/optimize/video.ts +++ b/frontend/src/features/optimize/video.ts @@ -1,14 +1,15 @@ import { ALL_FORMATS, - BlobSource, - BufferTarget, Conversion, Input, Mp4OutputFormat, Output, Quality, + StreamTarget, + UrlSource, WebMOutputFormat, getFirstEncodableVideoCodec, + type StreamTargetChunk, type VideoCodec, } from "mediabunny"; @@ -28,9 +29,98 @@ function even(value: number): number { return Math.max(2, value - (value % 2)); } -/** Transcode with WebCodecs (hardware-accelerated where available). */ +interface CollectedChunk { + blob: Blob; + position: number; + sequence: number; +} + +/** + * Stream the muxed output into Blob parts instead of one giant ArrayBuffer — + * a 4K file would otherwise blow the tab's memory. Chunks may arrive out of + * order (the muxer patches headers), so later writes win per byte range when + * the final blob is assembled. + */ +function createStreamOutput(): { + target: StreamTarget; + toBlob: (type: string) => Blob; +} { + const chunks: CollectedChunk[] = []; + let sequence = 0; + const writable = new WritableStream({ + write(chunk) { + chunks.push({ + blob: new Blob([chunk.data.slice()]), + position: chunk.position, + sequence: sequence++, + }); + }, + }); + return { + target: new StreamTarget(writable, { + chunked: true, + chunkSize: 8 * 1024 * 1024, + }), + toBlob: (type: string) => assembleBlob(chunks, type), + }; +} + +function assembleBlob(chunks: CollectedChunk[], type: string): Blob { + if (chunks.length === 0) { + throw new Error("Encoding produced no output."); + } + const sorted = [...chunks].sort( + (a, b) => a.position - b.position || a.sequence - b.sequence, + ); + + // Overwrite ranges so the newest write for each byte range survives. + interface Segment { + start: number; + end: number; + sequence: number; + blob: Blob; + } + let segments: Segment[] = []; + for (const chunk of sorted) { + const start = chunk.position; + const end = start + chunk.blob.size; + const next: Segment[] = []; + for (const segment of segments) { + if (segment.end <= start || segment.start >= end) { + next.push(segment); + continue; + } + if (segment.start < start) { + next.push({ ...segment, end: start }); + } + if (segment.end > end) { + next.push({ + ...segment, + start: end, + blob: segment.blob.slice(end - segment.start), + }); + } + } + next.push({ start, end, sequence: chunk.sequence, blob: chunk.blob }); + segments = next; + } + segments.sort((a, b) => a.start - b.start); + + const parts: Blob[] = []; + let cursor = 0; + for (const segment of segments) { + if (segment.start !== cursor) { + throw new Error("The muxer produced an incomplete file."); + } + parts.push(segment.blob); + cursor = segment.end; + } + return new Blob(parts, { type }); +} + +/** Transcode with WebCodecs, streaming the source and the result. */ export async function optimizeVideo( - blob: Blob, + url: string, filename: string, options: OptimizeOptions, report: ProgressReporter, @@ -38,7 +128,7 @@ export async function optimizeVideo( const extension = extensionOf(filename); report(0.03, "Reading the video…"); const input = new Input({ - source: new BlobSource(blob), + source: new UrlSource(url, { maxCacheSize: 4 * 1024 * 1024 }), formats: ALL_FORMATS, }); const track = await input.getPrimaryVideoTrack(); @@ -56,8 +146,6 @@ export async function optimizeVideo( : options.container; const format = container === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); - const target = new BufferTarget(); - const output = new Output({ format, target }); const quality = new Quality(options.qualityLevel); let codec: VideoCodec | undefined = @@ -78,6 +166,9 @@ export async function optimizeVideo( const outputHeight = options.height ?? sourceHeight; const outputWidth = Math.round(sourceWidth * (outputHeight / sourceHeight)); + const { target, toBlob } = createStreamOutput(); + const output = new Output({ format, target }); + report(0.06, "Preparing the encoder…"); const conversion = await Conversion.init({ input, @@ -101,17 +192,21 @@ export async function optimizeVideo( conversion.onProgress = (progress: number) => { report(0.08 + Math.min(Math.max(progress, 0), 1) * 0.85, "Encoding…"); }; - await conversion.execute(); - - const buffer = target.buffer; - if (!buffer) { - throw new Error("Encoding produced no output."); + try { + await conversion.execute(); + } catch (error) { + throw new Error( + `Video encoding failed: ${ + error instanceof Error ? error.message : "unknown error" + }`, + ); + } finally { + input.dispose?.(); } + report(0.97, "Packaging…"); return { - blob: new Blob([buffer], { - type: container === "webm" ? "video/webm" : "video/mp4", - }), + blob: toBlob(container === "webm" ? "video/webm" : "video/mp4"), width: even(outputWidth), height: even(outputHeight), note: `${codec} · ${container}`,