Fix the client optimizer: PNG init, GIF compositing, streaming video

- PNG failed because wasm-bindgen's init only accepts a URL string (or a
  module/buffer): passing { module_or_path } broke every PNG optimization.
- GIF/APNG showed flashing colors because patches were written with
  putImageData, which ignores the transparency that means "keep the
  previous frame". Patches now blend through drawImage with correct
  disposal handling, and quantization keeps a transparent palette entry.
- Video no longer downloads the whole file into memory: Mediabunny reads
  it with range requests (UrlSource) and the muxer streams into Blob
  chunks (StreamTarget) that are assembled with last-write-wins range
  merging. BufferTarget held the entire output in memory, which crashed
  the tab on large files.
This commit is contained in:
2026-09-17 20:10:31 -05:00
parent bab9904fc8
commit f7e2242e5f
4 changed files with 185 additions and 41 deletions
+55 -9
View File
@@ -79,24 +79,64 @@ async function decodeGif(
const height = parsed.lsd.height; const height = parsed.lsd.height;
const canvas = new OffscreenCanvas(width, height); const canvas = new OffscreenCanvas(width, height);
const context = context2d(canvas); const context = context2d(canvas);
const patchCanvas = new OffscreenCanvas(1, 1);
const frames: AnimationFrame[] = []; const frames: AnimationFrame[] = [];
let restoreNeeded = false; let previous: {
disposalType: number;
left: number;
top: number;
width: number;
height: number;
} | null = null;
let snapshot: ImageData | null = null;
for (const frame of decoded) { for (const frame of decoded) {
if (restoreNeeded) { // Apply the previous frame's disposal before drawing this one.
context.clearRect(0, 0, width, height); if (previous) {
restoreNeeded = false; if (previous.disposalType === 2) {
context.clearRect(
previous.left,
previous.top,
previous.width,
previous.height,
);
} else if (previous.disposalType === 3 && snapshot) {
context.putImageData(snapshot, 0, 0);
} }
const patch = new ImageData( snapshot = null;
}
if (frame.disposalType === 3) {
snapshot = context.getImageData(0, 0, width, height);
}
// GIF patches use transparency to mean "keep what is underneath", so the
// patch must be drawn (alpha blending), not written with putImageData.
patchCanvas.width = frame.dims.width;
patchCanvas.height = frame.dims.height;
const patchContext = context2d(patchCanvas);
patchContext.clearRect(0, 0, frame.dims.width, frame.dims.height);
patchContext.putImageData(
new ImageData(
new Uint8ClampedArray(frame.patch), new Uint8ClampedArray(frame.patch),
frame.dims.width, frame.dims.width,
frame.dims.height, frame.dims.height,
),
0,
0,
); );
context.putImageData(patch, frame.dims.left, frame.dims.top); context.drawImage(patchCanvas, frame.dims.left, frame.dims.top);
frames.push({ frames.push({
imageData: context.getImageData(0, 0, width, height), imageData: context.getImageData(0, 0, width, height),
delayMs: frame.delay || 100, delayMs: frame.delay || 100,
}); });
if (frame.disposalType === 2) restoreNeeded = true; previous = {
disposalType: frame.disposalType,
left: frame.dims.left,
top: frame.dims.top,
width: frame.dims.width,
height: frame.dims.height,
};
} }
return { frames, width, height }; return { frames, width, height };
} }
@@ -130,14 +170,20 @@ function encodeGif(
const encoder = GIFEncoder(); const encoder = GIFEncoder();
frames.forEach((frame, index) => { frames.forEach((frame, index) => {
const palette = quantize(frame.imageData.data, colors, { const palette = quantize(frame.imageData.data, colors, {
format: "rgb444", format: "rgba4444",
oneBitAlpha: true,
}); });
const indexed = applyPalette(frame.imageData.data, palette, "rgb444"); const indexed = applyPalette(frame.imageData.data, palette, "rgba4444");
const transparentIndex = palette.findIndex(
(entry) => entry.length >= 4 && entry[3] === 0,
);
encoder.writeFrame(indexed, width, height, { encoder.writeFrame(indexed, width, height, {
palette, palette,
delay: Math.max(20, Math.round(frame.delayMs)), delay: Math.max(20, Math.round(frame.delayMs)),
repeat: 0, repeat: 0,
first: index === 0, first: index === 0,
transparent: transparentIndex >= 0,
transparentIndex: transparentIndex >= 0 ? transparentIndex : 0,
}); });
if (index % 8 === 0) { if (index % 8 === 0) {
report(0.85 + 0.1 * (index / frames.length), "Encoding the GIF…"); report(0.85 + 0.1 * (index / frames.length), "Encoding the GIF…");
+2 -3
View File
@@ -30,9 +30,8 @@ async function ensureOxipng(): Promise<void> {
if (!oxipngReady) { if (!oxipngReady) {
const cores = globalThis.navigator?.hardwareConcurrency ?? 1; const cores = globalThis.navigator?.hardwareConcurrency ?? 1;
const useThreads = cores > 1 && (await threads()); const useThreads = cores > 1 && (await threads());
oxipngReady = initOxipng({ // wasm-bindgen's init only understands a URL string (or module/buffer).
module_or_path: useThreads ? oxipngMtWasmUrl : oxipngStWasmUrl, oxipngReady = initOxipng(useThreads ? oxipngMtWasmUrl : oxipngStWasmUrl);
});
} }
await oxipngReady; await oxipngReady;
} }
@@ -16,19 +16,23 @@ scope.onmessage = async (event: MessageEvent<OptimizeRequest>) => {
scope.postMessage({ type: "progress", progress, stage }); scope.postMessage({ type: "progress", progress, stage });
try { try {
let result;
if (kind === "video") {
// Mediabunny streams the source with range requests — never download a
// whole video into memory.
result = await optimizeVideo(url, filename, options, report);
} else {
report(0.01, "Downloading the original…"); report(0.01, "Downloading the original…");
const response = await fetch(url); const response = await fetch(url);
if (!response.ok) { if (!response.ok) {
throw new Error(`Could not load the file (HTTP ${response.status}).`); throw new Error(`Could not load the file (HTTP ${response.status}).`);
} }
const blob = await response.blob(); const blob = await response.blob();
result =
const result = kind === "animation"
kind === "video"
? await optimizeVideo(blob, filename, options, report)
: kind === "animation"
? await optimizeAnimation(blob, filename, options, report) ? await optimizeAnimation(blob, filename, options, report)
: await optimizeImage(blob, filename, options, report); : await optimizeImage(blob, filename, options, report);
}
scope.postMessage({ scope.postMessage({
type: "done", type: "done",
+109 -14
View File
@@ -1,14 +1,15 @@
import { import {
ALL_FORMATS, ALL_FORMATS,
BlobSource,
BufferTarget,
Conversion, Conversion,
Input, Input,
Mp4OutputFormat, Mp4OutputFormat,
Output, Output,
Quality, Quality,
StreamTarget,
UrlSource,
WebMOutputFormat, WebMOutputFormat,
getFirstEncodableVideoCodec, getFirstEncodableVideoCodec,
type StreamTargetChunk,
type VideoCodec, type VideoCodec,
} from "mediabunny"; } from "mediabunny";
@@ -28,9 +29,98 @@ function even(value: number): number {
return Math.max(2, value - (value % 2)); return Math.max(2, value - (value % 2));
} }
/** Transcode with WebCodecs (hardware-accelerated where available). */ interface CollectedChunk {
blob: Blob;
position: number;
sequence: number;
}
/**
* Stream the muxed output into Blob parts instead of one giant ArrayBuffer —
* a 4K file would otherwise blow the tab's memory. Chunks may arrive out of
* order (the muxer patches headers), so later writes win per byte range when
* the final blob is assembled.
*/
function createStreamOutput(): {
target: StreamTarget;
toBlob: (type: string) => Blob;
} {
const chunks: CollectedChunk[] = [];
let sequence = 0;
const writable = new WritableStream<StreamTargetChunk>({
write(chunk) {
chunks.push({
blob: new Blob([chunk.data.slice()]),
position: chunk.position,
sequence: sequence++,
});
},
});
return {
target: new StreamTarget(writable, {
chunked: true,
chunkSize: 8 * 1024 * 1024,
}),
toBlob: (type: string) => assembleBlob(chunks, type),
};
}
function assembleBlob(chunks: CollectedChunk[], type: string): Blob {
if (chunks.length === 0) {
throw new Error("Encoding produced no output.");
}
const sorted = [...chunks].sort(
(a, b) => a.position - b.position || a.sequence - b.sequence,
);
// Overwrite ranges so the newest write for each byte range survives.
interface Segment {
start: number;
end: number;
sequence: number;
blob: Blob;
}
let segments: Segment[] = [];
for (const chunk of sorted) {
const start = chunk.position;
const end = start + chunk.blob.size;
const next: Segment[] = [];
for (const segment of segments) {
if (segment.end <= start || segment.start >= end) {
next.push(segment);
continue;
}
if (segment.start < start) {
next.push({ ...segment, end: start });
}
if (segment.end > end) {
next.push({
...segment,
start: end,
blob: segment.blob.slice(end - segment.start),
});
}
}
next.push({ start, end, sequence: chunk.sequence, blob: chunk.blob });
segments = next;
}
segments.sort((a, b) => a.start - b.start);
const parts: Blob[] = [];
let cursor = 0;
for (const segment of segments) {
if (segment.start !== cursor) {
throw new Error("The muxer produced an incomplete file.");
}
parts.push(segment.blob);
cursor = segment.end;
}
return new Blob(parts, { type });
}
/** Transcode with WebCodecs, streaming the source and the result. */
export async function optimizeVideo( export async function optimizeVideo(
blob: Blob, url: string,
filename: string, filename: string,
options: OptimizeOptions, options: OptimizeOptions,
report: ProgressReporter, report: ProgressReporter,
@@ -38,7 +128,7 @@ export async function optimizeVideo(
const extension = extensionOf(filename); const extension = extensionOf(filename);
report(0.03, "Reading the video…"); report(0.03, "Reading the video…");
const input = new Input({ const input = new Input({
source: new BlobSource(blob), source: new UrlSource(url, { maxCacheSize: 4 * 1024 * 1024 }),
formats: ALL_FORMATS, formats: ALL_FORMATS,
}); });
const track = await input.getPrimaryVideoTrack(); const track = await input.getPrimaryVideoTrack();
@@ -56,8 +146,6 @@ export async function optimizeVideo(
: options.container; : options.container;
const format = const format =
container === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); container === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();
const target = new BufferTarget();
const output = new Output({ format, target });
const quality = new Quality(options.qualityLevel); const quality = new Quality(options.qualityLevel);
let codec: VideoCodec | undefined = let codec: VideoCodec | undefined =
@@ -78,6 +166,9 @@ export async function optimizeVideo(
const outputHeight = options.height ?? sourceHeight; const outputHeight = options.height ?? sourceHeight;
const outputWidth = Math.round(sourceWidth * (outputHeight / sourceHeight)); const outputWidth = Math.round(sourceWidth * (outputHeight / sourceHeight));
const { target, toBlob } = createStreamOutput();
const output = new Output({ format, target });
report(0.06, "Preparing the encoder…"); report(0.06, "Preparing the encoder…");
const conversion = await Conversion.init({ const conversion = await Conversion.init({
input, input,
@@ -101,17 +192,21 @@ export async function optimizeVideo(
conversion.onProgress = (progress: number) => { conversion.onProgress = (progress: number) => {
report(0.08 + Math.min(Math.max(progress, 0), 1) * 0.85, "Encoding…"); report(0.08 + Math.min(Math.max(progress, 0), 1) * 0.85, "Encoding…");
}; };
try {
await conversion.execute(); await conversion.execute();
} catch (error) {
const buffer = target.buffer; throw new Error(
if (!buffer) { `Video encoding failed: ${
throw new Error("Encoding produced no output."); error instanceof Error ? error.message : "unknown error"
}`,
);
} finally {
input.dispose?.();
} }
report(0.97, "Packaging…"); report(0.97, "Packaging…");
return { return {
blob: new Blob([buffer], { blob: toBlob(container === "webm" ? "video/webm" : "video/mp4"),
type: container === "webm" ? "video/webm" : "video/mp4",
}),
width: even(outputWidth), width: even(outputWidth),
height: even(outputHeight), height: even(outputHeight),
note: `${codec} · ${container}`, note: `${codec} · ${container}`,