From 33ae790bd0d9a2eaa43026880a98f58e91c617aa Mon Sep 17 00:00:00 2001 From: JakeBreath Date: Thu, 17 Sep 2026 21:04:58 -0500 Subject: [PATCH] Detect a missing video track and retry with another encoder Your symptom (audio kept, video 0x0) means the hardware H.264 encode silently produced no video samples while the audio was copied. The result check now parses the moov box and requires a 'vide' handler, so an audio-only output is treated as a failed attempt: the pipeline walks all codec x hardware/software configurations (starting with hardware when enabled) and only reports an error when every one fails, naming what each attempt returned. Resizing is also skipped entirely unless a smaller height was requested, keeping the scaler out of the pipeline. --- frontend/src/features/optimize/mp4.ts | 118 ++++++++++++++ frontend/src/features/optimize/video.ts | 207 +++++++++++------------- 2 files changed, 212 insertions(+), 113 deletions(-) create mode 100644 frontend/src/features/optimize/mp4.ts diff --git a/frontend/src/features/optimize/mp4.ts b/frontend/src/features/optimize/mp4.ts new file mode 100644 index 0000000..ab01d01 --- /dev/null +++ b/frontend/src/features/optimize/mp4.ts @@ -0,0 +1,118 @@ +/** + * Structural checks for muxer output. Kept dependency-free so it can also be + * exercised outside the browser. + */ + +function readUint32(view: DataView, offset: number): number { + return view.getUint32(offset); +} + +function readType(view: DataView, offset: number): string { + return String.fromCharCode( + view.getUint8(offset), + view.getUint8(offset + 1), + view.getUint8(offset + 2), + view.getUint8(offset + 3), + ); +} + +interface Box { + type: string; + start: number; // content start + size: number; // content size +} + +function childBoxes(view: DataView, start: number, end: number): Box[] { + const boxes: Box[] = []; + let offset = start; + while (offset + 8 <= end) { + const size = readUint32(view, offset); + const type = readType(view, offset + 4); + let boxSize = size; + let headerSize = 8; + if (size === 1 && offset + 16 <= end) { + boxSize = Number(view.getBigUint64(offset + 8)); + headerSize = 16; + } else if (size === 0) { + boxSize = end - offset; + } + if (boxSize < headerSize || offset + boxSize > end) break; + boxes.push({ + type, + start: offset + headerSize, + size: boxSize - headerSize, + }); + offset += boxSize; + } + return boxes; +} + +/** True when the moov payload declares a video track (hdlr === 'vide'). */ +export function moovHasVideoTrack( + view: DataView, + start: number, + end: number, +): boolean { + for (const trak of childBoxes(view, start, end)) { + if (trak.type !== "trak") continue; + for (const mdia of childBoxes(view, trak.start, trak.start + trak.size)) { + if (mdia.type !== "mdia") continue; + for (const hdlr of childBoxes(view, mdia.start, mdia.start + mdia.size)) { + if (hdlr.type !== "hdlr") continue; + const at = hdlr.start + 8; // version/flags + predefined + if (at + 4 <= hdlr.start + hdlr.size) { + if (readType(view, at) === "vide") return true; + } + } + } + } + return false; +} + +/** Throws when a muxed file is truncated or has no usable video track. */ +export async function assertPlayable( + file: Blob, + container: "mp4" | "webm", +): Promise { + const head = new Uint8Array(await file.slice(0, 12).arrayBuffer()); + if (container === "webm") { + const magic = [0x1a, 0x45, 0xdf, 0xa3]; + if (!magic.every((byte, index) => head[index] === byte)) { + throw new Error("the muxer produced an invalid WebM file"); + } + return; + } + let offset = 0; + for (let guard = 0; guard < 64 && offset + 8 <= file.size; guard++) { + const header = new Uint8Array( + await file.slice(offset, offset + 16).arrayBuffer(), + ); + const view = new DataView( + header.buffer, + header.byteOffset, + header.byteLength, + ); + const size = readUint32(view, 0); + const type = readType(view, 4); + let boxSize = size; + let headerSize = 8; + if (size === 1 && header.length >= 16) { + boxSize = Number(view.getBigUint64(8)); + headerSize = 16; + } else if (size === 0) { + boxSize = file.size - offset; + } + if (type === "moov") { + const payload = new Uint8Array( + await file.slice(offset + headerSize, offset + boxSize).arrayBuffer(), + ); + if (!moovHasVideoTrack(new DataView(payload.buffer), 0, payload.length)) { + throw new Error("the result has no video track"); + } + return; + } + if (!boxSize || boxSize < 8) break; + offset += boxSize; + } + throw new Error("the muxer produced an invalid MP4 file (no moov box)"); +} diff --git a/frontend/src/features/optimize/video.ts b/frontend/src/features/optimize/video.ts index dd1e242..25a24bc 100644 --- a/frontend/src/features/optimize/video.ts +++ b/frontend/src/features/optimize/video.ts @@ -10,6 +10,7 @@ import { type VideoCodec, } from "mediabunny"; +import { assertPlayable } from "./mp4"; import { createStreamOutput, type StreamOutput } from "./stream"; import { extensionOf, @@ -23,10 +24,21 @@ const CODEC_CANDIDATES: Record<"mp4" | "webm", VideoCodec[]> = { webm: ["vp9", "av1"], }; +const PREFERENCE_LABEL: Record = { + "prefer-hardware": "hardware", + "prefer-software": "software", + "no-preference": "auto", +}; + function even(value: number): number { return Math.max(2, value - (value % 2)); } +interface Attempt { + codec: VideoCodec; + preference: "prefer-hardware" | "prefer-software"; +} + /** Transcode with WebCodecs, streaming the source and the result. */ export async function optimizeVideo( url: string, @@ -62,10 +74,17 @@ export async function optimizeVideo( const format = container === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); - const outputHeight = even(options.height ?? sourceHeight); + // Only resize when a smaller height is requested — leaving it out keeps the + // scaler (and its failure modes) out of the pipeline entirely. + const downscale = + options.height !== null && options.height < sourceHeight + ? even(options.height) + : null; + const outputHeight = downscale ?? even(sourceHeight); const outputWidth = even( Math.round(sourceWidth * (outputHeight / sourceHeight)), ); + // Prefer bitrate-based encoding: hardware encoders often ignore the // per-frame quantizer that qualitative presets would otherwise use, which // made results inconsistent (and sometimes tiny). @@ -73,136 +92,98 @@ export async function optimizeVideo( quality: options.qualityLevel, preferBitrate: true, }); - const candidates: VideoCodec[] = + const codecs: VideoCodec[] = options.codec === "auto" ? CODEC_CANDIDATES[container] : [options.codec]; - const preferences: ("prefer-hardware" | "no-preference")[] = - options.preferHardware ? ["prefer-hardware", "no-preference"] : ["no-preference"]; - - report(0.06, "Preparing the encoder…"); - let winner: - | { - conversion: Conversion; - input: Input; - stream: StreamOutput; - codec: VideoCodec; - } - | null = null; - const tried: string[] = []; - for (const codec of candidates) { + const preferences: Attempt["preference"][] = options.preferHardware + ? ["prefer-hardware", "prefer-software"] + : ["prefer-software"]; + const attempts: Attempt[] = []; + for (const codec of codecs) { for (const preference of preferences) { - tried.push(`${codec}${preference === "prefer-hardware" ? " (hw)" : ""}`); - const input = new Input({ - source: new UrlSource(url, { maxCacheSize: 4 * 1024 * 1024 }), - formats: ALL_FORMATS, - }); - const stream = await createStreamOutput(container); - const output = new Output({ format, target: stream.target }); - const attempt = await Conversion.init({ + attempts.push({ codec, preference }); + } + } + + const issues: string[] = []; + for (const [index, attempt] of attempts.entries()) { + const label = `${attempt.codec} (${PREFERENCE_LABEL[attempt.preference]})`; + report( + 0.06 + index * 0.01, + index === 0 ? "Preparing the encoder…" : `Retrying with ${label}…`, + ); + const input = new Input({ + source: new UrlSource(url, { maxCacheSize: 4 * 1024 * 1024 }), + formats: ALL_FORMATS, + }); + const stream: StreamOutput = await createStreamOutput(container); + const output = new Output({ format, target: stream.target }); + let conversion: Conversion; + try { + conversion = await Conversion.init({ input, output, video: { - codec, + codec: attempt.codec, quality, - width: outputWidth, - height: outputHeight, - fit: "contain", - hardwareAcceleration: preference, + ...(downscale !== null + ? { width: outputWidth, height: outputHeight, fit: "contain" } + : {}), + hardwareAcceleration: attempt.preference, forceTranscode: true, }, }); - if (attempt.isValid) { - winner = { conversion: attempt, input, stream, codec }; - break; - } + } catch (error) { + issues.push(`${label}: ${messageOf(error)}`); input.dispose?.(); await stream.abort(); + continue; + } + if (!conversion.isValid) { + issues.push(`${label}: unsupported configuration`); + input.dispose?.(); + await stream.abort(); + continue; } - if (winner) break; - } - if (!winner) { - throw new Error( - `This browser could not configure ${container.toUpperCase()} encoding for ` + - `${outputWidth}×${outputHeight} (tried ${tried.join(", ")}). ` + - "Try another codec, container or resolution.", - ); - } - winner.conversion.onProgress = (progress: number) => { - report(0.08 + Math.min(Math.max(progress, 0), 1) * 0.85, "Encoding…"); - }; - try { - await winner.conversion.execute(); - } catch (error) { - await winner.stream.abort(); - throw new Error( - `Video encoding failed: ${ - error instanceof Error ? error.message : "unknown error" - }`, - ); - } finally { - winner.input.dispose?.(); - } - - report(0.97, "Checking the result…"); - try { - const { blob, storageKey } = await winner.stream.finalize( - container === "webm" ? "video/webm" : "video/mp4", - ); - await validateMedia(blob, container); - return { - blob, - width: outputWidth, - height: outputHeight, - note: `${winner.codec} · ${container}`, - storageKey, + conversion.onProgress = (progress: number) => { + report(0.08 + Math.min(Math.max(progress, 0), 1) * 0.85, "Encoding…"); }; - } catch (error) { - await winner.stream.abort(); - throw error; - } -} + try { + await conversion.execute(); + } catch (error) { + issues.push(`${label}: ${messageOf(error)}`); + input.dispose?.(); + await stream.abort(); + continue; + } + input.dispose?.(); -/** Reject a truncated or structurally broken muxer output up front. */ -async function validateMedia( - file: Blob, - container: "mp4" | "webm", -): Promise { - const head = new Uint8Array(await file.slice(0, 12).arrayBuffer()); - if (container === "webm") { - const magic = [0x1a, 0x45, 0xdf, 0xa3]; - if (!magic.every((byte, index) => head[index] === byte)) { - throw new Error("The muxer produced an invalid WebM file."); - } - return; - } - // Walk the top-level MP4 boxes until a moov box is found. - let offset = 0; - for (let guard = 0; guard < 64 && offset + 8 <= file.size; guard++) { - const header = new Uint8Array( - await file.slice(offset, offset + 16).arrayBuffer(), - ); - const size = - ((header[0] << 24) | (header[1] << 16) | (header[2] << 8) | header[3]) >>> - 0; - const type = String.fromCharCode( - header[4], - header[5], - header[6], - header[7], - ); - if (type === "moov") return; - let boxSize = size; - if (size === 1 && header.length >= 16) { - boxSize = Number( - new DataView(header.buffer, header.byteOffset, header.byteLength).getBigUint64( - 8, - ), + report(0.96, "Checking the result…"); + try { + const { blob, storageKey } = await stream.finalize( + container === "webm" ? "video/webm" : "video/mp4", ); + await assertPlayable(blob, container); + const software = attempt.preference === "prefer-software"; + return { + blob, + width: outputWidth, + height: outputHeight, + note: `${attempt.codec} · ${container}${software ? " · software" : ""}`, + storageKey, + }; + } catch (error) { + issues.push(`${label}: ${messageOf(error)}`); + await stream.abort(); } - if (!boxSize || boxSize < 8) break; - offset += boxSize; } + throw new Error( - "The muxer produced an invalid MP4 file (no moov box found).", + `Video encoding failed on every configuration (${issues.join("; ")}). ` + + "Try WebM output or a lower resolution.", ); } + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : "unknown error"; +}