Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 71 additions & 2 deletions packages/engine/src/services/streamingEncoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,7 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
const result = await encoder.close();
expect(result.success).toBe(false);
expect(result.failureReason).toBe("external_interruption");
expect(result.error).toContain("termination=exit");
expect(encoder.getExitFailureReason?.()).toBe("external_interruption");
});

Expand Down Expand Up @@ -661,7 +662,11 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
const result = await encoder.close();

expect(result.success).toBe(false);
expect(result.error).toBe("Streaming encode cancelled");
expect(result.error).toContain("Streaming encode cancelled");
expect(result.error).toContain("termination=abort");
expect(result.error).toContain("frames attempted=0, accepted=0");
expect(encoder.getExitError()).toBe(result.error);
expect(result.failureReason).toBeUndefined();
});

it("close() is idempotent: a second call still resolves to a result and does not throw", async () => {
Expand Down Expand Up @@ -921,6 +926,56 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
}
});

it("keeps progressing under backpressure beyond the total timeout budget", async () => {
vi.useFakeTimers();
try {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
const dir = mkdtempSync(join(tmpdir(), "se-slow-drain-"));
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions, undefined, {
ffmpegStreamingTimeout: 1000,
});
const proc = calls[0]!.proc;
proc.stdin.write = () => false;
for (let i = 0; i < 5; i++) {
const write = encoder.writeFrame(Buffer.from([i]));
vi.advanceTimersByTime(900);
proc.stdin.emit("drain");
await expect(write).resolves.toBe(true);
}
expect(proc.kill).not.toHaveBeenCalled();
proc.emit("close", 0);
expect((await encoder.close()).success).toBe(true);
} finally {
vi.useRealTimers();
}
});

it("reports watchdog termination as failure even when the child exits zero", async () => {
vi.useFakeTimers();
try {
const { spawn, calls } = createSpawnSpy();
vi.resetModules();
vi.doMock("child_process", () => ({ spawn }));
const { spawnStreamingEncoder } = await import("./streamingEncoder.js");
const dir = mkdtempSync(join(tmpdir(), "se-timeout-zero-"));
const encoder = await spawnStreamingEncoder(join(dir, "out.mp4"), baseOptions, undefined, {
ffmpegStreamingTimeout: 1000,
});
vi.advanceTimersByTime(1000);
calls[0]!.proc.emit("close", 0);
const result = await encoder.close();
expect(result.success).toBe(false);
expect(result.error).toContain("termination=inactivity");
expect(result.error).toContain("frames attempted=0, accepted=0");
expect(encoder.getExitError()).toBe(result.error);
} finally {
vi.useRealTimers();
}
});

it("inactivity timeout still fires when stdin is backpressured (stalled ffmpeg, live producer)", async () => {
vi.useFakeTimers();
try {
Expand Down Expand Up @@ -951,8 +1006,22 @@ describe("spawnStreamingEncoder lifecycle and cleanup", () => {
vi.advanceTimersByTime(1100);
expect(proc.kill).toHaveBeenCalledWith("SIGTERM");

proc.emit("close", null);
// FFmpeg commonly handles SIGTERM and exits 255 with this stderr.
proc.stderr.emit("data", Buffer.from("Exiting normally, received signal 15.\n"));
proc.emit("close", 255);
await expect(writePromise).resolves.toBe(false);
expect(encoder.getExitStatus()).toBe("error");
const errorAtExit = encoder.getExitError();
expect(errorAtExit).toContain("termination=inactivity");
expect(errorAtExit).toContain("ffmpegStreamingTimeout=1000 ms");
expect(errorAtExit).toContain("frames attempted=1, accepted=1, waitingForDrain=true");
expect(encoder.getExitFailureReason?.()).toBeUndefined();
// A late capture callback must not overwrite the counts at termination.
await expect(encoder.writeFrame(Buffer.from([1]))).resolves.toBe(false);
expect(encoder.getExitError()).toBe(errorAtExit);
const result = await encoder.close();
expect(result.error).toBe(errorAtExit);
expect(result.failureReason).toBeUndefined();
} finally {
vi.useRealTimers();
}
Expand Down
67 changes: 40 additions & 27 deletions packages/engine/src/services/streamingEncoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,15 @@ export async function spawnStreamingEncoder(
let exitSignal: NodeJS.Signals | null = null;
let terminationReason: ManagedProcessTerminationReason = "exit";

let framesAttempted = 0;
let framesAccepted = 0;
let waitingForDrain = false;
let terminationFrames: string | undefined;
// Accepted means queued by Node's writable, including writes returning false;
// it does not mean FFmpeg has encoded the frame.
const frameDiagnostics = () =>
`frames attempted=${framesAttempted}, accepted=${framesAccepted}, waitingForDrain=${waitingForDrain}`;

ffmpeg.stdin?.on("error", () => {});
ffmpeg.stdout?.on("error", () => {});

Expand All @@ -488,12 +497,16 @@ export async function spawnStreamingEncoder(
const managed = new ManagedChildProcess(ffmpeg, {
signal,
inactivityTimeoutMs: streamingTimeout,
onTerminationRequested: () => {
terminationFrames = frameDiagnostics();
},
});
const exitPromise = managed.wait().then((outcome) => {
exitCode = outcome.exitCode;
exitSignal = outcome.signal;
stderr = outcome.stderr;
terminationReason = outcome.reason;
terminationFrames ??= frameDiagnostics();
exitStatus = outcome.reason === "exit" && outcome.exitCode === 0 ? "success" : "error";
return outcome;
});
Expand Down Expand Up @@ -533,9 +546,23 @@ export async function spawnStreamingEncoder(
}
};

const formatExitError = (): string => {
const message =
terminationReason === "abort"
? "Streaming encode cancelled"
: formatFfmpegError(exitCode, stderr);
const timeout =
terminationReason === "inactivity"
? `; ffmpegStreamingTimeout=${streamingTimeout} ms without write progress`
: "";
return `${message}\nStreaming encoder termination=${terminationReason}${timeout}; ${terminationFrames ?? frameDiagnostics()}`;
};

const encoder: StreamingEncoder = {
writeFrame: async (buffer: Buffer): Promise<boolean> => {
framesAttempted++;
const stdin = ffmpeg.stdin;
if (terminationFrames !== undefined) await exitPromise;
if (exitStatus !== "running") {
return false;
}
Expand All @@ -554,22 +581,21 @@ export async function spawnStreamingEncoder(
// and flicker.
const copy = Buffer.from(buffer);
const accepted = stdin.write(copy);
// Reset inactivity timer immediately ONLY on `accepted === true`. `true`
// means the write went through to the kernel pipe without buffering in
// Node — proof FFmpeg is actually consuming. `false` means Node's writable
// stream had to buffer (FFmpeg hasn't drained the pipe yet); we await
// `drain` before letting callers produce the next frame, and only reset
// after drain proves consumption. We deliberately don't reset before
// drain so a hung FFmpeg with a still-producing Chrome can't keep us
// alive forever while Node's stdin buffer grows to OOM. If FFmpeg exits
// before draining, waitForDrainOrExit returns "exit", removes its
// one-shot listeners, and callers see `false` instead of hanging.
framesAccepted++;
// A false return is backpressure, not rejection. Wait for drain before
// producing another frame, and keep the watchdog armed during that wait.
if (accepted) {
managed.markActivity();
return true;
}

const drainResult = await waitForDrainOrExit(stdin);
waitingForDrain = true;
let drainResult: "drain" | "exit";
try {
drainResult = await waitForDrainOrExit(stdin);
} finally {
waitingForDrain = false;
}
if (drainResult !== "drain" || exitStatus !== "running") {
return false;
}
Expand Down Expand Up @@ -599,25 +625,12 @@ export async function spawnStreamingEncoder(
const outcome = await exitPromise;
const durationMs = outcome.durationMs;

if (terminationReason === "abort") {
return {
success: false,
durationMs,
fileSize: 0,
error: "Streaming encode cancelled",
};
}

if (exitCode !== 0) {
const inactivitySuffix =
terminationReason === "inactivity"
? `\nFFmpeg stopped after ${streamingTimeout} ms without consuming a frame.`
: "";
if (exitStatus === "error") {
return {
success: false,
durationMs,
fileSize: 0,
error: `${formatFfmpegError(exitCode, stderr)}${inactivitySuffix}`,
error: formatExitError(),
failureReason: isExternalFfmpegInterruption({
exitCode,
signal: exitSignal,
Expand All @@ -638,7 +651,7 @@ export async function spawnStreamingEncoder(

getExitError: () => {
if (exitStatus !== "error") return undefined;
return formatFfmpegError(exitCode, stderr);
return formatExitError();
},

getExitFailureReason: () => {
Expand Down
2 changes: 2 additions & 0 deletions packages/engine/src/utils/managedChildProcess.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ export interface ManagedChildProcessOptions {
terminationGraceMs?: number;
stderrMaxBytes?: number;
onStderr?: (chunk: string) => void;
onTerminationRequested?: () => void;
now?: () => number;
}

Expand Down Expand Up @@ -143,6 +144,7 @@ export class ManagedChildProcess {
): void {
if (this.settled || this.requestedReason) return;
this.requestedReason = reason;
this.options.onTerminationRequested?.();
try {
this.child.kill("SIGTERM");
} catch {
Expand Down
Loading