fix(stream): separate source-missing import failures and name the import list cap

.changeset/stream-import-missing-source.md

@@ -0,0 +1,6 @@
+---
+"@bunny.net/cli": patch
+"@bunny.net/stream-import": patch
+---
+
+fix(stream): stop suggesting --clean-failed alone for videos the source no longer lists, and name the 10,000-video cap in `stream import list`

packages/cli/src/commands/stream/import/list.ts

@@ -40,8 +40,18 @@ function renderListing(listing: ImportListing, output: OutputFormat): void {
         output,
       ),
     );
-    if (listing.truncated)
-      logger.dim(`Showing the first ${listing.videos.length} videos.`);
+    if (listing.truncated) {
+      // Only suggest a flag the source accepts: flat sources reject --folder.
+      const narrow =
+        listing.source === "s3"
+          ? " Narrow it with --prefix."
+          : listing.supportsFolders
+            ? " Narrow it with --folder."
+            : "";
+      logger.dim(
+        `Showing the first ${listing.videos.length.toLocaleString("en-US")} videos, the most one listing returns.${narrow}`,
+      );
+    }
     return;
   }
   if (!listing.supportsFolders) {

packages/cli/src/commands/stream/import/run.ts

@@ -255,12 +255,17 @@ function renderResult(
     logger.log("");
     for (const m of failed) logger.error(`${stripAnsi(m.name)}: ${m.error}`);
   }
-  const givenUp = failed.filter((f) => !f.retryable).length;
+  const missing = failed.filter((f) => f.missingAtSource).length;
+  const givenUp = failed.filter((f) => !f.retryable).length - missing;
   if (givenUp > 0)
     logger.info(
       `${givenUp} cannot be retried as they are: Bunny rejected them or they failed 3 times. Add --clean-failed to delete their copies and try again.`,
     );
-  if (failed.length > givenUp || result.status === "paused")
+  if (missing > 0)
+    logger.info(
+      `${missing} are no longer listed at the source. Restore them there, then add --clean-failed to try again.`,
+    );
+  if (failed.length > givenUp + missing || result.status === "paused")
     logger.info(
       `Resume with ${bunny(resumeCommand(result, libraryArg, includeTs))}`,
     );

packages/stream-import/src/constants.ts

@@ -18,6 +18,9 @@ export const MAX_VIDEO_SHORT_EDGE = 2_160;
 /** A resume stops retrying a video after this many fetches; a fresh run or `cleanFailed` tries it again. */
 export const MAX_IMPORT_ATTEMPTS = 3;
 
+/** The error on an entry the source stopped listing; only restoring it there lets a retry succeed. */
+export const SOURCE_MISSING_ERROR = "No longer listed at the source";
+
 export const VIDEO_EXTENSIONS = [
   ".mp4",
   ".mov",

packages/stream-import/src/migration.ts

@@ -16,6 +16,7 @@ import {
   MAX_VIDEO_DURATION_SECONDS,
   MAX_VIDEO_LONG_EDGE,
   MAX_VIDEO_SHORT_EDGE,
+  SOURCE_MISSING_ERROR,
 } from "./constants.ts";
 import type {
   Logger,
@@ -429,7 +430,7 @@ export class MigrationService {
           (!folderId || m.sourceFolderId === folderId);
         if (!gone || signal?.aborted) continue;
         m.status = "failed";
-        m.error = "No longer listed at the source";
+        m.error = SOURCE_MISSING_ERROR;
         m.permanent = true;
       }
       state.status = signal?.aborted
@@ -604,9 +605,14 @@ export class MigrationService {
       (m) => m.status === "failed" && inScope(m),
     );
     const givenUp = failed.filter((m) => this.givenUp(m));
-    if (givenUp.length > 0)
+    const missing = givenUp.filter((m) => m.error === SOURCE_MISSING_ERROR);
+    if (givenUp.length > missing.length)
       this.logger.warn(
-        `Not retrying ${givenUp.length} videos that cannot succeed or failed ${MAX_IMPORT_ATTEMPTS} times; a fresh import or \`cleanFailed\` tries them again.`,
+        `Not retrying ${givenUp.length - missing.length} videos that cannot succeed or failed ${MAX_IMPORT_ATTEMPTS} times; a fresh import or \`cleanFailed\` tries them again.`,
+      );
+    if (missing.length > 0)
+      this.logger.warn(
+        `Not retrying ${missing.length} videos the source no longer lists; restore them there, then a fresh import or \`cleanFailed\` tries them again.`,
       );
     const retry = failed.filter((m) => !this.givenUp(m));
     if (retry.length === 0) return;

packages/tools/src/stream/import/index.ts

@@ -17,6 +17,7 @@ import {
   type MigrationState,
   readMigrationState,
   refreshMigrationState,
+  SOURCE_MISSING_ERROR,
   videoStatusText,
 } from "@bunny.net/stream-import";
 import { z } from "zod";
@@ -480,9 +481,11 @@ export const streamImportRun = defineTool({
         name: m.videoName,
         sourceId: m.sourceVideoId,
         error: m.error,
+        missingAtSource: m.error === SOURCE_MISSING_ERROR,
         retryable:
-          Boolean(input.cleanFailed) ||
-          (!m.permanent && (m.attempts ?? 0) < MAX_IMPORT_ATTEMPTS),
+          m.error !== SOURCE_MISSING_ERROR &&
+          (Boolean(input.cleanFailed) ||
+            (!m.permanent && (m.attempts ?? 0) < MAX_IMPORT_ATTEMPTS)),
       })),
       collections: state.folderMappings,
       warnings,

packages/tools/src/stream/import/model.ts

@@ -87,6 +87,8 @@ export const ImportRunSchema = z.object({
       name: z.string(),
       sourceId: z.string(),
       error: z.string().nullable(),
+      /** The source no longer lists it, so only restoring it there can bring it back. */
+      missingAtSource: z.boolean(),
       /** False when a resume will not retry it: a retry cannot fix it, or it has failed too often. */
       retryable: z.boolean(),
     }),