diff --git a/.env.example b/.env.example index 69863e0..1cf4da6 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,13 @@ JWT_REFRESH_TTL=168h INTERNAL_TOKEN=change-me-internal-token-32chars # Upload limit (bytes). 5GB default for Phase 9, Phase 11 will use presigned multipart for bigger files UPLOAD_MAX_BYTES=5368709120 +# Upload reconciler: a video row is created before the bytes land, so an upload +# that never completes (closed tab, lost resume key) would hang in 'uploaded' +# forever. Every UPLOAD_RECONCILE_INTERVAL_SECONDS the upload service looks at +# 'uploaded' rows older than UPLOAD_STALE_TTL_SECONDS: it re-queues them when the +# raw file exists (lost event) or marks them failed (abandoned upload). +UPLOAD_STALE_TTL_SECONDS=3600 +UPLOAD_RECONCILE_INTERVAL_SECONDS=300 # Transcoder (Phase 10) — limit CPU/RAM per job, sequential renditions FFMPEG_THREADS=2 diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index e178a04..4e3c61b 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -239,6 +239,8 @@ services: INTERNAL_TOKEN: ${INTERNAL_TOKEN:?INTERNAL_TOKEN is required — set it in .env (see .env.example)} JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required — set it in .env (see .env.example)} UPLOAD_MAX_BYTES: ${UPLOAD_MAX_BYTES:-5368709120} + UPLOAD_STALE_TTL_SECONDS: ${UPLOAD_STALE_TTL_SECONDS:-3600} + UPLOAD_RECONCILE_INTERVAL_SECONDS: ${UPLOAD_RECONCILE_INTERVAL_SECONDS:-300} MINIO_PUBLIC_ENDPOINT: ${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000} # grace для Shutdown меньше stop_grace_period (issue #63) SHUTDOWN_GRACE: 4m diff --git a/docs/services-pipeline.md b/docs/services-pipeline.md index fdbb05f..ab6a82f 100644 --- a/docs/services-pipeline.md +++ b/docs/services-pipeline.md @@ -95,6 +95,8 @@ Flowix — MVP видеоплатформы с адаптивным стрими `uploaded` → (transcoder забрал) → `processing` → `ready` | `failed`. Frontend поллит `GET /api/v1/videos/:id` пока не `ready`. +Застрявшие загрузки само-лечатся: сервис upload раз в `UPLOAD_RECONCILE_INTERVAL_SECONDS` берёт записи в `uploaded` старше `UPLOAD_STALE_TTL_SECONDS` и переопубликовывает `video.uploaded` (если original есть в MinIO) либо помечает `failed` (загрузка брошена). Лента `GET /api/v1/videos` показывает всем только `public`+`ready`, владелец видит и свои не-`ready` (uploaded/processing/failed) для удаления/перезалива. + ### События (контракты) - `video.uploaded {video_id: UUID, s3_key: string, owner_id: UUID}` diff --git a/frontend/src/app/watch/[id]/page.tsx b/frontend/src/app/watch/[id]/page.tsx index ecdab43..3e74a88 100644 --- a/frontend/src/app/watch/[id]/page.tsx +++ b/frontend/src/app/watch/[id]/page.tsx @@ -143,7 +143,10 @@ export default function WatchPage() { ) : (
{video.status === "processing" || video.status === "uploaded" ? ( - Transcoding… status: {video.status} + + {video.status === "uploaded" ? "Waiting for upload…" : "Transcoding…"} status:{" "} + {video.status} + ) : video.status === "failed" ? ( Transcoding failed ) : ( diff --git a/scripts/e2e.sh b/scripts/e2e.sh index ecf2350..08353f9 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -500,6 +500,9 @@ if [ -n "${TOKEN:-}" ]; then code=$(curl -s -o "$TMP/body" -w '%{http_code}' -X PUT --data-binary "@$TMP/part2" -H "Authorization: Bearer $TOKEN" -H "Content-Type: video/mp4" -H "Content-Range: bytes $half-$end2/$total" "$GATEWAY/api/v1/videos/$RESUME2_ID/resumable") [ "$code" = "200" ] || fail "second half resumable PUT not 200: $code $(cat "$TMP/body")" say " 50% resume: ok (308→200)" + # complete so the video is queued and does not linger in 'uploaded' + code=$(curl -s -o "$TMP/body" -w '%{http_code}' -X POST "$GATEWAY/api/v1/videos/$RESUME2_ID/complete" -H "Authorization: Bearer $TOKEN") + [ "$code" = "200" ] || say " WARN: resume2 complete not 200: $code $(cat "$TMP/body")" else say " WARN: presign for resumable failed ($code)" fi diff --git a/services/auth/.python-version b/services/auth/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/services/auth/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/services/metadata/cmd/server/main.go b/services/metadata/cmd/server/main.go index a7c931a..66ceecf 100644 --- a/services/metadata/cmd/server/main.go +++ b/services/metadata/cmd/server/main.go @@ -151,6 +151,7 @@ func main() { r.Group(func(r chi.Router) { r.Use(mw.InternalAuth(internalToken)) r.Patch("/internal/videos/{id}/status", vh.UpdateStatus) + r.Get("/internal/videos/stale", vh.ListStale) r.Get("/internal/videos/{id}", vh.GetInternal) r.Get("/internal/videos/{id}/vod", vh.GetVODMapping) }) diff --git a/services/metadata/internal/handler/video.go b/services/metadata/internal/handler/video.go index 473e169..6b88921 100644 --- a/services/metadata/internal/handler/video.go +++ b/services/metadata/internal/handler/video.go @@ -28,6 +28,7 @@ type VideoStore interface { Delete(ctx context.Context, id, ownerID string) error UpdateStatus(ctx context.Context, id string, status model.VideoStatus, renditions []model.Rendition) error UpdateThumbnail(ctx context.Context, id string, thumbnailS3Key string) error + ListStaleUploaded(ctx context.Context, olderThan time.Duration, limit int) ([]model.StaleUpload, error) } // StorageRemover deletes S3 objects (raw + renditions + thumbnail). Nil = skip (tests). @@ -95,6 +96,7 @@ func (h *VideoHandler) Register(r chi.Router) { }) // internal (no auth) for transcoder/upload r.Patch("/internal/videos/{id}/status", h.UpdateStatus) + r.Get("/internal/videos/stale", h.ListStale) r.Get("/internal/videos/{id}", h.GetInternal) r.Get("/internal/videos/{id}/vod", h.GetVODMapping) } @@ -188,6 +190,33 @@ func (h *VideoHandler) getVideo(w http.ResponseWriter, r *http.Request, enforceV writeJSON(w, r, http.StatusOK, v) } +// ListStale godoc +// @Summary List videos stuck in 'uploaded' older than the given age (internal) +// @Description Used by the upload-service reconciler: a video row is created before the bytes land, so an upload that never completes stays in 'uploaded' forever. +// @Tags internal +// @Produce json +// @Param older_than_seconds query int false "Minimum age in seconds" default(3600) +// @Param limit query int false "Max rows (<=500)" default(100) +// @Success 200 {object} map[string]interface{} +// @Router /internal/videos/stale [get] +func (h *VideoHandler) ListStale(w http.ResponseWriter, r *http.Request) { + olderThan := 3600 + if v, err := strconv.Atoi(r.URL.Query().Get("older_than_seconds")); err == nil && v > 0 { + olderThan = v + } + limit := 100 + if v, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && v > 0 && v <= 500 { + limit = v + } + list, err := h.repo.ListStaleUploaded(r.Context(), time.Duration(olderThan)*time.Second, limit) + if err != nil { + slog.Error("list stale uploads failed", "error", err) + writeError(w, r, http.StatusInternalServerError, "internal error") + return + } + writeJSON(w, r, http.StatusOK, map[string]interface{}{"data": list}) +} + type vodMapping struct { Sequences []vodSequence `json:"sequences"` } diff --git a/services/metadata/internal/handler/video_test.go b/services/metadata/internal/handler/video_test.go index c2406a1..d7921ca 100644 --- a/services/metadata/internal/handler/video_test.go +++ b/services/metadata/internal/handler/video_test.go @@ -61,12 +61,15 @@ func (f *fakeStore) GetByID(_ context.Context, id string) (*model.Video, error) func (f *fakeStore) List(_ context.Context, limit, offset int, viewerID string) ([]model.Video, error) { out := []model.Video{} for _, v := range f.videos { - // mirror the SQL visibility filter (issue #44); empty = default public + // mirror the SQL feed filter (issue #44 + stale-upload fix): everyone + // sees only public+ready, the owner also sees their non-ready videos. vis := v.Visibility if vis == "" { vis = model.VisibilityPublic } - if vis != model.VisibilityPublic && (viewerID == "" || v.OwnerID != viewerID) { + isOwner := viewerID != "" && v.OwnerID == viewerID + publicReady := vis == model.VisibilityPublic && v.Status == model.StatusReady + if !publicReady && !isOwner { continue } out = append(out, *v) @@ -139,6 +142,19 @@ func (f *fakeStore) UpdateThumbnail(_ context.Context, id string, thumbnailS3Key v.ThumbnailURL = &u return nil } +func (f *fakeStore) ListStaleUploaded(_ context.Context, olderThan time.Duration, limit int) ([]model.StaleUpload, error) { + out := []model.StaleUpload{} + for _, v := range f.videos { + if v.Status != model.StatusUploaded || time.Since(v.CreatedAt) < olderThan { + continue + } + out = append(out, model.StaleUpload{ID: v.ID, OwnerID: v.OwnerID}) + if limit > 0 && len(out) >= limit { + break + } + } + return out, nil +} // Issue #58: wrappers that simulate a failing DB for handler 500-mapping tests. type failingGetStore struct { @@ -175,6 +191,7 @@ func testRouter(store VideoStore) chi.Router { r.With(optAuth).Get("/api/v1/videos", vh.List) r.With(optAuth).Get("/api/v1/videos/{id}", vh.Get) r.Patch("/internal/videos/{id}/status", vh.UpdateStatus) + r.Get("/internal/videos/stale", vh.ListStale) r.Get("/internal/videos/{id}", vh.GetInternal) r.Get("/internal/videos/{id}/vod", vh.GetVODMapping) return r @@ -344,6 +361,51 @@ func TestGetAndList(t *testing.T) { } } +// Feed must hide non-ready videos from everyone but their owner. +func TestListHidesNonReadyFromOthers(t *testing.T) { + store := newFake() + store.videos["pub-ready"] = &model.Video{ID: "pub-ready", OwnerID: "o1", Title: "a", Status: model.StatusReady, Visibility: model.VisibilityPublic} + store.videos["pub-failed"] = &model.Video{ID: "pub-failed", OwnerID: "o1", Title: "b", Status: model.StatusFailed, Visibility: model.VisibilityPublic} + store.videos["pub-uploaded"] = &model.Video{ID: "pub-uploaded", OwnerID: "o1", Title: "c", Status: model.StatusUploaded, Visibility: model.VisibilityPublic} + r := testRouter(store) + + ids := func(token string) map[string]bool { + req := httptest.NewRequest("GET", "/api/v1/videos?limit=50&offset=0", nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("want 200 got %d %s", w.Code, w.Body.String()) + } + var out struct { + Data []model.Video `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, v := range out.Data { + got[v.ID] = true + } + return got + } + + anon := ids("") + if !anon["pub-ready"] || anon["pub-failed"] || anon["pub-uploaded"] { + t.Fatalf("anonymous feed must show only ready: %v", anon) + } + owner := ids(signToken("o1")) + if !owner["pub-ready"] || !owner["pub-failed"] || !owner["pub-uploaded"] { + t.Fatalf("owner must see own non-ready videos: %v", owner) + } + other := ids(signToken("o2")) + if other["pub-failed"] || other["pub-uploaded"] { + t.Fatalf("other user must not see non-ready videos: %v", other) + } +} + func TestUpdateForbidden(t *testing.T) { store := newFake() store.videos["vid-1"] = &model.Video{ID: "vid-1", OwnerID: "owner1", Title: "t", Status: model.StatusUploaded} @@ -370,6 +432,30 @@ func TestUpdateForbidden(t *testing.T) { } } +// Reconciler endpoint: only 'uploaded' rows older than the threshold are listed. +func TestListStaleUploads(t *testing.T) { + store := newFake() + store.videos["old"] = &model.Video{ID: "old", OwnerID: "o1", Title: "t", Status: model.StatusUploaded, CreatedAt: time.Now().Add(-2 * time.Hour)} + store.videos["fresh"] = &model.Video{ID: "fresh", OwnerID: "o1", Title: "t", Status: model.StatusUploaded, CreatedAt: time.Now()} + store.videos["done"] = &model.Video{ID: "done", OwnerID: "o1", Title: "t", Status: model.StatusReady, CreatedAt: time.Now().Add(-2 * time.Hour)} + r := testRouter(store) + req := httptest.NewRequest("GET", "/internal/videos/stale?older_than_seconds=3600&limit=10", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("want 200 got %d %s", w.Code, w.Body.String()) + } + var out struct { + Data []model.StaleUpload `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if len(out.Data) != 1 || out.Data[0].ID != "old" || out.Data[0].OwnerID != "o1" { + t.Fatalf("want only 'old', got %+v", out.Data) + } +} + func TestDelete(t *testing.T) { store := newFake() store.videos["vid-1"] = &model.Video{ID: "vid-1", OwnerID: "owner1", Title: "t", Status: model.StatusUploaded} diff --git a/services/metadata/internal/model/video.go b/services/metadata/internal/model/video.go index d307d08..2b4e6e5 100644 --- a/services/metadata/internal/model/video.go +++ b/services/metadata/internal/model/video.go @@ -40,6 +40,14 @@ type Video struct { Renditions []Rendition `json:"renditions,omitempty"` } +// StaleUpload is a minimal view of a video still in 'uploaded' — the +// upload-service reconciler uses it to re-queue a lost event or fail an +// abandoned upload (a row is created before the bytes land). +type StaleUpload struct { + ID string `json:"id"` + OwnerID string `json:"owner_id"` +} + type Rendition struct { VideoID string `json:"video_id"` Quality string `json:"quality"` diff --git a/services/metadata/internal/repository/video.go b/services/metadata/internal/repository/video.go index 4abffef..28affd7 100644 --- a/services/metadata/internal/repository/video.go +++ b/services/metadata/internal/repository/video.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "time" "flowix/metadata/internal/model" "github.com/google/uuid" @@ -81,7 +82,10 @@ func (r *VideoRepo) GetByID(ctx context.Context, id string) (*model.Video, error func (r *VideoRepo) List(ctx context.Context, limit, offset int, viewerID string) ([]model.Video, error) { // $3 участвует и в текстовом сравнении ($3 <> ''), и в сравнении с uuid-колонкой: // без ::uuid Postgres выводит тип $3 как text и падает с SQLSTATE 42883 (uuid = text) - q := `SELECT v.id, v.owner_id, u.email, v.title, v.description, v.duration, v.status, COALESCE(v.visibility::text,'public'), v.thumbnail_s3_key, v.created_at, v.updated_at FROM videos v LEFT JOIN users u ON u.id=v.owner_id WHERE v.visibility='public' OR ($3 <> '' AND v.owner_id=$3::uuid) ORDER BY v.created_at DESC LIMIT $1 OFFSET $2` + // Лента: всем — только public+ready (неготовые/упавшие не светятся чужим); + // владелец видит свои в любом статусе (uploaded/processing/failed), чтобы + // удалить или перезалить (issue: зомби-видео засоряли ленту). + q := `SELECT v.id, v.owner_id, u.email, v.title, v.description, v.duration, v.status, COALESCE(v.visibility::text,'public'), v.thumbnail_s3_key, v.created_at, v.updated_at FROM videos v LEFT JOIN users u ON u.id=v.owner_id WHERE (v.visibility='public' AND v.status='ready') OR ($3 <> '' AND v.owner_id=$3::uuid) ORDER BY v.created_at DESC LIMIT $1 OFFSET $2` rows, err := r.pool.Query(ctx, q, limit, offset, viewerID) if err != nil { return nil, err @@ -153,6 +157,30 @@ func (r *VideoRepo) UpdateThumbnail(ctx context.Context, id string, thumbnailS3K return err } +// ListStaleUploaded returns videos still in 'uploaded' created more than +// olderThan ago. A row is inserted before the bytes are uploaded, so a video +// whose upload never completes sits in 'uploaded' forever; the upload-service +// reconciler uses this list to re-queue or fail such rows. +func (r *VideoRepo) ListStaleUploaded(ctx context.Context, olderThan time.Duration, limit int) ([]model.StaleUpload, error) { + if limit <= 0 || limit > 500 { + limit = 100 + } + rows, err := r.pool.Query(ctx, `SELECT id, owner_id FROM videos WHERE status='uploaded' AND created_at < now() - ($1::int * interval '1 second') ORDER BY created_at ASC LIMIT $2`, int(olderThan.Seconds()), limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := []model.StaleUpload{} + for rows.Next() { + var s model.StaleUpload + if err := rows.Scan(&s.ID, &s.OwnerID); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + func (r *VideoRepo) UpdateStatus(ctx context.Context, id string, status model.VideoStatus, renditions []model.Rendition) error { tx, err := r.pool.Begin(ctx) if err != nil { diff --git a/services/transcoder/.python-version b/services/transcoder/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/services/transcoder/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/services/upload/cmd/server/main.go b/services/upload/cmd/server/main.go index 55055b0..f44e68f 100644 --- a/services/upload/cmd/server/main.go +++ b/services/upload/cmd/server/main.go @@ -9,10 +9,12 @@ package main // @name Authorization // @description Type "Bearer {token}" import ( + "context" "log" "net/http" "os" "strconv" + "time" pkghttp "flowix/pkg/httpserver" pkglogger "flowix/pkg/logger" @@ -22,6 +24,7 @@ import ( "flowix/upload/internal/client" "flowix/upload/internal/handler" "flowix/upload/internal/queue" + "flowix/upload/internal/reconciler" "flowix/upload/internal/storage" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -73,6 +76,12 @@ func main() { rh := handler.NewResumableHandler(store, metaCl) mp := handler.NewMultipartHandler(store, pub, metaCl, metaCl) + // Self-heal videos stuck in 'uploaded': an upload row is created before the + // bytes land, so an upload that never completes hangs forever (see package + // reconciler). Runs for the lifetime of the process. + rec := reconciler.New(metaCl, store, pub, secondsEnv("UPLOAD_STALE_TTL_SECONDS", 3600), 100) + go rec.Run(context.Background(), secondsEnv("UPLOAD_RECONCILE_INTERVAL_SECONDS", 300)) + r := chi.NewRouter() r.Use(middleware.RequestID, middleware.Recoverer, pkgmw.RequestLogger("upload")) @@ -119,3 +128,11 @@ func requireEnv(k string) string { } return v } + +// secondsEnv reads a positive number of seconds from env, falling back to def. +func secondsEnv(k string, def int) time.Duration { + if n, err := strconv.Atoi(os.Getenv(k)); err == nil && n > 0 { + return time.Duration(n) * time.Second + } + return time.Duration(def) * time.Second +} diff --git a/services/upload/internal/client/metadata.go b/services/upload/internal/client/metadata.go index db80b51..860522f 100644 --- a/services/upload/internal/client/metadata.go +++ b/services/upload/internal/client/metadata.go @@ -59,6 +59,63 @@ func (c *MetadataClient) CreateVideo(token, title, description string) (string, return out.ID, nil } +// StaleUpload is a video stuck in 'uploaded' as returned by metadata +// GET /internal/videos/stale (id + owner for re-publishing video.uploaded). +type StaleUpload struct { + ID string `json:"id"` + OwnerID string `json:"owner_id"` +} + +// ListStaleUploaded returns videos still in 'uploaded' created more than +// olderThan ago — the reconciler filters out the ones that are genuinely done. +func (c *MetadataClient) ListStaleUploaded(olderThan time.Duration, limit int) ([]StaleUpload, error) { + u := fmt.Sprintf("%s/internal/videos/stale?older_than_seconds=%d&limit=%d", c.baseURL, int(olderThan.Seconds()), limit) + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + if c.internalToken != "" { + req.Header.Set("X-Internal-Token", c.internalToken) + } + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("metadata stale uploads: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("metadata stale uploads status %d", resp.StatusCode) + } + var out struct { + Data []StaleUpload `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, err + } + return out.Data, nil +} + +// SetStatus updates a video status via the metadata internal API. +func (c *MetadataClient) SetStatus(videoID, status string) error { + body, _ := json.Marshal(map[string]string{"status": status}) + req, err := http.NewRequest("PATCH", c.baseURL+"/internal/videos/"+url.PathEscape(videoID)+"/status", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if c.internalToken != "" { + req.Header.Set("X-Internal-Token", c.internalToken) + } + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("metadata set status: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("metadata set status %d", resp.StatusCode) + } + return nil +} + // GetVideoOwner resolves the owner of a video via metadata internal API // (upload handlers must verify ownership before writing raw/{id}/* — issue #46). func (c *MetadataClient) GetVideoOwner(videoID string) (string, error) { diff --git a/services/upload/internal/client/metadata_test.go b/services/upload/internal/client/metadata_test.go index 43adb17..d8d10ed 100644 --- a/services/upload/internal/client/metadata_test.go +++ b/services/upload/internal/client/metadata_test.go @@ -5,6 +5,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) func TestGetVideoOwner(t *testing.T) { @@ -74,3 +75,43 @@ func TestGetVideoOwnerOmitsEmptyToken(t *testing.T) { t.Fatal("X-Internal-Token must be omitted when internalToken is empty") } } + +func TestListStaleUploadedAndSetStatus(t *testing.T) { + var gotQuery, gotToken, gotMethod string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotToken = r.Header.Get("X-Internal-Token") + gotMethod = r.Method + switch r.URL.Path { + case "/internal/videos/stale": + gotQuery = r.URL.RawQuery + _, _ = w.Write([]byte(`{"data":[{"id":"v1","owner_id":"o1"}]}`)) + case "/internal/videos/v1/status": + _, _ = w.Write([]byte(`{"status":"ok"}`)) + default: + http.Error(w, "not found", http.StatusNotFound) + } + })) + defer srv.Close() + + c := NewMetadataClient(srv.URL, "tok") + list, err := c.ListStaleUploaded(2*time.Hour, 50) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(list) != 1 || list[0].ID != "v1" || list[0].OwnerID != "o1" { + t.Fatalf("unexpected stale list %+v", list) + } + if gotQuery != "older_than_seconds=7200&limit=50" { + t.Fatalf("query = %q", gotQuery) + } + if gotToken != "tok" { + t.Fatalf("X-Internal-Token = %q", gotToken) + } + + if err := c.SetStatus("v1", "failed"); err != nil { + t.Fatalf("unexpected err: %v", err) + } + if gotMethod != "PATCH" { + t.Fatalf("method = %q, want PATCH", gotMethod) + } +} diff --git a/services/upload/internal/reconciler/reconciler.go b/services/upload/internal/reconciler/reconciler.go new file mode 100644 index 0000000..84d4642 --- /dev/null +++ b/services/upload/internal/reconciler/reconciler.go @@ -0,0 +1,115 @@ +// Package reconciler self-heals videos stuck in status 'uploaded'. +// +// Every upload path creates the metadata row before the bytes land, and the +// only thing that moves a row out of 'uploaded' is the transcoder consuming the +// video.uploaded event. So an upload that never completes (closed tab, lost +// resume key, network drop, publish failure, a queue that was recreated) leaves +// a row that hangs forever — the UI then shows an endless "Transcoding…". +// +// The reconciler periodically looks at 'uploaded' rows older than ttl and: +// - re-publishes video.uploaded when the raw object exists (a lost event), or +// - aborts any dangling multipart session and marks the row failed when it +// does not (an abandoned upload). +package reconciler + +import ( + "context" + "time" + + "flowix/upload/internal/client" + "flowix/upload/internal/handler" + "github.com/rs/zerolog/log" +) + +// Metadata lists stale uploads and can fail them. +type Metadata interface { + ListStaleUploaded(olderThan time.Duration, limit int) ([]client.StaleUpload, error) + SetStatus(videoID, status string) error +} + +// Storage inspects and cleans the raw object / multipart session. +type Storage interface { + StatObjectSize(ctx context.Context, key string) (int64, error) + FindUploadID(ctx context.Context, key string) (string, error) + AbortMultipartUpload(ctx context.Context, key, uploadID string) error +} + +// Publisher re-emits video.uploaded (implemented by queue.Publisher). +type Publisher interface { + Publish(ctx context.Context, payload interface{}) error +} + +// Reconciler periodically reconciles stale 'uploaded' rows. +type Reconciler struct { + metadata Metadata + storage Storage + publisher Publisher + ttl time.Duration + limit int +} + +func New(m Metadata, s Storage, p Publisher, ttl time.Duration, limit int) *Reconciler { + if limit <= 0 { + limit = 100 + } + return &Reconciler{metadata: m, storage: s, publisher: p, ttl: ttl, limit: limit} +} + +// Run reconciles every interval until ctx is cancelled. +func (r *Reconciler) Run(ctx context.Context, interval time.Duration) { + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + r.reconcile(ctx) + } + } +} + +func rawKey(videoID string) string { return "raw/" + videoID + "/original.mp4" } + +// reconcile is exported for tests via reconcileOnce. +func (r *Reconciler) reconcile(ctx context.Context) { + stale, err := r.metadata.ListStaleUploaded(r.ttl, r.limit) + if err != nil { + log.Error().Err(err).Msg("reconciler: list stale uploads failed") + return + } + for _, v := range stale { + r.handle(ctx, v) + } +} + +func (r *Reconciler) handle(ctx context.Context, v client.StaleUpload) { + key := rawKey(v.ID) + // Object present → the upload finished but the event never reached the + // queue (publish failure / queue recreated). Re-publish; the transcoder + // skips videos already ready/failed. + if size, err := r.storage.StatObjectSize(ctx, key); err == nil && size > 0 { + ev := handler.VideoUploadedEvent{VideoID: v.ID, S3Key: key, OwnerID: v.OwnerID} + if err := r.publisher.Publish(ctx, ev); err != nil { + log.Error().Err(err).Str("video_id", v.ID).Msg("reconciler: republish failed") + return + } + log.Info().Str("video_id", v.ID).Msg("reconciler: re-published lost video.uploaded") + return + } + // No object → the upload never completed. Drop any dangling multipart + // session so its parts don't linger, then fail the row. + if uploadID, err := r.storage.FindUploadID(ctx, key); err == nil { + if err := r.storage.AbortMultipartUpload(ctx, key, uploadID); err != nil { + log.Warn().Err(err).Str("video_id", v.ID).Msg("reconciler: abort multipart failed") + } + } + if err := r.metadata.SetStatus(v.ID, "failed"); err != nil { + log.Error().Err(err).Str("video_id", v.ID).Msg("reconciler: mark failed failed") + return + } + log.Info().Str("video_id", v.ID).Msg("reconciler: marked abandoned upload failed") +} + +// ReconcileOnce runs a single pass (used by tests). +func (r *Reconciler) ReconcileOnce(ctx context.Context) { r.reconcile(ctx) } diff --git a/services/upload/internal/reconciler/reconciler_test.go b/services/upload/internal/reconciler/reconciler_test.go new file mode 100644 index 0000000..4cfe469 --- /dev/null +++ b/services/upload/internal/reconciler/reconciler_test.go @@ -0,0 +1,106 @@ +package reconciler + +import ( + "context" + "errors" + "testing" + "time" + + "flowix/upload/internal/client" +) + +type fakeMeta struct { + stale []client.StaleUpload + statuses map[string]string +} + +func (f *fakeMeta) ListStaleUploaded(time.Duration, int) ([]client.StaleUpload, error) { + return f.stale, nil +} +func (f *fakeMeta) SetStatus(id, status string) error { + f.statuses[id] = status + return nil +} + +type fakeStore struct { + size int64 + statErr error + upload string + aborted []string +} + +func (f *fakeStore) StatObjectSize(context.Context, string) (int64, error) { + return f.size, f.statErr +} +func (f *fakeStore) FindUploadID(context.Context, string) (string, error) { + if f.upload == "" { + return "", errors.New("multipart upload not found") + } + return f.upload, nil +} +func (f *fakeStore) AbortMultipartUpload(_ context.Context, key, id string) error { + f.aborted = append(f.aborted, key+"|"+id) + return nil +} + +type fakePub struct{ published []interface{} } + +func (f *fakePub) Publish(_ context.Context, payload interface{}) error { + f.published = append(f.published, payload) + return nil +} + +func newRec(m *fakeMeta, s *fakeStore, p *fakePub) *Reconciler { + return New(m, s, p, time.Hour, 100) +} + +// Object present → the upload finished but the event was lost: re-publish. +func TestReconcileRepublishesWhenObjectExists(t *testing.T) { + m := &fakeMeta{stale: []client.StaleUpload{{ID: "v1", OwnerID: "o1"}}, statuses: map[string]string{}} + s := &fakeStore{size: 1024} + p := &fakePub{} + newRec(m, s, p).ReconcileOnce(context.Background()) + + if len(p.published) != 1 { + t.Fatalf("want 1 republished event, got %d", len(p.published)) + } + if _, ok := m.statuses["v1"]; ok { + t.Fatalf("must not fail a completed upload, got %v", m.statuses) + } + if len(s.aborted) != 0 { + t.Fatalf("must not abort a completed upload") + } +} + +// No object, dangling multipart session → abort it and fail the row. +func TestReconcileAbortsSessionAndFails(t *testing.T) { + m := &fakeMeta{stale: []client.StaleUpload{{ID: "v2"}}, statuses: map[string]string{}} + s := &fakeStore{statErr: errors.New("not found"), upload: "up-1"} + p := &fakePub{} + newRec(m, s, p).ReconcileOnce(context.Background()) + + if m.statuses["v2"] != "failed" { + t.Fatalf("want failed, got %v", m.statuses) + } + if len(s.aborted) != 1 || s.aborted[0] != "raw/v2/original.mp4|up-1" { + t.Fatalf("want multipart abort, got %v", s.aborted) + } + if len(p.published) != 0 { + t.Fatalf("must not republish an upload with no object") + } +} + +// No object, no session → plain abandoned upload: mark failed. +func TestReconcileFailsAbandonedUpload(t *testing.T) { + m := &fakeMeta{stale: []client.StaleUpload{{ID: "v3"}}, statuses: map[string]string{}} + s := &fakeStore{statErr: errors.New("not found")} + p := &fakePub{} + newRec(m, s, p).ReconcileOnce(context.Background()) + + if m.statuses["v3"] != "failed" { + t.Fatalf("want failed, got %v", m.statuses) + } + if len(s.aborted) != 0 { + t.Fatalf("no session → no abort, got %v", s.aborted) + } +} diff --git a/services/upload/internal/storage/minio.go b/services/upload/internal/storage/minio.go index 85b9a94..b123cfd 100644 --- a/services/upload/internal/storage/minio.go +++ b/services/upload/internal/storage/minio.go @@ -219,6 +219,16 @@ func (m *MinioClient) FindUploadID(ctx context.Context, key string) (string, err return latest, nil } +// AbortMultipartUpload cancels an in-progress session, dropping its parts — +// used by the reconciler to clean up abandoned uploads. +func (m *MinioClient) AbortMultipartUpload(ctx context.Context, key, uploadID string) error { + core := minio.Core{Client: m.client} + if err := core.AbortMultipartUpload(ctx, m.bucket, key, uploadID); err != nil { + return fmt.Errorf("abort multipart upload %s: %w", key, err) + } + return nil +} + // ListParts returns uploaded parts ordered by part number. func (m *MinioClient) ListParts(ctx context.Context, key, uploadID string) ([]MultipartPart, error) { core := minio.Core{Client: m.client}