From 17e7e3602e1804c9f79ba4d99bcdc085abbb54d0 Mon Sep 17 00:00:00 2001 From: SlavaKuntsov Date: Thu, 22 Oct 2026 19:36:02 +0300 Subject: [PATCH 1/4] fix(upload,metadata): self-heal videos stuck in uploaded A video row is created before the bytes land, so an upload that never completes (closed tab, lost resume key, publish failure, recreated queue) stayed in 'uploaded' forever and the player spun endlessly. The upload service now reconciles stale rows: it re-publishes video.uploaded when the original exists in MinIO (lost event) and aborts the multipart session plus marks failed when it does not (abandoned upload), backed by a new internal /internal/videos/stale endpoint and UPLOAD_STALE_TTL_SECONDS / UPLOAD_RECONCILE_INTERVAL_SECONDS knobs. The feed also now hides non-ready videos from everyone but the owner. --- .env.example | 7 ++ deploy/docker-compose.yml | 2 + docs/services-pipeline.md | 2 + services/metadata/cmd/server/main.go | 1 + services/metadata/internal/handler/video.go | 29 +++++ .../metadata/internal/handler/video_test.go | 90 +++++++++++++- services/metadata/internal/model/video.go | 8 ++ .../metadata/internal/repository/video.go | 30 ++++- services/upload/cmd/server/main.go | 17 +++ services/upload/internal/client/metadata.go | 57 +++++++++ .../upload/internal/client/metadata_test.go | 41 +++++++ .../upload/internal/reconciler/reconciler.go | 115 ++++++++++++++++++ .../internal/reconciler/reconciler_test.go | 106 ++++++++++++++++ services/upload/internal/storage/minio.go | 10 ++ 14 files changed, 512 insertions(+), 3 deletions(-) create mode 100644 services/upload/internal/reconciler/reconciler.go create mode 100644 services/upload/internal/reconciler/reconciler_test.go diff --git a/.env.example b/.env.example index 69863e0..1cf4da6 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,13 @@ JWT_REFRESH_TTL=168h INTERNAL_TOKEN=change-me-internal-token-32chars # Upload limit (bytes). 5GB default for Phase 9, Phase 11 will use presigned multipart for bigger files UPLOAD_MAX_BYTES=5368709120 +# Upload reconciler: a video row is created before the bytes land, so an upload +# that never completes (closed tab, lost resume key) would hang in 'uploaded' +# forever. Every UPLOAD_RECONCILE_INTERVAL_SECONDS the upload service looks at +# 'uploaded' rows older than UPLOAD_STALE_TTL_SECONDS: it re-queues them when the +# raw file exists (lost event) or marks them failed (abandoned upload). +UPLOAD_STALE_TTL_SECONDS=3600 +UPLOAD_RECONCILE_INTERVAL_SECONDS=300 # Transcoder (Phase 10) — limit CPU/RAM per job, sequential renditions FFMPEG_THREADS=2 diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index e178a04..4e3c61b 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -239,6 +239,8 @@ services: INTERNAL_TOKEN: ${INTERNAL_TOKEN:?INTERNAL_TOKEN is required — set it in .env (see .env.example)} JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required — set it in .env (see .env.example)} UPLOAD_MAX_BYTES: ${UPLOAD_MAX_BYTES:-5368709120} + UPLOAD_STALE_TTL_SECONDS: ${UPLOAD_STALE_TTL_SECONDS:-3600} + UPLOAD_RECONCILE_INTERVAL_SECONDS: ${UPLOAD_RECONCILE_INTERVAL_SECONDS:-300} MINIO_PUBLIC_ENDPOINT: ${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000} # grace для Shutdown меньше stop_grace_period (issue #63) SHUTDOWN_GRACE: 4m diff --git a/docs/services-pipeline.md b/docs/services-pipeline.md index fdbb05f..ab6a82f 100644 --- a/docs/services-pipeline.md +++ b/docs/services-pipeline.md @@ -95,6 +95,8 @@ Flowix — MVP видеоплатформы с адаптивным стрими `uploaded` → (transcoder забрал) → `processing` → `ready` | `failed`. Frontend поллит `GET /api/v1/videos/:id` пока не `ready`. +Застрявшие загрузки само-лечатся: сервис upload раз в `UPLOAD_RECONCILE_INTERVAL_SECONDS` берёт записи в `uploaded` старше `UPLOAD_STALE_TTL_SECONDS` и переопубликовывает `video.uploaded` (если original есть в MinIO) либо помечает `failed` (загрузка брошена). Лента `GET /api/v1/videos` показывает всем только `public`+`ready`, владелец видит и свои не-`ready` (uploaded/processing/failed) для удаления/перезалива. + ### События (контракты) - `video.uploaded {video_id: UUID, s3_key: string, owner_id: UUID}` diff --git a/services/metadata/cmd/server/main.go b/services/metadata/cmd/server/main.go index a7c931a..66ceecf 100644 --- a/services/metadata/cmd/server/main.go +++ b/services/metadata/cmd/server/main.go @@ -151,6 +151,7 @@ func main() { r.Group(func(r chi.Router) { r.Use(mw.InternalAuth(internalToken)) r.Patch("/internal/videos/{id}/status", vh.UpdateStatus) + r.Get("/internal/videos/stale", vh.ListStale) r.Get("/internal/videos/{id}", vh.GetInternal) r.Get("/internal/videos/{id}/vod", vh.GetVODMapping) }) diff --git a/services/metadata/internal/handler/video.go b/services/metadata/internal/handler/video.go index 473e169..6b88921 100644 --- a/services/metadata/internal/handler/video.go +++ b/services/metadata/internal/handler/video.go @@ -28,6 +28,7 @@ type VideoStore interface { Delete(ctx context.Context, id, ownerID string) error UpdateStatus(ctx context.Context, id string, status model.VideoStatus, renditions []model.Rendition) error UpdateThumbnail(ctx context.Context, id string, thumbnailS3Key string) error + ListStaleUploaded(ctx context.Context, olderThan time.Duration, limit int) ([]model.StaleUpload, error) } // StorageRemover deletes S3 objects (raw + renditions + thumbnail). Nil = skip (tests). @@ -95,6 +96,7 @@ func (h *VideoHandler) Register(r chi.Router) { }) // internal (no auth) for transcoder/upload r.Patch("/internal/videos/{id}/status", h.UpdateStatus) + r.Get("/internal/videos/stale", h.ListStale) r.Get("/internal/videos/{id}", h.GetInternal) r.Get("/internal/videos/{id}/vod", h.GetVODMapping) } @@ -188,6 +190,33 @@ func (h *VideoHandler) getVideo(w http.ResponseWriter, r *http.Request, enforceV writeJSON(w, r, http.StatusOK, v) } +// ListStale godoc +// @Summary List videos stuck in 'uploaded' older than the given age (internal) +// @Description Used by the upload-service reconciler: a video row is created before the bytes land, so an upload that never completes stays in 'uploaded' forever. +// @Tags internal +// @Produce json +// @Param older_than_seconds query int false "Minimum age in seconds" default(3600) +// @Param limit query int false "Max rows (<=500)" default(100) +// @Success 200 {object} map[string]interface{} +// @Router /internal/videos/stale [get] +func (h *VideoHandler) ListStale(w http.ResponseWriter, r *http.Request) { + olderThan := 3600 + if v, err := strconv.Atoi(r.URL.Query().Get("older_than_seconds")); err == nil && v > 0 { + olderThan = v + } + limit := 100 + if v, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && v > 0 && v <= 500 { + limit = v + } + list, err := h.repo.ListStaleUploaded(r.Context(), time.Duration(olderThan)*time.Second, limit) + if err != nil { + slog.Error("list stale uploads failed", "error", err) + writeError(w, r, http.StatusInternalServerError, "internal error") + return + } + writeJSON(w, r, http.StatusOK, map[string]interface{}{"data": list}) +} + type vodMapping struct { Sequences []vodSequence `json:"sequences"` } diff --git a/services/metadata/internal/handler/video_test.go b/services/metadata/internal/handler/video_test.go index c2406a1..d7921ca 100644 --- a/services/metadata/internal/handler/video_test.go +++ b/services/metadata/internal/handler/video_test.go @@ -61,12 +61,15 @@ func (f *fakeStore) GetByID(_ context.Context, id string) (*model.Video, error) func (f *fakeStore) List(_ context.Context, limit, offset int, viewerID string) ([]model.Video, error) { out := []model.Video{} for _, v := range f.videos { - // mirror the SQL visibility filter (issue #44); empty = default public + // mirror the SQL feed filter (issue #44 + stale-upload fix): everyone + // sees only public+ready, the owner also sees their non-ready videos. vis := v.Visibility if vis == "" { vis = model.VisibilityPublic } - if vis != model.VisibilityPublic && (viewerID == "" || v.OwnerID != viewerID) { + isOwner := viewerID != "" && v.OwnerID == viewerID + publicReady := vis == model.VisibilityPublic && v.Status == model.StatusReady + if !publicReady && !isOwner { continue } out = append(out, *v) @@ -139,6 +142,19 @@ func (f *fakeStore) UpdateThumbnail(_ context.Context, id string, thumbnailS3Key v.ThumbnailURL = &u return nil } +func (f *fakeStore) ListStaleUploaded(_ context.Context, olderThan time.Duration, limit int) ([]model.StaleUpload, error) { + out := []model.StaleUpload{} + for _, v := range f.videos { + if v.Status != model.StatusUploaded || time.Since(v.CreatedAt) < olderThan { + continue + } + out = append(out, model.StaleUpload{ID: v.ID, OwnerID: v.OwnerID}) + if limit > 0 && len(out) >= limit { + break + } + } + return out, nil +} // Issue #58: wrappers that simulate a failing DB for handler 500-mapping tests. type failingGetStore struct { @@ -175,6 +191,7 @@ func testRouter(store VideoStore) chi.Router { r.With(optAuth).Get("/api/v1/videos", vh.List) r.With(optAuth).Get("/api/v1/videos/{id}", vh.Get) r.Patch("/internal/videos/{id}/status", vh.UpdateStatus) + r.Get("/internal/videos/stale", vh.ListStale) r.Get("/internal/videos/{id}", vh.GetInternal) r.Get("/internal/videos/{id}/vod", vh.GetVODMapping) return r @@ -344,6 +361,51 @@ func TestGetAndList(t *testing.T) { } } +// Feed must hide non-ready videos from everyone but their owner. +func TestListHidesNonReadyFromOthers(t *testing.T) { + store := newFake() + store.videos["pub-ready"] = &model.Video{ID: "pub-ready", OwnerID: "o1", Title: "a", Status: model.StatusReady, Visibility: model.VisibilityPublic} + store.videos["pub-failed"] = &model.Video{ID: "pub-failed", OwnerID: "o1", Title: "b", Status: model.StatusFailed, Visibility: model.VisibilityPublic} + store.videos["pub-uploaded"] = &model.Video{ID: "pub-uploaded", OwnerID: "o1", Title: "c", Status: model.StatusUploaded, Visibility: model.VisibilityPublic} + r := testRouter(store) + + ids := func(token string) map[string]bool { + req := httptest.NewRequest("GET", "/api/v1/videos?limit=50&offset=0", nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("want 200 got %d %s", w.Code, w.Body.String()) + } + var out struct { + Data []model.Video `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, v := range out.Data { + got[v.ID] = true + } + return got + } + + anon := ids("") + if !anon["pub-ready"] || anon["pub-failed"] || anon["pub-uploaded"] { + t.Fatalf("anonymous feed must show only ready: %v", anon) + } + owner := ids(signToken("o1")) + if !owner["pub-ready"] || !owner["pub-failed"] || !owner["pub-uploaded"] { + t.Fatalf("owner must see own non-ready videos: %v", owner) + } + other := ids(signToken("o2")) + if other["pub-failed"] || other["pub-uploaded"] { + t.Fatalf("other user must not see non-ready videos: %v", other) + } +} + func TestUpdateForbidden(t *testing.T) { store := newFake() store.videos["vid-1"] = &model.Video{ID: "vid-1", OwnerID: "owner1", Title: "t", Status: model.StatusUploaded} @@ -370,6 +432,30 @@ func TestUpdateForbidden(t *testing.T) { } } +// Reconciler endpoint: only 'uploaded' rows older than the threshold are listed. +func TestListStaleUploads(t *testing.T) { + store := newFake() + store.videos["old"] = &model.Video{ID: "old", OwnerID: "o1", Title: "t", Status: model.StatusUploaded, CreatedAt: time.Now().Add(-2 * time.Hour)} + store.videos["fresh"] = &model.Video{ID: "fresh", OwnerID: "o1", Title: "t", Status: model.StatusUploaded, CreatedAt: time.Now()} + store.videos["done"] = &model.Video{ID: "done", OwnerID: "o1", Title: "t", Status: model.StatusReady, CreatedAt: time.Now().Add(-2 * time.Hour)} + r := testRouter(store) + req := httptest.NewRequest("GET", "/internal/videos/stale?older_than_seconds=3600&limit=10", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("want 200 got %d %s", w.Code, w.Body.String()) + } + var out struct { + Data []model.StaleUpload `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if len(out.Data) != 1 || out.Data[0].ID != "old" || out.Data[0].OwnerID != "o1" { + t.Fatalf("want only 'old', got %+v", out.Data) + } +} + func TestDelete(t *testing.T) { store := newFake() store.videos["vid-1"] = &model.Video{ID: "vid-1", OwnerID: "owner1", Title: "t", Status: model.StatusUploaded} diff --git a/services/metadata/internal/model/video.go b/services/metadata/internal/model/video.go index d307d08..2b4e6e5 100644 --- a/services/metadata/internal/model/video.go +++ b/services/metadata/internal/model/video.go @@ -40,6 +40,14 @@ type Video struct { Renditions []Rendition `json:"renditions,omitempty"` } +// StaleUpload is a minimal view of a video still in 'uploaded' — the +// upload-service reconciler uses it to re-queue a lost event or fail an +// abandoned upload (a row is created before the bytes land). +type StaleUpload struct { + ID string `json:"id"` + OwnerID string `json:"owner_id"` +} + type Rendition struct { VideoID string `json:"video_id"` Quality string `json:"quality"` diff --git a/services/metadata/internal/repository/video.go b/services/metadata/internal/repository/video.go index 4abffef..28affd7 100644 --- a/services/metadata/internal/repository/video.go +++ b/services/metadata/internal/repository/video.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "time" "flowix/metadata/internal/model" "github.com/google/uuid" @@ -81,7 +82,10 @@ func (r *VideoRepo) GetByID(ctx context.Context, id string) (*model.Video, error func (r *VideoRepo) List(ctx context.Context, limit, offset int, viewerID string) ([]model.Video, error) { // $3 участвует и в текстовом сравнении ($3 <> ''), и в сравнении с uuid-колонкой: // без ::uuid Postgres выводит тип $3 как text и падает с SQLSTATE 42883 (uuid = text) - q := `SELECT v.id, v.owner_id, u.email, v.title, v.description, v.duration, v.status, COALESCE(v.visibility::text,'public'), v.thumbnail_s3_key, v.created_at, v.updated_at FROM videos v LEFT JOIN users u ON u.id=v.owner_id WHERE v.visibility='public' OR ($3 <> '' AND v.owner_id=$3::uuid) ORDER BY v.created_at DESC LIMIT $1 OFFSET $2` + // Лента: всем — только public+ready (неготовые/упавшие не светятся чужим); + // владелец видит свои в любом статусе (uploaded/processing/failed), чтобы + // удалить или перезалить (issue: зомби-видео засоряли ленту). + q := `SELECT v.id, v.owner_id, u.email, v.title, v.description, v.duration, v.status, COALESCE(v.visibility::text,'public'), v.thumbnail_s3_key, v.created_at, v.updated_at FROM videos v LEFT JOIN users u ON u.id=v.owner_id WHERE (v.visibility='public' AND v.status='ready') OR ($3 <> '' AND v.owner_id=$3::uuid) ORDER BY v.created_at DESC LIMIT $1 OFFSET $2` rows, err := r.pool.Query(ctx, q, limit, offset, viewerID) if err != nil { return nil, err @@ -153,6 +157,30 @@ func (r *VideoRepo) UpdateThumbnail(ctx context.Context, id string, thumbnailS3K return err } +// ListStaleUploaded returns videos still in 'uploaded' created more than +// olderThan ago. A row is inserted before the bytes are uploaded, so a video +// whose upload never completes sits in 'uploaded' forever; the upload-service +// reconciler uses this list to re-queue or fail such rows. +func (r *VideoRepo) ListStaleUploaded(ctx context.Context, olderThan time.Duration, limit int) ([]model.StaleUpload, error) { + if limit <= 0 || limit > 500 { + limit = 100 + } + rows, err := r.pool.Query(ctx, `SELECT id, owner_id FROM videos WHERE status='uploaded' AND created_at < now() - ($1::int * interval '1 second') ORDER BY created_at ASC LIMIT $2`, int(olderThan.Seconds()), limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := []model.StaleUpload{} + for rows.Next() { + var s model.StaleUpload + if err := rows.Scan(&s.ID, &s.OwnerID); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + func (r *VideoRepo) UpdateStatus(ctx context.Context, id string, status model.VideoStatus, renditions []model.Rendition) error { tx, err := r.pool.Begin(ctx) if err != nil { diff --git a/services/upload/cmd/server/main.go b/services/upload/cmd/server/main.go index 55055b0..f44e68f 100644 --- a/services/upload/cmd/server/main.go +++ b/services/upload/cmd/server/main.go @@ -9,10 +9,12 @@ package main // @name Authorization // @description Type "Bearer {token}" import ( + "context" "log" "net/http" "os" "strconv" + "time" pkghttp "flowix/pkg/httpserver" pkglogger "flowix/pkg/logger" @@ -22,6 +24,7 @@ import ( "flowix/upload/internal/client" "flowix/upload/internal/handler" "flowix/upload/internal/queue" + "flowix/upload/internal/reconciler" "flowix/upload/internal/storage" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -73,6 +76,12 @@ func main() { rh := handler.NewResumableHandler(store, metaCl) mp := handler.NewMultipartHandler(store, pub, metaCl, metaCl) + // Self-heal videos stuck in 'uploaded': an upload row is created before the + // bytes land, so an upload that never completes hangs forever (see package + // reconciler). Runs for the lifetime of the process. + rec := reconciler.New(metaCl, store, pub, secondsEnv("UPLOAD_STALE_TTL_SECONDS", 3600), 100) + go rec.Run(context.Background(), secondsEnv("UPLOAD_RECONCILE_INTERVAL_SECONDS", 300)) + r := chi.NewRouter() r.Use(middleware.RequestID, middleware.Recoverer, pkgmw.RequestLogger("upload")) @@ -119,3 +128,11 @@ func requireEnv(k string) string { } return v } + +// secondsEnv reads a positive number of seconds from env, falling back to def. +func secondsEnv(k string, def int) time.Duration { + if n, err := strconv.Atoi(os.Getenv(k)); err == nil && n > 0 { + return time.Duration(n) * time.Second + } + return time.Duration(def) * time.Second +} diff --git a/services/upload/internal/client/metadata.go b/services/upload/internal/client/metadata.go index db80b51..860522f 100644 --- a/services/upload/internal/client/metadata.go +++ b/services/upload/internal/client/metadata.go @@ -59,6 +59,63 @@ func (c *MetadataClient) CreateVideo(token, title, description string) (string, return out.ID, nil } +// StaleUpload is a video stuck in 'uploaded' as returned by metadata +// GET /internal/videos/stale (id + owner for re-publishing video.uploaded). +type StaleUpload struct { + ID string `json:"id"` + OwnerID string `json:"owner_id"` +} + +// ListStaleUploaded returns videos still in 'uploaded' created more than +// olderThan ago — the reconciler filters out the ones that are genuinely done. +func (c *MetadataClient) ListStaleUploaded(olderThan time.Duration, limit int) ([]StaleUpload, error) { + u := fmt.Sprintf("%s/internal/videos/stale?older_than_seconds=%d&limit=%d", c.baseURL, int(olderThan.Seconds()), limit) + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + if c.internalToken != "" { + req.Header.Set("X-Internal-Token", c.internalToken) + } + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("metadata stale uploads: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("metadata stale uploads status %d", resp.StatusCode) + } + var out struct { + Data []StaleUpload `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, err + } + return out.Data, nil +} + +// SetStatus updates a video status via the metadata internal API. +func (c *MetadataClient) SetStatus(videoID, status string) error { + body, _ := json.Marshal(map[string]string{"status": status}) + req, err := http.NewRequest("PATCH", c.baseURL+"/internal/videos/"+url.PathEscape(videoID)+"/status", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if c.internalToken != "" { + req.Header.Set("X-Internal-Token", c.internalToken) + } + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("metadata set status: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("metadata set status %d", resp.StatusCode) + } + return nil +} + // GetVideoOwner resolves the owner of a video via metadata internal API // (upload handlers must verify ownership before writing raw/{id}/* — issue #46). func (c *MetadataClient) GetVideoOwner(videoID string) (string, error) { diff --git a/services/upload/internal/client/metadata_test.go b/services/upload/internal/client/metadata_test.go index 43adb17..d8d10ed 100644 --- a/services/upload/internal/client/metadata_test.go +++ b/services/upload/internal/client/metadata_test.go @@ -5,6 +5,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) func TestGetVideoOwner(t *testing.T) { @@ -74,3 +75,43 @@ func TestGetVideoOwnerOmitsEmptyToken(t *testing.T) { t.Fatal("X-Internal-Token must be omitted when internalToken is empty") } } + +func TestListStaleUploadedAndSetStatus(t *testing.T) { + var gotQuery, gotToken, gotMethod string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotToken = r.Header.Get("X-Internal-Token") + gotMethod = r.Method + switch r.URL.Path { + case "/internal/videos/stale": + gotQuery = r.URL.RawQuery + _, _ = w.Write([]byte(`{"data":[{"id":"v1","owner_id":"o1"}]}`)) + case "/internal/videos/v1/status": + _, _ = w.Write([]byte(`{"status":"ok"}`)) + default: + http.Error(w, "not found", http.StatusNotFound) + } + })) + defer srv.Close() + + c := NewMetadataClient(srv.URL, "tok") + list, err := c.ListStaleUploaded(2*time.Hour, 50) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(list) != 1 || list[0].ID != "v1" || list[0].OwnerID != "o1" { + t.Fatalf("unexpected stale list %+v", list) + } + if gotQuery != "older_than_seconds=7200&limit=50" { + t.Fatalf("query = %q", gotQuery) + } + if gotToken != "tok" { + t.Fatalf("X-Internal-Token = %q", gotToken) + } + + if err := c.SetStatus("v1", "failed"); err != nil { + t.Fatalf("unexpected err: %v", err) + } + if gotMethod != "PATCH" { + t.Fatalf("method = %q, want PATCH", gotMethod) + } +} diff --git a/services/upload/internal/reconciler/reconciler.go b/services/upload/internal/reconciler/reconciler.go new file mode 100644 index 0000000..84d4642 --- /dev/null +++ b/services/upload/internal/reconciler/reconciler.go @@ -0,0 +1,115 @@ +// Package reconciler self-heals videos stuck in status 'uploaded'. +// +// Every upload path creates the metadata row before the bytes land, and the +// only thing that moves a row out of 'uploaded' is the transcoder consuming the +// video.uploaded event. So an upload that never completes (closed tab, lost +// resume key, network drop, publish failure, a queue that was recreated) leaves +// a row that hangs forever — the UI then shows an endless "Transcoding…". +// +// The reconciler periodically looks at 'uploaded' rows older than ttl and: +// - re-publishes video.uploaded when the raw object exists (a lost event), or +// - aborts any dangling multipart session and marks the row failed when it +// does not (an abandoned upload). +package reconciler + +import ( + "context" + "time" + + "flowix/upload/internal/client" + "flowix/upload/internal/handler" + "github.com/rs/zerolog/log" +) + +// Metadata lists stale uploads and can fail them. +type Metadata interface { + ListStaleUploaded(olderThan time.Duration, limit int) ([]client.StaleUpload, error) + SetStatus(videoID, status string) error +} + +// Storage inspects and cleans the raw object / multipart session. +type Storage interface { + StatObjectSize(ctx context.Context, key string) (int64, error) + FindUploadID(ctx context.Context, key string) (string, error) + AbortMultipartUpload(ctx context.Context, key, uploadID string) error +} + +// Publisher re-emits video.uploaded (implemented by queue.Publisher). +type Publisher interface { + Publish(ctx context.Context, payload interface{}) error +} + +// Reconciler periodically reconciles stale 'uploaded' rows. +type Reconciler struct { + metadata Metadata + storage Storage + publisher Publisher + ttl time.Duration + limit int +} + +func New(m Metadata, s Storage, p Publisher, ttl time.Duration, limit int) *Reconciler { + if limit <= 0 { + limit = 100 + } + return &Reconciler{metadata: m, storage: s, publisher: p, ttl: ttl, limit: limit} +} + +// Run reconciles every interval until ctx is cancelled. +func (r *Reconciler) Run(ctx context.Context, interval time.Duration) { + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + r.reconcile(ctx) + } + } +} + +func rawKey(videoID string) string { return "raw/" + videoID + "/original.mp4" } + +// reconcile is exported for tests via reconcileOnce. +func (r *Reconciler) reconcile(ctx context.Context) { + stale, err := r.metadata.ListStaleUploaded(r.ttl, r.limit) + if err != nil { + log.Error().Err(err).Msg("reconciler: list stale uploads failed") + return + } + for _, v := range stale { + r.handle(ctx, v) + } +} + +func (r *Reconciler) handle(ctx context.Context, v client.StaleUpload) { + key := rawKey(v.ID) + // Object present → the upload finished but the event never reached the + // queue (publish failure / queue recreated). Re-publish; the transcoder + // skips videos already ready/failed. + if size, err := r.storage.StatObjectSize(ctx, key); err == nil && size > 0 { + ev := handler.VideoUploadedEvent{VideoID: v.ID, S3Key: key, OwnerID: v.OwnerID} + if err := r.publisher.Publish(ctx, ev); err != nil { + log.Error().Err(err).Str("video_id", v.ID).Msg("reconciler: republish failed") + return + } + log.Info().Str("video_id", v.ID).Msg("reconciler: re-published lost video.uploaded") + return + } + // No object → the upload never completed. Drop any dangling multipart + // session so its parts don't linger, then fail the row. + if uploadID, err := r.storage.FindUploadID(ctx, key); err == nil { + if err := r.storage.AbortMultipartUpload(ctx, key, uploadID); err != nil { + log.Warn().Err(err).Str("video_id", v.ID).Msg("reconciler: abort multipart failed") + } + } + if err := r.metadata.SetStatus(v.ID, "failed"); err != nil { + log.Error().Err(err).Str("video_id", v.ID).Msg("reconciler: mark failed failed") + return + } + log.Info().Str("video_id", v.ID).Msg("reconciler: marked abandoned upload failed") +} + +// ReconcileOnce runs a single pass (used by tests). +func (r *Reconciler) ReconcileOnce(ctx context.Context) { r.reconcile(ctx) } diff --git a/services/upload/internal/reconciler/reconciler_test.go b/services/upload/internal/reconciler/reconciler_test.go new file mode 100644 index 0000000..4cfe469 --- /dev/null +++ b/services/upload/internal/reconciler/reconciler_test.go @@ -0,0 +1,106 @@ +package reconciler + +import ( + "context" + "errors" + "testing" + "time" + + "flowix/upload/internal/client" +) + +type fakeMeta struct { + stale []client.StaleUpload + statuses map[string]string +} + +func (f *fakeMeta) ListStaleUploaded(time.Duration, int) ([]client.StaleUpload, error) { + return f.stale, nil +} +func (f *fakeMeta) SetStatus(id, status string) error { + f.statuses[id] = status + return nil +} + +type fakeStore struct { + size int64 + statErr error + upload string + aborted []string +} + +func (f *fakeStore) StatObjectSize(context.Context, string) (int64, error) { + return f.size, f.statErr +} +func (f *fakeStore) FindUploadID(context.Context, string) (string, error) { + if f.upload == "" { + return "", errors.New("multipart upload not found") + } + return f.upload, nil +} +func (f *fakeStore) AbortMultipartUpload(_ context.Context, key, id string) error { + f.aborted = append(f.aborted, key+"|"+id) + return nil +} + +type fakePub struct{ published []interface{} } + +func (f *fakePub) Publish(_ context.Context, payload interface{}) error { + f.published = append(f.published, payload) + return nil +} + +func newRec(m *fakeMeta, s *fakeStore, p *fakePub) *Reconciler { + return New(m, s, p, time.Hour, 100) +} + +// Object present → the upload finished but the event was lost: re-publish. +func TestReconcileRepublishesWhenObjectExists(t *testing.T) { + m := &fakeMeta{stale: []client.StaleUpload{{ID: "v1", OwnerID: "o1"}}, statuses: map[string]string{}} + s := &fakeStore{size: 1024} + p := &fakePub{} + newRec(m, s, p).ReconcileOnce(context.Background()) + + if len(p.published) != 1 { + t.Fatalf("want 1 republished event, got %d", len(p.published)) + } + if _, ok := m.statuses["v1"]; ok { + t.Fatalf("must not fail a completed upload, got %v", m.statuses) + } + if len(s.aborted) != 0 { + t.Fatalf("must not abort a completed upload") + } +} + +// No object, dangling multipart session → abort it and fail the row. +func TestReconcileAbortsSessionAndFails(t *testing.T) { + m := &fakeMeta{stale: []client.StaleUpload{{ID: "v2"}}, statuses: map[string]string{}} + s := &fakeStore{statErr: errors.New("not found"), upload: "up-1"} + p := &fakePub{} + newRec(m, s, p).ReconcileOnce(context.Background()) + + if m.statuses["v2"] != "failed" { + t.Fatalf("want failed, got %v", m.statuses) + } + if len(s.aborted) != 1 || s.aborted[0] != "raw/v2/original.mp4|up-1" { + t.Fatalf("want multipart abort, got %v", s.aborted) + } + if len(p.published) != 0 { + t.Fatalf("must not republish an upload with no object") + } +} + +// No object, no session → plain abandoned upload: mark failed. +func TestReconcileFailsAbandonedUpload(t *testing.T) { + m := &fakeMeta{stale: []client.StaleUpload{{ID: "v3"}}, statuses: map[string]string{}} + s := &fakeStore{statErr: errors.New("not found")} + p := &fakePub{} + newRec(m, s, p).ReconcileOnce(context.Background()) + + if m.statuses["v3"] != "failed" { + t.Fatalf("want failed, got %v", m.statuses) + } + if len(s.aborted) != 0 { + t.Fatalf("no session → no abort, got %v", s.aborted) + } +} diff --git a/services/upload/internal/storage/minio.go b/services/upload/internal/storage/minio.go index 85b9a94..b123cfd 100644 --- a/services/upload/internal/storage/minio.go +++ b/services/upload/internal/storage/minio.go @@ -219,6 +219,16 @@ func (m *MinioClient) FindUploadID(ctx context.Context, key string) (string, err return latest, nil } +// AbortMultipartUpload cancels an in-progress session, dropping its parts — +// used by the reconciler to clean up abandoned uploads. +func (m *MinioClient) AbortMultipartUpload(ctx context.Context, key, uploadID string) error { + core := minio.Core{Client: m.client} + if err := core.AbortMultipartUpload(ctx, m.bucket, key, uploadID); err != nil { + return fmt.Errorf("abort multipart upload %s: %w", key, err) + } + return nil +} + // ListParts returns uploaded parts ordered by part number. func (m *MinioClient) ListParts(ctx context.Context, key, uploadID string) ([]MultipartPart, error) { core := minio.Core{Client: m.client} From 3f2acec4212000242fcb510397bd3d1e74822e4e Mon Sep 17 00:00:00 2001 From: SlavaKuntsov Date: Thu, 22 Oct 2026 19:36:06 +0300 Subject: [PATCH 2/4] fix(frontend): show waiting-for-upload state on watch page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While a video is in 'uploaded' the player said "Transcoding…", which hid that the upload itself had not finished (or was abandoned). Label the state honestly so the stuck-upload case is not mistaken for transcoding. --- frontend/src/app/watch/[id]/page.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/watch/[id]/page.tsx b/frontend/src/app/watch/[id]/page.tsx index 35ebe3c..5d44939 100644 --- a/frontend/src/app/watch/[id]/page.tsx +++ b/frontend/src/app/watch/[id]/page.tsx @@ -185,7 +185,8 @@ export default function WatchPage() { <> - Transcoding… {video.status} + {video.status === "uploaded" ? "Waiting for upload…" : "Transcoding…"}{" "} + {video.status} ) : video.status === "failed" ? ( From 600d81b6194bb936228032b9f8994722c562c9e5 Mon Sep 17 00:00:00 2001 From: SlavaKuntsov Date: Thu, 22 Oct 2026 19:36:10 +0300 Subject: [PATCH 3/4] test(repo): complete resumed upload in e2e The interrupted-resume case never called /complete, so every e2e run left a video stuck in 'uploaded'. Finish the upload so the test does not pollute the feed with abandoned rows. --- scripts/e2e.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/e2e.sh b/scripts/e2e.sh index ecf2350..08353f9 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -500,6 +500,9 @@ if [ -n "${TOKEN:-}" ]; then code=$(curl -s -o "$TMP/body" -w '%{http_code}' -X PUT --data-binary "@$TMP/part2" -H "Authorization: Bearer $TOKEN" -H "Content-Type: video/mp4" -H "Content-Range: bytes $half-$end2/$total" "$GATEWAY/api/v1/videos/$RESUME2_ID/resumable") [ "$code" = "200" ] || fail "second half resumable PUT not 200: $code $(cat "$TMP/body")" say " 50% resume: ok (308→200)" + # complete so the video is queued and does not linger in 'uploaded' + code=$(curl -s -o "$TMP/body" -w '%{http_code}' -X POST "$GATEWAY/api/v1/videos/$RESUME2_ID/complete" -H "Authorization: Bearer $TOKEN") + [ "$code" = "200" ] || say " WARN: resume2 complete not 200: $code $(cat "$TMP/body")" else say " WARN: presign for resumable failed ($code)" fi From b486a38057b9e26b7e0f1de35cfc6d630194f568 Mon Sep 17 00:00:00 2001 From: SlavaKuntsov Date: Thu, 22 Oct 2026 20:15:00 +0300 Subject: [PATCH 4/4] ci: pin python 3.14 for uv requires-python = ">=3.14" has no upper bound and the repo had no .python-version, so uv selected the newest interpreter (cpython-3.15.1) on clean CI runners. psycopg-binary 3.3.4 ships only cp314 wheels, so installing the auth env failed and took lint-py/test-py down (main red since 2026-10-09). Pin both services to 3.14, matching the python:3.14-slim images and the py314 black/mypy/ruff targets. Locally the breakage was masked because a 3.14 interpreter was already installed. --- services/auth/.python-version | 1 + services/transcoder/.python-version | 1 + 2 files changed, 2 insertions(+) create mode 100644 services/auth/.python-version create mode 100644 services/transcoder/.python-version diff --git a/services/auth/.python-version b/services/auth/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/services/auth/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/services/transcoder/.python-version b/services/transcoder/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/services/transcoder/.python-version @@ -0,0 +1 @@ +3.14