diff --git a/Makefile b/Makefile index b21e5b3..f3f9ff8 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ gc-acceptance: GIT_EXAMPLE_DIR = examples/git WAL_COMPONENT = examples/wal-component/Cargo.toml -GIT_TEST_FILES = access.go access_test.go auth.go auth_test.go repositories.go repositories_test.go imports.go imports_test.go receive_transport.go receive_transport_test.go request_config.go request_config_test.go import_pack.go import_pack_test.go import_pack_lifecycle_test.go limits.go limits_test.go index.go index_test.go codec.go codec_test.go delta.go delta_test.go read_retry.go read_retry_test.go retention.go retention_test.go retry.go retry_test.go fetch_policy.go fetch_policy_test.go validate_objects.go validate_objects_test.go validate.go validate_refname.go validate_refname_test.go validate_native_test.go +GIT_TEST_FILES = access.go access_test.go auth.go auth_test.go repositories.go repositories_test.go imports.go imports_test.go receive_transport.go receive_transport_test.go receive_progress.go receive_progress_test.go request_config.go request_config_test.go import_pack.go import_pack_test.go import_pack_lifecycle_test.go limits.go limits_test.go index.go index_test.go codec.go codec_test.go delta.go delta_test.go read_retry.go read_retry_test.go retention.go retention_test.go retry.go retry_test.go fetch_policy.go fetch_policy_test.go validate_objects.go validate_objects_test.go validate.go validate_refname.go validate_refname_test.go validate_native_test.go git-check: git-qualification-tools-test test -z "$$(gofmt -l $(GIT_EXAMPLE_DIR)/*.go $(GIT_EXAMPLE_DIR)/tests/*.go)" diff --git a/examples/git/README.md b/examples/git/README.md index 7fd930c..2c7640a 100644 --- a/examples/git/README.md +++ b/examples/git/README.md @@ -66,6 +66,12 @@ git clone http://127.0.0.1:19100/team/demo.git cloned-demo Git defaults to SHA-1. To use SHA-256, initialize the local repository with `git init --object-format=sha256`; its first push selects the remote format. +Push progress shows pack checks, imported-object counts, delta indexing, +validation, and publication. Import counts update about once a second as objects +finish; final push status follows durable publication. Use `git push --progress` +to request feedback when stderr is redirected. `-q` and `--no-progress` suppress +these messages. + Press Ctrl-C in the service terminal to stop Spin and MinIO and remove all demo data. Set `OBJECT_LOG_GIT_LOCAL_PORT` to choose another loopback port. To keep data or use an existing backend, build with `make git-build`, configure diff --git a/examples/git/main.go b/examples/git/main.go index 6d2a80f..2b2bf21 100644 --- a/examples/git/main.go +++ b/examples/git/main.go @@ -45,7 +45,10 @@ func advertise(w io.Writer, s *store, unknownFormat bool) error { return err } adv := &packp.AdvRefs{} - for _, feature := range []string{capability.ReportStatus, capability.DeleteRefs, capability.OFSDelta, capability.Atomic, capability.NoThin} { + for _, feature := range []string{ + capability.ReportStatus, capability.DeleteRefs, capability.OFSDelta, + capability.Atomic, capability.NoThin, capability.Sideband64k, capability.Quiet, + } { adv.Capabilities.Add(feature) } adv.Capabilities.Set(capability.ObjectFormat, s.meta.Format.String()) @@ -166,8 +169,9 @@ func serve(response http.ResponseWriter, r *http.Request) { http.Error(response, err.Error(), operationStatus(err)) return } + var pushCapabilities *capability.List if service == transport.ReceivePackService && method == http.MethodPost { - format, body, err := receiveFormat(r.Body, limits.negotiationBytes) + format, capabilities, body, err := receiveFormat(r.Body, limits.negotiationBytes) if err == nil && format != "" && route.Repository.Format != "" && route.Repository.Format != format { err = config.ErrInvalidObjectFormat } @@ -186,6 +190,7 @@ func serve(response http.ResponseWriter, r *http.Request) { return } route.Repository.Format = format + pushCapabilities = capabilities r.Body = gitio.NewReadCloser(body, r.Body) } @@ -352,12 +357,19 @@ func serve(response http.ResponseWriter, r *http.Request) { e = advertise(w, s, s.stateRoot == nil && route.Repository.Format == "") } else if service == transport.ReceivePackService { w.Header().Set("Content-Type", "application/x-git-receive-pack-result") + s.progress = newReceiveProgress(w, pushCapabilities, cancel) commands := &commandReader{LimitedReader: io.LimitedReader{R: r.Body, N: limits.negotiationBytes}} push := &receiveStore{Storer: s, commands: commands} e = transport.ReceivePack(r.Context(), push, gitio.NewReadCloser(commands, r.Body), gitio.WriteNopCloser(w), &transport.ReceivePackRequest{StatelessRPC: true, Hooks: transport.ReceivePackHooks{PreReceive: func(_ context.Context, info *transport.PreReceiveInfo) error { if len(info.Commands) == 0 { return nil } + if s.progress != nil { + s.progress.writer = info.Progress + } + if err := s.progress.message("Validating update...\n"); err != nil { + return err + } refs, e := validate(s, info.Commands) if e != nil { return e @@ -370,6 +382,9 @@ func serve(response http.ResponseWriter, r *http.Request) { } } } + if err := s.progress.message("Publishing update...\n"); err != nil { + return err + } e = s.publish(refs) return e }}}) diff --git a/examples/git/receive.go b/examples/git/receive.go index f246b66..7689871 100644 --- a/examples/git/receive.go +++ b/examples/git/receive.go @@ -46,6 +46,9 @@ func (p *incomingPack) Close() (err error) { if p.err != nil { return p.err } + if err := p.s.progress.message("Checking received pack...\n"); err != nil { + return err + } root, err := p.finish() if err != nil { return err @@ -58,7 +61,10 @@ func (p *incomingPack) Close() (err error) { } defer reader.Close() offsets := packOffsets{} - if err := importPack(p.s.ctx, reader, p.s, p.s.meta.Format, p.s.limits, offsets); err != nil { + if err := importPack(p.s.ctx, reader, p.s, p.s.meta.Format, p.s.limits, offsets, p.s.progress); err != nil { + return err + } + if err := p.s.progress.message("Indexing deltas...\n"); err != nil { return err } remaining := p.s.limits.catalogBytes diff --git a/examples/git/receive_progress.go b/examples/git/receive_progress.go new file mode 100644 index 0000000..bd9adba --- /dev/null +++ b/examples/git/receive_progress.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/protocol/capability" + "github.com/go-git/go-git/v6/plumbing/protocol/packp/sideband" +) + +// ReceivePack creates its hook progress writer after pack import. Use the same +// upstream mux for import, then its hook writer for validation and publication. +type receiveProgress struct { + writer io.Writer + cancel context.CancelFunc + total, done uint32 + last time.Time + err error +} + +func newReceiveProgress(w io.Writer, caps *capability.List, cancel context.CancelFunc) *receiveProgress { + if caps == nil || !caps.Supports(capability.Sideband64k) { + return nil + } + if caps.Supports(capability.Quiet) || caps.Supports(capability.NoProgress) { + return nil + } + return &receiveProgress{ + writer: progressChannel{mux: sideband.NewMuxer(sideband.Sideband64k, w)}, + cancel: cancel, + } +} + +type progressChannel struct{ mux *sideband.Muxer } + +func (w progressChannel) Write(data []byte) (int, error) { + return w.mux.WriteChannel(sideband.ProgressMessage, data) +} + +func (p *receiveProgress) message(text string) error { + if p == nil { + return nil + } + if p.err == nil { + _, p.err = io.WriteString(p.writer, text) + if p.err != nil && p.cancel != nil { + p.cancel() + } + p.last = time.Now() + } + return p.err +} + +func (p *receiveProgress) OnHeader(total uint32) error { + if p == nil { + return nil + } + p.total = total + return p.message(fmt.Sprintf("Importing objects: 0/%d\n", total)) +} + +func (p *receiveProgress) OnInflatedObjectHeader(plumbing.ObjectType, int64, int64) error { + if p == nil { + return nil + } + return p.err +} + +func (p *receiveProgress) OnInflatedObjectContent(plumbing.Hash, int64, uint32, []byte) error { + if p == nil { + return nil + } + p.done++ + if time.Since(p.last) < time.Second { + return p.err + } + return p.message(fmt.Sprintf("Importing objects: %d/%d\r", p.done, p.total)) +} + +func (p *receiveProgress) OnFooter(plumbing.Hash) error { + if p == nil { + return nil + } + return p.message(fmt.Sprintf("Importing objects: %d/%d\n", p.done, p.total)) +} diff --git a/examples/git/receive_progress_test.go b/examples/git/receive_progress_test.go new file mode 100644 index 0000000..2cbd08e --- /dev/null +++ b/examples/git/receive_progress_test.go @@ -0,0 +1,299 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "testing/synctest" + "time" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/format/config" + "github.com/go-git/go-git/v6/plumbing/format/pktline" + "github.com/go-git/go-git/v6/plumbing/protocol/capability" + "github.com/go-git/go-git/v6/plumbing/protocol/packp" + "github.com/go-git/go-git/v6/plumbing/protocol/packp/sideband" + "github.com/go-git/go-git/v6/plumbing/transport" + "github.com/go-git/go-git/v6/storage/memory" + gitio "github.com/go-git/go-git/v6/utils/ioutil" +) + +func TestReceiveProgressNegotiation(t *testing.T) { + for _, tc := range []struct { + name, caps string + progress bool + }{ + {name: "unnegotiated", caps: "report-status"}, + {name: "quiet", caps: "side-band-64k quiet"}, + {name: "no progress", caps: "side-band-64k no-progress"}, + {name: "sideband 64k", caps: "side-band-64k", progress: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var output bytes.Buffer + var caps capability.List + capability.DecodeList([]byte(tc.caps), &caps) + progress := newReceiveProgress(&output, &caps, nil) + if err := progress.message("Checking received pack...\n"); err != nil { + t.Fatal(err) + } + if !tc.progress { + if output.Len() != 0 { + t.Fatalf("unrequested progress: %q", output.String()) + } + return + } + scanner := pktline.NewScanner(&output) + if !scanner.Scan() || string(scanner.Bytes()) != "\x02Checking received pack...\n" { + t.Fatalf("invalid progress packet: %q, %v", scanner.Bytes(), scanner.Err()) + } + if scanner.Scan() || scanner.Err() != nil { + t.Fatalf("progress terminated the receive response: %v", scanner.Err()) + } + }) + } +} + +func TestReceiveProgressCountsImportedDeltaObjects(t *testing.T) { + for _, format := range []config.ObjectFormat{config.SHA1, config.SHA256} { + t.Run(format.String(), func(t *testing.T) { + var output bytes.Buffer + progress := &receiveProgress{writer: &output} + pack := fixturePack(t, format, []packFixtureEntry{ + {kind: plumbing.BlobObject, data: []byte("abc")}, + {kind: plumbing.OFSDeltaObject, data: []byte{3, 1, 0x90, 1}, ofs: 0}, + {kind: plumbing.REFDeltaObject, data: []byte{1, 1, 1, 'b'}, ref: blobID(format, []byte("a"))}, + }) + if err := importPack(t.Context(), bytes.NewReader(pack), newImportStorage(format), format, testPackLimits(1024), progress); err != nil { + t.Fatal(err) + } + if output.String() != "Importing objects: 0/3\nImporting objects: 3/3\n" { + t.Fatalf("inaccurate imported-object progress: %q", output.String()) + } + }) + } +} + +func TestReceiveProgressThrottlesObjectMessages(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var output bytes.Buffer + progress := &receiveProgress{writer: &output} + if err := progress.OnHeader(3); err != nil { + t.Fatal(err) + } + if err := progress.OnInflatedObjectContent(plumbing.ZeroHash, 0, 0, nil); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + if err := progress.OnInflatedObjectContent(plumbing.ZeroHash, 0, 0, nil); err != nil { + t.Fatal(err) + } + if output.String() != "Importing objects: 0/3\nImporting objects: 2/3\r" { + t.Fatalf("progress cadence: %q", output.String()) + } + }) +} + +type failedProgress struct { + err error + writes int +} + +func (w *failedProgress) Write([]byte) (int, error) { + w.writes++ + return 0, w.err +} + +type cancellableImportStorage struct { + *importStorage + ctx context.Context +} + +func (s *cancellableImportStorage) RawObjectWriter(kind plumbing.ObjectType, size int64) (io.WriteCloser, error) { + if err := s.ctx.Err(); err != nil { + return nil, err + } + return s.importStorage.RawObjectWriter(kind, size) +} + +func TestReceiveProgressWriteFailureStopsImport(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + failure := errors.New("response closed") + output := &failedProgress{err: failure} + progress := &receiveProgress{writer: output, cancel: cancel} + st := &cancellableImportStorage{importStorage: newImportStorage(config.SHA1), ctx: ctx} + pack := fixturePack(t, config.SHA1, []packFixtureEntry{ + {kind: plumbing.BlobObject, data: []byte("abc")}, + {kind: plumbing.BlobObject, data: []byte("def")}, + }) + err := importPack(ctx, bytes.NewReader(pack), st, config.SHA1, testPackLimits(1024), progress) + // Cancellation stops storage work even when upstream ignores callback errors. + if err == nil || ctx.Err() != context.Canceled || st.writes != 0 { + t.Fatalf("import continued: err=%v context=%v object writes=%d", err, ctx.Err(), st.writes) + } + if !errors.Is(progress.err, failure) || output.writes != 1 { + t.Fatalf("progress write error lost: err=%v writes=%d", progress.err, output.writes) + } + if err := progress.message("Publishing update...\n"); !errors.Is(err, failure) || output.writes != 1 { + t.Fatalf("progress retried a failed writer: err=%v writes=%d", err, output.writes) + } +} + +type progressPackStorage struct { + *memory.Storage + closePack func() error +} + +func (s *progressPackStorage) PackfileWriter() (io.WriteCloser, error) { + return gitio.NewWriteCloser(io.Discard, progressPackClose{s.closePack}), nil +} + +type progressPackClose struct{ close func() error } + +func (c progressPackClose) Close() error { return c.close() } + +func TestReceiveProgressPreservesReportStatus(t *testing.T) { + for _, tc := range []struct { + name, reason string + caps string + unpackError bool + plain bool + }{ + {name: "committed"}, + {name: "validation rejected", reason: "stale ref"}, + {name: "publication uncertain", reason: "publication pending"}, + {name: "pack rejected", reason: "malformed pack", unpackError: true}, + {name: "quiet committed", caps: "report-status side-band-64k quiet"}, + {name: "quiet uncertain", caps: "report-status side-band-64k quiet", reason: "publication pending"}, + {name: "plain committed", caps: "report-status", plain: true}, + {name: "plain uncertain", caps: "report-status", reason: "publication pending", plain: true}, + {name: "no progress", caps: "report-status side-band-64k no-progress", plain: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var output bytes.Buffer + var caps capability.List + features := tc.caps + if features == "" { + features = "report-status side-band-64k" + } + capability.DecodeList([]byte(features), &caps) + progress := newReceiveProgress(&output, &caps, nil) + st := &progressPackStorage{Storage: memory.NewStorage()} + st.closePack = func() error { + if err := progress.message("Importing objects...\n"); err != nil { + return err + } + if tc.unpackError { + return errors.New(tc.reason) + } + return nil + } + object := plumbing.NewMemoryObject(plumbing.FromObjectFormat(config.SHA1)) + object.SetType(plumbing.BlobObject) + writer, _ := object.Writer() + _, _ = io.WriteString(writer, "progress test") + _ = writer.Close() + tip, err := st.SetEncodedObject(object) + if err != nil { + t.Fatal(err) + } + request := packetForTest(strings.Repeat("0", 40) + " " + tip.String() + " refs/tags/progress\x00" + caps.String() + "\n") + request = append(request, []byte("0000pack bytes")...) + hookCalled := false + err = transport.ReceivePack( + t.Context(), + st, + io.NopCloser(bytes.NewReader(request)), + gitio.WriteNopCloser(&output), + &transport.ReceivePackRequest{StatelessRPC: true, Hooks: transport.ReceivePackHooks{ + PreReceive: func(_ context.Context, info *transport.PreReceiveInfo) error { + hookCalled = true + if progress != nil { + progress.writer = info.Progress + } + if err := progress.message("Publishing update...\n"); err != nil { + return err + } + if tc.reason != "" { + return errors.New(tc.reason) + } + return nil + }, + }}, + ) + if (err == nil) != (tc.reason == "") || hookCalled == tc.unpackError { + t.Fatalf("receive outcome: err=%v hooked=%t", err, hookCalled) + } + var messages bytes.Buffer + reportReader := io.Reader(bytes.NewReader(output.Bytes())) + if !tc.plain { + if !bytes.HasSuffix(output.Bytes(), []byte("0009\x0100000000")) { + t.Fatalf("sideband termination changed: %q", output.String()) + } + demux := sideband.NewDemuxer(sideband.Sideband64k, reportReader) + demux.Progress = &messages + reportReader = demux + } + var report packp.ReportStatus + if err := report.Decode(reportReader); err != nil { + t.Fatal(err) + } + if strings.Contains(messages.String(), "Importing objects") != (progress != nil) { + t.Fatalf("unexpected import feedback: %q", messages.String()) + } + if tc.unpackError { + if report.UnpackStatus == "ok" || len(report.CommandStatuses) != 0 { + t.Fatalf("pack failure reported success: %+v", report) + } + return + } + status := report.CommandStatuses[0] + want := "ok" + if tc.reason != "" { + want = tc.reason + } + if status.Status != want || status.ReferenceName != "refs/tags/progress" { + t.Fatalf("publication status changed: %+v", status) + } + ref, refErr := st.Reference("refs/tags/progress") + if tc.reason == "" && (refErr != nil || ref.Hash() != tip) { + t.Fatalf("committed ref missing: %v", refErr) + } + if tc.reason != "" && !errors.Is(refErr, plumbing.ErrReferenceNotFound) { + t.Fatalf("rejected or uncertain update moved a ref: %v", ref) + } + }) + } +} + +func TestReceiveProgressWriteFailurePreventsPublication(t *testing.T) { + failure := errors.New("response closed") + var caps capability.List + capability.DecodeList([]byte("report-status side-band-64k"), &caps) + output := &failedProgress{err: failure} + progress := newReceiveProgress(output, &caps, nil) + st := &progressPackStorage{Storage: memory.NewStorage(), closePack: func() error { + return progress.message("Checking received pack...\n") + }} + request := packetForTest(strings.Repeat("0", 40) + " " + strings.Repeat("1", 40) + " refs/heads/main\x00" + caps.String() + "\n") + request = append(request, []byte("0000pack bytes")...) + hooked := false + err := transport.ReceivePack( + t.Context(), + st, + io.NopCloser(bytes.NewReader(request)), + gitio.WriteNopCloser(output), + &transport.ReceivePackRequest{StatelessRPC: true, Hooks: transport.ReceivePackHooks{ + PreReceive: func(context.Context, *transport.PreReceiveInfo) error { + hooked = true + return nil + }, + }}, + ) + if !errors.Is(err, failure) || hooked { + t.Fatalf("progress failure reached publication: error=%v hook=%t", err, hooked) + } +} diff --git a/examples/git/receive_transport.go b/examples/git/receive_transport.go index 79ca375..c1d748a 100644 --- a/examples/git/receive_transport.go +++ b/examples/git/receive_transport.go @@ -17,7 +17,7 @@ import ( // Decode the first command with go-git, then replay its bounded prefix so the // ordinary receive-pack path still validates every command and the entire pack. -func receiveFormat(body io.Reader, limit int64) (config.ObjectFormat, io.Reader, error) { +func receiveFormat(body io.Reader, limit int64) (config.ObjectFormat, *capability.List, io.Reader, error) { var prefix bytes.Buffer scanner := pktline.NewScanner(&commandReader{io.LimitedReader{R: io.TeeReader(body, &prefix), N: limit}}) for scanner.Scan() { @@ -26,35 +26,35 @@ func receiveFormat(body io.Reader, limit int64) (config.ObjectFormat, io.Reader, } } if err := scanner.Err(); err != nil { - return "", nil, err + return "", nil, nil, err } replay := io.MultiReader(bytes.NewReader(prefix.Bytes()), body) if prefix.Len() == 4 && scanner.Len() == pktline.Flush { - return "", replay, nil + return "", nil, replay, nil } request := &packp.UpdateRequests{} if err := request.Decode(io.MultiReader(bytes.NewReader(prefix.Bytes()), strings.NewReader("0000"))); err != nil { - return "", nil, err + return "", nil, nil, err } if len(request.Commands) == 0 { - return "", replay, nil + return "", nil, replay, nil } formats := request.Capabilities.Get(capability.ObjectFormat) format := config.SHA1 if request.Capabilities.Supports(capability.ObjectFormat) && len(formats) != 1 { - return "", nil, config.ErrInvalidObjectFormat + return "", nil, nil, config.ErrInvalidObjectFormat } if len(formats) == 1 { format = config.ObjectFormat(formats[0]) } if format != config.SHA1 && format != config.SHA256 { - return "", nil, config.ErrInvalidObjectFormat + return "", nil, nil, config.ErrInvalidObjectFormat } command := request.Commands[0] if command.Old.HexSize() != format.HexSize() || command.New.HexSize() != format.HexSize() { - return "", nil, config.ErrInvalidObjectFormat + return "", nil, nil, config.ErrInvalidObjectFormat } - return format, replay, nil + return format, &request.Capabilities, replay, nil } // ReceivePack opens the pack writer after decoding commands and push options. diff --git a/examples/git/receive_transport_test.go b/examples/git/receive_transport_test.go index 7339d74..7d92518 100644 --- a/examples/git/receive_transport_test.go +++ b/examples/git/receive_transport_test.go @@ -98,7 +98,7 @@ func TestReceiveFormat(t *testing.T) { packet := bytes.Clone(prefix.Bytes()) input := append(bytes.Clone(packet), []byte("0000PACKpayload")...) body := bytes.NewBuffer(input) - format, replay, err := receiveFormat(body, int64(len(packet))) + format, caps, replay, err := receiveFormat(body, int64(len(packet))) if tc.want == "" { if err == nil { t.Fatal("invalid format accepted") @@ -108,18 +108,21 @@ func TestReceiveFormat(t *testing.T) { if err != nil || format != tc.want || body.Len() != len("0000PACKpayload") { t.Fatalf("format=%s error=%v unread=%d", format, err, body.Len()) } + if caps.String() != tc.caps { + t.Fatalf("capabilities changed: %s", caps.String()) + } data, err := io.ReadAll(replay) if err != nil || !bytes.Equal(data, input) { t.Fatalf("push body was changed: %v", err) } - if _, _, err := receiveFormat(bytes.NewReader(input), int64(len(packet)-1)); !errors.Is(err, errObjectLimit) { + if _, _, _, err := receiveFormat(bytes.NewReader(input), int64(len(packet)-1)); !errors.Is(err, errObjectLimit) { t.Fatalf("negotiation limit: %v", err) } }) } for _, input := range []string{"0000", string(packetForTest("shallow "+strings.Repeat("1", 40)+"\n")) + "0000"} { - format, replay, err := receiveFormat(strings.NewReader(input), int64(len(input))) - if err != nil || format != "" { + format, caps, replay, err := receiveFormat(strings.NewReader(input), int64(len(input))) + if err != nil || format != "" || caps != nil { t.Fatalf("no-op push: format=%s error=%v", format, err) } data, err := io.ReadAll(replay) diff --git a/examples/git/store.go b/examples/git/store.go index e21969f..c68452a 100644 --- a/examples/git/store.go +++ b/examples/git/store.go @@ -31,6 +31,7 @@ type store struct { writers []*byteWriter storage.Storer failure error + progress *receiveProgress tailEntries uint64 session *wal.Session recovery *wal.Recovery