diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b5fc4a..3194fdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ before paging, and include the route reconciler timer in platform health. - Permit the queue-only route reconciler timer through the observer's fixed systemd inventory when evaluating that platform-health requirement. +- Bound a remote outage to one retry sequence per exporter generation instead + of multiplying the network wait by every bundle already present in the WAL. - Added GARM derivative `v0.2.1-nddev.81`: all embedded bootstrap downloads now use the same three-total-attempt contract as provider `.78`, preventing diff --git a/internal/diagnosticexport/exporter.go b/internal/diagnosticexport/exporter.go index 0a016a9..b62d9cd 100644 --- a/internal/diagnosticexport/exporter.go +++ b/internal/diagnosticexport/exporter.go @@ -147,7 +147,11 @@ func (e Exporter) Run(ctx context.Context) (Summary, error) { firstFailure = "remote-head" } failures++ - continue + // Remote availability is shared by the whole batch. Preserve the + // remaining WAL and end this generation after one bounded probe; + // repeating the same retry budget for every bundle makes run time + // proportional to backlog size during an outage. + break } if remote.Exists { if !remoteMatches(remote, bundle) { diff --git a/internal/diagnosticexport/exporter_test.go b/internal/diagnosticexport/exporter_test.go index 4d6c38f..6d79723 100644 --- a/internal/diagnosticexport/exporter_test.go +++ b/internal/diagnosticexport/exporter_test.go @@ -88,6 +88,42 @@ func TestExporterRetriesRemoteHeadAcrossRouteConvergence(t *testing.T) { } } +func TestExporterBoundsRemoteOutageOncePerBatch(t *testing.T) { + exporter, remote, now := exporterFixture(t) + store := workerdiagnostics.Store{ + Directory: exporter.Config.SourceDirectory, Retention: 7 * 24 * time.Hour, + MaxBundleBytes: exporter.Config.MaxBundleBytes, MaxTotalBytes: 1024 * 1024 * 1024, + MaxArtifacts: workerdiagnostics.DefaultMaxArtifacts, + Now: func() time.Time { return now.Add(time.Second) }, + Random: strings.NewReader(strings.Repeat("abcdef", 600)), + } + for index := 0; index < 2; index++ { + _, err := store.Write(context.Background(), workerdiagnostics.Instance{ + Name: fmt.Sprintf("runner-outage-%d", index), ControllerID: "controller-test", + PoolID: "pool-test", PoolName: "nddev-linux-standard", ScaleSet: "nddev-linux-standard", + Repository: validConfig().Repositories[1], ImageFingerprint: strings.Repeat("a", 64), + RunnerVersion: "v2.336.0", ProviderVersion: "v0.1.5-nddev.3", + ProviderCommit: strings.Repeat("b", 40), State: "Stopped", + }, nil, nil) + if err != nil { + t.Fatal(err) + } + } + remote.headErrors = []error{ + errors.New("route absent"), errors.New("route absent"), + errors.New("route absent"), errors.New("route absent"), + } + exporter.Sleep = func(context.Context, time.Duration) error { return nil } + summary, err := exporter.Run(context.Background()) + var exportError ExportError + if !errors.As(err, &exportError) || exportError.Code != "remote-head" || exportError.Failed != 1 { + t.Fatalf("error=%v", err) + } + if summary.ScannedBundles != 3 || summary.PendingBundles != 3 || remote.headCalls != remoteHeadAttempts { + t.Fatalf("summary=%#v heads=%d", summary, remote.headCalls) + } +} + func TestExporterUploadsConfirmsAndUsesFreshJournal(t *testing.T) { exporter, remote, now := exporterFixture(t) summary, err := exporter.Run(context.Background())