Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
before paging, and include the route reconciler timer in platform health.
- Permit the queue-only route reconciler timer through the observer's fixed
systemd inventory when evaluating that platform-health requirement.
- Bound a remote outage to one retry sequence per exporter generation instead
of multiplying the network wait by every bundle already present in the WAL.

- Added GARM derivative `v0.2.1-nddev.81`: all embedded bootstrap downloads
now use the same three-total-attempt contract as provider `.78`, preventing
Expand Down
6 changes: 5 additions & 1 deletion internal/diagnosticexport/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,11 @@ func (e Exporter) Run(ctx context.Context) (Summary, error) {
firstFailure = "remote-head"
}
failures++
continue
// Remote availability is shared by the whole batch. Preserve the
// remaining WAL and end this generation after one bounded probe;
// repeating the same retry budget for every bundle makes run time
// proportional to backlog size during an outage.
break
}
if remote.Exists {
if !remoteMatches(remote, bundle) {
Expand Down
36 changes: 36 additions & 0 deletions internal/diagnosticexport/exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,42 @@ func TestExporterRetriesRemoteHeadAcrossRouteConvergence(t *testing.T) {
}
}

func TestExporterBoundsRemoteOutageOncePerBatch(t *testing.T) {
exporter, remote, now := exporterFixture(t)
store := workerdiagnostics.Store{
Directory: exporter.Config.SourceDirectory, Retention: 7 * 24 * time.Hour,
MaxBundleBytes: exporter.Config.MaxBundleBytes, MaxTotalBytes: 1024 * 1024 * 1024,
MaxArtifacts: workerdiagnostics.DefaultMaxArtifacts,
Now: func() time.Time { return now.Add(time.Second) },
Random: strings.NewReader(strings.Repeat("abcdef", 600)),
}
for index := 0; index < 2; index++ {
_, err := store.Write(context.Background(), workerdiagnostics.Instance{
Name: fmt.Sprintf("runner-outage-%d", index), ControllerID: "controller-test",
PoolID: "pool-test", PoolName: "nddev-linux-standard", ScaleSet: "nddev-linux-standard",
Repository: validConfig().Repositories[1], ImageFingerprint: strings.Repeat("a", 64),
RunnerVersion: "v2.336.0", ProviderVersion: "v0.1.5-nddev.3",
ProviderCommit: strings.Repeat("b", 40), State: "Stopped",
}, nil, nil)
if err != nil {
t.Fatal(err)
}
}
remote.headErrors = []error{
errors.New("route absent"), errors.New("route absent"),
errors.New("route absent"), errors.New("route absent"),
}
exporter.Sleep = func(context.Context, time.Duration) error { return nil }
summary, err := exporter.Run(context.Background())
var exportError ExportError
if !errors.As(err, &exportError) || exportError.Code != "remote-head" || exportError.Failed != 1 {
t.Fatalf("error=%v", err)
}
if summary.ScannedBundles != 3 || summary.PendingBundles != 3 || remote.headCalls != remoteHeadAttempts {
t.Fatalf("summary=%#v heads=%d", summary, remote.headCalls)
}
}

func TestExporterUploadsConfirmsAndUsesFreshJournal(t *testing.T) {
exporter, remote, now := exporterFixture(t)
summary, err := exporter.Run(context.Background())
Expand Down