diff --git a/internal/quotahistory/mistral_test.go b/internal/quotahistory/mistral_test.go index 5713490e424..ff6108f7f5b 100644 --- a/internal/quotahistory/mistral_test.go +++ b/internal/quotahistory/mistral_test.go @@ -308,7 +308,8 @@ func TestRecorderMergesMistralKeysOfOneCustomer(t *testing.T) { } customerOf := map[string]string{"fixture-vibe-key": "customer-a", "fixture-api-key": "customer-a", "fixture-other-key": "customer-b"} whoamiStatus := 200 - r := NewRecorder(Deps{ + var afterWhoami func(*coreauth.Auth) + deps := Deps{ ListAuths: func() []*coreauth.Auth { return auths }, DoRequest: func(_ context.Context, auth *coreauth.Auth, _, url string, _ map[string]string, _ []byte) (int, []byte, error) { switch url { @@ -317,30 +318,47 @@ func TestRecorderMergesMistralKeysOfOneCustomer(t *testing.T) { case MistralSubscriptionURL: return 200, []byte(mistralSubscriptionFixture), nil case MistralWhoamiURL: + if afterWhoami != nil { + afterWhoami(auth) + } return whoamiStatus, []byte(`{"customer_id":"` + customerOf[auth.Attributes["api_key"]] + `"}`), nil } return 404, nil, nil }, - }) - r.now = func() time.Time { return now } - r.startupDelay = time.Hour - if err := r.Apply(enabledSettings(t)); err != nil { - t.Fatal(err) } - t.Cleanup(r.Close) + var clock atomic.Int64 + clock.Store(now.UnixMilli()) + open := func(settings Settings) *Recorder { + t.Helper() + recorder := NewRecorder(deps) + recorder.now = func() time.Time { return time.UnixMilli(clock.Load()) } + recorder.startupDelay = time.Hour + if err := recorder.Apply(settings); err != nil { + t.Fatal(err) + } + t.Cleanup(recorder.Close) + return recorder + } + historyOf := func(recorder *Recorder) History { + t.Helper() + history, err := recorder.History(0, 0) + if err != nil { + t.Fatal(err) + } + return history + } + settings := enabledSettings(t) + r := open(settings) // The first round learns the customers as it goes, so a key polled before its // sibling may still be stored once; history hides it. if n := r.poll(context.Background(), r.store, 0, nil); n < 2 || n > 3 { t.Fatalf("first round stored %d samples, want 2 or 3", n) } - now = now.Add(time.Minute) + clock.Add(time.Minute.Milliseconds()) if n := r.poll(context.Background(), r.store, 0, nil); n != 2 { t.Fatalf("stored %d samples, want 2 (one per customer)", n) } - history, err := r.History(0, 0) - if err != nil { - t.Fatal(err) - } + history := historyOf(r) labels := map[string]bool{} for _, account := range history.Accounts { labels[account.Label] = true @@ -363,8 +381,58 @@ func TestRecorderMergesMistralKeysOfOneCustomer(t *testing.T) { // A customer that cannot be read keeps the last one known. whoamiStatus = 503 - now = now.Add(profileTTL) - if n := r.poll(context.Background(), r.store, 0, nil); n != 2 { + r.Close() + restarted := open(settings) + if reopened := historyOf(restarted); !reflect.DeepEqual(reopened, history) { + t.Fatalf("history before the first poll after restart changed: %d accounts and %d samples", len(reopened.Accounts), len(reopened.Samples)) + } + clock.Add(profileTTL.Milliseconds()) + if n := restarted.poll(context.Background(), restarted.store, 0, nil); n != 2 { t.Fatalf("stored %d samples after a failed customer read, want 2", n) } + + // A new database must retain a verified identity even for a disabled + // duplicate key that no longer gets polled. + auths[0].Disabled = true // mistral-vibe + settings.Path = filepath.Join(t.TempDir(), "new.db") + if err := restarted.Apply(settings); err != nil { + t.Fatal(err) + } + if n := restarted.poll(context.Background(), restarted.store, 0, nil); n != 2 { + t.Fatalf("stored %d samples after switching databases, want 2", n) + } + history = historyOf(restarted) + restarted.Close() + if reopened := historyOf(open(settings)); !reflect.DeepEqual(reopened, history) { + t.Fatalf("history before the first poll after switching databases and restarting changed: %d accounts and %d samples", len(reopened.Accounts), len(reopened.Samples)) + } + + // Shutdown can arrive just after the second key's lookup succeeds. Save + // the verified grouping while discarding the cancelled usage sample. + auths[0].Disabled = false + whoamiStatus = 200 + settings.Path = filepath.Join(t.TempDir(), "cancel.db") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cancelIndex := max(auths[0].EnsureIndex(), auths[1].EnsureIndex()) + afterWhoami = func(auth *coreauth.Auth) { + if auth.Index == cancelIndex { + cancel() + } + } + cancelled := open(settings) + cancelled.poll(ctx, cancelled.store, 0, nil) + if ctx.Err() == nil { + t.Fatal("the second same-customer lookup did not cancel the poll") + } + cancelled.Close() + history = historyOf(open(settings)) + if len(history.Accounts) != 2 { + t.Fatalf("accounts after cancelling and restarting = %+v, want two customers", history.Accounts) + } + for _, sample := range history.Samples { + if sample.AuthIndex == cancelIndex { + t.Fatal("stored the cancelled usage sample") + } + } } diff --git a/internal/quotahistory/providers.go b/internal/quotahistory/providers.go index 029e80a74ef..d8dc10723aa 100644 --- a/internal/quotahistory/providers.go +++ b/internal/quotahistory/providers.go @@ -2,6 +2,8 @@ package quotahistory import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "sort" @@ -119,8 +121,9 @@ func (r *Recorder) refreshMistralCustomer(ctx context.Context, auth *coreauth.Au return } if id := firstString(ojson.ParseBytes(body).Get("customer_id")); id != "" { + digest := sha256.Sum256([]byte(id)) r.plansMu.Lock() - r.customers[auth.Index] = id + r.customers[auth.Index] = hex.EncodeToString(digest[:]) r.plansMu.Unlock() } } diff --git a/internal/quotahistory/recorder.go b/internal/quotahistory/recorder.go index 0c501ce939d..ddba0d3e806 100644 --- a/internal/quotahistory/recorder.go +++ b/internal/quotahistory/recorder.go @@ -136,7 +136,7 @@ type Recorder struct { plansMu sync.Mutex plans map[string]string profileAttempts map[string]time.Time - // customers maps a Mistral key's auth index to the customer behind it. + // customers maps a Mistral key's auth index to its customer fingerprint. customers map[string]string customerAttempts map[string]time.Time } @@ -183,7 +183,20 @@ func (r *Recorder) Apply(s Settings) error { r.forgetUntracked(false) return errOpen } + customers, errCustomers := store.mistralCustomers() + if errCustomers != nil { + log.WithError(errCustomers).Warn("quota-history: restore Mistral customers") + } r.plansMu.Lock() + // Seed a new database even for known keys that are no longer polled. + if errSave := store.putMistralCustomers(r.customers); errSave != nil { + log.WithError(errSave).Warn("quota-history: seed Mistral customers") + } + for idx, customer := range customers { + if r.customers[idx] == "" { + r.customers[idx] = customer + } + } for idx, plan := range store.latestPlans(planSeedRecords) { if r.plans[idx] == "" { r.plans[idx] = plan @@ -354,11 +367,23 @@ func (r *Recorder) poll(ctx context.Context, store *Store, gap time.Duration, co continue } rec := r.sample(ctx, auth) + mistral := providerOf(auth) == "mistral" + if mistral { + // Keep verified identities even if cancellation discards the usage sample. + r.plansMu.Lock() + customer := r.customers[auth.Index] + r.plansMu.Unlock() + if customer != "" { + if errPut := store.putMistralCustomers(map[string]string{auth.Index: customer}); errPut != nil { + log.WithError(errPut).Warn("quota-history: store Mistral customer") + } + } + } if ctx.Err() != nil { // Shutting down or reconfiguring: do not record a cancellation as an error. return stored } - if providerOf(auth) == "mistral" && r.mistralDuplicates(r.trackable(r.pluginProviders(ctx, false)))[auth.Index] { + if mistral && r.mistralDuplicates(r.trackable(r.pluginProviders(ctx, false)))[auth.Index] { continue } if errPut := store.put(rec); errPut != nil { diff --git a/internal/quotahistory/store.go b/internal/quotahistory/store.go index 67e0fcc539f..2772d08aad9 100644 --- a/internal/quotahistory/store.go +++ b/internal/quotahistory/store.go @@ -12,7 +12,10 @@ import ( "go.etcd.io/bbolt" ) -var samplesBucket = []byte("samples") +var ( + samplesBucket = []byte("samples") + mistralCustomersBucket = []byte("mistral_customers") +) const boltOpenTimeout = 2 * time.Second @@ -146,6 +149,43 @@ func (s *Store) latestPlans(maxRecords int) map[string]string { return plans } +func (s *Store) putMistralCustomers(customers map[string]string) error { + if len(customers) == 0 { + return nil + } + return s.db.Update(func(tx *bbolt.Tx) error { + bucket, errCreate := tx.CreateBucketIfNotExists(mistralCustomersBucket) + if errCreate != nil { + return errCreate + } + for index, customer := range customers { + key := []byte(index) + if string(bucket.Get(key)) == customer { + continue + } + if errPut := bucket.Put(key, []byte(customer)); errPut != nil { + return errPut + } + } + return nil + }) +} + +func (s *Store) mistralCustomers() (map[string]string, error) { + customers := make(map[string]string) + errView := s.db.View(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(mistralCustomersBucket) + if bucket == nil { + return nil + } + return bucket.ForEach(func(index, customer []byte) error { + customers[string(index)] = string(customer) + return nil + }) + }) + return customers, errView +} + // prune deletes samples older than before (unix ms) and returns the count. func (s *Store) prune(before int64) (int, error) { deleted := 0