Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 82 additions & 14 deletions internal/quotahistory/mistral_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,8 @@ func TestRecorderMergesMistralKeysOfOneCustomer(t *testing.T) {
}
customerOf := map[string]string{"fixture-vibe-key": "customer-a", "fixture-api-key": "customer-a", "fixture-other-key": "customer-b"}
whoamiStatus := 200
r := NewRecorder(Deps{
var afterWhoami func(*coreauth.Auth)
deps := Deps{
ListAuths: func() []*coreauth.Auth { return auths },
DoRequest: func(_ context.Context, auth *coreauth.Auth, _, url string, _ map[string]string, _ []byte) (int, []byte, error) {
switch url {
Expand All @@ -317,30 +318,47 @@ func TestRecorderMergesMistralKeysOfOneCustomer(t *testing.T) {
case MistralSubscriptionURL:
return 200, []byte(mistralSubscriptionFixture), nil
case MistralWhoamiURL:
if afterWhoami != nil {
afterWhoami(auth)
}
return whoamiStatus, []byte(`{"customer_id":"` + customerOf[auth.Attributes["api_key"]] + `"}`), nil
}
return 404, nil, nil
},
})
r.now = func() time.Time { return now }
r.startupDelay = time.Hour
if err := r.Apply(enabledSettings(t)); err != nil {
t.Fatal(err)
}
t.Cleanup(r.Close)
var clock atomic.Int64
clock.Store(now.UnixMilli())
open := func(settings Settings) *Recorder {
t.Helper()
recorder := NewRecorder(deps)
recorder.now = func() time.Time { return time.UnixMilli(clock.Load()) }
recorder.startupDelay = time.Hour
if err := recorder.Apply(settings); err != nil {
t.Fatal(err)
}
t.Cleanup(recorder.Close)
return recorder
}
historyOf := func(recorder *Recorder) History {
t.Helper()
history, err := recorder.History(0, 0)
if err != nil {
t.Fatal(err)
}
return history
}
settings := enabledSettings(t)
r := open(settings)
// The first round learns the customers as it goes, so a key polled before its
// sibling may still be stored once; history hides it.
if n := r.poll(context.Background(), r.store, 0, nil); n < 2 || n > 3 {
t.Fatalf("first round stored %d samples, want 2 or 3", n)
}
now = now.Add(time.Minute)
clock.Add(time.Minute.Milliseconds())
if n := r.poll(context.Background(), r.store, 0, nil); n != 2 {
t.Fatalf("stored %d samples, want 2 (one per customer)", n)
}
history, err := r.History(0, 0)
if err != nil {
t.Fatal(err)
}
history := historyOf(r)
labels := map[string]bool{}
for _, account := range history.Accounts {
labels[account.Label] = true
Expand All @@ -363,8 +381,58 @@ func TestRecorderMergesMistralKeysOfOneCustomer(t *testing.T) {

// A customer that cannot be read keeps the last one known.
whoamiStatus = 503
now = now.Add(profileTTL)
if n := r.poll(context.Background(), r.store, 0, nil); n != 2 {
r.Close()
restarted := open(settings)
if reopened := historyOf(restarted); !reflect.DeepEqual(reopened, history) {
t.Fatalf("history before the first poll after restart changed: %d accounts and %d samples", len(reopened.Accounts), len(reopened.Samples))
}
clock.Add(profileTTL.Milliseconds())
if n := restarted.poll(context.Background(), restarted.store, 0, nil); n != 2 {
t.Fatalf("stored %d samples after a failed customer read, want 2", n)
}

// A new database must retain a verified identity even for a disabled
// duplicate key that no longer gets polled.
auths[0].Disabled = true // mistral-vibe
settings.Path = filepath.Join(t.TempDir(), "new.db")
if err := restarted.Apply(settings); err != nil {
t.Fatal(err)
}
if n := restarted.poll(context.Background(), restarted.store, 0, nil); n != 2 {
t.Fatalf("stored %d samples after switching databases, want 2", n)
}
history = historyOf(restarted)
restarted.Close()
if reopened := historyOf(open(settings)); !reflect.DeepEqual(reopened, history) {
t.Fatalf("history before the first poll after switching databases and restarting changed: %d accounts and %d samples", len(reopened.Accounts), len(reopened.Samples))
}

// Shutdown can arrive just after the second key's lookup succeeds. Save
// the verified grouping while discarding the cancelled usage sample.
auths[0].Disabled = false
whoamiStatus = 200
settings.Path = filepath.Join(t.TempDir(), "cancel.db")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cancelIndex := max(auths[0].EnsureIndex(), auths[1].EnsureIndex())
afterWhoami = func(auth *coreauth.Auth) {
if auth.Index == cancelIndex {
cancel()
}
}
cancelled := open(settings)
cancelled.poll(ctx, cancelled.store, 0, nil)
if ctx.Err() == nil {
t.Fatal("the second same-customer lookup did not cancel the poll")
}
cancelled.Close()
history = historyOf(open(settings))
if len(history.Accounts) != 2 {
t.Fatalf("accounts after cancelling and restarting = %+v, want two customers", history.Accounts)
}
for _, sample := range history.Samples {
if sample.AuthIndex == cancelIndex {
t.Fatal("stored the cancelled usage sample")
}
}
}
5 changes: 4 additions & 1 deletion internal/quotahistory/providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package quotahistory

import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"sort"
Expand Down Expand Up @@ -119,8 +121,9 @@ func (r *Recorder) refreshMistralCustomer(ctx context.Context, auth *coreauth.Au
return
}
if id := firstString(ojson.ParseBytes(body).Get("customer_id")); id != "" {
digest := sha256.Sum256([]byte(id))
r.plansMu.Lock()
r.customers[auth.Index] = id
r.customers[auth.Index] = hex.EncodeToString(digest[:])
r.plansMu.Unlock()
}
}
Expand Down
29 changes: 27 additions & 2 deletions internal/quotahistory/recorder.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ type Recorder struct {
plansMu sync.Mutex
plans map[string]string
profileAttempts map[string]time.Time
// customers maps a Mistral key's auth index to the customer behind it.
// customers maps a Mistral key's auth index to its customer fingerprint.
customers map[string]string
customerAttempts map[string]time.Time
}
Expand Down Expand Up @@ -183,7 +183,20 @@ func (r *Recorder) Apply(s Settings) error {
r.forgetUntracked(false)
return errOpen
}
customers, errCustomers := store.mistralCustomers()
if errCustomers != nil {
log.WithError(errCustomers).Warn("quota-history: restore Mistral customers")
}
r.plansMu.Lock()
// Seed a new database even for known keys that are no longer polled.
if errSave := store.putMistralCustomers(r.customers); errSave != nil {
log.WithError(errSave).Warn("quota-history: seed Mistral customers")
}
for idx, customer := range customers {
if r.customers[idx] == "" {
r.customers[idx] = customer
}
}
for idx, plan := range store.latestPlans(planSeedRecords) {
if r.plans[idx] == "" {
r.plans[idx] = plan
Expand Down Expand Up @@ -354,11 +367,23 @@ func (r *Recorder) poll(ctx context.Context, store *Store, gap time.Duration, co
continue
}
rec := r.sample(ctx, auth)
mistral := providerOf(auth) == "mistral"
if mistral {
// Keep verified identities even if cancellation discards the usage sample.
r.plansMu.Lock()
customer := r.customers[auth.Index]
r.plansMu.Unlock()
if customer != "" {
if errPut := store.putMistralCustomers(map[string]string{auth.Index: customer}); errPut != nil {
log.WithError(errPut).Warn("quota-history: store Mistral customer")
}
}
}
if ctx.Err() != nil {
// Shutting down or reconfiguring: do not record a cancellation as an error.
return stored
}
if providerOf(auth) == "mistral" && r.mistralDuplicates(r.trackable(r.pluginProviders(ctx, false)))[auth.Index] {
if mistral && r.mistralDuplicates(r.trackable(r.pluginProviders(ctx, false)))[auth.Index] {
continue
}
if errPut := store.put(rec); errPut != nil {
Expand Down
42 changes: 41 additions & 1 deletion internal/quotahistory/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@ import (
"go.etcd.io/bbolt"
)

var samplesBucket = []byte("samples")
var (
samplesBucket = []byte("samples")
mistralCustomersBucket = []byte("mistral_customers")
)

const boltOpenTimeout = 2 * time.Second

Expand Down Expand Up @@ -146,6 +149,43 @@ func (s *Store) latestPlans(maxRecords int) map[string]string {
return plans
}

func (s *Store) putMistralCustomers(customers map[string]string) error {
if len(customers) == 0 {
return nil
}
return s.db.Update(func(tx *bbolt.Tx) error {
bucket, errCreate := tx.CreateBucketIfNotExists(mistralCustomersBucket)
if errCreate != nil {
return errCreate
}
for index, customer := range customers {
key := []byte(index)
if string(bucket.Get(key)) == customer {
continue
}
if errPut := bucket.Put(key, []byte(customer)); errPut != nil {
return errPut
}
}
return nil
})
}

func (s *Store) mistralCustomers() (map[string]string, error) {
customers := make(map[string]string)
errView := s.db.View(func(tx *bbolt.Tx) error {
bucket := tx.Bucket(mistralCustomersBucket)
if bucket == nil {
return nil
}
return bucket.ForEach(func(index, customer []byte) error {
customers[string(index)] = string(customer)
return nil
})
})
return customers, errView
}

// prune deletes samples older than before (unix ms) and returns the count.
func (s *Store) prune(before int64) (int, error) {
deleted := 0
Expand Down