diff --git a/docs/coverage/aws/elasticache.md b/docs/coverage/aws/elasticache.md index 8e609d706..5ce9a6320 100644 --- a/docs/coverage/aws/elasticache.md +++ b/docs/coverage/aws/elasticache.md @@ -63,4 +63,8 @@ SubnetGroups is an OPTIONAL capability, discovered by type assertion. Cache ## Not in scope -_Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ +- AWS: ElastiCache clusters, replication groups, and subnet and parameter groups share one id namespace across regions, so the same id cannot exist in two regions at once (#1693). +- AWS: ElastiCache snapshot ARNs cannot be tagged yet; AddTagsToResource and ListTagsForResource on a `:snapshot:` ARN answer not-found (#1695). +- AWS: with a real cache engine, a cluster or replication group reports `creating` only while the engine is being started and `deleting` only while it is being stopped. Without one, both are immediate unless `--async-settle` is set. +- AWS: a persisted snapshot (`--persist`, `/_cloudemu/snapshot`) leaves out ElastiCache clusters and replication groups that are still `creating` or `deleting` with a real engine, so they are not restored. +- AWS: while a replication group is `deleting` with a real engine, its member clusters (`-001`, …) are already gone from DescribeCacheClusters, though the group itself still shows `deleting`. diff --git a/docs/coverage/azure/cache.md b/docs/coverage/azure/cache.md index c42085ec8..9e1f69d8f 100644 --- a/docs/coverage/azure/cache.md +++ b/docs/coverage/azure/cache.md @@ -54,4 +54,8 @@ RedisChildren is an OPTIONAL capability, discovered by type assertion. Azure ## Not in scope -_Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ +- AWS: ElastiCache clusters, replication groups, and subnet and parameter groups share one id namespace across regions, so the same id cannot exist in two regions at once (#1693). +- AWS: ElastiCache snapshot ARNs cannot be tagged yet; AddTagsToResource and ListTagsForResource on a `:snapshot:` ARN answer not-found (#1695). +- AWS: with a real cache engine, a cluster or replication group reports `creating` only while the engine is being started and `deleting` only while it is being stopped. Without one, both are immediate unless `--async-settle` is set. +- AWS: a persisted snapshot (`--persist`, `/_cloudemu/snapshot`) leaves out ElastiCache clusters and replication groups that are still `creating` or `deleting` with a real engine, so they are not restored. +- AWS: while a replication group is `deleting` with a real engine, its member clusters (`-001`, …) are already gone from DescribeCacheClusters, though the group itself still shows `deleting`. diff --git a/docs/coverage/gcp/memorystore.md b/docs/coverage/gcp/memorystore.md index cb31efc52..7dba3ff7a 100644 --- a/docs/coverage/gcp/memorystore.md +++ b/docs/coverage/gcp/memorystore.md @@ -27,4 +27,8 @@ GCP's `cache` service · portable interface `driver.Cache` · [GCP index](./READ ## Not in scope -_Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ +- AWS: ElastiCache clusters, replication groups, and subnet and parameter groups share one id namespace across regions, so the same id cannot exist in two regions at once (#1693). +- AWS: ElastiCache snapshot ARNs cannot be tagged yet; AddTagsToResource and ListTagsForResource on a `:snapshot:` ARN answer not-found (#1695). +- AWS: with a real cache engine, a cluster or replication group reports `creating` only while the engine is being started and `deleting` only while it is being stopped. Without one, both are immediate unless `--async-settle` is set. +- AWS: a persisted snapshot (`--persist`, `/_cloudemu/snapshot`) leaves out ElastiCache clusters and replication groups that are still `creating` or `deleting` with a real engine, so they are not restored. +- AWS: while a replication group is `deleting` with a real engine, its member clusters (`-001`, …) are already gone from DescribeCacheClusters, though the group itself still shows `deleting`. diff --git a/docs/coverage/nongoals/cache.md b/docs/coverage/nongoals/cache.md new file mode 100644 index 000000000..a26745609 --- /dev/null +++ b/docs/coverage/nongoals/cache.md @@ -0,0 +1,5 @@ +- AWS: ElastiCache clusters, replication groups, and subnet and parameter groups share one id namespace across regions, so the same id cannot exist in two regions at once (#1693). +- AWS: ElastiCache snapshot ARNs cannot be tagged yet; AddTagsToResource and ListTagsForResource on a `:snapshot:` ARN answer not-found (#1695). +- AWS: with a real cache engine, a cluster or replication group reports `creating` only while the engine is being started and `deleting` only while it is being stopped. Without one, both are immediate unless `--async-settle` is set. +- AWS: a persisted snapshot (`--persist`, `/_cloudemu/snapshot`) leaves out ElastiCache clusters and replication groups that are still `creating` or `deleting` with a real engine, so they are not restored. +- AWS: while a replication group is `deleting` with a real engine, its member clusters (`-001`, …) are already gone from DescribeCacheClusters, though the group itself still shows `deleting`. diff --git a/providers/aws/elasticache/concurrency_test.go b/providers/aws/elasticache/concurrency_test.go new file mode 100644 index 000000000..40f849f46 --- /dev/null +++ b/providers/aws/elasticache/concurrency_test.go @@ -0,0 +1,756 @@ +package elasticache + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + "testing" + "time" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + driver "github.com/stackshy/cloudemu/v2/services/cache/driver" + "github.com/stackshy/cloudemu/v2/services/scope" +) + +const raceGoroutines = 50 + +// gatedCacheEngine is a real-engine stand-in whose Provision and Deprovision +// block until released, so a test can act while a create or delete is mid-way +// through its engine call. +type gatedCacheEngine struct { + provisionEntered chan struct{} + provisionRelease chan struct{} + deprovisionEntered chan struct{} + deprovisionRelease chan struct{} +} + +func newGatedCacheEngine() *gatedCacheEngine { + return &gatedCacheEngine{ + provisionEntered: make(chan struct{}, 1), + provisionRelease: make(chan struct{}), + deprovisionEntered: make(chan struct{}, 1), + deprovisionRelease: make(chan struct{}), + } +} + +func (e *gatedCacheEngine) Provision(_ context.Context, _ config.CacheProvisionRequest) (config.ProvisionResult, error) { + e.provisionEntered <- struct{}{} + <-e.provisionRelease + + return config.ProvisionResult{Host: "127.0.0.1", Port: 6379}, nil +} + +func (e *gatedCacheEngine) Deprovision(_ context.Context, _ string) error { + e.deprovisionEntered <- struct{}{} + <-e.deprovisionRelease + + return nil +} + +func newGatedMock() (*Mock, *gatedCacheEngine) { + eng := newGatedCacheEngine() + fc := config.NewFakeClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)) + + return New(config.NewOptions(config.WithClock(fc), config.WithRegion("us-east-1"), config.WithCacheEngine(eng))), eng +} + +// faultCode returns the wire fault code carried by err, or "". +func faultCode(err error) string { + var f *Fault + if errors.As(err, &f) { + return f.AWSFault() + } + + return "" +} + +// runConcurrently runs fn on n goroutines started together and returns how +// many calls succeeded and the fault codes of the ones that failed. +func runConcurrently(n int, fn func() error) (successes int, codes []string) { + var ( + wg sync.WaitGroup + mu sync.Mutex + start = make(chan struct{}) + ) + + for range n { + wg.Add(1) + + go func() { + defer wg.Done() + + <-start + + err := fn() + + mu.Lock() + defer mu.Unlock() + + if err == nil { + successes++ + } else { + codes = append(codes, faultCode(err)) + } + }() + } + + close(start) + wg.Wait() + + return successes, codes +} + +func assertOneWinner(t *testing.T, successes int, codes []string, wantCode string) { + t.Helper() + + if successes != 1 { + t.Fatalf("successes = %d, want exactly 1", successes) + } + + for _, c := range codes { + if c != wantCode { + t.Fatalf("losing create got code %q, want %q", c, wantCode) + } + } +} + +func TestCreateCacheConcurrentDuplicate(t *testing.T) { + m, _ := newTestMock() + + successes, codes := runConcurrently(raceGoroutines, func() error { + _, err := m.CreateCache(context.Background(), driver.CacheConfig{Name: "dup"}) + return err + }) + + assertOneWinner(t, successes, codes, FaultCacheClusterAlreadyExists) +} + +func TestCreateReplicationGroupConcurrentDuplicate(t *testing.T) { + m, _ := newTestMock() + + successes, codes := runConcurrently(raceGoroutines, func() error { + _, err := m.CreateReplicationGroup(context.Background(), driver.ReplicationGroupConfig{ID: "dup"}) + return err + }) + + assertOneWinner(t, successes, codes, FaultReplicationGroupAlreadyExists) +} + +func TestCreateCacheSubnetGroupConcurrentDuplicate(t *testing.T) { + m, _ := newTestMock() + + successes, codes := runConcurrently(raceGoroutines, func() error { + _, err := m.CreateCacheSubnetGroup(context.Background(), driver.SubnetGroupConfig{ + Name: "dup", SubnetIDs: []string{"subnet-1"}, + }) + + return err + }) + + assertOneWinner(t, successes, codes, FaultCacheSubnetGroupAlreadyExists) +} + +// TestDeleteSubnetGroupVsCreateCacheRace races cluster creates against deletes +// of the subnet group they are placed in. Either the delete loses with +// CacheSubnetGroupInUse or the create loses with CacheSubnetGroupNotFoundFault; +// a cluster must never end up pointing at a deleted group. +func TestDeleteSubnetGroupVsCreateCacheRace(t *testing.T) { + ctx := context.Background() + m, _ := newTestMock() + + for i := range raceGoroutines { + sg := fmt.Sprintf("sg-%d", i) + if _, err := m.CreateCacheSubnetGroup(ctx, driver.SubnetGroupConfig{Name: sg, SubnetIDs: []string{"subnet-1"}}); err != nil { + t.Fatalf("CreateCacheSubnetGroup: %v", err) + } + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + _, _ = m.CreateCache(ctx, driver.CacheConfig{Name: "c-" + sg, SubnetGroupName: sg}) + }() + + go func() { + defer wg.Done() + + _ = m.DeleteCacheSubnetGroup(ctx, sg) + }() + + wg.Wait() + + _, cacheErr := m.GetCache(ctx, "c-"+sg) + _, sgErr := m.DescribeCacheSubnetGroups(ctx, []string{sg}) + + if cacheErr == nil && sgErr != nil { + t.Fatalf("cluster c-%s exists but its subnet group was deleted", sg) + } + } +} + +// TestDeleteSubnetGroupWhileCacheCreating proves a cluster still being +// provisioned counts as a user of its subnet group. +func TestDeleteSubnetGroupWhileCacheCreating(t *testing.T) { + ctx := context.Background() + m, eng := newGatedMock() + + if _, err := m.CreateCacheSubnetGroup(ctx, driver.SubnetGroupConfig{Name: "sg", SubnetIDs: []string{"subnet-1"}}); err != nil { + t.Fatalf("CreateCacheSubnetGroup: %v", err) + } + + done := make(chan error, 1) + + go func() { + _, err := m.CreateCache(ctx, driver.CacheConfig{Name: "c1", SubnetGroupName: "sg"}) + done <- err + }() + + <-eng.provisionEntered + + if code := faultCode(m.DeleteCacheSubnetGroup(ctx, "sg")); code != FaultCacheSubnetGroupInUse { + t.Fatalf("DeleteCacheSubnetGroup during create: code %q, want %q", code, FaultCacheSubnetGroupInUse) + } + + close(eng.provisionRelease) + + if err := <-done; err != nil { + t.Fatalf("CreateCache: %v", err) + } +} + +// TestDeleteParameterGroupWhileCacheCreating proves a create that has +// validated a parameter group keeps it from being deleted until it commits. +func TestDeleteParameterGroupWhileCacheCreating(t *testing.T) { + ctx := context.Background() + m, eng := newGatedMock() + + if _, err := m.CreateCacheParameterGroup(ctx, "pg", "redis7", "d"); err != nil { + t.Fatalf("CreateCacheParameterGroup: %v", err) + } + + done := make(chan error, 1) + + go func() { + _, err := m.CreateCache(ctx, driver.CacheConfig{Name: "c1", ParameterGroupName: "pg"}) + done <- err + }() + + <-eng.provisionEntered + + if code := faultCode(m.DeleteCacheParameterGroup(ctx, "pg")); code != FaultInvalidCacheParameterGroupState { + t.Fatalf("DeleteCacheParameterGroup during create: code %q, want %q", code, FaultInvalidCacheParameterGroupState) + } + + close(eng.provisionRelease) + + if err := <-done; err != nil { + t.Fatalf("CreateCache: %v", err) + } +} + +// TestDeleteParameterGroupVsCreateRace races creates that name a parameter +// group against deletes of that group: a committed cluster must never name a +// deleted group. +func TestDeleteParameterGroupVsCreateRace(t *testing.T) { + ctx := context.Background() + m, _ := newTestMock() + + for i := range raceGoroutines { + pg := fmt.Sprintf("pg-%d", i) + if _, err := m.CreateCacheParameterGroup(ctx, pg, "redis7", "d"); err != nil { + t.Fatalf("CreateCacheParameterGroup: %v", err) + } + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + _, _ = m.CreateCache(ctx, driver.CacheConfig{Name: "c-" + pg, ParameterGroupName: pg}) + }() + + go func() { + defer wg.Done() + + _ = m.DeleteCacheParameterGroup(ctx, pg) + }() + + wg.Wait() + + _, cacheErr := m.GetCache(ctx, "c-"+pg) + _, pgErr := m.DescribeCacheParameterGroups(ctx, []string{pg}) + + if cacheErr == nil && pgErr != nil { + t.Fatalf("cluster c-%s exists but its parameter group was deleted", pg) + } + } +} + +// TestDescribeReservedCreatingReturnsCreating proves a cluster whose engine is +// still being provisioned is describable with status creating. +func TestDescribeReservedCreatingReturnsCreating(t *testing.T) { + ctx := context.Background() + m, eng := newGatedMock() + + done := make(chan error, 1) + + go func() { + _, err := m.CreateCache(ctx, driver.CacheConfig{Name: "c1", Engine: "redis"}) + done <- err + }() + + <-eng.provisionEntered + + info, err := m.GetCache(ctx, "c1") + if err != nil || info.Status != statusCreating { + t.Fatalf("GetCache during create = %+v, %v; want status %q", info, err, statusCreating) + } + + list, err := m.ListCaches(ctx, info.Scope) + if err != nil || len(list) != 1 || list[0].Status != statusCreating { + t.Fatalf("ListCaches during create = %+v, %v; want one creating cluster", list, err) + } + + close(eng.provisionRelease) + + if err := <-done; err != nil { + t.Fatalf("CreateCache: %v", err) + } + + if info, _ = m.GetCache(ctx, "c1"); info.Status != statusAvailable { + t.Fatalf("status after create = %q, want %q", info.Status, statusAvailable) + } +} + +// TestModifyReservedCreatingIsInvalidState proves mutating calls on a cluster +// or replication group that is still being created report the invalid-state +// fault, not not-found. +func TestModifyReservedCreatingIsInvalidState(t *testing.T) { + ctx := context.Background() + m, eng := newGatedMock() + + cacheDone := make(chan error, 1) + + go func() { + _, err := m.CreateCache(ctx, driver.CacheConfig{Name: "c1", Engine: "redis"}) + cacheDone <- err + }() + + <-eng.provisionEntered + + _, err := m.ModifyCache(ctx, driver.ModifyCacheConfig{Name: "c1", NodeType: "cache.m5.large"}) + if code := faultCode(err); code != FaultInvalidCacheClusterState { + t.Fatalf("ModifyCache during create: code %q, want %q", code, FaultInvalidCacheClusterState) + } + + if code := faultCode(m.DeleteCache(ctx, "c1")); code != FaultInvalidCacheClusterState { + t.Fatalf("DeleteCache during create: code %q, want %q", code, FaultInvalidCacheClusterState) + } + + close(eng.provisionRelease) + + if err := <-cacheDone; err != nil { + t.Fatalf("CreateCache: %v", err) + } + + eng.provisionRelease = make(chan struct{}) + rgDone := make(chan error, 1) + + go func() { + _, err := m.CreateReplicationGroup(ctx, driver.ReplicationGroupConfig{ID: "rg1", Engine: "redis"}) + rgDone <- err + }() + + <-eng.provisionEntered + + _, err = m.ModifyReplicationGroup(ctx, "rg1", 2) + if code := faultCode(err); code != FaultInvalidReplicationGroupState { + t.Fatalf("ModifyReplicationGroup during create: code %q, want %q", code, FaultInvalidReplicationGroupState) + } + + groups, err := m.DescribeReplicationGroups(ctx, []string{"rg1"}) + if err != nil || groups[0].Status != statusCreating { + t.Fatalf("DescribeReplicationGroups during create = %+v, %v; want creating", groups, err) + } + + close(eng.provisionRelease) + + if err := <-rgDone; err != nil { + t.Fatalf("CreateReplicationGroup: %v", err) + } +} + +// TestRecreateWhileDeprovisioning proves an id stays taken until the real +// engine has been torn down, so the same engine slot is never provisioned +// twice, and that Describe reports the cluster as deleting meanwhile. +func TestRecreateWhileDeprovisioning(t *testing.T) { + ctx := context.Background() + m, eng := newGatedMock() + + close(eng.provisionRelease) + + if _, err := m.CreateCache(ctx, driver.CacheConfig{Name: "c1", Engine: "redis"}); err != nil { + t.Fatalf("CreateCache: %v", err) + } + + <-eng.provisionEntered + + done := make(chan error, 1) + + go func() { done <- m.DeleteCache(ctx, "c1") }() + + <-eng.deprovisionEntered + + info, err := m.GetCache(ctx, "c1") + if err != nil || info.Status != statusDeleting { + t.Fatalf("GetCache during delete = %+v, %v; want status %q", info, err, statusDeleting) + } + + _, err = m.CreateCache(ctx, driver.CacheConfig{Name: "c1", Engine: "redis"}) + if code := faultCode(err); code != FaultCacheClusterAlreadyExists { + t.Fatalf("CreateCache during delete: code %q, want %q", code, FaultCacheClusterAlreadyExists) + } + + if code := faultCode(m.DeleteCache(ctx, "c1")); code != FaultInvalidCacheClusterState { + t.Fatalf("DeleteCache during delete: code %q, want %q", code, FaultInvalidCacheClusterState) + } + + close(eng.deprovisionRelease) + + if err := <-done; err != nil { + t.Fatalf("DeleteCache: %v", err) + } + + if _, err := m.GetCache(ctx, "c1"); !cerrors.IsNotFound(err) { + t.Fatalf("GetCache after delete: %v, want not found", err) + } + + if _, err := m.CreateCache(ctx, driver.CacheConfig{Name: "c1", Engine: "redis"}); err != nil { + t.Fatalf("CreateCache after delete: %v", err) + } +} + +// TestDescribeDuringDeprovisionReturnsDeleting covers the replication-group +// side of the deleting reservation. +func TestDescribeDuringDeprovisionReturnsDeleting(t *testing.T) { + ctx := context.Background() + m, eng := newGatedMock() + + close(eng.provisionRelease) + + if _, err := m.CreateReplicationGroup(ctx, driver.ReplicationGroupConfig{ID: "rg1", Engine: "redis"}); err != nil { + t.Fatalf("CreateReplicationGroup: %v", err) + } + + <-eng.provisionEntered + + done := make(chan error, 1) + + go func() { done <- m.DeleteReplicationGroup(ctx, "rg1", driver.DeleteReplicationGroupOptions{}) }() + + <-eng.deprovisionEntered + + groups, err := m.DescribeReplicationGroups(ctx, nil) + if err != nil || len(groups) != 1 || groups[0].Status != statusDeleting { + t.Fatalf("DescribeReplicationGroups during delete = %+v, %v; want one deleting group", groups, err) + } + + _, err = m.CreateReplicationGroup(ctx, driver.ReplicationGroupConfig{ID: "rg1"}) + if code := faultCode(err); code != FaultReplicationGroupAlreadyExists { + t.Fatalf("CreateReplicationGroup during delete: code %q, want %q", code, FaultReplicationGroupAlreadyExists) + } + + close(eng.deprovisionRelease) + + if err := <-done; err != nil { + t.Fatalf("DeleteReplicationGroup: %v", err) + } + + if _, err := m.DescribeReplicationGroups(ctx, []string{"rg1"}); faultCode(err) != FaultReplicationGroupNotFound { + t.Fatalf("DescribeReplicationGroups after delete: %v, want ReplicationGroupNotFoundFault", err) + } +} + +// TestSnapshotRestoreSerialisedWithCreate runs persist Snapshot and Restore +// alongside creates and tag writes; under -race it proves they share mu. +func TestSnapshotRestoreSerialisedWithCreate(t *testing.T) { + ctx := context.Background() + src, _ := newTestMock() + dst, _ := newTestMock() + + var wg sync.WaitGroup + + for i := range raceGoroutines { + wg.Add(1) + + go func() { + defer wg.Done() + + name := fmt.Sprintf("c%d", i) + if _, err := src.CreateCache(ctx, driver.CacheConfig{Name: name, Tags: map[string]string{"k": "v"}}); err != nil { + t.Errorf("CreateCache: %v", err) + return + } + + _ = src.AddTags(ctx, src.cacheARN("us-east-1", name), map[string]string{"i": name}) + + raw, err := src.Snapshot(ctx, true) + if err != nil { + t.Errorf("Snapshot: %v", err) + return + } + + if err := dst.Restore(ctx, raw); err != nil { + t.Errorf("Restore: %v", err) + } + }() + } + + wg.Wait() + + raw, err := src.Snapshot(ctx, true) + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + + if err := dst.Restore(ctx, raw); err != nil { + t.Fatalf("Restore: %v", err) + } + + list, err := dst.ListCaches(ctx, scope.Scope{}) + if err != nil || len(list) != raceGoroutines { + t.Fatalf("restored %d clusters (%v), want %d", len(list), err, raceGoroutines) + } +} + +func TestCreateCacheParameterGroupConcurrentDuplicate(t *testing.T) { + m, _ := newTestMock() + + successes, codes := runConcurrently(raceGoroutines, func() error { + _, err := m.CreateCacheParameterGroup(context.Background(), "dup", "redis7", "d") + return err + }) + + assertOneWinner(t, successes, codes, FaultCacheParameterGroupAlreadyExists) +} + +// TestModifyVsResetParameterGroupRace interleaves modifies, resets and reads of +// one parameter group. Under -race it proves the overrides map is never +// written while a reader holds it. +func TestModifyVsResetParameterGroupRace(t *testing.T) { + ctx := context.Background() + m, _ := newTestMock() + + if _, err := m.CreateCacheParameterGroup(ctx, "pg", "redis7", "d"); err != nil { + t.Fatalf("CreateCacheParameterGroup: %v", err) + } + + var wg sync.WaitGroup + + for i := range raceGoroutines { + wg.Add(3) + + go func() { + defer wg.Done() + + if err := m.ModifyCacheParameterGroup(ctx, "pg", []ParameterUpdate{ + {Name: "maxmemory-policy", Value: "allkeys-lru"}, {Name: "timeout", Value: strconv.Itoa(i + 20)}, + }); err != nil { + t.Errorf("ModifyCacheParameterGroup: %v", err) + } + }() + + go func() { + defer wg.Done() + + if err := m.ResetCacheParameterGroup(ctx, "pg", i%2 == 0, []string{"timeout"}); err != nil { + t.Errorf("ResetCacheParameterGroup: %v", err) + } + }() + + go func() { + defer wg.Done() + + readParameterGroup(t, m, "pg") + }() + } + + wg.Wait() +} + +// readParameterGroup reads a group's overrides and parameters the way a +// Describe caller would. +func readParameterGroup(t *testing.T, m *Mock, name string) { + t.Helper() + + groups, err := m.DescribeCacheParameterGroups(context.Background(), []string{name}) + if err != nil { + t.Errorf("DescribeCacheParameterGroups: %v", err) + return + } + + seen := 0 + for range groups[0].Overrides { + seen++ + } + + if seen > 2 { + t.Errorf("overrides = %d entries, want at most 2", seen) + } + + if _, err := m.DescribeCacheParameters(context.Background(), name, ""); err != nil { + t.Errorf("DescribeCacheParameters: %v", err) + } +} + +// snapshotRaceResult holds the outcome of one round of +// TestSnapshotVsDeleteReplicationGroupRace. +type snapshotRaceResult struct { + manual, final, copied, deleted error +} + +// TestSnapshotVsDeleteReplicationGroupRace races snapshot create, copy and +// delete against deleting the replication group they back up. The manual +// snapshot and the group's final snapshot share a name, so exactly one of them +// wins; a snapshot of the group never outlives a failed lookup; and copy and +// delete of one snapshot are serialized. +func TestSnapshotVsDeleteReplicationGroupRace(t *testing.T) { + ctx := context.Background() + m, _ := newTestMock() + + for i := range raceGoroutines { + rg := fmt.Sprintf("rg%d", i) + if _, err := m.CreateReplicationGroup(ctx, driver.ReplicationGroupConfig{ID: rg, Engine: "redis"}); err != nil { + t.Fatalf("CreateReplicationGroup: %v", err) + } + + if _, err := m.CreateSnapshot(ctx, driver.SnapshotConfig{SnapshotName: "base-" + rg, ReplicationGroupID: rg}); err != nil { + t.Fatalf("CreateSnapshot: %v", err) + } + + var ( + wg sync.WaitGroup + res snapshotRaceResult + ) + + wg.Add(4) + + go func() { + defer wg.Done() + + _, res.manual = m.CreateSnapshot(ctx, driver.SnapshotConfig{SnapshotName: "snap-" + rg, ReplicationGroupID: rg}) + }() + + go func() { + defer wg.Done() + + res.final = m.DeleteReplicationGroup(ctx, rg, driver.DeleteReplicationGroupOptions{FinalSnapshotIdentifier: "snap-" + rg}) + }() + + go func() { + defer wg.Done() + + _, res.copied = m.CopySnapshot(ctx, driver.CopySnapshotConfig{SourceSnapshotName: "base-" + rg, TargetSnapshotName: "copy-" + rg}) + }() + + go func() { + defer wg.Done() + + _, res.deleted = m.DeleteSnapshot(ctx, "base-"+rg) + }() + + wg.Wait() + + checkSnapshotRaceOutcome(t, m, rg, &res) + } +} + +// checkSnapshotRaceOutcome asserts the invariants of one round. +func checkSnapshotRaceOutcome(t *testing.T, m *Mock, rg string, res *snapshotRaceResult) { + t.Helper() + + ctx := context.Background() + + // Manual first: it wins and the final snapshot then collides, so the group + // survives. Delete first: the final snapshot wins and the manual one finds + // no group. Never both, never neither. + if (res.manual == nil) == (res.final == nil) { + t.Fatalf("%s: manual snapshot err %v, delete err %v; want exactly one to succeed", rg, res.manual, res.final) + } + + // Losing to the delete, the manual snapshot finds the final snapshot's name + // already taken (the name is checked first) or, failing that, no group. + if code := faultCode(res.manual); res.manual != nil && + code != FaultSnapshotAlreadyExists && code != FaultReplicationGroupNotFound { + t.Fatalf("%s: manual snapshot error %v, want SnapshotAlreadyExistsFault or ReplicationGroupNotFoundFault", rg, res.manual) + } + + if res.final != nil && faultCode(res.final) != FaultSnapshotAlreadyExists { + t.Fatalf("%s: delete error %v, want SnapshotAlreadyExistsFault", rg, res.final) + } + + _, describeErr := m.DescribeReplicationGroups(ctx, []string{rg}) + if (res.final == nil) != (faultCode(describeErr) == FaultReplicationGroupNotFound) { + t.Fatalf("%s: delete err %v but describe err %v", rg, res.final, describeErr) + } + + if res.deleted != nil { + t.Fatalf("%s: DeleteSnapshot of base: %v", rg, res.deleted) + } + + if res.copied != nil && faultCode(res.copied) != FaultSnapshotNotFound { + t.Fatalf("%s: CopySnapshot error %v, want nil or SnapshotNotFoundFault", rg, res.copied) + } + + if _, err := m.DescribeSnapshots(ctx, driver.SnapshotFilter{SnapshotName: "base-" + rg}); faultCode(err) != FaultSnapshotNotFound { + t.Fatalf("%s: base snapshot survived its delete: %v", rg, err) + } +} + +// TestCreateCacheCollidingWithReservedGroupMember proves a replication group +// still being created owns its member ids, so a cluster named "-002" is +// refused during the window, and that a group whose member id is already a +// cluster is refused too. +func TestCreateCacheCollidingWithReservedGroupMember(t *testing.T) { + ctx := context.Background() + m, eng := newGatedMock() + + done := make(chan error, 1) + + go func() { + _, err := m.CreateReplicationGroup(ctx, driver.ReplicationGroupConfig{ID: "rg", Engine: "redis", NumCacheNodes: 2}) + done <- err + }() + + <-eng.provisionEntered + + _, err := m.CreateCache(ctx, driver.CacheConfig{Name: "rg-002", Engine: "memcached"}) + if code := faultCode(err); code != FaultCacheClusterAlreadyExists { + t.Fatalf("CreateCache(rg-002) during group create: code %q, want %q", code, FaultCacheClusterAlreadyExists) + } + + close(eng.provisionRelease) + + if err := <-done; err != nil { + t.Fatalf("CreateReplicationGroup: %v", err) + } + + plain, _ := newTestMock() + + if _, err := plain.CreateCache(ctx, driver.CacheConfig{Name: "g2-001", Engine: "memcached"}); err != nil { + t.Fatalf("CreateCache: %v", err) + } + + _, err = plain.CreateReplicationGroup(ctx, driver.ReplicationGroupConfig{ID: "g2"}) + if code := faultCode(err); code != FaultCacheClusterAlreadyExists { + t.Fatalf("CreateReplicationGroup over a cluster's id: code %q, want %q", code, FaultCacheClusterAlreadyExists) + } +} diff --git a/providers/aws/elasticache/elasticache.go b/providers/aws/elasticache/elasticache.go index 263d823f2..02bfa98dc 100644 --- a/providers/aws/elasticache/elasticache.go +++ b/providers/aws/elasticache/elasticache.go @@ -79,6 +79,9 @@ const ( // modify before settling to available. statusCreating = "creating" statusModifying = "modifying" + // statusDeleting is what a cluster or replication group reports from its + // delete call until it is gone. + statusDeleting = "deleting" defaultRedisVersion = "7.1" defaultMemcachedVersion = "1.6.22" @@ -135,8 +138,7 @@ func normalizeNodeCount(engine string, requested int) (int, error) { // check). An empty name places the cluster in the default subnet group. func (m *Mock) requireSubnetGroup(name string) error { if name != "" && !m.subnetGroups.Has(name) { - return errors.Newf(errors.NotFound, - "CacheSubnetGroupNotFoundFault: cache subnet group %q not found", name) + return subnetGroupNotFound(name) } return nil @@ -194,8 +196,31 @@ func defaultEngineVersion(engine string) string { return defaultRedisVersion } +// reservation holds an id whose resource is mid-way through a real-engine +// provision (state creating) or deprovision (state deleting). The id cannot be +// reused until the engine call returns, and Describe reports record with that +// status, the way a real resource in those states is still visible. +type reservation[T any] struct { + state string + record T +} + // Mock is an in-memory mock implementation of the AWS ElastiCache service. +// +// Lock order: mu, then a settle set's lock. A settle lock is only taken while +// holding mu or no lock at all, never the reverse. The settle sets compute +// state from the clock on read and never call back, so nothing re-enters mu. +// No lock is held across a call to another service (monitoring, the VPC subnet +// resolver) or to the real cache engine. type Mock struct { + // mu guards the control-plane stores and the reservations below, so every + // check-then-act (duplicate checks, parent validation, in-use scans, + // cascades, Snapshot/Restore) is atomic. The per-cache key/value stores + // (cacheData.items) are data plane and rely on memstore's own locking. + mu sync.Mutex + cacheRes map[string]*reservation[driver.CacheInfo] + rgRes map[string]*reservation[driver.ReplicationGroup] + caches *memstore.Store[*cacheData] subnetGroups *memstore.Store[driver.SubnetGroup] replicationGroups *memstore.Store[driver.ReplicationGroup] @@ -214,7 +239,7 @@ type Mock struct { cacheSettle *settle.Set rgSettle *settle.Set - tagMu sync.Mutex + // tagsByARN is guarded by mu like the stores above. tagsByARN map[string]map[string]string } @@ -259,6 +284,8 @@ func New(opts *config.Options) *Mock { snapshots: memstore.New[driver.Snapshot](), cacheSettle: settle.NewSet(), rgSettle: settle.NewSet(), + cacheRes: make(map[string]*reservation[driver.CacheInfo]), + rgRes: make(map[string]*reservation[driver.ReplicationGroup]), tagsByARN: make(map[string]map[string]string), opts: opts, } @@ -277,28 +304,119 @@ func (m *Mock) settleRGStatus(id, final string) string { } // CreateCache creates a new ElastiCache cluster. +// +// The id is reserved under mu while the request is validated, the optional +// real engine is provisioned with no lock held, and the cluster is committed +// under mu again. A concurrent create of the same id sees the reservation and +// reports CacheClusterAlreadyExists. +// +//nolint:gocritic // cfg is passed by value to match the driver.Cache signature. func (m *Mock) CreateCache(ctx context.Context, cfg driver.CacheConfig) (*driver.CacheInfo, error) { - if cfg.Name == "" { - return nil, errors.New(errors.InvalidArgument, "cache name is required") + info, err := m.reserveCache(ctx, &cfg) + if err != nil { + return nil, err + } + + // Opt-in: back the cache with a real server, replacing the synthetic + // endpoint with the real host:port a client connects to. + if err = cacheengine.Provision(ctx, m.opts.CacheEngine, &info); err != nil { + m.mu.Lock() + delete(m.cacheRes, info.Name) + m.mu.Unlock() + + return nil, err } - if m.caches.Has(cfg.Name) { - return nil, errors.Newf(errors.AlreadyExists, "cache %q already exists", cfg.Name) + return m.commitCache(&info), nil +} + +// cacheIDTaken reports whether name is already used by a cluster, a pending +// reservation, or a replication-group member node. The caller holds mu. +func (m *Mock) cacheIDTaken(name string) bool { + if m.caches.Has(name) { + return true + } + + if _, reserved := m.cacheRes[name]; reserved { + return true } // A replication group's member nodes ("-001", …) are describable as // single-node cache clusters without being backed by m.caches, so guard the // same id space here. Real ElastiCache rejects a create colliding with a // member id with CacheClusterAlreadyExists. - if _, member := m.lookupMember(cfg.Name); member { - return nil, errors.Newf(errors.AlreadyExists, "cache %q already exists", cfg.Name) + if _, member := m.lookupMember(name); member { + return true } + // A replication group still being created or deleted owns its member ids + // too, or a cluster named "-001" could slip in and collide with the + // member once the group commits. + for _, r := range m.rgRes { + if slices.Contains(r.record.MemberClusters, name) { + return true + } + } + + return false +} + +// reserveCache validates a create under mu, builds the cluster record, and +// reserves its id in the creating state. +func (m *Mock) reserveCache(ctx context.Context, cfg *driver.CacheConfig) (driver.CacheInfo, error) { + if cfg.Name == "" { + return driver.CacheInfo{}, errors.New(errors.InvalidArgument, "cache name is required") + } + + m.mu.Lock() + defer m.mu.Unlock() + + if m.cacheIDTaken(cfg.Name) { + return driver.CacheInfo{}, cacheAlreadyExists(cfg.Name) + } + + info, err := m.buildCacheInfo(ctx, cfg) + if err != nil { + return driver.CacheInfo{}, err + } + + m.cacheRes[cfg.Name] = &reservation[driver.CacheInfo]{state: statusCreating, record: info} + + return info, nil +} + +// commitCache stores a provisioned cluster and drops its reservation. +func (m *Mock) commitCache(info *driver.CacheInfo) *driver.CacheInfo { + m.mu.Lock() + defer m.mu.Unlock() + + delete(m.cacheRes, info.Name) + + m.caches.Set(info.Name, &cacheData{info: *info, items: memstore.New[cacheItem]()}) + m.seedTags(info.ARN, info.Tags) + + // Under AsyncSettle a fresh cluster reports creating until the window elapses, + // matching real ElastiCache (CreateCacheCluster → creating → available). With + // the default (AsyncSettle off) SettleDuration is 0 → inactive window → + // available immediately, so nothing changes for existing callers. + m.cacheSettle.Begin(info.Name, statusCreating, m.opts.Clock.Now(), + m.opts.SettleDuration(settle.DefaultCacheSettle)) + + result := *info + result.Tags = cloneTags(info.Tags) + result.Status = m.settleCacheStatus(info.Name, result.Status) + + return &result +} + +// buildCacheInfo applies the restore seed and defaults to a create request and +// validates it against its parents. The caller holds mu. +func (m *Mock) buildCacheInfo(ctx context.Context, cfg *driver.CacheConfig) (driver.CacheInfo, error) { // A restore (SnapshotName set) seeds the unset config fields from the // snapshot before defaults are applied, so the new cluster reproduces the // source's engine/version/node-type/count/port. - if err := m.seedCacheRestore(&cfg); err != nil { - return nil, err + if err := m.seedCacheRestore(cfg); err != nil { + return driver.CacheInfo{}, err } engine := cfg.Engine @@ -307,79 +425,66 @@ func (m *Mock) CreateCache(ctx context.Context, cfg driver.CacheConfig) (*driver } if err := validateEngine(engine, false); err != nil { - return nil, err - } - - nodeType := cfg.NodeType - if nodeType == "" { - nodeType = defaultNodeType - } - - engineVersion := cfg.EngineVersion - if engineVersion == "" { - engineVersion = defaultEngineVersion(engine) + return driver.CacheInfo{}, err } numNodes, err := normalizeNodeCount(engine, cfg.NumCacheNodes) if err != nil { - return nil, err + return driver.CacheInfo{}, err } if err := m.requireSubnetGroup(cfg.SubnetGroupName); err != nil { - return nil, err + return driver.CacheInfo{}, err } if err := m.requireParameterGroup(cfg.ParameterGroupName); err != nil { - return nil, err + return driver.CacheInfo{}, err } - region := regionctx.RegionOr(ctx, m.opts.Region) - endpoint := clusterEndpoint(cfg.Name, region, engine, resolvePort(engine, cfg.Port)) + nodeType := cfg.NodeType + if nodeType == "" { + nodeType = defaultNodeType + } - tags := cloneTags(cfg.Tags) + engineVersion := cfg.EngineVersion + if engineVersion == "" { + engineVersion = defaultEngineVersion(engine) + } - info := driver.CacheInfo{ + region := regionctx.RegionOr(ctx, m.opts.Region) + + return driver.CacheInfo{ Name: cfg.Name, Scope: cfg.Scope, NodeType: nodeType, Engine: engine, EngineVersion: engineVersion, Status: statusAvailable, - Endpoint: endpoint, + Endpoint: clusterEndpoint(cfg.Name, region, engine, resolvePort(engine, cfg.Port)), ARN: m.cacheARN(region, cfg.Name), CreatedAt: m.opts.Clock.Now().UTC().Format(time.RFC3339), - Tags: tags, + Tags: cloneTags(cfg.Tags), NumCacheNodes: numNodes, SubnetGroupName: cfg.SubnetGroupName, ParameterGroupName: cfg.ParameterGroupName, AutoMinorVersionUpgrade: boolOrDefault(cfg.AutoMinorVersionUpgrade, true), - } + }, nil +} - // Opt-in: back the cache with a real server, replacing the synthetic - // endpoint with the real host:port a client connects to. - if err := cacheengine.Provision(ctx, m.opts.CacheEngine, &info); err != nil { - return nil, err +// mutableCache returns the stored cluster a mutating call targets. A cluster +// that is still being created or is being deleted reports +// InvalidCacheClusterState, not CacheClusterNotFound: in AWS it exists in that +// state. The caller holds mu. +func (m *Mock) mutableCache(name string) (*cacheData, error) { + if cd, ok := m.caches.Get(name); ok { + return cd, nil } - cd := &cacheData{ - info: info, - items: memstore.New[cacheItem](), + if r, reserved := m.cacheRes[name]; reserved { + return nil, cacheInvalidState(name, r.state) } - m.caches.Set(cfg.Name, cd) - m.seedTags(info.ARN, tags) - - // Under AsyncSettle a fresh cluster reports creating until the window elapses, - // matching real ElastiCache (CreateCacheCluster → creating → available). With - // the default (AsyncSettle off) SettleDuration is 0 → inactive window → - // available immediately, so nothing changes for existing callers. - m.cacheSettle.Begin(cfg.Name, statusCreating, m.opts.Clock.Now(), - m.opts.SettleDuration(settle.DefaultCacheSettle)) - - result := info - result.Status = m.settleCacheStatus(cfg.Name, result.Status) - - return &result, nil + return nil, cacheNotFound(name) } // ModifyCache updates the mutable fields (node type, engine version, node @@ -388,9 +493,12 @@ func (m *Mock) CreateCache(ctx context.Context, cfg driver.CacheConfig) (*driver // change is re-validated against the cluster's engine (Memcached scales 1-40; // Redis/Valkey stays at 1). func (m *Mock) ModifyCache(_ context.Context, cfg driver.ModifyCacheConfig) (*driver.CacheInfo, error) { - cd, ok := m.caches.Get(cfg.Name) - if !ok { - return nil, errors.Newf(errors.NotFound, "cache %q not found", cfg.Name) + m.mu.Lock() + defer m.mu.Unlock() + + cd, err := m.mutableCache(cfg.Name) + if err != nil { + return nil, err } if cfg.NumCacheNodes != 0 { @@ -399,23 +507,7 @@ func (m *Mock) ModifyCache(_ context.Context, cfg driver.ModifyCacheConfig) (*dr } } - if cfg.NodeType != "" { - cd.info.NodeType = cfg.NodeType - } - - if cfg.EngineVersion != "" { - cd.info.EngineVersion = cfg.EngineVersion - } - - if cfg.NumCacheNodes > 0 { - cd.info.NumCacheNodes = cfg.NumCacheNodes - } - - if cfg.AutoMinorVersionUpgrade != nil { - cd.info.AutoMinorVersionUpgrade = *cfg.AutoMinorVersionUpgrade - } - - m.caches.Set(cfg.Name, cd) + applyCacheModify(&cd.info, &cfg) // Under AsyncSettle a modified cluster briefly reports modifying before // settling back to available (ModifyCacheCluster → modifying → available); a @@ -424,67 +516,128 @@ func (m *Mock) ModifyCache(_ context.Context, cfg driver.ModifyCacheConfig) (*dr m.opts.SettleDuration(settle.DefaultCacheModifySettle)) result := cd.info + result.Tags = cloneTags(cd.info.Tags) result.Status = m.settleCacheStatus(cfg.Name, result.Status) return &result, nil } +// applyCacheModify copies the non-empty fields of a modify request onto info. +func applyCacheModify(info *driver.CacheInfo, cfg *driver.ModifyCacheConfig) { + if cfg.NodeType != "" { + info.NodeType = cfg.NodeType + } + + if cfg.EngineVersion != "" { + info.EngineVersion = cfg.EngineVersion + } + + if cfg.NumCacheNodes > 0 { + info.NumCacheNodes = cfg.NumCacheNodes + } + + if cfg.AutoMinorVersionUpgrade != nil { + info.AutoMinorVersionUpgrade = *cfg.AutoMinorVersionUpgrade + } +} + // RebootCache reboots a cache cluster (ElastiCache RebootCacheCluster), the // path real deployments use to apply pending parameter-group changes. The // cluster stays available in the emulator; a reboot metric is emitted to mirror -// the lifecycle event. +// the lifecycle event, after mu is released since monitoring is another service. func (m *Mock) RebootCache(_ context.Context, name string) (*driver.CacheInfo, error) { - cd, ok := m.caches.Get(name) - if !ok { - return nil, errors.Newf(errors.NotFound, "cache %q not found", name) - } + m.mu.Lock() - m.emitMetric("Reboots", 1, map[string]string{"CacheClusterId": name}) + cd, err := m.mutableCache(name) + if err != nil { + m.mu.Unlock() + + return nil, err + } result := cd.info + result.Tags = cloneTags(cd.info.Tags) + m.mu.Unlock() + + m.emitMetric("Reboots", 1, map[string]string{"CacheClusterId": name}) return &result, nil } // DeleteCache deletes an ElastiCache cluster by name. +// +// The cluster is removed under mu and its id is held in the deleting state +// until the real engine (if any) has been torn down, so a create of the same id +// cannot provision the engine slot while the old one is still stopping. If the +// teardown fails the cluster is put back, as it was never deleted. func (m *Mock) DeleteCache(ctx context.Context, name string) error { - cd, ok := m.caches.Get(name) - if !ok { - return errors.Newf(errors.NotFound, "cache %q not found", name) - } + m.mu.Lock() + + cd, err := m.mutableCache(name) + if err != nil { + m.mu.Unlock() - // Tear down the real cache server backing the instance, if any. - if err := cacheengine.Deprovision(ctx, m.opts.CacheEngine, &cd.info); err != nil { return err } m.caches.Delete(name) m.cacheSettle.Clear(name) + last := cd.info + last.Status = statusDeleting + m.cacheRes[name] = &reservation[driver.CacheInfo]{state: statusDeleting, record: last} + m.mu.Unlock() + + // Tear down the real cache server backing the instance, if any. + derr := cacheengine.Deprovision(ctx, m.opts.CacheEngine, &cd.info) + + m.mu.Lock() + defer m.mu.Unlock() + + delete(m.cacheRes, name) + + if derr != nil { + m.caches.Set(name, cd) + + return derr + } + return nil } // GetCache retrieves information about an ElastiCache cluster. // -// A name that is not a standalone cluster may still be a member node of a -// replication group ("-001", …); real ElastiCache describes those as -// single-node cache clusters, so fall back to synthesizing the member from its -// group before reporting not-found. IaC that manages a replication group reads -// each member back this way. +// A cluster whose real engine is still being provisioned or torn down is +// reported with status creating or deleting. A name that is not a standalone +// cluster may still be a member node of a replication group ("-001", +// …); real ElastiCache describes those as single-node cache clusters, so fall +// back to synthesizing the member from its group before reporting not-found. +// IaC that manages a replication group reads each member back this way. func (m *Mock) GetCache(_ context.Context, name string) (*driver.CacheInfo, error) { - cd, ok := m.caches.Get(name) - if !ok { - if info, found := m.lookupMember(name); found { - return &info, nil - } + m.mu.Lock() + defer m.mu.Unlock() - return nil, errors.Newf(errors.NotFound, "cache %q not found", name) + if cd, ok := m.caches.Get(name); ok { + result := cd.info + result.Tags = cloneTags(cd.info.Tags) + result.Status = m.settleCacheStatus(name, result.Status) + + return &result, nil } - result := cd.info - result.Status = m.settleCacheStatus(name, result.Status) + if r, reserved := m.cacheRes[name]; reserved { + result := r.record + result.Tags = cloneTags(r.record.Tags) + result.Status = r.state - return &result, nil + return &result, nil + } + + if info, found := m.lookupMember(name); found { + return &info, nil + } + + return nil, cacheNotFound(name) } // lookupMember returns the synthesized cache-cluster view of a replication-group @@ -544,11 +697,16 @@ func (m *Mock) memberCacheInfo(rg *driver.ReplicationGroup, memberID string) dri // included as individual single-node cache clusters, matching real ElastiCache // (DescribeCacheClusters returns them alongside standalone clusters). The result // is sorted by cluster id so the wire order is deterministic (no map-iteration -// randomness) and matches the id-ordered listing AWS returns. +// randomness) and matches the id-ordered listing AWS returns. A cluster whose +// real engine is still being provisioned or torn down is listed with status +// creating or deleting, as AWS lists a cluster in those states. func (m *Mock) ListCaches(_ context.Context, filter scope.Scope) ([]driver.CacheInfo, error) { + m.mu.Lock() + defer m.mu.Unlock() + all := m.caches.SortedValues() - caches := make([]driver.CacheInfo, 0, len(all)) + caches := make([]driver.CacheInfo, 0, len(all)+len(m.cacheRes)) for _, cd := range all { if !cd.info.Scope.Matches(filter) { @@ -556,10 +714,22 @@ func (m *Mock) ListCaches(_ context.Context, filter scope.Scope) ([]driver.Cache } info := cd.info + info.Tags = cloneTags(cd.info.Tags) info.Status = m.settleCacheStatus(info.Name, info.Status) caches = append(caches, info) } + for _, r := range m.cacheRes { + if !r.record.Scope.Matches(filter) { + continue + } + + info := r.record + info.Tags = cloneTags(r.record.Tags) + info.Status = r.state + caches = append(caches, info) + } + groups := m.replicationGroups.SortedValues() for i := range groups { for _, memberID := range groups[i].MemberClusters { @@ -583,27 +753,31 @@ func (m *Mock) ListCaches(_ context.Context, filter scope.Scope) ([]driver.Cache // CreateOrUpdate-on-existing semantics (node type and tags come from the // request; identity, endpoint, and CreatedAt are preserved). func (m *Mock) UpdateCache(_ context.Context, cfg driver.CacheConfig) (*driver.CacheInfo, error) { - cd, ok := m.caches.Get(cfg.Name) - if !ok { - return nil, errors.Newf(errors.NotFound, "cache %q not found", cfg.Name) + m.mu.Lock() + defer m.mu.Unlock() + + cd, err := m.mutableCache(cfg.Name) + if err != nil { + return nil, err } if cfg.NodeType != "" { cd.info.NodeType = cfg.NodeType } + if cfg.Tags != nil { cd.info.Tags = maps.Clone(cfg.Tags) } + if !cfg.Scope.IsZero() { cd.info.Scope = cfg.Scope } - m.caches.Set(cfg.Name, cd) - m.cacheSettle.Begin(cfg.Name, statusModifying, m.opts.Clock.Now(), m.opts.SettleDuration(settle.DefaultCacheModifySettle)) result := cd.info + result.Tags = cloneTags(cd.info.Tags) result.Status = m.settleCacheStatus(cfg.Name, result.Status) return &result, nil diff --git a/providers/aws/elasticache/faults.go b/providers/aws/elasticache/faults.go new file mode 100644 index 000000000..c56d44f7f --- /dev/null +++ b/providers/aws/elasticache/faults.go @@ -0,0 +1,95 @@ +package elasticache + +import ( + cerrors "github.com/stackshy/cloudemu/v2/errors" +) + +// ElastiCache wire fault codes, as the AWS API (botocore error.code) spells +// them. A caller matching the SDK's typed errors sees this code, so it travels +// on the error as a typed field rather than inside the message text. +const ( + FaultCacheClusterNotFound = "CacheClusterNotFound" + FaultCacheClusterAlreadyExists = "CacheClusterAlreadyExists" + FaultInvalidCacheClusterState = "InvalidCacheClusterState" + FaultReplicationGroupNotFound = "ReplicationGroupNotFoundFault" + FaultReplicationGroupAlreadyExists = "ReplicationGroupAlreadyExists" + FaultInvalidReplicationGroupState = "InvalidReplicationGroupState" + FaultCacheSubnetGroupNotFound = "CacheSubnetGroupNotFoundFault" + FaultCacheSubnetGroupAlreadyExists = "CacheSubnetGroupAlreadyExists" + FaultCacheSubnetGroupInUse = "CacheSubnetGroupInUse" + FaultCacheParameterGroupNotFound = "CacheParameterGroupNotFound" + FaultCacheParameterGroupAlreadyExists = "CacheParameterGroupAlreadyExists" + FaultInvalidCacheParameterGroupState = "InvalidCacheParameterGroupState" + FaultSnapshotNotFound = "SnapshotNotFoundFault" + FaultSnapshotAlreadyExists = "SnapshotAlreadyExistsFault" + FaultSnapshotFeatureNotSupported = "SnapshotFeatureNotSupportedFault" +) + +// Fault is an ElastiCache error that carries its AWS wire fault code apart from +// the human message. Error() keeps the canonical cloudemu form; it unwraps to +// the canonical error, so cerrors.IsNotFound and friends keep working, and +// cerrors.Message returns the message without any code token, which is what +// real ElastiCache puts in . +type Fault struct { + err *cerrors.Error + code string +} + +// Error implements error. +func (f *Fault) Error() string { return f.err.Error() } + +// Unwrap returns the canonical cloudemu error. +func (f *Fault) Unwrap() error { return f.err } + +// AWSFault returns the ElastiCache wire fault code. +func (f *Fault) AWSFault() string { return f.code } + +// newFault builds a Fault of the given canonical kind and wire code. +func newFault(kind cerrors.Code, code, format string, args ...any) *Fault { + return &Fault{err: cerrors.Newf(kind, format, args...), code: code} +} + +func cacheNotFound(name string) *Fault { + return newFault(cerrors.NotFound, FaultCacheClusterNotFound, "cache cluster %q not found", name) +} + +func cacheAlreadyExists(name string) *Fault { + return newFault(cerrors.AlreadyExists, FaultCacheClusterAlreadyExists, "cache cluster %q already exists", name) +} + +// cacheInvalidState reports a mutating call on a cache cluster whose status +// does not allow it (it is still being created or is being deleted). +func cacheInvalidState(name, status string) *Fault { + return newFault(cerrors.FailedPrecondition, FaultInvalidCacheClusterState, + "cache cluster %q is not available (status %s)", name, status) +} + +func rgNotFound(id string) *Fault { + return newFault(cerrors.NotFound, FaultReplicationGroupNotFound, "replication group %q not found", id) +} + +func rgAlreadyExists(id string) *Fault { + return newFault(cerrors.AlreadyExists, FaultReplicationGroupAlreadyExists, + "replication group %q already exists", id) +} + +func rgInvalidState(id, status string) *Fault { + return newFault(cerrors.FailedPrecondition, FaultInvalidReplicationGroupState, + "replication group %q is not available (status %s)", id, status) +} + +func subnetGroupNotFound(name string) *Fault { + return newFault(cerrors.NotFound, FaultCacheSubnetGroupNotFound, "cache subnet group %q not found", name) +} + +func parameterGroupNotFound(name string) *Fault { + return newFault(cerrors.NotFound, FaultCacheParameterGroupNotFound, "cache parameter group %q not found", name) +} + +func snapshotNotFound(name string) *Fault { + return newFault(cerrors.NotFound, FaultSnapshotNotFound, "snapshot %q not found", name) +} + +func snapshotAlreadyExists(name string) *Fault { + return newFault(cerrors.AlreadyExists, FaultSnapshotAlreadyExists, "snapshot %q already exists", name) +} diff --git a/providers/aws/elasticache/parameters.go b/providers/aws/elasticache/parameters.go index 35cf33800..7c285cdbe 100644 --- a/providers/aws/elasticache/parameters.go +++ b/providers/aws/elasticache/parameters.go @@ -134,9 +134,12 @@ func defaultParameterIndex(family string) map[string]Parameter { // "engine-default" (only unmodified defaults). A missing group reports // CacheParameterGroupNotFound. func (m *Mock) DescribeCacheParameters(_ context.Context, name, source string) ([]Parameter, error) { + m.mu.Lock() pg, ok := m.lookupParameterGroup(name) + m.mu.Unlock() + if !ok { - return nil, cerrors.Newf(cerrors.NotFound, "cache parameter group %q not found", name) + return nil, parameterGroupNotFound(name) } defaults := defaultCacheParameters(pg.Family) @@ -186,9 +189,12 @@ func (m *Mock) ModifyCacheParameterGroup(_ context.Context, name string, updates return err } + m.mu.Lock() + defer m.mu.Unlock() + pg, ok := m.parameterGroups.Get(name) if !ok { - return cerrors.Newf(cerrors.NotFound, "cache parameter group %q not found", name) + return parameterGroupNotFound(name) } idx := defaultParameterIndex(pg.Family) @@ -204,9 +210,9 @@ func (m *Mock) ModifyCacheParameterGroup(_ context.Context, name string, updates } } - if pg.Overrides == nil { - pg.Overrides = make(map[string]string, len(updates)) - } + // Copy on write: a Describe result handed out earlier may still hold the old + // map, so it is never mutated in place. + pg.Overrides = cloneOverrides(pg.Overrides, len(updates)) for _, u := range updates { pg.Overrides[u.Name] = u.Value @@ -226,9 +232,12 @@ func (m *Mock) ResetCacheParameterGroup(_ context.Context, name string, resetAll return err } + m.mu.Lock() + defer m.mu.Unlock() + pg, ok := m.parameterGroups.Get(name) if !ok { - return cerrors.Newf(cerrors.NotFound, "cache parameter group %q not found", name) + return parameterGroupNotFound(name) } if resetAll { @@ -245,6 +254,8 @@ func (m *Mock) ResetCacheParameterGroup(_ context.Context, name string, resetAll } } + pg.Overrides = cloneOverrides(pg.Overrides, 0) + for _, n := range names { delete(pg.Overrides, n) } @@ -253,3 +264,14 @@ func (m *Mock) ResetCacheParameterGroup(_ context.Context, name string, resetAll return nil } + +// cloneOverrides returns a writable copy of a parameter group's overrides, +// sized for extra more entries. +func cloneOverrides(in map[string]string, extra int) map[string]string { + out := make(map[string]string, len(in)+extra) + for k, v := range in { + out[k] = v + } + + return out +} diff --git a/providers/aws/elasticache/persist_snapshot.go b/providers/aws/elasticache/persist_snapshot.go index c89f8e6e4..49af55f58 100644 --- a/providers/aws/elasticache/persist_snapshot.go +++ b/providers/aws/elasticache/persist_snapshot.go @@ -39,7 +39,15 @@ type cacheDataSnapshot struct { // Snapshot captures the mock's entire state as JSON. includeAssets is unused. Cache entry values // are the payload and are always captured. +// +// It holds mu for the whole capture (including the marshal, since the tag map +// is captured by reference), so it never sees a half-applied create or delete. +// A cluster or replication group whose real engine is still being provisioned +// or torn down is not part of the captured state. func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { + m.mu.Lock() + defer m.mu.Unlock() + var snap persistSnapshot if err := m.snapshotStores(&snap); err != nil { return nil, err @@ -52,9 +60,7 @@ func (m *Mock) Snapshot(_ context.Context, _ bool) (json.RawMessage, error) { snap.Caches = caches - m.tagMu.Lock() snap.TagsByARN = m.tagsByARN - m.tagMu.Unlock() return json.Marshal(snap) } @@ -109,6 +115,11 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { return fmt.Errorf("elasticache: parse snapshot: %w", err) } + // Restore swaps every store in one critical section, so it is serialized + // against in-flight creates and deletes. + m.mu.Lock() + defer m.mu.Unlock() + if err := m.restoreStores(&snap); err != nil { return err } @@ -118,9 +129,7 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { } if snap.TagsByARN != nil { - m.tagMu.Lock() m.tagsByARN = snap.TagsByARN - m.tagMu.Unlock() } return nil diff --git a/providers/aws/elasticache/replication_group.go b/providers/aws/elasticache/replication_group.go index d5166af56..968457c11 100644 --- a/providers/aws/elasticache/replication_group.go +++ b/providers/aws/elasticache/replication_group.go @@ -4,7 +4,9 @@ import ( "context" "fmt" "net" + "slices" "strconv" + "strings" "time" cerrors "github.com/stackshy/cloudemu/v2/errors" @@ -20,22 +22,102 @@ import ( // A duplicate id reports ReplicationGroupAlreadyExists: callers re-running a // provision treat that specific code as "already there, carry on", so folding // it into a generic error would turn a safe re-run into a hard failure. +// +// The id is reserved under mu while the request is validated, the optional real +// engine is provisioned with no lock held, and the group is committed under mu. func (m *Mock) CreateReplicationGroup( ctx context.Context, cfg cachedriver.ReplicationGroupConfig, ) (*cachedriver.ReplicationGroup, error) { + rg, err := m.reserveReplicationGroup(ctx, &cfg) + if err != nil { + return nil, err + } + + // Opt-in: back the primary with a real Redis server, replacing the synthetic + // hostname with the host:port a client connects to. Routes through the same + // cacheengine helper the cache-cluster path uses so the two cannot drift. + if err = m.provisionPrimaryEndpoint(ctx, &rg); err != nil { + m.mu.Lock() + delete(m.rgRes, rg.ID) + m.mu.Unlock() + + return nil, err + } + + return m.commitReplicationGroup(&rg), nil +} + +// reserveReplicationGroup validates a create under mu, builds the group, and +// reserves its id in the creating state. +func (m *Mock) reserveReplicationGroup( + ctx context.Context, cfg *cachedriver.ReplicationGroupConfig, +) (cachedriver.ReplicationGroup, error) { if cfg.ID == "" { - return nil, cerrors.New(cerrors.InvalidArgument, "ReplicationGroupId is required") + return cachedriver.ReplicationGroup{}, cerrors.New(cerrors.InvalidArgument, "ReplicationGroupId is required") + } + + m.mu.Lock() + defer m.mu.Unlock() + + if _, reserved := m.rgRes[cfg.ID]; reserved || m.replicationGroups.Has(cfg.ID) { + return cachedriver.ReplicationGroup{}, rgAlreadyExists(cfg.ID) + } + + rg, err := m.buildReplicationGroup(ctx, cfg) + if err != nil { + return cachedriver.ReplicationGroup{}, err } - if m.replicationGroups.Has(cfg.ID) { - return nil, cerrors.Newf(cerrors.AlreadyExists, - "ReplicationGroupAlreadyExists: replication group %q already exists", cfg.ID) + // Each member is a describable cache cluster, so a member id already used by + // a cluster (stored or reserved) is a CacheClusterAlreadyExists, as in AWS. + for _, member := range rg.MemberClusters { + if _, reserved := m.cacheRes[member]; reserved || m.caches.Has(member) { + return cachedriver.ReplicationGroup{}, cacheAlreadyExists(member) + } } + m.rgRes[cfg.ID] = &reservation[cachedriver.ReplicationGroup]{state: statusCreating, record: rg} + + return rg, nil +} + +// commitReplicationGroup stores a provisioned group and drops its reservation. +func (m *Mock) commitReplicationGroup(rg *cachedriver.ReplicationGroup) *cachedriver.ReplicationGroup { + m.mu.Lock() + defer m.mu.Unlock() + + delete(m.rgRes, rg.ID) + m.replicationGroups.Set(rg.ID, *rg) + + // Under AsyncSettle a fresh group reports creating until the window elapses + // (CreateReplicationGroup → creating → available); a no-op when settle is off. + m.rgSettle.Begin(rg.ID, statusCreating, m.opts.Clock.Now(), + m.opts.SettleDuration(settle.DefaultCacheSettle)) + + result := cloneReplicationGroup(rg) + result.Status = m.settleRGStatus(rg.ID, result.Status) + + return &result +} + +// cloneReplicationGroup copies a group so callers cannot alias its stored +// member list. +func cloneReplicationGroup(rg *cachedriver.ReplicationGroup) cachedriver.ReplicationGroup { + out := *rg + out.MemberClusters = append([]string(nil), rg.MemberClusters...) + + return out +} + +// buildReplicationGroup applies the restore seed and defaults to a create +// request and validates it against its parents. The caller holds mu. +func (m *Mock) buildReplicationGroup( + ctx context.Context, cfg *cachedriver.ReplicationGroupConfig, +) (cachedriver.ReplicationGroup, error) { // A restore (SnapshotName set) seeds the unset config fields from the // snapshot before defaults are applied. - if err := m.seedReplicationGroupRestore(&cfg); err != nil { - return nil, err + if err := m.seedReplicationGroupRestore(cfg); err != nil { + return cachedriver.ReplicationGroup{}, err } engine := cfg.Engine @@ -44,15 +126,15 @@ func (m *Mock) CreateReplicationGroup( } if err := validateEngine(engine, true); err != nil { - return nil, err + return cachedriver.ReplicationGroup{}, err } if err := validateReplicationNodeCount(cfg.NumCacheNodes); err != nil { - return nil, err + return cachedriver.ReplicationGroup{}, err } if err := m.requireParameterGroup(cfg.ParameterGroupName); err != nil { - return nil, err + return cachedriver.ReplicationGroup{}, err } nodeType := cfg.NodeType @@ -60,10 +142,7 @@ func (m *Mock) CreateReplicationGroup( nodeType = defaultNodeType } - nodes := cfg.NumCacheNodes - if nodes < 1 { - nodes = 1 - } + nodes := max(cfg.NumCacheNodes, 1) engineVersion := cfg.EngineVersion if engineVersion == "" { @@ -71,7 +150,8 @@ func (m *Mock) CreateReplicationGroup( } region := regionctx.RegionOr(ctx, m.opts.Region) - rg := cachedriver.ReplicationGroup{ + + return cachedriver.ReplicationGroup{ ID: cfg.ID, Description: cfg.Description, Status: statusAvailable, @@ -81,39 +161,17 @@ func (m *Mock) CreateReplicationGroup( NumCacheNodes: nodes, // Callers read the primary endpoint to build a connection string; a // group without one is indistinguishable from a broken provision. - PrimaryAddress: fmt.Sprintf("%s.%s.cache.amazonaws.com", - cfg.ID, region), - PrimaryPort: defaultRedisPort, + PrimaryAddress: fmt.Sprintf("%s.%s.cache.amazonaws.com", cfg.ID, region), + PrimaryPort: defaultRedisPort, // The reader endpoint lets clients scale reads across the replicas. - ReaderAddress: fmt.Sprintf("%s-ro.%s.cache.amazonaws.com", - cfg.ID, region), + ReaderAddress: fmt.Sprintf("%s-ro.%s.cache.amazonaws.com", cfg.ID, region), ReaderPort: defaultRedisPort, MemberClusters: memberClusters(cfg.ID, nodes), AutomaticFailover: failoverStatus(cfg.AutomaticFailoverEnabled), SubnetGroupName: cfg.SubnetGroupName, ParameterGroupName: cfg.ParameterGroupName, - ARN: "arn:aws:elasticache:" + region + ":" + m.opts.AccountID + - ":replicationgroup:" + cfg.ID, - } - - // Opt-in: back the primary with a real Redis server, replacing the synthetic - // hostname with the host:port a client connects to. Routes through the same - // cacheengine helper the cache-cluster path uses so the two cannot drift. - if err := m.provisionPrimaryEndpoint(ctx, &rg); err != nil { - return nil, err - } - - m.replicationGroups.Set(cfg.ID, rg) - - // Under AsyncSettle a fresh group reports creating until the window elapses - // (CreateReplicationGroup → creating → available); a no-op when settle is off. - m.rgSettle.Begin(cfg.ID, statusCreating, m.opts.Clock.Now(), - m.opts.SettleDuration(settle.DefaultCacheSettle)) - - result := rg - result.Status = m.settleRGStatus(cfg.ID, result.Status) - - return &result, nil + ARN: "arn:aws:elasticache:" + region + ":" + m.opts.AccountID + ":replicationgroup:" + cfg.ID, + }, nil } // maxReplicationGroupNodes bounds the member-cluster count a replication group @@ -198,42 +256,95 @@ func splitEndpoint(endpoint string) (host string, port int, err error) { } // DescribeReplicationGroups returns the named groups, or all when none given. +// A group whose real engine is still being provisioned or torn down is reported +// with status creating or deleting. func (m *Mock) DescribeReplicationGroups( _ context.Context, ids []string, ) ([]cachedriver.ReplicationGroup, error) { + m.mu.Lock() + defer m.mu.Unlock() + if len(ids) == 0 { - all := m.replicationGroups.SortedValues() - for i := range all { - all[i].Status = m.settleRGStatus(all[i].ID, all[i].Status) + stored := m.replicationGroups.SortedValues() + out := make([]cachedriver.ReplicationGroup, 0, len(stored)+len(m.rgRes)) + + for i := range stored { + rg := cloneReplicationGroup(&stored[i]) + rg.Status = m.settleRGStatus(rg.ID, rg.Status) + out = append(out, rg) + } + + for _, r := range m.rgRes { + rg := cloneReplicationGroup(&r.record) + rg.Status = r.state + out = append(out, rg) } - return all, nil + slices.SortFunc(out, func(a, b cachedriver.ReplicationGroup) int { return strings.Compare(a.ID, b.ID) }) + + return out, nil } out := make([]cachedriver.ReplicationGroup, 0, len(ids)) for _, id := range ids { - rg, ok := m.replicationGroups.Get(id) - if !ok { - return nil, cerrors.Newf(cerrors.NotFound, - "ReplicationGroupNotFoundFault: replication group %q not found", id) + rg, err := m.describeReplicationGroup(id) + if err != nil { + return nil, err } - rg.Status = m.settleRGStatus(id, rg.Status) out = append(out, rg) } return out, nil } +// describeReplicationGroup reads one group, falling back to a pending +// reservation. The caller holds mu. +func (m *Mock) describeReplicationGroup(id string) (cachedriver.ReplicationGroup, error) { + if rg, ok := m.replicationGroups.Get(id); ok { + out := cloneReplicationGroup(&rg) + out.Status = m.settleRGStatus(id, out.Status) + + return out, nil + } + + if r, reserved := m.rgRes[id]; reserved { + out := cloneReplicationGroup(&r.record) + out.Status = r.state + + return out, nil + } + + return cachedriver.ReplicationGroup{}, rgNotFound(id) +} + +// mutableReplicationGroup returns the stored group a mutating call targets. A +// group that is still being created or is being deleted reports +// InvalidReplicationGroupState, not ReplicationGroupNotFoundFault. The caller +// holds mu. +func (m *Mock) mutableReplicationGroup(id string) (cachedriver.ReplicationGroup, error) { + if rg, ok := m.replicationGroups.Get(id); ok { + return rg, nil + } + + if r, reserved := m.rgRes[id]; reserved { + return cachedriver.ReplicationGroup{}, rgInvalidState(id, r.state) + } + + return cachedriver.ReplicationGroup{}, rgNotFound(id) +} + // ModifyReplicationGroup changes the node count of an existing group. func (m *Mock) ModifyReplicationGroup( _ context.Context, id string, numCacheNodes int, ) (*cachedriver.ReplicationGroup, error) { - rg, ok := m.replicationGroups.Get(id) - if !ok { - return nil, cerrors.Newf(cerrors.NotFound, - "ReplicationGroupNotFoundFault: replication group %q not found", id) + m.mu.Lock() + defer m.mu.Unlock() + + rg, err := m.mutableReplicationGroup(id) + if err != nil { + return nil, err } if err := validateReplicationNodeCount(numCacheNodes); err != nil { @@ -252,7 +363,7 @@ func (m *Mock) ModifyReplicationGroup( m.rgSettle.Begin(id, statusModifying, m.opts.Clock.Now(), m.opts.SettleDuration(settle.DefaultCacheModifySettle)) - result := rg + result := cloneReplicationGroup(&rg) result.Status = m.settleRGStatus(id, result.Status) return &result, nil @@ -264,10 +375,12 @@ func (m *Mock) ModifyReplicationGroup( func (m *Mock) ModifyReplicationGroupParameterGroup( _ context.Context, id, name string, ) (*cachedriver.ReplicationGroup, error) { - rg, ok := m.replicationGroups.Get(id) - if !ok { - return nil, cerrors.Newf(cerrors.NotFound, - "ReplicationGroupNotFoundFault: replication group %q not found", id) + m.mu.Lock() + defer m.mu.Unlock() + + rg, err := m.mutableReplicationGroup(id) + if err != nil { + return nil, err } if err := m.requireParameterGroup(name); err != nil { @@ -277,7 +390,7 @@ func (m *Mock) ModifyReplicationGroupParameterGroup( rg.ParameterGroupName = name m.replicationGroups.Set(id, rg) - result := rg + result := cloneReplicationGroup(&rg) result.Status = m.settleRGStatus(id, result.Status) return &result, nil @@ -288,48 +401,70 @@ func (m *Mock) ModifyReplicationGroupParameterGroup( // group (which then shows up in DescribeSnapshots, as real ElastiCache does). // When opts.RetainPrimaryCluster is set the primary node group is kept as a // standalone cache cluster instead of being torn down. +// +// Otherwise the group is removed under mu and its id is held in the deleting +// state until the real engine (if any) has been torn down. If the teardown +// fails the group is put back, as it was never deleted. func (m *Mock) DeleteReplicationGroup( ctx context.Context, id string, opts cachedriver.DeleteReplicationGroupOptions, ) error { - rg, ok := m.replicationGroups.Get(id) - if !ok { - return cerrors.Newf(cerrors.NotFound, - "ReplicationGroupNotFoundFault: replication group %q not found", id) + m.mu.Lock() + + rg, err := m.mutableReplicationGroup(id) + if err != nil { + m.mu.Unlock() + + return err } if opts.FinalSnapshotIdentifier != "" { - if _, err := m.CreateSnapshot(ctx, cachedriver.SnapshotConfig{ + if _, err := m.createSnapshotLocked(ctx, cachedriver.SnapshotConfig{ SnapshotName: opts.FinalSnapshotIdentifier, ReplicationGroupID: id, }); err != nil { + m.mu.Unlock() + return err } } + m.replicationGroups.Delete(id) + m.rgSettle.Clear(id) + if opts.RetainPrimaryCluster { m.retainPrimaryCluster(&rg) - m.replicationGroups.Delete(id) - m.rgSettle.Clear(id) + m.mu.Unlock() return nil } + last := cloneReplicationGroup(&rg) + last.Status = statusDeleting + m.rgRes[id] = &reservation[cachedriver.ReplicationGroup]{state: statusDeleting, record: last} + m.mu.Unlock() + info := cachedriver.CacheInfo{Name: rg.ID, Engine: rg.Engine} - if err := cacheengine.Deprovision(ctx, m.opts.CacheEngine, &info); err != nil { - return err - } + derr := cacheengine.Deprovision(ctx, m.opts.CacheEngine, &info) - m.replicationGroups.Delete(id) - m.rgSettle.Clear(id) + m.mu.Lock() + defer m.mu.Unlock() + + delete(m.rgRes, id) + + if derr != nil { + m.replicationGroups.Set(id, rg) + + return derr + } return nil } // retainPrimaryCluster keeps the primary node group of a deleted replication // group as a standalone cache cluster, so DescribeCacheClusters still returns -// it. The backing engine (if any) is handed over untouched. +// it. The backing engine (if any) is handed over untouched. The caller holds mu. func (m *Mock) retainPrimaryCluster(rg *cachedriver.ReplicationGroup) { - if m.caches.Has(rg.ID) { + if m.caches.Has(rg.ID) || m.cacheRes[rg.ID] != nil { return } diff --git a/providers/aws/elasticache/snapshot.go b/providers/aws/elasticache/snapshot.go index f48743ede..66a9f083a 100644 --- a/providers/aws/elasticache/snapshot.go +++ b/providers/aws/elasticache/snapshot.go @@ -20,14 +20,22 @@ func (m *Mock) snapshotARN(region, name string) string { // type, version, port, node count) is copied into the snapshot so a restore can // recreate a like-for-like cluster. func (m *Mock) CreateSnapshot(ctx context.Context, cfg cachedriver.SnapshotConfig) (*cachedriver.Snapshot, error) { + m.mu.Lock() + defer m.mu.Unlock() + + return m.createSnapshotLocked(ctx, cfg) +} + +// createSnapshotLocked is CreateSnapshot for a caller that already holds mu, +// such as DeleteReplicationGroup taking a final snapshot. +func (m *Mock) createSnapshotLocked(ctx context.Context, cfg cachedriver.SnapshotConfig) (*cachedriver.Snapshot, error) { name := strings.ToLower(cfg.SnapshotName) if name == "" { return nil, cerrors.New(cerrors.InvalidArgument, "SnapshotName is required") } if m.snapshots.Has(name) { - return nil, cerrors.Newf(cerrors.AlreadyExists, - "SnapshotAlreadyExistsFault: snapshot %q already exists", name) + return nil, snapshotAlreadyExists(name) } // A snapshot is a child of the cluster or replication group it backs up, so @@ -61,10 +69,9 @@ func (m *Mock) CreateSnapshot(ctx context.Context, cfg cachedriver.SnapshotConfi func (m *Mock) fillSnapshotSource(cfg cachedriver.SnapshotConfig, snap *cachedriver.Snapshot, fallbackRegion string) error { switch { case cfg.CacheClusterID != "": - cd, ok := m.caches.Get(cfg.CacheClusterID) - if !ok { - return cerrors.Newf(cerrors.NotFound, - "CacheClusterNotFound: cache cluster %q not found", cfg.CacheClusterID) + cd, lookupErr := m.mutableCache(cfg.CacheClusterID) + if lookupErr != nil { + return lookupErr } if err := checkSnapshotSupported(cd.info.Engine, cd.info.NodeType); err != nil { @@ -82,10 +89,9 @@ func (m *Mock) fillSnapshotSource(cfg cachedriver.SnapshotConfig, snap *cachedri snap.NumCacheNodes = cd.info.NumCacheNodes } case cfg.ReplicationGroupID != "": - rg, ok := m.replicationGroups.Get(cfg.ReplicationGroupID) - if !ok { - return cerrors.Newf(cerrors.NotFound, - "ReplicationGroupNotFoundFault: replication group %q not found", cfg.ReplicationGroupID) + rg, err := m.mutableReplicationGroup(cfg.ReplicationGroupID) + if err != nil { + return err } snap.ReplicationGroupID = cfg.ReplicationGroupID @@ -112,13 +118,13 @@ func (m *Mock) fillSnapshotSource(cfg cachedriver.SnapshotConfig, snap *cachedri // running on a cache.t1.micro node with SnapshotFeatureNotSupportedFault. func checkSnapshotSupported(engine, nodeType string) error { if engine == engineMemcached { - return cerrors.New(cerrors.FailedPrecondition, - "SnapshotFeatureNotSupportedFault: snapshots are not supported for Memcached clusters") + return newFault(cerrors.FailedPrecondition, FaultSnapshotFeatureNotSupported, + "snapshots are not supported for Memcached clusters") } if nodeType == "cache.t1.micro" { - return cerrors.New(cerrors.FailedPrecondition, - "SnapshotFeatureNotSupportedFault: snapshots are not supported for cache.t1.micro nodes") + return newFault(cerrors.FailedPrecondition, FaultSnapshotFeatureNotSupported, + "snapshots are not supported for cache.t1.micro nodes") } return nil @@ -148,10 +154,12 @@ func (m *Mock) CopySnapshot(ctx context.Context, cfg cachedriver.CopySnapshotCon return nil, cerrors.New(cerrors.InvalidArgument, "TargetSnapshotName is required") } + m.mu.Lock() + defer m.mu.Unlock() + src, ok := m.snapshots.Get(source) if !ok { - return nil, cerrors.Newf(cerrors.NotFound, - "SnapshotNotFoundFault: snapshot %q not found", source) + return nil, snapshotNotFound(source) } region := regionctx.RegionOr(ctx, m.opts.Region) @@ -166,8 +174,7 @@ func (m *Mock) CopySnapshot(ctx context.Context, cfg cachedriver.CopySnapshotCon // SetIfAbsent is the atomic create-if-new guard: a concurrent copy to the same // target loses the race and reports the duplicate, never silently overwrites. if !m.snapshots.SetIfAbsent(target, dst) { - return nil, cerrors.Newf(cerrors.AlreadyExists, - "SnapshotAlreadyExistsFault: snapshot %q already exists", target) + return nil, snapshotAlreadyExists(target) } return &dst, nil @@ -179,18 +186,15 @@ func (m *Mock) CopySnapshot(ctx context.Context, cfg cachedriver.CopySnapshotCon func (m *Mock) DeleteSnapshot(_ context.Context, name string) (*cachedriver.Snapshot, error) { key := strings.ToLower(name) + m.mu.Lock() + defer m.mu.Unlock() + snap, ok := m.snapshots.Get(key) if !ok { - return nil, cerrors.Newf(cerrors.NotFound, - "SnapshotNotFoundFault: snapshot %q not found", key) + return nil, snapshotNotFound(key) } - // Gate the response on the delete winning: if a concurrent caller removed the - // snapshot between the Get and here, report the not-found it now is. - if !m.snapshots.Delete(key) { - return nil, cerrors.Newf(cerrors.NotFound, - "SnapshotNotFoundFault: snapshot %q not found", key) - } + m.snapshots.Delete(key) snap.Status = "deleting" @@ -209,8 +213,7 @@ func (m *Mock) restoreSeed(name string) (*cachedriver.Snapshot, error) { snap, ok := m.snapshots.Get(key) if !ok { - return nil, cerrors.Newf(cerrors.NotFound, - "SnapshotNotFoundFault: snapshot %q not found", key) + return nil, snapshotNotFound(key) } return &snap, nil @@ -281,11 +284,13 @@ func (m *Mock) seedReplicationGroupRestore(cfg *cachedriver.ReplicationGroupConf func (m *Mock) DescribeSnapshots( _ context.Context, filter cachedriver.SnapshotFilter, ) ([]cachedriver.Snapshot, error) { + m.mu.Lock() + defer m.mu.Unlock() + if name := strings.ToLower(filter.SnapshotName); name != "" { snap, ok := m.snapshots.Get(name) if !ok { - return nil, cerrors.Newf(cerrors.NotFound, - "SnapshotNotFoundFault: snapshot %q not found", name) + return nil, snapshotNotFound(name) } return []cachedriver.Snapshot{snap}, nil diff --git a/providers/aws/elasticache/subnet_group.go b/providers/aws/elasticache/subnet_group.go index c8368f4c6..15c96b5a0 100644 --- a/providers/aws/elasticache/subnet_group.go +++ b/providers/aws/elasticache/subnet_group.go @@ -34,11 +34,8 @@ func (m *Mock) CreateCacheSubnetGroup( return nil, cerrors.New(cerrors.InvalidArgument, "at least one subnet is required") } - if m.subnetGroups.Has(cfg.Name) { - return nil, cerrors.Newf(cerrors.AlreadyExists, - "CacheSubnetGroupAlreadyExists: cache subnet group %q already exists", cfg.Name) - } - + // The VPC lookup is a call into the networking mock, so it runs before mu + // is taken. sg := cachedriver.SubnetGroup{ Name: cfg.Name, Description: cfg.Description, @@ -48,15 +45,28 @@ func (m *Mock) CreateCacheSubnetGroup( ARN: "arn:aws:elasticache:" + regionctx.RegionOr(ctx, m.opts.Region) + ":" + m.opts.AccountID + ":subnetgroup:" + cfg.Name, } - m.subnetGroups.Set(cfg.Name, sg) - return &sg, nil + m.mu.Lock() + defer m.mu.Unlock() + + if !m.subnetGroups.SetIfAbsent(cfg.Name, sg) { + return nil, newFault(cerrors.AlreadyExists, FaultCacheSubnetGroupAlreadyExists, + "cache subnet group %q already exists", cfg.Name) + } + + out := sg + out.SubnetIDs = append([]string(nil), sg.SubnetIDs...) + + return &out, nil } // DescribeCacheSubnetGroups returns the named groups, or all when none given. func (m *Mock) DescribeCacheSubnetGroups( _ context.Context, names []string, ) ([]cachedriver.SubnetGroup, error) { + m.mu.Lock() + defer m.mu.Unlock() + if len(names) == 0 { return m.subnetGroups.SortedValues(), nil } @@ -66,8 +76,7 @@ func (m *Mock) DescribeCacheSubnetGroups( for _, n := range names { sg, ok := m.subnetGroups.Get(n) if !ok { - return nil, cerrors.Newf(cerrors.NotFound, - "CacheSubnetGroupNotFoundFault: cache subnet group %q not found", n) + return nil, subnetGroupNotFound(n) } out = append(out, sg) @@ -77,31 +86,70 @@ func (m *Mock) DescribeCacheSubnetGroups( } // DeleteCacheSubnetGroup deletes a cache subnet group. +// +// Real ElastiCache refuses to delete a subnet group associated with any +// clusters: standalone cache clusters count, not only replication groups. A +// teardown that skipped this would leave the group deleted and a live cluster +// pointing at nothing. The scan and the delete run under mu, and clusters still +// being created count as users, so a concurrent create cannot slip in between. func (m *Mock) DeleteCacheSubnetGroup(_ context.Context, name string) error { - // Real ElastiCache refuses to delete a subnet group associated with any - // clusters: standalone cache clusters count, not only replication groups. - // A teardown that skipped this would leave the group deleted and a live - // cluster pointing at nothing. + m.mu.Lock() + defer m.mu.Unlock() + + if user, inUse := m.subnetGroupUser(name); inUse { + return newFault(cerrors.FailedPrecondition, FaultCacheSubnetGroupInUse, + "cache subnet group %q is in use by %q", name, user) + } + + if !m.subnetGroups.Delete(name) { + return subnetGroupNotFound(name) + } + + return nil +} + +// subnetGroupUser returns a cluster or replication group (stored or reserved) +// placed in the named subnet group. The caller holds mu. +func (m *Mock) subnetGroupUser(name string) (string, bool) { + return m.firstUser(func(info *cachedriver.CacheInfo) bool { + return info.SubnetGroupName == name + }, func(rg *cachedriver.ReplicationGroup) bool { + return rg.SubnetGroupName == name + }) +} + +// firstUser returns the id of the first cluster or replication group, stored or +// reserved, matching the predicates. The caller holds mu. +func (m *Mock) firstUser( + cacheUses func(*cachedriver.CacheInfo) bool, rgUses func(*cachedriver.ReplicationGroup) bool, +) (string, bool) { for _, cd := range m.caches.All() { - if cd.info.SubnetGroupName == name { - return cerrors.Newf(cerrors.FailedPrecondition, - "CacheSubnetGroupInUse: cache subnet group %q is in use by %q", name, cd.info.Name) + if cacheUses(&cd.info) { + return cd.info.Name, true } } - for _, rg := range m.replicationGroups.All() { - if rg.SubnetGroupName == name { - return cerrors.Newf(cerrors.FailedPrecondition, - "CacheSubnetGroupInUse: cache subnet group %q is in use by %q", name, rg.ID) + for _, r := range m.cacheRes { + if cacheUses(&r.record) { + return r.record.Name, true } } - if !m.subnetGroups.Delete(name) { - return cerrors.Newf(cerrors.NotFound, - "CacheSubnetGroupNotFoundFault: cache subnet group %q not found", name) + groups := m.replicationGroups.All() + for id := range groups { + rg := groups[id] + if rgUses(&rg) { + return id, true + } } - return nil + for _, r := range m.rgRes { + if rgUses(&r.record) { + return r.record.ID, true + } + } + + return "", false } // resolveVPCID reports the VPC the member subnets belong to. diff --git a/providers/aws/elasticache/tags.go b/providers/aws/elasticache/tags.go index 69ad2af14..ffabe2bf2 100644 --- a/providers/aws/elasticache/tags.go +++ b/providers/aws/elasticache/tags.go @@ -6,12 +6,16 @@ import ( "strings" cerrors "github.com/stackshy/cloudemu/v2/errors" + cachedriver "github.com/stackshy/cloudemu/v2/services/cache/driver" ) -// resourceExists reports whether an ElastiCache ARN names a resource this mock -// knows about (a cache cluster, replication group, or parameter group). Tag -// operations on an unknown ARN return NotFound, matching AWS. -func (m *Mock) resourceExists(arn string) bool { +// taggableResource reports whether an ElastiCache ARN names a resource this +// mock knows about (a cache cluster, replication group, parameter group, or +// subnet group), returning the resource's not-found fault when it does not. +// A cluster or replication group whose real engine is still being provisioned +// or torn down exists in AWS, so a tag write on it reports the invalid-state +// fault, while a tag read is allowed. The caller holds mu. +func (m *Mock) taggableResource(arn string, write bool) error { name := arn if i := strings.LastIndex(arn, ":"); i >= 0 { name = arn[i+1:] @@ -19,15 +23,43 @@ func (m *Mock) resourceExists(arn string) bool { switch { case strings.Contains(arn, ":cluster:"): - return m.caches.Has(name) + return reservedOr(m.caches.Has(name), m.cacheRes[name], write, + func(state string) error { return cacheInvalidState(name, state) }, cacheNotFound(name)) case strings.Contains(arn, ":replicationgroup:"): - return m.replicationGroups.Has(name) + return reservedOr(m.replicationGroups.Has(name), m.rgRes[name], write, + func(state string) error { return rgInvalidState(name, state) }, rgNotFound(name)) case strings.Contains(arn, ":parametergroup:"): - return m.parameterGroups.Has(name) + if m.parameterGroups.Has(name) { + return nil + } + + return parameterGroupNotFound(name) case strings.Contains(arn, ":subnetgroup:"): - return m.subnetGroups.Has(name) + if m.subnetGroups.Has(name) { + return nil + } + + return subnetGroupNotFound(name) + default: + // Snapshot ARNs are not taggable yet (#1695); an unknown type keeps + // answering not-found as before. + return newFault(cerrors.NotFound, FaultCacheClusterNotFound, "cache resource %q not found", arn) + } +} + +// reservedOr resolves a tag target that may be stored or reserved: stored is +// taggable, reserved is readable but not writable, and anything else is +// notFound. +func reservedOr[T any](stored bool, r *reservation[T], write bool, invalid func(string) error, notFound error) error { + switch { + case stored: + return nil + case r != nil && write: + return invalid(r.state) + case r != nil: + return nil default: - return false + return notFound } } @@ -38,9 +70,6 @@ func (m *Mock) seedTags(arn string, tags map[string]string) { return } - m.tagMu.Lock() - defer m.tagMu.Unlock() - stored := make(map[string]string, len(tags)) for k, v := range tags { stored[k] = v @@ -51,12 +80,12 @@ func (m *Mock) seedTags(arn string, tags map[string]string) { // AddTags adds or overwrites tags on an ElastiCache resource by ARN. func (m *Mock) AddTags(_ context.Context, arn string, tags map[string]string) error { - if !m.resourceExists(arn) { - return cerrors.Newf(cerrors.NotFound, "cache resource %q not found", arn) - } + m.mu.Lock() + defer m.mu.Unlock() - m.tagMu.Lock() - defer m.tagMu.Unlock() + if err := m.taggableResource(arn, true); err != nil { + return err + } stored := m.tagsByARN[arn] if stored == nil { @@ -73,12 +102,12 @@ func (m *Mock) AddTags(_ context.Context, arn string, tags map[string]string) er // ListTags returns the tags on an ElastiCache resource by ARN. func (m *Mock) ListTags(_ context.Context, arn string) (map[string]string, error) { - if !m.resourceExists(arn) { - return nil, cerrors.Newf(cerrors.NotFound, "cache resource %q not found", arn) - } + m.mu.Lock() + defer m.mu.Unlock() - m.tagMu.Lock() - defer m.tagMu.Unlock() + if err := m.taggableResource(arn, false); err != nil { + return nil, err + } out := make(map[string]string, len(m.tagsByARN[arn])) for k, v := range m.tagsByARN[arn] { @@ -90,12 +119,12 @@ func (m *Mock) ListTags(_ context.Context, arn string) (map[string]string, error // RemoveTags deletes the named tag keys from an ElastiCache resource by ARN. func (m *Mock) RemoveTags(_ context.Context, arn string, keys []string) error { - if !m.resourceExists(arn) { - return cerrors.Newf(cerrors.NotFound, "cache resource %q not found", arn) - } + m.mu.Lock() + defer m.mu.Unlock() - m.tagMu.Lock() - defer m.tagMu.Unlock() + if err := m.taggableResource(arn, true); err != nil { + return err + } stored := m.tagsByARN[arn] for _, k := range keys { @@ -111,8 +140,12 @@ func (m *Mock) CreateCacheParameterGroup(_ context.Context, name, family, descri return nil, cerrors.New(cerrors.InvalidArgument, "CacheParameterGroupName is required") } + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.lookupParameterGroup(name); exists { - return nil, cerrors.Newf(cerrors.AlreadyExists, "cache parameter group %q already exists", name) + return nil, newFault(cerrors.AlreadyExists, FaultCacheParameterGroupAlreadyExists, + "cache parameter group %q already exists", name) } pg := ParameterGroup{Name: name, Family: family, Description: description, Overrides: map[string]string{}} @@ -124,6 +157,9 @@ func (m *Mock) CreateCacheParameterGroup(_ context.Context, name, family, descri // DescribeCacheParameterGroups returns the named groups, or all when none given. // With no names it lists the built-in default groups too, as ElastiCache does. func (m *Mock) DescribeCacheParameterGroups(_ context.Context, names []string) ([]ParameterGroup, error) { + m.mu.Lock() + defer m.mu.Unlock() + if len(names) == 0 { out := m.parameterGroups.SortedValues() @@ -142,7 +178,7 @@ func (m *Mock) DescribeCacheParameterGroups(_ context.Context, names []string) ( for _, name := range names { pg, ok := m.lookupParameterGroup(name) if !ok { - return nil, cerrors.Newf(cerrors.NotFound, "cache parameter group %q not found", name) + return nil, parameterGroupNotFound(name) } out = append(out, pg) @@ -152,29 +188,30 @@ func (m *Mock) DescribeCacheParameterGroups(_ context.Context, names []string) ( } // DeleteCacheParameterGroup removes an ElastiCache cache parameter group. -// Real ElastiCache refuses to delete a default group or one a cluster still uses. +// Real ElastiCache refuses to delete a default group or one a cluster still +// uses. The in-use scan and the delete run under mu, and clusters or groups +// still being created count as users, so a create that has validated the +// parameter group cannot commit after the group is gone. func (m *Mock) DeleteCacheParameterGroup(_ context.Context, name string) error { if err := rejectDefaultGroupChange(name); err != nil { return err } - for _, cd := range m.caches.All() { - if cd.info.ParameterGroupName == name { - return cerrors.Newf(cerrors.FailedPrecondition, - "InvalidCacheParameterGroupState: cache parameter group %q is in use by %q", name, cd.info.Name) - } - } + m.mu.Lock() + defer m.mu.Unlock() - groups := m.replicationGroups.All() - for id := range groups { - if groups[id].ParameterGroupName == name { - return cerrors.Newf(cerrors.FailedPrecondition, - "InvalidCacheParameterGroupState: cache parameter group %q is in use by %q", name, id) - } + user, inUse := m.firstUser(func(info *cachedriver.CacheInfo) bool { + return info.ParameterGroupName == name + }, func(rg *cachedriver.ReplicationGroup) bool { + return rg.ParameterGroupName == name + }) + if inUse { + return newFault(cerrors.FailedPrecondition, FaultInvalidCacheParameterGroupState, + "cache parameter group %q is in use by %q", name, user) } if !m.parameterGroups.Delete(name) { - return cerrors.Newf(cerrors.NotFound, "cache parameter group %q not found", name) + return parameterGroupNotFound(name) } return nil diff --git a/providers/aws/elasticache/validation.go b/providers/aws/elasticache/validation.go index 41b0f2f79..26996f800 100644 --- a/providers/aws/elasticache/validation.go +++ b/providers/aws/elasticache/validation.go @@ -142,7 +142,7 @@ func (m *Mock) requireParameterGroup(name string) error { } if _, ok := m.lookupParameterGroup(name); !ok { - return cerrors.Newf(cerrors.NotFound, "cache parameter group %q not found", name) + return parameterGroupNotFound(name) } return nil diff --git a/server/aws/elasticache/faults.go b/server/aws/elasticache/faults.go new file mode 100644 index 000000000..df5995262 --- /dev/null +++ b/server/aws/elasticache/faults.go @@ -0,0 +1,60 @@ +package elasticache + +import ( + "errors" + "net/http" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + ecprovider "github.com/stackshy/cloudemu/v2/providers/aws/elasticache" + "github.com/stackshy/cloudemu/v2/server/wire/awsquery" +) + +// awsFaulter is an error that names its ElastiCache wire fault code. The +// provider's typed Fault implements it. +type awsFaulter interface { + AWSFault() string +} + +// faultStatus is the HTTP status ElastiCache answers each fault code with, from +// the botocore model (error.httpStatusCode). Note that CacheSubnetGroupNotFound +// is a 400 in the real API, unlike most not-found faults. +func faultStatus(code string) int { + switch code { + case ecprovider.FaultCacheClusterNotFound, + ecprovider.FaultReplicationGroupNotFound, + ecprovider.FaultCacheParameterGroupNotFound, + ecprovider.FaultSnapshotNotFound: + return http.StatusNotFound + default: + return http.StatusBadRequest + } +} + +// writeErr maps cloudemu errors to ElastiCache XML error responses. +// +// A typed fault carries its wire code; the message is written without any code +// token, since a real ElastiCache never begins with one. Errors with +// no fault code (request parsing, unsupported capabilities) fall back to a +// code chosen by the canonical error kind. +func writeErr(w http.ResponseWriter, err error) { + msg := cerrors.Message(err) + + var f awsFaulter + if errors.As(err, &f) { + awsquery.WriteXMLError(w, faultStatus(f.AWSFault()), f.AWSFault(), msg) + return + } + + switch { + case cerrors.IsNotFound(err): + awsquery.WriteXMLError(w, http.StatusNotFound, ecprovider.FaultCacheClusterNotFound, msg) + case cerrors.IsAlreadyExists(err): + awsquery.WriteXMLError(w, http.StatusBadRequest, ecprovider.FaultCacheClusterAlreadyExists, msg) + case cerrors.IsInvalidArgument(err): + awsquery.WriteXMLError(w, http.StatusBadRequest, "InvalidParameterValue", msg) + case cerrors.IsFailedPrecondition(err): + awsquery.WriteXMLError(w, http.StatusBadRequest, ecprovider.FaultInvalidCacheClusterState, msg) + default: + awsquery.WriteXMLError(w, http.StatusInternalServerError, "InternalFailure", msg) + } +} diff --git a/server/aws/elasticache/faults_sdk_roundtrip_test.go b/server/aws/elasticache/faults_sdk_roundtrip_test.go new file mode 100644 index 000000000..ed05e5654 --- /dev/null +++ b/server/aws/elasticache/faults_sdk_roundtrip_test.go @@ -0,0 +1,88 @@ +package elasticache_test + +import ( + "context" + "errors" + "net/http" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + awselasticache "github.com/aws/aws-sdk-go-v2/service/elasticache" + ectypes "github.com/aws/aws-sdk-go-v2/service/elasticache/types" + "github.com/aws/smithy-go" +) + +// assertFault checks the SDK error's code, HTTP status, and that its message +// carries no code token. +func assertFault(t *testing.T, op string, err error, wantCode string, wantStatus int) { + t.Helper() + + var apiErr smithy.APIError + if !errors.As(err, &apiErr) || apiErr.ErrorCode() != wantCode { + t.Fatalf("%s: got %v, want %s", op, err, wantCode) + } + + msg := apiErr.ErrorMessage() + if msg == "" || strings.HasPrefix(msg, wantCode) || strings.Contains(msg, "NotFound: ") || + strings.Contains(msg, "AlreadyExists: ") || strings.Contains(msg, "FailedPrecondition: ") { + t.Fatalf("%s: message %q is empty or leaks a code prefix", op, msg) + } + + var respErr *awshttp.ResponseError + if !errors.As(err, &respErr) || respErr.HTTPStatusCode() != wantStatus { + t.Fatalf("%s: HTTP status = %v, want %d", op, err, wantStatus) + } +} + +// TestTypedFaultsSurviveSDK proves the provider's typed faults reach the SDK as +// its modelled error types, with the botocore status and a prefix-free message. +func TestTypedFaultsSurviveSDK(t *testing.T) { + client := newSDKClient(t) + ctx := context.Background() + + in := &awselasticache.CreateCacheClusterInput{ + CacheClusterId: aws.String("dup"), Engine: aws.String("redis"), + CacheNodeType: aws.String("cache.t3.micro"), NumCacheNodes: aws.Int32(1), + } + if _, err := client.CreateCacheCluster(ctx, in); err != nil { + t.Fatalf("CreateCacheCluster: %v", err) + } + + _, err := client.CreateCacheCluster(ctx, in) + assertFault(t, "CreateCacheCluster(dup)", err, "CacheClusterAlreadyExists", http.StatusBadRequest) + + var exists *ectypes.CacheClusterAlreadyExistsFault + if !errors.As(err, &exists) { + t.Fatalf("duplicate create: %T is not *types.CacheClusterAlreadyExistsFault", err) + } + + _, err = client.DescribeReplicationGroups(ctx, &awselasticache.DescribeReplicationGroupsInput{ + ReplicationGroupId: aws.String("nope"), + }) + assertFault(t, "DescribeReplicationGroups(nope)", err, "ReplicationGroupNotFoundFault", http.StatusNotFound) + + var rgMissing *ectypes.ReplicationGroupNotFoundFault + if !errors.As(err, &rgMissing) { + t.Fatalf("missing group: %T is not *types.ReplicationGroupNotFoundFault", err) + } + + _, err = client.DeleteCacheSubnetGroup(ctx, &awselasticache.DeleteCacheSubnetGroupInput{ + CacheSubnetGroupName: aws.String("nope"), + }) + assertFault(t, "DeleteCacheSubnetGroup(nope)", err, "CacheSubnetGroupNotFoundFault", http.StatusBadRequest) + + if _, err = client.CreateCacheParameterGroup(ctx, &awselasticache.CreateCacheParameterGroupInput{ + CacheParameterGroupName: aws.String("pg"), CacheParameterGroupFamily: aws.String("redis7"), + Description: aws.String("d"), + }); err != nil { + t.Fatalf("CreateCacheParameterGroup: %v", err) + } + + _, err = client.CreateCacheParameterGroup(ctx, &awselasticache.CreateCacheParameterGroupInput{ + CacheParameterGroupName: aws.String("pg"), CacheParameterGroupFamily: aws.String("redis7"), + Description: aws.String("d"), + }) + assertFault(t, "CreateCacheParameterGroup(dup)", err, "CacheParameterGroupAlreadyExists", http.StatusBadRequest) +} diff --git a/server/aws/elasticache/faults_test.go b/server/aws/elasticache/faults_test.go new file mode 100644 index 000000000..c8d495129 --- /dev/null +++ b/server/aws/elasticache/faults_test.go @@ -0,0 +1,107 @@ +package elasticache + +import ( + "encoding/xml" + "net/http" + "net/http/httptest" + "strings" + "testing" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + ecprovider "github.com/stackshy/cloudemu/v2/providers/aws/elasticache" +) + +// testFault is a stand-in for the provider's typed Fault. +type testFault struct { + err *cerrors.Error + code string +} + +func (f testFault) Error() string { return f.err.Error() } +func (f testFault) Unwrap() error { return f.err } +func (f testFault) AWSFault() string { return f.code } + +type xmlErrorBody struct { + Code string `xml:"Error>Code"` + Message string `xml:"Error>Message"` +} + +func recordWriteErr(t *testing.T, err error) (status int, body xmlErrorBody) { + t.Helper() + + rec := httptest.NewRecorder() + writeErr(rec, err) + + if uerr := xml.Unmarshal(rec.Body.Bytes(), &body); uerr != nil { + t.Fatalf("unmarshal error body %q: %v", rec.Body.String(), uerr) + } + + return rec.Code, body +} + +// allFaultCodes lists every fault code the provider emits. +func allFaultCodes() []string { + return []string{ + ecprovider.FaultCacheClusterNotFound, ecprovider.FaultCacheClusterAlreadyExists, + ecprovider.FaultInvalidCacheClusterState, ecprovider.FaultReplicationGroupNotFound, + ecprovider.FaultReplicationGroupAlreadyExists, ecprovider.FaultInvalidReplicationGroupState, + ecprovider.FaultCacheSubnetGroupNotFound, ecprovider.FaultCacheSubnetGroupAlreadyExists, + ecprovider.FaultCacheSubnetGroupInUse, ecprovider.FaultCacheParameterGroupNotFound, + ecprovider.FaultCacheParameterGroupAlreadyExists, ecprovider.FaultInvalidCacheParameterGroupState, + ecprovider.FaultSnapshotNotFound, ecprovider.FaultSnapshotAlreadyExists, + ecprovider.FaultSnapshotFeatureNotSupported, + } +} + +// TestWriteErrStripsCodeFromMessage proves the wire carries only the +// human text: neither the canonical taxonomy token ("NotFound: ") nor the +// fault code itself, as a real ElastiCache message never starts with one. +func TestWriteErrStripsCodeFromMessage(t *testing.T) { + for _, code := range allFaultCodes() { + t.Run(code, func(t *testing.T) { + err := testFault{err: cerrors.New(cerrors.NotFound, "resource \"x\" not found"), code: code} + + _, body := recordWriteErr(t, err) + + if body.Code != code { + t.Fatalf("Code = %q, want %q", body.Code, code) + } + + if body.Message != `resource "x" not found` { + t.Fatalf("Message = %q, want the bare human message", body.Message) + } + + if strings.HasPrefix(body.Message, code) || strings.HasPrefix(body.Message, "NotFound") { + t.Fatalf("Message %q leaks a code prefix", body.Message) + } + }) + } +} + +// TestWriteErrStatusFromTable checks the HTTP status per fault code against the +// botocore model, including CacheSubnetGroupNotFoundFault answering 400. +func TestWriteErrStatusFromTable(t *testing.T) { + notFound := map[string]bool{ + ecprovider.FaultCacheClusterNotFound: true, + ecprovider.FaultReplicationGroupNotFound: true, + ecprovider.FaultCacheParameterGroupNotFound: true, + ecprovider.FaultSnapshotNotFound: true, + } + + for _, code := range allFaultCodes() { + want := http.StatusBadRequest + if notFound[code] { + want = http.StatusNotFound + } + + status, _ := recordWriteErr(t, testFault{err: cerrors.New(cerrors.NotFound, "m"), code: code}) + if status != want { + t.Errorf("%s: status %d, want %d", code, status, want) + } + } + + status, body := recordWriteErr(t, cerrors.New(cerrors.InvalidArgument, "bad input")) + if status != http.StatusBadRequest || body.Code != "InvalidParameterValue" || body.Message != "bad input" { + t.Fatalf("plain InvalidArgument = %d %+v, want 400 InvalidParameterValue", status, body) + } +} diff --git a/server/aws/elasticache/handler.go b/server/aws/elasticache/handler.go index 0e97812f9..542d0bda8 100644 --- a/server/aws/elasticache/handler.go +++ b/server/aws/elasticache/handler.go @@ -23,7 +23,6 @@ import ( "net/http" "strings" - cerrors "github.com/stackshy/cloudemu/v2/errors" "github.com/stackshy/cloudemu/v2/server/wire/awsauthz" "github.com/stackshy/cloudemu/v2/server/wire/awsquery" cachedriver "github.com/stackshy/cloudemu/v2/services/cache/driver" @@ -240,86 +239,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -// writeErr maps cloudemu errors to ElastiCache XML error responses. -func writeErr(w http.ResponseWriter, err error) { - msg := cerrors.Message(err) - - switch { - case cerrors.IsNotFound(err): - awsquery.WriteXMLError(w, http.StatusNotFound, notFoundCode(err), msg) - case cerrors.IsAlreadyExists(err): - awsquery.WriteXMLError(w, http.StatusBadRequest, alreadyExistsCode(err), msg) - case cerrors.IsInvalidArgument(err): - awsquery.WriteXMLError(w, http.StatusBadRequest, "InvalidParameterValue", msg) - case cerrors.IsFailedPrecondition(err): - awsquery.WriteXMLError(w, http.StatusBadRequest, failedPreconditionCode(err), msg) - default: - awsquery.WriteXMLError(w, http.StatusInternalServerError, "InternalFailure", msg) - } -} - -// failedPreconditionCode picks the AWS-shaped error code for a failed -// precondition. The driver encodes the intended fault as a prefix on the -// message (a subnet group still in use, or an unsupported snapshot); a caller -// matching the SDK's typed errors sees the code, not the message, so it has to -// be surfaced as the wire Code. Anything else is a cache-cluster state error. -func failedPreconditionCode(err error) string { - msg := err.Error() - - switch { - case strings.Contains(msg, "CacheSubnetGroupInUse"): - return "CacheSubnetGroupInUse" - case strings.Contains(msg, "SnapshotFeatureNotSupportedFault"): - return "SnapshotFeatureNotSupportedFault" - case strings.Contains(msg, "InvalidCacheParameterGroupState"): - return "InvalidCacheParameterGroupState" - default: - return "InvalidCacheClusterState" - } -} - -// notFoundCode picks the AWS-shaped error code for the resource the message -// names. A caller matching the SDK's typed errors sees the code, not the -// message, so a generic one leaves it unable to tell a missing replication -// group from a missing cache cluster. -// -// Order matters: "cache subnet group" also contains "cache", so the more -// specific resources are checked first. -func notFoundCode(err error) string { - msg := err.Error() - - switch { - case strings.Contains(msg, "snapshot"): - return "SnapshotNotFoundFault" - case strings.Contains(msg, "replication group"): - return "ReplicationGroupNotFoundFault" - case strings.Contains(msg, "cache subnet group"): - return "CacheSubnetGroupNotFoundFault" - case strings.Contains(msg, "parameter group"): - return "CacheParameterGroupNotFound" - default: - return "CacheClusterNotFound" - } -} - -// alreadyExistsCode is the create-side counterpart of notFoundCode. Callers -// treat the specific already-exists code as "already provisioned, carry on", -// so collapsing it turns an idempotent re-run into a hard failure. -func alreadyExistsCode(err error) string { - msg := err.Error() - - switch { - case strings.Contains(msg, "snapshot"): - return "SnapshotAlreadyExistsFault" - case strings.Contains(msg, "replication group"): - return "ReplicationGroupAlreadyExists" - case strings.Contains(msg, "cache subnet group"): - return "CacheSubnetGroupAlreadyExists" - default: - return "CacheClusterAlreadyExists" - } -} - // IAMService returns the IAM service prefix of the operations this handler // serves. func (*Handler) IAMService() string { return "elasticache" }