diff --git a/docs/sdk-server.md b/docs/sdk-server.md index 52d6a2013..12808989d 100644 --- a/docs/sdk-server.md +++ b/docs/sdk-server.md @@ -184,7 +184,7 @@ The tables below list the main handlers. They are not the full list: the generat | **RDS / Aurora** *(query protocol)* | DBInstances (Create/Describe/Modify/Delete/Start/Stop/Reboot), DBClusters (Create/Describe/Modify/Delete/Start/Stop), DBSnapshots + DBClusterSnapshots (Create/Describe/Delete/Restore). One handler also serves the **Neptune** and **DocumentDB** engines: both reuse the same `aws-sdk-go-v2/service/{neptune,docdb}` client surface. | | **Redshift** *(query protocol)* | CreateCluster, DescribeClusters, ModifyCluster, DeleteCluster, RebootCluster, CreateClusterSnapshot, DescribeClusterSnapshots, DeleteClusterSnapshot, RestoreFromClusterSnapshot | | **MemoryDB** *(JSON 1.1, `AmazonMemoryDB.*`)* | Clusters (Create/Describe/Update/Delete/FailoverShard/ListAllowedNodeTypeUpdates), ACLs & Users, Parameter Groups, Subnet Groups, Snapshots (Create/Describe/Copy/Delete + restore), tags, engine-version & event catalogs. Optional (type-asserted): Multi-Region clusters, Reserved Nodes. Server-side `MaxResults`/`NextToken` pagination on every Describe. | -| **Keyspaces** *(JSON 1.0, `KeyspacesService.*`)* | Keyspaces (Create/Get/List/Update/Delete, single/multi-region replication), Tables (Create/Get/List/Update/Delete/Restore: full schema, capacity, encryption, PITR, TTL, CDC), user-defined types, tags. Optional: `GetTableAutoScalingSettings`. Responses use lower-camel keys so the case-sensitive SDK deserializer decodes them; pagination on every list. | +| **Keyspaces** *(JSON 1.0, `KeyspacesService.*`, and Smithy rpc-v2-cbor)* | Keyspaces (Create/Get/List/Update/Delete, single/multi-region replication), Tables (Create/Get/List/Update/Delete/Restore: full schema, capacity, encryption, PITR, TTL, CDC), user-defined types, tags. Optional: `GetTableAutoScalingSettings`. Responses use lower-camel keys so the case-sensitive SDK deserializer decodes them; pagination on every list. | | **EKS** *(REST + JSON)* | Clusters (Create/Describe/List/UpdateConfig/UpdateVersion/Delete), NodeGroups (Create/Describe/List/UpdateConfig/UpdateVersion/Delete), Fargate Profiles (Create/Describe/List/Delete), Addons (Create/Describe/List/Update/Delete). The kubeconfig points at the shared [Kubernetes data plane](#kubernetes). | | **IAM** *(query protocol)* | Users (Create/Get/List/Delete), Roles (Create/Get/List/Delete), Policies (Create/Get/List/Delete), Attach/Detach/ListAttached for both Users and Roles, Groups (Create/Get/List/Delete + AddUserToGroup/RemoveUserFromGroup/ListGroupsForUser), AccessKeys (Create/List/Delete), InstanceProfiles (Create/Get/List/Delete + AddRoleToInstanceProfile/RemoveRoleFromInstanceProfile). Errors surface as typed `*types.NoSuchEntityException` / `*types.EntityAlreadyExistsException`. | | **Resource Explorer 2** *(JSON)* | Search: free-text plus filter expression over the cross-service inventory; results include ARN, resource type, region, owning account, and tags | @@ -302,13 +302,13 @@ Each handler matches on a different signal, so dispatch within a provider is una | AWS DynamoDB | `X-Amz-Target: DynamoDB_20120810.*` header | | AWS SQS | `X-Amz-Target: AmazonSQS.*` header | | AWS MemoryDB | `X-Amz-Target: AmazonMemoryDB.*` header | -| AWS Keyspaces | `X-Amz-Target: KeyspacesService.*` header | +| AWS Keyspaces | `X-Amz-Target: KeyspacesService.*` header, or `Smithy-Protocol: rpc-v2-cbor` on `/service/KeyspacesService/operation/*` (registered before CloudWatch) | | AWS Lambda | URL prefix `/2015-03-31/functions` | | AWS EKS | URL prefix `/clusters` | | AWS RDS | Form-encoded POST whose `Action=` is a known RDS operation (registered before EC2) | | AWS Redshift | Form-encoded POST whose `Action=` is a known Redshift operation (registered before EC2) | | AWS EC2 | `Action=…` in URL query or `Content-Type: application/x-www-form-urlencoded` POST | -| AWS CloudWatch | `Smithy-Protocol: rpc-v2-cbor` header | +| AWS CloudWatch | `Smithy-Protocol: rpc-v2-cbor` header (any other `/service/` path) | | AWS Bedrock | URL prefix `/foundation-models`, `/model-customization-jobs`, `/custom-models`, `/guardrails`, `/provisioned-model-throughput`, `/logging/modelinvocations`, `/tagResource`, `/untagResource`, `/listTagsForResource`, `/model-import-jobs`, `/model-copy-jobs`, `/evaluation-jobs`, `/evaluation-job/`, `/inference-profiles`, `/prompt-routers`, `/marketplace-model/endpoints`, `/automated-reasoning-policies`, `/create-foundation-model-agreement`, `/delete-foundation-model-agreement`, `/list-foundation-model-agreement-offers/`, `/foundation-model-availability/`, or bedrock-runtime `/model/{id}/{invoke,converse,converse-stream,invoke-with-response-stream,count-tokens}`, `/guardrail/{id}/version/{version}/apply`, and `/async-invoke` | | AWS Bedrock Agent | Runtime routes are registered before the control plane and the query-protocol fallback, so they match whatever the Content-Type (except the roots that are also valid S3 bucket names, `/sessions`, `/rerank` and `/optimize-prompt`, which are claimed only for requests signed for `bedrock` or unsigned JSON requests): `/agents/{id}/agentAliases/{a}/sessions/{s}/text` (InvokeAgent), POST `/agents/{sessionId}` without a trailing slash (InvokeInlineAgent), `/agents/{id}/agentAliases/{a}/memories`, `/knowledgebases/{id}/retrieve`, `/knowledgebases/{id}/datasources/{ds}/documents/{doc}/content`, `/retrieveAndGenerate[Stream]`, `/rerank`, `/generateQuery`, `/optimize-prompt`, `/agenticRetrieveStream`, `/sessions...`, `/flows/{id}/aliases/{a}[/executions...]`, `/flows/{id}/executions`, and `/tags/{bedrock session ARN}`. Everything else under `/agents`, `/knowledgebases`, `/flows`, `/prompts` is the control plane | | AWS S3 | Fallback (everything else REST-shaped) | diff --git a/providers/aws/keyspaces/keyspaces_test.go b/providers/aws/keyspaces/keyspaces_test.go index aa1135c89..45381c274 100644 --- a/providers/aws/keyspaces/keyspaces_test.go +++ b/providers/aws/keyspaces/keyspaces_test.go @@ -145,7 +145,7 @@ func TestTableLifecycleAndValidation(t *testing.T) { t.Fatalf("CreateTable: %v", err) } - // Default capacity mode is PAY_PER_REQUEST; PITR/TTL default DISABLED. + // Default capacity mode is PAY_PER_REQUEST; PITR defaults to DISABLED. if tbl.CapacitySpecification.ThroughputMode != ksdriver.ThroughputPayPerRequest { t.Fatalf("default throughput mode: %+v", tbl.CapacitySpecification) } diff --git a/providers/aws/keyspaces/snapshot.go b/providers/aws/keyspaces/snapshot.go index 9410446c8..441c00775 100644 --- a/providers/aws/keyspaces/snapshot.go +++ b/providers/aws/keyspaces/snapshot.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/stackshy/cloudemu/v2/internal/snapshot" + ksdriver "github.com/stackshy/cloudemu/v2/services/keyspaces/driver" ) var _ snapshot.Snapshottable = (*Mock)(nil) @@ -86,6 +87,10 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { } } + m.mu.Lock() + m.normalizeRestoredTablesLocked() + m.mu.Unlock() + if snap.Tags != nil { m.mu.Lock() m.tags = m.canonicalTagsLocked(snap.Tags) @@ -95,6 +100,22 @@ func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { return nil } +// normalizeRestoredTablesLocked clears the ttl and clientSideTimestamps +// status of restored tables that never enabled them. An older build stored +// DISABLED for both on every table, which GetTable then reported although real +// Keyspaces omits the members (see enabledOnly). The caller holds the write +// lock, and the tables store is already restored. +func (m *Mock) normalizeRestoredTablesLocked() { + for _, key := range m.tables.Keys() { + m.tables.Update(key, func(t ksdriver.Table) ksdriver.Table { + t.TTLStatus = enabledOnly(t.TTLStatus) + t.ClientSideTimestamps = enabledOnly(t.ClientSideTimestamps) + + return t + }) + } +} + // canonicalTagsLocked keys restored tags by the canonical ARN of the keyspace // or table they belong to. Tag reads and writes resolve to the canonical ARN, // but an older build stored tags under whatever ARN the caller passed (for diff --git a/providers/aws/keyspaces/table_settings_test.go b/providers/aws/keyspaces/table_settings_test.go new file mode 100644 index 000000000..5c3884458 --- /dev/null +++ b/providers/aws/keyspaces/table_settings_test.go @@ -0,0 +1,151 @@ +package keyspaces + +import ( + "context" + "encoding/json" + "testing" + + ksdriver "github.com/stackshy/cloudemu/v2/services/keyspaces/driver" +) + +// The ttl and clientSideTimestamps enums hold only ENABLED, so a table that +// never enabled them records "" (not set) and GetTable omits the members. An +// explicit DISABLED is the documented default and records "" too, and once +// enabled neither setting can be turned off. +func TestTTLAndClientSideTimestampsTrackSet(t *testing.T) { + ctx := context.Background() + m := newTestMock() + mustKeyspace(t, m, "app") + + create := func(name, ttl, cst string) *ksdriver.Table { + t.Helper() + + tbl, err := m.CreateTable(ctx, ksdriver.CreateTableConfig{ + KeyspaceName: "app", Name: name, SchemaDefinition: sampleSchema(), + TTLStatus: ttl, ClientSideTimestamps: cst, + }) + if err != nil { + t.Fatalf("CreateTable %s: %v", name, err) + } + + return tbl + } + + check := func(name, wantTTL, wantCST string) { + t.Helper() + + got, err := m.GetTable(ctx, "app", name) + if err != nil { + t.Fatalf("GetTable %s: %v", name, err) + } + + if got.TTLStatus != wantTTL || got.ClientSideTimestamps != wantCST { + t.Fatalf("%s: ttl=%q cst=%q, want ttl=%q cst=%q", name, got.TTLStatus, got.ClientSideTimestamps, wantTTL, wantCST) + } + } + + create("unset", "", "") + check("unset", "", "") + + create("disabled", "DISABLED", "DISABLED") + check("disabled", "", "") + + create("enabled", "ENABLED", "ENABLED") + check("enabled", "ENABLED", "ENABLED") + + // Enabling later records the setting. + if _, err := m.UpdateTable(ctx, ksdriver.UpdateTableConfig{ + KeyspaceName: "app", Name: "unset", TTLStatus: "ENABLED", ClientSideTimestamps: "ENABLED", + }); err != nil { + t.Fatalf("UpdateTable enable: %v", err) + } + + check("unset", "ENABLED", "ENABLED") + + // DISABLED never turns an enabled setting back off. + if _, err := m.UpdateTable(ctx, ksdriver.UpdateTableConfig{ + KeyspaceName: "app", Name: "enabled", TTLStatus: "DISABLED", ClientSideTimestamps: "DISABLED", + }); err != nil { + t.Fatalf("UpdateTable disable: %v", err) + } + + check("enabled", "ENABLED", "ENABLED") +} + +// A snapshot keeps whether each setting was set, and a snapshot from an older +// build, which stored DISABLED on every table, restores as not set. +func TestSnapshotTracksTTLAndClientSideTimestamps(t *testing.T) { + ctx := context.Background() + src := newTestMock() + mustKeyspace(t, src, "app") + + for name, status := range map[string]string{"plain": "", "on": "ENABLED"} { + if _, err := src.CreateTable(ctx, ksdriver.CreateTableConfig{ + KeyspaceName: "app", Name: name, SchemaDefinition: sampleSchema(), + TTLStatus: status, ClientSideTimestamps: status, + }); err != nil { + t.Fatalf("CreateTable %s: %v", name, err) + } + } + + raw, err := src.Snapshot(ctx, false) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + + legacy := legacyDisabledSnapshot(t, raw) + + for label, data := range map[string]json.RawMessage{"current": raw, "legacy": legacy} { + dst := newTestMock() + if err := dst.Restore(ctx, data); err != nil { + t.Fatalf("%s restore: %v", label, err) + } + + for name, want := range map[string]string{"plain": "", "on": "ENABLED"} { + got, err := dst.GetTable(ctx, "app", name) + if err != nil { + t.Fatalf("%s GetTable %s: %v", label, name, err) + } + + if got.TTLStatus != want || got.ClientSideTimestamps != want { + t.Fatalf("%s %s: ttl=%q cst=%q, want %q", label, name, got.TTLStatus, got.ClientSideTimestamps, want) + } + } + } +} + +// legacyDisabledSnapshot rewrites raw the way an older build stored it: every +// table that never set ttl or clientSideTimestamps carries DISABLED. +func legacyDisabledSnapshot(t *testing.T, raw json.RawMessage) json.RawMessage { + t.Helper() + + var snap keyspacesSnapshot + if err := json.Unmarshal(raw, &snap); err != nil { + t.Fatalf("parse snapshot: %v", err) + } + + var tables map[string]ksdriver.Table + if err := json.Unmarshal(snap.Tables, &tables); err != nil { + t.Fatalf("parse tables: %v", err) + } + + for k, tbl := range tables { + tbl.TTLStatus = disabledIfEmpty(tbl.TTLStatus) + tbl.ClientSideTimestamps = disabledIfEmpty(tbl.ClientSideTimestamps) + tables[k] = tbl + } + + b, err := json.Marshal(tables) + if err != nil { + t.Fatalf("encode tables: %v", err) + } + + snap.Tables = b + + out, err := json.Marshal(snap) + if err != nil { + t.Fatalf("encode snapshot: %v", err) + } + + return out +} diff --git a/providers/aws/keyspaces/tables.go b/providers/aws/keyspaces/tables.go index c97de5e36..35cd445f2 100644 --- a/providers/aws/keyspaces/tables.go +++ b/providers/aws/keyspaces/tables.go @@ -51,7 +51,26 @@ func cloneTable(in *ksdriver.Table) ksdriver.Table { // disabledIfEmpty defaults an unset status field to "DISABLED". func disabledIfEmpty(v string) string { if v == "" { - return "DISABLED" + return statusDisabled + } + + return v +} + +// statusDisabled is the not-enabled value callers may send for the TTL and +// client-side timestamps settings. +const statusDisabled = "DISABLED" + +// enabledOnly normalizes the ttl and clientSideTimestamps status. Their model +// enums (TimeToLiveStatus, ClientSideTimestampsStatus) hold only ENABLED, so +// real Keyspaces never reports DISABLED: a table that never enabled the +// setting omits the member from GetTable. "" records "not set"; DISABLED is +// the documented default and maps to "" as well. The settings cannot be +// turned off once enabled, so on UpdateTable a DISABLED request is kept as a +// no-op by orKeep. +func enabledOnly(v string) string { + if strings.EqualFold(v, statusDisabled) { + return "" } return v @@ -146,9 +165,9 @@ func (m *Mock) CreateTable(_ context.Context, cfg ksdriver.CreateTableConfig) (* CapacitySpecification: resolveCapacity(cfg.CapacitySpecification), EncryptionSpecification: resolveEncryption(cfg.EncryptionSpecification), PointInTimeRecoveryStatus: disabledIfEmpty(cfg.PointInTimeRecovery), - TTLStatus: disabledIfEmpty(cfg.TTLStatus), + TTLStatus: enabledOnly(cfg.TTLStatus), DefaultTimeToLive: cfg.DefaultTimeToLive, - ClientSideTimestamps: disabledIfEmpty(cfg.ClientSideTimestamps), + ClientSideTimestamps: enabledOnly(cfg.ClientSideTimestamps), CdcStatus: disabledIfEmpty(cfg.CdcStatus), Comment: cfg.Comment, ReplicaRegions: append([]string(nil), cfg.ReplicaRegions...), @@ -287,8 +306,8 @@ func (m *Mock) UpdateTable(_ context.Context, cfg ksdriver.UpdateTableConfig) (* } t.PointInTimeRecoveryStatus = orKeep(cfg.PointInTimeRecovery, t.PointInTimeRecoveryStatus) - t.TTLStatus = orKeep(cfg.TTLStatus, t.TTLStatus) - t.ClientSideTimestamps = orKeep(cfg.ClientSideTimestamps, t.ClientSideTimestamps) + t.TTLStatus = orKeep(enabledOnly(cfg.TTLStatus), t.TTLStatus) + t.ClientSideTimestamps = orKeep(enabledOnly(cfg.ClientSideTimestamps), t.ClientSideTimestamps) t.CdcStatus = orKeep(cfg.CdcStatus, t.CdcStatus) t.Comment = orKeep(cfg.Comment, t.Comment) diff --git a/server/aws/authz_completeness_test.go b/server/aws/authz_completeness_test.go index 3109fc553..e38a243f0 100644 --- a/server/aws/authz_completeness_test.go +++ b/server/aws/authz_completeness_test.go @@ -93,7 +93,7 @@ func TestHandlerIAMServicesMatchTable(t *testing.T) { "*athena.Handler": "athena", "*cognito.Handler": "cognito-idp", "*cognito.WellKnown": "cognito-idp", "*configservice.Handler": "config", "*wafv2.Handler": "wafv2", "*ecs.Handler": "ecs", "*ecr.Handler": "ecr", "*route53resolver.Handler": "route53resolver", "*eventbridge.Handler": "events", "*cloudwatchlogs.Handler": "logs", "*secretsmanager.Handler": "secretsmanager", - "*keyspaces.Handler": "cassandra", "*memorydb.Handler": "memorydb", "*networkfirewall.Handler": "network-firewall", + "*keyspaces.Handler": "cassandra", "*keyspaces.CBORHandler": "cassandra", "*memorydb.Handler": "memorydb", "*networkfirewall.Handler": "network-firewall", "*resourcegroupstaggingapi.Handler": "tag", "*transfer.Handler": "transfer", "*timestreamwrite.Handler": "timestream", "*healthlake.Handler": "healthlake", "*apprunner.Handler": "apprunner", "*globalaccelerator.Handler": "globalaccelerator", "*costexplorer.Handler": "ce", "*servicequotas.Handler": "servicequotas", "*emr.Handler": "elasticmapreduce", diff --git a/server/aws/aws.go b/server/aws/aws.go index ffe0533fe..a861a308b 100644 --- a/server/aws/aws.go +++ b/server/aws/aws.go @@ -587,7 +587,8 @@ func NewFromProvider(p *awsprovider.Provider) *server.Server { // non-nil driver in d. Handlers are registered most-specific-first so the // dispatch is unambiguous: // -// - CloudWatch matches on Smithy-Protocol: rpc-v2-cbor header. +// - Keyspaces rpc-v2-cbor matches its own /service/KeyspacesService/ path; +// CloudWatch then matches any other Smithy-Protocol: rpc-v2-cbor request. // - DynamoDB matches on X-Amz-Target header (JSON-RPC). // - RDS matches form-encoded POSTs whose Action is one of the known RDS // operations. It must register before EC2 because both speak the AWS @@ -658,6 +659,14 @@ func newServer(d Drivers) (*server.Server, authzSets) { srv.Register(bedrockagentruntime.New(d.BedrockAgentRuntime)) } + // Keyspaces rpc-v2-cbor (aws-sdk-go-v2/service/keyspaces v1.37.0+, so + // hashicorp/aws 6.69+) matches only /service/KeyspacesService/operation/*. + // It registers before CloudWatch, which claims every other rpc-v2-cbor + // request under /service/. The AWS JSON 1.0 handler registers below. + if d.Keyspaces != nil { + srv.Register(keyspacessrv.NewCBOR(d.Keyspaces)) + } + if d.CloudWatch != nil { // The VPC driver optionally supplies derived AWS/IPAM metrics; surface // them through CloudWatch when it implements the capability. diff --git a/server/aws/keyspaces/cbor.go b/server/aws/keyspaces/cbor.go new file mode 100644 index 000000000..e2e2a3170 --- /dev/null +++ b/server/aws/keyspaces/cbor.go @@ -0,0 +1,325 @@ +package keyspaces + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "reflect" + "strings" + "time" + + "github.com/fxamacker/cbor/v2" + + "github.com/stackshy/cloudemu/v2/server/wire/awsauthz" + ksdriver "github.com/stackshy/cloudemu/v2/services/keyspaces/driver" +) + +// Smithy RPC v2 CBOR. aws-sdk-go-v2/service/keyspaces v1.37.0 added it beside +// AWS JSON 1.0 and prefers it, so hashicorp/aws 6.69+ sends every Keyspaces +// call as +// +// POST /service/KeyspacesService/operation/ +// Smithy-Protocol: rpc-v2-cbor +// Content-Type: application/cbor +// +// with a CBOR body that uses the same member names as the JSON protocol. +const ( + smithyProtocolHeader = "Smithy-Protocol" + rpcV2CBOR = "rpc-v2-cbor" + cborContentType = "application/cbor" + cborPathPrefix = "/service/KeyspacesService/operation/" + jsonContentType = "application/x-amz-json-1.0" + maxCBORBodyBytes = 1 << 20 +) + +var errNotCBORMap = errors.New("request body must be a CBOR map") + +// timestampMembers are the Timestamp members of the Keyspaces model. JSON +// carries them as epoch-second numbers; rpc-v2-cbor requires CBOR tag 1, and +// the SDK rejects an untagged number. +// +//nolint:gochecknoglobals // static protocol lookup table +var timestampMembers = map[string]bool{ + "creationTimestamp": true, + "lastModifiedTimestamp": true, + "earliestRestorableTimestamp": true, + "lastUpdateToPayPerRequestTimestamp": true, +} + +//nolint:gochecknoglobals // reused codec modes +var ( + cborDecMode = mustCBORDecMode() + cborEncMode = mustCBOREncMode() +) + +func mustCBORDecMode() cbor.DecMode { + mode, err := cbor.DecOptions{DefaultMapType: reflect.TypeOf(map[string]any(nil))}.DecMode() + if err != nil { + panic(err) + } + + return mode +} + +// mustCBOREncMode writes time.Time as tag 1 over epoch seconds, the Smithy +// timestamp encoding. +func mustCBOREncMode() cbor.EncMode { + mode, err := cbor.EncOptions{Time: cbor.TimeUnixDynamic, TimeTag: cbor.EncTagRequired}.EncMode() + if err != nil { + panic(err) + } + + return mode +} + +// CBORHandler serves the rpc-v2-cbor form of the Keyspaces API. It only +// converts the wire format: each request is re-encoded as AWS JSON 1.0 and run +// through the JSON Handler, and the answer is re-encoded as CBOR, so both +// protocols share one dispatch and one set of business rules. +type CBORHandler struct { + json *Handler +} + +// NewCBOR returns an rpc-v2-cbor Keyspaces handler backed by db. +func NewCBOR(db ksdriver.Keyspaces) *CBORHandler { + return &CBORHandler{json: New(db)} +} + +// Matches claims rpc-v2-cbor requests addressed to the KeyspacesService. +func (*CBORHandler) Matches(r *http.Request) bool { + return r.Header.Get(smithyProtocolHeader) == rpcV2CBOR && strings.HasPrefix(r.URL.Path, cborPathPrefix) +} + +// cborOperation returns the operation named in the URL path, or "". +func cborOperation(r *http.Request) string { + op := strings.TrimPrefix(r.URL.Path, cborPathPrefix) + if strings.Contains(op, "/") { + return "" + } + + return op +} + +// ServeHTTP converts the CBOR request to JSON, runs the JSON handler, and +// converts its response back to CBOR. +func (h *CBORHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + op := cborOperation(r) + if op == "" { + writeCBORError(w, http.StatusBadRequest, "ValidationException", "missing operation in path") + return + } + + raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxCBORBodyBytes)) + if err != nil { + writeCBORError(w, http.StatusBadRequest, "SerializationException", "cannot read body: "+err.Error()) + return + } + + body, err := cborToJSON(raw) + if err != nil { + writeCBORError(w, http.StatusBadRequest, "SerializationException", "invalid CBOR: "+err.Error()) + return + } + + jr := r.Clone(r.Context()) + jr.Body = io.NopCloser(bytes.NewReader(body)) + jr.ContentLength = int64(len(body)) + jr.Header.Set("X-Amz-Target", targetPrefix+op) + jr.Header.Set("Content-Type", jsonContentType) + + rec := &bufferedResponse{header: http.Header{}, status: http.StatusOK} + h.json.ServeHTTP(rec, jr) + + writeCBORFromJSON(w, rec) +} + +// IAMService returns the IAM service prefix of the operations this handler +// serves. +func (*CBORHandler) IAMService() string { return iamService } + +// IAMChecks names the IAM action of a request from the operation in the URL +// path, the same value ServeHTTP dispatches on. The resource is unknown, as for +// the JSON protocol. +func (h *CBORHandler) IAMChecks(r *http.Request, _ awsauthz.Scope) ([]awsauthz.Check, bool) { + op := cborOperation(r) + if op == "" { + return nil, false + } + + return awsauthz.Single(h.IAMService()+":"+op, ""), true +} + +// WriteAccessDenied writes the authorization 403 as an rpc-v2-cbor error. +func (*CBORHandler) WriteAccessDenied(w http.ResponseWriter, _ *http.Request, msg string) { + writeCBORError(w, http.StatusForbidden, "AccessDeniedException", msg) +} + +// bufferedResponse captures the JSON handler's response for re-encoding. +type bufferedResponse struct { + header http.Header + status int + body bytes.Buffer +} + +func (b *bufferedResponse) Header() http.Header { return b.header } +func (b *bufferedResponse) Write(p []byte) (int, error) { return b.body.Write(p) } +func (b *bufferedResponse) WriteHeader(status int) { b.status = status } + +// writeCBORFromJSON re-encodes a captured JSON response as rpc-v2-cbor. An +// error keeps its status and carries __type and message, as the Smithy +// protocol expects. +func writeCBORFromJSON(w http.ResponseWriter, rec *bufferedResponse) { + dec := json.NewDecoder(&rec.body) + dec.UseNumber() + + var decoded any + if err := dec.Decode(&decoded); err != nil { + writeCBORError(w, http.StatusInternalServerError, "InternalServerException", "cannot re-encode response: "+err.Error()) + return + } + + if rec.status >= http.StatusBadRequest { + obj, _ := decoded.(map[string]any) + errType, _ := obj["__type"].(string) + msg, _ := obj["Message"].(string) + writeCBORError(w, rec.status, errType, msg) + + return + } + + payload, err := jsonValueToCBOR("", decoded) + if err != nil { + writeCBORError(w, http.StatusInternalServerError, "InternalServerException", err.Error()) + return + } + + writeCBOR(w, http.StatusOK, payload) +} + +func writeCBORError(w http.ResponseWriter, status int, errType, msg string) { + writeCBOR(w, status, map[string]any{"__type": errType, "message": msg}) +} + +func writeCBOR(w http.ResponseWriter, status int, payload any) { + body, err := cborEncMode.Marshal(payload) + if err != nil { + status = http.StatusInternalServerError + body, _ = cborEncMode.Marshal(map[string]any{"__type": "InternalServerException", "message": err.Error()}) + } + + w.Header().Set(smithyProtocolHeader, rpcV2CBOR) + w.Header().Set("Content-Type", cborContentType) + w.WriteHeader(status) + _, _ = w.Write(body) +} + +// jsonValueToCBOR converts a decoded JSON response value for CBOR encoding: +// integral numbers become integers (the SDK reads Integer and Long members +// only from CBOR integers), other numbers float64, and the Timestamp members +// time.Time so they encode with tag 1. key is the member name v belongs to. +func jsonValueToCBOR(key string, v any) (any, error) { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + + for k, e := range t { + c, err := jsonValueToCBOR(k, e) + if err != nil { + return nil, err + } + + out[k] = c + } + + return out, nil + case []any: + out := make([]any, len(t)) + + for i, e := range t { + c, err := jsonValueToCBOR(key, e) + if err != nil { + return nil, err + } + + out[i] = c + } + + return out, nil + case json.Number: + return jsonNumberToCBOR(key, t) + default: + return v, nil + } +} + +func jsonNumberToCBOR(key string, n json.Number) (any, error) { + if timestampMembers[key] { + f, err := n.Float64() + if err != nil { + return nil, fmt.Errorf("member %s: %w", key, err) + } + + sec, frac := math.Modf(f) + + return time.Unix(int64(sec), int64(frac*float64(time.Second))).UTC(), nil + } + + if i, err := n.Int64(); err == nil { + return i, nil + } + + f, err := n.Float64() + if err != nil { + return nil, fmt.Errorf("member %s: %w", key, err) + } + + return f, nil +} + +// cborToJSON re-encodes a CBOR request body as AWS JSON 1.0: a tag-1 +// timestamp becomes epoch seconds and a byte string base64, the JSON forms of +// those Smithy types. An empty body is an empty input structure. +func cborToJSON(raw []byte) ([]byte, error) { + if len(raw) == 0 { + return []byte("{}"), nil + } + + var v any + if err := cborDecMode.Unmarshal(raw, &v); err != nil { + return nil, err + } + + if _, ok := v.(map[string]any); !ok { + return nil, fmt.Errorf("%w, got %T", errNotCBORMap, v) + } + + return json.Marshal(cborValueToJSON(v)) +} + +func cborValueToJSON(v any) any { + switch t := v.(type) { + case map[string]any: + for k, e := range t { + t[k] = cborValueToJSON(e) + } + + return t + case []any: + for i, e := range t { + t[i] = cborValueToJSON(e) + } + + return t + case time.Time: + return float64(t.UnixNano()) / float64(time.Second) + case []byte: + return base64.StdEncoding.EncodeToString(t) + default: + return v + } +} diff --git a/server/aws/keyspaces/cbor_test.go b/server/aws/keyspaces/cbor_test.go new file mode 100644 index 000000000..9b22c3154 --- /dev/null +++ b/server/aws/keyspaces/cbor_test.go @@ -0,0 +1,229 @@ +package keyspaces_test + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/fxamacker/cbor/v2" + + "github.com/stackshy/cloudemu/v2/config" + ksprovider "github.com/stackshy/cloudemu/v2/providers/aws/keyspaces" + ksserver "github.com/stackshy/cloudemu/v2/server/aws/keyspaces" +) + +// cborClient sends raw rpc-v2-cbor requests the way aws-sdk-go-v2 keyspaces +// v1.37+ does (see its request_snapshot/*.request.snap). +type cborClient struct { + t *testing.T + url string +} + +func newCBORClient(t *testing.T) *cborClient { + t.Helper() + + opts := config.NewOptions(config.WithRegion("us-east-1"), config.WithAccountID("123456789012")) + srv := httptest.NewServer(ksserver.NewCBOR(ksprovider.New(opts))) + t.Cleanup(srv.Close) + + return &cborClient{t: t, url: srv.URL} +} + +// call returns the status and the decoded CBOR body. Tag-1 timestamps decode +// as time.Time and integers as uint64/int64, so a test can check the types. +func (c *cborClient) call(op string, in map[string]any) (int, map[string]any) { + c.t.Helper() + + payload, err := cbor.Marshal(in) + if err != nil { + c.t.Fatalf("%s: encode: %v", op, err) + } + + req, err := http.NewRequest(http.MethodPost, c.url+"/service/KeyspacesService/operation/"+op, bytes.NewReader(payload)) + if err != nil { + c.t.Fatalf("%s: request: %v", op, err) + } + + req.Header.Set("Smithy-Protocol", "rpc-v2-cbor") + req.Header.Set("Content-Type", "application/cbor") + req.Header.Set("Accept", "application/cbor") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + c.t.Fatalf("%s: do: %v", op, err) + } + defer resp.Body.Close() + + if got := resp.Header.Get("Smithy-Protocol"); got != "rpc-v2-cbor" { + c.t.Fatalf("%s: Smithy-Protocol header %q", op, got) + } + + if got := resp.Header.Get("Content-Type"); got != "application/cbor" { + c.t.Fatalf("%s: Content-Type %q", op, got) + } + + raw, err := io.ReadAll(resp.Body) + if err != nil { + c.t.Fatalf("%s: read: %v", op, err) + } + + var out map[string]any + if err := cbor.Unmarshal(raw, &out); err != nil { + c.t.Fatalf("%s: decode %x: %v", op, raw, err) + } + + return resp.StatusCode, out +} + +func (c *cborClient) ok(op string, in map[string]any) map[string]any { + c.t.Helper() + + status, out := c.call(op, in) + if status != http.StatusOK { + c.t.Fatalf("%s: status %d: %v", op, status, out) + } + + return out +} + +func cborSchema() map[string]any { + return map[string]any{ + "allColumns": []any{map[string]any{"name": "id", "type": "int"}}, + "partitionKeys": []any{map[string]any{"name": "id"}}, + } +} + +// The rpc-v2-cbor form of the API serves the same operations as AWS JSON 1.0, +// with Integer/Long members as CBOR integers and Timestamp members as tag 1. +func TestCBORTableRoundTrip(t *testing.T) { + c := newCBORClient(t) + + out := c.ok("CreateKeyspace", map[string]any{"keyspaceName": "app"}) + if out["resourceArn"] != "arn:aws:cassandra:us-east-1:123456789012:/keyspace/app/" { + t.Fatalf("CreateKeyspace resourceArn: %v", out) + } + + c.ok("CreateTable", map[string]any{ + "keyspaceName": "app", "tableName": "t1", "schemaDefinition": cborSchema(), + "capacitySpecification": map[string]any{"throughputMode": "PROVISIONED", "readCapacityUnits": 5, "writeCapacityUnits": 7}, + "pointInTimeRecovery": map[string]any{"status": "ENABLED"}, + "ttl": map[string]any{"status": "ENABLED"}, + }) + + got := c.ok("GetTable", map[string]any{"keyspaceName": "app", "tableName": "t1"}) + + if _, ok := got["creationTimestamp"].(time.Time); !ok { + t.Fatalf("creationTimestamp = %T %v, want a tag-1 timestamp", got["creationTimestamp"], got["creationTimestamp"]) + } + + capSpec, _ := got["capacitySpecification"].(map[any]any) + if capSpec["readCapacityUnits"] != uint64(5) || capSpec["writeCapacityUnits"] != uint64(7) { + t.Fatalf("capacitySpecification = %#v, want integer units 5/7", capSpec) + } + + if ttl, _ := got["ttl"].(map[any]any); ttl["status"] != "ENABLED" { + t.Fatalf("ttl = %v", got["ttl"]) + } + + if _, ok := got["clientSideTimestamps"]; ok { + t.Fatalf("clientSideTimestamps present on a table that never set it: %v", got) + } + + // A tag-1 restoreTimestamp decodes like the JSON epoch number. + c.ok("RestoreTable", map[string]any{ + "sourceKeyspaceName": "app", "sourceTableName": "t1", + "targetKeyspaceName": "app", "targetTableName": "t2", + "restoreTimestamp": time.Now().Add(-time.Minute), + }) + + list := c.ok("ListTables", map[string]any{"keyspaceName": "app"}) + if tables, _ := list["tables"].([]any); len(tables) != 2 { + t.Fatalf("ListTables = %v, want 2 tables", list) + } + + c.ok("DeleteTable", map[string]any{"keyspaceName": "app", "tableName": "t2"}) +} + +// Errors keep the JSON protocol's status and code, as a CBOR __type/message +// map. +func TestCBORErrors(t *testing.T) { + c := newCBORClient(t) + + cases := []struct { + name, op string + in map[string]any + status int + code string + }{ + {"missing table", "GetTable", map[string]any{"keyspaceName": "nope", "tableName": "t"}, http.StatusBadRequest, "ResourceNotFoundException"}, + {"bad name", "CreateKeyspace", map[string]any{"keyspaceName": "bad-name"}, http.StatusBadRequest, "ValidationException"}, + {"future restore", "RestoreTable", map[string]any{ + "sourceKeyspaceName": "a", "sourceTableName": "b", "targetKeyspaceName": "a", "targetTableName": "c", + "restoreTimestamp": time.Now().Add(time.Hour), + }, http.StatusBadRequest, "ValidationException"}, + {"unknown operation", "Frobnicate", map[string]any{}, http.StatusBadRequest, "ValidationException"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + status, out := c.call(tc.op, tc.in) + if status != tc.status || out["__type"] != tc.code { + t.Fatalf("status %d body %v, want %d %s", status, out, tc.status, tc.code) + } + + if msg, _ := out["message"].(string); msg == "" { + t.Fatalf("empty message: %v", out) + } + }) + } +} + +func TestCBORRejectsMalformedBody(t *testing.T) { + c := newCBORClient(t) + + req, _ := http.NewRequest(http.MethodPost, c.url+"/service/KeyspacesService/operation/ListKeyspaces", bytes.NewReader([]byte{0xff, 0x00})) + req.Header.Set("Smithy-Protocol", "rpc-v2-cbor") + req.Header.Set("Content-Type", "application/cbor") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer resp.Body.Close() + + var out map[string]any + if err := cbor.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decode: %v", err) + } + + if resp.StatusCode != http.StatusBadRequest || out["__type"] != "SerializationException" { + t.Fatalf("status %d body %v, want 400 SerializationException", resp.StatusCode, out) + } +} + +func TestCBORMatches(t *testing.T) { + h := ksserver.NewCBOR(nil) + + cases := []struct { + name, path, proto string + want bool + }{ + {"keyspaces cbor", "/service/KeyspacesService/operation/GetTable", "rpc-v2-cbor", true}, + {"cloudwatch cbor", "/service/GraniteServiceVersion20100801/operation/ListMetrics", "rpc-v2-cbor", false}, + {"keyspaces path without the protocol header", "/service/KeyspacesService/operation/GetTable", "", false}, + } + + for _, tc := range cases { + req := httptest.NewRequest(http.MethodPost, tc.path, nil) + if tc.proto != "" { + req.Header.Set("Smithy-Protocol", tc.proto) + } + + if got := h.Matches(req); got != tc.want { + t.Errorf("%s: Matches = %v, want %v", tc.name, got, tc.want) + } + } +} diff --git a/server/aws/keyspaces/handler.go b/server/aws/keyspaces/handler.go index 449912229..269ce0f53 100644 --- a/server/aws/keyspaces/handler.go +++ b/server/aws/keyspaces/handler.go @@ -14,7 +14,10 @@ import ( ksdriver "github.com/stackshy/cloudemu/v2/services/keyspaces/driver" ) -const targetPrefix = "KeyspacesService." +const ( + targetPrefix = "KeyspacesService." + iamService = "cassandra" +) // Handler serves Keyspaces requests against a keyspaces driver. type Handler struct { @@ -110,4 +113,4 @@ func wireMessage(err error) string { // IAMService returns the IAM service prefix of the operations this handler // serves. -func (*Handler) IAMService() string { return "cassandra" } +func (*Handler) IAMService() string { return iamService } diff --git a/server/aws/keyspaces/tables.go b/server/aws/keyspaces/tables.go index 7d80b49cb..cb00d72f1 100644 --- a/server/aws/keyspaces/tables.go +++ b/server/aws/keyspaces/tables.go @@ -134,10 +134,19 @@ func tableOutput(t *ksdriver.Table) keyspaces.GetTableOutput { EncryptionSpecification: toWireEncryption(t.EncryptionSpecification), DefaultTimeToLive: aws.Int32(int32(t.DefaultTimeToLive)), //nolint:gosec // mock TTL seconds never overflow int32. PointInTimeRecovery: &types.PointInTimeRecoverySummary{Status: types.PointInTimeRecoveryStatus(t.PointInTimeRecoveryStatus)}, - Ttl: &types.TimeToLive{Status: types.TimeToLiveStatus(t.TTLStatus)}, - ClientSideTimestamps: &types.ClientSideTimestamps{Status: types.ClientSideTimestampsStatus(t.ClientSideTimestamps)}, CdcSpecification: &types.CdcSpecificationSummary{Status: types.CdcStatus(t.CdcStatus)}, } + // ttl and clientSideTimestamps are reported only once enabled: their enums + // hold only ENABLED, and real Keyspaces omits them on a table that never + // set them. + if t.TTLStatus != "" { + out.Ttl = &types.TimeToLive{Status: types.TimeToLiveStatus(t.TTLStatus)} + } + + if t.ClientSideTimestamps != "" { + out.ClientSideTimestamps = &types.ClientSideTimestamps{Status: types.ClientSideTimestampsStatus(t.ClientSideTimestamps)} + } + if t.Comment != "" { out.Comment = &types.Comment{Message: aws.String(t.Comment)} } diff --git a/server/aws/keyspaces/tables_test.go b/server/aws/keyspaces/tables_test.go index 4922733c9..adb03c44f 100644 --- a/server/aws/keyspaces/tables_test.go +++ b/server/aws/keyspaces/tables_test.go @@ -118,3 +118,52 @@ func TestGetTableOmitsUnsetFields(t *testing.T) { t.Fatalf("provisioned capacity: %v", pcap) } } + +// GetTable reports ttl and clientSideTimestamps only once they are enabled, +// so a Terraform aws_keyspaces_table that sets neither reads back no block +// and plans no replacement. +func TestSDKGetTableOmitsUnsetTTLAndClientSideTimestamps(t *testing.T) { + client := newSDKClient(t) + ctx := context.Background() + + if _, err := client.CreateKeyspace(ctx, &awskeyspaces.CreateKeyspaceInput{KeyspaceName: aws.String("app")}); err != nil { + t.Fatalf("CreateKeyspace: %v", err) + } + + if _, err := client.CreateTable(ctx, &awskeyspaces.CreateTableInput{ + KeyspaceName: aws.String("app"), TableName: aws.String("plain"), SchemaDefinition: sampleSchema(), + }); err != nil { + t.Fatalf("CreateTable plain: %v", err) + } + + if _, err := client.CreateTable(ctx, &awskeyspaces.CreateTableInput{ + KeyspaceName: aws.String("app"), TableName: aws.String("on"), SchemaDefinition: sampleSchema(), + Ttl: &kstypes.TimeToLive{Status: kstypes.TimeToLiveStatusEnabled}, + ClientSideTimestamps: &kstypes.ClientSideTimestamps{Status: kstypes.ClientSideTimestampsStatusEnabled}, + }); err != nil { + t.Fatalf("CreateTable on: %v", err) + } + + plain, err := client.GetTable(ctx, &awskeyspaces.GetTableInput{KeyspaceName: aws.String("app"), TableName: aws.String("plain")}) + if err != nil { + t.Fatalf("GetTable plain: %v", err) + } + + if plain.Ttl != nil || plain.ClientSideTimestamps != nil { + t.Fatalf("plain table: ttl=%+v clientSideTimestamps=%+v, want both omitted", plain.Ttl, plain.ClientSideTimestamps) + } + + if plain.PointInTimeRecovery == nil || plain.PointInTimeRecovery.Status != kstypes.PointInTimeRecoveryStatusDisabled { + t.Fatalf("plain table pointInTimeRecovery: %+v", plain.PointInTimeRecovery) + } + + on, err := client.GetTable(ctx, &awskeyspaces.GetTableInput{KeyspaceName: aws.String("app"), TableName: aws.String("on")}) + if err != nil { + t.Fatalf("GetTable on: %v", err) + } + + if on.Ttl == nil || on.Ttl.Status != kstypes.TimeToLiveStatusEnabled || + on.ClientSideTimestamps == nil || on.ClientSideTimestamps.Status != kstypes.ClientSideTimestampsStatusEnabled { + t.Fatalf("enabled table: ttl=%+v clientSideTimestamps=%+v", on.Ttl, on.ClientSideTimestamps) + } +} diff --git a/server/aws/keyspaces_cbor_test.go b/server/aws/keyspaces_cbor_test.go new file mode 100644 index 000000000..7880c9e02 --- /dev/null +++ b/server/aws/keyspaces_cbor_test.go @@ -0,0 +1,92 @@ +package aws + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + "github.com/fxamacker/cbor/v2" + + cloudemu "github.com/stackshy/cloudemu/v2" +) + +// keyspacesCBOR is a Keyspaces rpc-v2-cbor request as aws-sdk-go-v2 keyspaces +// v1.37+ (hashicorp/aws 6.69+) sends it. +func keyspacesCBOR(t *testing.T, op string, in map[string]any) sreq { + t.Helper() + + payload, err := cbor.Marshal(in) + if err != nil { + t.Fatalf("cbor: %v", err) + } + + return sreq{ + path: "/service/KeyspacesService/operation/" + op, ctype: "application/cbor", body: string(payload), + service: "cassandra", header: map[string]string{"Smithy-Protocol": "rpc-v2-cbor"}, + } +} + +func decodeCBORMap(t *testing.T, body string) map[string]any { + t.Helper() + + var out map[string]any + if err := cbor.Unmarshal([]byte(body), &out); err != nil { + t.Fatalf("decode %q: %v", body, err) + } + + return out +} + +// The full AWS server routes Keyspaces rpc-v2-cbor to Keyspaces, not to +// CloudWatch, which claims every other rpc-v2-cbor request. +func TestKeyspacesCBORDispatch(t *testing.T) { + ts := httptest.NewServer(New(DriversFrom(cloudemu.NewAWS()))) + t.Cleanup(ts.Close) + + rq := keyspacesCBOR(t, "CreateKeyspace", map[string]any{"keyspaceName": "app"}) + + req, err := http.NewRequest(http.MethodPost, ts.URL+rq.path, bytes.NewReader([]byte(rq.body))) + if err != nil { + t.Fatalf("request: %v", err) + } + + req.Header.Set("Content-Type", rq.ctype) + req.Header.Set("Smithy-Protocol", "rpc-v2-cbor") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer resp.Body.Close() + + var out map[string]any + if err := cbor.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decode: %v", err) + } + + if resp.StatusCode != http.StatusOK || out["resourceArn"] == nil { + t.Fatalf("CreateKeyspace over rpc-v2-cbor: %d %v", resp.StatusCode, out) + } +} + +// Under --enforce-auth a Keyspaces rpc-v2-cbor request is authorized as the +// cassandra: named in its path, and a deny is a CBOR error. +func TestKeyspacesCBORAuthorization(t *testing.T) { + ts, cloud := matrixServer(t, nil) + lister := userWithPolicy(t, cloud, "kslister", allow("cassandra:ListKeyspaces")) + + status, body := doSigned(t, ts, lister, keyspacesCBOR(t, "ListKeyspaces", map[string]any{})) + if status != http.StatusOK { + t.Fatalf("ListKeyspaces: %d %v", status, decodeCBORMap(t, body)) + } + + status, body = doSigned(t, ts, lister, keyspacesCBOR(t, "CreateKeyspace", map[string]any{"keyspaceName": "app"})) + if out := decodeCBORMap(t, body); status != http.StatusForbidden || out["__type"] != accessDeny { + t.Fatalf("CreateKeyspace: %d %v, want 403 %s", status, out, accessDeny) + } + + if _, err := cloud.Keyspaces.GetKeyspace(t.Context(), "app"); err == nil { + t.Fatal("a denied CreateKeyspace created the keyspace") + } +}