From 241a69ab4648af6dd12d20e14de4e24ff927b4d6 Mon Sep 17 00:00:00 2001 From: John Cavanaugh <59479+cavanaug@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:10:04 -0700 Subject: [PATCH 1/6] feat(firehose): opt-in Direct PUT throughput limits (revision 1 baseline) --- crates/fakecloud-core/src/quota.rs | 9 + .../tests/firehose_throughput.rs | 540 ++++++++++++++++++ crates/fakecloud-firehose/src/engine.rs | 85 ++- .../src/engine/tests/throughput.rs | 502 ++++++++++++++++ crates/fakecloud-firehose/src/lib.rs | 1 + crates/fakecloud-firehose/src/service.rs | 2 +- crates/fakecloud-firehose/src/throughput.rs | 514 +++++++++++++++++ crates/fakecloud-server/src/main.rs | 1 + crates/fakecloud-servicequotas/src/catalog.rs | 72 ++- .../fakecloud-servicequotas/src/provider.rs | 40 +- crates/fakecloud-servicequotas/src/service.rs | 8 +- .../src/service/introspection.rs | 2 +- .../src/service/tests.rs | 234 ++++++++ website/content/docs/services/firehose.md | 59 +- .../content/docs/services/servicequotas.md | 12 +- 15 files changed, 2064 insertions(+), 17 deletions(-) create mode 100644 crates/fakecloud-e2e/tests/firehose_throughput.rs create mode 100644 crates/fakecloud-firehose/src/engine/tests/throughput.rs create mode 100644 crates/fakecloud-firehose/src/throughput.rs diff --git a/crates/fakecloud-core/src/quota.rs b/crates/fakecloud-core/src/quota.rs index 5190118b1..e96af38e0 100644 --- a/crates/fakecloud-core/src/quota.rs +++ b/crates/fakecloud-core/src/quota.rs @@ -24,6 +24,15 @@ pub const DEFAULT_SECURITY_GROUPS_PER_INTERFACE: usize = 5; /// AWS default for [`RULES_PER_SECURITY_GROUP`]. pub const DEFAULT_RULES_PER_SECURITY_GROUP: usize = 60; +/// Service code of the Amazon Data Firehose quotas. +pub const FIREHOSE_SERVICE_CODE: &str = "firehose"; +/// Combined `PutRecord` and `PutRecordBatch` requests per second per stream. +pub const FIREHOSE_PUT_REQUESTS_PER_SECOND: &str = "L-7681C790"; +/// Records per second per Firehose delivery stream. +pub const FIREHOSE_RECORDS_PER_SECOND: &str = "L-17D4EADE"; +/// Mebibytes per second per Firehose delivery stream. +pub const FIREHOSE_MIB_PER_SECOND: &str = "L-937C879B"; + /// Service code of the IAM quotas. pub const IAM_SERVICE_CODE: &str = "iam"; diff --git a/crates/fakecloud-e2e/tests/firehose_throughput.rs b/crates/fakecloud-e2e/tests/firehose_throughput.rs new file mode 100644 index 000000000..4c04edbf2 --- /dev/null +++ b/crates/fakecloud-e2e/tests/firehose_throughput.rs @@ -0,0 +1,540 @@ +//! Public Firehose admission contract, using real HTTP and the official AWS SDK. +//! Fixed-window refill/concurrency are covered by the deterministic engine tests; +//! these tests use impossible single-call costs or zero capacity, not clock races. + +use std::future::Future; +use std::io::Read; +use std::time::Duration; + +use aws_sdk_firehose::config::retry::RetryConfig; +use aws_sdk_firehose::error::SdkError; +use aws_sdk_firehose::operation::put_record::PutRecordError; +use aws_sdk_firehose::operation::put_record_batch::PutRecordBatchError; +use aws_sdk_firehose::primitives::Blob; +use aws_sdk_firehose::types::{ + BufferingHints, CompressionFormat, DeliveryStreamType, ExtendedS3DestinationConfiguration, + Record, +}; +use fakecloud_sdk::types::{PutServiceQuotaRequest, QuotaEnforcement}; +use fakecloud_sdk::FakeCloud; +use fakecloud_testkit::TestServer; + +const REQUESTS: &str = "L-7681C790"; +const RECORDS: &str = "L-17D4EADE"; +const MIB: &str = "L-937C879B"; +const REGION: &str = "us-west-2"; + +async fn bounded(test: impl Future) { + tokio::time::timeout(Duration::from_secs(60), test) + .await + .expect("Firehose throughput E2E exceeded 60s"); +} + +fn client(sdk: &aws_config::SdkConfig, attempts: u32) -> aws_sdk_firehose::Client { + aws_sdk_firehose::Client::from_conf( + aws_sdk_firehose::config::Builder::from(sdk) + .retry_config(RetryConfig::standard().with_max_attempts(attempts)) + .build(), + ) +} + +fn record(bytes: &[u8]) -> Record { + Record::builder().data(Blob::new(bytes)).build().unwrap() +} + +async fn setup( + server: &TestServer, + bucket: &str, + gzip: bool, +) -> (aws_sdk_firehose::Client, aws_sdk_s3::Client) { + let sdk = server.aws_config_in(REGION).await; + let firehose = client(&sdk, 1); + let s3 = aws_sdk_s3::Client::from_conf( + aws_sdk_s3::config::Builder::from(&sdk) + .force_path_style(true) + .build(), + ); + s3.create_bucket() + .bucket(bucket) + .create_bucket_configuration( + aws_sdk_s3::types::CreateBucketConfiguration::builder() + .location_constraint(aws_sdk_s3::types::BucketLocationConstraint::UsWest2) + .build(), + ) + .send() + .await + .expect("create destination bucket"); + create_stream(&firehose, bucket, bucket, gzip).await; + (firehose, s3) +} + +async fn create_stream( + firehose: &aws_sdk_firehose::Client, + stream: &str, + bucket: &str, + gzip: bool, +) { + firehose + .create_delivery_stream() + .delivery_stream_name(stream) + .delivery_stream_type(DeliveryStreamType::DirectPut) + .extended_s3_destination_configuration( + ExtendedS3DestinationConfiguration::builder() + .role_arn("arn:aws:iam::123456789012:role/firehose") + .bucket_arn(format!("arn:aws:s3:::{bucket}")) + .compression_format(if gzip { + CompressionFormat::Gzip + } else { + CompressionFormat::Uncompressed + }) + .buffering_hints( + BufferingHints::builder() + .size_in_mbs(5) + .interval_in_seconds(0) + .build(), + ) + .build() + .unwrap(), + ) + .send() + .await + .expect("create DirectPut delivery stream"); +} + +async fn limit(fc: &FakeCloud, code: &str, value: f64) { + let quota = fc + .service_quotas() + .put_quota( + "firehose", + code, + &PutServiceQuotaRequest { + region: Some(REGION.into()), + value: Some(value), + enforce: Some(QuotaEnforcement::Enforce), + ..Default::default() + }, + ) + .await + .expect("set enforced quota through local SDK"); + assert_eq!(quota.applied_value, value); + assert!(quota.enforced); +} + +fn assert_single_unavailable(err: SdkError) { + let SdkError::ServiceError(context) = err else { + panic!("expected modeled PutRecord service error, got {err:?}"); + }; + assert_eq!(context.raw().status().as_u16(), 500); + assert!( + matches!( + context.err(), + PutRecordError::ServiceUnavailableException(_) + ), + "expected modeled ServiceUnavailableException, got {:?}", + context.err() + ); +} + +fn assert_batch_unavailable(err: SdkError) { + let SdkError::ServiceError(context) = err else { + panic!("expected modeled PutRecordBatch service error, got {err:?}"); + }; + assert_eq!(context.raw().status().as_u16(), 500); + assert!( + matches!( + context.err(), + PutRecordBatchError::ServiceUnavailableException(_) + ), + "expected modeled ServiceUnavailableException, got {:?}", + context.err() + ); +} + +async fn put(firehose: &aws_sdk_firehose::Client, stream: &str, bytes: &[u8]) { + let response = firehose + .put_record() + .delivery_stream_name(stream) + .record(record(bytes)) + .send() + .await + .expect("PutRecord succeeds"); + assert!(!response.record_id().is_empty()); +} + +async fn batch(firehose: &aws_sdk_firehose::Client, stream: &str, records: Vec) { + let count = records.len(); + let response = firehose + .put_record_batch() + .delivery_stream_name(stream) + .set_records(Some(records)) + .send() + .await + .expect("whole batch succeeds"); + assert_eq!(response.failed_put_count(), 0); + assert_eq!(response.request_responses().len(), count); + assert!(response + .request_responses() + .iter() + .all(|r| r.record_id().is_some_and(|id| !id.is_empty()) && r.error_code().is_none())); +} + +async fn assert_no_objects(s3: &aws_sdk_s3::Client, bucket: &str) { + // Observe across a worker tick, not just before an asynchronous write starts. + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + loop { + let listing = s3.list_objects_v2().bucket(bucket).send().await.unwrap(); + assert!( + listing.contents().is_empty(), + "rejection must not deliver any records" + ); + if tokio::time::Instant::now() >= deadline { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +async fn payloads(s3: &aws_sdk_s3::Client, bucket: &str, count: usize, gzip: bool) -> Vec> { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let listing = s3.list_objects_v2().bucket(bucket).send().await.unwrap(); + assert!( + listing.contents().len() <= count, + "unexpected extra delivery objects" + ); + if listing.contents().len() == count { + let mut payloads = Vec::new(); + for object in listing.contents() { + let object = s3 + .get_object() + .bucket(bucket) + .key(object.key().unwrap()) + .send() + .await + .expect("read delivered object"); + let bytes = object.body.collect().await.unwrap().into_bytes(); + let bytes = if gzip { + assert!( + bytes.len() < 512, + "fixture must actually compress on output" + ); + let mut raw = Vec::new(); + flate2::read::GzDecoder::new(bytes.as_ref()) + .read_to_end(&mut raw) + .expect("decode delivered GZIP"); + raw + } else { + bytes.to_vec() + }; + payloads.push(bytes); + } + payloads.sort(); + return payloads; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("delivery objects within 10s") +} + +#[tokio::test] +async fn zero_request_capacity_returns_modeled_http_500_for_both_apis_then_live_raise_recovers() { + bounded(async { + let server = TestServer::start().await; + let bucket = "fh-throughput-requests"; + let (firehose, s3) = setup(&server, bucket, false).await; + let fc = FakeCloud::new(server.endpoint()); + limit(&fc, REQUESTS, 0.0).await; + assert_single_unavailable( + firehose + .put_record() + .delivery_stream_name(bucket) + .record(record(b"single")) + .send() + .await + .expect_err("zero request capacity rejects PutRecord"), + ); + assert_batch_unavailable( + firehose + .put_record_batch() + .delivery_stream_name(bucket) + .records(record(b"batch-a")) + .records(record(b"batch-b")) + .send() + .await + .expect_err("zero request capacity rejects whole batch"), + ); + assert_no_objects(&s3, bucket).await; + limit(&fc, REQUESTS, 100.0).await; + put(&firehose, bucket, b"single").await; + batch( + &firehose, + bucket, + vec![record(b"batch-a"), record(b"batch-b")], + ) + .await; + assert_eq!( + payloads(&s3, bucket, 2, false).await, + vec![b"batch-abatch-b".to_vec(), b"single".to_vec()] + ); + }) + .await; +} + +#[tokio::test] +async fn record_capacity_rejects_entire_batch_without_s3_side_effects_then_live_raise_delivers() { + bounded(async { + let server = TestServer::start().await; + let bucket = "fh-throughput-records"; + let (firehose, s3) = setup(&server, bucket, false).await; + let fc = FakeCloud::new(server.endpoint()); + limit(&fc, RECORDS, 1.0).await; + assert_batch_unavailable( + firehose + .put_record_batch() + .delivery_stream_name(bucket) + .records(record(b"first")) + .records(record(b"second")) + .send() + .await + .expect_err("two records cannot fit in capacity one"), + ); + assert_no_objects(&s3, bucket).await; + limit(&fc, RECORDS, 2.0).await; + batch(&firehose, bucket, vec![record(b"first"), record(b"second")]).await; + assert_eq!( + payloads(&s3, bucket, 1, false).await, + vec![b"firstsecond".to_vec()] + ); + }) + .await; +} + +#[tokio::test] +async fn byte_capacity_counts_decoded_raw_bytes_not_base64_or_gzip_and_rejects_whole_batch() { + bounded(async { + let server = TestServer::start().await; + let bucket = "fh-throughput-bytes"; + let (firehose, s3) = setup(&server, bucket, true).await; + let fc = FakeCloud::new(server.endpoint()); + let a = vec![b'a'; 4096]; + let b = vec![b'b'; 4096]; + // Each individual record fits in 6144 bytes, but the 8192-byte batch does not. + // GZIP output is tiny; base64 on the wire is larger than either capacity. + limit(&fc, MIB, 6144.0 / 1_048_576.0).await; + assert_batch_unavailable( + firehose + .put_record_batch() + .delivery_stream_name(bucket) + .records(record(&a)) + .records(record(&b)) + .send() + .await + .expect_err("raw decoded batch exceeds byte capacity"), + ); + assert_no_objects(&s3, bucket).await; + limit(&fc, MIB, 8192.0 / 1_048_576.0).await; + batch(&firehose, bucket, vec![record(&a), record(&b)]).await; + assert_eq!(payloads(&s3, bucket, 1, true).await, vec![[a, b].concat()]); + }) + .await; +} + +#[tokio::test] +async fn sustained_zero_capacity_exhausts_bounded_standard_sdk_retries_for_both_apis() { + bounded(async { + let server = TestServer::start().await; + let bucket = "fh-throughput-retries"; + let (_, s3) = setup(&server, bucket, false).await; + let fc = FakeCloud::new(server.endpoint()); + limit(&fc, REQUESTS, 0.0).await; + let firehose = client(&server.aws_config_in(REGION).await, 3); + tokio::time::timeout(Duration::from_secs(20), async { + assert_single_unavailable( + firehose + .put_record() + .delivery_stream_name(bucket) + .record(record(b"single")) + .send() + .await + .expect_err("sustained throttling outlasts SDK attempts"), + ); + assert_batch_unavailable( + firehose + .put_record_batch() + .delivery_stream_name(bucket) + .records(record(b"batch")) + .send() + .await + .expect_err("sustained throttling outlasts batch SDK attempts"), + ); + }) + .await + .expect("three-attempt SDK clients must finish within 20s"); + assert_no_objects(&s3, bucket).await; + }) + .await; +} + +#[tokio::test] +async fn configured_zero_values_are_not_enforced_by_default() { + bounded(async { + let server = TestServer::start().await; + let bucket = "fh-throughput-disabled"; + let (firehose, s3) = setup(&server, bucket, false).await; + let fc = FakeCloud::new(server.endpoint()); + for code in [REQUESTS, RECORDS, MIB] { + let quota = fc + .service_quotas() + .put_quota( + "firehose", + code, + &PutServiceQuotaRequest { + region: Some(REGION.into()), + value: Some(0.0), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(quota.applied_value, 0.0); + assert!(!quota.enforced); + } + put(&firehose, bucket, b"single").await; + batch(&firehose, bucket, vec![record(b"batch")]).await; + assert_eq!( + payloads(&s3, bucket, 2, false).await, + vec![b"batch".to_vec(), b"single".to_vec()] + ); + }) + .await; +} + +#[tokio::test] +async fn other_account_and_region_quota_values_do_not_reject_the_producer() { + bounded(async { + let server = TestServer::start().await; + let bucket = "fh-throughput-scope"; + let (firehose, s3) = setup(&server, bucket, false).await; + let fc = FakeCloud::new(server.endpoint()); + let account = fc + .service_quotas() + .get_quotas(None, Some(REGION), Some("firehose")) + .await + .unwrap() + .account_id; + let other = "210987654321"; + fc.service_quotas() + .put_quota( + "firehose", + REQUESTS, + &PutServiceQuotaRequest { + account_id: Some(other.into()), + region: Some(REGION.into()), + value: Some(0.0), + enforce: Some(QuotaEnforcement::Enforce), + }, + ) + .await + .unwrap(); + put(&firehose, bucket, b"other-account-isolated").await; + + let (access, secret) = server.create_admin(other, "firehose-producer").await; + let sdk = server.aws_config_in(REGION).await; + let other_sdk = sdk + .into_builder() + .credentials_provider( + aws_credential_types::provider::SharedCredentialsProvider::new( + aws_credential_types::Credentials::new(access, secret, None, None, "test"), + ), + ) + .build(); + let other_firehose = client(&other_sdk, 1); + create_stream(&other_firehose, bucket, bucket, false).await; + assert_single_unavailable( + other_firehose + .put_record() + .delivery_stream_name(bucket) + .record(record(b"other-account-rejected")) + .send() + .await + .unwrap_err(), + ); + + fc.service_quotas() + .put_quota( + "firehose", + REQUESTS, + &PutServiceQuotaRequest { + account_id: Some(account), + region: Some("us-east-2".into()), + value: Some(0.0), + enforce: Some(QuotaEnforcement::Enforce), + }, + ) + .await + .unwrap(); + put(&firehose, bucket, b"other-region-isolated").await; + let regional = client(&server.aws_config_in("us-east-2").await, 1); + create_stream(®ional, bucket, bucket, false).await; + assert_batch_unavailable( + regional + .put_record_batch() + .delivery_stream_name(bucket) + .records(record(b"other-region-rejected")) + .send() + .await + .unwrap_err(), + ); + assert_eq!( + payloads(&s3, bucket, 2, false).await, + vec![ + b"other-account-isolated".to_vec(), + b"other-region-isolated".to_vec() + ] + ); + }) + .await; +} + +#[tokio::test] +async fn service_quotas_reports_regional_firehose_defaults_over_official_sdk() { + bounded(async { + let server = TestServer::start().await; + for (region, values) in [ + ("us-west-2", [2000.0, 500_000.0, 5.0]), + ("eu-west-1", [2000.0, 500_000.0, 5.0]), + ("us-east-2", [1000.0, 100_000.0, 1.0]), + ] { + let sq = aws_sdk_servicequotas::Client::new(&server.aws_config_in(region).await); + for (code, expected) in [REQUESTS, RECORDS, MIB].into_iter().zip(values) { + let default = sq + .get_aws_default_service_quota() + .service_code("firehose") + .quota_code(code) + .send() + .await + .unwrap(); + assert_eq!( + default.quota().unwrap().value(), + Some(expected), + "{region}/{code}" + ); + let applied = sq + .get_service_quota() + .service_code("firehose") + .quota_code(code) + .send() + .await + .unwrap(); + assert_eq!( + applied.quota().unwrap().value(), + Some(expected), + "{region}/{code}" + ); + } + } + }) + .await; +} diff --git a/crates/fakecloud-firehose/src/engine.rs b/crates/fakecloud-firehose/src/engine.rs index ab2118ad8..50fe3bece 100644 --- a/crates/fakecloud-firehose/src/engine.rs +++ b/crates/fakecloud-firehose/src/engine.rs @@ -4,6 +4,10 @@ use std::time::Duration; use bytes::Bytes; use chrono::{DateTime, Utc}; +use fakecloud_core::quota::{ + QuotaProvider, FIREHOSE_MIB_PER_SECOND, FIREHOSE_PUT_REQUESTS_PER_SECOND, + FIREHOSE_RECORDS_PER_SECOND, FIREHOSE_SERVICE_CODE, +}; use fakecloud_core::service::AwsServiceError; use fakecloud_persistence::S3Store; use fakecloud_s3::{delivery::DeliveryWriteOutcome, SharedS3State}; @@ -14,6 +18,7 @@ use tokio::task::{Id, JoinHandle, JoinSet}; use tokio::time::Instant; use crate::output::object_key; +use crate::throughput::{ThroughputLimits, ThroughputWindow}; use crate::writer::{BatchWriter, DeliveryBatch, S3BatchWriter}; use crate::{delivery_config, DeliveryConfig, DeliveryStream, SharedFirehoseState, StreamKey}; @@ -148,6 +153,9 @@ impl StreamBuffer { #[derive(Default)] struct Registry { streams: BTreeMap, + // Ingestion accounting outlives a drained S3 buffer and also covers streams + // with configuration-only destinations. Never used by delivery retries. + windows: BTreeMap, // Last assigned stream, not last scanned. A removed generation remains a // valid ordering cursor; range selection skips it and wraps naturally. last_scheduled: Option, @@ -161,6 +169,7 @@ struct EngineCore { registry: Mutex, clock: Arc, writer: RwLock>>, + quota_provider: RwLock>>, store: Arc>>>, wake: Arc, } @@ -198,6 +207,7 @@ impl FirehoseEngine { registry: Mutex::new(Registry::default()), clock: Arc::new(SystemClock), writer: RwLock::new(None), + quota_provider: RwLock::new(None), store: Arc::new(RwLock::new(None)), wake: Arc::new(Notify::new()), }), @@ -220,12 +230,52 @@ impl FirehoseEngine { self.core.wake.notify_one(); } + pub fn set_quota_provider(&self, provider: Arc) { + *self.core.quota_provider.write() = Some(provider); + } + pub fn enqueue( &self, account: &str, region: &str, name: &str, records: Vec, + ) -> Result<(), AwsServiceError> { + self.enqueue_admission(account, region, name, records, None) + } + + pub(crate) fn enqueue_put( + &self, + account: &str, + region: &str, + name: &str, + records: Vec, + ) -> Result<(), AwsServiceError> { + // Provider implementations can consult other services (including + // Firehose). Release even the provider guard before resolving limits, + // and never call them under configuration or registry locks. + let provider = self.core.quota_provider.read().clone(); + let limits = provider.map(|provider| { + let limit = + |code| provider.enforced_limit(account, region, FIREHOSE_SERVICE_CODE, code); + ThroughputLimits { + requests: limit(FIREHOSE_PUT_REQUESTS_PER_SECOND), + records: limit(FIREHOSE_RECORDS_PER_SECOND), + mib: limit(FIREHOSE_MIB_PER_SECOND), + } + }); + let limits = + limits.filter(|l| l.requests.is_some() || l.records.is_some() || l.mib.is_some()); + self.enqueue_admission(account, region, name, records, limits) + } + + fn enqueue_admission( + &self, + account: &str, + region: &str, + name: &str, + records: Vec, + limits: Option, ) -> Result<(), AwsServiceError> { // Every path holding both takes configuration before the registry. Keep // the read guard until admission and immediate detachment both commit. @@ -258,18 +308,37 @@ impl FirehoseEngine { "This operation is not permitted on KinesisStreamAsSource delivery stream type.", )); } - let Some(destination) = stream_delivery(stream)? else { - return Ok(()); - }; + // Complete destination preflight before touching usage or buffers. + let destination = stream_delivery(stream)?; let key = StreamKey { account_id: account.into(), region: region.into(), name: name.into(), instance_id: stream.instance_id, }; + let now = self.core.clock.now(); + if let Some(limits) = limits { + let bytes = records.iter().map(Bytes::len).sum(); + if !registry + .windows + .entry(key.clone()) + .or_insert_with(|| ThroughputWindow::new(now.monotonic)) + .try_admit(now.monotonic, limits, records.len(), bytes) + { + return Err(AwsServiceError::aws_error( + StatusCode::INTERNAL_SERVER_ERROR, + "ServiceUnavailableException", + "Throughput limit exceeded for the delivery stream. Back off and retry.", + )); + } + } + // Non-S3 streams retain the configuration-only acknowledgement, but + // public ingestion is still metered even without an S3 backend. + let Some(destination) = destination else { + return Ok(()); + }; let buffer = registry.streams.entry(key.clone()).or_default(); if !records.is_empty() { - let now = self.core.clock.now(); let active = buffer.active.get_or_insert_with(|| ActiveBuffer { records: Vec::new(), input_bytes: 0, @@ -454,6 +523,9 @@ impl EngineCore { registry .streams .retain(|key, _| current(&config, key).is_some()); + registry + .windows + .retain(|key, _| current(&config, key).is_some()); for (key, buffer) in ®istry.streams { if blocked.contains(key) || buffer.in_flight { continue; @@ -501,6 +573,9 @@ impl EngineCore { registry .streams .retain(|key, _| current(&config, key).is_some()); + registry + .windows + .retain(|key, _| current(&config, key).is_some()); if registry.closed && !force { return Vec::new(); } @@ -658,6 +733,8 @@ fn current<'a>(config: &'a crate::FirehoseAccounts, key: &StreamKey) -> Option<& #[cfg(test)] mod tests { + mod throughput; + use super::*; use crate::{config::parse_s3_destination, DeliveryStream, FirehoseAccounts, S3Destination}; use serde_json::json; diff --git a/crates/fakecloud-firehose/src/engine/tests/throughput.rs b/crates/fakecloud-firehose/src/engine/tests/throughput.rs new file mode 100644 index 000000000..cdf352ea2 --- /dev/null +++ b/crates/fakecloud-firehose/src/engine/tests/throughput.rs @@ -0,0 +1,502 @@ +use super::*; +use crate::throughput::ThroughputLimits; +use fakecloud_core::quota::{ + QuotaProvider, FIREHOSE_MIB_PER_SECOND, FIREHOSE_PUT_REQUESTS_PER_SECOND, + FIREHOSE_RECORDS_PER_SECOND, FIREHOSE_SERVICE_CODE, +}; + +struct LimitsProvider { + limits: RwLock, + engine: Weak, + probe_locks: std::sync::atomic::AtomicBool, +} + +impl QuotaProvider for LimitsProvider { + fn applied_value(&self, _: &str, _: &str, _: &str, _: &str) -> Option { + panic!("admission must use enforced_limit") + } + + fn enforced_limit( + &self, + account: &str, + region: &str, + service: &str, + code: &str, + ) -> Option { + assert_eq!(service, FIREHOSE_SERVICE_CODE); + if self.probe_locks.load(std::sync::atomic::Ordering::Relaxed) { + let engine = self.engine.upgrade().unwrap(); + // Fail promptly rather than deadlocking if resolution happens under + // either Firehose lock. Taking write also detects an existing read guard. + let state = engine + .core + .state + .try_write() + .expect("provider under config lock"); + assert!(state.get(account).and_then(|s| s.streams(region)).is_some()); + let _registry = engine + .core + .registry + .try_lock() + .expect("provider under registry lock"); + } + let limits = *self.limits.read(); + match code { + FIREHOSE_PUT_REQUESTS_PER_SECOND => limits.requests, + FIREHOSE_RECORDS_PER_SECOND => limits.records, + FIREHOSE_MIB_PER_SECOND => limits.mib, + _ => panic!("unexpected quota {code}"), + } + } +} + +fn enforce( + f: &Fixture, + requests: Option, + records: Option, + mib: Option, +) -> Arc { + let provider = Arc::new(LimitsProvider { + limits: RwLock::new(ThroughputLimits { + requests, + records, + mib, + }), + engine: Arc::downgrade(&f.engine), + probe_locks: std::sync::atomic::AtomicBool::new(false), + }); + f.engine.set_quota_provider(provider.clone()); + provider +} + +fn put(f: &Fixture, records: Vec) -> Result<(), AwsServiceError> { + crate::FirehoseService::default() + .with_engine(f.engine.clone()) + .deliver_records( + ACCOUNT, + REGION, + NAME, + records.into_iter().map(|b| b.to_vec()).collect(), + ) +} + +fn throttled(result: Result<(), AwsServiceError>) { + let error = result.unwrap_err(); + assert_eq!(error.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(error.code(), "ServiceUnavailableException"); +} + +fn buffered(f: &Fixture) -> Vec { + f.engine + .core + .registry + .lock() + .streams + .values() + .flat_map(|buffer| { + buffer + .queue + .iter() + .flat_map(|batch| batch.batch.records.iter()) + .chain( + buffer + .active + .iter() + .flat_map(|active| active.records.iter()), + ) + .cloned() + }) + .collect() +} + +#[test] +fn provider_resolution_precedes_firehose_locks() { + let f = Fixture::new(60); + let provider = enforce(&f, Some(1.0), None, None); + provider + .probe_locks + .store(true, std::sync::atomic::Ordering::Relaxed); + put(&f, vec![record(b'a', 1)]).unwrap(); + throttled(put(&f, vec![record(b'x', 1)])); +} + +#[tokio::test] +async fn single_and_batch_share_all_dimensions_with_atomic_rejection() { + for dimension in ["requests", "records", "bytes"] { + let f = Fixture::new(60); + enforce( + &f, + Some(3.0), + Some(if dimension == "bytes" { 5.0 } else { 4.0 }), + Some(12.0 / 1_048_576.0), + ); + put(&f, vec![record(b'a', 4)]).unwrap(); + put(&f, vec![record(b'b', 2), record(b'c', 2)]).unwrap(); + let before = buffered(&f); + let rejected = match dimension { + "requests" => vec![], // Consume the third request, then exceed it below. + "records" => vec![record(b'x', 1), record(b'y', 1)], + _ => vec![record(b'x', 3), record(b'y', 2)], + }; + if dimension == "requests" { + put(&f, rejected).unwrap(); + throttled(put(&f, vec![record(b'x', 1)])); + assert_eq!(buffered(&f), before); + } else { + throttled(put(&f, rejected)); + assert_eq!(buffered(&f), before); + // A rejected call consumed neither request, record nor byte capacity. + put(&f, vec![record(b'd', 4)]).unwrap(); + } + f.advance(60); + f.turn().await; + let records = f.batches()[0].records.clone(); + let mut expected = vec![record(b'a', 4), record(b'b', 2), record(b'c', 2)]; + if dimension != "requests" { + expected.push(record(b'd', 4)); + } + assert_eq!(records, expected); + } +} + +#[test] +fn budgets_are_stream_account_region_isolated_and_concurrent() { + let f = Fixture::new(60); + f.add_stream(ACCOUNT, REGION, "other", 60); + f.add_stream("222222222222", REGION, NAME, 60); + f.add_stream(ACCOUNT, "us-west-2", NAME, 60); + enforce(&f, Some(7.0), Some(7.0), None); + let barrier = std::sync::Barrier::new(40); + std::thread::scope(|scope| { + let handles: Vec<_> = (0..40) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + put(&f, vec![record(b'a', 1)]).is_ok() + }) + }) + .collect(); + assert_eq!( + handles + .into_iter() + .map(|h| h.join().unwrap()) + .filter(|ok| *ok) + .count(), + 7 + ); + }); + let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); + for (account, region, name) in [ + (ACCOUNT, REGION, "other"), + ("222222222222", REGION, NAME), + (ACCOUNT, "us-west-2", NAME), + ] { + svc.deliver_records(account, region, name, vec![vec![1]; 7]) + .unwrap(); + throttled(svc.deliver_records(account, region, name, vec![vec![1]])); + } +} + +#[test] +fn empty_records_and_dynamic_limits_preserve_all_usage() { + let f = Fixture::new(60); + let provider = enforce(&f, Some(10.0), None, Some(0.0)); + put(&f, vec![Bytes::new(), Bytes::new()]).unwrap(); + provider.limits.write().records = Some(2.0); + throttled(put(&f, vec![Bytes::new()])); + provider.limits.write().records = Some(3.0); + put(&f, vec![Bytes::new()]).unwrap(); + throttled(put(&f, vec![Bytes::new()])); + f.advance(1); + put(&f, vec![Bytes::new(); 3]).unwrap(); + throttled(put(&f, vec![record(b'a', 1)])); +} + +#[test] +fn validation_precedes_debit_and_config_only_and_missing_backend_are_metered() { + let f = Fixture::new(60); + enforce(&f, Some(1.0), None, None); + let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); + assert_eq!( + svc.deliver_records(ACCOUNT, REGION, "missing", vec![vec![1]]) + .unwrap_err() + .code(), + "ResourceNotFoundException" + ); + f.update(NAME, |s| s.stream_type = "KinesisStreamAsSource".into()); + assert_eq!( + put(&f, vec![record(b'a', 1)]).unwrap_err().code(), + "InvalidArgumentException" + ); + f.update(NAME, |s| { + s.stream_type = "DirectPut".into(); + s.destination.as_mut().unwrap().compression_format = Some("unsupported".into()); + }); + assert_eq!( + put(&f, vec![record(b'a', 1)]).unwrap_err().code(), + "NotImplemented" + ); + f.update(NAME, |s| s.destination = None); + put(&f, vec![record(b'a', 1)]).unwrap(); + throttled(put(&f, vec![record(b'a', 1)])); + f.advance(1); + f.update(NAME, |s| s.destination = Some(destination(60))); + *f.engine.core.writer.write() = None; + put(&f, vec![record(b'a', 1)]).unwrap(); + throttled(put(&f, vec![record(b'a', 1)])); +} + +#[tokio::test] +async fn zero_public_capacity_does_not_drop_internal_enqueue_and_disabled_bypasses() { + let f = Fixture::new(60); + let provider = enforce(&f, Some(0.0), Some(0.0), Some(0.0)); + throttled(put(&f, vec![record(b'x', 1)])); + f.enqueue(vec![record(b'i', 1)]); + *provider.limits.write() = ThroughputLimits::default(); + for _ in 0..20 { + put(&f, vec![record(b'p', 1)]).unwrap(); + } + provider.limits.write().requests = Some(1.0); + put(&f, vec![record(b'z', 1)]).unwrap(); + throttled(put(&f, vec![record(b'x', 1)])); + assert_eq!(f.engine.shutdown().await.undelivered_records, 0); + let records = &f.batches()[0].records; + assert_eq!(records.len(), 22); + assert_eq!(records[0], record(b'i', 1)); + assert_eq!(records[21], record(b'z', 1)); +} + +#[tokio::test] +async fn retries_do_not_redebit_ingestion() { + let f = Fixture::new(0); + enforce(&f, Some(1.0), Some(1.0), Some(1.0 / 1_048_576.0)); + let mut entered = f.controlled(); + put(&f, vec![record(b'a', 1)]).unwrap(); + let mut jobs = Writes::default(); + f.schedule(&mut jobs, false); + let first = next_attempt(&mut entered).await; + first.reply.send(Action::Fail).unwrap(); + f.finish(&mut jobs).await; + throttled(put(&f, vec![record(b'x', 1)])); + f.advance(1); + f.schedule(&mut jobs, true); + let retry = next_attempt(&mut entered).await; + assert_eq!(first.batch.object_key, retry.batch.object_key); + retry.reply.send(Action::Commit).unwrap(); + f.finish(&mut jobs).await; + put(&f, vec![record(b'b', 1)]).unwrap(); + throttled(put(&f, vec![record(b'x', 1)])); +} + +#[tokio::test] +async fn delete_reset_restore_start_new_generation_budgets() { + for operation in ["delete", "reset", "restore"] { + let f = Fixture::new(60); + enforce(&f, Some(1.0), None, None); + put(&f, vec![record(b'a', 1)]).unwrap(); + throttled(put(&f, vec![record(b'x', 1)])); + match operation { + "delete" => { + f.engine + .core + .state + .write() + .get_or_create(ACCOUNT, REGION) + .streams_mut(REGION) + .remove(NAME); + } + "reset" => f.engine.core.state.write().accounts.clear(), + _ => { + let serialized = serde_json::to_vec(&*f.engine.core.state.read()).unwrap(); + *f.engine.core.state.write() = serde_json::from_slice(&serialized).unwrap(); + } + } + if operation != "restore" { + f.add_stream(ACCOUNT, REGION, NAME, 60); + } + put(&f, vec![record(b'b', 1)]).unwrap(); + throttled(put(&f, vec![record(b'x', 1)])); + f.turn().await; + assert_eq!(f.engine.shutdown().await.undelivered_records, 0); + assert_eq!(f.batches()[0].records, vec![record(b'b', 1)]); + } +} + +fn request(action: &str, body: serde_json::Value) -> fakecloud_core::service::AwsRequest { + fakecloud_core::service::AwsRequest { + service: "firehose".into(), + action: action.into(), + region: REGION.into(), + account_id: ACCOUNT.into(), + request_id: "throughput-test".into(), + headers: http::HeaderMap::new(), + query_params: HashMap::new(), + body: Bytes::from(serde_json::to_vec(&body).unwrap()), + body_stream: Mutex::new(None), + path_segments: Vec::new(), + raw_path: "/".into(), + raw_query: String::new(), + method: http::Method::POST, + is_query_protocol: false, + access_key_id: None, + principal: None, + } +} + +#[tokio::test] +async fn public_decoding_charges_only_valid_records_and_decoded_bytes() { + use fakecloud_core::service::AwsService; + let f = Fixture::new(60); + enforce(&f, Some(2.0), Some(2.0), Some(4.0 / 1_048_576.0)); + let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); + svc.handle(request( + "PutRecord", + json!({"DeliveryStreamName": NAME, "Record": {"Data": "YQ=="}}), + )) + .await + .unwrap(); + let response = svc + .handle(request( + "PutRecordBatch", + json!({"DeliveryStreamName": NAME, + "Records": [{"Data": "%%%"}, {"Data": "YmNk"}]}), + )) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(response.body.expect_bytes()).unwrap(); + assert_eq!(body["FailedPutCount"], 1); + let error = svc + .handle(request( + "PutRecord", + json!({"DeliveryStreamName": NAME, "Record": {"Data": ""}}), + )) + .await + .err() + .expect("must throttle"); + assert_eq!(error.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(error.code(), "ServiceUnavailableException"); + f.advance(60); + f.turn().await; + assert_eq!( + f.batches()[0].records, + vec![Bytes::from_static(b"a"), Bytes::from_static(b"bcd")] + ); +} + +#[tokio::test] +async fn public_request_validation_never_debits_or_partially_admits() { + use base64::Engine; + use fakecloud_core::service::AwsService; + let f = Fixture::new(60); + enforce(&f, Some(1.0), Some(1.0), None); + let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); + let oversized = base64::engine::general_purpose::STANDARD.encode(vec![0; 1_024_001]); + for (action, body) in [ + ( + "PutRecord", + json!({"DeliveryStreamName": NAME, "Record": {"Data": "%%%"}}), + ), + ( + "PutRecord", + json!({"DeliveryStreamName": NAME, "Record": {"Data": oversized}}), + ), + ( + "PutRecordBatch", + json!({"DeliveryStreamName": NAME, "Records": []}), + ), + ( + "PutRecordBatch", + json!({"DeliveryStreamName": NAME, "Records": [{"Data": "YQ=="}, {"Data": oversized}]}), + ), + ] { + assert_eq!( + svc.handle(request(action, body)) + .await + .err() + .expect("must validate") + .code(), + "InvalidArgumentException" + ); + } + svc.handle(request( + "PutRecord", + json!({"DeliveryStreamName": NAME, "Record": {"Data": "Yg=="}}), + )) + .await + .unwrap(); + let error = svc + .handle(request( + "PutRecordBatch", + json!({"DeliveryStreamName": NAME, "Records": [{"Data": "Yw=="}, {"Data": "ZA=="}]}), + )) + .await + .err() + .expect("must throttle"); + assert_eq!(error.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(error.code(), "ServiceUnavailableException"); + f.advance(60); + f.turn().await; + assert_eq!(f.batches()[0].records, vec![Bytes::from_static(b"b")]); +} + +#[tokio::test] +async fn all_invalid_public_batch_consumes_one_request_but_no_records() { + use fakecloud_core::service::AwsService; + let f = Fixture::new(60); + enforce(&f, Some(1.0), Some(0.0), Some(0.0)); + let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); + svc.handle(request( + "PutRecordBatch", + json!({"DeliveryStreamName": NAME, "Records": [{"Data": "%%%"}]}), + )) + .await + .unwrap(); + throttled(put(&f, vec![])); +} + +#[tokio::test] +async fn stale_windows_are_pruned_without_buffered_records() { + for final_scan in [false, true] { + let f = Fixture::new(60); + f.update(NAME, |s| s.destination = None); + enforce(&f, Some(1.0), None, None); + put(&f, vec![record(b'a', 1)]).unwrap(); + let saved = f.engine.core.state.read().clone(); + f.engine.core.state.write().accounts.clear(); + if final_scan { + assert_eq!( + f.engine.core.finish_shutdown(&BTreeSet::new()), + Some(DrainReport::default()) + ); + } else { + f.turn().await; + } + // Reinstall the same identity to detect leaked windows independently + // of the fresh UUID behavior tested by delete/reset/restore above. + *f.engine.core.state.write() = saved; + put(&f, vec![record(b'b', 1)]).unwrap(); + throttled(put(&f, vec![record(b'x', 1)])); + } +} + +#[tokio::test] +async fn successful_delivery_and_destination_updates_do_not_reset_budget() { + let f = Fixture::new(0); + enforce(&f, Some(1.0), None, None); + put(&f, vec![record(b'a', 1)]).unwrap(); + f.turn().await; + assert!(f.engine.core.registry.lock().streams.is_empty()); + f.update(NAME, |s| { + s.version_id = "2".into(); + s.destination.as_mut().unwrap().prefix = Some("updated/".into()); + }); + throttled(put(&f, vec![record(b'x', 1)])); + f.advance(1); + put(&f, vec![record(b'b', 1)]).unwrap(); + f.turn().await; + let batches = f.batches(); + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].records, vec![record(b'a', 1)]); + assert_eq!(batches[1].records, vec![record(b'b', 1)]); + assert!(batches[1].object_key.starts_with("updated/")); +} diff --git a/crates/fakecloud-firehose/src/lib.rs b/crates/fakecloud-firehose/src/lib.rs index b484990ed..d106c7594 100644 --- a/crates/fakecloud-firehose/src/lib.rs +++ b/crates/fakecloud-firehose/src/lib.rs @@ -9,6 +9,7 @@ pub(crate) mod state; #[cfg(test)] #[path = "../tests/fixtures/xerial_decoder.rs"] mod test_xerial_decoder; +mod throughput; pub mod writer; pub use config::{buffering_policy, delivery_config, BufferingPolicy, DeliveryConfig}; diff --git a/crates/fakecloud-firehose/src/service.rs b/crates/fakecloud-firehose/src/service.rs index 245b3f341..683f5e676 100644 --- a/crates/fakecloud-firehose/src/service.rs +++ b/crates/fakecloud-firehose/src/service.rs @@ -1145,7 +1145,7 @@ impl FirehoseService { stream_name: &str, datas: Vec>, ) -> Result<(), AwsServiceError> { - self.engine.enqueue( + self.engine.enqueue_put( account_id, region, stream_name, diff --git a/crates/fakecloud-firehose/src/throughput.rs b/crates/fakecloud-firehose/src/throughput.rs new file mode 100644 index 000000000..00a75d081 --- /dev/null +++ b/crates/fakecloud-firehose/src/throughput.rs @@ -0,0 +1,514 @@ +use std::time::Duration; +use tokio::time::Instant; + +#[derive(Clone, Copy, Default)] +pub(crate) struct ThroughputLimits { + pub requests: Option, + pub records: Option, + pub mib: Option, +} + +/// Stream-local, one-second ingestion accounting; callers supply monotonic time. +/// +/// The first successful admission, and a successful admission at or after the +/// boundary, starts a fresh window at `now`. +/// Windows do not accrue idle credits, but full capacity immediately before and +/// after a boundary can produce a short burst of twice the per-window allowance. +pub(crate) struct ThroughputWindow { + started_at: Instant, + requests: u64, + records: usize, + bytes: usize, +} + +impl ThroughputWindow { + pub(crate) fn new(now: Instant) -> Self { + Self { + started_at: now, + requests: 0, + records: 0, + bytes: 0, + } + } + + pub(crate) fn try_admit( + &mut self, + now: Instant, + limits: ThroughputLimits, + records: usize, + bytes: usize, + ) -> bool { + if limits.requests.is_none() && limits.records.is_none() && limits.mib.is_none() { + return true; + } + + let rollover = now.duration_since(self.started_at) >= Duration::from_secs(1); + let (used_requests, used_records, used_bytes) = if rollover { + (0_u64, 0_usize, 0_usize) + } else { + (self.requests, self.records, self.bytes) + }; + // Track every dimension when any limit is enabled, including dimensions + // that might become enabled by a later same-window quota change. + let Some(requests) = used_requests.checked_add(1) else { + return false; + }; + let Some(records) = used_records.checked_add(records) else { + return false; + }; + let Some(bytes) = used_bytes.checked_add(bytes) else { + return false; + }; + + // Limits are finite and non-negative (validated by quota setters). Scale + // usage, not limits: huge finite MiB limits must not overflow a capacity. + if limits + .requests + .is_some_and(|limit| count_as_f64_ceil(requests as u128) > limit) + || limits + .records + .is_some_and(|limit| count_as_f64_ceil(records as u128) > limit) + || limits + .mib + .is_some_and(|limit| count_as_f64_ceil(bytes as u128) / 1_048_576.0 > limit) + { + return false; + } + + // Commit only after every check passes, including rollover. Rejections + // must not reset either the old counters or the old window's anchor. + if rollover || self.requests == 0 { + self.started_at = now; + } + self.requests = requests; + self.records = records; + self.bytes = bytes; + true + } +} + +/// Smallest representable f64 not below the count. An ordinary cast can round +/// down above 2^53 and incorrectly admit one unit over a representable limit. +/// Dividing this by the power-of-two bytes-per-MiB scale preserves that bound. +fn count_as_f64_ceil(count: u128) -> f64 { + let rounded = count as f64; + if (rounded as u128) < count { + f64::from_bits(rounded.to_bits() + 1) + } else { + rounded + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn usage(window: &ThroughputWindow) -> (Instant, u64, usize, usize) { + ( + window.started_at, + window.requests, + window.records, + window.bytes, + ) + } + + #[test] + fn request_capacity_recovers_at_one_second() { + let now = Instant::now(); + let limits = ThroughputLimits { + requests: Some(1.0), + ..Default::default() + }; + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, 1, 3)); + assert!(!window.try_admit(now, limits, 1, 3)); + assert!(!window.try_admit(now + Duration::from_millis(999), limits, 1, 3)); + assert!(window.try_admit(now + Duration::from_secs(1), limits, 1, 3)); + } + + #[test] + fn first_success_anchors_window_after_initial_rejection() { + let now = Instant::now(); + let limits = ThroughputLimits { + records: Some(1.0), + ..Default::default() + }; + let mut window = ThroughputWindow::new(now); + assert!(!window.try_admit(now, limits, 2, 1)); + assert!(window.try_admit(now + Duration::from_millis(500), limits, 1, 1)); + assert!(!window.try_admit(now + Duration::from_secs(1), limits, 1, 1)); + assert!(window.try_admit(now + Duration::from_millis(1_500), limits, 1, 1)); + } + + #[test] + fn records_accept_exact_limit_and_reject_one_over() { + let now = Instant::now(); + let limits = ThroughputLimits { + records: Some(3.0), + ..Default::default() + }; + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, 2, 10)); + assert!(window.try_admit(now, limits, 1, 10)); + assert!(!window.try_admit(now, limits, 1, 0)); + assert!(window.try_admit(now + Duration::from_secs(1), limits, 3, 10)); + } + + #[test] + fn fractional_mib_accepts_exact_bytes_and_rejects_one_over() { + let now = Instant::now(); + let limits = ThroughputLimits { + mib: Some(0.5), + ..Default::default() + }; + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, 1, 262_144)); + assert!(window.try_admit(now, limits, 1, 262_144)); + assert!(!window.try_admit(now, limits, 1, 1)); + assert!(window.try_admit(now + Duration::from_secs(1), limits, 1, 524_288)); + } + + #[test] + fn rejected_bytes_leave_capacity_for_smaller_request() { + let now = Instant::now(); + let limits = ThroughputLimits { + requests: Some(2.0), + records: Some(3.0), + mib: Some(0.5), + }; + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, 1, 262_144)); + assert!(!window.try_admit(now, limits, 2, 262_145)); + assert!(window.try_admit(now, limits, 2, 262_144)); + assert!(!window.try_admit(now, limits, 0, 0)); + } + + #[test] + fn each_enabled_dimension_can_reject_without_consuming_other_capacity() { + let now = Instant::now(); + let generous = ThroughputLimits { + requests: Some(3.0), + records: Some(3.0), + mib: Some(1.0), + }; + for strict in [ + ThroughputLimits { + requests: Some(0.0), + ..generous + }, + ThroughputLimits { + records: Some(0.0), + ..generous + }, + ThroughputLimits { + mib: Some(0.0), + ..generous + }, + ] { + let mut window = ThroughputWindow::new(now); + assert!(!window.try_admit(now, strict, 1, 1)); + assert!(window.try_admit(now, generous, 3, 1_048_576)); + } + } + + #[test] + fn empty_payload_still_consumes_a_request() { + let now = Instant::now(); + let limits = ThroughputLimits { + requests: Some(1.0), + records: Some(0.0), + mib: Some(0.0), + }; + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, 0, 0)); + assert!(!window.try_admit(now, limits, 0, 0)); + } + + #[test] + fn zero_record_and_byte_limits_allow_only_zero_usage() { + let now = Instant::now(); + let limits = ThroughputLimits { + records: Some(0.0), + mib: Some(0.0), + ..Default::default() + }; + let mut window = ThroughputWindow::new(now); + assert!(!window.try_admit(now, limits, 1, 0)); + assert!(!window.try_admit(now, limits, 0, 1)); + assert!(window.try_admit(now, limits, 0, 0)); + } + + #[test] + fn zero_request_limit_rejects_even_empty_payloads() { + let now = Instant::now(); + let limits = ThroughputLimits { + requests: Some(0.0), + ..Default::default() + }; + let mut window = ThroughputWindow::new(now); + assert!(!window.try_admit(now, limits, 0, 0)); + } + + #[test] + fn disabled_limits_bypass_without_changing_usage_or_window() { + let now = Instant::now(); + let limits = ThroughputLimits { + requests: Some(1.0), + ..Default::default() + }; + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, 1, 1)); + let before = usage(&window); + assert!(window.try_admit(now, ThroughputLimits::default(), usize::MAX, usize::MAX)); + assert!(window.try_admit( + now + Duration::from_secs(2), + ThroughputLimits::default(), + usize::MAX, + usize::MAX + )); + assert_eq!(usage(&window), before); + assert!(!window.try_admit(now, limits, 0, 0)); + } + + #[test] + fn ignored_dimensions_are_counted_when_another_dimension_is_enabled() { + let now = Instant::now(); + for initially_enabled in [ + ThroughputLimits { + requests: Some(10.0), + ..Default::default() + }, + ThroughputLimits { + records: Some(10.0), + ..Default::default() + }, + ThroughputLimits { + mib: Some(10.0), + ..Default::default() + }, + ] { + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, initially_enabled, 3, 524_288)); + assert!(!window.try_admit( + now, + ThroughputLimits { + requests: Some(1.0), + ..Default::default() + }, + 0, + 0 + )); + assert!(!window.try_admit( + now, + ThroughputLimits { + records: Some(3.0), + ..Default::default() + }, + 1, + 0 + )); + assert!(!window.try_admit( + now, + ThroughputLimits { + mib: Some(0.5), + ..Default::default() + }, + 0, + 1 + )); + } + } + + #[test] + fn runtime_limit_increases_and_decreases_preserve_same_window_usage() { + let now = Instant::now(); + let mut window = ThroughputWindow::new(now); + let limits = ThroughputLimits { + requests: Some(1.0), + records: Some(2.0), + mib: Some(0.5), + }; + assert!(window.try_admit(now, limits, 2, 524_288)); + assert!(!window.try_admit(now, limits, 0, 0)); + let raised = ThroughputLimits { + requests: Some(3.0), + records: Some(4.0), + mib: Some(1.0), + }; + assert!(window.try_admit(now, raised, 1, 262_144)); + assert!(!window.try_admit(now, limits, 0, 0)); + assert!(window.try_admit(now, raised, 1, 262_144)); + assert!(!window.try_admit(now, raised, 0, 0)); + } + + #[test] + fn rejected_rollover_preserves_usage_and_success_anchors_at_now() { + let now = Instant::now(); + let limits = ThroughputLimits { + records: Some(1.0), + ..Default::default() + }; + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, 1, 1)); + let before = usage(&window); + assert!(!window.try_admit(now + Duration::from_secs(1), limits, 2, 1)); + assert_eq!(usage(&window), before); + assert!(window.try_admit(now + Duration::from_millis(1_100), limits, 1, 1)); + assert!(!window.try_admit(now + Duration::from_secs(2), limits, 1, 1)); + assert!(window.try_admit(now + Duration::from_millis(2_100), limits, 1, 1)); + } + + #[test] + fn idle_time_does_not_accumulate_credits() { + let now = Instant::now(); + let limits = ThroughputLimits { + requests: Some(1.0), + ..Default::default() + }; + let mut window = ThroughputWindow::new(now); + let later = now + Duration::from_secs(100); + assert!(window.try_admit(later, limits, 1, 1)); + assert!(!window.try_admit(later, limits, 1, 1)); + assert!(!window.try_admit(later + Duration::from_millis(999), limits, 1, 1)); + assert!(window.try_admit(later + Duration::from_secs(1), limits, 1, 1)); + } + + #[test] + fn windows_have_independent_usage() { + let now = Instant::now(); + let limits = ThroughputLimits { + requests: Some(1.0), + ..Default::default() + }; + let mut first = ThroughputWindow::new(now); + let mut second = ThroughputWindow::new(now); + assert!(first.try_admit(now, limits, 1, 1)); + assert!(!first.try_admit(now, limits, 1, 1)); + assert!(second.try_admit(now, limits, 1, 1)); + } + + #[test] + fn fractional_count_limits_do_not_round_up() { + let now = Instant::now(); + for limits in [ + ThroughputLimits { + requests: Some(0.5), + ..Default::default() + }, + ThroughputLimits { + records: Some(0.5), + ..Default::default() + }, + ] { + let mut window = ThroughputWindow::new(now); + assert!(!window.try_admit(now, limits, 1, 0)); + } + } + + #[test] + fn sub_byte_mib_limit_does_not_allow_a_whole_byte() { + let now = Instant::now(); + let mut window = ThroughputWindow::new(now); + let limits = ThroughputLimits { + mib: Some(f64::MIN_POSITIVE), + ..Default::default() + }; + assert!(!window.try_admit(now, limits, 0, 1)); + assert!(window.try_admit(now, limits, 0, 0)); + } + + #[test] + fn huge_finite_limits_do_not_overflow_capacity_calculations() { + let now = Instant::now(); + let limits = ThroughputLimits { + requests: Some(f64::MAX), + records: Some(f64::MAX), + mib: Some(f64::MAX), + }; + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, usize::MAX, usize::MAX)); + assert!(!window.try_admit(now, limits, 1, 0)); + } + + #[test] + fn count_and_byte_overflow_reject_atomically_even_when_dimension_is_ignored() { + let now = Instant::now(); + let limits = ThroughputLimits { + requests: Some(10.0), + ..Default::default() + }; + for (records, bytes, extra_records, extra_bytes) in + [(usize::MAX, 0, 1, 0), (0, usize::MAX, 0, 1)] + { + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, records, bytes)); + let before = usage(&window); + assert!(!window.try_admit(now, limits, extra_records, extra_bytes)); + assert_eq!(usage(&window), before); + assert!(window.try_admit(now, limits, 0, 0)); + assert!(window.try_admit( + now + Duration::from_secs(1), + limits, + extra_records, + extra_bytes + )); + } + } + + #[test] + fn request_counter_overflow_rejects_without_mutation() { + let now = Instant::now(); + let mut window = ThroughputWindow::new(now); + window.requests = u64::MAX; + let limits = ThroughputLimits { + records: Some(1.0), + ..Default::default() + }; + let before = usage(&window); + assert!(!window.try_admit(now, limits, 1, 1)); + assert_eq!(usage(&window), before); + assert!(window.try_admit(now + Duration::from_secs(1), limits, 1, 1)); + } + + #[test] + fn large_request_count_is_not_rounded_down_to_the_limit() { + let now = Instant::now(); + let mut window = ThroughputWindow::new(now); + window.requests = 9_007_199_254_740_992; + let limits = ThroughputLimits { + requests: Some(9_007_199_254_740_992.0), + ..Default::default() + }; + assert!(!window.try_admit(now, limits, 0, 0)); + assert!(window.try_admit( + now, + ThroughputLimits { + requests: Some(9_007_199_254_740_994.0), + ..Default::default() + }, + 0, + 0 + )); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn large_record_and_byte_counts_are_not_rounded_down_to_limits() { + let now = Instant::now(); + for limits in [ + ThroughputLimits { + records: Some(9_007_199_254_740_992.0), + ..Default::default() + }, + ThroughputLimits { + mib: Some(8_589_934_592.0), + ..Default::default() + }, + ] { + let mut window = ThroughputWindow::new(now); + assert!(window.try_admit(now, limits, 9_007_199_254_740_992, 9_007_199_254_740_992)); + assert!(!window.try_admit(now, limits, 1, 1)); + } + } +} diff --git a/crates/fakecloud-server/src/main.rs b/crates/fakecloud-server/src/main.rs index 0282c29f2..f7bb47bc9 100644 --- a/crates/fakecloud-server/src/main.rs +++ b/crates/fakecloud-server/src/main.rs @@ -1375,6 +1375,7 @@ async fn main() { firehose_state.clone(), )); firehose_engine.set_s3(s3_state.clone()); + firehose_engine.set_quota_provider(quota_provider.clone()); let firehose_delivery_for_logs = Arc::new( fakecloud_firehose::FirehoseDeliveryImpl::with_engine(firehose_engine.clone()), ); diff --git a/crates/fakecloud-servicequotas/src/catalog.rs b/crates/fakecloud-servicequotas/src/catalog.rs index 04a9dfabe..21ec793cb 100644 --- a/crates/fakecloud-servicequotas/src/catalog.rs +++ b/crates/fakecloud-servicequotas/src/catalog.rs @@ -79,7 +79,9 @@ pub struct QuotaDef { } pub use fakecloud_core::quota::{ - RULES_PER_SECURITY_GROUP, SECURITY_GROUPS_PER_INTERFACE, VPC_SERVICE_CODE as VPC, + FIREHOSE_MIB_PER_SECOND, FIREHOSE_PUT_REQUESTS_PER_SECOND, FIREHOSE_RECORDS_PER_SECOND, + FIREHOSE_SERVICE_CODE, RULES_PER_SECURITY_GROUP, SECURITY_GROUPS_PER_INTERFACE, + VPC_SERVICE_CODE as VPC, }; pub const EC2: &str = "ec2"; @@ -102,6 +104,9 @@ pub const ENFORCEABLE: &[(&str, &str)] = &[ ("ec2", "L-7295265B"), // Running On-Demand X instances ("ec2", "L-74FC7D96"), // Running On-Demand F instances ("ec2", "L-DB2E81BA"), // Running On-Demand G and VT instances + (FIREHOSE_SERVICE_CODE, FIREHOSE_RECORDS_PER_SECOND), // Records per second per stream + (FIREHOSE_SERVICE_CODE, FIREHOSE_PUT_REQUESTS_PER_SECOND), // Put requests per second per stream + (FIREHOSE_SERVICE_CODE, FIREHOSE_MIB_PER_SECOND), // MiB per second per stream ("iam", "L-0DA4ABF3"), // Managed policies per role ("iam", "L-384571C4"), // Managed policies per group ("iam", "L-4019AD8B"), // Managed policies per user @@ -397,6 +402,29 @@ pub fn quota(service_code: &str, quota_code: &str) -> Option<&'static QuotaDef> c.index(service_code, quota_code).map(|i| &c.quotas[i]) } +/// AWS default in the requested region. Only Firehose's three per-stream +/// throughput quotas differ from the vendored us-east-1 catalog. +pub fn default_value(region: &str, def: &QuotaDef) -> f64 { + if def.service_code == FIREHOSE_SERVICE_CODE { + let high_throughput = matches!(region, "us-east-1" | "us-west-2" | "eu-west-1"); + match def.quota_code { + FIREHOSE_PUT_REQUESTS_PER_SECOND => { + return if high_throughput { 2000.0 } else { 1000.0 }; + } + FIREHOSE_RECORDS_PER_SECOND => { + return if high_throughput { + 500_000.0 + } else { + 100_000.0 + }; + } + FIREHOSE_MIB_PER_SECOND => return if high_throughput { 5.0 } else { 1.0 }, + _ => {} + } + } + def.default +} + /// Every quota of `service_code`, ordered by quota code. Empty for an /// unknown service. pub fn quotas_of(service_code: &str) -> &'static [QuotaDef] { @@ -416,6 +444,48 @@ mod tests { use super::*; + #[test] + fn firehose_defaults_follow_the_requested_region() { + for (code, high, low) in [ + ("L-7681C790", 2000.0, 1000.0), + ("L-17D4EADE", 500_000.0, 100_000.0), + ("L-937C879B", 5.0, 1.0), + ] { + let def = quota("firehose", code).unwrap(); + for (region, expected) in [ + ("us-east-1", high), + ("us-west-2", high), + ("eu-west-1", high), + ("us-east-2", low), + ("ap-southeast-2", low), + ] { + assert_eq!(default_value(region, def), expected, "{region}/{code}"); + } + assert!(!def.adjustable); + assert!(!def.global); + assert!(def.enforceable); + assert!(is_enforceable("firehose", code)); + } + // Do not regionalize other Firehose quotas or unrelated services. + for (service, code, expected) in [ + ("firehose", "L-14BB0BE7", 5000.0), + ("firehose", "L-040D05FF", 5.0), + ("vpc", "L-F678F1CE", 5.0), + ] { + for region in ["us-east-1", "us-west-2", "eu-west-1", "us-east-2"] { + assert_eq!( + default_value(region, quota(service, code).unwrap()), + expected + ); + } + } + // A matching quota code in another service must keep its own default. + let mut other = *quota("firehose", "L-937C879B").unwrap(); + other.service_code = "other"; + other.default = 42.0; + assert_eq!(default_value("us-east-2", &other), 42.0); + } + #[test] fn the_vendored_dump_decodes_into_every_listed_service() { let services = services(); diff --git a/crates/fakecloud-servicequotas/src/provider.rs b/crates/fakecloud-servicequotas/src/provider.rs index 9d976434c..4c75a0918 100644 --- a/crates/fakecloud-servicequotas/src/provider.rs +++ b/crates/fakecloud-servicequotas/src/provider.rs @@ -29,7 +29,7 @@ pub fn applied_value(data: Option<&ServiceQuotasData>, region: &str, def: &Quota )) .copied() }) - .unwrap_or(def.default) + .unwrap_or_else(|| catalog::default_value(region, def)) } /// The region a quota's ARN and applied value live in: global quotas have @@ -324,6 +324,44 @@ mod tests { ) } + #[test] + fn firehose_provider_defaults_are_regional_and_enforcement_is_opt_in() { + let p = provider(Default::default()); + let before = catalog::catalog_reads(); + for code in ["L-7681C790", "L-17D4EADE", "L-937C879B"] { + assert_eq!( + p.enforced_limit("000000000000", "us-east-2", "firehose", code), + None + ); + } + assert_eq!(catalog::catalog_reads(), before); + for (code, high, low) in [ + ("L-7681C790", 2000.0, 1000.0), + ("L-17D4EADE", 500_000.0, 100_000.0), + ("L-937C879B", 5.0, 1.0), + ] { + for (region, expected) in [ + ("us-east-1", high), + ("us-west-2", high), + ("eu-west-1", high), + ("us-east-2", low), + ] { + for account in ["000000000000", "222222222222"] { + assert_eq!( + p.applied_value(account, region, "firehose", code), + Some(expected) + ); + p.settings.write().enforce_all = true; + assert_eq!( + p.enforced_limit(account, region, "firehose", code), + Some(expected) + ); + p.settings.write().enforce_all = false; + } + } + } + } + /// With enforcement off, an enforcing service's limit lookup on its /// request path never touches (or decodes) the catalog. #[test] diff --git a/crates/fakecloud-servicequotas/src/service.rs b/crates/fakecloud-servicequotas/src/service.rs index c4eacbab7..ee1795e84 100644 --- a/crates/fakecloud-servicequotas/src/service.rs +++ b/crates/fakecloud-servicequotas/src/service.rs @@ -520,7 +520,9 @@ impl ServiceQuotasService { b: &Value, ) -> Result { let def = quota_from(b)?; - ok(json!({ "Quota": quota_json(&req.region, "", def, def.default) })) + ok( + json!({ "Quota": quota_json(&req.region, "", def, catalog::default_value(&req.region, def)) }), + ) } fn list_aws_default_service_quotas( @@ -532,7 +534,7 @@ impl ServiceQuotasService { let (items, token) = page(catalog::quotas_of(svc), b, 100, 100)?; let quotas: Vec = items .iter() - .map(|d| quota_json(&req.region, "", d, d.default)) + .map(|d| quota_json(&req.region, "", d, catalog::default_value(&req.region, d))) .collect(); let mut out = Map::new(); out.insert("Quotas".into(), Value::Array(quotas)); @@ -1280,7 +1282,7 @@ impl ServiceQuotasService { .map(|m| m.namespace) .unwrap_or("AWS/Usage"), "Utilization": utilization, - "DefaultValue": def.map(|d| d.default).unwrap_or(0.0), + "DefaultValue": def.map(|d| catalog::default_value(&req.region, d)).unwrap_or(0.0), "AppliedValue": e.applied_value, "ServiceName": catalog::service_name(&e.service_code), "Adjustable": def.is_some_and(|d| d.adjustable), diff --git a/crates/fakecloud-servicequotas/src/service/introspection.rs b/crates/fakecloud-servicequotas/src/service/introspection.rs index aca1db38e..6ce7659be 100644 --- a/crates/fakecloud-servicequotas/src/service/introspection.rs +++ b/crates/fakecloud-servicequotas/src/service/introspection.rs @@ -220,7 +220,7 @@ impl ServiceQuotasService { "global": def.global, "adjustable": def.adjustable, "unit": def.unit, - "defaultValue": def.default, + "defaultValue": catalog::default_value(region, def), "appliedValue": applied_value(data, region, def), "usage": usage, "enforceable": def.enforceable, diff --git a/crates/fakecloud-servicequotas/src/service/tests.rs b/crates/fakecloud-servicequotas/src/service/tests.rs index 02507719a..942a226e8 100644 --- a/crates/fakecloud-servicequotas/src/service/tests.rs +++ b/crates/fakecloud-servicequotas/src/service/tests.rs @@ -877,6 +877,240 @@ fn provider(s: &ServiceQuotasService) -> crate::ServiceQuotasProvider { const ACCT: &str = "000000000000"; +fn firehose_call( + s: &ServiceQuotasService, + account: &str, + region: &str, + action: &str, + body: Value, +) -> Value { + let resp = dispatch(s, &req_in(account, region, action, body)).expect("op ok"); + serde_json::from_slice(resp.body.expect_bytes()).unwrap() +} + +#[test] +fn firehose_quota_gets_lists_and_introspection_use_regional_defaults() { + let s = svc(); + for (region, expected) in [ + ("us-east-1", [2000.0, 500_000.0, 5.0]), + ("us-west-2", [2000.0, 500_000.0, 5.0]), + ("eu-west-1", [2000.0, 500_000.0, 5.0]), + ("us-east-2", [1000.0, 100_000.0, 1.0]), + ] { + for (code, expected) in ["L-7681C790", "L-17D4EADE", "L-937C879B"] + .into_iter() + .zip(expected) + { + let body = json!({"ServiceCode": "firehose", "QuotaCode": code}); + for action in ["GetAWSDefaultServiceQuota", "GetServiceQuota"] { + let q = firehose_call(&s, ACCT, region, action, body.clone()); + assert_eq!(q["Quota"]["Value"], expected, "{action}/{region}/{code}"); + assert_eq!(q["Quota"]["Adjustable"], false); + assert_eq!(q["Quota"]["GlobalQuota"], false); + } + for action in ["ListAWSDefaultServiceQuotas", "ListServiceQuotas"] { + let list = + firehose_call(&s, ACCT, region, action, json!({"ServiceCode": "firehose"})); + let q = list["Quotas"] + .as_array() + .unwrap() + .iter() + .find(|q| q["QuotaCode"] == code) + .unwrap(); + assert_eq!(q["Value"], expected, "{action}/{region}/{code}"); + assert_eq!(q["Adjustable"], false); + } + let list = s + .introspect_quotas(None, Some(region), Some("firehose")) + .unwrap(); + let q = list["quotas"] + .as_array() + .unwrap() + .iter() + .find(|q| q["quotaCode"] == code) + .unwrap(); + assert_eq!(q["defaultValue"], expected, "{region}/{code}"); + assert_eq!(q["appliedValue"], expected); + assert_eq!(q["enforceable"], true); + assert_eq!(q["enforced"], false); + } + // Nearby quotas must not acquire Firehose throughput defaults. + for (service, code, expected) in [ + ("firehose", "L-14BB0BE7", 5000.0), + ("vpc", "L-F678F1CE", 5.0), + ] { + for action in ["GetAWSDefaultServiceQuota", "GetServiceQuota"] { + let q = firehose_call( + &s, + ACCT, + region, + action, + json!({"ServiceCode": service, "QuotaCode": code}), + ); + assert_eq!(q["Quota"]["Value"], expected); + } + } + } +} + +#[test] +fn firehose_local_values_are_unrestricted_scoped_and_deleted_to_regional_defaults() { + use fakecloud_core::quota::QuotaProvider; + let s = svc(); + let p = provider(&s); + for (code, high, low) in [ + ("L-7681C790", 2000.0, 1000.0), + ("L-17D4EADE", 500_000.0, 100_000.0), + ("L-937C879B", 5.0, 1.0), + ] { + for (region, default) in [("us-east-1", high), ("us-east-2", low)] { + for value in [0.25, 0.0, 1e12, f64::MAX] { + let view = s + .introspect_put_quota( + "firehose", + code, + &PutQuotaRequest { + account_id: Some(ACCT.into()), + region: Some(region.into()), + value: Some(value), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(view["appliedValue"], value); + assert_eq!(view["defaultValue"], default); + assert_eq!(view["enforced"], false); + assert_eq!(p.applied_value(ACCT, region, "firehose", code), Some(value)); + assert_eq!(p.enforced_limit(ACCT, region, "firehose", code), None); + for action in ["GetServiceQuota", "ListServiceQuotas"] { + let q = firehose_call( + &s, + ACCT, + region, + action, + json!({"ServiceCode": "firehose", "QuotaCode": code}), + ); + let q = if action == "GetServiceQuota" { + &q["Quota"] + } else { + &q["Quotas"][0] + }; + assert_eq!(q["Value"].as_f64(), Some(value)); + } + let aws_default = firehose_call( + &s, + ACCT, + region, + "GetAWSDefaultServiceQuota", + json!({"ServiceCode": "firehose", "QuotaCode": code}), + ); + assert_eq!(aws_default["Quota"]["Value"], default); + assert_eq!( + p.applied_value(MEMBER, region, "firehose", code), + Some(default) + ); + let other_region = if region == "us-east-1" { + "us-east-2" + } else { + "us-east-1" + }; + let other_default = if region == "us-east-1" { low } else { high }; + assert_eq!( + p.applied_value(ACCT, other_region, "firehose", code), + Some(other_default) + ); + } + s.introspect_put_quota( + "firehose", + code, + &PutQuotaRequest { + account_id: Some(ACCT.into()), + region: Some(region.into()), + enforce: Some(Some(true)), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + p.enforced_limit(ACCT, region, "firehose", code), + Some(f64::MAX) + ); + assert_eq!(p.enforced_limit(MEMBER, region, "firehose", code), None); + let deleted = s + .introspect_delete_quota("firehose", code, Some(ACCT), Some(region)) + .unwrap(); + assert_eq!(deleted["defaultValue"], default); + assert_eq!(deleted["appliedValue"], default); + assert_eq!(deleted["enforced"], false); + assert_eq!( + p.applied_value(ACCT, region, "firehose", code), + Some(default) + ); + assert_eq!(p.enforced_limit(ACCT, region, "firehose", code), None); + } + } +} + +#[test] +fn firehose_throughput_quotas_remain_non_adjustable() { + let s = svc(); + for code in ["L-7681C790", "L-17D4EADE", "L-937C879B"] { + let err = call_err( + &s, + "RequestServiceQuotaIncrease", + json!({ + "ServiceCode": "firehose", "QuotaCode": code, "DesiredValue": 1_000_000.0, + }), + ); + assert_eq!(err.code(), "IllegalArgumentException"); + assert!(err.message().contains("not adjustable")); + for value in [-1.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let err = s + .introspect_put_quota( + "firehose", + code, + &PutQuotaRequest { + value: Some(value), + ..Default::default() + }, + ) + .unwrap_err(); + assert_eq!(err.status, StatusCode::BAD_REQUEST); + } + } +} + +#[test] +fn firehose_utilization_reports_use_regional_defaults() { + struct FirehoseUsage; + impl QuotaUsageSource for FirehoseUsage { + fn service_codes(&self) -> &[&str] { + &["firehose"] + } + fn usage(&self, _: &str, _: &str, _: &str, code: &str) -> Option { + (code == "L-937C879B").then_some(0.5) + } + } + let s = svc().with_usage_source(Arc::new(FirehoseUsage)); + let started = firehose_call( + &s, + ACCT, + "us-east-2", + "StartQuotaUtilizationReport", + json!({}), + ); + let report = firehose_call( + &s, + ACCT, + "us-east-2", + "GetQuotaUtilizationReport", + json!({"ReportId": started["ReportId"]}), + ); + assert_eq!(report["Quotas"][0]["DefaultValue"], 1.0); + assert_eq!(report["Quotas"][0]["AppliedValue"], 1.0); + assert_eq!(report["Quotas"][0]["Utilization"], 50.0); +} + #[test] fn nothing_is_enforced_until_switched_on() { use fakecloud_core::quota::QuotaProvider; diff --git a/website/content/docs/services/firehose.md b/website/content/docs/services/firehose.md index 8bc5b8ddc..684b7633a 100644 --- a/website/content/docs/services/firehose.md +++ b/website/content/docs/services/firehose.md @@ -33,6 +33,63 @@ The interval starts with the oldest arrival in a new buffer. After that buffer i `UpdateDestination` affects data still in the active buffer: when that data becomes eligible, it uses the latest supported destination, hints, output settings, and stream version. Once a batch has detached, its destination, version, codec, destination encryption key, timestamps, and object key are captured and reused for every retry. An unsupported update pauses already accepted active data until the configuration becomes supported again; it does not discard that data. Detached batches continue with their captured configuration. Deleting a stream or resetting the server discards its pending data; recreating the same stream name does not recover the old buffer. +## Opt-in Direct PUT throughput limits + +Ingestion quotas control whether public `PutRecord` and `PutRecordBatch` calls are accepted. They are separate from `BufferingHints`, which control when accepted data becomes eligible for S3 delivery. [`DirectPutSourceConfiguration.ThroughputHintInMBs`](https://docs.aws.amazon.com/firehose/latest/APIReference/API_DirectPutSourceConfiguration.html) is informational, not an enforcement control. + +The existing [Service Quotas controls](/docs/services/servicequotas/#enforcement) support three `firehose` quotas: + +| Quota code | Budget per stream per second | `us-east-1`, `us-west-2`, `eu-west-1` default | All other regions default | +| --- | --- | --- | --- | +| `L-7681C790` | Combined `PutRecord` and `PutRecordBatch` requests | 2,000 | 1,000 | +| `L-17D4EADE` | Records | 500,000 | 100,000 | +| `L-937C879B` | MiB of decoded record data | 5 | 1 | + +One MiB is 1,048,576 bytes. Both public operations share all three budgets: a batch costs one request, plus its valid records and their decoded bytes. Invalid-base64 batch entries do not consume records or bytes; even an all-invalid batch consumes one request. Producer-supplied delimiters count as input bytes; base64 transport encoding, output-added newlines, and compression do not affect the byte budget. + +Applied values are configured per account and region, but consumption is **independent for each delivery-stream instance**, not pooled across the region. Streams in the same account and region each receive the full configured budget. Destination updates, buffer flushes, and delivery retries do not reset or consume ingestion capacity. Deleting and recreating a stream starts a new budget; usage windows are process-local and are not persisted. + +### Enable and change limits + +**Enforcement is off by default.** Use `--enforce-quotas` globally, `--enforce-quota firehose/L-937C879B` for one quota, or the existing introspection API and its Service Quotas SDK wrappers. `--ignore-quota` and per-account overrides retain their existing precedence: account override, then server-wide override, then the global switch. The corresponding `FAKECLOUD_ENFORCE_QUOTAS`, `FAKECLOUD_ENFORCE_QUOTA`, and `FAKECLOUD_IGNORE_QUOTA` environment variables are also supported. + +For a running local server, set a 0.5 MiB/second budget for every stream in account `000000000000`, region `us-east-1`, and enforce that quota for the account: + +```sh +curl -fsS -X PUT \ + http://localhost:4566/_fakecloud/service-quotas/quotas/firehose/L-937C879B \ + -H 'Content-Type: application/json' \ + -d '{"accountId":"000000000000","region":"us-east-1","value":0.5,"enforce":true}' +``` + +Use the account and region of your producer's AWS client. Omitting `accountId` or `region` selects the server's configured account or region for the applied value, not the caller's AWS identity or signing region. There is an additional scope distinction: with `accountId`, `enforce` changes that account's override across regions; without `accountId`, it changes the server-wide override across accounts and regions. The example changes only the byte quota; requests and records remain unenforced unless separately enabled. + +Setting `value` alone leaves enforcement unchanged: on a default server it **does not turn throttling on**. `enforce: false` ignores the quota, `enforce: null` clears the override, and omitting `enforce` preserves it. No new Firehose-specific administration endpoint is required. + +The local setter accepts any finite, nonnegative value, including values below the AWS default, zero, and fractional MiB. Fractional request/record limits do not round up to a whole request/record. A zero request limit refuses every metered call; zero records permits no valid records, and zero MiB permits only zero decoded bytes (so an empty record can pass the byte check). All enabled dimensions must allow the call. + +Limits are read on each call. Raising a value applies to the next call without clearing usage already consumed in the current window; lowering it can block further admission until replenishment. Previously accepted buffers, delivery retries, and shutdown drains are unaffected. These Firehose quotas keep AWS's non-adjustable Service Quotas metadata: `RequestServiceQuotaIncrease` still refuses them. The local setter is a testing control, not a simulated AWS Support quota-increase workflow. + +### Window timing and request errors + +fakecloud approximates throughput with fixed, monotonic **one-second windows**. The first successful metered admission anchors a window; the first successful admission at or after its one-second boundary starts the next window at that call's time. Rejection does not consume capacity or move the anchor. Idle time accrues no credits. This is not a token bucket or a rolling-window guarantee: full allowances immediately before and after a boundary can produce a short burst of twice the per-window budget. + +There is no automatic scaling: configured capacity remains fixed until changed locally. A single record or batch exceeding an entire configured window cannot succeed merely by backing off and retrying; reduce its size or raise the relevant limit. + +If any enabled budget would be exceeded, the **whole call returns HTTP 500 `ServiceUnavailableException`**, before any records enter a buffer or trigger delivery. A rejected batch is not partially admitted and does not receive an HTTP 200 per-entry throughput-failure response. Existing input validation and unsupported-destination errors still precede quota consumption. + +With the official AWS SDK for Rust, request retries can delay the result or succeed after a window replenishes. If retryable failures continue until the configured retry attempts are exhausted, `.send().await` returns `Err`, not a successful output containing `FailedPutCount`. See the [Rust SDK retry configuration](https://docs.rs/aws-sdk-firehose/latest/aws_sdk_firehose/config/retry/struct.RetryConfig.html). + +By contrast, an HTTP 200 `PutRecordBatch` response can still contain existing per-entry validation failures, such as invalid base64, with results in input order and valid entries admitted. Inspect `FailedPutCount` and `RequestResponses`: the Rust SDK does **not** automatically resubmit failed entries from a successful HTTP 200 batch response. Request retries and per-entry results are separate contracts; neither an API acknowledgment nor successful retry means S3 delivery is durable. + +This increment meters only the public Direct PUT APIs, including their configuration-only non-S3 acknowledgments. Internal CloudWatch Logs and SES handoffs still share the buffering engine but are explicitly **unmetered** by these throughput controls. `KinesisStreamAsSource` streams still reject Direct PUT calls; Kinesis-source polling is not implemented. + +### AWS references and local differences + +The [Firehose quotas guide](https://docs.aws.amazon.com/firehose/latest/dev/limits.html) and [regional quota reference](https://docs.aws.amazon.com/general/latest/gr/fh.html) define the regional defaults and shared per-stream Direct PUT limits. AWS describes proportional quota increases and automatic scaling with unspecified timing; fakecloud does not reproduce either workflow or AWS's internal admission algorithm. Local controls can change each dimension independently. + +The [`PutRecord` reference](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecord.html) documents HTTP 500 `ServiceUnavailableException`, but its introductory 5,000-records/second figure conflicts with the 500,000 figure in the quota references for the three high-throughput regions. The table above follows the quota references, not that introductory number. The [`PutRecordBatch` reference](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecordBatch.html) distinguishes whole-call errors from ordered per-record results; it does not specify the exact threshold-allocation algorithm. The fixed windows and whole-call local throughput rejection described here are explicit local approximations. + ## S3 object bytes and names Create the destination bucket before sending records. Records `abc`, `def\n`, and the binary bytes `00 ff` in one delivery window become exactly `abcdef\n` followed by `00 ff`, whether submitted in one batch or separate calls. For compressed output, those concatenated bytes are encoded in one GZIP or Xerial Snappy stream. Producers must supply their own delimiters, such as a newline for each JSON record, or enable `AppendDelimiterToRecord`, which writes the same records as exactly `abc\ndef\n\n` followed by `00 ff 0a` before any compression. @@ -157,7 +214,7 @@ An acknowledgment means accepted into process-local memory, not durably delivere Runtime retries reuse the same captured key and output configuration. As AWS documents for S3 destinations, a failed head is retried every 5 seconds, with no backoff. Retries start on the worker's 1-second tick, so each actual delay can run up to one second longer. Each failed retry logs a warning with the attempt number and the next retry delay. As on AWS for `DirectPut` streams, retries stop 24 hours after a batch's oldest record arrived: the next failed attempt after that discards the batch with a warning naming its record and byte counts, and later batches in the stream proceed. The shutdown drain still makes its single final attempt regardless of age. -**Memory policy:** failed batches remain in memory for up to 24 hours while new valid records continue to be accepted. There is no memory limit, backpressure, or durable pending spool, so memory use can grow with everything accepted during that window of prolonged failure; a process crash loses pending data. Persistent mode stores stream configuration and completed S3 objects, not buffers or retry queues. +**Memory policy:** failed batches remain in memory for up to 24 hours while new valid records continue to be accepted, subject to any opt-in ingestion quotas. Those quotas are not backlog-aware backpressure: there is no memory limit or durable pending spool, so memory use can grow with everything accepted during that window of prolonged failure; a process crash loses pending data. Persistent mode stores stream configuration and completed S3 objects, not buffers or retry queues. **Shutdown policy:** graceful shutdown first lets the HTTP server finish in-flight requests, so API callers never see the closed admission. It then closes admission, waits for in-flight writes, and makes one final drain of supported buffered data without waiting for normal size/time hints or the retry interval. If a final head fails, later batches in that stream are not attempted; shutdown reports undelivered record/decoded-byte counts and does not start another retry loop. Unsupported active data is also reported undelivered. Late internal handoffs are refused with `ServiceUnavailableException` (HTTP 503) and dropped. The final drain has no time limit: a large backlog or slow store prolongs shutdown, and a supervisor that kills the process first (for example, after Docker's default 10-second stop timeout) loses whatever was not yet written. With a healthy bucket and store, graceful restart drains buffered records into durable objects; failed final-drain data is not recovered after restart. No flush/admin endpoint is provided. diff --git a/website/content/docs/services/servicequotas.md b/website/content/docs/services/servicequotas.md index b87689838..a25bd71af 100644 --- a/website/content/docs/services/servicequotas.md +++ b/website/content/docs/services/servicequotas.md @@ -10,11 +10,11 @@ persists across restarts in persistent mode. The wire protocol is awsJson1.1 (x-amz-target `ServiceQuotasV20190624.`), signing as `servicequotas`. Quotas are not just reported. Once you switch enforcement on, EC2, IAM, -DynamoDB, KMS, S3 and Lambda read the applied value of the quotas they check +DynamoDB, KMS, S3, Lambda and Firehose read the applied value of the quotas they check (for EC2 and VPC: security groups and their rules, VPCs, subnets, gateways, route tables, network ACLs, network interfaces, endpoints, peering, Elastic IPs, VPN connections and the On-Demand and Spot vCPU quotas) from Service Quotas, so -raising one with `RequestServiceQuotaIncrease` changes what the service accepts. +raising an adjustable one with `RequestServiceQuotaIncrease` changes what the service accepts. Enforcement is **opt-in**: by default nothing is enforced, so a local test suite that creates more resources than a fresh AWS account allows keeps working. See @@ -201,6 +201,7 @@ persist across restarts in persistent mode. | Customer Master Keys (`kms`/`L-C2F1777E`) | `CreateKey`, `ReplicateKey` (counted in the replica's region) (`LimitExceededException`); customer managed keys in any key state count, including pending deletion, AWS managed keys do not | | General purpose buckets (`s3`/`L-DC2B2D3D`) | `CreateBucket` (`TooManyBuckets`); a global quota: one applied value per account, counting buckets across all regions | | Function and layer storage (`lambda`/`L-2ACBD22F`) | `CreateFunction`, `UpdateFunctionCode`, `PublishVersion`, `PublishLayerVersion` (`CodeStorageExceededException`); the code of every function's `$LATEST`, published version and layer version counts, container images do not | +| Firehose requests, records and MiB per second per stream (`firehose`/`L-7681C790`, `L-17D4EADE`, `L-937C879B`) | Public `PutRecord` and `PutRecordBatch` share independent per-stream budgets; whole-call HTTP 500 `ServiceUnavailableException`. Non-adjustable through AWS Service Quotas; use the local setter. See [Firehose throughput limits](/docs/services/firehose/#opt-in-direct-put-throughput-limits) | IAM refusals are HTTP 409 with IAM's `Cannot exceed quota for : ` message. Every count quota refuses the request that would take the count past @@ -344,6 +345,7 @@ that concurrency limit, as Lambda always does. `GetServiceQuota` and `RequestServiceQuotaIncrease` refuse a `ContextId` (saying so) and `ListServiceQuotas` with `QuotaAppliedAtLevel` `RESOURCE` returns nothing. -- Default values are the ones AWS publishes in `us-east-1`. AWS sets some - defaults per region; fakecloud reports the `us-east-1` value in every - region. +- Default values are the ones AWS publishes in `us-east-1`, except for the + three [Firehose throughput quotas](/docs/services/firehose/#opt-in-direct-put-throughput-limits), + which report regional defaults. For other region-dependent defaults, + fakecloud reports the `us-east-1` value in every region. From 12a8bcff151c701d3901a6a2eef4937cc8a09722 Mon Sep 17 00:00:00 2001 From: John Cavanaugh <59479+cavanaug@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:31:49 -0700 Subject: [PATCH 2/6] test(firehose): align throughput fixtures with merged buffering behavior --- crates/fakecloud-firehose/src/engine/tests/throughput.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/fakecloud-firehose/src/engine/tests/throughput.rs b/crates/fakecloud-firehose/src/engine/tests/throughput.rs index cdf352ea2..438d3cb94 100644 --- a/crates/fakecloud-firehose/src/engine/tests/throughput.rs +++ b/crates/fakecloud-firehose/src/engine/tests/throughput.rs @@ -234,7 +234,7 @@ fn validation_precedes_debit_and_config_only_and_missing_backend_are_metered() { }); assert_eq!( put(&f, vec![record(b'a', 1)]).unwrap_err().code(), - "NotImplemented" + "InvalidArgumentException" ); f.update(NAME, |s| s.destination = None); put(&f, vec![record(b'a', 1)]).unwrap(); @@ -278,7 +278,7 @@ async fn retries_do_not_redebit_ingestion() { first.reply.send(Action::Fail).unwrap(); f.finish(&mut jobs).await; throttled(put(&f, vec![record(b'x', 1)])); - f.advance(1); + f.advance(5); f.schedule(&mut jobs, true); let retry = next_attempt(&mut entered).await; assert_eq!(first.batch.object_key, retry.batch.object_key); From 54c3d1508897d35d098be29a8a8c3f31f3ec88be Mon Sep 17 00:00:00 2001 From: John Cavanaugh <59479+cavanaug@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:38:18 -0700 Subject: [PATCH 3/6] feat(servicequotas): activate auto-scaling quotas only with explicit values --- crates/fakecloud-core/src/quota.rs | 42 ++- crates/fakecloud-sdk/src/types.rs | 5 +- crates/fakecloud-servicequotas/src/catalog.rs | 165 ++++++++++-- .../fakecloud-servicequotas/src/provider.rs | 254 ++++++++++++++++-- .../src/service/introspection.rs | 2 +- .../src/service/tests.rs | 217 +++++++++++++++ .../fakecloud-servicequotas/src/settings.rs | 204 ++++++++++++-- crates/fakecloud-servicequotas/src/state.rs | 16 +- sdks/dotnet/src/FakeCloud/Types.cs | 2 +- sdks/go/types.go | 2 +- .../src/main/java/dev/fakecloud/Types.java | 4 +- sdks/php/src/Types.php | 2 +- sdks/python/src/fakecloud/types.py | 3 +- sdks/typescript/src/types.ts | 3 +- 14 files changed, 830 insertions(+), 91 deletions(-) diff --git a/crates/fakecloud-core/src/quota.rs b/crates/fakecloud-core/src/quota.rs index e96af38e0..4dea7508b 100644 --- a/crates/fakecloud-core/src/quota.rs +++ b/crates/fakecloud-core/src/quota.rs @@ -5,9 +5,12 @@ //! quota request template, or set directly through the introspection API) and //! whether fakecloud enforces it. Enforcement is opt-in: a quota is enforced //! only when the user turned it on, for every quota (`--enforce-quotas`) or -//! for that quota alone. Services that enforce a quota ask for its limit -//! through [`QuotaProvider::enforced_limit`], so raising a quota through the -//! Service Quotas API changes what the enforcing service accepts. +//! for that quota alone. A quota whose capacity AWS scales automatically +//! (Firehose throughput) is additionally enforced only at a value the user set +//! explicitly; its AWS default is never a limit. Services that enforce a quota +//! ask for its limit through [`QuotaProvider::enforced_limit`], so raising a +//! quota through the Service Quotas API changes what the enforcing service +//! accepts. //! //! The reverse direction is [`QuotaUsageSource`]: a service that can count its //! own resources reports the usage behind a quota, which Service Quotas turns @@ -118,8 +121,9 @@ pub trait QuotaProvider: Send + Sync { /// The applied value of `service_code`/`quota_code` when fakecloud /// enforces that quota for `account_id`, or `None` when it does not (the - /// quota is not switched on, or unknown). A service refuses a request - /// that would take it past the returned limit. + /// quota is not switched on, has no explicit value while AWS scales it + /// automatically, or is unknown). A service refuses a request that would + /// take it past the returned limit. fn enforced_limit( &self, account_id: &str, @@ -127,6 +131,21 @@ pub trait QuotaProvider: Send + Sync { service_code: &str, quota_code: &str, ) -> Option; + + /// [`QuotaProvider::enforced_limit`] of several quotas of one service, + /// in `quota_codes` order, for a service that checks them together. + fn enforced_limits( + &self, + account_id: &str, + region: &str, + service_code: &str, + quota_codes: &[&str], + ) -> Vec> { + quota_codes + .iter() + .map(|code| self.enforced_limit(account_id, region, service_code, code)) + .collect() + } } /// Reports how much of a quota an account currently uses. @@ -254,6 +273,19 @@ mod tests { assert_eq!(enforced_count(None, "a", "r", "s", "neg"), None); } + #[test] + fn enforced_limits_resolves_each_code_in_order() { + let p = FixedQuotas::default() + .with("s", "a", 1.0) + .with("s", "b", 2.0); + assert_eq!( + p.enforced_limits("acct", "r", "s", &["b", "missing", "a"]), + vec![Some(2.0), None, Some(1.0)] + ); + assert_eq!(p.enforced_limits("acct", "r", "other", &["a"]), vec![None]); + assert!(p.enforced_limits("acct", "r", "s", &[]).is_empty()); + } + #[test] fn has_room_below_the_limit_only() { assert!(has_room(None, usize::MAX)); diff --git a/crates/fakecloud-sdk/src/types.rs b/crates/fakecloud-sdk/src/types.rs index 5d8d121fe..3e22afc74 100644 --- a/crates/fakecloud-sdk/src/types.rs +++ b/crates/fakecloud-sdk/src/types.rs @@ -2569,8 +2569,7 @@ pub struct Ec2InstanceNetworksResponse { // ── Service Quotas ────────────────────────────────────────────────── /// One quota as `/_fakecloud/service-quotas/quotas` reports it for an account -/// and region. `enforcementSource` is one of `not_enforceable`, -/// `account_override`, `override`, `global`. +/// and region. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ServiceQuota { @@ -2587,6 +2586,8 @@ pub struct ServiceQuota { /// Whether a fakecloud service checks requests against this quota. pub enforceable: bool, pub enforced: bool, + /// What decided `enforced`: `not_enforceable`, `account_override`, + /// `override`, `global` or `auto_scaling_default`. pub enforcement_source: String, } diff --git a/crates/fakecloud-servicequotas/src/catalog.rs b/crates/fakecloud-servicequotas/src/catalog.rs index 21ec793cb..3c134d629 100644 --- a/crates/fakecloud-servicequotas/src/catalog.rs +++ b/crates/fakecloud-servicequotas/src/catalog.rs @@ -9,13 +9,15 @@ //! enforceable?) are answered from the static tables without decoding, so a //! server that enforces nothing never pays for the decode on a request path. //! -//! Two things AWS does not publish through the API are a hand-maintained -//! overlay here: which quotas a fakecloud service checks -//! ([`QuotaDef::enforceable`], see [`ENFORCEABLE`]) and the documented maximum -//! an increase request can be approved for ([`QuotaDef::max_value`], see -//! [`MAX_VALUES`]). The enforcing service reads an enforceable quota's applied -//! value through [`fakecloud_core::quota::QuotaProvider`] once the user -//! switched enforcement on. +//! What AWS does not publish through the API is a hand-maintained overlay +//! here: which quotas a fakecloud service checks ([`QuotaDef::enforceable`], +//! see [`ENFORCEABLE`]), which of those AWS scales automatically +//! ([`AUTO_SCALING`]), the documented maximum an increase request can be +//! approved for ([`QuotaDef::max_value`], see [`MAX_VALUES`]) and defaults +//! that differ by region ([`REGIONAL_DEFAULTS`]). The enforcing service reads +//! an enforceable quota's applied value through +//! [`fakecloud_core::quota::QuotaProvider`] once the user switched +//! enforcement on. use std::collections::{BTreeMap, HashMap}; use std::ops::Range; @@ -140,6 +142,47 @@ pub const ENFORCEABLE: &[(&str, &str)] = &[ ("vpc", "L-FE5A380F"), // NAT gateways per Availability Zone ]; +/// Enforceable quotas whose capacity AWS scales automatically, so their AWS +/// default describes a starting point rather than a limit. One is enforced +/// only at an applied value the account set explicitly for the region (see +/// [`crate::settings::AccountEnforcement`]). Sorted like [`ENFORCEABLE`], so +/// [`is_auto_scaling`] can binary search it; every entry is also there. +pub const AUTO_SCALING: &[(&str, &str)] = &[ + (FIREHOSE_SERVICE_CODE, FIREHOSE_RECORDS_PER_SECOND), // Records per second per stream + (FIREHOSE_SERVICE_CODE, FIREHOSE_PUT_REQUESTS_PER_SECOND), // Put requests per second per stream + (FIREHOSE_SERVICE_CODE, FIREHOSE_MIB_PER_SECOND), // MiB per second per stream +]; + +/// Regions where Firehose's Direct PUT throughput defaults are higher. +const FIREHOSE_HIGH_THROUGHPUT_REGIONS: &[&str] = &["us-east-1", "us-west-2", "eu-west-1"]; + +/// AWS defaults that differ by region, which the us-east-1 dump cannot +/// carry: `(service, quota, regions, value in those regions, value +/// elsewhere)`. Every other quota's default is the same in every region. +pub const REGIONAL_DEFAULTS: &[(&str, &str, &[&str], f64, f64)] = &[ + ( + FIREHOSE_SERVICE_CODE, + FIREHOSE_PUT_REQUESTS_PER_SECOND, + FIREHOSE_HIGH_THROUGHPUT_REGIONS, + 2000.0, + 1000.0, + ), + ( + FIREHOSE_SERVICE_CODE, + FIREHOSE_RECORDS_PER_SECOND, + FIREHOSE_HIGH_THROUGHPUT_REGIONS, + 500_000.0, + 100_000.0, + ), + ( + FIREHOSE_SERVICE_CODE, + FIREHOSE_MIB_PER_SECOND, + FIREHOSE_HIGH_THROUGHPUT_REGIONS, + 5.0, + 1.0, + ), +]; + /// Maximum values AWS documents for adjustable quotas (Amazon VPC quotas, /// IAM and AWS STS quotas). An increase request above one is not approved. pub const MAX_VALUES: &[(&str, &str, f64)] = &[ @@ -368,6 +411,14 @@ pub fn is_enforceable(service_code: &str, quota_code: &str) -> bool { .is_ok() } +/// Whether AWS scales `service_code/quota_code` automatically (the +/// [`AUTO_SCALING`] overlay). Answered without decoding the catalog. +pub fn is_auto_scaling(service_code: &str, quota_code: &str) -> bool { + AUTO_SCALING + .binary_search_by(|&(s, q)| s.cmp(service_code).then_with(|| q.cmp(quota_code))) + .is_ok() +} + fn catalog() -> &'static Catalog { #[cfg(test)] CATALOG_READS.with(|c| c.set(c.get() + 1)); @@ -402,27 +453,19 @@ pub fn quota(service_code: &str, quota_code: &str) -> Option<&'static QuotaDef> c.index(service_code, quota_code).map(|i| &c.quotas[i]) } -/// AWS default in the requested region. Only Firehose's three per-stream -/// throughput quotas differ from the vendored us-east-1 catalog. +/// AWS default in the requested region: the [`REGIONAL_DEFAULTS`] entry for +/// the quota, else the vendored us-east-1 value. pub fn default_value(region: &str, def: &QuotaDef) -> f64 { - if def.service_code == FIREHOSE_SERVICE_CODE { - let high_throughput = matches!(region, "us-east-1" | "us-west-2" | "eu-west-1"); - match def.quota_code { - FIREHOSE_PUT_REQUESTS_PER_SECOND => { - return if high_throughput { 2000.0 } else { 1000.0 }; + REGIONAL_DEFAULTS + .iter() + .find(|&&(s, q, ..)| s == def.service_code && q == def.quota_code) + .map_or(def.default, |&(_, _, regions, inside, elsewhere)| { + if regions.contains(®ion) { + inside + } else { + elsewhere } - FIREHOSE_RECORDS_PER_SECOND => { - return if high_throughput { - 500_000.0 - } else { - 100_000.0 - }; - } - FIREHOSE_MIB_PER_SECOND => return if high_throughput { 5.0 } else { 1.0 }, - _ => {} - } - } - def.default + }) } /// Every quota of `service_code`, ordered by quota code. Empty for an @@ -486,6 +529,76 @@ mod tests { assert_eq!(default_value("us-east-2", &other), 42.0); } + #[test] + fn regional_defaults_name_real_quotas_once_and_resolve_by_region() { + let mut seen = HashSet::new(); + for &(service, code, regions, inside, elsewhere) in REGIONAL_DEFAULTS { + let def = quota(service, code) + .unwrap_or_else(|| panic!("regional default {service}/{code} missing")); + assert!(seen.insert((service, code)), "duplicate {service}/{code}"); + assert!(!def.global, "{service}/{code} is global"); + assert!(!regions.is_empty(), "{service}/{code}"); + for region in regions.iter() { + assert_eq!(default_value(region, def), inside, "{region}/{code}"); + } + for region in ["ap-southeast-2", "us-gov-west-1", "us-east-2"] { + assert!(!regions.contains(®ion)); + assert_eq!(default_value(region, def), elsewhere, "{region}/{code}"); + } + // The vendored dump is us-east-1's view. + assert_eq!(def.default, default_value("us-east-1", def), "{code}"); + } + assert_eq!( + REGIONAL_DEFAULTS + .iter() + .map(|&(s, q, r, inside, elsewhere)| (s, q, r.to_vec(), inside, elsewhere)) + .collect::>(), + [ + ("L-7681C790", 2000.0, 1000.0), + ("L-17D4EADE", 500_000.0, 100_000.0), + ("L-937C879B", 5.0, 1.0), + ] + .map(|(code, inside, elsewhere)| ( + "firehose", + code, + vec!["us-east-1", "us-west-2", "eu-west-1"], + inside, + elsewhere + )) + ); + // Every other quota keeps its vendored default everywhere. + for def in quotas() { + if !seen.contains(&(def.service_code, def.quota_code)) { + assert_eq!(default_value("ap-southeast-2", def), def.default); + } + } + } + + #[test] + fn auto_scaling_quotas_are_sorted_enforceable_and_answered_without_decoding() { + assert!( + AUTO_SCALING.windows(2).all(|w| w[0] < w[1]), + "AUTO_SCALING must be sorted" + ); + let before = catalog_reads(); + for &(svc, code) in AUTO_SCALING { + assert!(is_enforceable(svc, code), "{svc}/{code} not enforceable"); + assert!(is_auto_scaling(svc, code), "{svc}/{code}"); + } + assert!(!is_auto_scaling(VPC, "L-F678F1CE")); + assert!(!is_auto_scaling("firehose", "L-14BB0BE7")); + assert!(!is_auto_scaling("other", FIREHOSE_MIB_PER_SECOND)); + assert_eq!(catalog_reads(), before); + assert_eq!( + AUTO_SCALING, + [ + ("firehose", "L-17D4EADE"), + ("firehose", "L-7681C790"), + ("firehose", "L-937C879B"), + ] + ); + } + #[test] fn the_vendored_dump_decodes_into_every_listed_service() { let services = services(); diff --git a/crates/fakecloud-servicequotas/src/provider.rs b/crates/fakecloud-servicequotas/src/provider.rs index 4c75a0918..2a8e1795d 100644 --- a/crates/fakecloud-servicequotas/src/provider.rs +++ b/crates/fakecloud-servicequotas/src/provider.rs @@ -11,7 +11,7 @@ use fakecloud_organizations::SharedOrganizationsState; use fakecloud_persistence::SnapshotHook; use crate::catalog::{self, QuotaDef}; -use crate::settings::{enforcement_of, RequestApproval, SharedQuotaSettings}; +use crate::settings::{AccountEnforcement, RequestApproval, SharedQuotaSettings}; use crate::state::{ applied_key, QuotaRequest, ServiceQuotasData, SharedServiceQuotasState, TemplateEntry, }; @@ -252,10 +252,12 @@ impl ServiceQuotasProvider { impl ServiceQuotasProvider { /// Apply the organization's template to `account_id` if it is due, - /// persisting the change. - fn apply_template(&self, account_id: &str) { + /// persisting the change. Whether it changed anything. + fn apply_template(&self, account_id: &str) -> bool { let approval = self.settings.read().request_approval; - if apply_template_if_new(&self.state, &self.orgs, approval, account_id, Utc::now()) { + let applied = + apply_template_if_new(&self.state, &self.orgs, approval, account_id, Utc::now()); + if applied { // The lookup is synchronous; the snapshot write runs on the // runtime the enforcing service is called from. if let (Some(hook), Ok(rt)) = ( @@ -265,6 +267,25 @@ impl ServiceQuotasProvider { rt.spawn(hook()); } } + applied + } + + /// The enforced limits of `quota_codes`, from one settings and state + /// snapshot. + fn snapshot_limits( + &self, + account_id: &str, + region: &str, + service_code: &str, + quota_codes: &[&str], + ) -> Vec> { + let settings = self.settings.read(); + let guard = self.state.read(); + let mut account = AccountEnforcement::new(&settings, guard.get(account_id), region); + quota_codes + .iter() + .map(|code| account.limit(service_code, code)) + .collect() } } @@ -289,20 +310,37 @@ impl QuotaProvider for ServiceQuotasProvider { service_code: &str, quota_code: &str, ) -> Option { - // Whether the quota is enforced is answered from the static overlay - // and the settings; the catalog is only decoded for a quota that is. - if !catalog::is_enforceable(service_code, quota_code) { - return None; + self.enforced_limits(account_id, region, service_code, &[quota_code]) + .pop() + .flatten() + } + + fn enforced_limits( + &self, + account_id: &str, + region: &str, + service_code: &str, + quota_codes: &[&str], + ) -> Vec> { + // Whether a quota is enforced is answered from the static overlay and + // the settings; the catalog is only decoded for a quota that is. + if !quota_codes + .iter() + .any(|code| catalog::is_enforceable(service_code, code)) + { + return vec![None; quota_codes.len()]; } - self.apply_template(account_id); - let settings = self.settings.read(); - let guard = self.state.read(); - let data = guard.get(account_id); - if !enforcement_of(&settings, data, service_code, quota_code).0 { - return None; + let limits = self.snapshot_limits(account_id, region, service_code, quota_codes); + #[cfg(test)] + tests::after_initial_snapshot(); + if !limits.iter().any(Option::is_some) { + return limits; } - let def = catalog::quota(service_code, quota_code)?; - Some(applied_value(data, region, def)) + // Enforcing something touches the account's quotas, which applies a + // due organization template (see `apply_template_if_new`); answer + // from the values it left, even if another lookup applied it. + self.apply_template(account_id); + self.snapshot_limits(account_id, region, service_code, quota_codes) } } @@ -310,6 +348,19 @@ impl QuotaProvider for ServiceQuotasProvider { mod tests { use super::*; + thread_local! { + // One-shot scheduling seam, isolated to the lookup's test thread. + static AFTER_INITIAL_SNAPSHOT: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; + } + + pub(super) fn after_initial_snapshot() { + let hook = AFTER_INITIAL_SNAPSHOT.with(|hook| hook.borrow_mut().take()); + if let Some(hook) = hook { + hook(); + } + } + fn provider(settings: crate::settings::QuotaSettings) -> ServiceQuotasProvider { use parking_lot::RwLock; use std::sync::Arc; @@ -325,7 +376,7 @@ mod tests { } #[test] - fn firehose_provider_defaults_are_regional_and_enforcement_is_opt_in() { + fn firehose_provider_reports_regional_defaults_but_enforces_none() { let p = provider(Default::default()); let before = catalog::catalog_reads(); for code in ["L-7681C790", "L-17D4EADE", "L-937C879B"] { @@ -351,17 +402,169 @@ mod tests { p.applied_value(account, region, "firehose", code), Some(expected) ); + // AWS scales this capacity: its default is no limit. p.settings.write().enforce_all = true; - assert_eq!( - p.enforced_limit(account, region, "firehose", code), - Some(expected) - ); + assert_eq!(p.enforced_limit(account, region, "firehose", code), None); p.settings.write().enforce_all = false; } } } } + /// One batched lookup answers exactly what one lookup per code does. + #[test] + fn enforced_limits_match_one_lookup_per_code() { + use fakecloud_core::quota::{ + FIREHOSE_MIB_PER_SECOND as MIB, FIREHOSE_PUT_REQUESTS_PER_SECOND as REQUESTS, + FIREHOSE_RECORDS_PER_SECOND as RECORDS, + }; + const ACCT: &str = "000000000000"; + let p = provider(Default::default()); + let codes = [REQUESTS, RECORDS, MIB, "L-14BB0BE7", "L-NOPE"]; + let limits = |p: &ServiceQuotasProvider| { + for (account, region) in [ + (ACCT, "us-east-1"), + (ACCT, "us-east-2"), + ("222222222222", "us-east-2"), + ] { + let one_by_one: Vec<_> = codes + .iter() + .map(|code| p.enforced_limit(account, region, "firehose", code)) + .collect(); + assert_eq!( + p.enforced_limits(account, region, "firehose", &codes), + one_by_one, + "{account}/{region}" + ); + } + p.enforced_limits(ACCT, "us-east-2", "firehose", &codes) + }; + let explicit = |code: &str, v: f64| { + p.state + .write() + .get_or_create(ACCT) + .applied + .insert(applied_key("us-east-2", false, "firehose", code), v); + }; + + assert_eq!(limits(&p), [None; 5]); + p.settings.write().enforce_all = true; + assert_eq!(limits(&p), [None; 5]); + explicit(REQUESTS, 10.0); + explicit(MIB, 0.5); + assert_eq!(limits(&p), [Some(10.0), None, Some(0.5), None, None]); + p.settings + .write() + .overrides + .insert(format!("firehose/{MIB}"), false); + assert_eq!(limits(&p), [Some(10.0), None, None, None, None]); + p.settings.write().enforce_all = false; + explicit(RECORDS, 7.0); + p.state + .write() + .get_or_create(ACCT) + .enforcement + .insert(format!("firehose/{RECORDS}"), true); + assert_eq!(limits(&p), [None, Some(7.0), None, None, None]); + assert_eq!( + p.enforced_limits(ACCT, "us-east-2", "firehose", &[]), + Vec::>::new() + ); + + // Quotas AWS does not scale are enforced at their applied value. + p.settings.write().enforce_all = true; + assert_eq!( + p.enforced_limits( + ACCT, + "us-east-2", + catalog::VPC, + &["L-0EA8095F", "L-2AFB9258"] + ), + [Some(60.0), Some(5.0)] + ); + } + + #[test] + fn enforced_limits_resnapshot_when_another_lookup_applies_the_template() { + use fakecloud_organizations::{MemberAccount, OrganizationState}; + use std::sync::mpsc; + + const MGMT: &str = "111111111111"; + const MEMBER: &str = "222222222222"; + const REGION: &str = "us-east-1"; + const CODES: &[&str] = &["L-2AFB9258", "L-0EA8095F", "L-NOPE"]; + let p = provider(crate::settings::QuotaSettings { + enforce_all: true, + ..Default::default() + }); + let associated_at = DateTime::from_timestamp(1_700_000_000, 0).unwrap(); + let mut org = OrganizationState::bootstrap_in(REGION, MGMT); + org.accounts.insert( + MEMBER.to_string(), + MemberAccount { + id: MEMBER.to_string(), + arn: org.account_arn(MEMBER), + email: format!("{MEMBER}@example.com"), + name: "member".into(), + status: "ACTIVE".into(), + joined_method: "CREATED".into(), + joined_timestamp: associated_at + chrono::Duration::seconds(1), + parent_id: org.root_id.clone(), + gov_cloud_mirror: false, + }, + ); + p.orgs.write().insert(org); + { + let mut guard = p.state.write(); + let management = guard.get_or_create(MGMT); + management.template_associated_at = Some(associated_at); + management.template.insert( + crate::state::template_key(catalog::VPC, CODES[0], REGION), + TemplateEntry { + service_code: catalog::VPC.into(), + quota_code: CODES[0].into(), + aws_region: REGION.into(), + desired_value: 8.0, + }, + ); + } + assert_eq!( + p.snapshot_limits(MEMBER, REGION, catalog::VPC, CODES), + [Some(5.0), Some(60.0), None] + ); + + std::thread::scope(|scope| { + let (snapshotted_tx, snapshotted_rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let p = &p; + let losing_lookup = scope.spawn(move || { + AFTER_INITIAL_SNAPSHOT.with(|hook| { + *hook.borrow_mut() = Some(Box::new(move || { + snapshotted_tx.send(()).unwrap(); + resume_rx.recv().unwrap(); + })); + }); + p.enforced_limits(MEMBER, REGION, catalog::VPC, CODES) + }); + + // Both lookups snapshot the defaults before the winner applies + // the real template. The paused lookup then sees its marker and + // gets false from apply_template, but must still return 8, not 5. + snapshotted_rx.recv().unwrap(); + let winning_limits = p.enforced_limits(MEMBER, REGION, catalog::VPC, CODES); + resume_tx.send(()).unwrap(); + let losing_limits = losing_lookup.join().unwrap(); + assert_eq!(winning_limits, [Some(8.0), Some(60.0), None]); + assert_eq!(losing_limits, [Some(8.0), Some(60.0), None]); + }); + assert!(!p.apply_template(MEMBER)); + let guard = p.state.read(); + let member = guard.get(MEMBER).unwrap(); + assert!(member.template_checked.is_some()); + assert_eq!(member.requests.len(), 1, "template applied more than once"); + assert_eq!(member.requests.values().next().unwrap().status, "APPROVED"); + } + /// With enforcement off, an enforcing service's limit lookup on its /// request path never touches (or decodes) the catalog. #[test] @@ -380,6 +583,15 @@ mod tests { None ); } + assert_eq!( + p.enforced_limits( + "000000000000", + "us-east-1", + "firehose", + &["L-7681C790", "L-17D4EADE", "L-937C879B", "L-14BB0BE7"] + ), + [None; 4] + ); assert_eq!(catalog::catalog_reads(), before); } diff --git a/crates/fakecloud-servicequotas/src/service/introspection.rs b/crates/fakecloud-servicequotas/src/service/introspection.rs index 6ce7659be..bef1ff50f 100644 --- a/crates/fakecloud-servicequotas/src/service/introspection.rs +++ b/crates/fakecloud-servicequotas/src/service/introspection.rs @@ -212,7 +212,7 @@ impl ServiceQuotasService { def: &QuotaDef, usage: Option, ) -> Value { - let (enforced, source) = enforcement(settings, data, def); + let (enforced, source) = enforcement(settings, data, region, def); json!({ "serviceCode": def.service_code, "quotaCode": def.quota_code, diff --git a/crates/fakecloud-servicequotas/src/service/tests.rs b/crates/fakecloud-servicequotas/src/service/tests.rs index 942a226e8..a03b7c8d0 100644 --- a/crates/fakecloud-servicequotas/src/service/tests.rs +++ b/crates/fakecloud-servicequotas/src/service/tests.rs @@ -1111,6 +1111,223 @@ fn firehose_utilization_reports_use_regional_defaults() { assert_eq!(report["Quotas"][0]["Utilization"], 50.0); } +const FIREHOSE_THROUGHPUT: [&str; 3] = ["L-7681C790", "L-17D4EADE", "L-937C879B"]; + +fn put_value(s: &ServiceQuotasService, account: &str, region: &str, code: &str, v: f64) -> Value { + s.introspect_put_quota( + "firehose", + code, + &PutQuotaRequest { + account_id: Some(account.into()), + region: Some(region.into()), + value: Some(v), + ..Default::default() + }, + ) + .unwrap() +} + +fn set_override(s: &ServiceQuotasService, code: &str, account: Option<&str>, on: Option) { + s.introspect_put_enforcement(&PutEnforcementRequest { + enforce_all: None, + overrides: vec![OverrideChange { + service_code: "firehose".into(), + quota_code: code.into(), + account_id: account.map(Into::into), + enforce: on, + }], + }) + .unwrap(); +} + +fn enforce_all(s: &ServiceQuotasService, on: bool) { + s.introspect_put_enforcement(&PutEnforcementRequest { + enforce_all: Some(on), + overrides: vec![], + }) + .unwrap(); +} + +/// AWS scales Firehose throughput automatically, so switching enforcement on +/// is not enough: only a value the account set explicitly in the region is a +/// limit. +#[test] +fn firehose_throughput_is_enforced_only_at_explicit_values() { + use fakecloud_core::quota::QuotaProvider; + let s = svc(); + let p = provider(&s); + let limit = |account: &str, region: &str, code: &str| { + p.enforced_limit(account, region, "firehose", code) + }; + + enforce_all(&s, true); + for code in FIREHOSE_THROUGHPUT { + for region in ["us-east-1", "us-east-2"] { + assert_eq!(limit(ACCT, region, code), None, "{region}/{code}"); + } + } + // Other enforceable quotas are enforced at their applied value as before. + assert_eq!( + p.enforced_limit(ACCT, "us-east-1", "vpc", "L-0EA8095F"), + Some(60.0) + ); + + enforce_all(&s, false); + let values = [10.0, 200.0, 0.5]; + for (code, v) in FIREHOSE_THROUGHPUT.into_iter().zip(values) { + put_value(&s, ACCT, "us-east-2", code, v); + assert_eq!(limit(ACCT, "us-east-2", code), None, "not switched on"); + } + + enforce_all(&s, true); + for (code, v) in FIREHOSE_THROUGHPUT.into_iter().zip(values) { + assert_eq!(limit(ACCT, "us-east-2", code), Some(v), "{code}"); + // The value belongs to its account and region only. + assert_eq!(limit(ACCT, "us-east-1", code), None); + assert_eq!(limit(MEMBER, "us-east-2", code), None); + } + // A value equal to the AWS default is explicit too. + put_value(&s, ACCT, "us-east-1", "L-7681C790", 2000.0); + assert_eq!(limit(ACCT, "us-east-1", "L-7681C790"), Some(2000.0)); + + // Overrides switch it on and off as for any enforceable quota. + enforce_all(&s, false); + set_override(&s, "L-17D4EADE", None, Some(true)); + assert_eq!(limit(ACCT, "us-east-2", "L-17D4EADE"), Some(200.0)); + assert_eq!(limit(ACCT, "us-east-2", "L-7681C790"), None); + put_value(&s, MEMBER, "us-east-2", "L-937C879B", 3.0); + assert_eq!(limit(MEMBER, "us-east-2", "L-937C879B"), None); + set_override(&s, "L-937C879B", Some(MEMBER), Some(true)); + assert_eq!(limit(MEMBER, "us-east-2", "L-937C879B"), Some(3.0)); + assert_eq!(limit(ACCT, "us-east-2", "L-937C879B"), None); + set_override(&s, "L-17D4EADE", Some(ACCT), Some(false)); + assert_eq!(limit(ACCT, "us-east-2", "L-17D4EADE"), None); + + // Deleting the explicit value stops enforcement, even when switched on. + set_override(&s, "L-17D4EADE", Some(ACCT), None); + enforce_all(&s, true); + for (code, v) in FIREHOSE_THROUGHPUT.into_iter().zip(values) { + assert_eq!(limit(ACCT, "us-east-2", code), Some(v)); + s.introspect_delete_quota("firehose", code, None, Some("us-east-2")) + .unwrap(); + assert_eq!(limit(ACCT, "us-east-2", code), None, "{code} deleted"); + } +} + +#[test] +fn firehose_introspection_reports_auto_scaling_defaults_as_unenforced() { + let s = svc(); + let view = |account: &str, region: &str, code: &str| -> Value { + s.introspect_quotas(Some(account), Some(region), Some("firehose")) + .unwrap()["quotas"] + .as_array() + .unwrap() + .iter() + .find(|q| q["quotaCode"] == code) + .unwrap() + .clone() + }; + let state = |v: &Value| (v["enforced"].clone(), v["enforcementSource"].clone()); + let off = |source: &str| (json!(false), json!(source)); + let on = |source: &str| (json!(true), json!(source)); + + for code in FIREHOSE_THROUGHPUT { + assert_eq!(state(&view(ACCT, "us-east-2", code)), off("global")); + } + enforce_all(&s, true); + for code in FIREHOSE_THROUGHPUT { + assert_eq!( + state(&view(ACCT, "us-east-2", code)), + off("auto_scaling_default"), + "{code}" + ); + } + let vpc = s + .introspect_quotas(Some(ACCT), Some("us-east-2"), Some("vpc")) + .unwrap(); + let rules = vpc["quotas"] + .as_array() + .unwrap() + .iter() + .find(|q| q["quotaCode"] == "L-0EA8095F") + .unwrap(); + assert_eq!(state(rules), on("global")); + + let code = "L-937C879B"; + let put = put_value(&s, ACCT, "us-east-2", code, 2.0); + assert_eq!(state(&put), on("global")); + assert_eq!(state(&view(ACCT, "us-east-2", code)), on("global")); + assert_eq!( + state(&view(ACCT, "us-east-1", code)), + off("auto_scaling_default") + ); + assert_eq!( + state(&view(MEMBER, "us-east-2", code)), + off("auto_scaling_default") + ); + set_override(&s, code, None, Some(true)); + assert_eq!(state(&view(ACCT, "us-east-2", code)), on("override")); + assert_eq!( + state(&view(ACCT, "us-east-1", code)), + off("auto_scaling_default") + ); + set_override(&s, code, Some(ACCT), Some(true)); + assert_eq!( + state(&view(ACCT, "us-east-2", code)), + on("account_override") + ); + set_override(&s, code, Some(ACCT), Some(false)); + assert_eq!( + state(&view(ACCT, "us-east-2", code)), + off("account_override") + ); + set_override(&s, code, Some(ACCT), None); + set_override(&s, code, None, None); + enforce_all(&s, false); + assert_eq!(state(&view(ACCT, "us-east-2", code)), off("global")); + + enforce_all(&s, true); + let deleted = s + .introspect_delete_quota("firehose", code, Some(ACCT), Some("us-east-2")) + .unwrap(); + assert_eq!(state(&deleted), off("auto_scaling_default")); +} + +/// Another service's limit lookup applies a due Organizations template only +/// when it enforces something; an unenforced lookup leaves the account alone. +#[test] +fn enforced_limit_applies_the_template_only_once_enforced() { + use fakecloud_core::quota::QuotaProvider; + let o = org_with_member(3600); + let s = svc_with(o); + run( + &s, + MGMT, + "PutServiceQuotaIncreaseRequestIntoTemplate", + json!({ "ServiceCode": "vpc", "QuotaCode": "L-2AFB9258", "AwsRegion": "us-east-1", "DesiredValue": 8.0 }), + ) + .unwrap(); + run(&s, MGMT, "AssociateServiceQuotaTemplate", json!({})).unwrap(); + let p = provider(&s); + assert_eq!( + p.enforced_limit(MEMBER, "us-east-1", "vpc", "L-2AFB9258"), + None + ); + assert!(s.state.read().get(MEMBER).is_none(), "template applied"); + + enforce_all(&s, true); + assert_eq!( + p.enforced_limits(MEMBER, "us-east-1", "vpc", &["L-2AFB9258", "L-0EA8095F"]), + [Some(8.0), Some(60.0)] + ); + assert_eq!(s.state.read().get(MEMBER).unwrap().requests.len(), 1); + assert_eq!( + p.enforced_limit(MEMBER, "us-east-1", "vpc", "L-2AFB9258"), + Some(8.0) + ); + assert_eq!(s.state.read().get(MEMBER).unwrap().requests.len(), 1); +} + #[test] fn nothing_is_enforced_until_switched_on() { use fakecloud_core::quota::QuotaProvider; diff --git a/crates/fakecloud-servicequotas/src/settings.rs b/crates/fakecloud-servicequotas/src/settings.rs index f24064d5a..ed1b1b769 100644 --- a/crates/fakecloud-servicequotas/src/settings.rs +++ b/crates/fakecloud-servicequotas/src/settings.rs @@ -9,10 +9,13 @@ //! Nothing is enforced by default, so a local test suite that creates more //! resources than a fresh AWS account allows keeps working until the user //! asks for limit fidelity. Only quotas a fakecloud service can check -//! ([`QuotaDef::enforceable`]) are ever enforced. +//! ([`QuotaDef::enforceable`]) are ever enforced, and a quota whose capacity +//! AWS scales automatically ([`catalog::AUTO_SCALING`]) only once the account +//! set an explicit applied value for it in the region. //! //! The server-wide part lives in [`QuotaSettings`]; per-account overrides live //! in [`ServiceQuotasData::enforcement`] next to the account's applied values. +//! [`AccountEnforcement`] combines them. use std::collections::BTreeMap; use std::sync::Arc; @@ -20,8 +23,8 @@ use std::sync::Arc; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; -use crate::catalog::QuotaDef; -use crate::state::ServiceQuotasData; +use crate::catalog::{self, QuotaDef}; +use crate::state::{write_applied_key, ServiceQuotasData}; /// How `RequestServiceQuotaIncrease` (and template entries applied to new /// accounts) are decided. @@ -81,6 +84,10 @@ pub enum EnforcementSource { Override, /// The global switch. Global, + /// Switched on, but AWS scales this quota's capacity automatically and + /// the account set no explicit value for it in the region, so there is + /// no limit to enforce. + AutoScalingDefault, } impl EnforcementSource { @@ -90,6 +97,7 @@ impl EnforcementSource { Self::AccountOverride => "account_override", Self::Override => "override", Self::Global => "global", + Self::AutoScalingDefault => "auto_scaling_default", } } } @@ -102,7 +110,14 @@ pub fn quota_ref(def: &QuotaDef) -> String { /// [`quota_ref`] from the codes. pub fn quota_ref_of(service_code: &str, quota_code: &str) -> String { - format!("{service_code}/{quota_code}") + let mut key = String::new(); + write_quota_ref(&mut key, service_code, quota_code); + key +} + +fn write_quota_ref(buf: &mut String, service_code: &str, quota_code: &str) { + buf.clear(); + buf.extend([service_code, "/", quota_code]); } /// Resolve a `service_code/quota_code` reference to an enforceable quota (as @@ -126,46 +141,117 @@ pub fn resolve_enforceable(reference: &str) -> Result { Ok(quota_ref(def)) } -/// Whether `def` is enforced for the account holding `data`, and why. +/// Whether `def` is enforced for the account holding `data` in `region`, and +/// why. See [`AccountEnforcement::of`]. pub fn enforcement( settings: &QuotaSettings, data: Option<&ServiceQuotasData>, + region: &str, def: &QuotaDef, ) -> (bool, EnforcementSource) { - enforcement_of(settings, data, def.service_code, def.quota_code) + AccountEnforcement::new(settings, data, region).of(def.service_code, def.quota_code) } -/// [`enforcement`] by codes. Reads only the static enforceable overlay, never -/// the decoded catalog, so the request path of a service that enforces nothing -/// stays cheap. -pub fn enforcement_of( - settings: &QuotaSettings, - data: Option<&ServiceQuotasData>, - service_code: &str, - quota_code: &str, -) -> (bool, EnforcementSource) { - if !crate::catalog::is_enforceable(service_code, quota_code) { - return (false, EnforcementSource::NotEnforceable); +/// Effective enforcement for one account in one region, read from one +/// settings and state snapshot: the single place both the +/// [`fakecloud_core::quota::QuotaProvider`] and the introspection view decide +/// whether a quota is enforced. Lookups reuse one key buffer, so resolving +/// several quotas allocates at most once. +pub struct AccountEnforcement<'a> { + settings: &'a QuotaSettings, + data: Option<&'a ServiceQuotasData>, + region: &'a str, + key: String, +} + +impl<'a> AccountEnforcement<'a> { + pub fn new( + settings: &'a QuotaSettings, + data: Option<&'a ServiceQuotasData>, + region: &'a str, + ) -> Self { + Self { + settings, + data, + region, + key: String::new(), + } } - let key = quota_ref_of(service_code, quota_code); - if let Some(on) = data.and_then(|d| d.enforcement.get(&key)) { - return (*on, EnforcementSource::AccountOverride); + + /// Whether `service_code/quota_code` is enforced, and why: the override + /// precedence, except that a quota AWS scales automatically + /// ([`crate::catalog::AUTO_SCALING`]) is enforced only at an applied value + /// the account set explicitly for the region. The catalog is decoded only + /// for such a quota once it is switched on, so the request path of a + /// service that enforces nothing stays cheap. + pub fn of(&mut self, service_code: &str, quota_code: &str) -> (bool, EnforcementSource) { + if !catalog::is_enforceable(service_code, quota_code) { + return (false, EnforcementSource::NotEnforceable); + } + let (on, source) = self.switch(service_code, quota_code); + if on + && catalog::is_auto_scaling(service_code, quota_code) + && catalog::quota(service_code, quota_code) + .and_then(|def| self.explicit_value(def)) + .is_none() + { + return (false, EnforcementSource::AutoScalingDefault); + } + (on, source) } - if let Some(on) = settings.overrides.get(&key) { - return (*on, EnforcementSource::Override); + + /// The limit fakecloud enforces for `service_code/quota_code`: its + /// applied value while [`AccountEnforcement::of`] says it is enforced. + pub fn limit(&mut self, service_code: &str, quota_code: &str) -> Option { + if !self.of(service_code, quota_code).0 { + return None; + } + let def = catalog::quota(service_code, quota_code)?; + Some( + self.explicit_value(def) + .unwrap_or_else(|| catalog::default_value(self.region, def)), + ) + } + + /// The account switch, else the server-wide override, else the global + /// switch. + fn switch(&mut self, service_code: &str, quota_code: &str) -> (bool, EnforcementSource) { + write_quota_ref(&mut self.key, service_code, quota_code); + let key = self.key.as_str(); + if let Some(on) = self.data.and_then(|d| d.enforcement.get(key)) { + return (*on, EnforcementSource::AccountOverride); + } + if let Some(on) = self.settings.overrides.get(key) { + return (*on, EnforcementSource::Override); + } + (self.settings.enforce_all, EnforcementSource::Global) + } + + /// The applied value the account set for `def` in the region (an + /// approved request or the introspection API), if any. + fn explicit_value(&mut self, def: &QuotaDef) -> Option { + let data = self.data?; + write_applied_key( + &mut self.key, + self.region, + def.global, + def.service_code, + def.quota_code, + ); + data.applied.get(self.key.as_str()).copied() } - (settings.enforce_all, EnforcementSource::Global) } #[cfg(test)] mod tests { use super::*; use crate::catalog; + use crate::state::applied_key; #[test] fn nothing_is_enforced_by_default() { for def in catalog::quotas() { - assert!(!enforcement(&QuotaSettings::default(), None, def).0); + assert!(!enforcement(&QuotaSettings::default(), None, "us-east-1", def).0); } } @@ -177,18 +263,18 @@ mod tests { ..Default::default() }; assert_eq!( - enforcement(&settings, None, def), + enforcement(&settings, None, "us-east-1", def), (true, EnforcementSource::Global) ); settings.overrides.insert(quota_ref(def), false); assert_eq!( - enforcement(&settings, None, def), + enforcement(&settings, None, "us-east-1", def), (false, EnforcementSource::Override) ); let mut data = ServiceQuotasData::default(); data.enforcement.insert(quota_ref(def), true); assert_eq!( - enforcement(&settings, Some(&data), def), + enforcement(&settings, Some(&data), "us-east-1", def), (true, EnforcementSource::AccountOverride) ); } @@ -203,11 +289,73 @@ mod tests { }; settings.overrides.insert(quota_ref(def), true); assert_eq!( - enforcement(&settings, None, def), + enforcement(&settings, None, "us-east-1", def), (false, EnforcementSource::NotEnforceable) ); } + #[test] + fn an_auto_scaling_quota_is_enforced_only_at_an_explicit_value() { + use EnforcementSource::*; + let def = catalog::quota( + catalog::FIREHOSE_SERVICE_CODE, + catalog::FIREHOSE_RECORDS_PER_SECOND, + ) + .unwrap(); + let mut settings = QuotaSettings { + enforce_all: true, + ..Default::default() + }; + let mut data = ServiceQuotasData::default(); + assert_eq!( + enforcement(&settings, Some(&data), "us-east-2", def), + (false, AutoScalingDefault) + ); + assert_eq!(AutoScalingDefault.as_str(), "auto_scaling_default"); + data.applied.insert( + applied_key("us-east-2", false, def.service_code, def.quota_code), + 9.0, + ); + let effective = |settings: &QuotaSettings, data: &ServiceQuotasData, region| { + enforcement(settings, Some(data), region, def) + }; + assert_eq!(effective(&settings, &data, "us-east-2"), (true, Global)); + assert_eq!( + effective(&settings, &data, "us-east-1"), + (false, AutoScalingDefault) + ); + assert_eq!( + enforcement(&settings, None, "us-east-2", def), + (false, AutoScalingDefault) + ); + // Switched off, the switch says why, value or not. + settings.overrides.insert(quota_ref(def), false); + assert_eq!(effective(&settings, &data, "us-east-2"), (false, Override)); + assert_eq!(effective(&settings, &data, "us-east-1"), (false, Override)); + data.enforcement.insert(quota_ref(def), true); + assert_eq!( + effective(&settings, &data, "us-east-2"), + (true, AccountOverride) + ); + assert_eq!( + effective(&settings, &data, "us-east-1"), + (false, AutoScalingDefault) + ); + + let mut account = AccountEnforcement::new(&settings, Some(&data), "us-east-2"); + assert_eq!(account.limit(def.service_code, def.quota_code), Some(9.0)); + assert_eq!( + account.limit(def.service_code, catalog::FIREHOSE_MIB_PER_SECOND), + None + ); + // Quotas AWS does not scale are enforced at the default. + assert_eq!( + account.limit(catalog::VPC, catalog::RULES_PER_SECURITY_GROUP), + Some(60.0) + ); + assert_eq!(account.limit("lambda", "L-B99A9384"), None); + } + #[test] fn enforce_flag_references_resolve_through_the_vendored_data() { assert_eq!( diff --git a/crates/fakecloud-servicequotas/src/state.rs b/crates/fakecloud-servicequotas/src/state.rs index c13386927..a3661fed3 100644 --- a/crates/fakecloud-servicequotas/src/state.rs +++ b/crates/fakecloud-servicequotas/src/state.rs @@ -136,8 +136,22 @@ impl AccountState for ServiceQuotasData { /// Key of an applied value. Global quotas apply in every region, so their key /// carries no region. pub fn applied_key(region: &str, global: bool, service_code: &str, quota_code: &str) -> String { + let mut key = String::new(); + write_applied_key(&mut key, region, global, service_code, quota_code); + key +} + +/// [`applied_key`] written over `buf`, so repeated lookups reuse one buffer. +pub(crate) fn write_applied_key( + buf: &mut String, + region: &str, + global: bool, + service_code: &str, + quota_code: &str, +) { let region = if global { "" } else { region }; - format!("{region}|{service_code}|{quota_code}") + buf.clear(); + buf.extend([region, "|", service_code, "|", quota_code]); } pub fn template_key(service_code: &str, quota_code: &str, aws_region: &str) -> String { diff --git a/sdks/dotnet/src/FakeCloud/Types.cs b/sdks/dotnet/src/FakeCloud/Types.cs index 5687863a1..dfcee4d7c 100644 --- a/sdks/dotnet/src/FakeCloud/Types.cs +++ b/sdks/dotnet/src/FakeCloud/Types.cs @@ -1407,7 +1407,7 @@ public enum QuotaEnforcement /// null when no fakecloud service measures the quota. /// EnforcementSource says which setting decided Enforced: one of /// not_enforceable, account_override, override, -/// global. +/// global, auto_scaling_default. /// public sealed record ServiceQuota( string? ServiceCode, diff --git a/sdks/go/types.go b/sdks/go/types.go index 586b8af99..79302da6a 100644 --- a/sdks/go/types.go +++ b/sdks/go/types.go @@ -1956,7 +1956,7 @@ type ServiceQuota struct { Enforceable bool `json:"enforceable"` Enforced bool `json:"enforced"` // EnforcementSource says what decided Enforced: "not_enforceable", - // "account_override", "override" or "global". + // "account_override", "override", "global" or "auto_scaling_default". EnforcementSource string `json:"enforcementSource"` } diff --git a/sdks/java/src/main/java/dev/fakecloud/Types.java b/sdks/java/src/main/java/dev/fakecloud/Types.java index 27be59808..163025013 100644 --- a/sdks/java/src/main/java/dev/fakecloud/Types.java +++ b/sdks/java/src/main/java/dev/fakecloud/Types.java @@ -1752,8 +1752,8 @@ public Boolean toWire() { * One quota from {@code GET /_fakecloud/service-quotas/quotas} (or the * single quota returned by a put/delete). {@code usage} is {@code null} * when no fakecloud service measures it. {@code enforcementSource} is one - * of {@code not_enforceable}, {@code account_override}, {@code override} - * or {@code global}. + * of {@code not_enforceable}, {@code account_override}, {@code override}, + * {@code global} or {@code auto_scaling_default}. */ @JsonIgnoreProperties(ignoreUnknown = true) public record ServiceQuota( diff --git a/sdks/php/src/Types.php b/sdks/php/src/Types.php index 35fa9733e..fc7719ec6 100644 --- a/sdks/php/src/Types.php +++ b/sdks/php/src/Types.php @@ -4475,7 +4475,7 @@ public function __construct( /** Whether a fakecloud service checks requests against this quota. */ public readonly bool $enforceable, public readonly bool $enforced, - /** `not_enforceable`, `account_override`, `override` or `global`. */ + /** `not_enforceable`, `account_override`, `override`, `global` or `auto_scaling_default`. */ public readonly string $enforcementSource, ) {} diff --git a/sdks/python/src/fakecloud/types.py b/sdks/python/src/fakecloud/types.py index 4215bdf24..d28268564 100644 --- a/sdks/python/src/fakecloud/types.py +++ b/sdks/python/src/fakecloud/types.py @@ -3849,7 +3849,8 @@ class ServiceQuota: ``usage`` is ``None`` when no fakecloud service measures the quota. ``enforcement_source`` is one of ``"not_enforceable"``, - ``"account_override"``, ``"override"`` or ``"global"``. + ``"account_override"``, ``"override"``, ``"global"`` or + ``"auto_scaling_default"``. """ service_code: str diff --git a/sdks/typescript/src/types.ts b/sdks/typescript/src/types.ts index a7a86d72d..e5cd274e9 100644 --- a/sdks/typescript/src/types.ts +++ b/sdks/typescript/src/types.ts @@ -1792,7 +1792,8 @@ export type QuotaEnforcementSource = | "not_enforceable" | "account_override" | "override" - | "global"; + | "global" + | "auto_scaling_default"; /** One Service Quotas catalog quota as seen by an account and region. */ export interface Quota { From 4d4014bea5a9c0e760ec9a5847bc96ca1dd6a57b Mon Sep 17 00:00:00 2001 From: John Cavanaugh <59479+cavanaug@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:29:31 -0700 Subject: [PATCH 4/6] feat(firehose): complete throughput admission and partial failures --- crates/fakecloud-e2e/tests/firehose.rs | 15 +- .../tests/firehose_throughput.rs | 286 ++++++++- crates/fakecloud-firehose/src/engine.rs | 60 +- .../src/engine/tests/throughput.rs | 178 ++++-- crates/fakecloud-firehose/src/service.rs | 143 ++--- crates/fakecloud-firehose/src/throughput.rs | 604 +++++++----------- website/content/docs/services/firehose.md | 28 +- .../content/docs/services/servicequotas.md | 10 +- 8 files changed, 751 insertions(+), 573 deletions(-) diff --git a/crates/fakecloud-e2e/tests/firehose.rs b/crates/fakecloud-e2e/tests/firehose.rs index 058793c75..801aaa473 100644 --- a/crates/fakecloud-e2e/tests/firehose.rs +++ b/crates/fakecloud-e2e/tests/firehose.rs @@ -641,8 +641,7 @@ async fn firehose_put_record_batch_reports_invalid_records() { .expect("put batch"); assert_eq!(resp.failed_put_count(), 0); - // Wire-level invalid-base64 records should bump FailedPutCount and - // surface an ErrorCode on the corresponding response entry. + // Invalid base64 rejects the whole request, not individual entries. let http = reqwest::Client::new(); let url = format!("{}/", server.endpoint()); let body = serde_json::json!({ @@ -666,13 +665,13 @@ async fn firehose_put_record_batch_reports_invalid_records() { .send() .await .unwrap(); - assert_eq!(r.status(), 200); + assert_eq!(r.status(), 400); let json: serde_json::Value = r.json().await.unwrap(); - assert_eq!(json["FailedPutCount"], 1); - let resps = json["RequestResponses"].as_array().unwrap(); - assert_eq!(resps.len(), 2); - assert_eq!(resps[0]["ErrorCode"], "InvalidArgumentException"); - assert!(resps[1].get("RecordId").is_some()); + assert!(json["__type"] + .as_str() + .unwrap() + .ends_with("SerializationException")); + assert!(json.get("RequestResponses").is_none()); } #[tokio::test] diff --git a/crates/fakecloud-e2e/tests/firehose_throughput.rs b/crates/fakecloud-e2e/tests/firehose_throughput.rs index 4c04edbf2..c3db37bfa 100644 --- a/crates/fakecloud-e2e/tests/firehose_throughput.rs +++ b/crates/fakecloud-e2e/tests/firehose_throughput.rs @@ -1,12 +1,13 @@ //! Public Firehose admission contract, using real HTTP and the official AWS SDK. -//! Fixed-window refill/concurrency are covered by the deterministic engine tests; -//! these tests use impossible single-call costs or zero capacity, not clock races. +//! Burst repayment/concurrency are covered by deterministic engine tests; +//! HTTP assertions use single-call costs or zero capacity, not clock races. use std::future::Future; use std::io::Read; use std::time::Duration; use aws_sdk_firehose::config::retry::RetryConfig; +use aws_sdk_firehose::error::ProvideErrorMetadata; use aws_sdk_firehose::error::SdkError; use aws_sdk_firehose::operation::put_record::PutRecordError; use aws_sdk_firehose::operation::put_record_batch::PutRecordBatchError; @@ -125,6 +126,7 @@ fn assert_single_unavailable(err: SdkError) { panic!("expected modeled PutRecord service error, got {err:?}"); }; assert_eq!(context.raw().status().as_u16(), 500); + assert_eq!(context.err().message(), Some("Slow down.")); assert!( matches!( context.err(), @@ -140,6 +142,7 @@ fn assert_batch_unavailable(err: SdkError) { panic!("expected modeled PutRecordBatch service error, got {err:?}"); }; assert_eq!(context.raw().status().as_u16(), 500); + assert_eq!(context.err().message(), Some("Slow down.")); assert!( matches!( context.err(), @@ -283,36 +286,86 @@ async fn zero_request_capacity_returns_modeled_http_500_for_both_apis_then_live_ } #[tokio::test] -async fn record_capacity_rejects_entire_batch_without_s3_side_effects_then_live_raise_delivers() { +async fn record_capacity_partially_accepts_batch_and_only_failed_entries_are_resubmitted() { bounded(async { let server = TestServer::start().await; let bucket = "fh-throughput-records"; let (firehose, s3) = setup(&server, bucket, false).await; let fc = FakeCloud::new(server.endpoint()); limit(&fc, RECORDS, 1.0).await; - assert_batch_unavailable( - firehose + let response = firehose + .put_record_batch() + .delivery_stream_name(bucket) + .records(record(b"first")) + .records(record(b"second")) + .send() + .await + .expect("record exhaustion returns HTTP 200"); + assert_eq!(response.failed_put_count(), 1); + let entries = response.request_responses(); + assert!(entries[0].record_id().is_some()); + assert!(entries[0].error_code().is_none()); + assert!(entries[1].record_id().is_none()); + assert_eq!(entries[1].error_code(), Some("ServiceUnavailableException")); + assert_eq!(entries[1].error_message(), Some("Slow down.")); + assert_eq!( + payloads(&s3, bucket, 1, false).await, + vec![b"first".to_vec()] + ); + // A successful batch response does not cause SDK retries of failed + // entries. The producer explicitly resubmits only that entry. + limit(&fc, RECORDS, 2.0).await; + batch(&firehose, bucket, vec![record(b"second")]).await; + assert_eq!( + payloads(&s3, bucket, 2, false).await, + vec![b"first".to_vec(), b"second".to_vec()] + ); + }) + .await; +} + +#[tokio::test] +async fn zero_record_or_byte_capacity_fails_single_calls_but_returns_all_failed_batches() { + bounded(async { + for (code, bucket) in [ + (RECORDS, "fh-throughput-zero-records"), + (MIB, "fh-throughput-zero-bytes"), + ] { + let server = TestServer::start().await; + let (firehose, s3) = setup(&server, bucket, false).await; + limit(&FakeCloud::new(server.endpoint()), code, 0.0).await; + assert_single_unavailable( + firehose + .put_record() + .delivery_stream_name(bucket) + .record(record(b"single")) + .send() + .await + .unwrap_err(), + ); + let response = firehose .put_record_batch() .delivery_stream_name(bucket) .records(record(b"first")) .records(record(b"second")) .send() .await - .expect_err("two records cannot fit in capacity one"), - ); - assert_no_objects(&s3, bucket).await; - limit(&fc, RECORDS, 2.0).await; - batch(&firehose, bucket, vec![record(b"first"), record(b"second")]).await; - assert_eq!( - payloads(&s3, bucket, 1, false).await, - vec![b"firstsecond".to_vec()] - ); + .expect("record/byte limits must not reject a batch as a whole-call error"); + assert_eq!(response.failed_put_count(), 2); + assert_eq!(response.request_responses().len(), 2); + for entry in response.request_responses() { + assert!(entry.record_id().is_none()); + assert_eq!(entry.error_code(), Some("ServiceUnavailableException")); + assert_eq!(entry.error_message(), Some("Slow down.")); + } + assert_no_objects(&s3, bucket).await; + } }) .await; } #[tokio::test] -async fn byte_capacity_counts_decoded_raw_bytes_not_base64_or_gzip_and_rejects_whole_batch() { +async fn byte_capacity_counts_decoded_bytes_and_admits_one_boundary_crossing_record() { bounded(async { let server = TestServer::start().await; let bucket = "fh-throughput-bytes"; @@ -320,23 +373,27 @@ async fn byte_capacity_counts_decoded_raw_bytes_not_base64_or_gzip_and_rejects_w let fc = FakeCloud::new(server.endpoint()); let a = vec![b'a'; 4096]; let b = vec![b'b'; 4096]; - // Each individual record fits in 6144 bytes, but the 8192-byte batch does not. - // GZIP output is tiny; base64 on the wire is larger than either capacity. - limit(&fc, MIB, 6144.0 / 1_048_576.0).await; - assert_batch_unavailable( - firehose - .put_record_batch() - .delivery_stream_name(bucket) - .records(record(&a)) - .records(record(&b)) - .send() - .await - .expect_err("raw decoded batch exceeds byte capacity"), + // The first 4096-byte record crosses 3072 bytes of capacity. Its base64 + // encoding is larger and GZIP output is tiny; neither sets the budget. + limit(&fc, MIB, 3072.0 / 1_048_576.0).await; + let response = firehose + .put_record_batch() + .delivery_stream_name(bucket) + .records(record(&a)) + .records(record(&b)) + .send() + .await + .expect("byte exhaustion is per-entry"); + assert_eq!(response.failed_put_count(), 1); + assert!(response.request_responses()[0].record_id().is_some()); + assert_eq!( + response.request_responses()[1].error_code(), + Some("ServiceUnavailableException") ); - assert_no_objects(&s3, bucket).await; + assert_eq!(payloads(&s3, bucket, 1, true).await, vec![a.clone()]); limit(&fc, MIB, 8192.0 / 1_048_576.0).await; - batch(&firehose, bucket, vec![record(&a), record(&b)]).await; - assert_eq!(payloads(&s3, bucket, 1, true).await, vec![[a, b].concat()]); + batch(&firehose, bucket, vec![record(&b)]).await; + assert_eq!(payloads(&s3, bucket, 2, true).await, vec![a, b]); }) .await; } @@ -411,6 +468,175 @@ async fn configured_zero_values_are_not_enforced_by_default() { .await; } +#[tokio::test] +async fn enabling_default_throughput_does_not_impose_the_reported_aws_limit() { + bounded(async { + let server = TestServer::start_full(&[], &["--enforce-quotas"]).await; + let region = "us-east-2"; // Reported default is 1 MiB/s. + let firehose = client(&server.aws_config_in(region).await, 1); + let fc = FakeCloud::new(server.endpoint()); + let quotas = fc + .service_quotas() + .get_quotas(None, Some(region), Some("firehose")) + .await + .unwrap(); + for quota in quotas + .quotas + .iter() + .filter(|q| [REQUESTS, RECORDS, MIB].contains(&q.quota_code.as_str())) + { + assert!(!quota.enforced); + assert_eq!(quota.enforcement_source, "auto_scaling_default"); + } + create_stream( + &firehose, + "default-throughput", + "unused-default-bucket", + false, + ) + .await; + // Several maximum-size batches exceed the AWS default without explicit + // applied values. Limits alone must not introduce failed entries. + let bytes = vec![b'x'; 800_000]; + for _ in 0..3 { + batch(&firehose, "default-throughput", vec![record(&bytes); 5]).await; + } + }) + .await; +} + +#[tokio::test] +async fn same_named_streams_in_two_enforced_accounts_each_get_their_own_allowance() { + bounded(async { + let server = TestServer::start().await; + let bucket = "fh-throughput-two-accounts"; + let (firehose, s3) = setup(&server, bucket, false).await; + let fc = FakeCloud::new(server.endpoint()); + limit(&fc, RECORDS, 1.0).await; + let other = "210987654321"; + let (access, secret) = server.create_admin(other, "throughput-account").await; + let sdk = server + .aws_config_in(REGION) + .await + .into_builder() + .credentials_provider( + aws_credential_types::provider::SharedCredentialsProvider::new( + aws_credential_types::Credentials::new(access, secret, None, None, "test"), + ), + ) + .build(); + let other_firehose = client(&sdk, 1); + let other_s3 = aws_sdk_s3::Client::from_conf( + aws_sdk_s3::config::Builder::from(&sdk) + .force_path_style(true) + .build(), + ); + let other_bucket = "fh-throughput-other-account"; + other_s3 + .create_bucket() + .bucket(other_bucket) + .create_bucket_configuration( + aws_sdk_s3::types::CreateBucketConfiguration::builder() + .location_constraint(aws_sdk_s3::types::BucketLocationConstraint::UsWest2) + .build(), + ) + .send() + .await + .unwrap(); + create_stream(&other_firehose, bucket, other_bucket, false).await; + fc.service_quotas() + .put_quota( + "firehose", + RECORDS, + &PutServiceQuotaRequest { + account_id: Some(other.into()), + region: Some(REGION.into()), + value: Some(1.0), + enforce: Some(QuotaEnforcement::Enforce), + }, + ) + .await + .unwrap(); + for (producer, accepted, rejected) in [ + ( + &firehose, + b"first-account".as_slice(), + b"reject-first".as_slice(), + ), + ( + &other_firehose, + b"second-account".as_slice(), + b"reject-second".as_slice(), + ), + ] { + let response = producer + .put_record_batch() + .delivery_stream_name(bucket) + .records(record(accepted)) + .records(record(rejected)) + .send() + .await + .unwrap(); + assert_eq!(response.failed_put_count(), 1); + assert!(response.request_responses()[0].record_id().is_some()); + assert_eq!( + response.request_responses()[1].error_code(), + Some("ServiceUnavailableException") + ); + } + assert_eq!( + payloads(&s3, bucket, 1, false).await, + vec![b"first-account".to_vec()] + ); + assert_eq!( + payloads(&other_s3, other_bucket, 1, false).await, + vec![b"second-account".to_vec()] + ); + }) + .await; +} + +#[tokio::test] +async fn decoded_byte_budget_excludes_base64_and_compression() { + bounded(async { + let server = TestServer::start().await; + let bucket = "fh-throughput-decoded"; + let (firehose, s3) = setup(&server, bucket, true).await; + limit( + &FakeCloud::new(server.endpoint()), + MIB, + 8192.0 / 1_048_576.0, + ) + .await; + let a = vec![b'a'; 4096]; + let b = vec![b'b'; 2048]; + let c = vec![b'c'; 2048]; + let response = firehose + .put_record_batch() + .delivery_stream_name(bucket) + .records(record(&a)) + .records(record(&b)) + .records(record(&c)) + .records(record(b"d")) + .send() + .await + .unwrap(); + assert_eq!(response.failed_put_count(), 1); + assert!(response.request_responses()[..3] + .iter() + .all(|r| r.record_id().is_some())); + assert_eq!( + response.request_responses()[3].error_code(), + Some("ServiceUnavailableException") + ); + assert_eq!( + payloads(&s3, bucket, 1, true).await, + vec![[a, b, c].concat()] + ); + }) + .await; +} + #[tokio::test] async fn other_account_and_region_quota_values_do_not_reject_the_producer() { bounded(async { diff --git a/crates/fakecloud-firehose/src/engine.rs b/crates/fakecloud-firehose/src/engine.rs index 50fe3bece..18bd6af6c 100644 --- a/crates/fakecloud-firehose/src/engine.rs +++ b/crates/fakecloud-firehose/src/engine.rs @@ -242,6 +242,7 @@ impl FirehoseEngine { records: Vec, ) -> Result<(), AwsServiceError> { self.enqueue_admission(account, region, name, records, None) + .map(|_| ()) } pub(crate) fn enqueue_put( @@ -250,18 +251,26 @@ impl FirehoseEngine { region: &str, name: &str, records: Vec, - ) -> Result<(), AwsServiceError> { + ) -> Result, AwsServiceError> { // Provider implementations can consult other services (including // Firehose). Release even the provider guard before resolving limits, // and never call them under configuration or registry locks. let provider = self.core.quota_provider.read().clone(); let limits = provider.map(|provider| { - let limit = - |code| provider.enforced_limit(account, region, FIREHOSE_SERVICE_CODE, code); + let values = provider.enforced_limits( + account, + region, + FIREHOSE_SERVICE_CODE, + &[ + FIREHOSE_PUT_REQUESTS_PER_SECOND, + FIREHOSE_RECORDS_PER_SECOND, + FIREHOSE_MIB_PER_SECOND, + ], + ); ThroughputLimits { - requests: limit(FIREHOSE_PUT_REQUESTS_PER_SECOND), - records: limit(FIREHOSE_RECORDS_PER_SECOND), - mib: limit(FIREHOSE_MIB_PER_SECOND), + requests: values[0], + records: values[1], + mib: values[2], } }); let limits = @@ -276,7 +285,7 @@ impl FirehoseEngine { name: &str, records: Vec, limits: Option, - ) -> Result<(), AwsServiceError> { + ) -> Result, AwsServiceError> { // Every path holding both takes configuration before the registry. Keep // the read guard until admission and immediate detachment both commit. let config = self.core.state.read(); @@ -317,34 +326,35 @@ impl FirehoseEngine { instance_id: stream.instance_id, }; let now = self.core.clock.now(); + let mut accepted = vec![true; records.len()]; if let Some(limits) = limits { - let bytes = records.iter().map(Bytes::len).sum(); - if !registry + let window = registry .windows .entry(key.clone()) - .or_insert_with(|| ThroughputWindow::new(now.monotonic)) - .try_admit(now.monotonic, limits, records.len(), bytes) - { - return Err(AwsServiceError::aws_error( - StatusCode::INTERNAL_SERVER_ERROR, - "ServiceUnavailableException", - "Throughput limit exceeded for the delivery stream. Back off and retry.", - )); + .or_insert_with(|| ThroughputWindow::new(now.monotonic)); + if !window.try_request(now.monotonic, limits) { + return Err(throughput_error()); + } + for (admitted, record) in accepted.iter_mut().zip(&records) { + *admitted = window.try_record(limits, record.len()); } } // Non-S3 streams retain the configuration-only acknowledgement, but // public ingestion is still metered even without an S3 backend. let Some(destination) = destination else { - return Ok(()); + return Ok(accepted); }; let buffer = registry.streams.entry(key.clone()).or_default(); - if !records.is_empty() { + if accepted.iter().any(|admitted| *admitted) { let active = buffer.active.get_or_insert_with(|| ActiveBuffer { records: Vec::new(), input_bytes: 0, first: now.clone(), }); - for record in records { + for (record, admitted) in records.into_iter().zip(&accepted) { + if !admitted { + continue; + } active.input_bytes += record.len(); active.records.push(record); } @@ -358,7 +368,7 @@ impl FirehoseEngine { drop(registry); drop(config); self.core.wake.notify_one(); - Ok(()) + Ok(accepted) } /// Start once inside an active Tokio runtime. Admission can precede startup. @@ -718,6 +728,14 @@ impl EngineCore { /// The one delivery decision shared by admission, scheduling, and the final /// drain, so they cannot disagree about whether a stream can deliver. /// `Ok(None)`: no destination is configured, so nothing is written. +pub(crate) fn throughput_error() -> AwsServiceError { + AwsServiceError::aws_error( + StatusCode::INTERNAL_SERVER_ERROR, + "ServiceUnavailableException", + "Slow down.", + ) +} + fn stream_delivery(stream: &DeliveryStream) -> Result, AwsServiceError> { crate::admission::validate_stream_destination(stream)?; stream.destination.as_ref().map(delivery_config).transpose() diff --git a/crates/fakecloud-firehose/src/engine/tests/throughput.rs b/crates/fakecloud-firehose/src/engine/tests/throughput.rs index 438d3cb94..ec9c496a9 100644 --- a/crates/fakecloud-firehose/src/engine/tests/throughput.rs +++ b/crates/fakecloud-firehose/src/engine/tests/throughput.rs @@ -16,13 +16,17 @@ impl QuotaProvider for LimitsProvider { panic!("admission must use enforced_limit") } - fn enforced_limit( + fn enforced_limit(&self, _: &str, _: &str, _: &str, _: &str) -> Option { + panic!("admission must resolve all limits in one batched lookup") + } + + fn enforced_limits( &self, account: &str, region: &str, service: &str, - code: &str, - ) -> Option { + codes: &[&str], + ) -> Vec> { assert_eq!(service, FIREHOSE_SERVICE_CODE); if self.probe_locks.load(std::sync::atomic::Ordering::Relaxed) { let engine = self.engine.upgrade().unwrap(); @@ -41,12 +45,15 @@ impl QuotaProvider for LimitsProvider { .expect("provider under registry lock"); } let limits = *self.limits.read(); - match code { - FIREHOSE_PUT_REQUESTS_PER_SECOND => limits.requests, - FIREHOSE_RECORDS_PER_SECOND => limits.records, - FIREHOSE_MIB_PER_SECOND => limits.mib, - _ => panic!("unexpected quota {code}"), - } + codes + .iter() + .map(|code| match *code { + FIREHOSE_PUT_REQUESTS_PER_SECOND => limits.requests, + FIREHOSE_RECORDS_PER_SECOND => limits.records, + FIREHOSE_MIB_PER_SECOND => limits.mib, + _ => panic!("unexpected quota {code}"), + }) + .collect() } } @@ -70,14 +77,12 @@ fn enforce( } fn put(f: &Fixture, records: Vec) -> Result<(), AwsServiceError> { - crate::FirehoseService::default() - .with_engine(f.engine.clone()) - .deliver_records( - ACCOUNT, - REGION, - NAME, - records.into_iter().map(|b| b.to_vec()).collect(), - ) + let admitted = f.engine.enqueue_put(ACCOUNT, REGION, NAME, records)?; + if admitted.iter().all(|admitted| *admitted) { + Ok(()) + } else { + Err(throughput_error()) + } } fn throttled(result: Result<(), AwsServiceError>) { @@ -121,14 +126,14 @@ fn provider_resolution_precedes_firehose_locks() { } #[tokio::test] -async fn single_and_batch_share_all_dimensions_with_atomic_rejection() { +async fn single_and_batch_share_all_dimensions_with_ordered_admission() { for dimension in ["requests", "records", "bytes"] { let f = Fixture::new(60); enforce( &f, Some(3.0), Some(if dimension == "bytes" { 5.0 } else { 4.0 }), - Some(12.0 / 1_048_576.0), + Some(10.0 / 1_048_576.0), ); put(&f, vec![record(b'a', 4)]).unwrap(); put(&f, vec![record(b'b', 2), record(b'c', 2)]).unwrap(); @@ -143,17 +148,21 @@ async fn single_and_batch_share_all_dimensions_with_atomic_rejection() { throttled(put(&f, vec![record(b'x', 1)])); assert_eq!(buffered(&f), before); } else { - throttled(put(&f, rejected)); - assert_eq!(buffered(&f), before); - // A rejected call consumed neither request, record nor byte capacity. - put(&f, vec![record(b'd', 4)]).unwrap(); + assert_eq!( + f.engine + .enqueue_put(ACCOUNT, REGION, NAME, rejected) + .unwrap(), + vec![true, false] + ); + // The request was charged once, even though its second entry failed. + throttled(put(&f, vec![record(b'd', 1)])); } f.advance(60); f.turn().await; let records = f.batches()[0].records.clone(); let mut expected = vec![record(b'a', 4), record(b'b', 2), record(b'c', 2)]; if dimension != "requests" { - expected.push(record(b'd', 4)); + expected.push(record(b'x', if dimension == "records" { 1 } else { 3 })); } assert_eq!(records, expected); } @@ -185,15 +194,23 @@ fn budgets_are_stream_account_region_isolated_and_concurrent() { 7 ); }); - let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); for (account, region, name) in [ (ACCOUNT, REGION, "other"), ("222222222222", REGION, NAME), (ACCOUNT, "us-west-2", NAME), ] { - svc.deliver_records(account, region, name, vec![vec![1]; 7]) - .unwrap(); - throttled(svc.deliver_records(account, region, name, vec![vec![1]])); + assert_eq!( + f.engine + .enqueue_put(account, region, name, vec![record(b'a', 1); 7]) + .unwrap(), + vec![true; 7] + ); + assert_eq!( + f.engine + .enqueue_put(account, region, name, vec![record(b'x', 1)]) + .unwrap(), + vec![false] + ); } } @@ -343,6 +360,99 @@ fn request(action: &str, body: serde_json::Value) -> fakecloud_core::service::Aw } } +#[tokio::test] +async fn public_batch_burst_is_ordered_and_debt_survives_window_rollover() { + use fakecloud_core::service::AwsService; + let f = Fixture::new(60); + enforce(&f, None, None, Some(2.0 / 1_048_576.0)); + let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); + let send = || { + request( + "PutRecordBatch", + json!({"DeliveryStreamName": NAME, + "Records": [{"Data": "YWJjZGU="}, {"Data": "Zg=="}, {"Data": ""}]}), + ) + }; + let response = svc.handle(send()).await.unwrap(); + assert_eq!(response.status, StatusCode::OK); + let body: serde_json::Value = serde_json::from_slice(response.body.expect_bytes()).unwrap(); + assert_eq!(body["FailedPutCount"], 1); + assert!(body["RequestResponses"][0]["RecordId"].is_string()); + assert_eq!( + body["RequestResponses"][1], + json!({"ErrorCode": "ServiceUnavailableException", "ErrorMessage": "Slow down."}) + ); + assert!(body["RequestResponses"][2]["RecordId"].is_string()); + assert_eq!( + buffered(&f), + vec![Bytes::from_static(b"abcde"), Bytes::new()] + ); + + // Five bytes against two bytes/s leave three bytes carried at t+1. + // At t+2 only one byte remains charged, leaving room for the retry. + { + f.advance(1); + let response = svc.handle(send()).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(response.body.expect_bytes()).unwrap(); + assert_eq!(body["FailedPutCount"], 2); + } + f.advance(1); + let response = svc + .handle(request( + "PutRecordBatch", + json!({"DeliveryStreamName": NAME, + "Records": [{"Data": "Zg=="}]}), + )) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(response.body.expect_bytes()).unwrap(); + assert_eq!(body["FailedPutCount"], 0); + assert_eq!(buffered(&f).last().unwrap(), &Bytes::from_static(b"f")); +} + +#[tokio::test] +async fn malformed_base64_rejects_whole_call_without_consuming_capacity() { + use fakecloud_core::service::AwsService; + let f = Fixture::new(60); + enforce(&f, Some(1.0), Some(1.0), None); + let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); + for (action, body) in [ + ( + "PutRecord", + json!({"DeliveryStreamName": NAME, "Record": {"Data": "%%%"}}), + ), + ( + "PutRecordBatch", + json!({"DeliveryStreamName": NAME, "Records": [{"Data": "YQ=="}, {"Data": "%%%"}]}), + ), + ] { + let error = svc.handle(request(action, body)).await.err().unwrap(); + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + assert_eq!(error.code(), "SerializationException"); + assert!(buffered(&f).is_empty()); + } + svc.handle(request( + "PutRecord", + json!({"DeliveryStreamName": NAME, + "Record": {"Data": "Yg=="}}), + )) + .await + .unwrap(); + let error = svc + .handle(request( + "PutRecordBatch", + json!({"DeliveryStreamName": NAME, + "Records": [{"Data": "Yw=="}]}), + )) + .await + .err() + .unwrap(); + assert_eq!(error.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(error.code(), "ServiceUnavailableException"); + assert_eq!(error.message(), "Slow down."); + assert_eq!(buffered(&f), vec![Bytes::from_static(b"b")]); +} + #[tokio::test] async fn public_decoding_charges_only_valid_records_and_decoded_bytes() { use fakecloud_core::service::AwsService; @@ -359,12 +469,12 @@ async fn public_decoding_charges_only_valid_records_and_decoded_bytes() { .handle(request( "PutRecordBatch", json!({"DeliveryStreamName": NAME, - "Records": [{"Data": "%%%"}, {"Data": "YmNk"}]}), + "Records": [{"Data": "YmNk"}]}), )) .await .unwrap(); let body: serde_json::Value = serde_json::from_slice(response.body.expect_bytes()).unwrap(); - assert_eq!(body["FailedPutCount"], 1); + assert_eq!(body["FailedPutCount"], 0); let error = svc .handle(request( "PutRecord", @@ -392,10 +502,6 @@ async fn public_request_validation_never_debits_or_partially_admits() { let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); let oversized = base64::engine::general_purpose::STANDARD.encode(vec![0; 1_024_001]); for (action, body) in [ - ( - "PutRecord", - json!({"DeliveryStreamName": NAME, "Record": {"Data": "%%%"}}), - ), ( "PutRecord", json!({"DeliveryStreamName": NAME, "Record": {"Data": oversized}}), @@ -440,14 +546,14 @@ async fn public_request_validation_never_debits_or_partially_admits() { } #[tokio::test] -async fn all_invalid_public_batch_consumes_one_request_but_no_records() { +async fn all_throttled_public_batch_consumes_one_request_but_no_records() { use fakecloud_core::service::AwsService; let f = Fixture::new(60); enforce(&f, Some(1.0), Some(0.0), Some(0.0)); let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); svc.handle(request( "PutRecordBatch", - json!({"DeliveryStreamName": NAME, "Records": [{"Data": "%%%"}]}), + json!({"DeliveryStreamName": NAME, "Records": [{"Data": "YQ=="}]}), )) .await .unwrap(); diff --git a/crates/fakecloud-firehose/src/service.rs b/crates/fakecloud-firehose/src/service.rs index 683f5e676..4e52f426e 100644 --- a/crates/fakecloud-firehose/src/service.rs +++ b/crates/fakecloud-firehose/src/service.rs @@ -835,7 +835,7 @@ impl FirehoseService { .map_err(|_| { AwsServiceError::aws_error( StatusCode::BAD_REQUEST, - "InvalidArgumentException", + "SerializationException", "Record.Data must be valid base64", ) })?; @@ -845,7 +845,15 @@ impl FirehoseService { )); } let record_id = format!("{}", Uuid::new_v4()); - self.deliver_records(&req.account_id, &req.region, &name, vec![data])?; + let accepted = self.engine.enqueue_put( + &req.account_id, + &req.region, + &name, + vec![Bytes::from(data)], + )?; + if !accepted[0] { + return Err(crate::engine::throughput_error()); + } Ok(AwsResponse::ok_json(json!({ "RecordId": record_id, "Encrypted": false, @@ -867,8 +875,6 @@ impl FirehoseService { )); } let mut datas = Vec::with_capacity(records.len()); - let mut response_records = Vec::with_capacity(records.len()); - let mut failed = 0; let mut decoded_bytes = 0; for r in records { let b64 = r["Data"].as_str(); @@ -883,23 +889,30 @@ impl FirehoseService { if decoded_bytes > 4_194_304 { return Err(invalid_argument("Records exceed 4,194,304 decoded bytes")); } - datas.push(data); - response_records.push(json!({ - "RecordId": Uuid::new_v4().to_string(), - })); + datas.push(Bytes::from(data)); } _ => { - failed += 1; - response_records.push(json!({ - "ErrorCode": "InvalidArgumentException", - "ErrorMessage": "Record.Data must be a base64-encoded string", - })); + return Err(AwsServiceError::aws_error( + StatusCode::BAD_REQUEST, + "SerializationException", + "Record.Data must be a base64-encoded string", + )); } } } // One handoff after all decoding/limits: request-wide rejection cannot - // partially admit data. Empty/all-invalid requests still run preflight. - self.deliver_records(&req.account_id, &req.region, &name, datas)?; + // partially admit data. Only throughput refusal can fail individual entries. + let accepted = self + .engine + .enqueue_put(&req.account_id, &req.region, &name, datas)?; + let failed = accepted.iter().filter(|admitted| !**admitted).count(); + let response_records: Vec<_> = accepted.into_iter().map(|admitted| { + if admitted { + json!({"RecordId": Uuid::new_v4().to_string()}) + } else { + json!({"ErrorCode": "ServiceUnavailableException", "ErrorMessage": "Slow down."}) + } + }).collect(); Ok(AwsResponse::ok_json(json!({ "FailedPutCount": failed, "Encrypted": false, @@ -1138,6 +1151,8 @@ impl FirehoseService { Ok(AwsResponse::ok_json(json!({}))) } + /// Internal decoded-record handoff, unmetered like the Logs/SES adapters. + /// Public Direct PUT APIs use the engine's quota-aware admission instead. pub fn deliver_records( &self, account_id: &str, @@ -1145,7 +1160,7 @@ impl FirehoseService { stream_name: &str, datas: Vec>, ) -> Result<(), AwsServiceError> { - self.engine.enqueue_put( + self.engine.enqueue( account_id, region, stream_name, @@ -1234,8 +1249,8 @@ mod tests { } for records in [ json!([]), - json!([{"Data": "%%%"}]), - json!([{"Data": "eA=="}, {"Data": "%%%"}]), + json!([{"Data": ""}]), + json!([{"Data": "eA=="}, {"Data": ""}]), ] { let empty = records.as_array().unwrap().is_empty(); let error = svc @@ -2562,26 +2577,15 @@ mod tests { } #[test] - fn task5_empty_and_malformed_batch_entry_order() { + fn malformed_batch_is_request_wide_even_after_valid_or_empty_records() { let (svc, _) = memory_fixture(); - let response = batch( - &svc, - vec![ - json!({"Data": "%%%"}), - record(b""), - record(b"valid"), - json!({}), - ], - ) - .unwrap(); - let response: Value = serde_json::from_slice(response.body.expect_bytes()).unwrap(); - assert_eq!(response["FailedPutCount"], 2); - let entries = response["RequestResponses"].as_array().unwrap(); - assert_eq!(entries.len(), 4); - assert_eq!(entries[0]["ErrorCode"], "InvalidArgumentException"); - assert!(entries[1]["RecordId"].as_str().is_some()); - assert!(entries[2]["RecordId"].as_str().is_some()); - assert_eq!(entries[3]["ErrorCode"], "InvalidArgumentException"); + for malformed in [json!({"Data": "%%%"}), json!({}), json!({"Data": 1})] { + let error = batch(&svc, vec![record(b""), record(b"valid"), malformed]) + .err() + .unwrap(); + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + assert_eq!(error.code(), "SerializationException"); + } let error = svc .put_record_batch(&request( "PutRecordBatch", @@ -2591,7 +2595,7 @@ mod tests { )) .err() .unwrap(); - assert_eq!(error.code(), "ResourceNotFoundException"); + assert_eq!(error.code(), "SerializationException"); } #[tokio::test] @@ -2831,13 +2835,11 @@ mod tests { let (svc, s3) = memory_fixture(); let mut exact = exact; exact.push(record(&vec![2; 98_304])); - // Malformed entries are not included in the decoded-byte sum. - exact.insert(0, json!({"Data": "%%%"})); let response = batch(&svc, exact).unwrap(); assert_eq!( serde_json::from_slice::(response.body.expect_bytes()).unwrap() ["FailedPutCount"], - 1 + 0 ); assert_eq!(svc.delivery_engine().shutdown().await.delivered_batches, 1); let s3 = s3.read(); @@ -2847,7 +2849,7 @@ mod tests { } #[tokio::test] - async fn malformed_batch_admits_only_valid_bytes_and_empty_blobs_create_no_objects() { + async fn malformed_batch_admits_nothing_and_empty_blobs_create_no_objects() { let (svc, s3) = memory_fixture(); batch( &svc, @@ -2859,17 +2861,13 @@ mod tests { record(b"b"), ], ) - .unwrap(); - assert_eq!(svc.delivery_engine().shutdown().await.delivered_batches, 1); + .err() + .expect("malformed batch must reject"); + assert_eq!(svc.delivery_engine().shutdown().await.delivered_batches, 0); let objects = s3.read().get(S3_ACCOUNT).unwrap().buckets["buffered"] .objects .clone(); - assert_eq!(objects.len(), 1); - let fakecloud_persistence::BodyRef::Memory(body) = &objects.values().next().unwrap().body - else { - panic!("memory fixture") - }; - assert_eq!(body.as_ref(), b"ab"); + assert!(objects.is_empty()); let (svc, s3) = memory_fixture(); put(&svc, b"").unwrap(); let response = batch(&svc, vec![record(b""); 500]).unwrap(); @@ -2898,7 +2896,7 @@ mod tests { for result in [ put(&svc, b"late"), batch(&svc, vec![record(b"late")]), - batch(&svc, vec![json!({"Data": "%%%"})]), + batch(&svc, vec![record(b"")]), ] { let error = result.err().unwrap(); assert_eq!(error.status(), StatusCode::SERVICE_UNAVAILABLE); @@ -3277,7 +3275,7 @@ mod tests { for records in [ json!([{ "Data": "eA==" }]), - json!([{ "Data": "eA==" }, { "Data": "@@@" }]), + json!([{ "Data": "eA==" }, { "Data": "" }]), ] { assert_source_rejected( svc.put_record_batch(&request( @@ -3300,8 +3298,8 @@ mod tests { )) .unwrap(); - // Empty arrays fail the request-wide 1-500 entry check; malformed or - // empty blobs in a nonempty array must still run source admission. + // Empty arrays fail the request-wide 1-500 entry check; valid empty + // blobs in a nonempty array must still run source admission. let err = svc .put_record_batch(&request( "PutRecordBatch", @@ -3312,7 +3310,10 @@ mod tests { .err() .unwrap(); assert_eq!(err.code(), "InvalidArgumentException"); - for records in [json!([{ "Data": "@@@" }, {}]), json!([{ "Data": "" }])] { + for records in [ + json!([{ "Data": "" }, {"Data": ""}]), + json!([{ "Data": "" }]), + ] { assert_source_rejected( svc.put_record_batch(&request( "PutRecordBatch", @@ -3468,7 +3469,7 @@ mod tests { } #[tokio::test] - async fn direct_put_record_and_batch_keep_success_and_partial_failure_behavior() { + async fn direct_put_record_succeeds_and_malformed_batches_do_not_deliver() { let (svc, s3_state, disk, _tmp) = firehose_with_disk_s3("fh-bucket"); create_s3_stream(&svc, "direct-stream", "fh-bucket"); let resp = svc @@ -3478,33 +3479,19 @@ mod tests { let body: Value = serde_json::from_slice(resp.body.expect_bytes()).unwrap(); assert!(!body["RecordId"].as_str().unwrap().is_empty()); - for (records, failed) in [ - (json!([{ "Data": "eA==" }, { "Data": "@@@" }]), 1), - (json!([{ "Data": "@@@" }]), 1), + for records in [ + json!([{ "Data": "eA==" }, { "Data": "@@@" }]), + json!([{ "Data": "@@@" }]), ] { - let resp = svc + let error = svc .put_record_batch(&request( "PutRecordBatch", json!({ "DeliveryStreamName": "direct-stream", "Records": records }), )) + .err() .unwrap(); - assert_eq!(resp.status, StatusCode::OK); - let body: Value = serde_json::from_slice(resp.body.expect_bytes()).unwrap(); - assert_eq!(body["FailedPutCount"], failed); - assert_eq!( - body["RequestResponses"].as_array().unwrap().len(), - records.as_array().unwrap().len() - ); - if records.as_array().unwrap().len() == 2 { - assert!(!body["RequestResponses"][0]["RecordId"] - .as_str() - .unwrap() - .is_empty()); - assert_eq!( - body["RequestResponses"][1]["ErrorCode"], - "InvalidArgumentException" - ); - } + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + assert_eq!(error.code(), "SerializationException"); } assert_eq!(svc.delivery_engine().shutdown().await.delivered_batches, 1); let s3 = s3_state.read(); @@ -3522,7 +3509,7 @@ mod tests { .next() .unwrap() .size, - 2 + 1 ); } diff --git a/crates/fakecloud-firehose/src/throughput.rs b/crates/fakecloud-firehose/src/throughput.rs index 00a75d081..1eb15dd46 100644 --- a/crates/fakecloud-firehose/src/throughput.rs +++ b/crates/fakecloud-firehose/src/throughput.rs @@ -1,4 +1,3 @@ -use std::time::Duration; use tokio::time::Instant; #[derive(Clone, Copy, Default)] @@ -8,92 +7,132 @@ pub(crate) struct ThroughputLimits { pub mib: Option, } -/// Stream-local, one-second ingestion accounting; callers supply monotonic time. -/// -/// The first successful admission, and a successful admission at or after the -/// boundary, starts a fresh window at `now`. -/// Windows do not accrue idle credits, but full capacity immediately before and -/// after a boundary can produce a short burst of twice the per-window allowance. +/// Stream-local admission policy, not a model of AWS's unpublished algorithm. +/// Request capacity resets each second. Record/byte excess carries forward: +/// a positive remaining allowance admits one whole record, even if it crosses +/// the boundary. Elapsed whole seconds repay debt but never accrue idle credit. pub(crate) struct ThroughputWindow { started_at: Instant, + active: bool, + window: u64, requests: u64, - records: usize, - bytes: usize, + records: Debt, + bytes: Debt, } impl ThroughputWindow { pub(crate) fn new(now: Instant) -> Self { Self { started_at: now, + active: false, + window: 0, requests: 0, - records: 0, - bytes: 0, + records: Debt::default(), + bytes: Debt::default(), } } - pub(crate) fn try_admit( - &mut self, - now: Instant, - limits: ThroughputLimits, - records: usize, - bytes: usize, - ) -> bool { - if limits.requests.is_none() && limits.records.is_none() && limits.mib.is_none() { - return true; + /// Charge the whole call before considering individual records. + pub(crate) fn try_request(&mut self, now: Instant, limits: ThroughputLimits) -> bool { + if self.active { + let window = now.duration_since(self.started_at).as_secs(); + if window > self.window { + self.window = window; + self.requests = 0; + } } - - let rollover = now.duration_since(self.started_at) >= Duration::from_secs(1); - let (used_requests, used_records, used_bytes) = if rollover { - (0_u64, 0_usize, 0_usize) - } else { - (self.requests, self.records, self.bytes) - }; - // Track every dimension when any limit is enabled, including dimensions - // that might become enabled by a later same-window quota change. - let Some(requests) = used_requests.checked_add(1) else { + let Some(requests) = self.requests.checked_add(1) else { return false; }; - let Some(records) = used_records.checked_add(records) else { - return false; - }; - let Some(bytes) = used_bytes.checked_add(bytes) else { - return false; - }; - - // Limits are finite and non-negative (validated by quota setters). Scale - // usage, not limits: huge finite MiB limits must not overflow a capacity. if limits .requests .is_some_and(|limit| count_as_f64_ceil(requests as u128) > limit) - || limits - .records - .is_some_and(|limit| count_as_f64_ceil(records as u128) > limit) - || limits - .mib - .is_some_and(|limit| count_as_f64_ceil(bytes as u128) / 1_048_576.0 > limit) { return false; } - - // Commit only after every check passes, including rollover. Rejections - // must not reset either the old counters or the old window's anchor. - if rollover || self.requests == 0 { + if !self.active { self.started_at = now; + self.active = true; } self.requests = requests; - self.records = records; - self.bytes = bytes; + true + } + + /// Rejected entries debit neither dimension, leaving room for later entries + /// (notably empty records when the byte allowance is exhausted). + pub(crate) fn try_record(&mut self, limits: ThroughputLimits, bytes: usize) -> bool { + if !self.records.has_room(self.window, limits.records, 1.0) + || (bytes > 0 && !self.bytes.has_room(self.window, limits.mib, 1_048_576.0)) + { + return false; + } + // Keep accounting even for currently ignored dimensions when another + // quota is enabled, so a live quota change does not erase usage. + self.records.charge(self.window, limits.records, 1.0, 1.0); + if bytes > 0 { + self.bytes.charge( + self.window, + limits.mib, + 1_048_576.0, + count_as_f64_ceil(bytes as u128), + ); + } true } } -/// Smallest representable f64 not below the count. An ordinary cast can round -/// down above 2^53 and incorrectly admit one unit over a representable limit. -/// Dividing this by the power-of-two bytes-per-MiB scale preserves that bound. +/// Keep the last charge as an immutable repayment anchor. Rejected retries and +/// zero-byte records must not repeatedly round byte debt. Compare total usage +/// with the cumulative allowance before subtracting: e.g. 0.5 MiB against five +/// 0.1 MiB allowances is exactly full, not a tiny positive burst credit. +#[derive(Default)] +struct Debt { + used: f64, + window: u64, +} + +impl Debt { + fn has_room(&self, window: u64, limit: Option, scale: f64) -> bool { + limit.is_none_or(|limit| self.used / scale < limit * ((window - self.window) as f64 + 1.0)) + } + + fn charge(&mut self, window: u64, limit: Option, scale: f64, cost: f64) { + let seconds = window - self.window; + let used = if seconds == 0 { + self.used + } else if let Some(limit) = limit { + let allowance = limit * seconds as f64; + // Divide before multiplying to avoid overflow at huge MiB limits. + if self.used / scale <= allowance { + 0.0 + } else { + self.used - allowance * scale + } + } else { + 0.0 + }; + self.used = add_ceil(used, cost); + self.window = window; + } +} + +/// Round an addition upward when its exact result lies between floats. Usage +/// must not round down and accidentally create room above large finite quotas. +fn add_ceil(a: f64, b: f64) -> f64 { + let sum = a + b; + let b_rounded = sum - a; + let error = (a - (sum - b_rounded)) + (b - b_rounded); + if error > 0.0 { + sum.next_up() + } else { + sum + } +} + fn count_as_f64_ceil(count: u128) -> f64 { let rounded = count as f64; if (rounded as u128) < count { - f64::from_bits(rounded.to_bits() + 1) + rounded.next_up() } else { rounded } @@ -104,411 +143,204 @@ mod tests { use super::*; use std::time::Duration; - fn usage(window: &ThroughputWindow) -> (Instant, u64, usize, usize) { - ( - window.started_at, - window.requests, - window.records, - window.bytes, - ) - } - #[test] - fn request_capacity_recovers_at_one_second() { + fn request_limit_is_hard_and_first_success_anchors_window() { let now = Instant::now(); + let mut w = ThroughputWindow::new(now); let limits = ThroughputLimits { requests: Some(1.0), ..Default::default() }; - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, 1, 3)); - assert!(!window.try_admit(now, limits, 1, 3)); - assert!(!window.try_admit(now + Duration::from_millis(999), limits, 1, 3)); - assert!(window.try_admit(now + Duration::from_secs(1), limits, 1, 3)); + assert!(!w.try_request( + now, + ThroughputLimits { + requests: Some(0.0), + ..limits + } + )); + assert!(w.try_request(now + Duration::from_millis(500), limits)); + assert!(!w.try_request(now + Duration::from_secs(1), limits)); + assert!(w.try_request(now + Duration::from_millis(1500), limits)); + assert!(!w.try_request(now + Duration::from_millis(2499), limits)); + assert!(w.try_request(now + Duration::from_millis(2500), limits)); } #[test] - fn first_success_anchors_window_after_initial_rejection() { + fn large_record_burst_carries_debt_across_multiple_windows() { let now = Instant::now(); let limits = ThroughputLimits { - records: Some(1.0), + mib: Some(2.0 / 1_048_576.0), ..Default::default() }; - let mut window = ThroughputWindow::new(now); - assert!(!window.try_admit(now, limits, 2, 1)); - assert!(window.try_admit(now + Duration::from_millis(500), limits, 1, 1)); - assert!(!window.try_admit(now + Duration::from_secs(1), limits, 1, 1)); - assert!(window.try_admit(now + Duration::from_millis(1_500), limits, 1, 1)); + let mut w = ThroughputWindow::new(now); + assert!(w.try_request(now, limits)); + assert!(w.try_record(limits, 5)); + assert!(!w.try_record(limits, 1)); + assert!(w.try_record(limits, 0)); + assert!(w.try_request(now + Duration::from_secs(1), limits)); + assert!(!w.try_record(limits, 1)); + assert!(w.try_request(now + Duration::from_secs(2), limits)); + assert!(w.try_record(limits, 1)); + assert!(!w.try_record(limits, 1)); } #[test] - fn records_accept_exact_limit_and_reject_one_over() { + fn skipped_seconds_repay_debt_without_idle_credit() { let now = Instant::now(); let limits = ThroughputLimits { - records: Some(3.0), + records: Some(1.0), + mib: Some(1.0 / 1_048_576.0), ..Default::default() }; - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, 2, 10)); - assert!(window.try_admit(now, limits, 1, 10)); - assert!(!window.try_admit(now, limits, 1, 0)); - assert!(window.try_admit(now + Duration::from_secs(1), limits, 3, 10)); + let mut w = ThroughputWindow::new(now); + assert!(w.try_request(now, limits)); + assert!(w.try_record(limits, 5)); + assert!(w.try_request(now + Duration::from_secs(100), limits)); + assert!(w.try_record(limits, 1)); + assert!(!w.try_record(limits, 1)); } #[test] - fn fractional_mib_accepts_exact_bytes_and_rejects_one_over() { + fn fractional_positive_allowances_admit_one_record_then_repay() { let now = Instant::now(); let limits = ThroughputLimits { - mib: Some(0.5), + records: Some(0.5), + mib: Some(0.25 / 1_048_576.0), ..Default::default() }; - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, 1, 262_144)); - assert!(window.try_admit(now, limits, 1, 262_144)); - assert!(!window.try_admit(now, limits, 1, 1)); - assert!(window.try_admit(now + Duration::from_secs(1), limits, 1, 524_288)); - } - - #[test] - fn rejected_bytes_leave_capacity_for_smaller_request() { - let now = Instant::now(); - let limits = ThroughputLimits { - requests: Some(2.0), - records: Some(3.0), - mib: Some(0.5), - }; - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, 1, 262_144)); - assert!(!window.try_admit(now, limits, 2, 262_145)); - assert!(window.try_admit(now, limits, 2, 262_144)); - assert!(!window.try_admit(now, limits, 0, 0)); - } - - #[test] - fn each_enabled_dimension_can_reject_without_consuming_other_capacity() { - let now = Instant::now(); - let generous = ThroughputLimits { - requests: Some(3.0), - records: Some(3.0), - mib: Some(1.0), - }; - for strict in [ - ThroughputLimits { - requests: Some(0.0), - ..generous - }, - ThroughputLimits { - records: Some(0.0), - ..generous - }, - ThroughputLimits { - mib: Some(0.0), - ..generous - }, - ] { - let mut window = ThroughputWindow::new(now); - assert!(!window.try_admit(now, strict, 1, 1)); - assert!(window.try_admit(now, generous, 3, 1_048_576)); + let mut w = ThroughputWindow::new(now); + assert!(w.try_request(now, limits)); + assert!(w.try_record(limits, 1)); + for seconds in [1, 2, 3] { + assert!(w.try_request(now + Duration::from_secs(seconds), limits)); + assert!(!w.try_record(limits, 1)); } + assert!(w.try_request(now + Duration::from_secs(4), limits)); + assert!(w.try_record(limits, 1)); } #[test] - fn empty_payload_still_consumes_a_request() { + fn decimal_debt_recovery_does_not_depend_on_rejected_retry_frequency() { let now = Instant::now(); let limits = ThroughputLimits { - requests: Some(1.0), - records: Some(0.0), - mib: Some(0.0), - }; - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, 0, 0)); - assert!(!window.try_admit(now, limits, 0, 0)); - } - - #[test] - fn zero_record_and_byte_limits_allow_only_zero_usage() { - let now = Instant::now(); - let limits = ThroughputLimits { - records: Some(0.0), - mib: Some(0.0), + mib: Some(0.1), ..Default::default() }; - let mut window = ThroughputWindow::new(now); - assert!(!window.try_admit(now, limits, 1, 0)); - assert!(!window.try_admit(now, limits, 0, 1)); - assert!(window.try_admit(now, limits, 0, 0)); + let mut frequent = ThroughputWindow::new(now); + let mut delayed = ThroughputWindow::new(now); + for window in [&mut frequent, &mut delayed] { + assert!(window.try_request(now, limits)); + assert!(window.try_record(limits, 524_288)); + } + for second in 1..4 { + assert!(frequent.try_request(now + Duration::from_secs(second), limits)); + assert!(!frequent.try_record(limits, 1)); + } + for window in [&mut frequent, &mut delayed] { + assert!(window.try_request(now + Duration::from_secs(4), limits)); + // The entire fifth allowance is still needed to repay the burst. + assert!(!window.try_record(limits, 1)); + assert!(window.try_request(now + Duration::from_secs(5), limits)); + assert!(window.try_record(limits, 1)); + } } #[test] - fn zero_request_limit_rejects_even_empty_payloads() { + fn zero_limits_and_rejection_do_not_debit_other_dimensions() { let now = Instant::now(); + let mut w = ThroughputWindow::new(now); let limits = ThroughputLimits { - requests: Some(0.0), + records: Some(1.0), + mib: Some(0.0), ..Default::default() }; - let mut window = ThroughputWindow::new(now); - assert!(!window.try_admit(now, limits, 0, 0)); + assert!(w.try_request(now, limits)); + assert!(!w.try_record(limits, 1)); + assert!(w.try_record(limits, 0)); + assert!(!w.try_record(limits, 0)); + let mut w = ThroughputWindow::new(now); + assert!(!w.try_record( + ThroughputLimits { + records: Some(0.0), + ..Default::default() + }, + 0 + )); } #[test] - fn disabled_limits_bypass_without_changing_usage_or_window() { + fn live_changes_preserve_usage_and_track_ignored_dimensions() { let now = Instant::now(); + let mut w = ThroughputWindow::new(now); let limits = ThroughputLimits { - requests: Some(1.0), + requests: Some(10.0), ..Default::default() }; - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, 1, 1)); - let before = usage(&window); - assert!(window.try_admit(now, ThroughputLimits::default(), usize::MAX, usize::MAX)); - assert!(window.try_admit( - now + Duration::from_secs(2), - ThroughputLimits::default(), - usize::MAX, - usize::MAX - )); - assert_eq!(usage(&window), before); - assert!(!window.try_admit(now, limits, 0, 0)); - } - - #[test] - fn ignored_dimensions_are_counted_when_another_dimension_is_enabled() { - let now = Instant::now(); - for initially_enabled in [ + assert!(w.try_request(now, limits)); + assert!(w.try_record(limits, 5)); + assert!(!w.try_record( ThroughputLimits { - requests: Some(10.0), - ..Default::default() + mib: Some(5.0 / 1_048_576.0), + ..limits }, + 1 + )); + assert!(!w.try_record( ThroughputLimits { - records: Some(10.0), - ..Default::default() + records: Some(1.0), + ..limits }, + 0 + )); + assert!(w.try_record( ThroughputLimits { - mib: Some(10.0), - ..Default::default() + records: Some(2.0), + mib: Some(6.0 / 1_048_576.0), + ..limits }, - ] { - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, initially_enabled, 3, 524_288)); - assert!(!window.try_admit( - now, - ThroughputLimits { - requests: Some(1.0), - ..Default::default() - }, - 0, - 0 - )); - assert!(!window.try_admit( - now, - ThroughputLimits { - records: Some(3.0), - ..Default::default() - }, - 1, - 0 - )); - assert!(!window.try_admit( - now, - ThroughputLimits { - mib: Some(0.5), - ..Default::default() - }, - 0, - 1 - )); - } + 1 + )); } #[test] - fn runtime_limit_increases_and_decreases_preserve_same_window_usage() { + fn rejected_requests_do_not_charge_records_or_move_boundary() { let now = Instant::now(); - let mut window = ThroughputWindow::new(now); + let mut w = ThroughputWindow::new(now); let limits = ThroughputLimits { requests: Some(1.0), - records: Some(2.0), - mib: Some(0.5), - }; - assert!(window.try_admit(now, limits, 2, 524_288)); - assert!(!window.try_admit(now, limits, 0, 0)); - let raised = ThroughputLimits { - requests: Some(3.0), - records: Some(4.0), - mib: Some(1.0), - }; - assert!(window.try_admit(now, raised, 1, 262_144)); - assert!(!window.try_admit(now, limits, 0, 0)); - assert!(window.try_admit(now, raised, 1, 262_144)); - assert!(!window.try_admit(now, raised, 0, 0)); - } - - #[test] - fn rejected_rollover_preserves_usage_and_success_anchors_at_now() { - let now = Instant::now(); - let limits = ThroughputLimits { records: Some(1.0), ..Default::default() }; - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, 1, 1)); - let before = usage(&window); - assert!(!window.try_admit(now + Duration::from_secs(1), limits, 2, 1)); - assert_eq!(usage(&window), before); - assert!(window.try_admit(now + Duration::from_millis(1_100), limits, 1, 1)); - assert!(!window.try_admit(now + Duration::from_secs(2), limits, 1, 1)); - assert!(window.try_admit(now + Duration::from_millis(2_100), limits, 1, 1)); - } - - #[test] - fn idle_time_does_not_accumulate_credits() { - let now = Instant::now(); - let limits = ThroughputLimits { - requests: Some(1.0), - ..Default::default() - }; - let mut window = ThroughputWindow::new(now); - let later = now + Duration::from_secs(100); - assert!(window.try_admit(later, limits, 1, 1)); - assert!(!window.try_admit(later, limits, 1, 1)); - assert!(!window.try_admit(later + Duration::from_millis(999), limits, 1, 1)); - assert!(window.try_admit(later + Duration::from_secs(1), limits, 1, 1)); - } - - #[test] - fn windows_have_independent_usage() { - let now = Instant::now(); - let limits = ThroughputLimits { - requests: Some(1.0), - ..Default::default() - }; - let mut first = ThroughputWindow::new(now); - let mut second = ThroughputWindow::new(now); - assert!(first.try_admit(now, limits, 1, 1)); - assert!(!first.try_admit(now, limits, 1, 1)); - assert!(second.try_admit(now, limits, 1, 1)); - } - - #[test] - fn fractional_count_limits_do_not_round_up() { - let now = Instant::now(); - for limits in [ - ThroughputLimits { - requests: Some(0.5), - ..Default::default() - }, - ThroughputLimits { - records: Some(0.5), - ..Default::default() - }, - ] { - let mut window = ThroughputWindow::new(now); - assert!(!window.try_admit(now, limits, 1, 0)); - } - } - - #[test] - fn sub_byte_mib_limit_does_not_allow_a_whole_byte() { - let now = Instant::now(); - let mut window = ThroughputWindow::new(now); - let limits = ThroughputLimits { - mib: Some(f64::MIN_POSITIVE), - ..Default::default() - }; - assert!(!window.try_admit(now, limits, 0, 1)); - assert!(window.try_admit(now, limits, 0, 0)); + assert!(w.try_request(now, limits)); + assert!(!w.try_request(now + Duration::from_millis(999), limits)); + assert!(w.try_record(limits, 0)); + assert!(w.try_request(now + Duration::from_secs(1), limits)); + assert!(w.try_record(limits, 0)); } #[test] - fn huge_finite_limits_do_not_overflow_capacity_calculations() { + fn huge_finite_limits_and_large_counts_do_not_overflow_or_round_down() { let now = Instant::now(); + let mut w = ThroughputWindow::new(now); let limits = ThroughputLimits { requests: Some(f64::MAX), records: Some(f64::MAX), mib: Some(f64::MAX), }; - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, usize::MAX, usize::MAX)); - assert!(!window.try_admit(now, limits, 1, 0)); - } - - #[test] - fn count_and_byte_overflow_reject_atomically_even_when_dimension_is_ignored() { - let now = Instant::now(); - let limits = ThroughputLimits { - requests: Some(10.0), - ..Default::default() - }; - for (records, bytes, extra_records, extra_bytes) in - [(usize::MAX, 0, 1, 0), (0, usize::MAX, 0, 1)] - { - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, records, bytes)); - let before = usage(&window); - assert!(!window.try_admit(now, limits, extra_records, extra_bytes)); - assert_eq!(usage(&window), before); - assert!(window.try_admit(now, limits, 0, 0)); - assert!(window.try_admit( - now + Duration::from_secs(1), - limits, - extra_records, - extra_bytes - )); - } - } - - #[test] - fn request_counter_overflow_rejects_without_mutation() { - let now = Instant::now(); - let mut window = ThroughputWindow::new(now); - window.requests = u64::MAX; - let limits = ThroughputLimits { - records: Some(1.0), - ..Default::default() - }; - let before = usage(&window); - assert!(!window.try_admit(now, limits, 1, 1)); - assert_eq!(usage(&window), before); - assert!(window.try_admit(now + Duration::from_secs(1), limits, 1, 1)); - } - - #[test] - fn large_request_count_is_not_rounded_down_to_the_limit() { - let now = Instant::now(); - let mut window = ThroughputWindow::new(now); - window.requests = 9_007_199_254_740_992; - let limits = ThroughputLimits { - requests: Some(9_007_199_254_740_992.0), - ..Default::default() - }; - assert!(!window.try_admit(now, limits, 0, 0)); - assert!(window.try_admit( - now, + assert!(w.try_request(now, limits)); + assert!(w.try_record(limits, usize::MAX)); + assert!(w.try_request(now + Duration::from_secs(100), limits)); + assert!(w.try_record(limits, 1)); + w.requests = 9_007_199_254_740_992; + assert!(!w.try_request( + now + Duration::from_secs(100), ThroughputLimits { - requests: Some(9_007_199_254_740_994.0), - ..Default::default() - }, - 0, - 0 + requests: Some(9_007_199_254_740_992.0), + ..limits + } )); - } - - #[test] - #[cfg(target_pointer_width = "64")] - fn large_record_and_byte_counts_are_not_rounded_down_to_limits() { - let now = Instant::now(); - for limits in [ - ThroughputLimits { - records: Some(9_007_199_254_740_992.0), - ..Default::default() - }, - ThroughputLimits { - mib: Some(8_589_934_592.0), - ..Default::default() - }, - ] { - let mut window = ThroughputWindow::new(now); - assert!(window.try_admit(now, limits, 9_007_199_254_740_992, 9_007_199_254_740_992)); - assert!(!window.try_admit(now, limits, 1, 1)); - } + w.requests = u64::MAX; + assert!(!w.try_request(now + Duration::from_secs(100), limits)); + assert!(add_ceil(9_007_199_254_740_992.0, 1.0) > 9_007_199_254_740_992.0); } } diff --git a/website/content/docs/services/firehose.md b/website/content/docs/services/firehose.md index 684b7633a..33c3f38a1 100644 --- a/website/content/docs/services/firehose.md +++ b/website/content/docs/services/firehose.md @@ -19,7 +19,7 @@ fakecloud implements Amazon Data Firehose's JSON 1.1 control plane and buffered Incomplete pairs, non-integers, and out-of-range values return `InvalidArgumentException`. CloudFormation also accepts quoted integers such as `"60"`, because `Ref` parameters resolve to strings; it coerces them, and every other quoted modeled boolean or number in the destination block (including inside `S3BackupConfiguration`), before validating. A value that does not match its modeled type makes the resource fail, naming the member's path: a quoted string that does not parse exactly (such as `IntervalInSeconds: "soon"`, or `" 60 "` with surrounding whitespace), an integer outside the model's 32-bit range, or a JSON value of the wrong type (such as a number for `Enabled`). Interval zero makes nonempty input eligible at admission; writing is still asynchronous, so poll S3 for completion rather than assuming an acknowledgment means the object exists. - **Destination strings**: the AWS model's length limits apply: `Prefix` and `ErrorOutputPrefix` allow at most 1,024 characters, `CustomTimeZone` at most 50, `RoleARN` 1-512, and `BucketARN` 1-2,048. ARN and time-zone patterns are not enforced here. Other values return `InvalidArgumentException` from `CreateDeliveryStream` and `UpdateDestination`, and make a CloudFormation resource fail. - **Records**: `PutRecord` / `PutRecordBatch` assign per-record `RecordId`s after supported S3 admission. Binary bytes and producer-supplied delimiters are preserved exactly before compression; no separators are added unless the `AppendDelimiterToRecord` processor is enabled. Without it, a delivery window holding only empty records produces no object. Public API and internal Logs/SES delivery use the same buffering engine. - Each API record is limited to 1,024,000 decoded bytes; a batch contains 1-500 records and at most 4,194,304 decoded bytes. Request-wide limit violations reject the whole call before admission. Invalid-base64 batch entries receive individual errors, while valid entries can still be admitted. + Each API record is limited to 1,024,000 decoded bytes; a batch contains 1-500 records and at most 4,194,304 decoded bytes. Request-wide limit violations reject the whole call before admission. Malformed base64 in either API rejects the whole call with HTTP 400 `SerializationException`; no entries are admitted. Opt-in throughput limits can instead produce partial batch failures, as described below. - **New line delimiter**: when `ProcessingConfiguration` has `Enabled: true` and every entry in its `Processors` is `AppendDelimiterToRecord`, a newline (`\n`) is written after every record, including the last one and empty records, before any compression; a delivery window holding only empty records becomes an object of newlines. This applies to API deliveries, to internal CloudWatch Logs and SES deliveries, and to streams created or updated through CloudFormation, which store `ProcessingConfiguration` like `CreateDeliveryStream`. The optional `Delimiter` parameter is stored but not interpreted: AWS documents only a new line delimiter. Appended newlines do not count toward the size threshold. Lambda and other processors are not run; enabled processing that includes them, or lists no processors, rejects record admission with HTTP 400 `InvalidArgumentException` naming the unsupported setting (see [Caveats](#caveats)). - **S3 output**: static prefixes, UTC date hierarchy, stream/version/time/UUID object names, and `FileExtension` overrides. Objects have `Content-Type: application/octet-stream` and the encoded body's MD5 as their `ETag`. GZIP output has `Content-Encoding: gzip`; Snappy output has `Content-Encoding: snappy-java`; uncompressed output has no content encoding. Completed objects' bytes and metadata survive persistent-mode restart, including internal adapter delivery. Bucket versioning, default Object Lock retention, bucket-default encryption (including SSE-KMS), replication, and S3 event notifications apply through the shared S3 write path as for ordinary S3 writes, including for compressed objects. `EncryptionConfiguration.KMSEncryptionConfig` writes objects with SSE-KMS under the configured key (see [Caveats](#caveats)). Pending buffers and retry queues are process-local and are not persisted. - **Tags**: `ListTagsForDeliveryStream` / `TagDeliveryStream` / `UntagDeliveryStream`. Keyed by stream ARN. @@ -27,7 +27,7 @@ fakecloud implements Amazon Data Firehose's JSON 1.1 control plane and buffered ## Buffering and destination updates -Size accounting uses decoded record bytes, before compression or base64 transport encoding; newlines added by `AppendDelimiterToRecord` are not counted. Delivery becomes eligible at **greater than or equal to** the configured threshold. All valid records in one API call are admitted together: a batch crossing the threshold stays whole, so the resulting object can overshoot the hint. Highly compressible input can trigger delivery even when the encoded object is much smaller than the threshold. +Size accounting uses decoded record bytes, before compression or base64 transport encoding; newlines added by `AppendDelimiterToRecord` are not counted. Delivery becomes eligible at **greater than or equal to** the configured threshold. All accepted records in one API call enter the buffer together: a batch crossing the threshold stays whole, so the resulting object can overshoot the hint. Entries refused by an opt-in throughput limit never enter the buffer. Highly compressible input can trigger delivery even when the encoded object is much smaller than the threshold. The interval starts with the oldest arrival in a new buffer. After that buffer is detached for delivery, subsequent records start a fresh size/time window. Buffers are isolated by account, region, and delivery-stream instance. @@ -45,13 +45,15 @@ The existing [Service Quotas controls](/docs/services/servicequotas/#enforcement | `L-17D4EADE` | Records | 500,000 | 100,000 | | `L-937C879B` | MiB of decoded record data | 5 | 1 | -One MiB is 1,048,576 bytes. Both public operations share all three budgets: a batch costs one request, plus its valid records and their decoded bytes. Invalid-base64 batch entries do not consume records or bytes; even an all-invalid batch consumes one request. Producer-supplied delimiters count as input bytes; base64 transport encoding, output-added newlines, and compression do not affect the byte budget. +One MiB is 1,048,576 bytes. Both public operations share all three budgets: a validated batch costs one request, plus its accepted records and their decoded bytes. A batch whose entries all fail throughput admission still consumes one request. Malformed base64 and request-wide validation failures consume no capacity. Producer-supplied delimiters count as input bytes; base64 transport encoding, output-added newlines, and compression do not affect the byte budget. -Applied values are configured per account and region, but consumption is **independent for each delivery-stream instance**, not pooled across the region. Streams in the same account and region each receive the full configured budget. Destination updates, buffer flushes, and delivery retries do not reset or consume ingestion capacity. Deleting and recreating a stream starts a new budget; usage windows are process-local and are not persisted. +Applied values are configured per account and region, but consumption is **independent for each delivery-stream instance**, not pooled across the region. Streams in the same account and region each receive the full configured budget. Destination updates, buffer flushes, and delivery retries do not reset or consume ingestion capacity. Deleting and recreating a stream starts a new budget. Usage and burst debt are process-local and are not persisted; applied values and account enforcement overrides survive persistent-mode restarts. Server-wide enforcement settings are restored from startup flags on restart or reset, as with other Service Quotas controls. ### Enable and change limits -**Enforcement is off by default.** Use `--enforce-quotas` globally, `--enforce-quota firehose/L-937C879B` for one quota, or the existing introspection API and its Service Quotas SDK wrappers. `--ignore-quota` and per-account overrides retain their existing precedence: account override, then server-wide override, then the global switch. The corresponding `FAKECLOUD_ENFORCE_QUOTAS`, `FAKECLOUD_ENFORCE_QUOTA`, and `FAKECLOUD_IGNORE_QUOTA` environment variables are also supported. +**Enforcement requires both an explicit applied value and an enabled enforcement switch.** AWS defaults are reported but never imposed, even with `--enforce-quotas`; introspection reports `enforced: false` and `enforcementSource: "auto_scaling_default"` until an explicit account/region value exists. This is a local convenience, not a simulation of AWS's automatic scaling latency. + +Use `--enforce-quotas` globally, `--enforce-quota firehose/L-937C879B` for one quota, or the existing introspection API and its Service Quotas SDK wrappers. `--ignore-quota` and per-account overrides retain their existing precedence: account override, then server-wide override, then the global switch. The corresponding `FAKECLOUD_ENFORCE_QUOTAS`, `FAKECLOUD_ENFORCE_QUOTA`, and `FAKECLOUD_IGNORE_QUOTA` environment variables are also supported. For a running local server, set a 0.5 MiB/second budget for every stream in account `000000000000`, region `us-east-1`, and enforce that quota for the account: @@ -66,21 +68,21 @@ Use the account and region of your producer's AWS client. Omitting `accountId` o Setting `value` alone leaves enforcement unchanged: on a default server it **does not turn throttling on**. `enforce: false` ignores the quota, `enforce: null` clears the override, and omitting `enforce` preserves it. No new Firehose-specific administration endpoint is required. -The local setter accepts any finite, nonnegative value, including values below the AWS default, zero, and fractional MiB. Fractional request/record limits do not round up to a whole request/record. A zero request limit refuses every metered call; zero records permits no valid records, and zero MiB permits only zero decoded bytes (so an empty record can pass the byte check). All enabled dimensions must allow the call. +The local setter accepts any finite, nonnegative value, including values below the AWS default, zero, and fractional MiB. Fractional request limits do not round up to a whole request. Positive record/byte limits allow a boundary-crossing record, with excess carried forward. A zero request limit refuses every metered call; zero records permits no records, and zero MiB permits only zero decoded bytes (so an empty record can pass the byte check). All enabled dimensions must allow each record. Limits are read on each call. Raising a value applies to the next call without clearing usage already consumed in the current window; lowering it can block further admission until replenishment. Previously accepted buffers, delivery retries, and shutdown drains are unaffected. These Firehose quotas keep AWS's non-adjustable Service Quotas metadata: `RequestServiceQuotaIncrease` still refuses them. The local setter is a testing control, not a simulated AWS Support quota-increase workflow. -### Window timing and request errors +### Burst recovery and partial failures -fakecloud approximates throughput with fixed, monotonic **one-second windows**. The first successful metered admission anchors a window; the first successful admission at or after its one-second boundary starts the next window at that call's time. Rejection does not consume capacity or move the anchor. Idle time accrues no credits. This is not a token bucket or a rolling-window guarantee: full allowances immediately before and after a boundary can produce a short burst of twice the per-window budget. +fakecloud uses monotonic **one-second accounting windows**, anchored by the first metered request admitted under the request quota. Request capacity resets at each boundary. Record and byte usage instead subtract one allowance for each elapsed second, retaining any excess and never accumulating idle credits. Rejected requests do not move the boundary; rejected records consume neither records nor bytes. -There is no automatic scaling: configured capacity remains fixed until changed locally. A single record or batch exceeding an entire configured window cannot succeed merely by backing off and retrying; reduce its size or raise the relevant limit. +Records are considered in input order. If both record and byte budgets have room, the whole record is accepted, even if it crosses a positive allowance. Later records wait for that burst to be repaid. For example, a five-byte record under a two-byte/second limit is accepted initially; one second later usage is three bytes and nonempty records are refused; after two seconds usage is one byte and another byte can be accepted. An empty record needs record capacity but no byte capacity, including while byte debt remains. Configured capacity does not automatically scale. -If any enabled budget would be exceeded, the **whole call returns HTTP 500 `ServiceUnavailableException`**, before any records enter a buffer or trigger delivery. A rejected batch is not partially admitted and does not receive an HTTP 200 per-entry throughput-failure response. Existing input validation and unsupported-destination errors still precede quota consumption. +**Request-rate exhaustion** rejects the whole call with HTTP 500 `ServiceUnavailableException`, message `Slow down.`, before any records enter a buffer. **Record/byte exhaustion** gives `PutRecord` the same error, but `PutRecordBatch` returns HTTP 200 with ordered `RequestResponses`: accepted entries have `RecordId`, refused entries have `ErrorCode: "ServiceUnavailableException"` and `ErrorMessage: "Slow down."` without a record ID. `FailedPutCount` counts refused entries, even when every entry fails. Existing input validation and unsupported-destination errors precede quota consumption. -With the official AWS SDK for Rust, request retries can delay the result or succeed after a window replenishes. If retryable failures continue until the configured retry attempts are exhausted, `.send().await` returns `Err`, not a successful output containing `FailedPutCount`. See the [Rust SDK retry configuration](https://docs.rs/aws-sdk-firehose/latest/aws_sdk_firehose/config/retry/struct.RetryConfig.html). +With the official AWS SDK for Rust, whole-request retries can delay the result or succeed after capacity replenishes. If retryable whole-call failures continue until the configured retry attempts are exhausted, `.send().await` returns `Err`. See the [Rust SDK retry configuration](https://docs.rs/aws-sdk-firehose/latest/aws_sdk_firehose/config/retry/struct.RetryConfig.html). -By contrast, an HTTP 200 `PutRecordBatch` response can still contain existing per-entry validation failures, such as invalid base64, with results in input order and valid entries admitted. Inspect `FailedPutCount` and `RequestResponses`: the Rust SDK does **not** automatically resubmit failed entries from a successful HTTP 200 batch response. Request retries and per-entry results are separate contracts; neither an API acknowledgment nor successful retry means S3 delivery is durable. +Inspect `FailedPutCount` and `RequestResponses`: the Rust SDK does **not** automatically resubmit failed entries from a successful HTTP 200 batch response. Back off and explicitly resubmit only failed entries, not records that already received IDs. Request retries and per-entry results are separate contracts; neither an API acknowledgment nor successful retry means S3 delivery is durable. This increment meters only the public Direct PUT APIs, including their configuration-only non-S3 acknowledgments. Internal CloudWatch Logs and SES handoffs still share the buffering engine but are explicitly **unmetered** by these throughput controls. `KinesisStreamAsSource` streams still reject Direct PUT calls; Kinesis-source polling is not implemented. @@ -88,7 +90,7 @@ This increment meters only the public Direct PUT APIs, including their configura The [Firehose quotas guide](https://docs.aws.amazon.com/firehose/latest/dev/limits.html) and [regional quota reference](https://docs.aws.amazon.com/general/latest/gr/fh.html) define the regional defaults and shared per-stream Direct PUT limits. AWS describes proportional quota increases and automatic scaling with unspecified timing; fakecloud does not reproduce either workflow or AWS's internal admission algorithm. Local controls can change each dimension independently. -The [`PutRecord` reference](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecord.html) documents HTTP 500 `ServiceUnavailableException`, but its introductory 5,000-records/second figure conflicts with the 500,000 figure in the quota references for the three high-throughput regions. The table above follows the quota references, not that introductory number. The [`PutRecordBatch` reference](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecordBatch.html) distinguishes whole-call errors from ordered per-record results; it does not specify the exact threshold-allocation algorithm. The fixed windows and whole-call local throughput rejection described here are explicit local approximations. +The [`PutRecord` reference](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecord.html) documents HTTP 500 `ServiceUnavailableException`, but its introductory 5,000-records/second figure conflicts with the 500,000 figure in the quota references for the three high-throughput regions. The table above follows the quota references, not that introductory number. The [`PutRecordBatch` reference](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecordBatch.html) distinguishes whole-call errors from ordered per-record results; it does not specify the exact threshold-allocation algorithm. One-second accounting, burst carry-over, and the dimension-to-error mapping here are explicit local testing policies, not claims about AWS's unpublished throttling algorithm. ## S3 object bytes and names diff --git a/website/content/docs/services/servicequotas.md b/website/content/docs/services/servicequotas.md index a25bd71af..043992952 100644 --- a/website/content/docs/services/servicequotas.md +++ b/website/content/docs/services/servicequotas.md @@ -164,6 +164,14 @@ a restart or `POST /_fakecloud/reset`, which puts them back to the flags. Applied values, per-account overrides and requests are account data and persist across restarts in persistent mode. +Firehose's three automatically scaled throughput quotas additionally require an +explicit applied value for the producer's account and region. Their reported AWS +defaults never throttle, even with global enforcement enabled; introspection then +reports `enforced: false` and `enforcementSource: "auto_scaling_default"`. An +explicit value equal to the default still counts as explicit. This local default +bypass does not simulate AWS's scaling latency. Usage and burst debt are not +persisted, unlike applied values and account overrides. + ### Enforceable quotas | Quota | Enforced by | @@ -201,7 +209,7 @@ persist across restarts in persistent mode. | Customer Master Keys (`kms`/`L-C2F1777E`) | `CreateKey`, `ReplicateKey` (counted in the replica's region) (`LimitExceededException`); customer managed keys in any key state count, including pending deletion, AWS managed keys do not | | General purpose buckets (`s3`/`L-DC2B2D3D`) | `CreateBucket` (`TooManyBuckets`); a global quota: one applied value per account, counting buckets across all regions | | Function and layer storage (`lambda`/`L-2ACBD22F`) | `CreateFunction`, `UpdateFunctionCode`, `PublishVersion`, `PublishLayerVersion` (`CodeStorageExceededException`); the code of every function's `$LATEST`, published version and layer version counts, container images do not | -| Firehose requests, records and MiB per second per stream (`firehose`/`L-7681C790`, `L-17D4EADE`, `L-937C879B`) | Public `PutRecord` and `PutRecordBatch` share independent per-stream budgets; whole-call HTTP 500 `ServiceUnavailableException`. Non-adjustable through AWS Service Quotas; use the local setter. See [Firehose throughput limits](/docs/services/firehose/#opt-in-direct-put-throughput-limits) | +| Firehose requests, records and MiB per second per stream (`firehose`/`L-7681C790`, `L-17D4EADE`, `L-937C879B`) | Public `PutRecord` and `PutRecordBatch` share independent per-stream budgets. Request exhaustion returns whole-call HTTP 500 `ServiceUnavailableException`; record/byte exhaustion produces ordered HTTP 200 batch-entry failures (HTTP 500 for `PutRecord`), with `Slow down.` messages. Positive record/byte allowances tolerate one boundary-crossing record and carry excess forward. Non-adjustable through AWS Service Quotas; use the local setter. See [Firehose throughput limits](/docs/services/firehose/#opt-in-direct-put-throughput-limits) | IAM refusals are HTTP 409 with IAM's `Cannot exceed quota for : ` message. Every count quota refuses the request that would take the count past From 71d42171b22d52b7a0f00c25e2f917473dbc9bca Mon Sep 17 00:00:00 2001 From: John Cavanaugh <59479+cavanaug@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:05:55 -0700 Subject: [PATCH 5/6] fix(firehose): address throughput review findings Share record-data validation across both public PUT APIs and cover request-wide rejection without quota consumption. Restore delivery documentation, share throughput error text, and clarify status codes and quota behavior in the service docs. --- AGENTS.md | 2 +- crates/fakecloud-e2e/tests/firehose.rs | 85 +++++++++----- .../tests/firehose_throughput.rs | 2 +- crates/fakecloud-firehose/src/engine.rs | 12 +- .../src/engine/tests/throughput.rs | 54 +++++++++ crates/fakecloud-firehose/src/service.rs | 109 +++++++++--------- crates/fakecloud-firehose/src/throughput.rs | 3 + .../fakecloud-servicequotas/src/provider.rs | 3 + website/content/docs/parity.md | 2 +- website/content/docs/services/_index.md | 2 +- website/content/docs/services/firehose.md | 6 +- .../content/docs/services/servicequotas.md | 9 ++ 12 files changed, 195 insertions(+), 94 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f391992e6..66ce3d7fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,5 +129,5 @@ Startup time, idle memory, and binary size are tracked as constants in `scripts/ - ECS task-role credentials: task-role containers get `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI=/v2/credentials/` and reach the agent's `169.254.170.2:80` in-network, as on ECS: an nftables/iptables DNAT in the task's netns sends it to fakecloud's main listener, where `ecs_creds::link_local_host_middleware` answers `Host: 169.254.170.2` requests with the agent surface (nothing listens on :80 in the task, so app ports stay free). Docker/Podman: a per-container NET_ADMIN holder (`fakecloud-ecs-netns--`, locally built `fakecloud-ecs-creds-helper:` Alpine+nftables image) owns the netns + network flags and the app joins it with `--network container:`; k8s: a first `fakecloud-ecs-creds` NET_ADMIN initContainer (default `alpine:3.20` + runtime `apk add nftables`). `none` network mode gets the relative URI with no holder/route (no network, as on ECS); `host` is emulated on a bridge so it gets a holder too. `FAKECLOUD_ECS_CREDS_HELPER_IMAGE` overrides the image on both (`fakecloud_ecs` `runtime::task_creds`). Fail-safe: holder/initContainer failure or Pod refusal -> warning + `AWS_CONTAINER_CREDENTIALS_FULL_URI` at `GET /_fakecloud/ecs/creds/{task_id}`. The endpoint vends a session for the task role named after the task ID (`assumed-role//`), minted + registered like AssumeRole and cached per task until near expiry (`fakecloud-server` `ecs_creds`, `WorkloadCredentialCache`); revoked when the task stops (1s sweep). Full-URI task containers also get `AWS_CONTAINER_AUTHORIZATION_TOKEN` (`fakecloud_ecs::runtime::task_credentials_token`, HMAC of the task ID under a per-process key); under `--iam strict` the full-URI endpoint requires it as the `Authorization` header (else HTTP 401 `AccessDenied`). The relative-URI `169.254.170.2/v2/credentials/` surface never requires it, as on ECS; under strict the main port serves that surface only to container-network peers (`ecs_creds::relative_uri_source_allowed`: private v4 / CGNAT / ULA / link-local, not loopback). Stopped/unknown/role-less tasks get the ECS agent's HTTP 400 `{"code":"InvalidIdInRequest","message":"CredentialsV2Request: Credentials not found","HTTPErrorCode":400}`. RegisterTaskDefinition/RunTask overrides/CFN apply `fakecloud_ecs::validate_task_role` (trust policy must name `ecs-tasks.amazonaws.com`; cross-account refused under `--iam strict`) with ECS's `ClientException` "ECS was unable to assume the role ...". - Container data-plane reachability (`fakecloud_core::dataplane`): AWS-visible target addresses (an `awsvpc` task's ENI, a real requester-managed EC2 ENI created via the delivery bus's `Ec2NetworkLookup` -> `vpc_lookup::create_service_eni` and deleted when the task stops; an `i-*` instance) are kept in every response, and the ELBv2 data plane / prober resolve them to a reachable socket -- ECS publishes `awsvpc` container ports ephemerally and registers `:`; the EC2 runtime installs an instance resolver that starts a socat forwarder per instance port on demand (serialized per instance+port, refused for stopped instances, its subnet IP counted as a member of the LB's security groups in the nft firewall model). EC2 instance containers get IMDS at `169.254.169.254` through a sidecar in their netns (nft redirect -> loopback nginx -> `/_fakecloud/ec2/imds//latest/*`, served by the server `ec2_imds` module with the instance's identity + instance-profile role); user-data waits on `/run/fakecloud/imds-ready`. Container env rewriting is `container_net::rewrite_loopback_value` (URLs, bare host / host:port, DSNs; MSK's container listener port swapped in via `dataplane::register_container_port`). ECR pulls by ECS and Lambda use `container_net::ecr_registry_host` (loopback, podman-machine alias, `FAKECLOUD_ECR_REGISTRY_HOST`), never the sibling host. - `--dns` runs a real DNS resolver (UDP+TCP, `--dns-addr` default `0.0.0.0:53`, port 53 needs root) answering the wire-encodable types (A/AAAA/CNAME/MX/TXT/NS/PTR/SPF/CAA/SRV/SOA) from the Route 53 records created in fakecloud. Longest-suffix zone match across all accounts; CNAME chase into local zones with the requested type appended, and an external CNAME target forward-resolved upstream so a stub client still gets an address. Names in no local zone forward to `--dns-upstream` (default first non-loopback `/etc/resolv.conf` nameserver, else `8.8.8.8:53`); forward socket is connect()ed + txid-checked. UDP honors EDNS0 size and sets TC to force TCP fallback past it. Pure resolution logic in `fakecloud_route53::resolver`, hand-rolled wire codec in the server `dns` module reusing `dnssec::encode_rdata`/`type_code`. Best-effort detached listener; a bind failure is logged and the main server is unaffected. The same resolution is exposed for tests at `GET /_fakecloud/dns/resolve?name=&type=` (defaults `A`, unsupported type -> 400) returning `{name,type,status(ANSWERED|NODATA|NXDOMAIN|NOT_AUTHORITATIVE),authoritative,records:[{name,type,ttl,value}],external_cname}` (`external_cname` names the target when an A/AAAA CNAME chain exits all local zones, since this path does no upstream I/O), wrapped as `dns_resolve`/`dnsResolve` in all seven SDKs -- no socket needed. -- Service Quotas enforcement is opt-in: nothing is enforced unless `--enforce-quotas` (global), `--enforce-quota SVC/CODE` / `--ignore-quota SVC/CODE`, or `/_fakecloud/service-quotas/*` (server-wide or per-account overrides; precedence account > server > global) switch it on. Enforcing services call `QuotaProvider::enforced_limit` (`None` = not enforced) and only catalog quotas marked `enforceable` can be switched on. The catalog is AWS's published defaults for every service AWS lists (us-east-1 dump vendored as `crates/fakecloud-servicequotas/data/quotas.json.gz`, regenerated by `scripts/gen-service-quotas-catalog.py`); `enforceable` and documented maximums are a hand-maintained overlay in `catalog.rs` (`ENFORCEABLE`, `MAX_VALUES`). The introspection API also sets applied values below the AWS default and approves/denies requests held `PENDING` under `--quota-requests manual`. Server-wide settings come from the flags at every start (not persisted; reset restores them); applied values and per-account overrides are persisted account data. +- Service Quotas enforcement is opt-in: nothing is enforced unless `--enforce-quotas` (global), `--enforce-quota SVC/CODE` / `--ignore-quota SVC/CODE`, or `/_fakecloud/service-quotas/*` (server-wide or per-account overrides; precedence account > server > global) switch it on. Enforcing services call `QuotaProvider::enforced_limit` (`None` = not enforced); use `enforced_limits` when checking several quotas together so Service Quotas resolves them from one settings/state snapshot. Only catalog quotas marked `enforceable` can be switched on. The catalog is AWS's published defaults for every service AWS lists (us-east-1 dump vendored as `crates/fakecloud-servicequotas/data/quotas.json.gz`, regenerated by `scripts/gen-service-quotas-catalog.py`); maintain the overlays in `catalog.rs`: `ENFORCEABLE` (checked quotas), `MAX_VALUES` (documented maximums), `AUTO_SCALING` (requires an explicit account/region applied value as well as an enabled switch), and `REGIONAL_DEFAULTS` (region-dependent defaults). Never enforce an `AUTO_SCALING` quota's reported AWS default merely because global enforcement is on. Firehose's three throughput quotas use both new overlays and meter each delivery-stream instance independently; internal Logs/SES handoffs are unmetered. The introspection API also sets applied values below the AWS default and approves/denies requests held `PENDING` under `--quota-requests manual`. Server-wide settings come from the flags at every start (not persisted; reset restores them); applied values and per-account overrides are persisted account data. - When adding new service behavior, prefer complete, realistic implementations over placeholder API coverage. diff --git a/crates/fakecloud-e2e/tests/firehose.rs b/crates/fakecloud-e2e/tests/firehose.rs index 801aaa473..ef0d10ff0 100644 --- a/crates/fakecloud-e2e/tests/firehose.rs +++ b/crates/fakecloud-e2e/tests/firehose.rs @@ -614,7 +614,7 @@ async fn firehose_destination_update_changes_output_version_and_extension() { } #[tokio::test] -async fn firehose_put_record_batch_reports_invalid_records() { +async fn firehose_put_apis_reject_invalid_data_request_wide() { let server = TestServer::start().await; let firehose = setup(&server, "fh-bucket-invalid").await; let bucket_arn = "arn:aws:s3:::fh-bucket-invalid"; @@ -627,7 +627,7 @@ async fn firehose_put_record_batch_reports_invalid_records() { .await .expect("create"); - // Mix valid + valid + valid records (SDK base64-encodes Blob for us, so + // Mix two valid records (SDK base64-encodes Blob for us, so // we can't construct invalid base64 via the SDK). Validate happy-path // counts here; invalid-record handling is exercised at the wire level // below by talking directly to the JSON endpoint. @@ -641,37 +641,60 @@ async fn firehose_put_record_batch_reports_invalid_records() { .expect("put batch"); assert_eq!(resp.failed_put_count(), 0); - // Invalid base64 rejects the whole request, not individual entries. + // Missing, wrong-type and invalid-base64 data reject the whole request. + // A valid leading batch entry must not turn a validation error into a + // successful partial response. let http = reqwest::Client::new(); let url = format!("{}/", server.endpoint()); - let body = serde_json::json!({ - "DeliveryStreamName": "fh-invalid-stream", - "Records": [ - {"Data": "@@@not-base64@@@"}, - {"Data": "Z29vZA=="} - ] - }); - let r = http - .post(&url) - .header("Content-Type", "application/x-amz-json-1.1") - .header("X-Amz-Target", "Firehose_20150804.PutRecordBatch") - .header( - "Authorization", - "AWS4-HMAC-SHA256 \ - Credential=AKIAIOSFODNN7EXAMPLE/20260501/us-east-1/firehose/aws4_request, \ - SignedHeaders=host;x-amz-target, Signature=deadbeef", - ) - .body(body.to_string()) - .send() - .await - .unwrap(); - assert_eq!(r.status(), 400); - let json: serde_json::Value = r.json().await.unwrap(); - assert!(json["__type"] - .as_str() - .unwrap() - .ends_with("SerializationException")); - assert!(json.get("RequestResponses").is_none()); + for (record, expected_code) in [ + (serde_json::json!({}), "InvalidArgumentException"), + ( + serde_json::json!({"Data": null}), + "InvalidArgumentException", + ), + (serde_json::json!({"Data": 1}), "SerializationException"), + (serde_json::json!({"Data": false}), "SerializationException"), + (serde_json::json!({"Data": []}), "SerializationException"), + (serde_json::json!({"Data": {}}), "SerializationException"), + ( + serde_json::json!({"Data": "@@@not-base64@@@"}), + "SerializationException", + ), + ] { + for action in ["PutRecord", "PutRecordBatch"] { + let body = if action == "PutRecord" { + serde_json::json!({ + "DeliveryStreamName": "fh-invalid-stream", "Record": record, + }) + } else { + serde_json::json!({ + "DeliveryStreamName": "fh-invalid-stream", + "Records": [{"Data": "Z29vZA=="}, record], + }) + }; + let r = http + .post(&url) + .header("Content-Type", "application/x-amz-json-1.1") + .header("X-Amz-Target", format!("Firehose_20150804.{action}")) + .header( + "Authorization", + "AWS4-HMAC-SHA256 \ + Credential=AKIAIOSFODNN7EXAMPLE/20260501/us-east-1/firehose/aws4_request, \ + SignedHeaders=host;x-amz-target, Signature=deadbeef", + ) + .json(&body) + .send() + .await + .unwrap(); + assert_eq!(r.status(), 400, "{action}: {record}"); + let json: serde_json::Value = r.json().await.unwrap(); + assert!( + json["__type"].as_str().unwrap().ends_with(expected_code), + "{action}: {record}: {json}", + ); + assert!(json.get("RequestResponses").is_none()); + } + } } #[tokio::test] diff --git a/crates/fakecloud-e2e/tests/firehose_throughput.rs b/crates/fakecloud-e2e/tests/firehose_throughput.rs index c3db37bfa..9827b8d03 100644 --- a/crates/fakecloud-e2e/tests/firehose_throughput.rs +++ b/crates/fakecloud-e2e/tests/firehose_throughput.rs @@ -242,7 +242,7 @@ async fn payloads(s3: &aws_sdk_s3::Client, bucket: &str, count: usize, gzip: boo } #[tokio::test] -async fn zero_request_capacity_returns_modeled_http_500_for_both_apis_then_live_raise_recovers() { +async fn zero_request_capacity_returns_http_500_for_both_apis_then_live_raise_recovers() { bounded(async { let server = TestServer::start().await; let bucket = "fh-throughput-requests"; diff --git a/crates/fakecloud-firehose/src/engine.rs b/crates/fakecloud-firehose/src/engine.rs index 18bd6af6c..f8592f935 100644 --- a/crates/fakecloud-firehose/src/engine.rs +++ b/crates/fakecloud-firehose/src/engine.rs @@ -725,17 +725,19 @@ impl EngineCore { } } -/// The one delivery decision shared by admission, scheduling, and the final -/// drain, so they cannot disagree about whether a stream can deliver. -/// `Ok(None)`: no destination is configured, so nothing is written. +/// Throttle response using the API reference's HTTP 500, not the model's 503. +/// Closed admission is a separate, local shutdown condition and retains 503. pub(crate) fn throughput_error() -> AwsServiceError { AwsServiceError::aws_error( StatusCode::INTERNAL_SERVER_ERROR, - "ServiceUnavailableException", - "Slow down.", + crate::throughput::THROUGHPUT_ERROR_CODE, + crate::throughput::THROUGHPUT_ERROR_MESSAGE, ) } +/// The one delivery decision shared by admission, scheduling, and the final +/// drain, so they cannot disagree about whether a stream can deliver. +/// `Ok(None)`: no destination is configured, so nothing is written. fn stream_delivery(stream: &DeliveryStream) -> Result, AwsServiceError> { crate::admission::validate_stream_destination(stream)?; stream.destination.as_ref().map(delivery_config).transpose() diff --git a/crates/fakecloud-firehose/src/engine/tests/throughput.rs b/crates/fakecloud-firehose/src/engine/tests/throughput.rs index ec9c496a9..8676a9bd2 100644 --- a/crates/fakecloud-firehose/src/engine/tests/throughput.rs +++ b/crates/fakecloud-firehose/src/engine/tests/throughput.rs @@ -410,6 +410,60 @@ async fn public_batch_burst_is_ordered_and_debt_survives_window_rollover() { assert_eq!(buffered(&f).last().unwrap(), &Bytes::from_static(b"f")); } +#[tokio::test] +async fn public_record_data_validation_is_consistent_and_never_consumes_capacity() { + use fakecloud_core::service::AwsService; + + for (invalid, expected_code) in [ + (json!({}), "InvalidArgumentException"), + (json!({"Data": null}), "InvalidArgumentException"), + (json!({"Data": 1}), "SerializationException"), + (json!({"Data": false}), "SerializationException"), + (json!({"Data": []}), "SerializationException"), + (json!({"Data": {}}), "SerializationException"), + (json!({"Data": "%%%"}), "SerializationException"), + ] { + for action in ["PutRecord", "PutRecordBatch"] { + let f = Fixture::new(60); + enforce(&f, Some(1.0), Some(1.0), Some(1.0 / 1_048_576.0)); + let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); + let body = if action == "PutRecord" { + json!({"DeliveryStreamName": NAME, "Record": invalid}) + } else { + // A valid leading entry must not be admitted before validation finishes. + json!({"DeliveryStreamName": NAME, "Records": [{"Data": "YQ=="}, invalid]}) + }; + let error = svc.handle(request(action, body)).await.err().unwrap(); + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + assert_eq!(error.code(), expected_code, "{action}: {invalid}"); + assert!(buffered(&f).is_empty(), "{action}: {invalid}"); + + let valid = if action == "PutRecord" { + json!({"DeliveryStreamName": NAME, "Record": {"Data": "Yg=="}}) + } else { + json!({"DeliveryStreamName": NAME, "Records": [{"Data": "Yg=="}]}) + }; + let response = svc.handle(request(action, valid.clone())).await.unwrap(); + assert_eq!(response.status, StatusCode::OK); + let body: serde_json::Value = + serde_json::from_slice(response.body.expect_bytes()).unwrap(); + if action == "PutRecord" { + assert!(body["RecordId"].is_string()); + } else { + assert_eq!(body["FailedPutCount"], 0); + assert!(body["RequestResponses"][0]["RecordId"].is_string()); + } + assert_eq!(buffered(&f), vec![Bytes::from_static(b"b")]); + + // Successful validation still debits the shared request budget. + let error = svc.handle(request(action, valid)).await.err().unwrap(); + assert_eq!(error.code(), "ServiceUnavailableException"); + assert_eq!(error.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(buffered(&f), vec![Bytes::from_static(b"b")]); + } + } +} + #[tokio::test] async fn malformed_base64_rejects_whole_call_without_consuming_capacity() { use fakecloud_core::service::AwsService; diff --git a/crates/fakecloud-firehose/src/service.rs b/crates/fakecloud-firehose/src/service.rs index 4e52f426e..695224b49 100644 --- a/crates/fakecloud-firehose/src/service.rs +++ b/crates/fakecloud-firehose/src/service.rs @@ -224,6 +224,34 @@ fn missing(field: &str) -> AwsServiceError { ) } +fn decode_record_data(value: &Value) -> Result { + if value.is_null() { + return Err(missing("Record.Data")); + } + let encoded = value.as_str().ok_or_else(|| { + AwsServiceError::aws_error( + StatusCode::BAD_REQUEST, + "SerializationException", + "Record.Data must be a base64-encoded string", + ) + })?; + let data = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|_| { + AwsServiceError::aws_error( + StatusCode::BAD_REQUEST, + "SerializationException", + "Record.Data must be valid base64", + ) + })?; + if data.len() > 1_024_000 { + return Err(invalid_argument( + "Record.Data exceeds 1,024,000 decoded bytes", + )); + } + Ok(Bytes::from(data)) +} + /// Non-S3 destination base names. Firehose names each destination's input /// `Configuration` (create) / `Update` (update) and reports it as /// `Description`. fakecloud models S3/ExtendedS3 field-by-field; these @@ -827,30 +855,11 @@ impl FirehoseService { .as_str() .ok_or_else(|| missing("DeliveryStreamName"))? .to_string(); - let data_b64 = body["Record"]["Data"] - .as_str() - .ok_or_else(|| missing("Record.Data"))?; - let data = base64::engine::general_purpose::STANDARD - .decode(data_b64) - .map_err(|_| { - AwsServiceError::aws_error( - StatusCode::BAD_REQUEST, - "SerializationException", - "Record.Data must be valid base64", - ) - })?; - if data.len() > 1_024_000 { - return Err(invalid_argument( - "Record.Data exceeds 1,024,000 decoded bytes", - )); - } + let data = decode_record_data(&body["Record"]["Data"])?; let record_id = format!("{}", Uuid::new_v4()); - let accepted = self.engine.enqueue_put( - &req.account_id, - &req.region, - &name, - vec![Bytes::from(data)], - )?; + let accepted = self + .engine + .enqueue_put(&req.account_id, &req.region, &name, vec![data])?; if !accepted[0] { return Err(crate::engine::throughput_error()); } @@ -877,28 +886,12 @@ impl FirehoseService { let mut datas = Vec::with_capacity(records.len()); let mut decoded_bytes = 0; for r in records { - let b64 = r["Data"].as_str(); - match b64.map(|s| base64::engine::general_purpose::STANDARD.decode(s)) { - Some(Ok(data)) => { - if data.len() > 1_024_000 { - return Err(invalid_argument( - "Record.Data exceeds 1,024,000 decoded bytes", - )); - } - decoded_bytes += data.len(); - if decoded_bytes > 4_194_304 { - return Err(invalid_argument("Records exceed 4,194,304 decoded bytes")); - } - datas.push(Bytes::from(data)); - } - _ => { - return Err(AwsServiceError::aws_error( - StatusCode::BAD_REQUEST, - "SerializationException", - "Record.Data must be a base64-encoded string", - )); - } + let data = decode_record_data(&r["Data"])?; + decoded_bytes += data.len(); + if decoded_bytes > 4_194_304 { + return Err(invalid_argument("Records exceed 4,194,304 decoded bytes")); } + datas.push(data); } // One handoff after all decoding/limits: request-wide rejection cannot // partially admit data. Only throughput refusal can fail individual entries. @@ -906,13 +899,19 @@ impl FirehoseService { .engine .enqueue_put(&req.account_id, &req.region, &name, datas)?; let failed = accepted.iter().filter(|admitted| !**admitted).count(); - let response_records: Vec<_> = accepted.into_iter().map(|admitted| { - if admitted { - json!({"RecordId": Uuid::new_v4().to_string()}) - } else { - json!({"ErrorCode": "ServiceUnavailableException", "ErrorMessage": "Slow down."}) - } - }).collect(); + let response_records: Vec<_> = accepted + .into_iter() + .map(|admitted| { + if admitted { + json!({"RecordId": Uuid::new_v4().to_string()}) + } else { + json!({ + "ErrorCode": crate::throughput::THROUGHPUT_ERROR_CODE, + "ErrorMessage": crate::throughput::THROUGHPUT_ERROR_MESSAGE, + }) + } + }) + .collect(); Ok(AwsResponse::ok_json(json!({ "FailedPutCount": failed, "Encrypted": false, @@ -2579,12 +2578,16 @@ mod tests { #[test] fn malformed_batch_is_request_wide_even_after_valid_or_empty_records() { let (svc, _) = memory_fixture(); - for malformed in [json!({"Data": "%%%"}), json!({}), json!({"Data": 1})] { + for (malformed, expected_code) in [ + (json!({"Data": "%%%"}), "SerializationException"), + (json!({}), "InvalidArgumentException"), + (json!({"Data": 1}), "SerializationException"), + ] { let error = batch(&svc, vec![record(b""), record(b"valid"), malformed]) .err() .unwrap(); assert_eq!(error.status(), StatusCode::BAD_REQUEST); - assert_eq!(error.code(), "SerializationException"); + assert_eq!(error.code(), expected_code); } let error = svc .put_record_batch(&request( diff --git a/crates/fakecloud-firehose/src/throughput.rs b/crates/fakecloud-firehose/src/throughput.rs index 1eb15dd46..d37840d7f 100644 --- a/crates/fakecloud-firehose/src/throughput.rs +++ b/crates/fakecloud-firehose/src/throughput.rs @@ -1,5 +1,8 @@ use tokio::time::Instant; +pub(crate) const THROUGHPUT_ERROR_CODE: &str = "ServiceUnavailableException"; +pub(crate) const THROUGHPUT_ERROR_MESSAGE: &str = "Slow down."; + #[derive(Clone, Copy, Default)] pub(crate) struct ThroughputLimits { pub requests: Option, diff --git a/crates/fakecloud-servicequotas/src/provider.rs b/crates/fakecloud-servicequotas/src/provider.rs index 2a8e1795d..b0078c922 100644 --- a/crates/fakecloud-servicequotas/src/provider.rs +++ b/crates/fakecloud-servicequotas/src/provider.rs @@ -334,6 +334,9 @@ impl QuotaProvider for ServiceQuotasProvider { #[cfg(test)] tests::after_initial_snapshot(); if !limits.iter().any(Option::is_some) { + // This fast path applies to every enforcing service: an ignored + // quota must not trigger the lazy organization-template fallback. + // Explicit applied-value lookups still apply a due template. return limits; } // Enforcing something touches the account's quotas, which applies a diff --git a/website/content/docs/parity.md b/website/content/docs/parity.md index e373757a5..64948da14 100644 --- a/website/content/docs/parity.md +++ b/website/content/docs/parity.md @@ -103,7 +103,7 @@ fakecloud implements **107 AWS services** with **7,657 operations**. **258,545/2 | [ACM PCA](@/docs/services/acm-pca.md) | 23 | JSON 1.1 | Full | Full | Real private CA hierarchy. `CreateCertificateAuthority` mints a genuine CA key pair and (for a ROOT CA) a self-signed certificate; a SUBORDINATE CA starts `PENDING_CERTIFICATE` and serves a real PEM CSR from `GetCertificateAuthorityCsr` until its signed chain is installed via `ImportCertificateAuthorityCertificate`. `IssueCertificate` signs real end-entity certificates from the caller's CSR that verify against the CA (`rcgen`). Revocation, audit reports, resource permissions, and resource policies are implemented. CA private keys are persisted so issued certs still verify after a restart. | | [Config](@/docs/services/config.md) | 102 | JSON 1.1 | Full | Partial | Real configuration recorder: a running recorder snapshots the live state of other fakecloud services (S3, EC2, IAM) into genuine `ConfigurationItem` history that `GetResourceConfigHistory` / `BatchGetResourceConfig` / `ListDiscoveredResources` return. `PutResourceConfig` records external resources. AWS managed rules (`S3_BUCKET_VERSIONING_ENABLED`, `S3_BUCKET_SERVER_SIDE_ENCRYPTION_ENABLED`, `S3_BUCKET_PUBLIC_READ/WRITE_PROHIBITED`, `IAM_USER_NO_POLICIES_CHECK`, `EC2_INSTANCE_NO_PUBLIC_IP`, `VPC_DEFAULT_SECURITY_GROUP_CLOSED`, `INCOMING_SSH_DISABLED`, `RESTRICTED_INCOMING_TRAFFIC`, `REQUIRED_TAGS`) run real evaluation against the recorded items; custom `CUSTOM_LAMBDA` rules invoke the referenced Lambda via fakecloud-lambda. `SelectResourceConfig` runs a real SQL-subset query over the recorded items. Compliance queries, conformance packs, aggregators, remediation, retention, and stored queries are implemented. Unimplemented managed rules evaluate as `INSUFFICIENT_DATA` rather than a fabricated result. | | [CloudWatch (Metrics & Alarms)](@/docs/services/cloudwatch.md) | 55 | RPC v2 CBOR, JSON 1.0, Query | Full | Partial | Metrics, statistics, dashboards, metric/composite alarms, anomaly detectors, insight rules, metric streams, alarm mute rules, contributor insights, OTel enrichment, dataset KMS key management, and tagging are all implemented with persisted in-memory state. Alarm threshold transitions trigger SNS/AppAS/EC2 actions. `GetMetricWidgetImage` returns a deterministic image blob. Metrics do not persist across server restarts. | -| [Firehose](@/docs/services/firehose.md) | 12 | JSON 1.1 | Full | Partial | Buffered S3-only delivery across API/internal calls: defaults 5 MiB/300 s, paired hints 1-128 MiB/0-900 s, decoded-byte accounting before compression (appended delimiters are not counted), whole-call threshold overshoot, idle interval flush, and async zero-interval delivery. Uncompressed/GZIP/Xerial Snappy preserve exact bytes, oldest-arrival UTC prefixes, delivery-time suffixes, codec metadata, MD5 ETags, and validated custom extensions; no delimiters are inserted unless the `AppendDelimiterToRecord` processor adds `\n` after every record. Active data uses the latest supported update; detached batches retry with captured configuration/key. All modeled codecs and destination settings remain accepted by Create/UpdateDestination and CloudFormation; unsupported S3 codecs (`ZIP`, `HADOOP_SNAPPY`), enabled processing other than delimiter-only processing, format conversion, dynamic partitioning, prefix expressions, non-UTC time zones, S3 backup, and non-S3 destinations reject record admission with HTTP 400 `InvalidArgumentException` naming the setting before acknowledgment; unsupported updates pause accepted active data. Completed objects persist; pending queues do not. A destination `KMSEncryptionConfig` writes objects with SSE-KMS under its key, otherwise bucket-default encryption applies, and replication and S3 notifications apply to encoded objects through the shared S3 write path. Failed batches stay in memory and retry every 5 seconds for up to 24 hours from arrival, as on AWS, then are discarded with a warning; graceful shutdown closes late admission and performs one final drain, reporting undelivered data on failure. Kinesis-source polling is not implemented. Server-side encryption configuration persists and surfaces in `DescribeDeliveryStream`. | +| [Firehose](@/docs/services/firehose.md) | 12 | JSON 1.1 | Full | Partial | Buffered S3-only delivery across API/internal calls: defaults 5 MiB/300 s, paired hints 1-128 MiB/0-900 s, decoded-byte accounting before compression (appended delimiters are not counted), whole-call threshold overshoot, idle interval flush, and async zero-interval delivery. Public Direct PUT APIs share opt-in per-stream request, record and decoded-MiB budgets, requiring explicit account/region applied values plus enabled Service Quotas switches; reported AWS defaults never throttle. Request exhaustion returns HTTP 500 `ServiceUnavailableException`; record/byte exhaustion produces ordered partial batch failures (HTTP 500 for `PutRecord`). Missing data, wrong types and malformed base64 reject the whole call before admission or quota consumption. Internal Logs/SES handoffs are unmetered. Uncompressed/GZIP/Xerial Snappy preserve exact bytes, oldest-arrival UTC prefixes, delivery-time suffixes, codec metadata, MD5 ETags, and validated custom extensions; no delimiters are inserted unless the `AppendDelimiterToRecord` processor adds `\n` after every record. Active data uses the latest supported update; detached batches retry with captured configuration/key. All modeled codecs and destination settings remain accepted by Create/UpdateDestination and CloudFormation; unsupported S3 codecs (`ZIP`, `HADOOP_SNAPPY`), enabled processing other than delimiter-only processing, format conversion, dynamic partitioning, prefix expressions, non-UTC time zones, S3 backup, and non-S3 destinations reject record admission with HTTP 400 `InvalidArgumentException` naming the setting before acknowledgment; unsupported updates pause accepted active data. Completed objects persist; pending queues do not. A destination `KMSEncryptionConfig` writes objects with SSE-KMS under its key, otherwise bucket-default encryption applies, and replication and S3 notifications apply to encoded objects through the shared S3 write path. Failed batches stay in memory and retry every 5 seconds for up to 24 hours from arrival, as on AWS, then are discarded with a warning; graceful shutdown closes late admission and performs one final drain, reporting undelivered data on failure. Kinesis-source polling is not implemented. Server-side encryption configuration persists and surfaces in `DescribeDeliveryStream`. | | [Glue](@/docs/services/glue.md) | 302 | JSON 1.1 | Full | Partial | Full control plane: Data Catalog (databases, tables, partitions with `GetPartitions` `Expression` pruning), jobs, crawlers, classifiers, connections, triggers, workflows, blueprints, dev endpoints, schema registry, interactive sessions, ML transforms, data quality, user-defined functions, usage profiles, column statistics, and tagging. Status transitions are real (crawler `READY`↔`RUNNING`, trigger/workflow/run lifecycles). Job/crawler/Spark *execution* itself is synthesized — fakecloud is not a Spark engine. | | [Organizations](@/docs/services/organizations.md) | 63 | JSON 1.1 | Full | Full | Full org tree (roots, OUs, accounts), policies with SCP enforcement, handshakes, delegated administrators, service access, tagging, and a resource policy. Billing responsibility transfers ride handshake-backed records. `CreateAccount` transitions `IN_PROGRESS` -> `SUCCEEDED` after a short synthetic delay. | | [EC2](@/docs/services/ec2.md) | 807 | ec2Query | Full | Partial | Full 807-op control plane: VPCs, subnets, security groups, route tables, gateways, ENIs, instances, EBS volumes/snapshots, AMIs (+ watermarks), network ACLs, VPC peering/endpoints, flow logs, launch templates, spot/fleet, capacity/reserved/dedicated hosts, transit gateways (+ multicast/peering/metering/policy-table entries), VPN + Client VPN, IPAM, Verified Access, Network Insights, Outpost/local-gateway/CoIP, and Instance Connect. Instances run as real containers — Docker/Podman by default or native Kubernetes Pods (`FAKECLOUD_EC2_BACKEND=k8s`) — running user-data at boot, with the instance lifecycle mapped to the container lifecycle and `GetConsoleOutput` returning the container log; the control plane degrades to metadata-only when no container runtime is present. A few model ops absent from the vendored SDK are validated via raw ec2Query. | diff --git a/website/content/docs/services/_index.md b/website/content/docs/services/_index.md index a8d9c544b..da766d18f 100644 --- a/website/content/docs/services/_index.md +++ b/website/content/docs/services/_index.md @@ -76,7 +76,7 @@ fakecloud implements 107 AWS services with 7,657 total operations. 258,545/258,5 | ACM PCA | 23 | **Full control plane + real certificate data plane.** Private CA hierarchy: `CreateCertificateAuthority` mints a genuine CA key pair (RSA 2048/3072/4096, EC P-256/P-384); a `ROOT` CA self-signs while a `SUBORDINATE` CA starts `PENDING_CERTIFICATE` and serves a real PEM CSR from `GetCertificateAuthorityCsr` until its signed chain is installed via `ImportCertificateAuthorityCertificate`. `IssueCertificate` parses the caller's CSR and signs a real end-entity certificate that verifies against the CA (`rcgen`); `GetCertificate` returns the signed PEM + chain. Revocation (`RevokeCertificate`), audit reports (`Create`/`DescribeCertificateAuthorityAuditReport`, JSON/CSV), resource-share permissions (`Create`/`List`/`DeletePermission`), resource policies (`Put`/`Get`/`DeletePolicy`), enable/disable + delete/restore lifecycle, and ARN-keyed tags. CA private keys are persisted so issued certs still verify after a restart. JSON 1.1 protocol (all 23 ops) | | Athena | 70 | **Full control plane.** Workgroups (default `primary` seeded), Data Catalogs (default `AwsDataCatalog` seeded), Named Queries, Prepared Statements (keyed by `(workgroup, statement_name)`), Query Executions, Notebooks, Sessions + Calculations, Capacity Reservations + Capacity Assignment Configuration. `StartQueryExecution` runs through a minimal SQL evaluator: `SHOW DATABASES` / `SHOW TABLES` / `DESCRIBE` and trivial `SELECT col FROM db.table WHERE col = 'lit' LIMIT N` resolve against the Glue Data Catalog (databases + tables created via `glue:CreateTable` are visible immediately under `AwsDataCatalog`). `ExecutionParameters` performs positional `?` substitution against prepared statements. Anything outside the supported subset (joins, aggregates, window functions, Parquet scans) falls back to a synthesized single-row `[["1"]]` result so callers can still fetch via `GetQueryResults` without polling. `DeleteWorkGroup` rejects `primary` and refuses non-empty workgroups unless `RecursiveDeleteOption=true`. `DeleteDataCatalog` rejects `AwsDataCatalog`. Statement classification (`DML` / `DDL` / `UTILITY`) inferred from the leading SQL keyword. Tags keyed by ARN across workgroup / datacatalog / capacity-reservation resources. `ListEngineVersions` / `ListApplicationDPUSizes` / `ListExecutors` / `GetResourceDashboard` return read-only catalog data. JSON 1.1 protocol. fakecloud is not a full SQL engine — complex queries fall back to the synthesized `[["1"]]` result | | Glue | 302 | **Full control plane.** Data Catalog (databases / tables / partitions with **`GetPartitions` Expression pruning** — `=` / `!=` / `<>` / comparisons / `IN` / `BETWEEN` / `LIKE` / `IS [NOT] NULL` with `AND` / `OR` / `NOT`, type-aware over `string` / `int` / `bigint` / `date` / `timestamp`), catalogs + encryption settings + resource policies, jobs + job runs + bookmarks, crawlers + classifiers + schedules (real `READY` ↔ `RUNNING`, `CrawlerRunningException` / `CrawlerNotRunningException`), connections, triggers, workflows + runs, blueprints + runs, dev endpoints, schema registry (registries / schemas / versions / metadata), interactive sessions + statements, ML transforms + task runs, data quality (rulesets / evaluation + recommendation runs / results), user-defined functions, usage profiles, column statistics + stats tasks, table optimizers, custom entity types, security configurations, tagging. Smithy `@length` / `@range` / enum constraints enforced server-side. Same Data Catalog store backs Athena's `ListDatabases` / `GetTableMetadata`. JSON 1.1 protocol. Job / crawler / ML / data-quality *execution* is synthesized — runs reach their terminal state immediately; fakecloud is not a Spark engine | -| Firehose | 12 | **Delivery stream control plane + buffered S3-only delivery.** Stream CRUD, destination round-trips, tags, and encryption configuration. API/internal records share a buffer: defaults 5 MiB/300 s; paired hints 1-128 MiB/0-900 s. Size counts decoded bytes before compression and permits whole-call overshoot; idle intervals flush, zero interval delivers asynchronously. No delimiters are inserted unless the `AppendDelimiterToRecord` processor is enabled (then `\n` follows every record, before compression). Uncompressed/GZIP/Xerial Snappy output supports static oldest-arrival UTC prefixes, delivery-time suffixes, and validated custom extensions (`gzip`/`snappy-java` encoding headers); a destination `KMSEncryptionConfig` writes objects with SSE-KMS under its key, otherwise bucket-default encryption applies, and replication and S3 notifications apply to encoded objects. Active data uses the latest supported configuration; detached retries keep captured settings/key. `ZIP`/`HADOOP_SNAPPY`, prefix expressions, non-UTC time zones, processing other than `AppendDelimiterToRecord` alone, format conversion, dynamic partitioning, S3 backup, and non-S3 destinations round-trip as configuration, but record admission returns HTTP 400 `InvalidArgumentException` naming the setting, and an unsupported update pauses already accepted active data. Completed objects/metadata persist, pending buffers do not. Failed batches stay in memory and retry every 5 seconds for up to 24 hours; graceful shutdown closes late admission and performs one final drain with undelivered counts. Kinesis-source polling is not implemented. See [Firehose](@/docs/services/firehose.md) for lifecycle and pipeline limits. JSON 1.1 protocol | +| Firehose | 12 | **Delivery stream control plane + buffered S3-only delivery.** Stream CRUD, destination round-trips, tags, and encryption configuration. Public Direct PUT APIs share opt-in per-stream request, record and decoded-MiB quotas: enforcement needs explicit account/region values plus enabled switches, never reported AWS defaults. Request exhaustion returns HTTP 500 `ServiceUnavailableException`; record/byte exhaustion yields ordered partial batch failures (HTTP 500 for `PutRecord`). Invalid record data rejects the whole call without admitting entries or consuming quotas. Internal Logs/SES handoffs are unmetered. API/internal records share a buffer: defaults 5 MiB/300 s; paired hints 1-128 MiB/0-900 s. Size counts decoded bytes before compression and permits whole-call overshoot; idle intervals flush, zero interval delivers asynchronously. No delimiters are inserted unless the `AppendDelimiterToRecord` processor is enabled (then `\n` follows every record, before compression). Uncompressed/GZIP/Xerial Snappy output supports static oldest-arrival UTC prefixes, delivery-time suffixes, and validated custom extensions (`gzip`/`snappy-java` encoding headers); a destination `KMSEncryptionConfig` writes objects with SSE-KMS under its key, otherwise bucket-default encryption applies, and replication and S3 notifications apply to encoded objects. Active data uses the latest supported configuration; detached retries keep captured settings/key. `ZIP`/`HADOOP_SNAPPY`, prefix expressions, non-UTC time zones, processing other than `AppendDelimiterToRecord` alone, format conversion, dynamic partitioning, S3 backup, and non-S3 destinations round-trip as configuration, but record admission returns HTTP 400 `InvalidArgumentException` naming the setting, and an unsupported update pauses already accepted active data. Completed objects/metadata persist, pending buffers do not. Failed batches stay in memory and retry every 5 seconds for up to 24 hours; graceful shutdown closes late admission and performs one final drain with undelivered counts. Kinesis-source polling is not implemented. See [Firehose](@/docs/services/firehose.md) for lifecycle and pipeline limits. JSON 1.1 protocol | | Amazon Timestream | 30 | **Full 30-op surface (Write + Query).** One crate serves both SDK clients on the shared `Timestream_20181101.` JSON 1.0 target prefix and one account-partitioned store. Databases (`Create`/`Describe`/`List`/`Update`/`Delete`, `database/` ARNs, `KmsKeyId`, `TableCount`; non-empty delete rejected), tables (`RetentionProperties`/`MagneticStoreWriteProperties`/`Schema` echoed, status `ACTIVE`, `ListTables` filtered by database), `WriteRecords` (validates + merges `CommonAttributes`, buffers ingested points, `RecordsIngested`, `RejectedRecordsException`), `Query`/`PrepareQuery`/`CancelQuery`, scheduled queries (`Create`/`Describe`/`List`/`Update`/`Delete`/`ExecuteScheduledQuery`), batch-load tasks (`Create`/`Describe`/`List`/`Resume`), account settings (`MaxQueryTCU`/`QueryPricingModel`), `DescribeEndpoints` (echoes the fakecloud host so endpoint discovery resolves back to fakecloud), and ARN-keyed tagging. Model-driven validation with the declared exceptions (`ValidationException`/`ResourceNotFoundException`/`ConflictException`/`RejectedRecordsException`/`ThrottlingException`/`InvalidEndpointException`). Account-partitioned and persisted. Honest gap: the `Query` SQL engine is a bounded interpreter (`SELECT *` / `COUNT(*)` over `"db"."table"`, `WHERE time ago(...)`, `ORDER BY time`, `LIMIT`) returning ingested points as AWS-shaped rows; unsupported shapes return a `ValidationException` rather than a wrong result, and there is no real time-series engine underneath | | AWS IoT Core | 272 | Full 272-op AWS IoT Core control plane (signed as `iot`, routed by HTTP method + `@http` URI path; route table, HTTP bindings, input constraints, and output shapes all generated from AWS's Smithy model). Every resource family mints proper ARNs + ids, persists attributes, and round-trips on read/list with paginating `nextToken`s: things, thing types, thing groups (static + dynamic), billing groups (with membership), policies (+ versions + v2/legacy attachments), certificates (`CreateKeysAndCertificate`/`CreateCertificateFromCsr` mint a 64-hex id + ARN + shaped PEM + RSA key pair; register/describe/transfer; thing-principal attachment), jobs + job templates, topic rules + destinations (SQL + actions stored verbatim), Device Defender (security profiles, scheduled audits, audit configuration, mitigation actions, custom metrics, dimensions), provisioning templates (+ versions), domain configurations, fleet metrics, role aliases, authorizers (+ default authorizer), streams, OTA updates, packages (+ versions), certificate providers, commands, `DescribeEndpoint` (deterministic per-account host), and ARN-keyed tagging. Model-derived `required`/`length`/`range`/`enum` validation with declared exceptions; account-partitioned and persisted. Honest gaps: no MQTT broker or device connectivity (the control plane is real but no message is routed, no rule action executes, no device attaches -- the data plane is the separate `iotdata` service); `SearchIndex`/aggregations use a bounded query subset (`*`, `thingName:`, bare name) and return `InvalidQueryException` otherwise; certificates are structurally-valid PEM placeholders, not real X.509 chains | | AWS IoT Data Plane | 11 | Full 11-op AWS IoT Data Plane device-shadow + retained-message data plane (signed as `iotdata`, routed by HTTP method + `@http` URI path). Device shadows (classic + named): `UpdateThingShadow` accepts a shadow document as the raw `@httpPayload` body and deep-merges `state.desired`/`state.reported` (a `null` leaf deletes the key), bumps `version`, stamps per-leaf `metadata` timestamps, and recomputes `state.delta`; `GetThingShadow` returns the merged document; `DeleteThingShadow` returns the deletion payload; `ListNamedShadowsForThing` paginates named-shadow names. A mismatched shadow `version` is `ConflictException`; a missing shadow is `ResourceNotFoundException`. `Publish` stores retained messages (empty payload clears the topic) and no-ops non-retained publishes; `GetRetainedMessage`/`ListRetainedMessages` read them back. Model-derived validation, account-partitioned, persisted. Honest gap: no MQTT broker, so publishes are never fanned out to live subscribers and the connection-introspection ops (`GetConnection`/`DeleteConnection`/`ListSubscriptions`/`SendDirectMessage`) return `ResourceNotFoundException` for every client id (no client is ever connected) | diff --git a/website/content/docs/services/firehose.md b/website/content/docs/services/firehose.md index 33c3f38a1..6981d8e9f 100644 --- a/website/content/docs/services/firehose.md +++ b/website/content/docs/services/firehose.md @@ -19,7 +19,7 @@ fakecloud implements Amazon Data Firehose's JSON 1.1 control plane and buffered Incomplete pairs, non-integers, and out-of-range values return `InvalidArgumentException`. CloudFormation also accepts quoted integers such as `"60"`, because `Ref` parameters resolve to strings; it coerces them, and every other quoted modeled boolean or number in the destination block (including inside `S3BackupConfiguration`), before validating. A value that does not match its modeled type makes the resource fail, naming the member's path: a quoted string that does not parse exactly (such as `IntervalInSeconds: "soon"`, or `" 60 "` with surrounding whitespace), an integer outside the model's 32-bit range, or a JSON value of the wrong type (such as a number for `Enabled`). Interval zero makes nonempty input eligible at admission; writing is still asynchronous, so poll S3 for completion rather than assuming an acknowledgment means the object exists. - **Destination strings**: the AWS model's length limits apply: `Prefix` and `ErrorOutputPrefix` allow at most 1,024 characters, `CustomTimeZone` at most 50, `RoleARN` 1-512, and `BucketARN` 1-2,048. ARN and time-zone patterns are not enforced here. Other values return `InvalidArgumentException` from `CreateDeliveryStream` and `UpdateDestination`, and make a CloudFormation resource fail. - **Records**: `PutRecord` / `PutRecordBatch` assign per-record `RecordId`s after supported S3 admission. Binary bytes and producer-supplied delimiters are preserved exactly before compression; no separators are added unless the `AppendDelimiterToRecord` processor is enabled. Without it, a delivery window holding only empty records produces no object. Public API and internal Logs/SES delivery use the same buffering engine. - Each API record is limited to 1,024,000 decoded bytes; a batch contains 1-500 records and at most 4,194,304 decoded bytes. Request-wide limit violations reject the whole call before admission. Malformed base64 in either API rejects the whole call with HTTP 400 `SerializationException`; no entries are admitted. Opt-in throughput limits can instead produce partial batch failures, as described below. + Each API record is limited to 1,024,000 decoded bytes; a batch contains 1-500 records and at most 4,194,304 decoded bytes. Request-wide limit violations reject the whole call before admission. Missing or null `Data` in either API rejects the whole call with HTTP 400 `InvalidArgumentException`; non-string `Data` or malformed base64 rejects it with HTTP 400 `SerializationException`. No entries are admitted and no quota capacity is consumed by these validation failures. Opt-in throughput limits can instead produce partial batch failures, as described below. - **New line delimiter**: when `ProcessingConfiguration` has `Enabled: true` and every entry in its `Processors` is `AppendDelimiterToRecord`, a newline (`\n`) is written after every record, including the last one and empty records, before any compression; a delivery window holding only empty records becomes an object of newlines. This applies to API deliveries, to internal CloudWatch Logs and SES deliveries, and to streams created or updated through CloudFormation, which store `ProcessingConfiguration` like `CreateDeliveryStream`. The optional `Delimiter` parameter is stored but not interpreted: AWS documents only a new line delimiter. Appended newlines do not count toward the size threshold. Lambda and other processors are not run; enabled processing that includes them, or lists no processors, rejects record admission with HTTP 400 `InvalidArgumentException` naming the unsupported setting (see [Caveats](#caveats)). - **S3 output**: static prefixes, UTC date hierarchy, stream/version/time/UUID object names, and `FileExtension` overrides. Objects have `Content-Type: application/octet-stream` and the encoded body's MD5 as their `ETag`. GZIP output has `Content-Encoding: gzip`; Snappy output has `Content-Encoding: snappy-java`; uncompressed output has no content encoding. Completed objects' bytes and metadata survive persistent-mode restart, including internal adapter delivery. Bucket versioning, default Object Lock retention, bucket-default encryption (including SSE-KMS), replication, and S3 event notifications apply through the shared S3 write path as for ordinary S3 writes, including for compressed objects. `EncryptionConfiguration.KMSEncryptionConfig` writes objects with SSE-KMS under the configured key (see [Caveats](#caveats)). Pending buffers and retry queues are process-local and are not persisted. - **Tags**: `ListTagsForDeliveryStream` / `TagDeliveryStream` / `UntagDeliveryStream`. Keyed by stream ARN. @@ -88,10 +88,14 @@ This increment meters only the public Direct PUT APIs, including their configura ### AWS references and local differences +**Compatibility change:** malformed batch entries no longer produce individual `InvalidArgumentException` results alongside accepted records. Both public PUT APIs validate every record before admission and reject the entire call for missing, wrong-type, or invalid-base64 data. Per-entry batch errors are reserved for throughput refusal; callers relying on partial acceptance of malformed batches must correct the input and retry the whole rejected call. + The [Firehose quotas guide](https://docs.aws.amazon.com/firehose/latest/dev/limits.html) and [regional quota reference](https://docs.aws.amazon.com/general/latest/gr/fh.html) define the regional defaults and shared per-stream Direct PUT limits. AWS describes proportional quota increases and automatic scaling with unspecified timing; fakecloud does not reproduce either workflow or AWS's internal admission algorithm. Local controls can change each dimension independently. The [`PutRecord` reference](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecord.html) documents HTTP 500 `ServiceUnavailableException`, but its introductory 5,000-records/second figure conflicts with the 500,000 figure in the quota references for the three high-throughput regions. The table above follows the quota references, not that introductory number. The [`PutRecordBatch` reference](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecordBatch.html) distinguishes whole-call errors from ordered per-record results; it does not specify the exact threshold-allocation algorithm. One-second accounting, burst carry-over, and the dimension-to-error mapping here are explicit local testing policies, not claims about AWS's unpublished throttling algorithm. +There is also an HTTP-status discrepancy: both API references document HTTP 500 for `ServiceUnavailableException`, while the vendored Smithy model (`aws-models/firehose.json`) specifies HTTP 503. Throttling follows the API references and returns HTTP 500; official-SDK tests check both that status and the modeled error type. Closed admission during local shutdown is a distinct condition and retains HTTP 503 `ServiceUnavailableException` with message `Firehose admission is closed`, not `Slow down.`. + ## S3 object bytes and names Create the destination bucket before sending records. Records `abc`, `def\n`, and the binary bytes `00 ff` in one delivery window become exactly `abcdef\n` followed by `00 ff`, whether submitted in one batch or separate calls. For compressed output, those concatenated bytes are encoded in one GZIP or Xerial Snappy stream. Producers must supply their own delimiters, such as a newline for each JSON record, or enable `AppendDelimiterToRecord`, which writes the same records as exactly `abc\ndef\n\n` followed by `00 ff 0a` before any compression. diff --git a/website/content/docs/services/servicequotas.md b/website/content/docs/services/servicequotas.md index 043992952..8118ca8b2 100644 --- a/website/content/docs/services/servicequotas.md +++ b/website/content/docs/services/servicequotas.md @@ -120,6 +120,15 @@ from an account outside an organization return approval). Accounts that were already members, or that joined by invitation, are left alone. +As a fallback, a due template is also applied when the account's quota state is +next accessed through Service Quotas, an applied-value lookup, or a lookup that +actually enforces a limit. An enforcement-only lookup with no active limits +returns without applying the template. This rule applies across enforcing +services, not just Firehose. A lookup that applies a template rereads the limits +afterward, including changes made by a concurrent lookup. Firehose's three +throughput quotas are non-adjustable and cannot be included in an increase +request template. + ### Tags **`TagResource`**, **`UntagResource`** and **`ListTagsForResource`** tag the From 7a3daab5ecf99e7f28f700507b619b707a886853 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Thu, 29 Oct 2026 12:08:51 -0300 Subject: [PATCH 6/6] refactor(firehose): simplify throughput accounting; restore quota template order - Service Quotas: apply a due organization template before snapshotting limits on every enforceable lookup, as before, so enforcing services keep their behavior with enforcement off. Drop the double-snapshot path and its test-only hook. - Firehose throughput: replace the compensated float helpers and the Debt anchor with a plain per-window Usage counter; same admission semantics (hard request cap, boundary-crossing record admitted, excess carried forward, no idle credit). - Prune a deleted stream's throughput window on DeleteDeliveryStream and on the metered path, not only when the S3 scheduler runs. - Revert the servicequotas docs paragraph on lazy template timing. --- crates/fakecloud-firehose/src/engine.rs | 19 +- .../src/engine/tests/throughput.rs | 44 +++++ crates/fakecloud-firehose/src/service.rs | 2 + crates/fakecloud-firehose/src/throughput.rs | 168 +++++------------- .../fakecloud-servicequotas/src/provider.rs | 144 ++------------- .../src/service/tests.rs | 28 ++- .../fakecloud-servicequotas/src/settings.rs | 44 ++--- crates/fakecloud-servicequotas/src/state.rs | 16 +- .../content/docs/services/servicequotas.md | 10 +- 9 files changed, 163 insertions(+), 312 deletions(-) diff --git a/crates/fakecloud-firehose/src/engine.rs b/crates/fakecloud-firehose/src/engine.rs index f8592f935..4f701acb4 100644 --- a/crates/fakecloud-firehose/src/engine.rs +++ b/crates/fakecloud-firehose/src/engine.rs @@ -234,6 +234,17 @@ impl FirehoseEngine { *self.core.quota_provider.write() = Some(provider); } + /// Forget the ingestion usage of streams that no longer exist, so a + /// deleted stream's usage goes with it even when nothing is buffered. + pub(crate) fn prune_throughput(&self) { + let config = self.core.state.read(); + self.core + .registry + .lock() + .windows + .retain(|key, _| current(&config, key).is_some()); + } + pub fn enqueue( &self, account: &str, @@ -328,10 +339,12 @@ impl FirehoseEngine { let now = self.core.clock.now(); let mut accepted = vec![true; records.len()]; if let Some(limits) = limits { - let window = registry + // Drop the usage of deleted or reset streams here too: the + // scheduler's pass never runs for a stream without S3 delivery. + registry .windows - .entry(key.clone()) - .or_insert_with(|| ThroughputWindow::new(now.monotonic)); + .retain(|key, _| current(&config, key).is_some()); + let window = registry.windows.entry(key.clone()).or_default(); if !window.try_request(now.monotonic, limits) { return Err(throughput_error()); } diff --git a/crates/fakecloud-firehose/src/engine/tests/throughput.rs b/crates/fakecloud-firehose/src/engine/tests/throughput.rs index 8676a9bd2..a1215c52b 100644 --- a/crates/fakecloud-firehose/src/engine/tests/throughput.rs +++ b/crates/fakecloud-firehose/src/engine/tests/throughput.rs @@ -660,3 +660,47 @@ async fn successful_delivery_and_destination_updates_do_not_reset_budget() { assert_eq!(batches[1].records, vec![record(b'b', 1)]); assert!(batches[1].object_key.starts_with("updated/")); } + +/// Without S3 delivery the scheduler never runs for a stream, so its usage +/// must go when the stream is deleted (or reset away) on its own. +#[tokio::test] +async fn deleted_streams_without_s3_delivery_drop_their_usage_without_a_scan() { + use fakecloud_core::service::AwsService; + let windows = |f: &Fixture| f.engine.core.registry.lock().windows.len(); + + // DeleteDeliveryStream prunes immediately. + let f = Fixture::new(60); + f.update(NAME, |s| s.destination = None); + enforce(&f, Some(10.0), None, None); + let svc = crate::FirehoseService::default().with_engine(f.engine.clone()); + put(&f, vec![record(b'a', 1)]).unwrap(); + assert_eq!(windows(&f), 1); + svc.handle(request( + "DeleteDeliveryStream", + json!({"DeliveryStreamName": NAME}), + )) + .await + .unwrap(); + assert_eq!(windows(&f), 0); + + // A reset only touches configuration; the next metered call prunes it. + let f = Fixture::new(60); + f.update(NAME, |s| s.destination = None); + enforce(&f, Some(10.0), None, None); + put(&f, vec![record(b'a', 1)]).unwrap(); + f.engine.core.state.write().accounts.clear(); + f.add_stream(ACCOUNT, REGION, "other", 60); + f.update("other", |s| s.destination = None); + f.engine + .enqueue_put(ACCOUNT, REGION, "other", vec![record(b'b', 1)]) + .unwrap(); + let registry = f.engine.core.registry.lock(); + assert_eq!( + registry + .windows + .keys() + .map(|key| key.name.as_str()) + .collect::>(), + ["other"] + ); +} diff --git a/crates/fakecloud-firehose/src/service.rs b/crates/fakecloud-firehose/src/service.rs index 695224b49..302feb717 100644 --- a/crates/fakecloud-firehose/src/service.rs +++ b/crates/fakecloud-firehose/src/service.rs @@ -846,6 +846,8 @@ impl FirehoseService { if state.streams_mut(&req.region).remove(name).is_none() { return Err(not_found(name)); } + drop(accounts); + self.engine.prune_throughput(); Ok(AwsResponse::ok_json(json!({}))) } diff --git a/crates/fakecloud-firehose/src/throughput.rs b/crates/fakecloud-firehose/src/throughput.rs index d37840d7f..d6489262f 100644 --- a/crates/fakecloud-firehose/src/throughput.rs +++ b/crates/fakecloud-firehose/src/throughput.rs @@ -3,6 +3,8 @@ use tokio::time::Instant; pub(crate) const THROUGHPUT_ERROR_CODE: &str = "ServiceUnavailableException"; pub(crate) const THROUGHPUT_ERROR_MESSAGE: &str = "Slow down."; +const BYTES_PER_MIB: f64 = 1_048_576.0; + #[derive(Clone, Copy, Default)] pub(crate) struct ThroughputLimits { pub requests: Option, @@ -11,133 +13,85 @@ pub(crate) struct ThroughputLimits { } /// Stream-local admission policy, not a model of AWS's unpublished algorithm. -/// Request capacity resets each second. Record/byte excess carries forward: +/// Time is split into one-second windows from the first admitted request. +/// Request capacity resets each window. Record/byte excess carries forward: /// a positive remaining allowance admits one whole record, even if it crosses -/// the boundary. Elapsed whole seconds repay debt but never accrue idle credit. +/// the boundary. Elapsed windows repay debt but never accrue idle credit. +#[derive(Default)] pub(crate) struct ThroughputWindow { - started_at: Instant, - active: bool, + started_at: Option, window: u64, requests: u64, - records: Debt, - bytes: Debt, + records: Usage, + bytes: Usage, } impl ThroughputWindow { - pub(crate) fn new(now: Instant) -> Self { - Self { - started_at: now, - active: false, - window: 0, - requests: 0, - records: Debt::default(), - bytes: Debt::default(), - } - } - /// Charge the whole call before considering individual records. pub(crate) fn try_request(&mut self, now: Instant, limits: ThroughputLimits) -> bool { - if self.active { - let window = now.duration_since(self.started_at).as_secs(); + if let Some(started_at) = self.started_at { + let window = now.duration_since(started_at).as_secs(); if window > self.window { self.window = window; self.requests = 0; } } - let Some(requests) = self.requests.checked_add(1) else { - return false; - }; if limits .requests - .is_some_and(|limit| count_as_f64_ceil(requests as u128) > limit) + .is_some_and(|limit| (self.requests + 1) as f64 > limit) { return false; } - if !self.active { - self.started_at = now; - self.active = true; - } - self.requests = requests; + self.started_at.get_or_insert(now); + self.requests += 1; true } /// Rejected entries debit neither dimension, leaving room for later entries /// (notably empty records when the byte allowance is exhausted). pub(crate) fn try_record(&mut self, limits: ThroughputLimits, bytes: usize) -> bool { - if !self.records.has_room(self.window, limits.records, 1.0) - || (bytes > 0 && !self.bytes.has_room(self.window, limits.mib, 1_048_576.0)) + let bytes_per_second = limits.mib.map(|mib| mib * BYTES_PER_MIB); + if !self.records.has_room(self.window, limits.records) + || (bytes > 0 && !self.bytes.has_room(self.window, bytes_per_second)) { return false; } // Keep accounting even for currently ignored dimensions when another // quota is enabled, so a live quota change does not erase usage. - self.records.charge(self.window, limits.records, 1.0, 1.0); + self.records.charge(self.window, limits.records, 1.0); if bytes > 0 { - self.bytes.charge( - self.window, - limits.mib, - 1_048_576.0, - count_as_f64_ceil(bytes as u128), - ); + self.bytes + .charge(self.window, bytes_per_second, bytes as f64); } true } } -/// Keep the last charge as an immutable repayment anchor. Rejected retries and -/// zero-byte records must not repeatedly round byte debt. Compare total usage -/// with the cumulative allowance before subtracting: e.g. 0.5 MiB against five -/// 0.1 MiB allowances is exactly full, not a tiny positive burst credit. +/// Usage of one dimension since the window of its last charge. Only a charge +/// repays elapsed windows, so rejected retries cannot change recovery timing. #[derive(Default)] -struct Debt { +struct Usage { used: f64, window: u64, } -impl Debt { - fn has_room(&self, window: u64, limit: Option, scale: f64) -> bool { - limit.is_none_or(|limit| self.used / scale < limit * ((window - self.window) as f64 + 1.0)) +impl Usage { + /// Whether usage is below the allowance of every window since the last + /// charge, up to and including `window`. + fn has_room(&self, window: u64, per_second: Option) -> bool { + per_second.is_none_or(|limit| self.used < limit * (window - self.window + 1) as f64) } - fn charge(&mut self, window: u64, limit: Option, scale: f64, cost: f64) { - let seconds = window - self.window; - let used = if seconds == 0 { - self.used - } else if let Some(limit) = limit { - let allowance = limit * seconds as f64; - // Divide before multiplying to avoid overflow at huge MiB limits. - if self.used / scale <= allowance { - 0.0 - } else { - self.used - allowance * scale - } - } else { - 0.0 - }; - self.used = add_ceil(used, cost); - self.window = window; - } -} - -/// Round an addition upward when its exact result lies between floats. Usage -/// must not round down and accidentally create room above large finite quotas. -fn add_ceil(a: f64, b: f64) -> f64 { - let sum = a + b; - let b_rounded = sum - a; - let error = (a - (sum - b_rounded)) + (b - b_rounded); - if error > 0.0 { - sum.next_up() - } else { - sum - } -} - -fn count_as_f64_ceil(count: u128) -> f64 { - let rounded = count as f64; - if (rounded as u128) < count { - rounded.next_up() - } else { - rounded + fn charge(&mut self, window: u64, per_second: Option, cost: f64) { + let elapsed = window - self.window; + if elapsed > 0 { + // Each elapsed window repays up to its allowance; an unlimited + // dimension starts over. Unused allowance is not banked. + let repaid = per_second.map_or(self.used, |limit| limit * elapsed as f64); + self.used = (self.used - repaid).max(0.0); + self.window = window; + } + self.used += cost; } } @@ -149,7 +103,7 @@ mod tests { #[test] fn request_limit_is_hard_and_first_success_anchors_window() { let now = Instant::now(); - let mut w = ThroughputWindow::new(now); + let mut w = ThroughputWindow::default(); let limits = ThroughputLimits { requests: Some(1.0), ..Default::default() @@ -175,7 +129,7 @@ mod tests { mib: Some(2.0 / 1_048_576.0), ..Default::default() }; - let mut w = ThroughputWindow::new(now); + let mut w = ThroughputWindow::default(); assert!(w.try_request(now, limits)); assert!(w.try_record(limits, 5)); assert!(!w.try_record(limits, 1)); @@ -195,7 +149,7 @@ mod tests { mib: Some(1.0 / 1_048_576.0), ..Default::default() }; - let mut w = ThroughputWindow::new(now); + let mut w = ThroughputWindow::default(); assert!(w.try_request(now, limits)); assert!(w.try_record(limits, 5)); assert!(w.try_request(now + Duration::from_secs(100), limits)); @@ -211,7 +165,7 @@ mod tests { mib: Some(0.25 / 1_048_576.0), ..Default::default() }; - let mut w = ThroughputWindow::new(now); + let mut w = ThroughputWindow::default(); assert!(w.try_request(now, limits)); assert!(w.try_record(limits, 1)); for seconds in [1, 2, 3] { @@ -229,8 +183,8 @@ mod tests { mib: Some(0.1), ..Default::default() }; - let mut frequent = ThroughputWindow::new(now); - let mut delayed = ThroughputWindow::new(now); + let mut frequent = ThroughputWindow::default(); + let mut delayed = ThroughputWindow::default(); for window in [&mut frequent, &mut delayed] { assert!(window.try_request(now, limits)); assert!(window.try_record(limits, 524_288)); @@ -251,7 +205,7 @@ mod tests { #[test] fn zero_limits_and_rejection_do_not_debit_other_dimensions() { let now = Instant::now(); - let mut w = ThroughputWindow::new(now); + let mut w = ThroughputWindow::default(); let limits = ThroughputLimits { records: Some(1.0), mib: Some(0.0), @@ -261,7 +215,7 @@ mod tests { assert!(!w.try_record(limits, 1)); assert!(w.try_record(limits, 0)); assert!(!w.try_record(limits, 0)); - let mut w = ThroughputWindow::new(now); + let mut w = ThroughputWindow::default(); assert!(!w.try_record( ThroughputLimits { records: Some(0.0), @@ -274,7 +228,7 @@ mod tests { #[test] fn live_changes_preserve_usage_and_track_ignored_dimensions() { let now = Instant::now(); - let mut w = ThroughputWindow::new(now); + let mut w = ThroughputWindow::default(); let limits = ThroughputLimits { requests: Some(10.0), ..Default::default() @@ -308,7 +262,7 @@ mod tests { #[test] fn rejected_requests_do_not_charge_records_or_move_boundary() { let now = Instant::now(); - let mut w = ThroughputWindow::new(now); + let mut w = ThroughputWindow::default(); let limits = ThroughputLimits { requests: Some(1.0), records: Some(1.0), @@ -320,30 +274,4 @@ mod tests { assert!(w.try_request(now + Duration::from_secs(1), limits)); assert!(w.try_record(limits, 0)); } - - #[test] - fn huge_finite_limits_and_large_counts_do_not_overflow_or_round_down() { - let now = Instant::now(); - let mut w = ThroughputWindow::new(now); - let limits = ThroughputLimits { - requests: Some(f64::MAX), - records: Some(f64::MAX), - mib: Some(f64::MAX), - }; - assert!(w.try_request(now, limits)); - assert!(w.try_record(limits, usize::MAX)); - assert!(w.try_request(now + Duration::from_secs(100), limits)); - assert!(w.try_record(limits, 1)); - w.requests = 9_007_199_254_740_992; - assert!(!w.try_request( - now + Duration::from_secs(100), - ThroughputLimits { - requests: Some(9_007_199_254_740_992.0), - ..limits - } - )); - w.requests = u64::MAX; - assert!(!w.try_request(now + Duration::from_secs(100), limits)); - assert!(add_ceil(9_007_199_254_740_992.0, 1.0) > 9_007_199_254_740_992.0); - } } diff --git a/crates/fakecloud-servicequotas/src/provider.rs b/crates/fakecloud-servicequotas/src/provider.rs index b0078c922..5231eb32f 100644 --- a/crates/fakecloud-servicequotas/src/provider.rs +++ b/crates/fakecloud-servicequotas/src/provider.rs @@ -252,12 +252,10 @@ impl ServiceQuotasProvider { impl ServiceQuotasProvider { /// Apply the organization's template to `account_id` if it is due, - /// persisting the change. Whether it changed anything. - fn apply_template(&self, account_id: &str) -> bool { + /// persisting the change. + fn apply_template(&self, account_id: &str) { let approval = self.settings.read().request_approval; - let applied = - apply_template_if_new(&self.state, &self.orgs, approval, account_id, Utc::now()); - if applied { + if apply_template_if_new(&self.state, &self.orgs, approval, account_id, Utc::now()) { // The lookup is synchronous; the snapshot write runs on the // runtime the enforcing service is called from. if let (Some(hook), Ok(rt)) = ( @@ -267,25 +265,6 @@ impl ServiceQuotasProvider { rt.spawn(hook()); } } - applied - } - - /// The enforced limits of `quota_codes`, from one settings and state - /// snapshot. - fn snapshot_limits( - &self, - account_id: &str, - region: &str, - service_code: &str, - quota_codes: &[&str], - ) -> Vec> { - let settings = self.settings.read(); - let guard = self.state.read(); - let mut account = AccountEnforcement::new(&settings, guard.get(account_id), region); - quota_codes - .iter() - .map(|code| account.limit(service_code, code)) - .collect() } } @@ -330,20 +309,17 @@ impl QuotaProvider for ServiceQuotasProvider { { return vec![None; quota_codes.len()]; } - let limits = self.snapshot_limits(account_id, region, service_code, quota_codes); - #[cfg(test)] - tests::after_initial_snapshot(); - if !limits.iter().any(Option::is_some) { - // This fast path applies to every enforcing service: an ignored - // quota must not trigger the lazy organization-template fallback. - // Explicit applied-value lookups still apply a due template. - return limits; - } - // Enforcing something touches the account's quotas, which applies a - // due organization template (see `apply_template_if_new`); answer - // from the values it left, even if another lookup applied it. + // Enforcing touches the account's quotas, which applies a due + // organization template (see `apply_template_if_new`) before the + // limits are read. self.apply_template(account_id); - self.snapshot_limits(account_id, region, service_code, quota_codes) + let settings = self.settings.read(); + let guard = self.state.read(); + let account = AccountEnforcement::new(&settings, guard.get(account_id), region); + quota_codes + .iter() + .map(|code| account.limit(service_code, code)) + .collect() } } @@ -351,19 +327,6 @@ impl QuotaProvider for ServiceQuotasProvider { mod tests { use super::*; - thread_local! { - // One-shot scheduling seam, isolated to the lookup's test thread. - static AFTER_INITIAL_SNAPSHOT: std::cell::RefCell>> = - const { std::cell::RefCell::new(None) }; - } - - pub(super) fn after_initial_snapshot() { - let hook = AFTER_INITIAL_SNAPSHOT.with(|hook| hook.borrow_mut().take()); - if let Some(hook) = hook { - hook(); - } - } - fn provider(settings: crate::settings::QuotaSettings) -> ServiceQuotasProvider { use parking_lot::RwLock; use std::sync::Arc; @@ -487,87 +450,6 @@ mod tests { ); } - #[test] - fn enforced_limits_resnapshot_when_another_lookup_applies_the_template() { - use fakecloud_organizations::{MemberAccount, OrganizationState}; - use std::sync::mpsc; - - const MGMT: &str = "111111111111"; - const MEMBER: &str = "222222222222"; - const REGION: &str = "us-east-1"; - const CODES: &[&str] = &["L-2AFB9258", "L-0EA8095F", "L-NOPE"]; - let p = provider(crate::settings::QuotaSettings { - enforce_all: true, - ..Default::default() - }); - let associated_at = DateTime::from_timestamp(1_700_000_000, 0).unwrap(); - let mut org = OrganizationState::bootstrap_in(REGION, MGMT); - org.accounts.insert( - MEMBER.to_string(), - MemberAccount { - id: MEMBER.to_string(), - arn: org.account_arn(MEMBER), - email: format!("{MEMBER}@example.com"), - name: "member".into(), - status: "ACTIVE".into(), - joined_method: "CREATED".into(), - joined_timestamp: associated_at + chrono::Duration::seconds(1), - parent_id: org.root_id.clone(), - gov_cloud_mirror: false, - }, - ); - p.orgs.write().insert(org); - { - let mut guard = p.state.write(); - let management = guard.get_or_create(MGMT); - management.template_associated_at = Some(associated_at); - management.template.insert( - crate::state::template_key(catalog::VPC, CODES[0], REGION), - TemplateEntry { - service_code: catalog::VPC.into(), - quota_code: CODES[0].into(), - aws_region: REGION.into(), - desired_value: 8.0, - }, - ); - } - assert_eq!( - p.snapshot_limits(MEMBER, REGION, catalog::VPC, CODES), - [Some(5.0), Some(60.0), None] - ); - - std::thread::scope(|scope| { - let (snapshotted_tx, snapshotted_rx) = mpsc::channel(); - let (resume_tx, resume_rx) = mpsc::channel(); - let p = &p; - let losing_lookup = scope.spawn(move || { - AFTER_INITIAL_SNAPSHOT.with(|hook| { - *hook.borrow_mut() = Some(Box::new(move || { - snapshotted_tx.send(()).unwrap(); - resume_rx.recv().unwrap(); - })); - }); - p.enforced_limits(MEMBER, REGION, catalog::VPC, CODES) - }); - - // Both lookups snapshot the defaults before the winner applies - // the real template. The paused lookup then sees its marker and - // gets false from apply_template, but must still return 8, not 5. - snapshotted_rx.recv().unwrap(); - let winning_limits = p.enforced_limits(MEMBER, REGION, catalog::VPC, CODES); - resume_tx.send(()).unwrap(); - let losing_limits = losing_lookup.join().unwrap(); - assert_eq!(winning_limits, [Some(8.0), Some(60.0), None]); - assert_eq!(losing_limits, [Some(8.0), Some(60.0), None]); - }); - assert!(!p.apply_template(MEMBER)); - let guard = p.state.read(); - let member = guard.get(MEMBER).unwrap(); - assert!(member.template_checked.is_some()); - assert_eq!(member.requests.len(), 1, "template applied more than once"); - assert_eq!(member.requests.values().next().unwrap().status, "APPROVED"); - } - /// With enforcement off, an enforcing service's limit lookup on its /// request path never touches (or decodes) the catalog. #[test] diff --git a/crates/fakecloud-servicequotas/src/service/tests.rs b/crates/fakecloud-servicequotas/src/service/tests.rs index a03b7c8d0..a33383dde 100644 --- a/crates/fakecloud-servicequotas/src/service/tests.rs +++ b/crates/fakecloud-servicequotas/src/service/tests.rs @@ -1293,10 +1293,11 @@ fn firehose_introspection_reports_auto_scaling_defaults_as_unenforced() { assert_eq!(state(&deleted), off("auto_scaling_default")); } -/// Another service's limit lookup applies a due Organizations template only -/// when it enforces something; an unenforced lookup leaves the account alone. +/// Another service's limit lookup of an enforceable quota applies a due +/// Organizations template, whether or not it enforces anything, and applies +/// it once. #[test] -fn enforced_limit_applies_the_template_only_once_enforced() { +fn enforced_limit_applies_a_due_template_even_when_unenforced() { use fakecloud_core::quota::QuotaProvider; let o = org_with_member(3600); let s = svc_with(o); @@ -1313,7 +1314,26 @@ fn enforced_limit_applies_the_template_only_once_enforced() { p.enforced_limit(MEMBER, "us-east-1", "vpc", "L-2AFB9258"), None ); - assert!(s.state.read().get(MEMBER).is_none(), "template applied"); + assert_eq!( + s.state.read().get(MEMBER).unwrap().requests.len(), + 1, + "template not applied" + ); + // A quota no service checks never touches the account's quotas. + let s2 = svc_with(org_with_member(3600)); + run( + &s2, + MGMT, + "PutServiceQuotaIncreaseRequestIntoTemplate", + json!({ "ServiceCode": "vpc", "QuotaCode": "L-2AFB9258", "AwsRegion": "us-east-1", "DesiredValue": 8.0 }), + ) + .unwrap(); + run(&s2, MGMT, "AssociateServiceQuotaTemplate", json!({})).unwrap(); + assert_eq!( + provider(&s2).enforced_limit(MEMBER, "us-east-1", "lambda", "L-B99A9384"), + None + ); + assert!(s2.state.read().get(MEMBER).is_none(), "template applied"); enforce_all(&s, true); assert_eq!( diff --git a/crates/fakecloud-servicequotas/src/settings.rs b/crates/fakecloud-servicequotas/src/settings.rs index ed1b1b769..ce45cac3f 100644 --- a/crates/fakecloud-servicequotas/src/settings.rs +++ b/crates/fakecloud-servicequotas/src/settings.rs @@ -24,7 +24,7 @@ use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use crate::catalog::{self, QuotaDef}; -use crate::state::{write_applied_key, ServiceQuotasData}; +use crate::state::{applied_key, ServiceQuotasData}; /// How `RequestServiceQuotaIncrease` (and template entries applied to new /// accounts) are decided. @@ -110,14 +110,7 @@ pub fn quota_ref(def: &QuotaDef) -> String { /// [`quota_ref`] from the codes. pub fn quota_ref_of(service_code: &str, quota_code: &str) -> String { - let mut key = String::new(); - write_quota_ref(&mut key, service_code, quota_code); - key -} - -fn write_quota_ref(buf: &mut String, service_code: &str, quota_code: &str) { - buf.clear(); - buf.extend([service_code, "/", quota_code]); + format!("{service_code}/{quota_code}") } /// Resolve a `service_code/quota_code` reference to an enforceable quota (as @@ -155,13 +148,11 @@ pub fn enforcement( /// Effective enforcement for one account in one region, read from one /// settings and state snapshot: the single place both the /// [`fakecloud_core::quota::QuotaProvider`] and the introspection view decide -/// whether a quota is enforced. Lookups reuse one key buffer, so resolving -/// several quotas allocates at most once. +/// whether a quota is enforced. pub struct AccountEnforcement<'a> { settings: &'a QuotaSettings, data: Option<&'a ServiceQuotasData>, region: &'a str, - key: String, } impl<'a> AccountEnforcement<'a> { @@ -174,7 +165,6 @@ impl<'a> AccountEnforcement<'a> { settings, data, region, - key: String::new(), } } @@ -184,7 +174,7 @@ impl<'a> AccountEnforcement<'a> { /// the account set explicitly for the region. The catalog is decoded only /// for such a quota once it is switched on, so the request path of a /// service that enforces nothing stays cheap. - pub fn of(&mut self, service_code: &str, quota_code: &str) -> (bool, EnforcementSource) { + pub fn of(&self, service_code: &str, quota_code: &str) -> (bool, EnforcementSource) { if !catalog::is_enforceable(service_code, quota_code) { return (false, EnforcementSource::NotEnforceable); } @@ -202,7 +192,7 @@ impl<'a> AccountEnforcement<'a> { /// The limit fakecloud enforces for `service_code/quota_code`: its /// applied value while [`AccountEnforcement::of`] says it is enforced. - pub fn limit(&mut self, service_code: &str, quota_code: &str) -> Option { + pub fn limit(&self, service_code: &str, quota_code: &str) -> Option { if !self.of(service_code, quota_code).0 { return None; } @@ -215,13 +205,12 @@ impl<'a> AccountEnforcement<'a> { /// The account switch, else the server-wide override, else the global /// switch. - fn switch(&mut self, service_code: &str, quota_code: &str) -> (bool, EnforcementSource) { - write_quota_ref(&mut self.key, service_code, quota_code); - let key = self.key.as_str(); - if let Some(on) = self.data.and_then(|d| d.enforcement.get(key)) { + fn switch(&self, service_code: &str, quota_code: &str) -> (bool, EnforcementSource) { + let key = quota_ref_of(service_code, quota_code); + if let Some(on) = self.data.and_then(|d| d.enforcement.get(&key)) { return (*on, EnforcementSource::AccountOverride); } - if let Some(on) = self.settings.overrides.get(key) { + if let Some(on) = self.settings.overrides.get(&key) { return (*on, EnforcementSource::Override); } (self.settings.enforce_all, EnforcementSource::Global) @@ -229,16 +218,9 @@ impl<'a> AccountEnforcement<'a> { /// The applied value the account set for `def` in the region (an /// approved request or the introspection API), if any. - fn explicit_value(&mut self, def: &QuotaDef) -> Option { - let data = self.data?; - write_applied_key( - &mut self.key, - self.region, - def.global, - def.service_code, - def.quota_code, - ); - data.applied.get(self.key.as_str()).copied() + fn explicit_value(&self, def: &QuotaDef) -> Option { + let key = applied_key(self.region, def.global, def.service_code, def.quota_code); + self.data?.applied.get(&key).copied() } } @@ -342,7 +324,7 @@ mod tests { (false, AutoScalingDefault) ); - let mut account = AccountEnforcement::new(&settings, Some(&data), "us-east-2"); + let account = AccountEnforcement::new(&settings, Some(&data), "us-east-2"); assert_eq!(account.limit(def.service_code, def.quota_code), Some(9.0)); assert_eq!( account.limit(def.service_code, catalog::FIREHOSE_MIB_PER_SECOND), diff --git a/crates/fakecloud-servicequotas/src/state.rs b/crates/fakecloud-servicequotas/src/state.rs index a3661fed3..c13386927 100644 --- a/crates/fakecloud-servicequotas/src/state.rs +++ b/crates/fakecloud-servicequotas/src/state.rs @@ -136,22 +136,8 @@ impl AccountState for ServiceQuotasData { /// Key of an applied value. Global quotas apply in every region, so their key /// carries no region. pub fn applied_key(region: &str, global: bool, service_code: &str, quota_code: &str) -> String { - let mut key = String::new(); - write_applied_key(&mut key, region, global, service_code, quota_code); - key -} - -/// [`applied_key`] written over `buf`, so repeated lookups reuse one buffer. -pub(crate) fn write_applied_key( - buf: &mut String, - region: &str, - global: bool, - service_code: &str, - quota_code: &str, -) { let region = if global { "" } else { region }; - buf.clear(); - buf.extend([region, "|", service_code, "|", quota_code]); + format!("{region}|{service_code}|{quota_code}") } pub fn template_key(service_code: &str, quota_code: &str, aws_region: &str) -> String { diff --git a/website/content/docs/services/servicequotas.md b/website/content/docs/services/servicequotas.md index 8118ca8b2..e81a614d1 100644 --- a/website/content/docs/services/servicequotas.md +++ b/website/content/docs/services/servicequotas.md @@ -120,14 +120,8 @@ from an account outside an organization return approval). Accounts that were already members, or that joined by invitation, are left alone. -As a fallback, a due template is also applied when the account's quota state is -next accessed through Service Quotas, an applied-value lookup, or a lookup that -actually enforces a limit. An enforcement-only lookup with no active limits -returns without applying the template. This rule applies across enforcing -services, not just Firehose. A lookup that applies a template rereads the limits -afterward, including changes made by a concurrent lookup. Firehose's three -throughput quotas are non-adjustable and cannot be included in an increase -request template. +Firehose's three throughput quotas are non-adjustable and cannot be included +in an increase request template. ### Tags