From b5946b90afb22f794fdebe320f1f7831f047edb0 Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Tue, 3 Nov 2026 20:11:16 +0100 Subject: [PATCH 1/2] feat: add Pod-scoped S3 gateway commands --- Cargo.lock | 13 +- Cargo.toml | 1 + README.md | 56 ++- src/cmd/mod.rs | 29 +- src/config.rs | 2 + src/describe.rs | 14 + src/main.rs | 15 +- src/output.rs | 5 + src/s3.rs | 598 ++++++++++++++++++++++++++ src/s3/multipart.rs | 533 +++++++++++++++++++++++ src/s3/test-certificate.pem | 18 + src/s3/test-private-key.pem | 28 ++ src/s3/tests.rs | 816 ++++++++++++++++++++++++++++++++++++ 13 files changed, 2123 insertions(+), 5 deletions(-) create mode 100644 src/s3.rs create mode 100644 src/s3/multipart.rs create mode 100644 src/s3/test-certificate.pem create mode 100644 src/s3/test-private-key.pem create mode 100644 src/s3/tests.rs diff --git a/Cargo.lock b/Cargo.lock index faf7c8c..8eef30a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,6 +257,7 @@ dependencies = [ "prost", "prost-types", "protox", + "quick-xml", "reqwest", "rustls", "serde", @@ -1411,6 +1412,16 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.45" @@ -1978,7 +1989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index 2ea6afb..efe826d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ flate2 = "1.1.2" getrandom = "0.3.4" prost = "0.13.5" prost-types = "0.13.5" +quick-xml = { version = "0.38.4", features = ["serialize"] } reqwest = { version = "0.13.4", default-features = false, features = ["charset", "http2", "json", "query", "rustls-no-provider", "stream", "system-proxy"] } rustls = { version = "0.23.40", default-features = false, features = ["ring", "std", "tls12"] } serde = { version = "1.0.228", features = ["derive"] } diff --git a/README.md b/README.md index 0565de0..94f3d6c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ # Brainpod CLI -A non-interactive CLI for managing Brainpod pods, images, blueprints, revisions, resources, deployments, database tunnels, and events. Its default output is deterministic line-oriented text suitable for LLMs and shell tools. Add `--json` to receive machine-readable JSON; login, database tunnels, and event watches use NDJSON. +A non-interactive CLI for managing Brainpod pods, images, blueprints, revisions, resources, deployments, object storage, database tunnels, and events. Its default output is deterministic line-oriented text suitable for LLMs and shell tools. Add `--json` to receive machine-readable JSON; login, database tunnels, and event watches use NDJSON. The CLI builds application images locally from an existing Dockerfile or with Railpack, then pushes them directly to the selected pod's private Brainpod registry namespace. Image builds probe the API's cluster architectures, prefer amd64 and then arm64, and store the selected default architecture in the configuration. Use `--platform linux/arm64` for a one-off override. @@ -373,3 +373,57 @@ Create and deploy: brainpod resource create --file resources.json --json brainpod deploy --summary "Configure application resources" --json ``` + +## S3 gateway + +Use the normal Brainpod API token and selected Pod (`--pod`, `BRAINPOD_POD`, or configured default). The key needs explicit `s3:list`, `s3:read`, and/or `s3:write` policies for that Pod; write includes replacement and deletion. Existing keys are not expanded automatically. Buckets must already be provisioned through the resource workflow. + +```sh +brainpod --pod my-pod s3 ls +brainpod --pod my-pod s3 ls s3://uploads/images/ +brainpod --pod my-pod s3 cp ./photo.jpg s3://uploads/images/photo.jpg +brainpod --pod my-pod s3 upload ./photo.jpg s3://uploads/photo.jpg +brainpod --pod my-pod s3 cp s3://uploads/photo.jpg ./downloaded.jpg +brainpod --pod my-pod s3 cp s3://uploads/photo.jpg s3://archive/photo.jpg +brainpod --pod my-pod s3 mv s3://uploads/photo.jpg s3://uploads/renamed.jpg +brainpod --pod my-pod s3 rm s3://uploads/renamed.jpg +``` + +Remote paths are `s3:///` within the selected Pod, not provider bucket names or URLs. Keys are literal UTF-8: quote shell metacharacters, and do not pre-percent-encode spaces, plus signs, or percent signs. Repeated and trailing slashes are preserved; dot-segment object keys are rejected to prevent URL normalization. Listings show one level of objects/prefixes and automatically fetch every page. `--json` returns an `items` array or a transfer/deletion result. + +Transfers require explicit file/key destinations (no implicit basename or directory expansion). Local destinations must not already exist; downloads are saved atomically after success. Remote destinations replace existing keys. All `cp`/`mv` transfers, including downloads and gateway-to-gateway copies, and file-based `upload` are limited to **25 MiB** and buffered in bounded memory. Explicit streaming upload/download modes are described below. Gateway copies download then upload, not native CopyObject. `mv` deletes its source only after a successful copy; a deletion failure reports that both copies may remain. Moves are not atomic: avoid concurrent changes to the source. `rm` deletes one exact key, including a trailing-slash marker, never prefix contents. + +The default gateway origin is `https://s3.brainpod.io`; the CLI prepends the selected Pod name to its hostname. Override the base origin with `--s3-endpoint`, `BRAINPOD_S3_ENDPOINT`, or `brainpod config set s3-endpoint https://s3.example.test` (in that precedence order). For local development, use `http://s3.brainpod.localhost` with working wildcard DNS. HTTP is unencrypted and should only be used locally. The API endpoint does not change the S3 origin. + +Downloads explicitly accept the gateway's 307 redirect and use a separate unauthenticated HTTPS request to the short-lived provider URL. Brainpod Bearer credentials are never sent to the provider; further redirects are rejected. Presigned URLs are not printed. There is no AWS SigV4 client authentication, public HEAD, recursive/sync transfer or deletion, bucket administration, ranged reads, or metadata preservation for `cp`/`mv`. Multipart is reserved for explicit stdin streaming uploads. Transfer errors return a nonzero exit status; interrupted uploads can have an uncertain remote outcome, so inspect the key before retrying. + +### Streaming transfers + +`upload -` uses sequential **8 MiB** multipart parts (up to 10,000 parts, about 78 GiB), rather than buffering the entire object. It requires a multipart-capable endpoint and `s3:write`. + +EOF alone cannot prove that an upstream producer succeeded, so stdin uploads require `--require-success-file PATH`. After EOF, completion is allowed only when that regular file contains exactly `commit` followed by one newline. Use a fresh private directory for each attempt and write the marker only after the entire producer pipeline succeeds. Keep the producer subshell's stdout open until after writing the marker, so EOF cannot race its creation. + +```bash +set -euo pipefail +run_dir=$(mktemp -d) +trap 'rm -rf "$run_dir"' EXIT +( + set -euo pipefail + cat ./large-object.bin + printf 'commit\n' > "$run_dir/success" +) | brainpod s3 upload - s3://uploads/large-object.bin \ + --require-success-file "$run_dir/success" \ + --metadata project=example +``` + +Generic object metadata is accepted on multipart initiation using repeated `--metadata KEY=VALUE`. Keys must be nonempty valid HTTP header names; values must be nonempty printable ASCII. Metadata names and values together may total at most 2048 bytes, and duplicate keys are rejected case-insensitively. Endpoint-specific metadata policies are enforced by the server. + +Missing or invalid markers, input errors, rejected parts, malformed completion responses (including HTTP 200 ``), SIGINT, and SIGTERM fail the upload and attempt to abort it. Completion/network failures can have an uncertain outcome; inspect the exact key before retrying. Abrupt process termination or failed aborts can leave incomplete uploads requiring endpoint cleanup. + +Stream an object to stdout without the small-copy limit: + +```sh +brainpod s3 download s3://uploads/large-object.bin - > ./large-object.bin +``` + +`download` currently requires destination `-`. Stdout contains only raw object bytes, even with `--json`; diagnostics stay on stderr. Provider requests use unauthenticated HTTPS with redirects disabled. An interrupted transfer can emit a partial object, so check the complete pipeline's exit status; early consumer closure follows normal Unix broken-pipe behavior. Network inactivity is bounded, but streaming downloads have no whole-object timeout or size cap. Input/output use bounded worker handoffs so cancellation does not wait for an unresponsive pipe reader/writer. diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 4877aa4..6541d08 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -33,6 +33,8 @@ pub enum Command { Blueprint(BlueprintArgs), /// Manage container images Image(ImageArgs), + /// Transfer and list objects through the Pod-scoped S3 gateway + S3(crate::s3::S3Args), /// Inspect pod revisions Revision(RevisionArgs), /// Create a local TCP tunnel to a deployed database @@ -95,6 +97,7 @@ enum ConfigKey { Endpoint, ControlPlaneEndpoint, RegistryEndpoint, + S3Endpoint, ApiToken, Pod, Architecture, @@ -497,7 +500,11 @@ pub fn needs_api_token(command: &Command) -> bool { pub fn needs_client(command: &Command) -> bool { !matches!( command, - Command::Describe(_) | Command::Agent(_) | Command::Login(_) | Command::Config(_) + Command::S3(_) + | Command::Describe(_) + | Command::Agent(_) + | Command::Login(_) + | Command::Config(_) ) } @@ -509,6 +516,7 @@ pub struct CommandContext<'a> { pub pod: Option<&'a str>, pub api_token: Option<&'a str>, pub registry_endpoint: &'a str, + pub s3_endpoint: &'a str, pub config: &'a mut Config, pub config_path: &'a Path, pub show_progress: bool, @@ -524,6 +532,7 @@ pub async fn handle(command: Command, context: CommandContext<'_>) -> Result) -> Result handle_config(args, config, config_path), + Command::S3(args) => { + crate::s3::handle( + args, + s3_endpoint, + pod_required(pod)?, + api_token.ok_or_else(|| anyhow!("API token is required"))?, + ) + .await + } Command::Whoami => Ok(CommandOutput::new( client_required(client)?.get(&["v1", "me"], &[]).await?, View::Whoami, @@ -652,6 +670,7 @@ fn handle_config(args: ConfigArgs, config: &mut Config, path: &Path) -> Result Result { + config.s3_endpoint = Some(value); + "s3Endpoint" + } ConfigKey::RegistryEndpoint => { config.registry_endpoint = Some(value); "registryEndpoint" @@ -708,6 +731,10 @@ fn handle_config(args: ConfigArgs, config: &mut Config, path: &Path) -> Result { + config.s3_endpoint = None; + "s3Endpoint" + } ConfigKey::RegistryEndpoint => { config.registry_endpoint = None; "registryEndpoint" diff --git a/src/config.rs b/src/config.rs index 99a0b2c..a60fb76 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; pub const DEFAULT_ENDPOINT: &str = "https://api.brainpod.io"; pub const DEFAULT_CONTROL_PLANE_ENDPOINT: &str = "https://control-plane.brainpod.io"; pub const DEFAULT_DASHBOARD_ENDPOINT: &str = "https://console.brainpod.io"; +pub const DEFAULT_S3_ENDPOINT: &str = "https://s3.brainpod.io"; pub const DEFAULT_REGISTRY_ENDPOINT: &str = "https://registry.brainpod.io"; #[derive(Clone, Debug, Default, Deserialize, Serialize)] @@ -14,6 +15,7 @@ pub const DEFAULT_REGISTRY_ENDPOINT: &str = "https://registry.brainpod.io"; pub struct Config { pub endpoint: Option, pub control_plane_endpoint: Option, + pub s3_endpoint: Option, pub registry_endpoint: Option, pub api_token: Option, pub pod: Option, diff --git a/src/describe.rs b/src/describe.rs index bb6796c..f5b0405 100644 --- a/src/describe.rs +++ b/src/describe.rs @@ -38,6 +38,7 @@ pub fn generate(mut root: Command, path: &[String]) -> Result { "eventWatchJson": "Event watches emit one JSON value per line as NDJSON.", "tunnelJson": "Tunnels emit listening, optional credentials, and closed events as NDJSON.", "waitProgress": "Interactive waits report unhealthy-to-healthy transitions on stderr; progress is suppressed when stderr is redirected or --json is used.", + "s3Download": "s3 download - writes raw bytes only, even with --json; errors remain on stderr.", "errors": "Errors are written to stderr and return a non-zero exit code; --json also makes errors JSON." }, "guidance": [ @@ -251,6 +252,7 @@ fn requirements(path: &[&str]) -> (Option, Option) { ["blueprint", _] | ["whoami"] | ["cluster"] | ["cluster", _] | ["pod"] | ["pod", _] => { (Some(true), Some(false)) } + ["s3"] | ["s3", _] => (Some(true), Some(true)), ["image"] | ["image", _] | ["revision"] @@ -288,6 +290,18 @@ fn effect(path: &[&str]) -> (&'static str, &'static str) { | ["resource", "list"] | ["resource", "get"] | ["resource", "variables"] => ("read", "Reads remote state without changing it."), + ["s3", "ls"] => ( + "read", + "Lists gateway buckets or one level of object prefixes.", + ), + ["s3", "download"] => ( + "read-stream", + "Streams provider object bytes to stdout without a text or JSON summary.", + ), + ["s3", _] => ( + "local-and-remote-write", + "Transfers one bounded object or deletes one exact key; moves delete only after a successful copy.", + ), ["events"] => ( "read-or-stream", "Reads or continuously streams remote events.", diff --git a/src/main.rs b/src/main.rs index 630be05..5488e46 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,13 +15,14 @@ mod describe; mod image; mod openapi; mod output; +mod s3; mod tunnel; use client::{ApiError, Client}; use cmd::Command; use config::{ Config, DEFAULT_CONTROL_PLANE_ENDPOINT, DEFAULT_DASHBOARD_ENDPOINT, DEFAULT_ENDPOINT, - DEFAULT_REGISTRY_ENDPOINT, + DEFAULT_REGISTRY_ENDPOINT, DEFAULT_S3_ENDPOINT, }; const UPGRADE_URL: &str = "https://console.brainpod.io/onboarding?upgrade=1"; @@ -30,7 +31,7 @@ const UPGRADE_URL: &str = "https://console.brainpod.io/onboarding?upgrade=1"; #[command( name = "brainpod", version, - about = "Manage Brainpod deployments, images, resources, and database tunnels", + about = "Manage Brainpod deployments, images, resources, object storage, and database tunnels", after_help = "For machine-readable command metadata, run `brainpod describe --json`." )] pub(crate) struct Opts { @@ -54,6 +55,10 @@ pub(crate) struct Opts { #[arg(long, global = true)] registry_endpoint: Option, + /// S3 gateway base origin; the selected Pod is prepended to its hostname + #[arg(long, global = true)] + s3_endpoint: Option, + /// Default pod for pod-scoped commands #[arg(long, global = true)] pod: Option, @@ -152,6 +157,11 @@ async fn run(opts: Opts) -> Result { .or_else(|| environment("BRAINPOD_REGISTRY_ENDPOINT")) .or_else(|| config.registry_endpoint.clone()) .unwrap_or_else(|| DEFAULT_REGISTRY_ENDPOINT.to_owned()); + let s3_endpoint = opts + .s3_endpoint + .or_else(|| environment("BRAINPOD_S3_ENDPOINT")) + .or_else(|| config.s3_endpoint.clone()) + .unwrap_or_else(|| DEFAULT_S3_ENDPOINT.to_owned()); let pod = opts .pod .or_else(|| environment("BRAINPOD_POD")) @@ -189,6 +199,7 @@ async fn run(opts: Opts) -> Result { pod: pod.as_deref(), api_token: api_token.as_deref(), registry_endpoint: ®istry_endpoint, + s3_endpoint: &s3_endpoint, config: &mut config, config_path: &config_path, show_progress, diff --git a/src/output.rs b/src/output.rs index cf6c0bd..b864d56 100644 --- a/src/output.rs +++ b/src/output.rs @@ -6,6 +6,7 @@ use serde_json::{Value, json}; use crate::client::{EventStreamMessage, EventWatch}; pub enum CommandOutput { + Silent, Buffered { value: Value, view: View }, EventWatch(Box), } @@ -22,6 +23,7 @@ impl CommandOutput { #[derive(Clone, Copy)] pub enum View { + S3, Describe, ResourceSchema, AgentStart, @@ -60,6 +62,7 @@ pub enum View { pub async fn write(output: CommandOutput, json: bool) -> Result<()> { match output { + CommandOutput::Silent => Ok(()), CommandOutput::Buffered { value, view } => write_buffered(&value, view, json), CommandOutput::EventWatch(watch) => write_event_watch(*watch, json).await, } @@ -159,6 +162,7 @@ fn stream_error(data: &str) -> anyhow::Error { fn render(value: &Value, view: View, color: bool) -> Vec { match view { + View::S3 => crate::s3::render(value), View::Describe => render_describe(value), View::ResourceSchema => render_resource_schema(value), View::AgentStart => crate::agent::render_start(value), @@ -514,6 +518,7 @@ fn render_config_show(value: &Value) -> Vec { field(value, "controlPlaneEndpoint") ), format!("Registry endpoint: {}", field(value, "registryEndpoint")), + format!("S3 endpoint: {}", field(value, "s3Endpoint")), format!( "API token configured: {}", yes_no(value_at(value, "apiTokenConfigured")) diff --git a/src/s3.rs b/src/s3.rs new file mode 100644 index 0000000..b207af9 --- /dev/null +++ b/src/s3.rs @@ -0,0 +1,598 @@ +use std::collections::HashSet; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use clap::{Args, Subcommand}; +use reqwest::{Method, StatusCode, Url}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::output::{CommandOutput, View}; + +const MAX_OBJECT: usize = 25 * 1024 * 1024; +const MAX_XML: usize = 4 * 1024 * 1024; + +#[derive(Debug, Args)] +pub struct S3Args { + #[command(subcommand)] + command: S3Command, +} + +#[derive(Debug, Subcommand)] +enum S3Command { + /// List buckets, or one level beneath s3://bucket/prefix (all pages) + Ls { path: Option }, + /// Copy one object (at most 25 MiB); remote paths require s3://bucket/key + Cp { source: String, destination: String }, + /// Copy one object, then delete its source only after success + Mv { source: String, destination: String }, + /// Upload one local file, or multipart stdin (-) with an explicit producer commit file + Upload { + source: String, + destination: String, + /// Required for stdin: private per-run file containing commit followed by a newline + #[arg(long)] + require_success_file: Option, + /// Object metadata; repeat KEY=VALUE + #[arg(long = "metadata", value_name = "KEY=VALUE")] + metadata: Vec, + }, + /// Stream a remote object to stdout (-), without the cp size limit + Download { source: String, destination: String }, + /// Delete exactly one key, never its prefix contents + Rm { path: String }, +} + +#[derive(Debug, PartialEq, Eq)] +struct Remote { + bucket: String, + key: String, +} + +impl Remote { + fn parse(value: &str) -> Result> { + let Some(path) = value.strip_prefix("s3://") else { + return Ok(None); + }; + let (bucket, key) = path.split_once('/').unwrap_or((path, "")); + if bucket.is_empty() + || !bucket + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'.') + || matches!(bucket, "." | "..") + { + bail!("invalid logical bucket in S3 path"); + } + Ok(Some(Self { + bucket: bucket.to_owned(), + key: key.to_owned(), + })) + } + + fn object(&self) -> Result<()> { + if self.key.is_empty() || self.key.len() > 1024 { + bail!("object paths require an explicit key of 1–1024 bytes"); + } + if self.key.split('/').any(|s| matches!(s, "." | "..")) { + bail!( + "dot-segment object keys are unsupported because URL handling would normalize them" + ); + } + Ok(()) + } +} + +struct Gateway<'a> { + http: reqwest::Client, + origin: Url, + token: &'a str, +} + +impl<'a> Gateway<'a> { + fn new(endpoint: &str, pod: &str, token: &'a str) -> Result { + let mut origin = Url::parse(endpoint).context("invalid S3 endpoint")?; + if !matches!(origin.scheme(), "http" | "https") + || !origin.username().is_empty() + || origin.password().is_some() + || origin.path() != "/" + || origin.query().is_some() + || origin.fragment().is_some() + { + bail!( + "S3 endpoint must be an HTTP(S) origin without credentials, path, query, or fragment" + ); + } + if pod.is_empty() || !pod.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') { + bail!("S3 requires a Pod hostname name, not an ID or URL"); + } + let host = origin + .host_str() + .ok_or_else(|| anyhow!("S3 endpoint has no hostname"))?; + origin.set_host(Some(&format!("{pod}.{host}")))?; + Ok(Self { + http: crate::http_client_builder()? + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(30)) + .read_timeout(Duration::from_secs(60)) + .build()?, + origin, + token, + }) + } + + fn url(&self, remote: Option<&Remote>) -> Result { + let mut url = self.origin.clone(); + if let Some(remote) = remote { + let mut segments = url + .path_segments_mut() + .map_err(|_| anyhow!("invalid gateway URL"))?; + segments.clear().push(&remote.bucket); + if !remote.key.is_empty() { + remote.object()?; + for segment in remote.key.split('/') { + segments.push(segment); + } + } + } + Ok(url) + } + + async fn request( + &self, + method: Method, + url: Url, + body: Option>, + ) -> Result { + let mut request = self + .http + .request(method, url) + .bearer_auth(self.token) + .timeout(Duration::from_secs(120)); + if let Some(body) = body { + request = request + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(body); + } + request + .send() + .await + .map_err(|_| anyhow!("S3 gateway request failed")) + } + + async fn delete(&self, remote: &Remote) -> Result<()> { + remote.object()?; + let response = self + .request(Method::DELETE, self.url(Some(remote))?, None) + .await?; + expect(response, StatusCode::NO_CONTENT).await?; + Ok(()) + } + + async fn download(&self, remote: &Remote) -> Result> { + bounded_body(self.provider_response(remote, false).await?, MAX_OBJECT).await + } + + async fn provider_response( + &self, + remote: &Remote, + streaming: bool, + ) -> Result { + remote.object()?; + let response = self + .request(Method::GET, self.url(Some(remote))?, None) + .await?; + let response = expect(response, StatusCode::TEMPORARY_REDIRECT).await?; + let location = response + .headers() + .get(reqwest::header::LOCATION) + .ok_or_else(|| anyhow!("gateway redirect has no Location"))? + .to_str()?; + let url = provider_url(location)?; + let mut request = self.http.get(url); + if !streaming { + request = request.timeout(Duration::from_secs(120)); + } + let response = request + .send() + .await + .map_err(|_| anyhow!("provider download request failed"))?; + expect(response, StatusCode::OK).await + } + + async fn copy( + &self, + source: &str, + destination: &str, + moving: bool, + upload: bool, + ) -> Result { + let src = Remote::parse(source)?; + let dst = Remote::parse(destination)?; + if src.is_none() && dst.is_none() { + bail!("at least one path must use s3://bucket/key"); + } + if upload && (src.is_some() || dst.is_none()) { + bail!("upload requires a local source and s3://bucket/key destination"); + } + if src.is_some() && src == dst { + bail!("source and destination must differ"); + } + if let Some(dst) = &dst { + dst.object()?; + } + if dst.is_none() && Path::new(destination).exists() { + bail!("local destination already exists; choose a new file path"); + } + let bytes = match &src { + Some(src) => self.download(src).await?, + None => { + let file = std::fs::File::open(source).context("failed to open local source")?; + if !file.metadata()?.is_file() { + bail!("source must be a regular file; recursive transfers are unsupported"); + } + let mut bytes = Vec::new(); + file.take((MAX_OBJECT + 1) as u64).read_to_end(&mut bytes)?; + check_size(bytes.len(), MAX_OBJECT)?; + bytes + } + }; + let size = bytes.len(); + match dst { + Some(dst) => { + let response = self + .request(Method::PUT, self.url(Some(&dst))?, Some(bytes)) + .await?; + expect(response, StatusCode::OK).await?; + } + None => { + let path = Path::new(destination); + let parent = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let mut temporary = tempfile::NamedTempFile::new_in(parent)?; + temporary.write_all(&bytes)?; + temporary.as_file().sync_all()?; + temporary + .persist_noclobber(path) + .context("failed to save local destination")?; + } + } + if moving { + let deleted = match src { + Some(src) => self.delete(&src).await, + None => std::fs::remove_file(source).context("failed to delete local source"), + }; + deleted + .context("copy succeeded, but source deletion failed; both copies may remain")?; + } + Ok( + json!({"operation": if moving { "mv" } else { "cp" }, "source": source, "destination": destination, "size": size}), + ) + } + + async fn list(&self, path: Option<&str>) -> Result { + let remote = path + .map(|p| { + Remote::parse(p)?.ok_or_else(|| anyhow!("listing path must use s3://bucket/prefix")) + }) + .transpose()?; + let mut items = Vec::new(); + let mut token: Option = None; + let mut seen = HashSet::new(); + loop { + let bucket = remote.as_ref().map(|r| Remote { + bucket: r.bucket.clone(), + key: String::new(), + }); + let mut url = self.url(bucket.as_ref())?; + { + let mut query = url.query_pairs_mut(); + if let Some(remote) = &remote { + query.extend_pairs([ + ("list-type", "2"), + ("encoding-type", "url"), + ("delimiter", "/"), + ("max-keys", "1000"), + ("prefix", remote.key.as_str()), + ]); + } + if let Some(token) = &token { + query.append_pair("continuation-token", token); + } + } + let response = self.request(Method::GET, url, None).await?; + let xml = bounded_body(expect(response, StatusCode::OK).await?, MAX_XML).await?; + let page = parse_listing(&xml, remote.is_some())?; + if remote.is_some() { + let encoded = page.encoding_type.as_deref() == Some("url"); + for object in page.contents { + items.push(json!({"kind": "object", "key": decode_field(&object.key, encoded)?, "size": object.size, "lastModified": object.last_modified})); + } + for prefix in page.common_prefixes { + items.push( + json!({"kind": "prefix", "key": decode_field(&prefix.prefix, encoded)?}), + ); + } + } else { + for bucket in page.buckets.bucket { + items.push(json!({"kind": "bucket", "name": bucket.name})); + } + } + token = if remote.is_some() { + if page.is_truncated { + page.next_continuation_token + } else { + None + } + } else { + page.continuation_token.or(page.next_continuation_token) + } + .filter(|t| !t.is_empty()); + if page.is_truncated && token.is_none() { + bail!("truncated gateway listing has no continuation token"); + } + let Some(next) = &token else { + break; + }; + if !seen.insert(next.clone()) { + bail!("gateway repeated a pagination token"); + } + } + Ok(json!({"items": items})) + } +} + +fn parse_listing(xml: &[u8], objects: bool) -> Result { + let mut reader = quick_xml::Reader::from_reader(xml); + reader.config_mut().trim_text(true); + let expected = if objects { + "ListBucketResult" + } else { + "ListAllMyBucketsResult" + }; + loop { + match reader.read_event().context("invalid gateway listing XML")? { + quick_xml::events::Event::Decl(_) | quick_xml::events::Event::Comment(_) => continue, + quick_xml::events::Event::Start(tag) | quick_xml::events::Event::Empty(tag) + if tag.local_name().as_ref() == expected.as_bytes() => + { + break; + } + _ => bail!("unexpected gateway listing XML root"), + } + } + let page: Listing = quick_xml::de::from_reader(xml).context("invalid gateway listing XML")?; + if page + .encoding_type + .as_deref() + .is_some_and(|encoding| encoding != "url") + { + bail!("unsupported gateway listing encoding"); + } + Ok(page) +} + +fn provider_url(location: &str) -> Result { + let url = Url::parse(location).context("invalid provider redirect URL")?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + { + bail!("provider redirect must be an absolute HTTPS URL without credentials or fragment"); + } + Ok(url) +} + +fn check_size(size: usize, max: usize) -> Result<()> { + if size > max { + bail!( + "response or file exceeds the {} byte limit; object transfers support at most 25 MiB, without multipart", + max + ); + } + Ok(()) +} + +async fn bounded_body(mut response: reqwest::Response, max: usize) -> Result> { + if response.content_length().is_some_and(|n| n > max as u64) { + check_size(max + 1, max)?; + } + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow!("failed to read transfer body"))? + { + check_size(bytes.len().saturating_add(chunk.len()), max)?; + bytes.extend_from_slice(&chunk); + } + Ok(bytes) +} + +async fn expect(response: reqwest::Response, status: StatusCode) -> Result { + if response.status() == status { + return Ok(response); + } + let actual = response.status(); + let code = bounded_body(response, MAX_XML) + .await + .ok() + .and_then(|xml| quick_xml::de::from_reader::<_, S3Error>(xml.as_slice()).ok()) + .map(|e| e.code); + bail!( + "S3 request returned HTTP {}{}", + actual.as_u16(), + code.map(|c| format!(" ({c})")).unwrap_or_default() + ) +} + +fn decode_field(value: &str, encoded: bool) -> Result { + if !encoded { + return Ok(value.to_owned()); + } + let bytes = value.as_bytes(); + let mut decoded = Vec::new(); + let mut index = 0; + while index < bytes.len() { + match bytes[index] { + b'+' => decoded.push(b' '), + b'%' => { + let pair = bytes + .get(index + 1..index + 3) + .ok_or_else(|| anyhow!("invalid listing percent escape"))?; + let hex = std::str::from_utf8(pair)?; + decoded + .push(u8::from_str_radix(hex, 16).context("invalid listing percent escape")?); + index += 2; + } + byte => decoded.push(byte), + } + index += 1; + } + String::from_utf8(decoded).context("listing field is not UTF-8") +} + +#[derive(Default, Deserialize)] +#[serde(rename_all = "PascalCase")] +struct Listing { + #[serde(default)] + buckets: Buckets, + #[serde(default)] + contents: Vec, + #[serde(default)] + common_prefixes: Vec, + encoding_type: Option, + #[serde(default)] + is_truncated: bool, + next_continuation_token: Option, + continuation_token: Option, +} +#[derive(Default, Deserialize)] +struct Buckets { + #[serde(rename = "Bucket", default)] + bucket: Vec, +} +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct Bucket { + name: String, +} +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct Object { + key: String, + size: u64, + last_modified: Option, +} +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct Prefix { + prefix: String, +} +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct S3Error { + code: String, +} + +pub async fn handle(args: S3Args, endpoint: &str, pod: &str, token: &str) -> Result { + let gateway = Gateway::new(endpoint, pod, token)?; + let value = match args.command { + S3Command::Ls { path } => gateway.list(path.as_deref()).await?, + S3Command::Cp { + source, + destination, + } => gateway.copy(&source, &destination, false, false).await?, + S3Command::Mv { + source, + destination, + } => gateway.copy(&source, &destination, true, false).await?, + S3Command::Upload { + source, + destination, + require_success_file, + metadata, + } => { + if source == "-" { + let remote = Remote::parse(&destination)? + .ok_or_else(|| anyhow!("upload requires s3://bucket/key"))?; + let require_success_file = require_success_file.ok_or_else(|| anyhow!("stdin uploads require --require-success-file; pipeline EOF alone cannot prove producer success"))?; + gateway + .upload_stream( + multipart::cancellable_stdin(), + &remote, + &require_success_file, + &metadata, + multipart::cancellation()?, + ) + .await? + } else { + if require_success_file.is_some() || !metadata.is_empty() { + bail!("--require-success-file and --metadata require streaming stdin (-)"); + } + gateway.copy(&source, &destination, false, true).await? + } + } + S3Command::Download { + source, + destination, + } => { + if destination != "-" { + bail!( + "streaming download destination must be -; use cp for a bounded file download" + ); + } + let remote = Remote::parse(&source)? + .ok_or_else(|| anyhow!("download requires s3://bucket/key"))?; + let mut stdout = multipart::cancellable_output(std::io::stdout()); + tokio::select! { + result = gateway.download_stream(&remote, &mut stdout) => { result?; } + _ = multipart::cancellation()? => bail!("download cancelled; stdout may contain a partial object"), + } + return Ok(CommandOutput::Silent); + } + S3Command::Rm { path } => { + let remote = + Remote::parse(&path)?.ok_or_else(|| anyhow!("rm requires s3://bucket/key"))?; + gateway.delete(&remote).await?; + json!({"operation": "rm", "path": path}) + } + }; + Ok(CommandOutput::new(value, View::S3)) +} + +pub fn render(value: &Value) -> Vec { + if let Some(items) = value.get("items").and_then(Value::as_array) { + return items + .iter() + .map(|item| match item["kind"].as_str() { + Some("bucket") => format!("Bucket: {}", item["name"].as_str().unwrap_or_default()), + Some("prefix") => format!("Prefix: {}", item["key"].as_str().unwrap_or_default()), + _ => format!( + "{}\t{}", + item["size"], + item["key"].as_str().unwrap_or_default() + ), + }) + .collect(); + } + vec![format!( + "{}: {}", + value["operation"].as_str().unwrap_or_default(), + value + .get("destination") + .or_else(|| value.get("path")) + .and_then(Value::as_str) + .unwrap_or_default() + )] +} + +mod multipart; + +#[cfg(test)] +mod tests; diff --git a/src/s3/multipart.rs b/src/s3/multipart.rs new file mode 100644 index 0000000..dd17387 --- /dev/null +++ b/src/s3/multipart.rs @@ -0,0 +1,533 @@ +use std::future::Future; + +use reqwest::header::{CONTENT_TYPE, ETAG, HeaderMap, HeaderName, HeaderValue}; +use serde::Serialize; +use tokio::io::{AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; + +use super::*; + +const PART_SIZE: usize = 8 * 1024 * 1024; +const MAX_PARTS: usize = 10_000; +const MAX_COMPLETION: usize = 2 * 1024 * 1024; +pub(super) fn cancellation() -> Result> { + #[cfg(unix)] + let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + Ok(async move { + #[cfg(unix)] + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + _ = terminate.recv() => {}, + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } + }) +} + +fn metadata_headers(metadata: &[String]) -> Result { + let mut headers = HeaderMap::new(); + let mut total = 0; + for entry in metadata { + let (key, value) = entry + .split_once('=') + .ok_or_else(|| anyhow!("metadata must use KEY=VALUE"))?; + if key.is_empty() { + bail!("metadata keys must not be empty"); + } + if value.is_empty() || !value.bytes().all(|b| (0x20..=0x7e).contains(&b)) { + bail!("metadata values must be nonempty printable ASCII"); + } + total += key.len() + value.len(); + if total > 2048 { + bail!("metadata names and values exceed 2048 bytes"); + } + let name = HeaderName::from_bytes(format!("x-amz-meta-{key}").as_bytes()) + .context("invalid metadata key")?; + if headers + .insert(name, HeaderValue::from_str(value)?) + .is_some() + { + bail!("duplicate metadata key: {key}"); + } + } + Ok(headers) +} + +fn parse_xml(xml: &[u8], expected: &str) -> Result { + let mut reader = quick_xml::Reader::from_reader(xml); + reader.config_mut().trim_text(true); + let mut depth = 0usize; + let mut root = false; + loop { + match reader + .read_event() + .context("invalid multipart response XML")? + { + quick_xml::events::Event::Decl(_) | quick_xml::events::Event::Comment(_) => {} + quick_xml::events::Event::Start(tag) => { + if depth == 0 { + if root || tag.local_name().as_ref() != expected.as_bytes() { + bail!("unexpected multipart response XML root (expected {expected})"); + } + root = true; + } + depth += 1; + if depth > 16 { + bail!("multipart response XML is too deeply nested"); + } + } + quick_xml::events::Event::Empty(tag) => { + if depth == 0 { + if root || tag.local_name().as_ref() != expected.as_bytes() { + bail!("unexpected multipart response XML root (expected {expected})"); + } + root = true; + } + } + quick_xml::events::Event::End(_) => { + depth = depth + .checked_sub(1) + .ok_or_else(|| anyhow!("invalid multipart XML nesting"))?; + } + quick_xml::events::Event::Text(text) + if depth > 0 || text.iter().all(u8::is_ascii_whitespace) => {} + quick_xml::events::Event::CData(_) | quick_xml::events::Event::GeneralRef(_) + if depth > 0 => {} + quick_xml::events::Event::Eof if root && depth == 0 => break, + _ => bail!("invalid or incomplete multipart response XML"), + } + } + quick_xml::de::from_reader(xml).context("invalid multipart response XML fields") +} + +fn validate_etag(etag: &str) -> Result<()> { + if !(3..=256).contains(&etag.len()) + || !etag.starts_with('"') + || !etag.ends_with('"') + || !etag[1..etag.len() - 1] + .bytes() + .all(|b| (0x21..=0x7e).contains(&b) && b != b'"') + { + bail!("multipart response requires a quoted ETag of at most 256 bytes"); + } + Ok(()) +} + +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct Initiated { + bucket: String, + key: String, + upload_id: String, +} + +#[derive(Serialize)] +#[serde(rename = "CompleteMultipartUpload")] +struct Completion { + #[serde(rename = "Part")] + parts: Vec, +} + +#[derive(Serialize)] +struct Part { + #[serde(rename = "PartNumber")] + number: usize, + #[serde(rename = "ETag")] + etag: String, +} + +#[derive(Deserialize)] +struct Completed { + #[serde(rename = "Bucket")] + bucket: String, + #[serde(rename = "Key")] + key: String, + #[serde(rename = "ETag")] + etag: String, +} + +impl Gateway<'_> { + fn multipart_url(&self, remote: &Remote, upload_id: &str) -> Result { + let mut url = self.url(Some(remote))?; + url.query_pairs_mut().append_pair("uploadId", upload_id); + Ok(url) + } + + pub(super) async fn upload_stream>( + &self, + mut input: R, + remote: &Remote, + success_file: &Path, + metadata: &[String], + cancelled: C, + ) -> Result { + remote.object()?; + let metadata = metadata_headers(metadata)?; + let mut upload_id = None; + let mut completing = false; + let result = tokio::select! { + biased; + _ = cancelled => Err(anyhow!("multipart upload cancelled")), + result = self.multipart_transfer(&mut input, remote, success_file, metadata, &mut upload_id, &mut completing) => result, + }; + match result { + Ok(value) => Ok(value), + Err(error) => { + if let Some(upload_id) = upload_id { + let abort = async { + let response = self + .request( + Method::DELETE, + self.multipart_url(remote, &upload_id)?, + None, + ) + .await?; + expect(response, StatusCode::NO_CONTENT).await?; + Ok::<_, anyhow::Error>(()) + }; + if !matches!( + tokio::time::timeout(Duration::from_secs(30), abort).await, + Ok(Ok(())) + ) { + eprintln!("warning: multipart abort failed; incomplete upload may remain"); + } + } else { + eprintln!( + "warning: initiation did not yield an upload ID; any allocated multipart upload cannot be aborted" + ); + } + if completing { + Err(error.context("completion outcome may be uncertain; inspect the exact key before retrying")) + } else { + Err(error) + } + } + } + } + + async fn multipart_transfer( + &self, + input: &mut R, + remote: &Remote, + success_file: &Path, + metadata: HeaderMap, + upload_id: &mut Option, + completing: &mut bool, + ) -> Result { + let mut url = self.url(Some(remote))?; + url.set_query(Some("uploads")); + let response = self + .http + .post(url) + .bearer_auth(self.token) + .headers(metadata) + .header(CONTENT_TYPE, "application/octet-stream") + .body(Vec::new()) + .timeout(Duration::from_secs(120)) + .send() + .await + .map_err(|_| anyhow!("multipart initiation request failed"))?; + let xml = bounded_body(expect(response, StatusCode::OK).await?, MAX_XML).await?; + let initiated: Initiated = parse_xml(&xml, "InitiateMultipartUploadResult")?; + if initiated.upload_id.is_empty() || initiated.upload_id.len() > 8192 { + bail!("gateway returned an invalid multipart upload ID"); + } + *upload_id = Some(initiated.upload_id); + if initiated.bucket != remote.bucket || initiated.key != remote.key { + bail!("gateway initiated a different object"); + } + let id = upload_id + .as_deref() + .ok_or_else(|| anyhow!("multipart upload has no ID"))?; + let mut parts = Vec::new(); + let mut size = 0u64; + loop { + let mut bytes = vec![0; PART_SIZE]; + let mut filled = 0; + while filled < PART_SIZE { + let count = input + .read(&mut bytes[filled..]) + .await + .context("failed to read upload input; not completing")?; + if count == 0 { + break; + } + filled += count; + } + if filled == 0 && !parts.is_empty() { + break; + } + if parts.len() == MAX_PARTS { + bail!("stream exceeds the 10000-part limit (8 MiB per part)"); + } + bytes.truncate(filled); + let number = parts.len() + 1; + let mut url = self.multipart_url(remote, id)?; + url.query_pairs_mut() + .append_pair("partNumber", &number.to_string()); + let response = self.request(Method::PUT, url, Some(bytes)).await?; + let response = expect(response, StatusCode::OK).await?; + let etag = response + .headers() + .get(ETAG) + .ok_or_else(|| anyhow!("uploaded part response has no ETag"))? + .to_str()? + .to_owned(); + validate_etag(&etag)?; + bounded_body(response, MAX_XML).await?; + parts.push(Part { number, etag }); + size += filled as u64; + if filled < PART_SIZE { + break; + } + } + let marker_metadata = tokio::fs::metadata(success_file) + .await + .context("producer success file missing; aborting without completion")?; + if !marker_metadata.is_file() { + bail!("producer success path is not a regular file"); + } + let mut marker = Vec::new(); + tokio::fs::File::open(success_file) + .await + .context("producer success file missing; aborting without completion")? + .take(8) + .read_to_end(&mut marker) + .await + .context("cannot read producer success file")?; + if marker != b"commit\n" { + bail!("invalid producer success file; aborting without completion"); + } + let body = quick_xml::se::to_string(&Completion { parts }) + .context("failed to encode multipart completion")?; + check_size(body.len(), MAX_COMPLETION)?; + *completing = true; + let response = self + .http + .post(self.multipart_url(remote, id)?) + .bearer_auth(self.token) + .header(CONTENT_TYPE, "application/xml") + .body(body) + .timeout(Duration::from_secs(120)) + .send() + .await + .map_err(|_| anyhow!("multipart completion request failed"))?; + let xml = bounded_body(expect(response, StatusCode::OK).await?, MAX_COMPLETION).await?; + let completed: Completed = parse_xml(&xml, "CompleteMultipartUploadResult")?; + if completed.bucket != remote.bucket || completed.key != remote.key { + bail!("gateway completed a different object"); + } + validate_etag(&completed.etag)?; + Ok( + json!({"operation": "upload", "destination": format!("s3://{}/{}", remote.bucket, remote.key), "size": size}), + ) + } + + pub(super) async fn download_stream( + &self, + remote: &Remote, + output: &mut W, + ) -> Result { + let mut response = self.provider_response(remote, true).await?; + let mut size = 0u64; + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow!("provider download interrupted; output may be partial"))? + { + output + .write_all(&chunk) + .await + .context("failed to write download output; output may be partial")?; + size = size + .checked_add(chunk.len() as u64) + .ok_or_else(|| anyhow!("download size overflow"))?; + } + output + .flush() + .await + .context("failed to flush download output")?; + Ok(size) + } +} + +pub(super) fn cancellable_stdin() -> impl AsyncRead + Unpin { + let (sender, receiver) = tokio::sync::mpsc::channel(1); + std::thread::spawn(move || { + let stdin = std::io::stdin(); + let mut stdin = stdin.lock(); + loop { + let mut bytes = vec![0; 64 * 1024]; + let result = match std::io::Read::read(&mut stdin, &mut bytes) { + Ok(0) => break, + Ok(count) => { + bytes.truncate(count); + Ok(tokio_util::bytes::Bytes::from(bytes)) + } + Err(error) => Err(error), + }; + let failed = result.is_err(); + if sender.blocking_send(result).is_err() || failed { + break; + } + } + }); + tokio_util::io::StreamReader::new(tokio_stream::wrappers::ReceiverStream::new(receiver)) +} + +pub(super) struct CancellableOutput { + sender: std::sync::mpsc::Sender<(Vec, tokio::sync::oneshot::Sender>)>, + pending: Option<(usize, tokio::sync::oneshot::Receiver>)>, +} + +impl tokio::io::AsyncWrite for CancellableOutput { + fn poll_write( + mut self: std::pin::Pin<&mut Self>, + context: &mut std::task::Context<'_>, + bytes: &[u8], + ) -> std::task::Poll> { + if self.pending.is_none() { + let size = bytes.len().min(64 * 1024); + let (sender, receiver) = tokio::sync::oneshot::channel(); + if self.sender.send((bytes[..size].to_vec(), sender)).is_err() { + return std::task::Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "stdout worker stopped", + ))); + } + self.pending = Some((size, receiver)); + } + let (size, receiver) = self + .pending + .as_mut() + .ok_or_else(|| std::io::Error::other("no pending stdout write"))?; + let size = *size; + match std::pin::Pin::new(receiver).poll(context) { + std::task::Poll::Pending => std::task::Poll::Pending, + std::task::Poll::Ready(result) => { + self.pending = None; + std::task::Poll::Ready( + result + .unwrap_or_else(|_| Err(std::io::Error::other("stdout worker stopped"))) + .map(|()| size), + ) + } + } + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + _: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + context: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.poll_flush(context) + } +} + +pub(super) fn cancellable_output( + mut output: W, +) -> CancellableOutput { + let (sender, receiver) = + std::sync::mpsc::channel::<(Vec, tokio::sync::oneshot::Sender>)>(); + std::thread::spawn(move || { + while let Ok((bytes, acknowledge)) = receiver.recv() { + let result = output.write_all(&bytes).and_then(|()| output.flush()); + let failed = result.is_err(); + if acknowledge.send(result).is_err() || failed { + break; + } + } + }); + CancellableOutput { + sender, + pending: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_generic_metadata_names_values_and_bounds() { + let headers = metadata_headers(&["project=example".into(), "Source=cli".into()]).unwrap(); + assert_eq!(headers.len(), 2); + assert_eq!(headers["x-amz-meta-project"], "example"); + assert_eq!(headers["x-amz-meta-source"], "cli"); + assert!(metadata_headers(&[format!("description={}", "x".repeat(1024))]).is_ok()); + for metadata in [ + vec!["missing-equals".into()], + vec!["=value".into()], + vec!["bad key=value".into()], + vec!["bad:key=value".into()], + vec!["name=".into()], + vec!["name=a\r\nbad".into()], + vec!["name=雪".into()], + vec![format!("description={}", "x".repeat(2048))], + vec!["name=a".into(), "NAME=b".into()], + (0..5) + .map(|n| format!("field{n}={}", "x".repeat(512))) + .collect(), + ] { + assert!(metadata_headers(&metadata).is_err()); + } + } + + #[test] + fn completion_manifest_supports_10000_ordered_parts() { + let completion = Completion { + parts: (1..=MAX_PARTS) + .map(|number| Part { + number, + etag: "\"opaque&etag\"".into(), + }) + .collect(), + }; + let xml = quick_xml::se::to_string(&completion).unwrap(); + assert!(xml.len() < MAX_COMPLETION); + assert!(xml.contains("opaque&etag")); + assert_eq!(xml.matches("").count(), MAX_PARTS); + assert!(xml.contains("10000")); + } + + #[tokio::test] + async fn output_worker_preserves_raw_bytes_and_reports_write_errors() { + struct Writer(Arc>>); + use std::sync::Arc; + impl std::io::Write for Writer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let bytes = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut output = cancellable_output(Writer(bytes.clone())); + output.write_all(&vec![0xff; 128 * 1024]).await.unwrap(); + output.flush().await.unwrap(); + assert_eq!(*bytes.lock().unwrap(), vec![0xff; 128 * 1024]); + struct Broken; + impl std::io::Write for Broken { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let error = cancellable_output(Broken) + .write_all(b"raw bytes") + .await + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::BrokenPipe); + } +} diff --git a/src/s3/test-certificate.pem b/src/s3/test-certificate.pem new file mode 100644 index 0000000..d36da7c --- /dev/null +++ b/src/s3/test-certificate.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC6DCCAdCgAwIBAgIJANBNJopqUmWRMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0yNjExMDIwOTA0MTNaGA8yMTI2MTAwOTA5MDQxM1ow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAus7FOhd9TubvQauWCttScTXQBezI1yKu4ImpHtnD59QsMvQVsfPdhJ4n +BWBogmu4RPSUhOu1+5Oj8EKKtw6OwMvb0Vuqn/5Ktl8tzlou2y0jAw2TqMJ91IQA +m+X0s/dXZQZTChT0DwjWynYU6nXAoT17m8mFm2zY1w+8FWs4ZNhMQ6zx/WO9RDva +L4pDTZk/a5WQOQYhoy9oZLIzX6Q9YRUW4+KrvX0GfWlzB/nvNDYS/DPV7Kj/MIB4 +eM/WMH7LD689H+ed96M261T+yWXALWsGPmsuPVZpeD+C8Y65cF+c+bG6csZ+TAh4 +G5TJH8FRuqAhopoUDg85ttJXbuVS5QIDAQABozswOTAUBgNVHREEDTALgglsb2Nh +bGhvc3QwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggrBgEFBQcDATANBgkqhkiG +9w0BAQsFAAOCAQEAcW5Es1kUtj4Ciuf7LLTuMQhRjWkGXF7FFs3Ca/JWbI/7XAzS +fdSCRLUaNdGrWWJsnjp9cZN45VWnmO9n5nLs4Zbo0GWuOVnAWA7s6E247pOVaQDi +BvGCzROZ+5ZQog6Ttxm6ZcRfDfipVkcu0BMY8DUJeh0qiF6ZrdAEI4LH+EEV1vMV +i1hHeau8TBBHm+SGfWw5tOVz+kgbZZBhyAXSxKQj+AIzFIS/268s4+WTAOeMI6y7 +qFFcTRTuUdowxQ+mLQ6pNwRU/FXLPa6skqM641SQHnON6oW40xbkdAqWHSSIxaLC +8LRWw2b9W2WMu8iusYn6AqdDM3Y+lEg5bhBn0A== +-----END CERTIFICATE----- diff --git a/src/s3/test-private-key.pem b/src/s3/test-private-key.pem new file mode 100644 index 0000000..27f5293 --- /dev/null +++ b/src/s3/test-private-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC6zsU6F31O5u9B +q5YK21JxNdAF7MjXIq7giake2cPn1Cwy9BWx892EnicFYGiCa7hE9JSE67X7k6Pw +Qoq3Do7Ay9vRW6qf/kq2Xy3OWi7bLSMDDZOown3UhACb5fSz91dlBlMKFPQPCNbK +dhTqdcChPXubyYWbbNjXD7wVazhk2ExDrPH9Y71EO9ovikNNmT9rlZA5BiGjL2hk +sjNfpD1hFRbj4qu9fQZ9aXMH+e80NhL8M9XsqP8wgHh4z9YwfssPrz0f5533ozbr +VP7JZcAtawY+ay49Vml4P4LxjrlwX5z5sbpyxn5MCHgblMkfwVG6oCGimhQODzm2 +0ldu5VLlAgMBAAECggEAIE3DU2FefPd4Zk3Ku4PjxMn+eBfTP0izHwcPJNOZigxp +vEHzQnueUeGbc9WRHQSNr/bve/Da3m9QiSqonDE7RGG8WH7WiaZKuzDIVgfIVISA +qnV9lFBVvBs7k7n3T+u0xDaMw5COB8+KfSZgRY8pSXi9WFpH7XGsKbyB1N5euSlV +/Tf/WeVvAPxU5h3tYw0qzUVlV2F7kmXBJAsSl0xkTbr4Z8Oqvr7aE6mnjo6SeVBQ +5lzhgIpIAiuV4GfCkNO5LXn0kTCGj3kcWyN+0kw2fkX3X9tImkUS+d6OoCopAR8N +7gYlKhkN6obwIAvN3rh/CbPdqr5gk3JcNdvzhwSRwQKBgQD4nfdetaZCKmnRX8ht +kSCVNwPanwG8Xv23QRkG6FYnnXYUdvlMLBiz1AnDBwfJpDBwMTMphBVXgI3ssx1/ +nds56gvHRa848GNTld3EDoHAB6U34JiG/EY0wK87BldLKf8OwKTk0ZwzwKxT2+Jq +I4zWXN/6QjDEbHl3FM7WgrEBtQKBgQDAWurwpH67LRTJh6hGhegZZo3PbQBgNYyT +2IfVbDEROJSMOUWR1200UCdYiOIKRxwbZwGroYc9x8Pr42L76QjyI6XPAe3ygKwd +hGipFNyHQ2vpBoB8tO3XuEY+9RHzs4C4k0S83dtMKFsn0XbrQBlV4wEzlVvXQkis +D8T9y3iKcQKBgQC35UCAkaR09KJo+IxsXxjWxmbXYx4Jk8nfrwswDNMNYm9hG8+z +BDsRaRn/ZEfiYsxhcEMDxmty2egg2SxMpRrvb6terrC5UTZnMUdkKcbXn9AZ0MuB +rUDQ7GWxJzOSqXG8G8rcZh9AOJg/PznugHVAalpR5nvkDsiZyI5mJS2AsQKBgFFE +2GhNh4G7fUHcP9tz7ej1bErL6dG8LiW04UbpjB2TgydvdxY0J15n0wfT5XSkMJrf +PVRvTX1rrjo2evalsXLkXsSEMy028gESlUmCl93KPYhZh+WoZ6ixoXQuBKSrHWJe +TjdhoKqfjkvXct/X2BxyaWqK2x+qUJvpmbgExGvxAoGAV74qbEqdx/HB89v7uLGE +KzPoD6b819RK75peB6yrUvH9QYq9P5YHupNR14BzLzbS06XC17liPqwLlaK9cwuT +iwHAIigUfPmx+YQj+Vlou2eTJxxVV1Zw5fv2y9p0/8dh3GDG6L/bbx8rOccp5E7A +t+d8bZO1DgPljy/fFx7CIRs= +-----END PRIVATE KEY----- diff --git a/src/s3/tests.rs b/src/s3/tests.rs new file mode 100644 index 0000000..c176f11 --- /dev/null +++ b/src/s3/tests.rs @@ -0,0 +1,816 @@ +use std::sync::{Arc, Mutex}; + +use axum::{Router, body::Body, extract::State, http::Request, response::Response, routing::any}; +use clap::Parser as _; + +use super::*; + +type MockResponse = (u16, Vec<(&'static str, String)>, String); + +type Requests = Arc)>>>; + +async fn mock( + responses: Vec, +) -> (Gateway<'static>, Requests, tokio::task::JoinHandle<()>) { + let requests: Requests = Arc::default(); + let responses = Arc::new(Mutex::new(std::collections::VecDeque::from(responses))); + let router = Router::new() + .fallback(any({ + let responses = responses.clone(); + move |State(requests): State, request: Request| { + let responses = responses.clone(); + async move { + assert_eq!(request.headers()["authorization"], "Bearer test-key"); + let method = request.method().to_string(); + let uri = request.uri().to_string(); + let length = request + .headers() + .get("content-length") + .map(|v| v.to_str().unwrap().parse::().unwrap()); + let body = axum::body::to_bytes(request.into_body(), MAX_OBJECT + 1) + .await + .unwrap() + .to_vec(); + if method == "PUT" { + assert_eq!(length, Some(body.len())); + } + requests.lock().unwrap().push((method, uri, body)); + let (status, headers, body) = responses + .lock() + .unwrap() + .pop_front() + .expect("unexpected request"); + let mut response = Response::builder().status(status); + for (name, value) in headers { + response = response.header(name, value); + } + response.body(Body::from(body)).unwrap() + } + } + })) + .with_state(requests.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = Url::parse(&format!("http://{}", listener.local_addr().unwrap())).unwrap(); + let task = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + let gateway = Gateway { + origin, + token: "test-key", + http: crate::http_client_builder() + .unwrap() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(), + }; + (gateway, requests, task) +} + +#[test] +fn parses_commands_and_contract() { + for args in [ + vec!["s3", "ls"], + vec!["s3", "ls", "s3://files/a/"], + vec!["s3", "cp", "a", "s3://files/a"], + vec!["s3", "mv", "s3://files/a", "a"], + vec!["s3", "upload", "a", "s3://files/a"], + vec![ + "s3", + "upload", + "-", + "s3://files/a", + "--require-success-file", + "/tmp/success", + "--metadata", + "project=example", + ], + vec!["s3", "download", "s3://files/a", "-"], + vec!["s3", "rm", "s3://files/a"], + ] { + let opts = crate::Opts::try_parse_from(std::iter::once("brainpod").chain(args)).unwrap(); + assert!(crate::cmd::needs_api_token(&opts.command)); + assert!(!crate::cmd::needs_client(&opts.command)); + } + assert!( + crate::Opts::try_parse_from(["brainpod", "s3", "cp", "--recursive", "a", "s3://files/a"]) + .is_err() + ); +} + +#[test] +fn preserves_literal_keys_and_rejects_normalization() { + let gateway = Gateway::new("https://s3.example.test:443", "my-pod", "key").unwrap(); + let remote = Remote::parse("s3://files/a//雪 +%2F?#/end/") + .unwrap() + .unwrap(); + let url = gateway.url(Some(&remote)).unwrap(); + assert_eq!(url.host_str(), Some("my-pod.s3.example.test")); + assert_eq!(url.path(), "/files/a//%E9%9B%AA%20+%252F%3F%23/end/"); + assert!(url.query().is_none()); + for path in ["s3://files/a/../b", "s3://files/./a", "s3://files/"] { + assert!(Remote::parse(path).unwrap().unwrap().object().is_err()); + } + for endpoint in [ + "https://user:pass@example.test", + "https://example.test/path", + "https://example.test/?x=1", + "file:///tmp/x", + ] { + assert!(Gateway::new(endpoint, "pod", "key").is_err()); + } + assert!(Gateway::new("https://example.test", "evil.example", "key").is_err()); +} + +#[test] +fn decodes_listing_fields_once_using_form_semantics() { + assert_eq!( + decode_field("a+b%20c%2B%252F%E9%9B%AA", true).unwrap(), + "a b c+%2F雪" + ); + assert_eq!(decode_field("a+%2F", false).unwrap(), "a+%2F"); + for field in ["%", "%ZZ", "%FF"] { + assert!(decode_field(field, true).is_err()); + } +} + +#[test] +fn only_accepts_safe_provider_redirects() { + for location in [ + "http://provider.test/a", + "/relative", + "https://user:pass@provider.test/a", + "https://provider.test/a#fragment", + "file:///tmp/a", + ] { + assert!(provider_url(location).is_err()); + } + let gateway = Gateway::new("https://s3.example.test", "pod", "key").unwrap(); + let request = gateway + .http + .get(provider_url("https://provider.test/key?signature=secret").unwrap()) + .build() + .unwrap(); + assert!(!request.headers().contains_key("authorization")); +} + +#[tokio::test] +async fn follows_object_pages_with_opaque_tokens_without_reusing_echoed_token() { + let page1 = r#"urltruea+/=&%2Fa%2B%252F42dir%2F"#; + let page2 = r#"urlfalsea+/=&%2Flast+file0"#; + let (gateway, requests, task) = mock(vec![ + (200, vec![], page1.into()), + (200, vec![], page2.into()), + ]) + .await; + let result = gateway.list(Some("s3://files/dot/../+%/")).await.unwrap(); + assert_eq!(result["items"][0]["key"], "a+%2F"); + assert_eq!(result["items"][1]["key"], "dir/"); + assert_eq!(result["items"][2]["key"], "last file"); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + let url = Url::parse(&format!("http://localhost{}", requests[1].1)).unwrap(); + let pairs: std::collections::HashMap<_, _> = url.query_pairs().collect(); + assert_eq!(pairs["continuation-token"], "a+/=&%2F"); + assert_eq!(pairs["prefix"], "dot/../+%/"); + task.abort(); +} + +#[tokio::test] +async fn rejects_broken_and_repeated_pagination() { + for xml in [ + "true", + "truesame", + "", + "AccessDenied", + ] { + let (gateway, _, task) = + mock(vec![(200, vec![], xml.into()), (200, vec![], xml.into())]).await; + assert!(gateway.list(Some("s3://files/")).await.is_err()); + task.abort(); + } +} + +#[tokio::test] +async fn lists_buckets_with_optional_pagination() { + let (gateway, requests, task) = mock(vec![ + (200, vec![], "onenext".into()), + (200, vec![], "two".into()), + ]).await; + let result = gateway.list(None).await.unwrap(); + assert_eq!(result["items"].as_array().unwrap().len(), 2); + assert_eq!(requests.lock().unwrap().len(), 2); + task.abort(); +} + +#[tokio::test] +async fn upload_and_move_only_remove_local_source_after_success() { + for (status, moving) in [(200, false), (200, true), (403, true)] { + let source = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(source.path(), b"hello").unwrap(); + let (gateway, requests, task) = mock(vec![( + status, + vec![], + "AccessDenied".into(), + )]) + .await; + let result = gateway + .copy( + source.path().to_str().unwrap(), + "s3://files/a//b%?", + moving, + false, + ) + .await; + assert_eq!(result.is_ok(), status == 200); + assert_eq!(source.path().exists(), !(moving && status == 200)); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0], + ("PUT".into(), "/files/a//b%25%3F".into(), b"hello".to_vec()) + ); + task.abort(); + } +} + +#[tokio::test] +async fn rejects_oversized_and_unsupported_copies_before_mutation() { + let source = tempfile::NamedTempFile::new().unwrap(); + source.as_file().set_len((MAX_OBJECT + 1) as u64).unwrap(); + let (gateway, requests, task) = mock(vec![]).await; + assert!( + gateway + .copy(source.path().to_str().unwrap(), "s3://files/a", true, false) + .await + .is_err() + ); + assert!(source.path().exists()); + assert!( + gateway + .copy("s3://files/a", "s3://files/a", true, false) + .await + .is_err() + ); + assert!( + gateway + .copy("s3://files/a", "s3://files/b", false, true) + .await + .is_err() + ); + assert!(gateway.copy("a", "b", false, false).await.is_err()); + assert!(requests.lock().unwrap().is_empty()); + task.abort(); +} + +#[tokio::test] +async fn failed_download_never_deletes_source_or_creates_destination() { + for (status, headers) in [ + ( + 307, + vec![("location", "http://untrusted.invalid/object".into())], + ), + (404, vec![]), + ( + 302, + vec![("location", "https://untrusted.invalid/object".into())], + ), + ] { + let destination = tempfile::tempdir().unwrap().path().join("download"); + let (gateway, requests, task) = mock(vec![(status, headers, String::new())]).await; + assert!( + gateway + .copy("s3://files/a", destination.to_str().unwrap(), true, false) + .await + .is_err() + ); + assert!(!destination.exists()); + assert_eq!(requests.lock().unwrap().len(), 1); + task.abort(); + } +} + +#[tokio::test] +async fn deletion_is_exact_and_status_checked() { + for status in [204, 500] { + let (gateway, requests, task) = mock(vec![(status, vec![], String::new())]).await; + assert_eq!( + gateway + .delete(&Remote::parse("s3://files/dir//%2F/").unwrap().unwrap()) + .await + .is_ok(), + status == 204 + ); + assert_eq!(requests.lock().unwrap()[0].1, "/files/dir//%252F/"); + task.abort(); + } +} + +#[tokio::test] +async fn bounds_listing_responses() { + let (gateway, _, task) = mock(vec![(200, vec![], "x".repeat(MAX_XML + 1))]).await; + assert!( + gateway + .list(None) + .await + .unwrap_err() + .to_string() + .contains("limit") + ); + task.abort(); +} + +fn provider(body: &[u8], declared_length: usize) -> (String, std::thread::JoinHandle) { + let body = body.to_vec(); + use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject as _}; + crate::install_crypto_provider().unwrap(); + let certificate = + CertificateDer::from_pem_slice(include_bytes!("test-certificate.pem")).unwrap(); + let key = PrivateKeyDer::from_pem_slice(include_bytes!("test-private-key.pem")).unwrap(); + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![certificate], key) + .unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!( + "https://localhost:{}/object?signature=test", + listener.local_addr().unwrap().port() + ); + let thread = std::thread::spawn(move || { + let (socket, _) = listener.accept().unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + socket + .set_write_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let connection = rustls::ServerConnection::new(Arc::new(config)).unwrap(); + let mut stream = rustls::StreamOwned::new(connection, socket); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut byte = [0]; + stream.read_exact(&mut byte).unwrap(); + request.push(byte[0]); + } + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {declared_length}\r\nConnection: close\r\n\r\n" + ) + .unwrap(); + stream.write_all(&body).unwrap(); + stream.flush().unwrap(); + String::from_utf8(request).unwrap() + }); + (url, thread) +} + +fn trust_provider(gateway: &mut Gateway<'_>) { + let certificate = + reqwest::Certificate::from_pem(include_bytes!("test-certificate.pem")).unwrap(); + gateway.http = crate::http_client_builder() + .unwrap() + .no_proxy() + .tls_certs_only([certificate]) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); +} + +#[tokio::test] +async fn downloads_and_gateway_copies_without_forwarding_bearer_and_moves_after_success() { + for (local, moving, put_status, delete_status) in [ + (true, false, 200, 204), + (true, true, 200, 204), + (false, false, 200, 204), + (false, true, 200, 204), + (false, true, 500, 204), + (false, true, 200, 500), + ] { + let (url, provider) = provider(b"payload", 7); + let mut responses = vec![(307, vec![("location", url)], String::new())]; + if !local { + responses.push((put_status, vec![], String::new())); + } + if moving && (local || put_status == 200) { + responses.push((delete_status, vec![], String::new())); + } + let (mut gateway, requests, task) = mock(responses).await; + trust_provider(&mut gateway); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("download"); + let destination = if local { + path.to_str().unwrap() + } else { + "s3://archive/copied" + }; + let result = gateway + .copy("s3://files/original", destination, moving, false) + .await; + assert_eq!(result.is_ok(), put_status == 200 && delete_status == 204); + if local { + assert_eq!(std::fs::read(&path).unwrap(), b"payload"); + } + let request = provider.join().unwrap().to_ascii_lowercase(); + assert!(request.starts_with("get /object?signature=test ")); + assert!(!request.contains("authorization:")); + assert!(!request.contains("test-key")); + let requests = requests.lock().unwrap(); + assert_eq!(requests[0].0, "GET"); + if !local { + assert_eq!( + requests[1], + ("PUT".into(), "/archive/copied".into(), b"payload".to_vec()) + ); + } + if moving && (local || put_status == 200) { + assert_eq!(requests.last().unwrap().0, "DELETE"); + assert_eq!(requests.last().unwrap().1, "/files/original"); + } else { + assert!(!requests.iter().any(|request| request.0 == "DELETE")); + } + task.abort(); + } +} + +#[tokio::test] +async fn oversized_provider_download_preserves_source_and_destination() { + let (url, provider) = provider(b"", MAX_OBJECT + 1); + let (mut gateway, requests, task) = + mock(vec![(307, vec![("location", url)], String::new())]).await; + trust_provider(&mut gateway); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("download"); + assert!( + gateway + .copy("s3://files/original", path.to_str().unwrap(), true, false) + .await + .unwrap_err() + .to_string() + .contains("limit") + ); + assert!(!path.exists()); + assert_eq!(requests.lock().unwrap().len(), 1); + provider.join().unwrap(); + task.abort(); +} + +#[tokio::test] +async fn bounds_bodies_even_without_content_length() { + let response = axum::http::Response::builder() + .status(200) + .body(vec![0; 17]) + .unwrap(); + assert!( + bounded_body(reqwest::Response::from(response), 16) + .await + .is_err() + ); + let response = axum::http::Response::builder() + .status(200) + .body(vec![0; 16]) + .unwrap(); + assert_eq!( + bounded_body(reqwest::Response::from(response), 16) + .await + .unwrap() + .len(), + 16 + ); +} + +#[test] +fn configuration_and_machine_description_include_s3() { + use clap::CommandFactory as _; + let config: crate::config::Config = + toml::from_str("s3_endpoint = 'https://s3.example.test'").unwrap(); + assert_eq!( + config.s3_endpoint.as_deref(), + Some("https://s3.example.test") + ); + let description = + crate::describe::generate(crate::Opts::command(), &["s3".into(), "ls".into()]).unwrap(); + assert_eq!(description["command"]["requirements"]["pod"], true); + assert_eq!(description["command"]["requirements"]["apiToken"], true); + assert_eq!(description["command"]["effect"], "read"); +} + +fn initiated() -> MockResponse { + (200, vec![], "filesobject.binopaque+/=&%2F".into()) +} + +fn part_response(number: usize) -> MockResponse { + ( + 200, + vec![("etag", format!("\"part-{number}\""))], + String::new(), + ) +} + +fn completed() -> MockResponse { + (200, vec![], "filesobject.bin\"complete\"".into()) +} + +fn streaming_remote() -> Remote { + Remote::parse("s3://files/object.bin").unwrap().unwrap() +} + +#[tokio::test] +async fn multipart_streams_more_than_25mib_with_ordered_etags_and_opaque_id() { + use tokio::io::AsyncReadExt as _; + let marker = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(marker.path(), b"commit\n").unwrap(); + let size = 26 * 1024 * 1024 + 17; + let (gateway, requests, task) = mock(vec![ + initiated(), + part_response(1), + part_response(2), + part_response(3), + part_response(4), + completed(), + ]) + .await; + let input = tokio::io::repeat(b'x').take(size as u64); + let result = gateway + .upload_stream( + input, + &streaming_remote(), + marker.path(), + &["project=example".into()], + std::future::pending(), + ) + .await + .unwrap(); + assert_eq!(result["size"], size); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 6); + assert_eq!(requests[0].1, "/files/object.bin?uploads"); + for (index, request) in requests[1..5].iter().enumerate() { + assert_eq!(request.0, "PUT"); + let url = Url::parse(&format!("http://localhost{}", request.1)).unwrap(); + let pairs: std::collections::HashMap<_, _> = url.query_pairs().collect(); + assert_eq!(pairs["uploadId"], "opaque+/=&%2F"); + assert_eq!(pairs["partNumber"], (index + 1).to_string()); + assert_eq!( + request.2.len(), + if index < 3 { + 8 * 1024 * 1024 + } else { + 2 * 1024 * 1024 + 17 + } + ); + assert!(request.2.iter().all(|b| *b == b'x')); + } + let xml = String::from_utf8(requests[5].2.clone()).unwrap(); + let first = xml.find("1").unwrap(); + let last = xml.find("4").unwrap(); + assert!(first < last); + assert!(xml.contains("part-1")); + assert_eq!(requests[5].0, "POST"); + task.abort(); +} + +#[tokio::test] +async fn multipart_never_completes_without_valid_producer_commit() { + for marker_contents in [None, Some(""), Some("commit"), Some("commit\nextra")] { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("success"); + if let Some(contents) = marker_contents { + std::fs::write(&marker, contents).unwrap(); + } + let (gateway, requests, task) = mock(vec![ + initiated(), + part_response(1), + (204, vec![], String::new()), + ]) + .await; + assert!( + gateway + .upload_stream( + &b"partial producer output"[..], + &streaming_remote(), + &marker, + &[], + std::future::pending() + ) + .await + .is_err() + ); + let requests = requests.lock().unwrap(); + assert_eq!( + requests.iter().map(|r| r.0.as_str()).collect::>(), + ["POST", "PUT", "DELETE"] + ); + task.abort(); + } +} + +struct FailingInput { + remaining: usize, +} + +impl tokio::io::AsyncRead for FailingInput { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + _: &mut std::task::Context<'_>, + buffer: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + if self.remaining == 0 { + return std::task::Poll::Ready(Err(std::io::Error::other("producer read failed"))); + } + let count = self.remaining.min(buffer.remaining()); + buffer.initialize_unfilled()[..count].fill(b'x'); + buffer.advance(count); + self.remaining -= count; + std::task::Poll::Ready(Ok(())) + } +} + +#[tokio::test] +async fn multipart_aborts_on_input_error_even_with_a_marker() { + let marker = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(marker.path(), b"commit\n").unwrap(); + let (gateway, requests, task) = mock(vec![ + initiated(), + part_response(1), + (204, vec![], String::new()), + ]) + .await; + let input = FailingInput { + remaining: 8 * 1024 * 1024 + 23, + }; + assert!( + gateway + .upload_stream( + input, + &streaming_remote(), + marker.path(), + &[], + std::future::pending() + ) + .await + .unwrap_err() + .to_string() + .contains("input") + ); + assert_eq!( + requests + .lock() + .unwrap() + .iter() + .map(|r| r.0.as_str()) + .collect::>(), + ["POST", "PUT", "DELETE"] + ); + task.abort(); +} + +#[tokio::test] +async fn multipart_aborts_failed_parts_and_error_or_malformed_completions() { + let marker = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(marker.path(), b"commit\n").unwrap(); + let failed_parts = [ + (500, vec![], String::new()), + (200, vec![], String::new()), + (200, vec![("etag", "unquoted".into())], String::new()), + ( + 200, + vec![("etag", format!("\"{}\"", "a".repeat(255)))], + String::new(), + ), + ]; + for failed in failed_parts { + let (gateway, requests, task) = + mock(vec![initiated(), failed, (204, vec![], String::new())]).await; + assert!( + gateway + .upload_stream( + &b"raw bytes"[..], + &streaming_remote(), + marker.path(), + &[], + std::future::pending() + ) + .await + .is_err() + ); + assert_eq!(requests.lock().unwrap().last().unwrap().0, "DELETE"); + task.abort(); + } + for xml in [ + "InternalError", + "filesobject.bin", + "otherobject.bin\"x\"", + "filesobject.bin\"x\"", + "filesobject.bin\"x\"", + ] { + let (gateway, requests, task) = mock(vec![ + initiated(), + part_response(1), + (200, vec![], xml.into()), + (204, vec![], String::new()), + ]) + .await; + assert!( + gateway + .upload_stream( + &b"raw bytes"[..], + &streaming_remote(), + marker.path(), + &[], + std::future::pending() + ) + .await + .unwrap_err() + .to_string() + .contains("uncertain") + ); + assert_eq!(requests.lock().unwrap().last().unwrap().0, "DELETE"); + task.abort(); + } +} + +struct WaitingInput { + read_started: Arc, +} + +impl tokio::io::AsyncRead for WaitingInput { + fn poll_read( + self: std::pin::Pin<&mut Self>, + _: &mut std::task::Context<'_>, + _: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + self.read_started.notify_one(); + std::task::Poll::Pending + } +} + +#[tokio::test] +async fn multipart_aborts_when_cancelled_while_waiting_for_input() { + let read_started = Arc::new(tokio::sync::Notify::new()); + let (gateway, requests, task) = mock(vec![initiated(), (204, vec![], String::new())]).await; + let input = WaitingInput { + read_started: read_started.clone(), + }; + assert!( + gateway + .upload_stream( + input, + &streaming_remote(), + Path::new("/unused"), + &[], + read_started.notified() + ) + .await + .unwrap_err() + .to_string() + .contains("cancelled") + ); + assert_eq!( + requests + .lock() + .unwrap() + .iter() + .map(|r| r.0.as_str()) + .collect::>(), + ["POST", "DELETE"] + ); + task.abort(); +} + +#[tokio::test] +async fn streaming_download_exceeds_small_copy_limit_without_credentials_or_summary() { + let body = vec![b'z'; 26 * 1024 * 1024]; + let (url, provider) = provider(&body, body.len()); + let (mut gateway, requests, task) = + mock(vec![(307, vec![("location", url)], String::new())]).await; + trust_provider(&mut gateway); + let mut output = tokio::io::sink(); + assert_eq!( + gateway + .download_stream(&streaming_remote(), &mut output) + .await + .unwrap(), + body.len() as u64 + ); + let request = provider.join().unwrap().to_ascii_lowercase(); + assert!(!request.contains("authorization:")); + assert!(!request.contains("test-key")); + assert_eq!(requests.lock().unwrap().len(), 1); + let mut bytes = Vec::new(); + let (url, provider) = self::provider(b"raw encrypted bytes", 19); + let (mut gateway, _, second_task) = + mock(vec![(307, vec![("location", url)], String::new())]).await; + trust_provider(&mut gateway); + assert_eq!( + gateway + .download_stream(&streaming_remote(), &mut bytes) + .await + .unwrap(), + 19 + ); + assert_eq!(bytes, b"raw encrypted bytes"); + provider.join().unwrap(); + task.abort(); + second_task.abort(); +} From 808eba045050b04e16c347274f1e4bfce5a11065 Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Tue, 3 Nov 2026 20:29:19 +0100 Subject: [PATCH 2/2] feat: use AWS SDK for S3 gateway transfers --- Cargo.lock | 673 +++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 5 +- README.md | 6 +- src/s3.rs | 282 +++++++------------ src/s3/multipart.rs | 257 +++++++---------- src/s3/sdk.rs | 271 ++++++++++++++++++ src/s3/sdk_tests.rs | 495 ++++++++++++++++++++++++++++++++ src/s3/tests.rs | 92 +++++- 8 files changed, 1698 insertions(+), 383 deletions(-) create mode 100644 src/s3/sdk.rs create mode 100644 src/s3/sdk_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 8eef30a..d5a438a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "anstream" version = "1.0.0" @@ -118,6 +124,289 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-credential-types" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dcd93c82209ac7413532388067dce79be5a8780c1786e5fae3df22e4dee2864" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "http-body 0.4.6", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.132.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5575840a3a6b11f6011463ebe359320dfe5b67babb5e9b06fed6ddf809a9ab40" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.2", + "http-body 1.0.1", + "lru", + "percent-encoding", + "regex-lite", + "sha2 0.11.1", + "tracing", + "url", +] + +[[package]] +name = "aws-sigv4" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68dc0b907359b120170613b5c09ccc61304eac3998ff6274b97d93ee6490115a" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.2", + "percent-encoding", + "sha2 0.11.1", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.64.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10efbbcec1e044b81600e2fc562a391951d291152d95b482d5b7e7132299d762" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 1.4.2", + "http-body 1.0.1", + "http-body-util", + "md-5", + "pin-project-lite", + "sha1", + "sha2 0.11.1", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf09d74e5e32f76b8762da505a3cd59303e367a664ca67295387baa8c1d7548" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.62.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0504b1ab12debb5959e5165ee5fe97dd387e7aa7ea6a477bfd7635dfe769a4f5" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71a13df6ada0aafbf21a73bdfcdf9324cfa9df77d96b8446045be3cde61b42e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.2", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "aws-smithy-types" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.2", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bbcaa9304ea40902d3d5f42a0428d1bd895a2b0f6999436fb279ffddc58ac" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.7.9" @@ -128,8 +417,8 @@ dependencies = [ "axum-core 0.4.5", "bytes", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "itoa", "matchit 0.7.3", @@ -155,8 +444,8 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-util", @@ -187,8 +476,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -206,8 +495,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -223,6 +512,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "beef" version = "0.5.2" @@ -244,11 +543,24 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "brainpod-cli" version = "0.1.0" dependencies = [ "anyhow", + "aws-sdk-s3", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "axum 0.8.9", "clap", "dirs", @@ -257,12 +569,11 @@ dependencies = [ "prost", "prost-types", "protox", - "quick-xml", "reqwest", "rustls", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tar", "tempfile", "tokio", @@ -287,6 +598,16 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.2.64" @@ -349,6 +670,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -365,6 +692,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "core-foundation" version = "0.9.4" @@ -400,6 +733,42 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc-fast" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" +dependencies = [ + "crc", + "digest 0.10.7", + "rustversion", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -419,14 +788,50 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2376ffd6f96d8a9db59875b4b6b0237d6900b5baf64279714e05493ce64828a" +dependencies = [ + "cmov", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -536,6 +941,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -660,7 +1071,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.4.2", "indexmap 2.14.0", "slab", "tokio", @@ -674,6 +1085,17 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -686,6 +1108,32 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.2" @@ -696,6 +1144,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -703,7 +1162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.2", ] [[package]] @@ -714,8 +1173,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", ] @@ -731,6 +1190,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -742,8 +1210,8 @@ dependencies = [ "futures-channel", "futures-core", "h2", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -759,7 +1227,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http", + "http 1.4.2", "hyper", "hyper-util", "rustls", @@ -791,8 +1259,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "hyper", "ipnet", "libc", @@ -1113,6 +1581,15 @@ dependencies = [ "logos-codegen", ] +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "matchit" version = "0.7.3" @@ -1125,6 +1602,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.2" @@ -1192,6 +1679,30 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "objc2" version = "0.6.4" @@ -1241,6 +1752,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1283,6 +1800,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "potential_utf" version = "0.1.5" @@ -1292,6 +1815,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1412,16 +1941,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "quick-xml" -version = "0.38.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quote" version = "1.0.45" @@ -1507,6 +2026,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" @@ -1525,8 +2050,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", @@ -1809,6 +2334,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1816,8 +2352,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -1890,6 +2437,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2035,6 +2588,36 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -2157,8 +2740,8 @@ dependencies = [ "base64", "bytes", "h2", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-timeout", @@ -2237,8 +2820,8 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", "tower 0.5.3", "tower-layer", @@ -2361,6 +2944,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -2742,6 +3331,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index efe826d..8cce05e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,13 +10,16 @@ path = "src/main.rs" [dependencies] anyhow = "1.0.102" axum = "0.8.8" +aws-sdk-s3 = { version = "=1.132.0", default-features = false, features = ["rt-tokio"] } +aws-smithy-http = "0.63.6" +aws-smithy-runtime-api = { version = "1.12.0", features = ["client", "http-1x"] } +aws-smithy-types = "1.4.7" clap = { version = "4.6.1", features = ["derive"] } dirs = "6.0.0" flate2 = "1.1.2" getrandom = "0.3.4" prost = "0.13.5" prost-types = "0.13.5" -quick-xml = { version = "0.38.4", features = ["serialize"] } reqwest = { version = "0.13.4", default-features = false, features = ["charset", "http2", "json", "query", "rustls-no-provider", "stream", "system-proxy"] } rustls = { version = "0.23.40", default-features = false, features = ["ring", "std", "tls12"] } serde = { version = "1.0.228", features = ["derive"] } diff --git a/README.md b/README.md index 94f3d6c..68d09d1 100644 --- a/README.md +++ b/README.md @@ -395,7 +395,9 @@ Transfers require explicit file/key destinations (no implicit basename or direct The default gateway origin is `https://s3.brainpod.io`; the CLI prepends the selected Pod name to its hostname. Override the base origin with `--s3-endpoint`, `BRAINPOD_S3_ENDPOINT`, or `brainpod config set s3-endpoint https://s3.example.test` (in that precedence order). For local development, use `http://s3.brainpod.localhost` with working wildcard DNS. HTTP is unencrypted and should only be used locally. The API endpoint does not change the S3 origin. -Downloads explicitly accept the gateway's 307 redirect and use a separate unauthenticated HTTPS request to the short-lived provider URL. Brainpod Bearer credentials are never sent to the provider; further redirects are rejected. Presigned URLs are not printed. There is no AWS SigV4 client authentication, public HEAD, recursive/sync transfer or deletion, bucket administration, ranged reads, or metadata preservation for `cp`/`mv`. Multipart is reserved for explicit stdin streaming uploads. Transfer errors return a nonzero exit status; interrupted uploads can have an uncertain remote outcome, so inspect the key before retrying. +Bucket/object listings, PUT/delete, and multipart initiation, parts, completion, and abort use typed `aws-sdk-s3` operations and SDK XML serialization/deserialization. A gateway transport keeps logical buckets path-style beneath the Pod hostname, uses unsigned SDK auth with Bearer authentication added only at the gateway, removes the SDK's `x-id` query parameter and nonsemantic `x-amz-user-agent` telemetry header, disables optional checksums, and requires buffered, non-`aws-chunked` request bodies. Requests are not automatically retried. Response byte limits remain in place; XML parsing and service-error detection are delegated to the SDK. + +Downloads explicitly accept the gateway's 307 redirect and use a separate unauthenticated HTTPS request to the short-lived provider URL. Both download requests remain implemented with `reqwest`, not SDK GetObject, to enforce this redirect contract. Brainpod Bearer credentials are never sent to the provider; further redirects are rejected. Presigned URLs are not printed. There is no AWS SigV4 client authentication, public HEAD, recursive/sync transfer or deletion, bucket administration, ranged reads, or metadata preservation for `cp`/`mv`. Multipart is reserved for explicit stdin streaming uploads. Transfer errors return a nonzero exit status; interrupted uploads can have an uncertain remote outcome, so inspect the key before retrying. ### Streaming transfers @@ -418,7 +420,7 @@ trap 'rm -rf "$run_dir"' EXIT Generic object metadata is accepted on multipart initiation using repeated `--metadata KEY=VALUE`. Keys must be nonempty valid HTTP header names; values must be nonempty printable ASCII. Metadata names and values together may total at most 2048 bytes, and duplicate keys are rejected case-insensitively. Endpoint-specific metadata policies are enforced by the server. -Missing or invalid markers, input errors, rejected parts, malformed completion responses (including HTTP 200 ``), SIGINT, and SIGTERM fail the upload and attempt to abort it. Completion/network failures can have an uncertain outcome; inspect the exact key before retrying. Abrupt process termination or failed aborts can leave incomplete uploads requiring endpoint cleanup. +Missing or invalid markers, input errors, rejected parts, completion errors reported by the SDK (including HTTP 200 ``), SIGINT, and SIGTERM fail the upload and attempt to abort it. Completion/network failures can have an uncertain outcome; inspect the exact key before retrying. Abrupt process termination or failed aborts can leave incomplete uploads requiring endpoint cleanup. Stream an object to stdout without the small-copy limit: diff --git a/src/s3.rs b/src/s3.rs index b207af9..fa16a83 100644 --- a/src/s3.rs +++ b/src/s3.rs @@ -6,7 +6,6 @@ use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; use clap::{Args, Subcommand}; use reqwest::{Method, StatusCode, Url}; -use serde::Deserialize; use serde_json::{Value, json}; use crate::output::{CommandOutput, View}; @@ -86,6 +85,7 @@ impl Remote { struct Gateway<'a> { http: reqwest::Client, + sdk: aws_sdk_s3::Client, origin: Url, token: &'a str, } @@ -111,12 +111,15 @@ impl<'a> Gateway<'a> { .host_str() .ok_or_else(|| anyhow!("S3 endpoint has no hostname"))?; origin.set_host(Some(&format!("{pod}.{host}")))?; + let http = crate::http_client_builder()? + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(30)) + .read_timeout(Duration::from_secs(60)) + .build()?; + let sdk = sdk::client(http.clone(), &origin, token)?; Ok(Self { - http: crate::http_client_builder()? - .redirect(reqwest::redirect::Policy::none()) - .connect_timeout(Duration::from_secs(30)) - .read_timeout(Duration::from_secs(60)) - .build()?, + http, + sdk, origin, token, }) @@ -125,48 +128,31 @@ impl<'a> Gateway<'a> { fn url(&self, remote: Option<&Remote>) -> Result { let mut url = self.origin.clone(); if let Some(remote) = remote { - let mut segments = url - .path_segments_mut() - .map_err(|_| anyhow!("invalid gateway URL"))?; - segments.clear().push(&remote.bucket); + let mut path = format!("/{}", remote.bucket); if !remote.key.is_empty() { remote.object()?; - for segment in remote.key.split('/') { - segments.push(segment); - } + path.push('/'); + path.push_str(&aws_smithy_http::label::fmt_string( + &remote.key, + aws_smithy_http::label::EncodingStrategy::Greedy, + )); } + url.set_path(&path); } Ok(url) } - async fn request( - &self, - method: Method, - url: Url, - body: Option>, - ) -> Result { - let mut request = self - .http - .request(method, url) - .bearer_auth(self.token) - .timeout(Duration::from_secs(120)); - if let Some(body) = body { - request = request - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(body); - } - request - .send() - .await - .map_err(|_| anyhow!("S3 gateway request failed")) - } - async fn delete(&self, remote: &Remote) -> Result<()> { remote.object()?; - let response = self - .request(Method::DELETE, self.url(Some(remote))?, None) - .await?; - expect(response, StatusCode::NO_CONTENT).await?; + self.sdk + .delete_object() + .bucket(&remote.bucket) + .key(sdk::key(&remote.key)) + .customize() + .interceptor(sdk::LiteralKey::new(&remote.key)) + .send() + .await + .map_err(sdk::error)?; Ok(()) } @@ -181,8 +167,13 @@ impl<'a> Gateway<'a> { ) -> Result { remote.object()?; let response = self - .request(Method::GET, self.url(Some(remote))?, None) - .await?; + .http + .get(self.url(Some(remote))?) + .bearer_auth(self.token) + .timeout(Duration::from_secs(120)) + .send() + .await + .map_err(|_| anyhow!("S3 gateway request failed"))?; let response = expect(response, StatusCode::TEMPORARY_REDIRECT).await?; let location = response .headers() @@ -241,10 +232,17 @@ impl<'a> Gateway<'a> { let size = bytes.len(); match dst { Some(dst) => { - let response = self - .request(Method::PUT, self.url(Some(&dst))?, Some(bytes)) - .await?; - expect(response, StatusCode::OK).await?; + self.sdk + .put_object() + .bucket(&dst.bucket) + .key(sdk::key(&dst.key)) + .content_type("application/octet-stream") + .body(aws_sdk_s3::primitives::ByteStream::from(bytes)) + .customize() + .interceptor(sdk::LiteralKey::new(&dst.key)) + .send() + .await + .map_err(sdk::error)?; } None => { let path = Path::new(destination); @@ -283,57 +281,73 @@ impl<'a> Gateway<'a> { let mut token: Option = None; let mut seen = HashSet::new(); loop { - let bucket = remote.as_ref().map(|r| Remote { - bucket: r.bucket.clone(), - key: String::new(), - }); - let mut url = self.url(bucket.as_ref())?; - { - let mut query = url.query_pairs_mut(); - if let Some(remote) = &remote { - query.extend_pairs([ - ("list-type", "2"), - ("encoding-type", "url"), - ("delimiter", "/"), - ("max-keys", "1000"), - ("prefix", remote.key.as_str()), - ]); - } - if let Some(token) = &token { - query.append_pair("continuation-token", token); + token = if let Some(remote) = &remote { + let page = self + .sdk + .list_objects_v2() + .bucket(&remote.bucket) + .prefix(&remote.key) + .delimiter("/") + .encoding_type(aws_sdk_s3::types::EncodingType::Url) + .max_keys(1000) + .set_continuation_token(token) + .send() + .await + .map_err(sdk::error)?; + let encoded = match page.encoding_type().map(|encoding| encoding.as_str()) { + Some("url") => true, + None => false, + _ => bail!("unsupported gateway listing encoding"), + }; + for object in page.contents() { + let key = object + .key() + .ok_or_else(|| anyhow!("listing object has no key"))?; + let size = object + .size() + .filter(|size| *size >= 0) + .ok_or_else(|| anyhow!("listing object has no valid size"))?; + let modified = object + .last_modified() + .map(|date| date.fmt(aws_smithy_types::date_time::Format::DateTime)) + .transpose()?; + items.push(json!({"kind": "object", "key": decode_field(key, encoded)?, "size": size, "lastModified": modified})); } - } - let response = self.request(Method::GET, url, None).await?; - let xml = bounded_body(expect(response, StatusCode::OK).await?, MAX_XML).await?; - let page = parse_listing(&xml, remote.is_some())?; - if remote.is_some() { - let encoded = page.encoding_type.as_deref() == Some("url"); - for object in page.contents { - items.push(json!({"kind": "object", "key": decode_field(&object.key, encoded)?, "size": object.size, "lastModified": object.last_modified})); - } - for prefix in page.common_prefixes { - items.push( - json!({"kind": "prefix", "key": decode_field(&prefix.prefix, encoded)?}), - ); - } - } else { - for bucket in page.buckets.bucket { - items.push(json!({"kind": "bucket", "name": bucket.name})); + for prefix in page.common_prefixes() { + let prefix = prefix + .prefix() + .ok_or_else(|| anyhow!("listing prefix has no key"))?; + items.push(json!({"kind": "prefix", "key": decode_field(prefix, encoded)?})); } - } - token = if remote.is_some() { - if page.is_truncated { - page.next_continuation_token + let next = if page.is_truncated() == Some(true) { + page.next_continuation_token() + .filter(|token| !token.is_empty()) + .map(str::to_owned) } else { None + }; + if page.is_truncated() == Some(true) && next.is_none() { + bail!("truncated gateway listing has no continuation token"); } + next } else { - page.continuation_token.or(page.next_continuation_token) - } - .filter(|t| !t.is_empty()); - if page.is_truncated && token.is_none() { - bail!("truncated gateway listing has no continuation token"); - } + let page = self + .sdk + .list_buckets() + .set_continuation_token(token) + .send() + .await + .map_err(sdk::error)?; + for bucket in page.buckets() { + let name = bucket + .name() + .ok_or_else(|| anyhow!("listing bucket has no name"))?; + items.push(json!({"kind": "bucket", "name": name})); + } + page.continuation_token() + .filter(|token| !token.is_empty()) + .map(str::to_owned) + }; let Some(next) = &token else { break; }; @@ -345,36 +359,6 @@ impl<'a> Gateway<'a> { } } -fn parse_listing(xml: &[u8], objects: bool) -> Result { - let mut reader = quick_xml::Reader::from_reader(xml); - reader.config_mut().trim_text(true); - let expected = if objects { - "ListBucketResult" - } else { - "ListAllMyBucketsResult" - }; - loop { - match reader.read_event().context("invalid gateway listing XML")? { - quick_xml::events::Event::Decl(_) | quick_xml::events::Event::Comment(_) => continue, - quick_xml::events::Event::Start(tag) | quick_xml::events::Event::Empty(tag) - if tag.local_name().as_ref() == expected.as_bytes() => - { - break; - } - _ => bail!("unexpected gateway listing XML root"), - } - } - let page: Listing = quick_xml::de::from_reader(xml).context("invalid gateway listing XML")?; - if page - .encoding_type - .as_deref() - .is_some_and(|encoding| encoding != "url") - { - bail!("unsupported gateway listing encoding"); - } - Ok(page) -} - fn provider_url(location: &str) -> Result { let url = Url::parse(location).context("invalid provider redirect URL")?; if url.scheme() != "https" @@ -419,16 +403,7 @@ async fn expect(response: reqwest::Response, status: StatusCode) -> Result(xml.as_slice()).ok()) - .map(|e| e.code); - bail!( - "S3 request returned HTTP {}{}", - actual.as_u16(), - code.map(|c| format!(" ({c})")).unwrap_or_default() - ) + bail!("S3 request returned HTTP {}", actual.as_u16()) } fn decode_field(value: &str, encoded: bool) -> Result { @@ -457,49 +432,6 @@ fn decode_field(value: &str, encoded: bool) -> Result { String::from_utf8(decoded).context("listing field is not UTF-8") } -#[derive(Default, Deserialize)] -#[serde(rename_all = "PascalCase")] -struct Listing { - #[serde(default)] - buckets: Buckets, - #[serde(default)] - contents: Vec, - #[serde(default)] - common_prefixes: Vec, - encoding_type: Option, - #[serde(default)] - is_truncated: bool, - next_continuation_token: Option, - continuation_token: Option, -} -#[derive(Default, Deserialize)] -struct Buckets { - #[serde(rename = "Bucket", default)] - bucket: Vec, -} -#[derive(Deserialize)] -#[serde(rename_all = "PascalCase")] -struct Bucket { - name: String, -} -#[derive(Deserialize)] -#[serde(rename_all = "PascalCase")] -struct Object { - key: String, - size: u64, - last_modified: Option, -} -#[derive(Deserialize)] -#[serde(rename_all = "PascalCase")] -struct Prefix { - prefix: String, -} -#[derive(Deserialize)] -#[serde(rename_all = "PascalCase")] -struct S3Error { - code: String, -} - pub async fn handle(args: S3Args, endpoint: &str, pod: &str, token: &str) -> Result { let gateway = Gateway::new(endpoint, pod, token)?; let value = match args.command { @@ -593,6 +525,10 @@ pub fn render(value: &Value) -> Vec { } mod multipart; +mod sdk; #[cfg(test)] mod tests; + +#[cfg(test)] +mod sdk_tests; diff --git a/src/s3/multipart.rs b/src/s3/multipart.rs index dd17387..9c2a74e 100644 --- a/src/s3/multipart.rs +++ b/src/s3/multipart.rs @@ -1,14 +1,14 @@ use std::future::Future; -use reqwest::header::{CONTENT_TYPE, ETAG, HeaderMap, HeaderName, HeaderValue}; -use serde::Serialize; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use tokio::io::{AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; use super::*; const PART_SIZE: usize = 8 * 1024 * 1024; const MAX_PARTS: usize = 10_000; -const MAX_COMPLETION: usize = 2 * 1024 * 1024; +pub(super) const MAX_COMPLETION: usize = 2 * 1024 * 1024; pub(super) fn cancellation() -> Result> { #[cfg(unix)] let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; @@ -54,53 +54,6 @@ fn metadata_headers(metadata: &[String]) -> Result { Ok(headers) } -fn parse_xml(xml: &[u8], expected: &str) -> Result { - let mut reader = quick_xml::Reader::from_reader(xml); - reader.config_mut().trim_text(true); - let mut depth = 0usize; - let mut root = false; - loop { - match reader - .read_event() - .context("invalid multipart response XML")? - { - quick_xml::events::Event::Decl(_) | quick_xml::events::Event::Comment(_) => {} - quick_xml::events::Event::Start(tag) => { - if depth == 0 { - if root || tag.local_name().as_ref() != expected.as_bytes() { - bail!("unexpected multipart response XML root (expected {expected})"); - } - root = true; - } - depth += 1; - if depth > 16 { - bail!("multipart response XML is too deeply nested"); - } - } - quick_xml::events::Event::Empty(tag) => { - if depth == 0 { - if root || tag.local_name().as_ref() != expected.as_bytes() { - bail!("unexpected multipart response XML root (expected {expected})"); - } - root = true; - } - } - quick_xml::events::Event::End(_) => { - depth = depth - .checked_sub(1) - .ok_or_else(|| anyhow!("invalid multipart XML nesting"))?; - } - quick_xml::events::Event::Text(text) - if depth > 0 || text.iter().all(u8::is_ascii_whitespace) => {} - quick_xml::events::Event::CData(_) | quick_xml::events::Event::GeneralRef(_) - if depth > 0 => {} - quick_xml::events::Event::Eof if root && depth == 0 => break, - _ => bail!("invalid or incomplete multipart response XML"), - } - } - quick_xml::de::from_reader(xml).context("invalid multipart response XML fields") -} - fn validate_etag(etag: &str) -> Result<()> { if !(3..=256).contains(&etag.len()) || !etag.starts_with('"') @@ -114,46 +67,7 @@ fn validate_etag(etag: &str) -> Result<()> { Ok(()) } -#[derive(Deserialize)] -#[serde(rename_all = "PascalCase")] -struct Initiated { - bucket: String, - key: String, - upload_id: String, -} - -#[derive(Serialize)] -#[serde(rename = "CompleteMultipartUpload")] -struct Completion { - #[serde(rename = "Part")] - parts: Vec, -} - -#[derive(Serialize)] -struct Part { - #[serde(rename = "PartNumber")] - number: usize, - #[serde(rename = "ETag")] - etag: String, -} - -#[derive(Deserialize)] -struct Completed { - #[serde(rename = "Bucket")] - bucket: String, - #[serde(rename = "Key")] - key: String, - #[serde(rename = "ETag")] - etag: String, -} - impl Gateway<'_> { - fn multipart_url(&self, remote: &Remote, upload_id: &str) -> Result { - let mut url = self.url(Some(remote))?; - url.query_pairs_mut().append_pair("uploadId", upload_id); - Ok(url) - } - pub(super) async fn upload_stream>( &self, mut input: R, @@ -176,14 +90,16 @@ impl Gateway<'_> { Err(error) => { if let Some(upload_id) = upload_id { let abort = async { - let response = self - .request( - Method::DELETE, - self.multipart_url(remote, &upload_id)?, - None, - ) - .await?; - expect(response, StatusCode::NO_CONTENT).await?; + self.sdk + .abort_multipart_upload() + .bucket(&remote.bucket) + .key(sdk::key(&remote.key)) + .upload_id(&upload_id) + .customize() + .interceptor(sdk::LiteralKey::new(&remote.key)) + .send() + .await + .map_err(sdk::error)?; Ok::<_, anyhow::Error>(()) }; if !matches!( @@ -215,31 +131,37 @@ impl Gateway<'_> { upload_id: &mut Option, completing: &mut bool, ) -> Result { - let mut url = self.url(Some(remote))?; - url.set_query(Some("uploads")); - let response = self - .http - .post(url) - .bearer_auth(self.token) - .headers(metadata) - .header(CONTENT_TYPE, "application/octet-stream") - .body(Vec::new()) - .timeout(Duration::from_secs(120)) + let metadata = metadata + .iter() + .map(|(name, value)| { + Ok(( + name.as_str()["x-amz-meta-".len()..].to_owned(), + value.to_str()?.to_owned(), + )) + }) + .collect::>>()?; + let initiated = self + .sdk + .create_multipart_upload() + .bucket(&remote.bucket) + .key(sdk::key(&remote.key)) + .set_metadata(Some(metadata)) + .content_type("application/octet-stream") + .customize() + .interceptor(sdk::LiteralKey::new(&remote.key)) .send() .await - .map_err(|_| anyhow!("multipart initiation request failed"))?; - let xml = bounded_body(expect(response, StatusCode::OK).await?, MAX_XML).await?; - let initiated: Initiated = parse_xml(&xml, "InitiateMultipartUploadResult")?; - if initiated.upload_id.is_empty() || initiated.upload_id.len() > 8192 { - bail!("gateway returned an invalid multipart upload ID"); - } - *upload_id = Some(initiated.upload_id); - if initiated.bucket != remote.bucket || initiated.key != remote.key { + .map_err(sdk::error)?; + let id = initiated + .upload_id() + .filter(|id| !id.is_empty() && id.len() <= 8192) + .ok_or_else(|| anyhow!("gateway returned an invalid multipart upload ID"))?; + *upload_id = Some(id.to_owned()); + if initiated.bucket() != Some(remote.bucket.as_str()) + || initiated.key() != Some(remote.key.as_str()) + { bail!("gateway initiated a different object"); } - let id = upload_id - .as_deref() - .ok_or_else(|| anyhow!("multipart upload has no ID"))?; let mut parts = Vec::new(); let mut size = 0u64; loop { @@ -263,20 +185,29 @@ impl Gateway<'_> { } bytes.truncate(filled); let number = parts.len() + 1; - let mut url = self.multipart_url(remote, id)?; - url.query_pairs_mut() - .append_pair("partNumber", &number.to_string()); - let response = self.request(Method::PUT, url, Some(bytes)).await?; - let response = expect(response, StatusCode::OK).await?; + let response = self + .sdk + .upload_part() + .bucket(&remote.bucket) + .key(sdk::key(&remote.key)) + .upload_id(id) + .part_number(number as i32) + .body(aws_sdk_s3::primitives::ByteStream::from(bytes)) + .customize() + .interceptor(sdk::LiteralKey::new(&remote.key)) + .send() + .await + .map_err(sdk::error)?; let etag = response - .headers() - .get(ETAG) - .ok_or_else(|| anyhow!("uploaded part response has no ETag"))? - .to_str()? - .to_owned(); - validate_etag(&etag)?; - bounded_body(response, MAX_XML).await?; - parts.push(Part { number, etag }); + .e_tag() + .ok_or_else(|| anyhow!("uploaded part response has no ETag"))?; + validate_etag(etag)?; + parts.push( + CompletedPart::builder() + .part_number(number as i32) + .e_tag(etag) + .build(), + ); size += filled as u64; if filled < PART_SIZE { break; @@ -299,26 +230,33 @@ impl Gateway<'_> { if marker != b"commit\n" { bail!("invalid producer success file; aborting without completion"); } - let body = quick_xml::se::to_string(&Completion { parts }) - .context("failed to encode multipart completion")?; - check_size(body.len(), MAX_COMPLETION)?; *completing = true; - let response = self - .http - .post(self.multipart_url(remote, id)?) - .bearer_auth(self.token) - .header(CONTENT_TYPE, "application/xml") - .body(body) - .timeout(Duration::from_secs(120)) + let completed = self + .sdk + .complete_multipart_upload() + .bucket(&remote.bucket) + .key(sdk::key(&remote.key)) + .upload_id(id) + .multipart_upload( + CompletedMultipartUpload::builder() + .set_parts(Some(parts)) + .build(), + ) + .customize() + .interceptor(sdk::LiteralKey::new(&remote.key)) .send() .await - .map_err(|_| anyhow!("multipart completion request failed"))?; - let xml = bounded_body(expect(response, StatusCode::OK).await?, MAX_COMPLETION).await?; - let completed: Completed = parse_xml(&xml, "CompleteMultipartUploadResult")?; - if completed.bucket != remote.bucket || completed.key != remote.key { + .map_err(sdk::error)?; + if completed.bucket() != Some(remote.bucket.as_str()) + || completed.key() != Some(remote.key.as_str()) + { bail!("gateway completed a different object"); } - validate_etag(&completed.etag)?; + validate_etag( + completed + .e_tag() + .ok_or_else(|| anyhow!("completion response has no ETag"))?, + )?; Ok( json!({"operation": "upload", "destination": format!("s3://{}/{}", remote.bucket, remote.key), "size": size}), ) @@ -482,19 +420,20 @@ mod tests { #[test] fn completion_manifest_supports_10000_ordered_parts() { - let completion = Completion { - parts: (1..=MAX_PARTS) - .map(|number| Part { - number, - etag: "\"opaque&etag\"".into(), - }) - .collect(), - }; - let xml = quick_xml::se::to_string(&completion).unwrap(); - assert!(xml.len() < MAX_COMPLETION); - assert!(xml.contains("opaque&etag")); - assert_eq!(xml.matches("").count(), MAX_PARTS); - assert!(xml.contains("10000")); + let completion = CompletedMultipartUpload::builder() + .set_parts(Some( + (1..=MAX_PARTS) + .map(|number| { + CompletedPart::builder() + .part_number(number as i32) + .e_tag("\"opaque&etag\"") + .build() + }) + .collect(), + )) + .build(); + assert_eq!(completion.parts().len(), MAX_PARTS); + assert_eq!(completion.parts()[MAX_PARTS - 1].part_number(), Some(10000)); } #[tokio::test] diff --git a/src/s3/sdk.rs b/src/s3/sdk.rs new file mode 100644 index 0000000..6f5b6f8 --- /dev/null +++ b/src/s3/sdk.rs @@ -0,0 +1,271 @@ +use aws_sdk_s3::config::{ + BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation, + retry::RetryConfig, +}; +use aws_smithy_runtime_api::client::{ + auth::AuthSchemeId, + http::{HttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn}, + orchestrator::{HttpRequest, HttpResponse}, + result::ConnectorError, +}; +use aws_smithy_types::body::SdkBody; + +use super::*; + +#[derive(Clone, Debug)] +struct Transport { + http: reqwest::Client, + origin: Url, + bearer: reqwest::header::HeaderValue, +} + +#[derive(Debug)] +struct TransportError(String); + +impl std::fmt::Display for TransportError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for TransportError {} + +fn failure(message: impl Into) -> ConnectorError { + ConnectorError::other(Box::new(TransportError(message.into())), None) +} + +pub(super) fn error(error: impl std::error::Error) -> anyhow::Error { + let mut source: Option<&(dyn std::error::Error + 'static)> = error.source(); + while let Some(current) = source { + if let Some(safe) = current.downcast_ref::() { + return anyhow!("{}", safe.0); + } + source = current.source(); + } + anyhow!("S3 SDK request failed") +} + +pub(super) fn client( + http: reqwest::Client, + origin: &Url, + token: &str, +) -> Result { + let mut bearer = reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")) + .map_err(|_| anyhow!("invalid S3 gateway token"))?; + bearer.set_sensitive(true); + let transport = SharedHttpConnector::new(Transport { + http, + origin: origin.clone(), + bearer, + }); + let config = aws_sdk_s3::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .endpoint_url(origin.as_str()) + .force_path_style(true) + .disable_s3_express_session_auth(true) + .allow_no_auth() + .auth_scheme_preference([AuthSchemeId::new("noAuth")]) + .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) + .response_checksum_validation(ResponseChecksumValidation::WhenRequired) + .retry_config(RetryConfig::disabled()) + .http_client(http_client_fn(move |_, _| transport.clone())) + .build(); + Ok(aws_sdk_s3::Client::from_conf(config)) +} + +impl HttpConnector for Transport { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let transport = self.clone(); + HttpConnectorFuture::new(async move { transport.send(request).await }) + } +} + +impl Transport { + async fn send( + &self, + request: HttpRequest, + ) -> std::result::Result { + let mut url = Url::parse(request.uri()).map_err(|_| failure("invalid SDK request URL"))?; + if url.origin() != self.origin.origin() + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + { + return Err(failure("SDK request must target the S3 gateway origin")); + } + let query = url.query().map(|query| { + query + .split('&') + .filter(|pair| pair.split('=').next() != Some("x-id")) + .collect::>() + .join("&") + }); + url.set_query(query.as_deref().filter(|query| !query.is_empty())); + let method = Method::from_bytes(request.method().as_bytes()) + .map_err(|_| failure("invalid SDK request method"))?; + let completion = + method == Method::POST && url.query_pairs().any(|(name, _)| name == "uploadId"); + let max = if completion { + multipart::MAX_COMPLETION + } else { + MAX_XML + }; + let body = request + .body() + .bytes() + .ok_or_else(|| failure("SDK gateway requests require a buffered body"))?; + if completion && body.len() > max { + return Err(failure("multipart completion exceeds its byte limit")); + } + let expected = if method == Method::DELETE { + StatusCode::NO_CONTENT + } else { + StatusCode::OK + }; + let mut outgoing = self + .http + .request(method, url) + .timeout(Duration::from_secs(120)); + for (name, value) in request.headers().iter() { + if name == "x-amz-user-agent" { + continue; + } + if name == "authorization" + || name == "transfer-encoding" + || name == "x-amz-trailer" + || name.starts_with("x-amz-checksum-") + || name == "x-amz-sdk-checksum-algorithm" + || (name == "content-encoding" && value.contains("aws-chunked")) + { + return Err(failure( + "unsupported SDK gateway authentication or body encoding", + )); + } + outgoing = outgoing.header(name, value); + } + let response = outgoing + .header(reqwest::header::AUTHORIZATION, self.bearer.clone()) + .body(body.to_vec()) + .send() + .await + .map_err(|_| failure("S3 gateway request failed"))?; + let status = response.status(); + if status != expected { + return Err(failure(format!( + "S3 request returned HTTP {}", + status.as_u16() + ))); + } + let headers = response.headers().clone(); + let bytes = bounded_body(response, max) + .await + .map_err(|_| failure("S3 gateway response failed or exceeds its byte limit"))?; + let mut response = HttpResponse::new(status.into(), SdkBody::from(bytes)); + for (name, value) in &headers { + let value = value + .to_str() + .map_err(|_| failure("invalid gateway response header"))?; + response + .headers_mut() + .try_insert(name.as_str().to_owned(), value.to_owned()) + .map_err(|_| failure("invalid gateway response header"))?; + } + Ok(response) + } +} + +pub(super) fn key(key: &str) -> &str { + if key.trim().is_empty() { "_" } else { key } +} + +#[derive(Debug)] +pub(super) struct LiteralKey(Option); + +impl LiteralKey { + pub(super) fn new(key: &str) -> Self { + Self(key.trim().is_empty().then(|| key.to_owned())) + } +} + +impl aws_smithy_runtime_api::client::interceptors::Intercept for LiteralKey { + fn name(&self) -> &'static str { + "LiteralS3Key" + } + + fn modify_before_serialization( + &self, + context: &mut aws_smithy_runtime_api::client::interceptors::context::BeforeSerializationInterceptorContextMut<'_>, + _: &aws_smithy_runtime_api::client::runtime_components::RuntimeComponents, + _: &mut aws_smithy_types::config_bag::ConfigBag, + ) -> std::result::Result<(), aws_smithy_runtime_api::box_error::BoxError> { + let Some(key) = &self.0 else { + return Ok(()); + }; + // SDK endpoint parameters reject whitespace-only keys; restore the literal input before SDK serialization. + macro_rules! restore { + ($($input:ty),+ $(,)?) => { $( + if let Some(input) = context.input_mut().downcast_mut::<$input>() { + input.key = Some(key.clone()); + return Ok(()); + } + )+ }; + } + restore!( + aws_sdk_s3::operation::put_object::PutObjectInput, + aws_sdk_s3::operation::delete_object::DeleteObjectInput, + aws_sdk_s3::operation::create_multipart_upload::CreateMultipartUploadInput, + aws_sdk_s3::operation::upload_part::UploadPartInput, + aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadInput, + aws_sdk_s3::operation::abort_multipart_upload::AbortMultipartUploadInput, + ); + Err("unsupported literal S3 key operation".into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn transport_rejects_other_origins_and_unsupported_wire_features_without_sending() { + let mut bearer = reqwest::header::HeaderValue::from_static("Bearer transport-secret"); + bearer.set_sensitive(true); + let transport = Transport { + http: crate::http_client_builder() + .unwrap() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(), + origin: Url::parse("http://127.0.0.1:0").unwrap(), + bearer, + }; + assert!(!format!("{transport:?}").contains("transport-secret")); + let error = transport + .call(HttpRequest::get("https://provider.invalid/object?signature=secret").unwrap()) + .await + .unwrap_err(); + assert!(!format!("{error:?}").contains("signature=secret")); + for (name, value) in [ + ("authorization", "AWS4-HMAC-SHA256 forbidden"), + ("x-amz-checksum-crc32", "AAAAAA=="), + ("x-amz-sdk-checksum-algorithm", "CRC32"), + ("x-amz-trailer", "checksum"), + ("content-encoding", "aws-chunked"), + ("transfer-encoding", "chunked"), + ] { + let mut request = HttpRequest::get("http://127.0.0.1:0/").unwrap(); + request.headers_mut().insert(name, value); + let error = transport.call(request).await.unwrap_err(); + assert!(super::error(error).to_string().contains("unsupported")); + } + let mut request = HttpRequest::get("http://127.0.0.1:0/").unwrap(); + *request.body_mut() = SdkBody::taken(); + assert!( + super::error(transport.call(request).await.unwrap_err()) + .to_string() + .contains("buffered") + ); + } +} diff --git a/src/s3/sdk_tests.rs b/src/s3/sdk_tests.rs new file mode 100644 index 0000000..b6dd61f --- /dev/null +++ b/src/s3/sdk_tests.rs @@ -0,0 +1,495 @@ +use super::tests::{ + completed, initiated, mock, mock_with_headers, part_response, streaming_remote, +}; +use super::*; + +#[tokio::test] +async fn sdk_preserves_literal_keys_and_empty_object_bodies() { + let source = tempfile::NamedTempFile::new().unwrap(); + for (key, wire) in [ + ( + "a//雪 +%2F?#/end/", + "/files/a//%E9%9B%AA%20%2B%252F%3F%23/end/", + ), + ("/leading/", "/files//leading/"), + (" ", "/files/%20"), + ("\t\n", "/files/%09%0A"), + ] { + let (gateway, requests, task) = mock(vec![ + (200, vec![], String::new()), + (204, vec![], String::new()), + ]) + .await; + let path = format!("s3://files/{key}"); + gateway + .copy(source.path().to_str().unwrap(), &path, false, false) + .await + .unwrap(); + gateway + .delete(&Remote::parse(&path).unwrap().unwrap()) + .await + .unwrap(); + let requests = requests.lock().unwrap(); + assert_eq!(requests[0], ("PUT".into(), wire.into(), Vec::new())); + assert_eq!(requests[1].1, wire); + task.abort(); + } +} + +#[tokio::test] +async fn sdk_serializes_generic_metadata_and_completion_xml() { + let headers: Arc>> = Arc::default(); + let marker = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(marker.path(), b"commit\n").unwrap(); + let (gateway, requests, task) = mock_with_headers( + vec![initiated(), part_response(1), completed()], + headers.clone(), + ) + .await; + gateway + .upload_stream( + &b""[..], + &streaming_remote(), + marker.path(), + &["Project=example".into(), "x-amz-meta-source=cli".into()], + std::future::pending(), + ) + .await + .unwrap(); + let headers = headers.lock().unwrap(); + assert_eq!(headers[0]["x-amz-meta-project"], "example"); + assert_eq!(headers[0]["x-amz-meta-x-amz-meta-source"], "cli"); + assert!(!headers[0].contains_key("x-amz-meta-source")); + assert_eq!(headers[0]["content-type"], "application/octet-stream"); + assert_eq!(headers[2]["content-type"], "application/xml"); + let requests = requests.lock().unwrap(); + assert!(requests[1].2.is_empty()); + assert_eq!( + requests[1].1, + "/files/object.bin?partNumber=1&uploadId=opaque%2B%2F%3D%26%252F" + ); + task.abort(); +} + +#[tokio::test] +async fn sdk_completion_wire_supports_10000_ordered_escaped_parts() { + use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; + let (gateway, requests, task) = mock(vec![completed()]).await; + gateway + .sdk + .complete_multipart_upload() + .bucket("files") + .key("object.bin") + .upload_id("opaque+/=&%2F") + .multipart_upload( + CompletedMultipartUpload::builder() + .set_parts(Some( + (1..=10000) + .map(|number| { + CompletedPart::builder() + .part_number(number) + .e_tag("\"opaque&etag\"") + .build() + }) + .collect(), + )) + .build(), + ) + .send() + .await + .unwrap(); + let requests = requests.lock().unwrap(); + let xml = std::str::from_utf8(&requests[0].2).unwrap(); + assert!(xml.len() < multipart::MAX_COMPLETION); + assert!(xml.contains("opaque&etag")); + assert_eq!(xml.matches("").count(), 10000); + assert!( + xml.find("1").unwrap() + < xml.find("10000").unwrap() + ); + task.abort(); +} + +#[tokio::test] +async fn sdk_rejects_http200_put_error_without_deleting_move_source() { + let source = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(source.path(), b"payload").unwrap(); + let (gateway, requests, task) = mock(vec![( + 200, + vec![], + "InternalError".into(), + )]) + .await; + assert!( + gateway + .copy( + source.path().to_str().unwrap(), + "s3://files/object", + true, + false + ) + .await + .is_err() + ); + assert!(source.path().exists()); + assert_eq!(requests.lock().unwrap().len(), 1); + task.abort(); +} + +#[tokio::test] +async fn sdk_errors_never_include_response_secrets_or_redirect_locations() { + let secret = "https://provider.invalid/object?signature=not-for-errors"; + for (status, headers, body) in [ + (307, vec![("location", secret.into())], String::new()), + ( + 403, + vec![], + format!("{secret}test-key"), + ), + ( + 200, + vec![], + format!( + "{secret}" + ), + ), + ] { + let (gateway, requests, task) = mock(vec![(status, headers, body)]).await; + let error = gateway.list(None).await.unwrap_err(); + let error = format!("{error:?}"); + assert!(!error.contains(secret)); + assert!(!error.contains("test-key")); + assert_eq!(requests.lock().unwrap().len(), 1); + task.abort(); + } + let (gateway, _, task) = mock(vec![( + 403, + vec![], + format!("{secret}"), + )]) + .await; + let error = gateway.download(&streaming_remote()).await.unwrap_err(); + assert!(!format!("{error:?}").contains(secret)); + task.abort(); +} + +#[tokio::test] +async fn sdk_rejects_invalid_listing_fields() { + for xml in [ + "1", + "a", + "a-1", + "unknown", + ] { + let (gateway, _, task) = mock(vec![(200, vec![], xml.into())]).await; + assert!(gateway.list(Some("s3://files/")).await.is_err(), "{xml}"); + task.abort(); + } +} + +#[tokio::test] +async fn sdk_bounds_multipart_bodies_and_aborts_failed_completion() { + let marker = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(marker.path(), b"commit\n").unwrap(); + let (gateway, requests, task) = mock(vec![ + initiated(), + part_response(1), + (200, vec![], " ".repeat(multipart::MAX_COMPLETION + 1)), + (204, vec![], String::new()), + ]) + .await; + let error = gateway + .upload_stream( + &b"payload"[..], + &streaming_remote(), + marker.path(), + &[], + std::future::pending(), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("uncertain")); + assert_eq!(requests.lock().unwrap().last().unwrap().0, "DELETE"); + task.abort(); +} + +use std::sync::Arc; + +#[tokio::test] +async fn sdk_multipart_preserves_whitespace_keys_on_success_and_abort() { + let marker = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(marker.path(), b"commit\n").unwrap(); + for success in [true, false] { + let responses = vec![ + (200, vec![], "files opaque".into()), + if success { part_response(1) } else { (500, vec![], String::new()) }, + if success { (200, vec![], "files \"done\"".into()) } else { (204, vec![], String::new()) }, + ]; + let (gateway, requests, task) = mock(responses).await; + let result = gateway + .upload_stream( + &b"payload"[..], + &Remote::parse("s3://files/ ").unwrap().unwrap(), + marker.path(), + &[], + std::future::pending(), + ) + .await; + assert_eq!(result.is_ok(), success, "{result:?}"); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert!( + requests + .iter() + .all(|request| request.1.starts_with("/files/%20?")) + ); + assert_eq!(requests[2].0, if success { "POST" } else { "DELETE" }); + task.abort(); + } +} + +#[tokio::test] +async fn sdk_completion_uses_sdk_response_parsing() { + for (xml, accepted) in [ + ( + "InternalError".to_owned(), + false, + ), + (completed().2, true), + ] { + let (gateway, _, task) = mock(vec![(200, vec![], xml)]).await; + let result = gateway + .sdk + .complete_multipart_upload() + .bucket("files") + .key("object.bin") + .upload_id("opaque") + .send() + .await; + assert_eq!(result.is_ok(), accepted, "{result:?}"); + task.abort(); + } +} + +#[tokio::test] +async fn sdk_uses_pod_hostname_and_path_style_logical_buckets() { + let headers: Arc>> = Arc::default(); + let (mock_gateway, requests, task) = mock_with_headers(vec![ + (200, vec![], "files".into()), + (200, vec![], "".into()), + (204, vec![], String::new()), + ], headers.clone()).await; + let address: std::net::SocketAddr = mock_gateway.origin.socket_addrs(|| None).unwrap()[0]; + let mut gateway = Gateway::new( + &format!("http://s3.example.test:{}", address.port()), + "my-pod", + "test-key", + ) + .unwrap(); + let http = crate::http_client_builder() + .unwrap() + .no_proxy() + .resolve("my-pod.s3.example.test", address) + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(); + gateway.sdk = sdk::client(http, &gateway.origin, gateway.token).unwrap(); + gateway.list(None).await.unwrap(); + gateway + .list(Some("s3://UPPER.Files/a// +%2F/")) + .await + .unwrap(); + gateway + .delete(&Remote::parse("s3://a/object").unwrap().unwrap()) + .await + .unwrap(); + let headers = headers.lock().unwrap(); + assert!( + headers + .iter() + .all(|headers| headers["host"] == format!("my-pod.s3.example.test:{}", address.port())) + ); + let requests = requests.lock().unwrap(); + assert_eq!(requests[0].1, "/"); + let url = Url::parse(&format!("http://localhost{}", requests[1].1)).unwrap(); + assert_eq!(url.path(), "/UPPER.Files/"); + let pairs: std::collections::HashMap<_, _> = url.query_pairs().collect(); + assert_eq!(pairs["list-type"], "2"); + assert_eq!(pairs["encoding-type"], "url"); + assert_eq!(pairs["max-keys"], "1000"); + assert_eq!(pairs["delimiter"], "/"); + assert_eq!(pairs["prefix"], "a// +%2F/"); + assert_eq!(requests[2].1, "/a/object"); + task.abort(); +} + +#[tokio::test] +async fn sdk_rejects_oversized_completion_manifests_before_sending() { + use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; + let (gateway, requests, task) = mock(vec![]).await; + let result = gateway + .sdk + .complete_multipart_upload() + .bucket("files") + .key("object.bin") + .upload_id("opaque") + .multipart_upload( + CompletedMultipartUpload::builder() + .set_parts(Some( + (1..=10000) + .map(|number| { + CompletedPart::builder() + .part_number(number) + .e_tag(format!("\"{}\"", "&".repeat(254))) + .build() + }) + .collect(), + )) + .build(), + ) + .send() + .await; + assert!( + sdk::error(result.unwrap_err()) + .to_string() + .contains("limit") + ); + assert!(requests.lock().unwrap().is_empty()); + task.abort(); +} + +#[tokio::test] +async fn sdk_writes_match_strict_gateway_header_contract() { + let headers: Arc>> = Arc::default(); + let (gateway, requests, task) = mock_with_headers( + vec![ + (200, vec![], String::new()), + (204, vec![], String::new()), + initiated(), + part_response(1), + completed(), + initiated(), + part_response(1), + (204, vec![], String::new()), + ], + headers.clone(), + ) + .await; + let source = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(source.path(), b"payload").unwrap(); + gateway + .copy( + source.path().to_str().unwrap(), + "s3://files/object.bin", + false, + false, + ) + .await + .unwrap(); + gateway.delete(&streaming_remote()).await.unwrap(); + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("success"); + std::fs::write(&marker, b"commit\n").unwrap(); + gateway + .upload_stream( + &b"payload"[..], + &streaming_remote(), + &marker, + &["project=example".into()], + std::future::pending(), + ) + .await + .unwrap(); + std::fs::remove_file(&marker).unwrap(); + assert!( + gateway + .upload_stream( + &b"payload"[..], + &streaming_remote(), + &marker, + &["project=example".into()], + std::future::pending() + ) + .await + .is_err() + ); + let requests = requests.lock().unwrap(); + assert_eq!( + requests + .iter() + .map(|request| request.0.as_str()) + .collect::>(), + [ + "PUT", "DELETE", "POST", "PUT", "POST", "POST", "PUT", "DELETE" + ] + ); + let headers = headers.lock().unwrap(); + assert_eq!(headers.len(), 8); + for (index, headers) in headers.iter().enumerate() { + assert!(!headers.contains_key("x-amz-user-agent")); + let metadata = headers + .keys() + .filter(|name| name.as_str().starts_with("x-amz-")) + .map(|name| name.as_str()) + .collect::>(); + assert_eq!( + metadata, + if [2, 5].contains(&index) { + vec!["x-amz-meta-project"] + } else { + Vec::new() + } + ); + if [2, 5].contains(&index) { + assert_eq!(headers["x-amz-meta-project"], "example"); + } + } + task.abort(); +} + +#[tokio::test] +async fn sdk_put_delete_and_gateway_get_encode_literal_keys_identically() { + let source = tempfile::NamedTempFile::new().unwrap(); + for (key, wire) in [ + ("café +%.bin", "/files/caf%C3%A9%20%2B%25.bin"), + ( + "a//雪 +%2F?#/end/", + "/files/a//%E9%9B%AA%20%2B%252F%3F%23/end/", + ), + ( + "reserved/:?#[]@!$&'()*+,;=%", + "/files/reserved/%3A%3F%23%5B%5D%40%21%24%26%27%28%29%2A%2B%2C%3B%3D%25", + ), + ("raw/%2F%252F/%2E", "/files/raw/%252F%25252F/%252E"), + ("unreserved/-._~AZaz09", "/files/unreserved/-._~AZaz09"), + ("/leading//trailing/", "/files//leading//trailing/"), + (" \t\n", "/files/%20%09%0A"), + ] { + let (gateway, requests, task) = mock(vec![ + (200, vec![], String::new()), + (404, vec![], String::new()), + (204, vec![], String::new()), + ]) + .await; + let path = format!("s3://files/{key}"); + let remote = Remote::parse(&path).unwrap().unwrap(); + gateway + .copy(source.path().to_str().unwrap(), &path, false, false) + .await + .unwrap(); + assert!(gateway.download(&remote).await.is_err()); + gateway.delete(&remote).await.unwrap(); + let requests = requests.lock().unwrap(); + assert_eq!( + requests + .iter() + .map(|request| request.0.as_str()) + .collect::>(), + ["PUT", "GET", "DELETE"] + ); + for request in requests.iter() { + assert_eq!(request.1, wire); + } + task.abort(); + } +} diff --git a/src/s3/tests.rs b/src/s3/tests.rs index c176f11..093769a 100644 --- a/src/s3/tests.rs +++ b/src/s3/tests.rs @@ -9,8 +9,17 @@ type MockResponse = (u16, Vec<(&'static str, String)>, String); type Requests = Arc)>>>; -async fn mock( +pub(super) async fn mock( responses: Vec, +) -> (Gateway<'static>, Requests, tokio::task::JoinHandle<()>) { + mock_with_headers(responses, Arc::default()).await +} + +type RequestHeaders = Arc>>; + +pub(super) async fn mock_with_headers( + responses: Vec, + headers: RequestHeaders, ) -> (Gateway<'static>, Requests, tokio::task::JoinHandle<()>) { let requests: Requests = Arc::default(); let responses = Arc::new(Mutex::new(std::collections::VecDeque::from(responses))); @@ -19,9 +28,60 @@ async fn mock( let responses = responses.clone(); move |State(requests): State, request: Request| { let responses = responses.clone(); + let headers = headers.clone(); async move { assert_eq!(request.headers()["authorization"], "Bearer test-key"); + for (name, value) in request.headers() { + assert!(!name.as_str().starts_with("x-amz-checksum-")); + assert!( + ![ + "x-amz-sdk-checksum-algorithm", + "x-amz-trailer", + "transfer-encoding", + "x-amz-content-sha256", + "x-amz-security-token", + "x-amz-date" + ] + .contains(&name.as_str()) + ); + assert!( + !value + .as_bytes() + .windows(11) + .any(|part| part == b"aws-chunked") + ); + } + assert!( + !request + .uri() + .query() + .unwrap_or_default() + .split('&') + .any(|pair| pair.starts_with("x-id=")) + ); + headers.lock().unwrap().push(request.headers().clone()); let method = request.method().to_string(); + let initiation = method == "POST" + && request + .uri() + .query() + .is_some_and(|query| query.split('&').any(|pair| pair == "uploads")); + if method != "GET" + && request.headers().keys().any(|name| { + name.as_str().starts_with("x-amz-") + && !["x-amz-date", "x-amz-content-sha256"].contains(&name.as_str()) + && !(initiation && name.as_str().starts_with("x-amz-meta-")) + }) + { + let mut names = request + .headers() + .keys() + .map(|name| name.as_str()) + .collect::>(); + names.sort_unstable(); + eprintln!("strict gateway rejected request header names: {names:?}"); + return Response::builder().status(501).body(Body::empty()).unwrap(); + } let uri = request.uri().to_string(); let length = request .headers() @@ -55,6 +115,17 @@ async fn mock( axum::serve(listener, router).await.unwrap(); }); let gateway = Gateway { + sdk: sdk::client( + crate::http_client_builder() + .unwrap() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(), + &origin, + "test-key", + ) + .unwrap(), origin, token: "test-key", http: crate::http_client_builder() @@ -106,7 +177,7 @@ fn preserves_literal_keys_and_rejects_normalization() { .unwrap(); let url = gateway.url(Some(&remote)).unwrap(); assert_eq!(url.host_str(), Some("my-pod.s3.example.test")); - assert_eq!(url.path(), "/files/a//%E9%9B%AA%20+%252F%3F%23/end/"); + assert_eq!(url.path(), "/files/a//%E9%9B%AA%20%2B%252F%3F%23/end/"); assert!(url.query().is_none()); for path in ["s3://files/a/../b", "s3://files/./a", "s3://files/"] { assert!(Remote::parse(path).unwrap().unwrap().object().is_err()); @@ -211,7 +282,11 @@ async fn upload_and_move_only_remove_local_source_after_success() { let (gateway, requests, task) = mock(vec![( status, vec![], - "AccessDenied".into(), + if status == 200 { + String::new() + } else { + "AccessDenied".into() + }, )]) .await; let result = gateway @@ -495,11 +570,11 @@ fn configuration_and_machine_description_include_s3() { assert_eq!(description["command"]["effect"], "read"); } -fn initiated() -> MockResponse { +pub(super) fn initiated() -> MockResponse { (200, vec![], "filesobject.binopaque+/=&%2F".into()) } -fn part_response(number: usize) -> MockResponse { +pub(super) fn part_response(number: usize) -> MockResponse { ( 200, vec![("etag", format!("\"part-{number}\""))], @@ -507,11 +582,11 @@ fn part_response(number: usize) -> MockResponse { ) } -fn completed() -> MockResponse { +pub(super) fn completed() -> MockResponse { (200, vec![], "filesobject.bin\"complete\"".into()) } -fn streaming_remote() -> Remote { +pub(super) fn streaming_remote() -> Remote { Remote::parse("s3://files/object.bin").unwrap().unwrap() } @@ -701,8 +776,7 @@ async fn multipart_aborts_failed_parts_and_error_or_malformed_completions() { "InternalError", "filesobject.bin", "otherobject.bin\"x\"", - "filesobject.bin\"x\"", - "filesobject.bin\"x\"", + "not XML", ] { let (gateway, requests, task) = mock(vec![ initiated(),