From 00bace0da725c6036982e51a626369d1b140b129 Mon Sep 17 00:00:00 2001 From: sanil-23 Date: Wed, 7 Oct 2026 23:32:32 +0530 Subject: [PATCH 1/9] Send a User-Agent from web_fetch and http_request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reqwest sends no `User-Agent` unless told to, and some APIs refuse a request without one. GitHub's REST API is the clearest case: 403 Request forbidden by administrative rules. Please make sure your request has a User-Agent header Reproducible on demand — the same URL in the same second answers 403 with no header and 200 with one. Every `api.github.com` call through these tools therefore failed, deterministically, and the 403 was then read downstream as a credentials problem, which sends whoever is debugging it after the wrong thing entirely. Adds `gate::USER_AGENT` (`tinytools/`) and sets it on both clients. Identifying rather than disguised: a server that wants to throttle or block this traffic should be able to name it, and a version makes builds distinguishable. The test asserts the header left the process, read back from `serve`'s raw request log, rather than asserting on the builder. Co-Authored-By: Claude Opus 5 --- crates/tinytools-std/src/network/gate.rs | 20 +++++++++++ .../tinytools-std/src/network/http_request.rs | 1 + .../src/network/http_request_tests.rs | 35 +++++++++++++++++++ crates/tinytools-std/src/network/web_fetch.rs | 3 +- 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-std/src/network/gate.rs b/crates/tinytools-std/src/network/gate.rs index 49d38d3..65339a7 100644 --- a/crates/tinytools-std/src/network/gate.rs +++ b/crates/tinytools-std/src/network/gate.rs @@ -12,6 +12,26 @@ //! already holding, so a host maps its own policy onto it without translating //! a vocabulary. +/// What these tools call themselves on the wire. +/// +/// reqwest sends no `User-Agent` unless told to, and a missing one is not a +/// cosmetic omission. GitHub's REST API refuses the request outright: +/// +/// ```text +/// 403 Request forbidden by administrative rules. +/// Please make sure your request has a User-Agent header +/// ``` +/// +/// Reproducible on demand — the same URL in the same second answers 403 with +/// no header and 200 with one — so every `api.github.com` call through these +/// tools failed, always, and the 403 was then read as a credentials problem. +/// Several other APIs require one too, and anonymous traffic is the first a +/// rate limiter penalises. +/// +/// Identifying rather than disguised: a server that wants to throttle or block +/// this traffic should be able to name it. +pub const USER_AGENT: &str = concat!("tinytools/", env!("CARGO_PKG_VERSION")); + /// The host policy a network tool consults before it acts. /// /// Implementations must be cheap to call: the tools ask on every invocation. diff --git a/crates/tinytools-std/src/network/http_request.rs b/crates/tinytools-std/src/network/http_request.rs index 39aac65..3307b6b 100644 --- a/crates/tinytools-std/src/network/http_request.rs +++ b/crates/tinytools-std/src/network/http_request.rs @@ -184,6 +184,7 @@ impl HttpRequestTool { let builder = reqwest::Client::builder() .timeout(Duration::from_secs(self.timeout_secs)) .connect_timeout(Duration::from_secs(10)) + .user_agent(super::gate::USER_AGENT) .redirect(reqwest::redirect::Policy::none()); let builder = self.gate.prepare_client("tool.http_request", builder); let client = builder.build()?; diff --git a/crates/tinytools-std/src/network/http_request_tests.rs b/crates/tinytools-std/src/network/http_request_tests.rs index 520cc33..0bd4ea7 100644 --- a/crates/tinytools-std/src/network/http_request_tests.rs +++ b/crates/tinytools-std/src/network/http_request_tests.rs @@ -544,3 +544,38 @@ fn the_test_gate_builds_a_client_with_the_requested_timeouts() { let gate = TestNetGate::supervised(); let _client = gate.timeout_client("svc", 5, 2); } + +/// Both network tools must name themselves on the wire. +/// +/// Not cosmetic: GitHub's REST API answers 403 to a request with no +/// `User-Agent`, so every `api.github.com` call through these tools failed +/// until this was set. `serve` records the raw request, which is the only way +/// to assert an outgoing header actually left the process. +#[tokio::test] +async fn an_outgoing_request_carries_a_user_agent() { + let (addr, seen) = serve(vec![ + "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".to_string(), + ]) + .await; + let tool = test_tool(vec![]); + let _ = tool + .execute_request( + &format!("http://{addr}/"), + reqwest::Method::GET, + vec![], + None, + ) + .await + .expect("the request reaches the server"); + + let request = seen.lock().expect("the log is readable")[0].clone(); + let lower = request.to_ascii_lowercase(); + assert!( + lower.contains("user-agent:"), + "no User-Agent was sent:\n{request}" + ); + assert!( + lower.contains("user-agent: tinytools/"), + "the header must identify this crate:\n{request}" + ); +} diff --git a/crates/tinytools-std/src/network/web_fetch.rs b/crates/tinytools-std/src/network/web_fetch.rs index 1775e0b..25b8572 100644 --- a/crates/tinytools-std/src/network/web_fetch.rs +++ b/crates/tinytools-std/src/network/web_fetch.rs @@ -11,7 +11,7 @@ //! budgets or retries on tool errors sees blocked and rate-limited pages for //! what they are. 3xx responses are not followed and stay successful reports. -use super::gate::{HttpLimits, NetGate, host_of}; +use super::gate::{HttpLimits, NetGate, USER_AGENT, host_of}; use crate::url_guard::{normalize_allowed_domains, validate_url_with_dns_check}; use async_trait::async_trait; use serde_json::json; @@ -237,6 +237,7 @@ impl WebFetchTool { // the caller so they can decide whether to refetch the new URL. let client = match reqwest::Client::builder() .timeout(Duration::from_secs(self.timeout_secs)) + .user_agent(USER_AGENT) .redirect(reqwest::redirect::Policy::none()) .build() { From eb7e9be66ebd7e77c74676e6e401acf90fa587c9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 14:15:44 +0300 Subject: [PATCH 2/9] test(network): assert outgoing requests carry a user agent Add a test that spins up a local listener, performs a validated fetch, and checks the captured request headers for a tinytools user agent. This guards against regressions where the header is dropped from outgoing requests. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/network/web_fetch_tests.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/crates/tinytools-std/src/network/web_fetch_tests.rs b/crates/tinytools-std/src/network/web_fetch_tests.rs index 239b5e9..6984e1d 100644 --- a/crates/tinytools-std/src/network/web_fetch_tests.rs +++ b/crates/tinytools-std/src/network/web_fetch_tests.rs @@ -253,6 +253,36 @@ async fn serve_once(response: &str) -> String { format!("http://{addr}/page") } +#[tokio::test] +async fn an_outgoing_request_carries_a_user_agent() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let seen = Arc::new(std::sync::Mutex::new(String::new())); + let request_log = Arc::clone(&seen); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buf = vec![0u8; 8192]; + let n = socket.read(&mut buf).await.unwrap(); + *request_log.lock().unwrap() = String::from_utf8_lossy(&buf[..n]).to_string(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .await + .unwrap(); + }); + + let tool = fetch(test_security(), vec![], None, None); + let result = tool + .fetch_validated(&format!("http://{addr}/"), 1_000_000, false) + .await + .unwrap(); + assert!(!result.is_error, "got: {}", result.output()); + server.await.unwrap(); + + let request = seen.lock().unwrap().to_ascii_lowercase(); + assert!(request.contains("user-agent: tinytools/"), "got: {request}"); +} + fn http_response(status_line: &str, headers: &str, body: &str) -> String { format!( "HTTP/1.1 {status_line}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", From f0e779f6f8f8ac77c41cb8f27a1f607fdd0099a8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 14:18:08 +0300 Subject: [PATCH 3/9] refactor(network): narrow user agent visibility and tidy test The USER_AGENT constant is now crate-visible rather than public, since it is only consumed within the network module. The user agent test was reworked to return a Result and propagate errors with ?, replacing the expect calls. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-std/src/network/gate.rs | 2 +- .../tinytools-std/src/network/http_request_tests.rs | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/tinytools-std/src/network/gate.rs b/crates/tinytools-std/src/network/gate.rs index 65339a7..c30bbc8 100644 --- a/crates/tinytools-std/src/network/gate.rs +++ b/crates/tinytools-std/src/network/gate.rs @@ -30,7 +30,7 @@ /// /// Identifying rather than disguised: a server that wants to throttle or block /// this traffic should be able to name it. -pub const USER_AGENT: &str = concat!("tinytools/", env!("CARGO_PKG_VERSION")); +pub(super) const USER_AGENT: &str = concat!("tinytools/", env!("CARGO_PKG_VERSION")); /// The host policy a network tool consults before it acts. /// diff --git a/crates/tinytools-std/src/network/http_request_tests.rs b/crates/tinytools-std/src/network/http_request_tests.rs index 0bd4ea7..8ee3f7e 100644 --- a/crates/tinytools-std/src/network/http_request_tests.rs +++ b/crates/tinytools-std/src/network/http_request_tests.rs @@ -552,7 +552,7 @@ fn the_test_gate_builds_a_client_with_the_requested_timeouts() { /// until this was set. `serve` records the raw request, which is the only way /// to assert an outgoing header actually left the process. #[tokio::test] -async fn an_outgoing_request_carries_a_user_agent() { +async fn an_outgoing_request_carries_a_user_agent() -> anyhow::Result<()> { let (addr, seen) = serve(vec![ "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".to_string(), ]) @@ -565,10 +565,12 @@ async fn an_outgoing_request_carries_a_user_agent() { vec![], None, ) - .await - .expect("the request reaches the server"); + .await?; - let request = seen.lock().expect("the log is readable")[0].clone(); + let request = seen + .lock() + .map_err(|error| anyhow::anyhow!("request log mutex poisoned: {error}"))?[0] + .clone(); let lower = request.to_ascii_lowercase(); assert!( lower.contains("user-agent:"), @@ -578,4 +580,5 @@ async fn an_outgoing_request_carries_a_user_agent() { lower.contains("user-agent: tinytools/"), "the header must identify this crate:\n{request}" ); + Ok(()) } From 55ce8e348ee65bfd5e50860baacd0ac48c6d63df Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 14:21:19 +0300 Subject: [PATCH 4/9] test(network): assert exact user agent header value The user agent test now extracts the header line and compares it against the exact expected value instead of a substring match, so a wrong or truncated version string is caught rather than passing on the prefix alone. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-std/src/network/web_fetch_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-std/src/network/web_fetch_tests.rs b/crates/tinytools-std/src/network/web_fetch_tests.rs index 6984e1d..5259bad 100644 --- a/crates/tinytools-std/src/network/web_fetch_tests.rs +++ b/crates/tinytools-std/src/network/web_fetch_tests.rs @@ -280,7 +280,10 @@ async fn an_outgoing_request_carries_a_user_agent() { server.await.unwrap(); let request = seen.lock().unwrap().to_ascii_lowercase(); - assert!(request.contains("user-agent: tinytools/"), "got: {request}"); + let user_agent = request + .lines() + .find_map(|line| line.strip_prefix("user-agent: ")); + assert_eq!(user_agent, Some(concat!("tinytools/", env!("CARGO_PKG_VERSION")))); } fn http_response(status_line: &str, headers: &str, body: &str) -> String { From 0ade83e5fab818d53b2af80edb19a1dd7bbf08a6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 14:21:24 +0300 Subject: [PATCH 5/9] test(web_fetch): reformat user agent assertion Reformat the user agent assertion in the outgoing request test to satisfy rustfmt line width limits. No behaviour change. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-std/src/network/web_fetch_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-std/src/network/web_fetch_tests.rs b/crates/tinytools-std/src/network/web_fetch_tests.rs index 5259bad..f0bc931 100644 --- a/crates/tinytools-std/src/network/web_fetch_tests.rs +++ b/crates/tinytools-std/src/network/web_fetch_tests.rs @@ -283,7 +283,10 @@ async fn an_outgoing_request_carries_a_user_agent() { let user_agent = request .lines() .find_map(|line| line.strip_prefix("user-agent: ")); - assert_eq!(user_agent, Some(concat!("tinytools/", env!("CARGO_PKG_VERSION")))); + assert_eq!( + user_agent, + Some(concat!("tinytools/", env!("CARGO_PKG_VERSION"))) + ); } fn http_response(status_line: &str, headers: &str, body: &str) -> String { From 61bda35247d9e9703e2440ad1fb75964e399c140 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 14:29:37 +0300 Subject: [PATCH 6/9] test(network): read full request headers in user agent test The test server now reads until the end of the header block instead of assuming a single read captures the whole request, so the assertion no longer fails when the request arrives split across packets. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-std/src/network/web_fetch_tests.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/tinytools-std/src/network/web_fetch_tests.rs b/crates/tinytools-std/src/network/web_fetch_tests.rs index f0bc931..54aa06f 100644 --- a/crates/tinytools-std/src/network/web_fetch_tests.rs +++ b/crates/tinytools-std/src/network/web_fetch_tests.rs @@ -262,9 +262,16 @@ async fn an_outgoing_request_carries_a_user_agent() { let request_log = Arc::clone(&seen); let server = tokio::spawn(async move { let (mut socket, _) = listener.accept().await.unwrap(); - let mut buf = vec![0u8; 8192]; - let n = socket.read(&mut buf).await.unwrap(); - *request_log.lock().unwrap() = String::from_utf8_lossy(&buf[..n]).to_string(); + let mut request = Vec::new(); + let mut buf = [0u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let n = socket.read(&mut buf).await.unwrap(); + if n == 0 { + break; + } + request.extend_from_slice(&buf[..n]); + } + *request_log.lock().unwrap() = String::from_utf8_lossy(&request).to_string(); socket .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") .await From 7e728b257f1103ffc4bc69311a70c5e417f648cd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 14:34:10 +0300 Subject: [PATCH 7/9] test(network): fail fast when peer closes before headers The user agent test previously broke out of the read loop when the peer closed early, which let the assertion run against a truncated request. It now panics with a clear message so the failure points at the incomplete read instead of a confusing mismatch. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-std/src/network/web_fetch_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinytools-std/src/network/web_fetch_tests.rs b/crates/tinytools-std/src/network/web_fetch_tests.rs index 54aa06f..5c7890f 100644 --- a/crates/tinytools-std/src/network/web_fetch_tests.rs +++ b/crates/tinytools-std/src/network/web_fetch_tests.rs @@ -267,7 +267,7 @@ async fn an_outgoing_request_carries_a_user_agent() { while !request.windows(4).any(|window| window == b"\r\n\r\n") { let n = socket.read(&mut buf).await.unwrap(); if n == 0 { - break; + panic!("peer closed before sending the complete HTTP headers"); } request.extend_from_slice(&buf[..n]); } From 2f9c968f547936db62b97cf5ee83d03556c9ed22 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 14:37:06 +0300 Subject: [PATCH 8/9] test(web_fetch): simplify early-close assertion Replace the explicit panic on a zero-length read with an assert! that carries the same message, keeping the test's intent while shortening the loop body. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-std/src/network/web_fetch_tests.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/tinytools-std/src/network/web_fetch_tests.rs b/crates/tinytools-std/src/network/web_fetch_tests.rs index 5c7890f..d5e64cb 100644 --- a/crates/tinytools-std/src/network/web_fetch_tests.rs +++ b/crates/tinytools-std/src/network/web_fetch_tests.rs @@ -266,9 +266,7 @@ async fn an_outgoing_request_carries_a_user_agent() { let mut buf = [0u8; 1024]; while !request.windows(4).any(|window| window == b"\r\n\r\n") { let n = socket.read(&mut buf).await.unwrap(); - if n == 0 { - panic!("peer closed before sending the complete HTTP headers"); - } + assert!(n != 0, "peer closed before sending the complete HTTP headers"); request.extend_from_slice(&buf[..n]); } *request_log.lock().unwrap() = String::from_utf8_lossy(&request).to_string(); From a25500ce912e0954124e76ed0c9ea9c1a2e14e96 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 14:37:12 +0300 Subject: [PATCH 9/9] test(network): reformat assertion in web fetch test Reformatted the assertion that checks the peer has not closed before sending complete HTTP headers so it matches the project's rustfmt style. No behaviour change. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-std/src/network/web_fetch_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-std/src/network/web_fetch_tests.rs b/crates/tinytools-std/src/network/web_fetch_tests.rs index d5e64cb..e6d36f8 100644 --- a/crates/tinytools-std/src/network/web_fetch_tests.rs +++ b/crates/tinytools-std/src/network/web_fetch_tests.rs @@ -266,7 +266,10 @@ async fn an_outgoing_request_carries_a_user_agent() { let mut buf = [0u8; 1024]; while !request.windows(4).any(|window| window == b"\r\n\r\n") { let n = socket.read(&mut buf).await.unwrap(); - assert!(n != 0, "peer closed before sending the complete HTTP headers"); + assert!( + n != 0, + "peer closed before sending the complete HTTP headers" + ); request.extend_from_slice(&buf[..n]); } *request_log.lock().unwrap() = String::from_utf8_lossy(&request).to_string();