feat(turnloop-http): https:// proxies, with TLS to the proxy nesting the tunnel

protocols/turnloop-http/src/asynchronous/client.rs

@@ -49,11 +49,14 @@ impl Default for Options {
         }
     }
 }
-enum Transport<B: Backend> {
-    Plain(AsyncIo<B>),
-    Tls(Box<TlsStream<AsyncIo<B>>>),
+/// One plaintext-or-TLS layer. A connection through an `https://` proxy stacks
+/// two: TLS to the proxy, then the target's own session inside the tunnel.
+enum Layer<S> {
+    Plain(S),
+    Tls(Box<TlsStream<S>>),
 }
-impl<B: Backend> AsyncRead for Transport<B> {
+type Transport<B> = Layer<Layer<AsyncIo<B>>>;
+impl<S: AsyncRead + AsyncWrite + Unpin> AsyncRead for Layer<S> {
     fn poll_read(
         self: Pin<&mut Self>,
         cx: &mut Context<'_>,
@@ -65,7 +68,7 @@ impl<B: Backend> AsyncRead for Transport<B> {
         }
     }
 }
-impl<B: Backend> AsyncWrite for Transport<B> {
+impl<S: AsyncRead + AsyncWrite + Unpin> AsyncWrite for Layer<S> {
     fn poll_write(
         self: Pin<&mut Self>,
         cx: &mut Context<'_>,
@@ -206,7 +209,7 @@ impl<B: Backend> Client<B> {
         let proxy = self
             .options
             .proxy
-            .proxy_for(&request.url)
+            .proxy_for_tls_capable(&request.url)
             .map_err(io::Error::other)?;
         let key = PoolKey::new(&request.url, proxy.as_ref());
         let mut route = Route::new(request.url.clone(), proxy);
@@ -239,6 +242,20 @@ impl<B: Backend> Client<B> {
         turnloop_io::deadline(&executor, at, async {
             if lease.connection.is_none() {
                 let stream = connect(&executor, &route).await?;
+                let stream = match route.proxy_tls() {
+                    Some(TransportRequest::UpgradeTls { server_name }) => {
+                        let name =
+                            turnloop_tls::rustls::pki_types::ServerName::try_from(server_name)
+                                .map_err(io::Error::other)?;
+                        // CONNECT and absolute-form requests are HTTP/1.1.
+                        let proxy_tls = tls.with_alpn(vec![b"http/1.1".to_vec()]);
+                        Layer::Tls(Box::new(
+                            TlsStream::connect(stream, &proxy_tls, name, &executor, at, time)
+                                .await?,
+                        ))
+                    }
+                    _ => Layer::Plain(stream),
+                };
                 let stream = if let Some(head) = route.connect_head(None) {
                     let mut tunnel = Http1::new(stream, Mode::Response);
                     tunnel.response_to("CONNECT");
@@ -277,10 +294,10 @@ impl<B: Backend> Client<B> {
                     let stream = TlsStream::connect(stream, tls, name, &executor, at, time).await?;
                     let protocol =
                         client::from_alpn(stream.alpn_protocol()).map_err(io::Error::other)?;
-                    (Transport::Tls(Box::new(stream)), protocol)
+                    (Layer::Tls(Box::new(stream)), protocol)
                 } else {
                     (
-                        Transport::Plain(stream),
+                        Layer::Plain(stream),
                         if options.http2_prior_knowledge {
                             Protocol::Http2
                         } else {

protocols/turnloop-http/src/client.rs

@@ -201,7 +201,17 @@ pub struct ProxyEnvironment {
 }
 impl ProxyEnvironment {
     /// Values are supplied by the host from HTTP_PROXY/HTTPS_PROXY/NO_PROXY (or lowercase overrides).
+    /// Only `http://` proxies are accepted; see [`Self::proxy_for_tls_capable`].
     pub fn proxy_for(&self, url: &Url) -> Result<Option<Url>> {
+        self.select(url, false)
+    }
+    /// [`Self::proxy_for`], also accepting `https://` proxies. The caller must
+    /// then secure the hop to the proxy itself: after connecting, perform the
+    /// [`Route::proxy_tls`] upgrade before anything else is written.
+    pub fn proxy_for_tls_capable(&self, url: &Url) -> Result<Option<Url>> {
+        self.select(url, true)
+    }
+    fn select(&self, url: &Url, https: bool) -> Result<Option<Url>> {
         if bypass(url, &self.no_proxy) {
             return Ok(None);
         }
@@ -215,10 +225,15 @@ impl ProxyEnvironment {
             .map(|p| {
                 let parsed =
                     Url::parse(p).map_err(|_| Error::new("ERR_INVALID_URL", "invalid proxy"))?;
-                if parsed.scheme() != "http" || parsed.host_str().is_none() {
+                let scheme_ok = parsed.scheme() == "http" || (https && parsed.scheme() == "https");
+                if !scheme_ok || parsed.host_str().is_none() {
                     return Err(Error::new(
                         "UND_ERR_NOT_SUPPORTED",
-                        "only HTTP proxies are supported",
+                        if https {
+                            "only HTTP and HTTPS proxies are supported"
+                        } else {
+                            "only HTTP proxies are supported"
+                        },
                     ));
                 }
                 Ok(parsed)
@@ -659,6 +674,16 @@ impl Route {
             port: peer.port_or_known_default().unwrap_or(80),
         }
     }
+    /// For an `https://` proxy, the TLS upgrade to the proxy itself. It comes
+    /// first, right after the connection is established: the CONNECT tunnel or
+    /// the absolute-form request then runs inside it, and an `https` target's
+    /// own TLS session nests within that tunnel.
+    pub fn proxy_tls(&self) -> Option<TransportRequest> {
+        let proxy = self.proxy.as_ref().filter(|p| p.scheme() == "https")?;
+        Some(TransportRequest::UpgradeTls {
+            server_name: proxy.host_str().unwrap_or("").into(),
+        })
+    }
     pub fn connect_head(&self, authorization: Option<&[u8]>) -> Option<Head> {
         if self.proxy.is_none() || self.target.scheme() != "https" {
             return None;

protocols/turnloop-http/tests/asynchronous.rs

@@ -926,6 +926,128 @@ origin.listen(0, '127.0.0.1', () => proxy.listen(0, '127.0.0.1', () => {
     assert!(node.0.wait().expect("Node exit").success());
 }
 
+#[cfg(not(target_arch = "wasm32"))]
+#[test]
+fn node_https_via_authenticated_https_proxy_nests_tls() {
+    use std::{
+        io::{BufRead, BufReader},
+        process::{Command, Stdio},
+    };
+    use turnloop_http::client::ProxyEnvironment;
+    let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).expect("certificate");
+    let script = r#"
+const assert = require('node:assert/strict');
+const https = require('node:https'), http = require('node:http'), net = require('node:net');
+let connects = 0, requests = 0;
+const watchdog = setTimeout(() => process.exit(7), 10000);
+const origin = https.createServer({key: process.env.KEY, cert: process.env.CERT}, (req, res) => {
+  assert.equal(req.method, 'POST'); assert.equal(req.url, '/through-proxy');
+  let body = ''; req.on('data', b => body += b); req.on('end', () => {
+    assert.equal(body, 'verified TLS upload'); requests++;
+    res.setHeader('connection', 'close'); res.end('verified TLS response');
+  });
+});
+// The proxy prefers h2: only the client's http/1.1-only offer keeps CONNECT on HTTP/1.1.
+const proxy = https.createServer({key: process.env.KEY, cert: process.env.CERT, ALPNProtocols: ['h2', 'http/1.1']});
+let proxyTls = 0;
+proxy.on('secureConnection', s => { proxyTls++; assert.equal(s.alpnProtocol, 'http/1.1'); });
+proxy.on('connect', (req, client, head) => {
+  assert.equal(req.url, 'localhost:' + origin.address().port);
+  assert.equal(req.headers['proxy-authorization'], 'Basic dXNlcjpwYXNz'); assert.equal(req.socket.encrypted, true); connects++;
+  const upstream = net.connect(origin.address().port, '127.0.0.1', () => {
+    client.write('HTTP/1.1 200 Connection Established\r\n\r\n');
+    if (head.length) upstream.write(head);
+    client.pipe(upstream); upstream.pipe(client);
+  });
+  client.on('error', () => upstream.destroy()); upstream.on('error', () => client.destroy());
+  client.on('close', () => {
+    upstream.destroy(); assert.equal(connects, 1); assert.equal(requests, 1); assert.equal(proxyTls, 1);
+    proxy.close(); origin.close(); clearTimeout(watchdog);
+  });
+});
+origin.listen(0, '127.0.0.1', () => proxy.listen(0, '127.0.0.1', () => {
+  console.log(origin.address().port + ' ' + proxy.address().port);
+}));
+"#;
+    struct OwnedChild(std::process::Child);
+    impl Drop for OwnedChild {
+        fn drop(&mut self) {
+            let _ = self.0.kill();
+            let _ = self.0.wait();
+        }
+    }
+    let mut node = OwnedChild(
+        Command::new("node")
+            .args(["-e", script])
+            .env("KEY", cert.key_pair.serialize_pem())
+            .env("CERT", cert.cert.pem())
+            .stdout(Stdio::piped())
+            .spawn()
+            .expect("Node required"),
+    );
+    let mut ports = String::new();
+    BufReader::new(node.0.stdout.take().expect("stdout"))
+        .read_line(&mut ports)
+        .expect("listener ports");
+    let ports: Vec<u16> = ports
+        .split_whitespace()
+        .map(|p| p.parse().expect("port"))
+        .collect();
+    assert_eq!(ports.len(), 2);
+    let mut executor = LocalExecutor::<Platform>::new(Config::default()).expect("executor");
+    let h = executor.handle();
+    let mut task = executor
+        .spawn_local(async move {
+            let tls = turnloop_tls::ClientConfig::new(
+                turnloop_tls::ClientOptions {
+                    ca: Some(vec![cert.cert.der().clone()]),
+                    alpn: vec![b"h2".to_vec(), b"http/1.1".to_vec()],
+                    ..Default::default()
+                },
+                1_789_344_000,
+            )
+            .expect("trusted TLS config");
+            let mut client = Client::new(
+                h,
+                tls,
+                1_789_344_000,
+                Options {
+                    proxy: ProxyEnvironment {
+                        https_proxy: Some(format!("https://user:pass@localhost:{}", ports[1])),
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                },
+            );
+            let mut req = Request::new(
+                &format!("https://localhost:{}/through-proxy", ports[0]),
+                "POST",
+            )
+            .expect("request");
+            req.body = b"verified TLS upload".to_vec();
+            let mut body = Vec::new();
+            let head = client
+                .request(&mut req, |bytes| {
+                    body.extend_from_slice(bytes);
+                    Ok(())
+                })
+                .await
+                .expect("TLS to the proxy, CONNECT, then TLS to the origin");
+            assert_eq!(head.status, 200);
+            assert_eq!(body, b"verified TLS response");
+            body.len()
+        })
+        .expect("spawn");
+    let end = executor.driver().now() + Duration::from_secs(10);
+    while !task.is_finished() {
+        assert!(executor.driver().now() < end);
+        executor.turn(Timeout::Until(end)).expect("turn");
+    }
+    assert_eq!(finish(&mut task), 21);
+    executor.turn(Timeout::Now).expect("close delivery");
+    assert!(node.0.wait().expect("Node exit").success());
+}
+
 #[test]
 fn cancelled_pooled_request_never_reuses_partial_response() {
     use std::cell::Cell;

protocols/turnloop-http/tests/codecs.rs

@@ -766,6 +766,84 @@ fn proxy_credentials_and_cross_origin_mixed_case_headers() {
     );
 }
 
+#[test]
+fn https_proxy_is_opt_in_and_its_tls_comes_first() {
+    let env = ProxyEnvironment {
+        http_proxy: Some("https://proxy.test".into()),
+        https_proxy: Some("https://us%65r:p%61ss@proxy.test:8443".into()),
+        no_proxy: String::new(),
+    };
+    let secure = url::Url::parse("https://origin.test/x").unwrap();
+    let plain = url::Url::parse("http://origin.test/x").unwrap();
+    // The original entry point still refuses a proxy it cannot be told to secure.
+    assert_eq!(
+        env.proxy_for(&secure).unwrap_err().code,
+        "UND_ERR_NOT_SUPPORTED"
+    );
+    let proxy = env.proxy_for_tls_capable(&secure).unwrap().unwrap();
+    assert_eq!(proxy.scheme(), "https");
+    let ftp = ProxyEnvironment {
+        https_proxy: Some("socks5://proxy.test".into()),
+        ..Default::default()
+    };
+    assert!(ftp.proxy_for_tls_capable(&secure).is_err());
+
+    // https target: TLS to the proxy, CONNECT inside it, then the target's TLS.
+    let mut route = Route::new(secure.clone(), Some(proxy));
+    assert_eq!(
+        route.resolve(),
+        TransportRequest::Resolve {
+            hostname: "proxy.test".into(),
+            port: 8443
+        }
+    );
+    assert_eq!(
+        route.proxy_tls(),
+        Some(TransportRequest::UpgradeTls {
+            server_name: "proxy.test".into()
+        })
+    );
+    let connect = route.connect_head(None).unwrap();
+    assert_eq!(connect.target, "origin.test:443");
+    assert_eq!(
+        connect.get("proxy-authorization"),
+        Some(b"Basic dXNlcjpwYXNz".as_slice())
+    );
+    assert_eq!(route.connected(), None);
+    assert_eq!(
+        route.tunnel_response(200).unwrap(),
+        TransportRequest::UpgradeTls {
+            server_name: "origin.test".into()
+        }
+    );
+
+    // http target: TLS to the proxy, then an absolute-form request, no CONNECT.
+    let proxy = env.proxy_for_tls_capable(&plain).unwrap().unwrap();
+    let route = Route::new(plain.clone(), Some(proxy));
+    assert_eq!(
+        route.resolve(),
+        TransportRequest::Resolve {
+            hostname: "proxy.test".into(),
+            port: 443
+        }
+    );
+    assert!(route.proxy_tls().is_some());
+    assert!(route.connect_head(None).is_none());
+    let request = Request::new(plain.as_str(), "GET").unwrap();
+    assert_eq!(
+        route.request_head(&request, None).target,
+        "http://origin.test/x"
+    );
+
+    // An http proxy, or none, needs no proxy TLS.
+    let route = Route::new(
+        secure.clone(),
+        Some(url::Url::parse("http://p.test").unwrap()),
+    );
+    assert_eq!(route.proxy_tls(), None);
+    assert_eq!(Route::new(secure, None).proxy_tls(), None);
+}
+
 // --- HTTP/2 stream-lifetime contract -----------------------------------------
 //
 // h2spec's strict suite drives the subject from the peer side only: every

protocols/turnloop-tls/src/lib.rs

@@ -176,6 +176,14 @@ impl ClientConfig {
     pub fn rustls_config(&self) -> &Arc<rustls::ClientConfig> {
         &self.config
     }
+    /// The same roots, verifier and host time, offering `alpn` instead. A hop
+    /// that only speaks one protocol (an HTTP/1.1 proxy) must not let the peer
+    /// pick another; the copy has its own session cache.
+    pub fn with_alpn(&self, alpn: Vec<Vec<u8>>) -> Self {
+        let mut config = (*self.config).clone();
+        config.alpn_protocols = alpn;
+        Self::from_rustls(Arc::new(config), self.time.clone())
+    }
     /// Reuse this configuration across connections to share the session cache.
     pub fn connect(&self, name: ServerName<'static>) -> Result<Client, Error> {
         Ok(Client {