Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,12 @@ Most browser device models share identical TLS and HTTP/2 configurations, differ
- <details>
<summary>Available browser emulations</summary>

Edge 149–158 are desktop profiles for Windows, macOS, and Linux.

| **Browser** | **Versions** |
| ----------- | ------------ |
| **Chrome** | `Chrome100`, `Chrome101`, `Chrome104`, `Chrome105`, `Chrome106`, `Chrome107`, `Chrome108`, `Chrome109`, `Chrome110`, `Chrome114`, `Chrome116`, `Chrome117`, `Chrome118`, `Chrome119`, `Chrome120`, `Chrome123`, `Chrome124`, `Chrome126`, `Chrome127`, `Chrome128`, `Chrome129`, `Chrome130`, `Chrome131`, `Chrome132`, `Chrome133`, `Chrome134`, `Chrome135`, `Chrome136`, `Chrome137`, `Chrome138`, `Chrome139`, `Chrome140`, `Chrome141`, `Chrome142`, `Chrome143`, `Chrome144`, `Chrome145`, `Chrome146`, `Chrome147`, `Chrome148`, `Chrome149`, `Chrome150`, `Chrome151`, `Chrome152`, `Chrome153`, `Chrome154`, `Chrome155`, `Chrome156`, `Chrome157`, `Chrome158` |
| **Edge** | `Edge101`, `Edge122`, `Edge127`, `Edge131`, `Edge134`, `Edge135`, `Edge136`, `Edge137`, `Edge138`, `Edge139`, `Edge140`, `Edge141`, `Edge142`, `Edge143`, `Edge144`, `Edge145`, `Edge146`, `Edge147`, `Edge148` |
| **Edge** | `Edge101`, `Edge122`, `Edge127`, `Edge131`, `Edge134`–`Edge158` |
| **Firefox** | `Firefox109`, `Firefox117`, `Firefox128`, `Firefox133`, `Firefox135`, `FirefoxPrivate135`, `FirefoxAndroid135`, `Firefox136`, `FirefoxPrivate136`, `Firefox139`, `Firefox142`, `Firefox143`, `Firefox144`, `Firefox145`, `Firefox146`, `Firefox147`, `Firefox148`, `Firefox149`, `Firefox150`, `Firefox151`, `Firefox152` |
| **Safari** | `Safari15_3`, `Safari15_5`, `Safari15_6_1`, `Safari16`, `Safari16_5`, `Safari17_0`, `Safari17_2_1`, `Safari17_4_1`, `Safari17_5`, `Safari17_6`, `Safari18`, `Safari18_2`, `Safari18_3`, `Safari18_3_1`, `Safari18_5`, `Safari26`, `Safari26_1`, `Safari26_2`, `SafariIos16_5`, `SafariIos17_2`, `SafariIos17_4_1`, `SafariIos18_1_1`, `SafariIos26`, `SafariIos26_2`, `SafariIPad18`, `SafariIPad26`, `SafariIpad26_2` |
| **OkHttp** | `Opera116`, `Opera117`, `Opera118`, `Opera119`, `Opera120`, `Opera121`, `Opera122`, `Opera123`, `Opera124`, `Opera125`, `Opera126`, `Opera127`, `Opera128`, `Opera129`, `Opera130` |
Expand Down
10 changes: 10 additions & 0 deletions crates/wreq-util/src/emulate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,16 @@ define_enum!(
Edge146 => ("edge_146", edge146::emulation),
Edge147 => ("edge_147", edge147::emulation),
Edge148 => ("edge_148", edge148::emulation),
Edge149 => ("edge_149", edge149::emulation),
Edge150 => ("edge_150", edge150::emulation),
Edge151 => ("edge_151", edge151::emulation),
Edge152 => ("edge_152", edge152::emulation),
Edge153 => ("edge_153", edge153::emulation),
Edge154 => ("edge_154", edge154::emulation),
Edge155 => ("edge_155", edge155::emulation),
Edge156 => ("edge_156", edge156::emulation),
Edge157 => ("edge_157", edge157::emulation),
Edge158 => ("edge_158", edge158::emulation),

// Opera versions
Opera116 => ("opera_116", opera116::emulation),
Expand Down
101 changes: 101 additions & 0 deletions crates/wreq-util/src/emulate/profile/chrome.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2302,3 +2302,104 @@ mod_generator!(
)
]
);

macro_rules! edge_desktop_ua {
($platform:literal, $major:literal) => {
concat!(
"Mozilla/5.0 (",
$platform,
") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/",
$major,
".0.0.0 Safari/537.36 Edg/",
$major,
".0.0.0"
)
};
}

macro_rules! edge_desktop_profile {
($name:ident, $major:literal, $brands:literal, $($transport:expr),+ $(,)?) => {
standard_mod_generator!($name, $($transport,)+ |emulation: &Emulation| {
emulation.headers.then(|| {
let (platform, ua) = match emulation.platform {
Platform::Windows => (
Platform::Windows,
edge_desktop_ua!("Windows NT 10.0; Win64; x64", $major),
),
Platform::Linux => (
Platform::Linux,
edge_desktop_ua!("X11; Linux x86_64", $major),
),
_ => (
Platform::MacOS,
edge_desktop_ua!("Macintosh; Intel Mac OS X 10_15_7", $major),
),
};
header_initializer_with_zstd_priority($brands, ua, platform)
})
});
};
}

// Unlike newer Chrome, Edge does not advertise JPEG XL in navigation requests.
edge_desktop_profile!(
edge149,
"149",
r#""Microsoft Edge";v="149", "Chromium";v="149", "Not)A;Brand";v="24""#,
v132::build_emulation
);
edge_desktop_profile!(
edge150,
"150",
r#""Not;A=Brand";v="8", "Chromium";v="150", "Microsoft Edge";v="150""#,
v150::build_emulation
);
edge_desktop_profile!(
edge151,
"151",
r#""Not=A?Brand";v="99", "Microsoft Edge";v="151", "Chromium";v="151""#,
v150::build_emulation
);
edge_desktop_profile!(
edge152,
"152",
r#""Chromium";v="152", "Not?A_Brand";v="24", "Microsoft Edge";v="152""#,
tls_options!(edge),
http2_options!(3)
);
edge_desktop_profile!(
edge153,
"153",
r#""Microsoft Edge";v="153", "Not_A Brand";v="8", "Chromium";v="153""#,
edge152::build_emulation
);
edge_desktop_profile!(
edge154,
"154",
r#""Chromium";v="154", "Microsoft Edge";v="154", "Not A(Brand";v="99""#,
edge152::build_emulation
);
edge_desktop_profile!(
edge155,
"155",
r#""Microsoft Edge";v="155", "Chromium";v="155", "Not(A:Brand";v="24""#,
edge152::build_emulation
);
edge_desktop_profile!(
edge156,
"156",
r#""Not:A-Brand";v="8", "Chromium";v="156", "Microsoft Edge";v="156""#,
edge152::build_emulation
);
edge_desktop_profile!(
edge157,
"157",
r#""Not-A.Brand";v="99", "Microsoft Edge";v="157", "Chromium";v="157""#,
edge152::build_emulation
);
edge_desktop_profile!(
edge158,
"158",
r#""Chromium";v="158", "Not.A/Brand";v="24", "Microsoft Edge";v="158""#,
edge152::build_emulation
);
12 changes: 12 additions & 0 deletions crates/wreq-util/src/emulate/profile/chrome/tls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,18 @@ macro_rules! tls_options {
$builder.build().into()
};

// Edge sends signature algorithm GREASE without Chromium's Trust Anchor IDs.
(edge) => {
tls_options!(@build ChromeTlsConfig::builder()
.permute_extensions(true)
.enable_ech_grease(true)
.pre_shared_key(true)
.curves(CURVES_3)
.sigalgs_list(NEW_SIGALGS_LIST)
.grease_sigalgs_enabled(true)
.alps_use_new_codepoint(true))
};

(1) => {
tls_options!(@build ChromeTlsConfig::builder())
};
Expand Down
44 changes: 43 additions & 1 deletion crates/wreq-util/tests/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,11 @@ async fn assert_emulation_headers(profile: Profile, expected: &'static [&'static
#[tokio::test]
async fn test_client_emulation_device() {
let server = server::http(move |req| async move {
let header = |name| req.headers().get(name).and_then(|value| value.to_str().ok());
let header = |name| {
req.headers()
.get(name)
.and_then(|value| value.to_str().ok())
};

assert_eq!(
header("user-agent"),
Expand Down Expand Up @@ -136,6 +140,44 @@ async fn test_chrome_default_header_order() {
assert_emulation_headers(Emulation::Chrome133, CHROMIUM_HEADER_ORDER).await;
}

#[tokio::test]
async fn test_edge158_desktop_headers() {
let server = server::http(|request| async move {
let headers = request.headers();
assert_eq!(
headers["user-agent"],
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/158.0.0.0 Safari/537.36 Edg/158.0.0.0"
);
assert_eq!(
headers["sec-ch-ua"],
r#""Chromium";v="158", "Not.A/Brand";v="24", "Microsoft Edge";v="158""#
);
assert_eq!(
headers["accept"],
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
);
#[cfg(feature = "emulation-compression")]
assert_eq!(headers["accept-encoding"], "gzip, deflate, br, zstd");
assert_eq!(headers["priority"], "u=0, i");
http::Response::default()
});

let response = Client::builder()
.emulation(
Emulation::builder()
.profile(Emulation::Edge158)
.platform(Platform::Windows)
.build(),
)
.build()
.unwrap()
.get(format!("http://{}/", server.addr()))
.send()
.await
.unwrap();
assert_eq!(response.status(), wreq::StatusCode::OK);
}

#[tokio::test]
async fn test_firefox_default_header_order() {
assert_emulation_headers(Emulation::Firefox109, FIREFOX_HEADER_ORDER).await;
Expand Down
7 changes: 7 additions & 0 deletions crates/wreq-util/tests/emulate_chrome.rs
Original file line number Diff line number Diff line change
Expand Up @@ -345,3 +345,10 @@ test_emulation!(
],
"52d84b11737d980aef856699f885ca86"
);

test_emulation!(
test_edge158,
Emulation::Edge158,
["t13d1516h2_8daaf6152771_806a8c22fdea"],
"52d84b11737d980aef856699f885ca86"
);
10 changes: 10 additions & 0 deletions lib/wreq_ruby/emulate.rb
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,16 @@ class Profile
Edge146 = nil
Edge147 = nil
Edge148 = nil
Edge149 = nil
Edge150 = nil
Edge151 = nil
Edge152 = nil
Edge153 = nil
Edge154 = nil
Edge155 = nil
Edge156 = nil
Edge157 = nil
Edge158 = nil

Firefox109 = nil
Firefox117 = nil
Expand Down
10 changes: 10 additions & 0 deletions src/emulate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,16 @@ define_ruby_enum!(
Edge146,
Edge147,
Edge148,
Edge149,
Edge150,
Edge151,
Edge152,
Edge153,
Edge154,
Edge155,
Edge156,
Edge157,
Edge158,
Firefox109,
Firefox117,
Firefox128,
Expand Down
43 changes: 34 additions & 9 deletions test/emulation_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,38 @@
require "timeout"

class EmulationTest < Minitest::Test
SIGNATURE_ALGORITHMS = 13
TRUST_ANCHOR_IDS = 51_764

def test_reused_emulation_retains_trust_anchors_across_clients_and_requests
[Wreq::Profile::Chrome152, Wreq::Profile::Chrome153, Wreq::Profile::Chrome154, Wreq::Profile::Chrome155, Wreq::Profile::Chrome156, Wreq::Profile::Chrome157].each do |profile|
emulation = Wreq::Emulation.new(profile: profile)
client = Wreq::Client.new(emulation: emulation, no_proxy: true)
expected = capture_trust_anchors { |url| client.get(url, timeout: 5) }
extensions = capture_client_hello_extensions { |url| client.get(url, timeout: 5) }
expected = extensions.fetch(TRUST_ANCHOR_IDS)
refute_empty expected, profile.to_s

request_client = Wreq::Client.new(no_proxy: true)
2.times do
actual = capture_trust_anchors do |url|
actual = capture_client_hello_extensions do |url|
request_client.get(url, emulation: emulation, timeout: 5)
end
assert_equal expected, actual, "#{profile}: reusing an object must retain its configuration"
assert_equal expected, actual.fetch(TRUST_ANCHOR_IDS), "#{profile}: reusing an object must retain its configuration"
end
end
end

def test_edge158_sends_signature_grease_without_trust_anchors
client = Wreq::Client.new(emulation: Wreq::Emulation.new(profile: Wreq::Profile::Edge158), no_proxy: true)
extensions = capture_client_hello_extensions { |url| client.get(url, timeout: 5) }

_, grease, *signatures = extensions.fetch(SIGNATURE_ALGORITHMS).unpack("n*")
assert_equal 0x0a0a, grease & 0x0f0f
assert_equal grease >> 8, grease & 0xff
assert_equal [0x0904, 0x0905, 0x0906], signatures.first(3) # ML-DSA
refute extensions.key?(TRUST_ANCHOR_IDS)
end

def test_all_emulation_device_constants_are_non_nil
Wreq::Profile.constants.each do |name|
const = Wreq::Profile.const_get(name)
Expand All @@ -38,6 +53,16 @@ def test_chrome151_profile_is_available
assert_instance_of Wreq::Emulation, Wreq::Emulation.new(profile: profile)
end

def test_all_edge_profiles_are_available
([101, 122, 127, 131] + (134..158).to_a).each do |version|
name = "Edge#{version}"
assert Wreq::Profile.const_defined?(name, false), "missing #{name}"
profile = Wreq::Profile.const_get(name, false)
assert_equal name, profile.to_s
assert_instance_of Wreq::Emulation, Wreq::Emulation.new(profile: profile)
end
end

def test_new_browser_profiles_are_available
{
Wreq::Profile::Chrome152 => "Chrome152",
Expand Down Expand Up @@ -66,7 +91,7 @@ def test_all_emulation_os_constants_are_non_nil

# Each listener forces a new connection. Stop after ClientHello so this test
# inspects the extension bytes without depending on certificates or the network.
def capture_trust_anchors
def capture_client_hello_extensions
server = TCPServer.new("127.0.0.1", 0)
thread = Thread.new do
socket = nil
Expand All @@ -81,7 +106,7 @@ def capture_trust_anchors
length = handshake.byteslice(1, 3).bytes.reduce(0) { |n, byte| (n << 8) | byte }
break if handshake.bytesize >= length + 4
end
trust_anchors_from_client_hello(handshake)
extensions_from_client_hello(handshake)
end
ensure
socket&.close
Expand All @@ -102,19 +127,19 @@ def read_tls_bytes(io, length)
bytes
end

def trust_anchors_from_client_hello(handshake)
def extensions_from_client_hello(handshake)
io = StringIO.new(handshake)
raise "expected ClientHello" unless read_tls_bytes(io, 1).unpack1("C") == 1
read_tls_bytes(io, 3 + 2 + 32) # Handshake length, legacy version and random.
read_tls_bytes(io, read_tls_bytes(io, 1).unpack1("C")) # Session ID.
read_tls_bytes(io, read_tls_bytes(io, 2).unpack1("n")) # Cipher suites.
read_tls_bytes(io, read_tls_bytes(io, 1).unpack1("C")) # Compression methods.
extensions = StringIO.new(read_tls_bytes(io, read_tls_bytes(io, 2).unpack1("n")))
parsed = {}
until extensions.eof?
type, length = read_tls_bytes(extensions, 4).unpack("nn")
data = read_tls_bytes(extensions, length)
return data if type == 51764
parsed[type] = read_tls_bytes(extensions, length)
end
raise "missing trust_anchors extension"
parsed
end
end
Loading