Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ Most browser device models share identical TLS and HTTP/2 configurations, differ
| ----------- | ------------ |
| **Chrome** | `Chrome100`, `Chrome101`, `Chrome104`, `Chrome105`, `Chrome106`, `Chrome107`, `Chrome108`, `Chrome109`, `Chrome110`, `Chrome114`, `Chrome116`, `Chrome117`, `Chrome118`, `Chrome119`, `Chrome120`, `Chrome123`, `Chrome124`, `Chrome126`, `Chrome127`, `Chrome128`, `Chrome129`, `Chrome130`, `Chrome131`, `Chrome132`, `Chrome133`, `Chrome134`, `Chrome135`, `Chrome136`, `Chrome137`, `Chrome138`, `Chrome139`, `Chrome140`, `Chrome141`, `Chrome142`, `Chrome143`, `Chrome144`, `Chrome145`, `Chrome146`, `Chrome147`, `Chrome148`, `Chrome149`, `Chrome150`, `Chrome151`, `Chrome152`, `Chrome153`, `Chrome154`, `Chrome155`, `Chrome156`, `Chrome157`, `Chrome158` |
| **Edge** | `Edge101`, `Edge122`, `Edge127`, `Edge131`, `Edge134`, `Edge135`, `Edge136`, `Edge137`, `Edge138`, `Edge139`, `Edge140`, `Edge141`, `Edge142`, `Edge143`, `Edge144`, `Edge145`, `Edge146`, `Edge147`, `Edge148` |
| **Firefox** | `Firefox109`, `Firefox117`, `Firefox128`, `Firefox133`, `Firefox135`, `FirefoxPrivate135`, `FirefoxAndroid135`, `Firefox136`, `FirefoxPrivate136`, `Firefox139`, `Firefox142`, `Firefox143`, `Firefox144`, `Firefox145`, `Firefox146`, `Firefox147`, `Firefox148`, `Firefox149`, `Firefox150`, `Firefox151`, `Firefox152` |
| **Firefox** | `Firefox109`, `Firefox117`, `Firefox128`, `Firefox133`, `Firefox135`, `FirefoxPrivate135`, `FirefoxAndroid135`, `Firefox136`, `FirefoxPrivate136`, `Firefox139`, `Firefox142`, `Firefox143`, `Firefox144`, `Firefox145`, `Firefox146`, `Firefox147`, `Firefox148`, `Firefox149`, `Firefox150`, `Firefox151`, `Firefox152`, `Firefox153`–`Firefox160` (desktop) |
| **Safari** | `Safari15_3`, `Safari15_5`, `Safari15_6_1`, `Safari16`, `Safari16_5`, `Safari17_0`, `Safari17_2_1`, `Safari17_4_1`, `Safari17_5`, `Safari17_6`, `Safari18`, `Safari18_2`, `Safari18_3`, `Safari18_3_1`, `Safari18_5`, `Safari26`, `Safari26_1`, `Safari26_2`, `SafariIos16_5`, `SafariIos17_2`, `SafariIos17_4_1`, `SafariIos18_1_1`, `SafariIos26`, `SafariIos26_2`, `SafariIPad18`, `SafariIPad26`, `SafariIpad26_2` |
| **OkHttp** | `Opera116`, `Opera117`, `Opera118`, `Opera119`, `Opera120`, `Opera121`, `Opera122`, `Opera123`, `Opera124`, `Opera125`, `Opera126`, `Opera127`, `Opera128`, `Opera129`, `Opera130` |
| **Opera** | `Opera116`, `Opera117`, `Opera118`, `Opera119` |
Expand Down
8 changes: 8 additions & 0 deletions crates/wreq-util/src/emulate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,14 @@ define_enum!(
Firefox150 => ("firefox_150", ff150::emulation),
Firefox151 => ("firefox_151", ff151::emulation),
Firefox152 => ("firefox_152", ff152::emulation),
Firefox153 => ("firefox_153", ff153::emulation),
Firefox154 => ("firefox_154", ff154::emulation),
Firefox155 => ("firefox_155", ff155::emulation),
Firefox156 => ("firefox_156", ff156::emulation),
Firefox157 => ("firefox_157", ff157::emulation),
Firefox158 => ("firefox_158", ff158::emulation),
Firefox159 => ("firefox_159", ff159::emulation),
Firefox160 => ("firefox_160", ff160::emulation),

// Safari versions
SafariIos17_2 => ("safari_ios_17.2", safari_ios_17_2::emulation),
Expand Down
75 changes: 73 additions & 2 deletions crates/wreq-util/src/emulate/profile/firefox.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,13 @@ mod_generator!(

mod_generator!(
ff135,
tls_options!(4, CIPHER_LIST_1, CURVES_2, KEY_SHARES_2),
tls_options!(
4,
CIPHER_LIST_1,
CURVES_2,
KEY_SHARES_2,
EchGrease::NssPre3127
),
http2_options!(1),
header_initializer_with_zstd,
[
Expand Down Expand Up @@ -463,7 +469,13 @@ mod_generator!(

mod_generator!(
ff150,
tls_options!(4, CIPHER_LIST_3, CURVES_2, KEY_SHARES_2),
tls_options!(
4,
CIPHER_LIST_3,
CURVES_2,
KEY_SHARES_2,
EchGrease::NssPre3127
),
http2_options!(1),
header_initializer_with_zstd_q09,
[
Expand Down Expand Up @@ -545,3 +557,62 @@ mod_generator!(
)
]
);

macro_rules! firefox_desktop_ua {
($platform:literal, $major:literal) => {
concat!(
"Mozilla/5.0 (",
$platform,
"; rv:",
$major,
".0) Gecko/20100101 Firefox/",
$major,
".0"
)
};
}

macro_rules! firefox_desktop_profile {
($name:ident, $major:literal, $($transport:expr),+ $(,)?) => {
standard_mod_generator!($name, $($transport,)+ |emulation: &Emulation| {
emulation.headers.then(|| {
let ua = match emulation.platform {
Platform::MacOS => firefox_desktop_ua!("Macintosh; Intel Mac OS X 10.15", $major),
Platform::Linux => firefox_desktop_ua!("X11; Linux x86_64", $major),
_ => firefox_desktop_ua!("Windows NT 10.0; Win64; x64", $major),
};
header_initializer_with_zstd_q09(ua)
})
});
};
}

firefox_desktop_profile!(ff153, "153", ff150::build_emulation);
firefox_desktop_profile!(
ff154,
"154",
tls_options!(
4,
CIPHER_LIST_2,
CURVES_2,
KEY_SHARES_2,
EchGrease::NssPre3127
),
http2_options!(1)
);
firefox_desktop_profile!(
ff155,
"155",
tls_options!(4, CIPHER_LIST_2, CURVES_2, KEY_SHARES_2, EchGrease::Nss3127),
http2_options!(1)
);
firefox_desktop_profile!(
ff156,
"156",
tls_options!(4, CIPHER_LIST_2, CURVES_3, KEY_SHARES_2, EchGrease::Nss3127),
http2_options!(1)
);
firefox_desktop_profile!(ff157, "157", ff156::build_emulation);
firefox_desktop_profile!(ff158, "158", ff156::build_emulation);
firefox_desktop_profile!(ff159, "159", ff156::build_emulation);
firefox_desktop_profile!(ff160, "160", ff156::build_emulation);
21 changes: 12 additions & 9 deletions crates/wreq-util/src/emulate/profile/firefox/tls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ macro_rules! tls_options {

(1, $cipher_list:expr, $curves:expr, $key_shares:expr) => {
tls_options!(@build tls_options!(@base FirefoxTlsConfig::builder(), $cipher_list, $curves)
.enable_ech_grease(true)
.ech_grease(EchGrease::NssPre3127)
.pre_shared_key(true)
.psk_skip_session_tickets(true)
.key_shares($key_shares)
Expand All @@ -25,13 +25,13 @@ macro_rules! tls_options {
(3, $cipher_list:expr, $curves:expr, $key_shares:expr) => {
tls_options!(@build tls_options!(@base FirefoxTlsConfig::builder(), $cipher_list, $curves)
.session_ticket(false)
.enable_ech_grease(true)
.ech_grease(EchGrease::NssPre3127)
.psk_dhe_ke(false)
.key_shares($key_shares))
};
(4, $cipher_list:expr, $curves:expr, $key_shares:expr) => {
(4, $cipher_list:expr, $curves:expr, $key_shares:expr, $ech_grease:expr) => {
tls_options!(@build tls_options!(@base FirefoxTlsConfig::builder(), $cipher_list, $curves)
.enable_ech_grease(true)
.ech_grease($ech_grease)
.enable_signed_cert_timestamps(true)
.session_ticket(true)
.pre_shared_key(true)
Expand All @@ -41,15 +41,15 @@ macro_rules! tls_options {
};
(5, $cipher_list:expr, $curves:expr, $key_shares:expr) => {
tls_options!(@build tls_options!(@base FirefoxTlsConfig::builder(), $cipher_list, $curves)
.enable_ech_grease(true)
.ech_grease(EchGrease::NssPre3127)
.pre_shared_key(true)
.psk_skip_session_tickets(true)
.key_shares($key_shares)
.certificate_compressors(CERTIFICATE_COMPRESSORS))
};
(6, $cipher_list:expr, $curves:expr, $key_shares:expr) => {
tls_options!(@build tls_options!(@base FirefoxTlsConfig::builder(), $cipher_list, $curves)
.enable_ech_grease(true)
.ech_grease(EchGrease::NssPre3127)
.enable_signed_cert_timestamps(true)
.session_ticket(false)
.psk_dhe_ke(false)
Expand Down Expand Up @@ -79,6 +79,9 @@ pub const CURVES_2: &str = join!(
"ffdhe3072"
);

// Firefox 156 stopped advertising FFDHE groups by default.
pub const CURVES_3: &str = join!(":", "X25519MLKEM768", "X25519", "P-256", "P-384", "P-521");

pub const KEY_SHARES_1: &[KeyShare] = &[KeyShare::X25519, KeyShare::P256];

pub const KEY_SHARES_2: &[KeyShare] =
Expand Down Expand Up @@ -206,8 +209,8 @@ pub struct FirefoxTlsConfig {
#[builder(default = true)]
session_ticket: bool,

#[builder(default = false, setter(into))]
enable_ech_grease: bool,
#[builder(default, setter(into))]
ech_grease: Option<EchGrease>,

#[builder(default = false, setter(into))]
enable_signed_cert_timestamps: bool,
Expand Down Expand Up @@ -245,7 +248,7 @@ impl From<FirefoxTlsConfig> for TlsOptions {
.delegated_credentials(val.delegated_credentials)
.record_size_limit(val.record_size_limit)
.enable_ocsp_stapling(true)
.ech_grease(val.enable_ech_grease.then_some(EchGrease::NssPre3127))
.ech_grease(val.ech_grease)
.enable_signed_cert_timestamps(val.enable_signed_cert_timestamps)
.alpn_protocols([AlpnProtocol::HTTP2, AlpnProtocol::HTTP1])
.min_tls_version(TlsVersion::TLS_1_2)
Expand Down
15 changes: 15 additions & 0 deletions crates/wreq-util/tests/emulate_firefox.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,13 @@ test_emulation!(
"6ea73faa8fc5aac76bded7bd238f6433"
);

test_emulation!(
test_firefox_160,
Emulation::Firefox160,
["t13d1517h2_8daaf6152771_3cbfd9057e0d"],
"6ea73faa8fc5aac76bded7bd238f6433"
);

/// Firefox 147 changed the generated `Accept-Language` q-value: the list used to divide 1.0
/// across its entries, giving `q=0.5` for two, and now decrements by 0.1 per entry, giving
/// `q=0.9`. See Bugzilla 2000765, landed in mozilla-firefox 0e050ae5116d.
Expand Down Expand Up @@ -146,6 +153,14 @@ async fn firefox_accept_language_matches_the_profile_version() {
("Firefox150", Emulation::Firefox150),
("Firefox151", Emulation::Firefox151),
("Firefox152", Emulation::Firefox152),
("Firefox153", Emulation::Firefox153),
("Firefox154", Emulation::Firefox154),
("Firefox155", Emulation::Firefox155),
("Firefox156", Emulation::Firefox156),
("Firefox157", Emulation::Firefox157),
("Firefox158", Emulation::Firefox158),
("Firefox159", Emulation::Firefox159),
("Firefox160", Emulation::Firefox160),
];

let server = support::server::http(|request| async move {
Expand Down
8 changes: 8 additions & 0 deletions lib/wreq_ruby/emulate.rb
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,14 @@ class Profile
Firefox150 = nil
Firefox151 = nil
Firefox152 = nil
Firefox153 = nil
Firefox154 = nil
Firefox155 = nil
Firefox156 = nil
Firefox157 = nil
Firefox158 = nil
Firefox159 = nil
Firefox160 = nil

SafariIos17_2 = nil
SafariIos17_4_1 = nil
Expand Down
8 changes: 8 additions & 0 deletions src/emulate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,14 @@ define_ruby_enum!(
Firefox150,
Firefox151,
Firefox152,
Firefox153,
Firefox154,
Firefox155,
Firefox156,
Firefox157,
Firefox158,
Firefox159,
Firefox160,
SafariIos17_2,
SafariIos17_4_1,
SafariIos16_5,
Expand Down
50 changes: 41 additions & 9 deletions test/emulation_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,22 @@
require "timeout"

class EmulationTest < Minitest::Test
TRUST_ANCHOR_IDS = 51_764

def test_reused_emulation_retains_trust_anchors_across_clients_and_requests
[Wreq::Profile::Chrome152, Wreq::Profile::Chrome153, Wreq::Profile::Chrome154, Wreq::Profile::Chrome155, Wreq::Profile::Chrome156, Wreq::Profile::Chrome157].each do |profile|
emulation = Wreq::Emulation.new(profile: profile)
client = Wreq::Client.new(emulation: emulation, no_proxy: true)
expected = capture_trust_anchors { |url| client.get(url, timeout: 5) }
extensions = capture_client_hello_extensions { |url| client.get(url, timeout: 5) }
expected = extensions.fetch(TRUST_ANCHOR_IDS)
refute_empty expected, profile.to_s

request_client = Wreq::Client.new(no_proxy: true)
2.times do
actual = capture_trust_anchors do |url|
actual = capture_client_hello_extensions do |url|
request_client.get(url, emulation: emulation, timeout: 5)
end
assert_equal expected, actual, "#{profile}: reusing an object must retain its configuration"
assert_equal expected, actual.fetch(TRUST_ANCHOR_IDS), "#{profile}: reusing an object must retain its configuration"
end
end
end
Expand Down Expand Up @@ -62,11 +65,40 @@ def test_all_emulation_os_constants_are_non_nil
end
end

def test_all_firefox_profiles_are_available
versions = [109, 117, 128, 133, 135, 136, 139, *(142..160)]
names = versions.map { |version| "Firefox#{version}" } + %w[FirefoxPrivate135 FirefoxAndroid135 FirefoxPrivate136]
names.each do |name|
assert Wreq::Profile.const_defined?(name, false), "missing #{name}"
profile = Wreq::Profile.const_get(name, false)
assert_equal name, profile.to_s
assert_instance_of Wreq::Emulation, Wreq::Emulation.new(profile: profile)
end
end

def test_firefox_client_hello_matches_nss_version
[
[Wreq::Profile::Firefox154, 15, [4588, 29, 23, 24, 25, 256, 257]],
[Wreq::Profile::Firefox155, 16, [4588, 29, 23, 24, 25, 256, 257]],
[Wreq::Profile::Firefox156, 16, [4588, 29, 23, 24, 25]]
].each do |profile, padding_remainder, groups|
client = Wreq::Client.new(emulation: Wreq::Emulation.new(profile: profile), no_proxy: true)
extensions = capture_client_hello_extensions { |url| client.get(url, timeout: 5) }

ech = StringIO.new(extensions.fetch(65_037))
read_tls_bytes(ech, 6) # Outer tag, HPKE suite and config ID.
read_tls_bytes(ech, read_tls_bytes(ech, 2).unpack1("n")) # Encapsulated key.
payload_size = read_tls_bytes(ech, 2).unpack1("n")
assert_equal padding_remainder, payload_size % 32, "#{profile}: ECH GREASE ciphertext"
assert_equal groups, extensions.fetch(10).unpack("n*").drop(1), "#{profile}: supported groups"
end
end

private

# Each listener forces a new connection. Stop after ClientHello so this test
# inspects the extension bytes without depending on certificates or the network.
def capture_trust_anchors
def capture_client_hello_extensions
server = TCPServer.new("127.0.0.1", 0)
thread = Thread.new do
socket = nil
Expand All @@ -81,7 +113,7 @@ def capture_trust_anchors
length = handshake.byteslice(1, 3).bytes.reduce(0) { |n, byte| (n << 8) | byte }
break if handshake.bytesize >= length + 4
end
trust_anchors_from_client_hello(handshake)
extensions_from_client_hello(handshake)
end
ensure
socket&.close
Expand All @@ -102,19 +134,19 @@ def read_tls_bytes(io, length)
bytes
end

def trust_anchors_from_client_hello(handshake)
def extensions_from_client_hello(handshake)
io = StringIO.new(handshake)
raise "expected ClientHello" unless read_tls_bytes(io, 1).unpack1("C") == 1
read_tls_bytes(io, 3 + 2 + 32) # Handshake length, legacy version and random.
read_tls_bytes(io, read_tls_bytes(io, 1).unpack1("C")) # Session ID.
read_tls_bytes(io, read_tls_bytes(io, 2).unpack1("n")) # Cipher suites.
read_tls_bytes(io, read_tls_bytes(io, 1).unpack1("C")) # Compression methods.
extensions = StringIO.new(read_tls_bytes(io, read_tls_bytes(io, 2).unpack1("n")))
parsed = {}
until extensions.eof?
type, length = read_tls_bytes(extensions, 4).unpack("nn")
data = read_tls_bytes(extensions, length)
return data if type == 51764
parsed[type] = read_tls_bytes(extensions, length)
end
raise "missing trust_anchors extension"
parsed
end
end
Loading