Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,12 @@ uv run --no-sync python bench/run.py \
--server bench/server/target/release/wreq-benchmark-server \
--sizes 10240,1048576 --concurrency 2 --requests 4 \
--rounds 1 --warmup 0 --samples 1 --output bench/data/smoke/RUN.json
uv run --no-sync python -m pytest bench/test_*.py
uv run --no-sync python -m pytest bench
```

Default `pytest` runs only `tests/`. The explicit command above checks the
benchmark tools without running a performance measurement.

## Results and interpretation

The JSON records the source SHA and dirty state, Python and package versions,
Expand Down Expand Up @@ -154,8 +157,8 @@ runner doesn't commit, push or enable benchmarks in CI.

`python docs/build.py` reads and validates the checked-in `bench/data/latest.json`.
It generates responsive light/dark SVG charts and fills
`docs/templates/benchmark.md` with body-size controls and expandable tables.
The built site includes a frozen raw JSON download. The build won't fetch
`docs/templates/benchmark.md` with body-size controls and measurement details.
The built site includes a frozen raw JSON copy. The build won't fetch
measurements or start a benchmark, and missing or invalid data stops it.
Read the Docs uses this same entry point.

Expand Down
40 changes: 16 additions & 24 deletions bench/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,12 @@ def safe_data_link(link: str) -> str:
return link


def render_markdown(document: dict, data_link: str | None = None) -> str:
def render_markdown(
document: dict,
data_link: str | None = None,
*,
environment_only: bool = False,
) -> str:
validate_document(document)
config = document["configuration"]
source = document["source"]
Expand Down Expand Up @@ -128,13 +133,19 @@ def render_markdown(document: dict, data_link: str | None = None) -> str:
f"requests/batch: {config['requests']}; warm-up batches/round: {config['warmup']} |",
"| Stream upload chunk | "
+ (
"Varies by payload; see body cases below"
(
"Varies by payload; recorded in raw JSON"
if environment_only
else "Varies by payload; see body cases below"
)
if "stream_chunk_bytes_by_payload" in config
else payload_label(config["stream_chunk_bytes"])
)
+ " |",
"",
]
if environment_only:
return "\n".join(lines)
if "stream_chunk_bytes_by_payload" in config:
lines += [
"### Body cases",
Expand Down Expand Up @@ -168,35 +179,16 @@ def render_markdown(document: dict, data_link: str | None = None) -> str:
f"{', '.join('HTTP/1.1' if p == 'h1' else 'HTTP/2' for p in capability['protocols'])} | "
f"{', '.join(k.title() for k in capability['body_kinds'])} |"
)
if any(
"response_read" in document["clients"][client] for client in ordered_clients
):
lines += [
"",
'<details markdown="1">',
"<summary>Adapter settings</summary>",
"",
"| Client | Response consumption | Connection pool |",
"| --- | --- | --- |",
]
for client in ordered_clients:
metadata = document["clients"][client]
pool = json.dumps(
metadata.get("pool", {"kind": "not recorded"}), sort_keys=True
)
lines.append(
f"| {escape(LABELS.get(client, client))} | "
f"{escape(metadata.get('response_read', 'Not recorded'))} | {escape(pool)} |"
)
lines += ["", "</details>"]
lines += [
"",
"### Throughput comparison",
"",
"Each value is total measured requests divided by total measured time "
"across all rounds, in requests per second (RPS). All timed samples count. "
"The raw JSON keeps individual timings, per-round results, response-payload "
"MB/s and native artifact SHA-256 hashes.",
"MB/s and native artifact SHA-256 hashes. Read chunking, buffering and "
"connection pools differ between adapters; their configurations are "
"recorded in the raw JSON.",
"",
]
results = {
Expand Down
31 changes: 25 additions & 6 deletions bench/test_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ def test_report_aggregation_and_metadata_escaping():
assert "`x`" not in text and "&#124;" in text
assert report.escape("CPU's name") == "CPU's name"
assert "javascript:source-repository" not in text
assert "Adapter settings" not in text and "<details" not in text
assert (
'<details markdown="1">' in text
and "<summary>Adapter settings</summary>" in text
"Read chunking, buffering and connection pools differ between adapters" in text
)
assert f"{report.REPOSITORY}/commit/{document['source']['commit']}" in text
for link in (
Expand Down Expand Up @@ -250,11 +250,21 @@ def no_network(*args, **kwargs):
snapshot = root / "docs/source/assets/benchmark/latest.json"
page = root / "docs/source/benchmark.md"
assert json.loads(snapshot.read_text(encoding="utf-8")) == document
assert "{{BENCHMARK_RESULTS}}" not in page.read_text(encoding="utf-8")
assert "{{BENCHMARK_CHARTS}}" not in page.read_text(encoding="utf-8")
content = page.read_text(encoding="utf-8")
assert "{{BENCHMARK_RESULTS}}" not in content
assert "{{BENCHMARK_CHARTS}}" not in content
assert "<details" not in content and "<summary>" not in content
assert report.render_markdown(document, environment_only=True) in content
assert "### Throughput comparison" not in content
assert "| Upload / echo payload | Concurrency" not in content
assert "### Measurement environment" in content
assert "### Client versions and runtimes" not in content
assert "### Body cases" not in content
assert "## Runtime and client differences" in content
charts = root / "docs/source/assets/benchmark/charts"
assert len(list(charts.glob("*.svg"))) == 16
assert "assets/benchmark/latest.json" in page.read_text(encoding="utf-8")
assert "Download this build's raw JSON" not in content
assert "Raw JSON</a>" not in content and "Download SVG" not in content
assert build.prepare(data=snapshot, root=root) == document
override = root / "override.json"
alternate = make_document(commit="d" * 40)
Expand Down Expand Up @@ -318,8 +328,17 @@ def test_chart_explorer_controls_fallback_and_inert_metadata(tmp_path):
assert 'value="h1" selected' in content and 'value="h2"' not in content
assert content.count("data-chart-payload=") == 1
assert content.count("data-chart-controls hidden") == 2
assert "<noscript>" in content and "Values for this chart" in content
assert "<noscript>" in content and "measurement details remain available" in content
assert "Values for this chart" not in content and "data-chart-values" not in content
assert "<table" not in content and "<details" not in content
assert "data-chart-download" not in content and "Download SVG" not in content
assert "Raw JSON</a>" not in content
assets = catalog["cases"][0]["assets"]["dark"]
assert f'src="assets/benchmark/charts/{assets["desktop"]}"' in content
assert f'srcset="assets/benchmark/charts/{assets["mobile"]}"' in content
assert "zero-based linear scale" in content and "requests/s (RPS)" in content
alt = re.search(r'<img data-chart-image[^>]* alt="([^"]*)"', content).group(1)
assert f': {catalog["cases"][0]["rows"][0]["rps"]:,.1f}' in alt
assert '<img src=x onerror="alert(1)">' not in content
assert "&lt;img src=x" in content
data = re.search(
Expand Down
23 changes: 10 additions & 13 deletions docs/benchmark_view.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,9 @@ def render_explorer(document, catalog, asset_prefix="assets/benchmark/charts"):
f'<button type="button" data-chart-payload="{size}" aria-pressed="{str(size == first["payload_bytes"]).lower()}">{payload_label(size)}</button>'
for size in payloads
)
assets = first["assets"]["light"]
assets = first["assets"]["dark"]
image = safe_data_link(f'{asset_prefix}/{assets["desktop"]}')
mobile = safe_data_link(f'{asset_prefix}/{assets["mobile"]}')
rows = "".join(
f'<tr><td>{escape(row["label"])}</td><td>{row["rps"]:,.1f}</td></tr>'
for row in first["rows"]
) + "".join(
f"<tr><td>{escape(label)}</td><td>N/A</td></tr>"
for label in first["unsupported"]
)
# This is inert JSON, not script. Escape HTML delimiters so metadata cannot end the tag.
data = json.dumps(catalog, ensure_ascii=True, separators=(",", ":"))
data = data.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026")
Expand All @@ -62,9 +55,14 @@ def render_explorer(document, catalog, asset_prefix="assets/benchmark/charts"):
if first["unsupported"]
else "All shown clients support this case."
)
values = "; ".join(
[f'{row["label"]}: {row["rps"]:,.1f}' for row in first["rows"]]
+ [f"{label}: N/A" for label in first["unsupported"]]
)
alt = f"{first['title']}. Throughput in requests per second. {values}."
return f"""<section class="wreq-bench" data-bench-explorer aria-label="Benchmark chart explorer">
<p class="wreq-bench-provenance">Measured <a href="{REPOSITORY}/commit/{source['commit']}">{source['commit'][:12]}</a> · {date:%Y-%m-%d %H:%M UTC}{dirty}</p>
<div class="wreq-bench-heading"><div><p class="wreq-bench-eyebrow">THROUGHPUT BY WORKLOAD</p><h3>Compare clients</h3></div><a href="assets/benchmark/latest.json">Raw JSON</a></div>
<div class="wreq-bench-heading"><p class="wreq-bench-eyebrow">THROUGHPUT BY WORKLOAD</p><h3>Compare clients</h3></div>
<p class="wreq-bench-intro">Pick a body size, then adjust the filters to compare clients. Full and Stream refer to the upload; we read every response to the end.</p>
<div class="wreq-bench-body-controls" data-chart-controls hidden>
<button type="button" data-chart-previous aria-label="Previous body size">←</button>
Expand All @@ -74,11 +72,10 @@ def render_explorer(document, catalog, asset_prefix="assets/benchmark/charts"):
<div class="wreq-bench-filters" data-chart-controls hidden>{''.join(controls)}</div>
<p class="wreq-bench-selection" data-chart-caption aria-live="polite">{escape(first['title'])}</p>
<figure class="wreq-bench-figure">
<picture><source media="(max-width: 600px)" data-chart-mobile srcset="{mobile}"><img data-chart-image src="{image}" alt="{escape(first['title'])}. Throughput in requests per second; values in the table below." width="900" decoding="async"></picture>
<figcaption><span data-chart-unsupported>{escape(unsupported)}</span><a data-chart-download href="{image}" download>Download SVG</a></figcaption>
<picture><source media="(max-width: 600px)" data-chart-mobile srcset="{mobile}"><img data-chart-image src="{image}" alt="{escape(alt)}" width="900" decoding="async"></picture>
<figcaption data-chart-unsupported>{escape(unsupported)}</figcaption>
</figure>
<p class="wreq-bench-note">Unit: requests/s (RPS). Higher is better. Each chart uses its own zero-based linear scale; compare the numbers, not bar lengths across different charts.</p>
<details class="wreq-bench-values"><summary>Values for this chart</summary><table><thead><tr><th>Client / runtime</th><th>Requests/s</th></tr></thead><tbody data-chart-values>{rows}</tbody></table></details>
<noscript><p>Chart switching needs JavaScript. The default chart and complete tables below remain available.</p></noscript>
<noscript><p>Chart switching needs JavaScript. The default chart and measurement details remain available.</p></noscript>
<script type="application/json" data-chart-catalog>{data}</script>
</section>"""
2 changes: 1 addition & 1 deletion docs/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def prepare(
)
document = decode_document(raw)
try:
content = render_markdown(document, data_link="assets/benchmark/latest.json")
content = render_markdown(document, environment_only=True)
catalog = write_charts(document, root / "docs/source/assets/benchmark/charts")
charts = render_explorer(document, catalog)
write_atomic(snapshot, document)
Expand Down
17 changes: 5 additions & 12 deletions docs/mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ repo_url: https://github.com/0x676e67/wreq-python
docs_dir: source

extra_css:
- stylesheets/theme.css
- stylesheets/home.css
- stylesheets/benchmarks.css

Expand All @@ -22,21 +23,13 @@ extra:

theme:
name: material
custom_dir: overrides
font:
code: Roboto Mono
palette:
- scheme: slate
primary: black
accent: indigo
toggle:
icon: material/toggle-switch-off
name: Switch to light mode
- scheme: default
primary: indigo
accent: indigo
toggle:
icon: material/toggle-switch
name: Switch to dark mode
scheme: slate
primary: black
accent: indigo
features:
- search.highlight
- search.share
Expand Down
1 change: 1 addition & 0 deletions docs/overrides/partials/javascripts/palette.html
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{# The site uses a fixed palette, so do not restore a previously saved theme. #}
2 changes: 1 addition & 1 deletion docs/source/guide/advanced.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# :star2: Advanced Features
# Advanced Features

!!! info "On this page"
- Header order
Expand Down
2 changes: 1 addition & 1 deletion docs/source/guide/auth.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# :lock: Authentication Guide
# Authentication Guide

!!! tip "Supported authentication methods"
- Basic Auth
Expand Down
2 changes: 1 addition & 1 deletion docs/source/guide/basic.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# :rocket: Basic Usage
# Basic Usage

!!! info "On this page"
- GET/POST requests
Expand Down
2 changes: 1 addition & 1 deletion docs/source/guide/blocking.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# :hourglass: Blocking/Sync API
# Blocking/Sync API

!!! info "On this page"
- Blocking GET
Expand Down
2 changes: 1 addition & 1 deletion docs/source/guide/emulation.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# :busts_in_silhouette: Emulation
# Emulation

!!! info "Emulation topics"
- Firefox/Chrome/Android
Expand Down
2 changes: 1 addition & 1 deletion docs/source/guide/proxy.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# :globe_with_meridians: Proxy Usage
# Proxy Usage

!!! info "On this page"
- HTTP/HTTPS proxy
Expand Down
2 changes: 1 addition & 1 deletion docs/source/guide/redirect-errors.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# :repeat: Redirects & Error Handling
# Redirects & Error Handling

!!! info "On this page"
- Custom redirect policy
Expand Down
2 changes: 1 addition & 1 deletion docs/source/guide/websocket.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# :satellite: WebSocket
# WebSocket

!!! info "On this page"
- HTTP/1.1 WebSocket
Expand Down
2 changes: 1 addition & 1 deletion docs/source/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ Choose a body size, protocol and concurrency level to see the results for your w

## Documentation and community

[Install wreq](getting-started/installation.md) · [Read the guides](guide/basic.md) · [Browse the API](api/wreq.md) · [Join the community](https://discord.gg/rfbvyFkgq3)
[Install wreq](getting-started/installation.md) · [Read the guides](guide/basic.md) · [Browse the API](api/wreq.md) · [Join the Discord](https://discord.gg/rfbvyFkgq3)

Browser profiles configure network behavior. They do not execute JavaScript or guarantee access to a protected website.
{ .wreq-fine-print }
Expand Down
36 changes: 4 additions & 32 deletions docs/source/javascripts/benchmarks.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,7 @@
(() => {
const initialized = new WeakSet();
const observers = new Map();

function initialize() {
observers.forEach((observer, section) => {
if (!section.isConnected) {
observer.disconnect();
observers.delete(section);
}
});
document.querySelectorAll("[data-bench-explorer]").forEach((section) => {
if (initialized.has(section)) return;
initialized.add(section);
Expand All @@ -28,28 +21,16 @@
&& item.api === values.api && item.protocol === values.protocol
&& item.body_kind === values.body_kind && item.concurrency === Number(values.concurrency));
if (!current) return;
const theme = document.body.getAttribute("data-md-color-scheme") === "slate" ? "dark" : "light";
const assets = current.assets[theme];
const assets = current.assets.dark;
image.src = `${prefix}${assets.desktop}`;
image.alt = `${current.title}. Throughput in requests per second; values in the table below.`;
const description = [...current.rows.map((row) => `${row.label}: ${number.format(row.rps)}`),
...current.unsupported.map((label) => `${label}: N/A`)].join("; ");
image.alt = `${current.title}. Throughput in requests per second. ${description}.`;
mobile.srcset = `${prefix}${assets.mobile}`;
const download = section.querySelector("[data-chart-download]");
download.href = image.src;
section.querySelector("[data-chart-caption]").textContent = current.title;
section.querySelector("[data-chart-unsupported]").textContent = current.unsupported.length
? `N/A: ${current.unsupported.join(", ")}` : "All shown clients support this case.";
payloadButtons.forEach((button) => button.setAttribute("aria-pressed", String(Number(button.dataset.chartPayload) === payload)));
const rows = [...current.rows.map((row) => [row.label, number.format(row.rps)]),
...current.unsupported.map((label) => [label, "N/A"])];
section.querySelector("[data-chart-values]").replaceChildren(...rows.map((values) => {
const row = document.createElement("tr");
values.forEach((value) => {
const cell = document.createElement("td");
cell.textContent = value;
row.append(cell);
});
return row;
}));
}

function move(direction) {
Expand All @@ -76,15 +57,6 @@
section.querySelector("[data-chart-previous]").addEventListener("click", () => move(-1));
section.querySelector("[data-chart-next]").addEventListener("click", () => move(1));
selects.forEach((select) => select.addEventListener("change", update));
const observer = new MutationObserver(() => {
if (!section.isConnected) {
observer.disconnect();
return;
}
update();
});
observer.observe(document.body, { attributes: true, attributeFilter: ["data-md-color-scheme"] });
observers.set(section, observer);
section.querySelectorAll("[data-chart-controls]").forEach((controls) => { controls.hidden = false; });
update();
});
Expand Down
Loading
Loading