diff --git a/bench/README.md b/bench/README.md
index 53c619ad..d5bf678c 100644
--- a/bench/README.md
+++ b/bench/README.md
@@ -85,9 +85,12 @@ uv run --no-sync python bench/run.py \
--server bench/server/target/release/wreq-benchmark-server \
--sizes 10240,1048576 --concurrency 2 --requests 4 \
--rounds 1 --warmup 0 --samples 1 --output bench/data/smoke/RUN.json
-uv run --no-sync python -m pytest bench/test_*.py
+uv run --no-sync python -m pytest bench
```
+Default `pytest` runs only `tests/`. The explicit command above checks the
+benchmark tools without running a performance measurement.
+
## Results and interpretation
The JSON records the source SHA and dirty state, Python and package versions,
@@ -154,8 +157,8 @@ runner doesn't commit, push or enable benchmarks in CI.
`python docs/build.py` reads and validates the checked-in `bench/data/latest.json`.
It generates responsive light/dark SVG charts and fills
-`docs/templates/benchmark.md` with body-size controls and expandable tables.
-The built site includes a frozen raw JSON download. The build won't fetch
+`docs/templates/benchmark.md` with body-size controls and measurement details.
+The built site includes a frozen raw JSON copy. The build won't fetch
measurements or start a benchmark, and missing or invalid data stops it.
Read the Docs uses this same entry point.
diff --git a/bench/report.py b/bench/report.py
index 8c464025..81966318 100644
--- a/bench/report.py
+++ b/bench/report.py
@@ -87,7 +87,12 @@ def safe_data_link(link: str) -> str:
return link
-def render_markdown(document: dict, data_link: str | None = None) -> str:
+def render_markdown(
+ document: dict,
+ data_link: str | None = None,
+ *,
+ environment_only: bool = False,
+) -> str:
validate_document(document)
config = document["configuration"]
source = document["source"]
@@ -128,13 +133,19 @@ def render_markdown(document: dict, data_link: str | None = None) -> str:
f"requests/batch: {config['requests']}; warm-up batches/round: {config['warmup']} |",
"| Stream upload chunk | "
+ (
- "Varies by payload; see body cases below"
+ (
+ "Varies by payload; recorded in raw JSON"
+ if environment_only
+ else "Varies by payload; see body cases below"
+ )
if "stream_chunk_bytes_by_payload" in config
else payload_label(config["stream_chunk_bytes"])
)
+ " |",
"",
]
+ if environment_only:
+ return "\n".join(lines)
if "stream_chunk_bytes_by_payload" in config:
lines += [
"### Body cases",
@@ -168,27 +179,6 @@ def render_markdown(document: dict, data_link: str | None = None) -> str:
f"{', '.join('HTTP/1.1' if p == 'h1' else 'HTTP/2' for p in capability['protocols'])} | "
f"{', '.join(k.title() for k in capability['body_kinds'])} |"
)
- if any(
- "response_read" in document["clients"][client] for client in ordered_clients
- ):
- lines += [
- "",
- '',
- "Adapter settings",
- "",
- "| Client | Response consumption | Connection pool |",
- "| --- | --- | --- |",
- ]
- for client in ordered_clients:
- metadata = document["clients"][client]
- pool = json.dumps(
- metadata.get("pool", {"kind": "not recorded"}), sort_keys=True
- )
- lines.append(
- f"| {escape(LABELS.get(client, client))} | "
- f"{escape(metadata.get('response_read', 'Not recorded'))} | {escape(pool)} |"
- )
- lines += ["", ""]
lines += [
"",
"### Throughput comparison",
@@ -196,7 +186,9 @@ def render_markdown(document: dict, data_link: str | None = None) -> str:
"Each value is total measured requests divided by total measured time "
"across all rounds, in requests per second (RPS). All timed samples count. "
"The raw JSON keeps individual timings, per-round results, response-payload "
- "MB/s and native artifact SHA-256 hashes.",
+ "MB/s and native artifact SHA-256 hashes. Read chunking, buffering and "
+ "connection pools differ between adapters; their configurations are "
+ "recorded in the raw JSON.",
"",
]
results = {
diff --git a/bench/test_report.py b/bench/test_report.py
index 6db7ad1f..3b6e95f6 100644
--- a/bench/test_report.py
+++ b/bench/test_report.py
@@ -55,9 +55,9 @@ def test_report_aggregation_and_metadata_escaping():
assert "`x`" not in text and "|" in text
assert report.escape("CPU's name") == "CPU's name"
assert "javascript:source-repository" not in text
+ assert "Adapter settings" not in text and "' in text
- and "Adapter settings" in text
+ "Read chunking, buffering and connection pools differ between adapters" in text
)
assert f"{report.REPOSITORY}/commit/{document['source']['commit']}" in text
for link in (
@@ -250,11 +250,21 @@ def no_network(*args, **kwargs):
snapshot = root / "docs/source/assets/benchmark/latest.json"
page = root / "docs/source/benchmark.md"
assert json.loads(snapshot.read_text(encoding="utf-8")) == document
- assert "{{BENCHMARK_RESULTS}}" not in page.read_text(encoding="utf-8")
- assert "{{BENCHMARK_CHARTS}}" not in page.read_text(encoding="utf-8")
+ content = page.read_text(encoding="utf-8")
+ assert "{{BENCHMARK_RESULTS}}" not in content
+ assert "{{BENCHMARK_CHARTS}}" not in content
+ assert "" not in content
+ assert report.render_markdown(document, environment_only=True) in content
+ assert "### Throughput comparison" not in content
+ assert "| Upload / echo payload | Concurrency" not in content
+ assert "### Measurement environment" in content
+ assert "### Client versions and runtimes" not in content
+ assert "### Body cases" not in content
+ assert "## Runtime and client differences" in content
charts = root / "docs/source/assets/benchmark/charts"
assert len(list(charts.glob("*.svg"))) == 16
- assert "assets/benchmark/latest.json" in page.read_text(encoding="utf-8")
+ assert "Download this build's raw JSON" not in content
+ assert "Raw JSON" not in content and "Download SVG" not in content
assert build.prepare(data=snapshot, root=root) == document
override = root / "override.json"
alternate = make_document(commit="d" * 40)
@@ -318,8 +328,17 @@ def test_chart_explorer_controls_fallback_and_inert_metadata(tmp_path):
assert 'value="h1" selected' in content and 'value="h2"' not in content
assert content.count("data-chart-payload=") == 1
assert content.count("data-chart-controls hidden") == 2
- assert "