From 57fef293a3b75b7163f4e7658f5389aab85c87a9 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:53:17 -0600 Subject: [PATCH 1/2] Add opt-in request-shape warmup to the DeepSeek launchers The DeepSeek-V4 pair and cycle launchers began serving without exercising request shapes the engine compiles on first use, so a cold compiler cache compiled known shapes during client traffic while ranks waited in a collective. Rank 0 now runs scripts/deepseek_v4_shape_warmup.py after /health and before the launcher returns: a concurrency ladder, medium and multi-chunk prefills, a thinking-off arm, and an exact token ladder verified through /tokenize, each request tagged with a leading nonce. MAX_NUM_SEQS, NUM_SPECULATIVE_TOKENS, BLOCK_SIZE, MAX_NUM_BATCHED_TOKENS and MAX_MODEL_LEN come from the rank env file, and SHAPE_WARMUP_API_KEY_FILE reuses the cycle controller's key-file rule for authenticated clusters. The bucket rule comes from measurements on a different serving image, so SHAPE_WARMUP defaults to 0 and the quickstart documents the cold-cache calibration procedure. A warmup failure logs a warning and never fails the launch. Validation: pytest scripts/test_deepseek_v4_shape_warmup.py scripts/test_deepseek_v4_pair_launcher.py runtime/host scripts/test_sparkring_cli.py -q (2189 passed, 86 skipped); ruff --select E,F,W --ignore E501; check_repository_layout.py; check_markdown_links.py. No hardware run: the sweep and its bucket mapping remain unverified on the served image. --- docs/operations/deepseek-0731.md | 111 ++- .../runtime.env.template | 8 + .../runtime.env.template | 8 + scripts/config/README.md | 10 + .../deepseek-v4-flash-0731-pair.env.example | 8 + .../config/deepseek-v4-flash-0731.env.example | 8 + scripts/deepseek_v4_cycle_serve.sh | 48 +- scripts/deepseek_v4_pair_serve.sh | 51 +- scripts/deepseek_v4_shape_warmup.py | 691 ++++++++++++++++++ scripts/test_deepseek_v4_pair_launcher.py | 173 +++++ scripts/test_deepseek_v4_shape_warmup.py | 539 ++++++++++++++ 11 files changed, 1647 insertions(+), 8 deletions(-) create mode 100644 scripts/deepseek_v4_shape_warmup.py create mode 100644 scripts/test_deepseek_v4_shape_warmup.py diff --git a/docs/operations/deepseek-0731.md b/docs/operations/deepseek-0731.md index b0292655c..eb45255e3 100644 --- a/docs/operations/deepseek-0731.md +++ b/docs/operations/deepseek-0731.md @@ -264,9 +264,11 @@ scripts/deepseek_v4_pair_serve.sh --run /path/to/rank-1.env scripts/deepseek_v4_pair_serve.sh --run /path/to/rank-0.env ``` -The pair environment exposes these operator-facing settings and uses the -defaults recorded in -[`recipes/deepseek-v4-flash-0731-pair.json`](../../recipes/deepseek-v4-flash-0731-pair.json): +The pair environment exposes these operator-facing settings. Serving defaults +match +[`recipes/deepseek-v4-flash-0731-pair.json`](../../recipes/deepseek-v4-flash-0731-pair.json); +the warmup settings default to the values below and are optional in the rank +env file: | Variable | Default | Purpose | |---|---:|---| @@ -279,6 +281,11 @@ defaults recorded in | `MAX_MODEL_LEN` | 1048576 | Per-request token limit | | `MAX_NUM_SEQS` | 32 | Scheduler admission ceiling | | `MAX_NUM_BATCHED_TOKENS` | 4096 | Scheduler budget and chunked-prefill size | +| `BLOCK_SIZE` | 256 | Key-value block size passed to the engine and used by the warmup sweep; change it only with a re-validated KV layout | +| `SHAPE_WARMUP` | 0 | Run the [request-shape warmup](#26-request-shape-warmup) on rank 0 | +| `SHAPE_WARMUP_WAIT_SECONDS` | 900 | Readiness budget for the warmup phase | +| `SHAPE_WARMUP_TIMEOUT_SECONDS` | 600 | Sweep budget after readiness | +| `SHAPE_WARMUP_API_KEY_FILE` | empty | Rank-0 file whose first non-whitespace line is the warmup bearer | The checkpoint's `dspark_block_size=5` is the minimum admitted draft depth. The [pinned runtime](https://github.com/local-inference-lab/vllm/blob/e2666d9a65f41fc376607531453cbd57c4c71016/vllm/config/speculative.py) @@ -295,8 +302,9 @@ and inside the running container when diagnosing shared-memory pressure. The cycle launcher consumes one resolved `deepseek-v4-flash-0731.env.example` copy per rank and exposes the same model, -cache, port, speculation, context, sequence, and scheduler settings as the pair -launcher. Validate all four ranks and compare their printed serving values: +cache, port, speculation, context, sequence, scheduler, block-size, and warmup +settings as the pair launcher. Validate all four ranks and compare their +printed serving values: ```bash scripts/deepseek_v4_cycle_serve.sh --check /path/to/rank-0.env @@ -364,6 +372,99 @@ arguments. Containers are removed with `docker rm -f`; there is no persistent service state. This controller only orchestrates the existing per-rank launcher and does not change the launch contract. +## 2.6 Request-shape warmup + +Status: **uncalibrated, disabled by default.** The sweep walks request shapes a +DeepSeek-V4-Flash deployment can schedule, so a cold compiler cache compiles +them during boot rather than during client traffic that holds ranks in a +collective. It does not establish that the covered shapes match the kernels +this image compiles; [Calibrating the sweep](#calibrating-the-sweep) does. + +With `SHAPE_WARMUP=1` in rank 0's env file, `--run` starts the container and then +runs [`scripts/deepseek_v4_shape_warmup.py`](../../scripts/deepseek_v4_shape_warmup.py) +in the launcher's foreground. The phase waits for `/health`, then fires these +arms in order: + +| Arm | Requests | Shape | +|---|---:|---| +| `concurrency-1` … `concurrency-MAX_NUM_SEQS` | 1, 2, 4, … then the sequence limit | Chat requests fired together | +| `prefill-medium` | 1 | Completion prompt of half `MAX_NUM_BATCHED_TOKENS` tokens | +| `prefill-multi-chunk` | 1 | Completion prompt of 8192 + `MAX_NUM_BATCHED_TOKENS` tokens, longer than the 8192-token boundary | +| `thinking-off` | 1 | Chat request with `chat_template_kwargs {"enable_thinking": false}` | +| `tokens-…` | 1 per bucket | Completion prompts sized so `min(BLOCK_SIZE, next_pow2(tokens + NUM_SPECULATIVE_TOKENS + 1))` reaches each power-of-two bucket through `BLOCK_SIZE` | + +Every request starts with a unique nonce, so prefix caching cannot satisfy the +prefill from an earlier request. Chat arms report the rendered prompt length +that `/tokenize` returns. Completion arms search filler length through +`/tokenize` until the prompt has exactly the bucket's token count; a target the +nonce alone exceeds is listed under `skipped` in the summary instead of being +approximated. `MAX_NUM_SEQS`, `BLOCK_SIZE`, `NUM_SPECULATIVE_TOKENS`, +`MAX_NUM_BATCHED_TOKENS` and `MAX_MODEL_LEN` come from the same rank env file +the container receives, and the module has no defaults for them. + +The phase is non-fatal. A failed request, an exhausted budget, a missing +`python3` or an unreadable key file logs a warning and the container keeps +serving; only the container command itself fails the launch. The launcher's +standard output carries one `shape_warmup` JSON summary with each arm's +completed and failed request counts, measured prompt lengths, compile buckets +and every skip. The pinned server serves `/v1/completions` for the +completion arms; a server that rejects it reports those arms as failures rather +than silently skipping them. + +On a cluster that enforces bearer authentication, set +`SHAPE_WARMUP_API_KEY_FILE` to the rank-0 file the +`deepseek_v4_cycle_ctl.py --api-key-file` probe reads. The warmup sends the +file's first non-whitespace line as `Authorization: Bearer`; `/health` is +probed without it. `cycle_ctl start` launches rank 0's launcher in the +background, so its API-ready line can precede the sweep. Before sending client +traffic, wait for the `shape_warmup` summary in +`$LOG_DIR/ctl-start-rank0.log`. + +### Calibrating the sweep + +Run this procedure on each image and topology before setting `SHAPE_WARMUP=1` +in a deployment. Measurements on an earlier DeepSeek-V4-Flash serving image +found the DSpark input-preparation kernel keyed by the token-count rule above +rather than by request concurrency; those measurements used a different +serving image than this repository pins, so the rule and the arm list both +need confirmation. + +1. Stop the deployment with the recovery procedure. Keep the model and + flight-recorder data, and move `CACHE_HOST_PATH/jit` aside so the next + launch starts with a cold compiler cache. +2. Render the exact rank commands with `--check` and add + `--jit-monitor-verbose` to the engine arguments on every rank. Run those + rendered commands instead of `--run`, starting the workers and then rank 0, + with `SHAPE_WARMUP=0`. Replay the intended client mix and record every + kernel compilation the rank logs report. This set is the target the sweep + must cover. +3. Stop the deployment and move `CACHE_HOST_PATH/jit` aside again. +4. Launch the same way, then run the sweep by hand against the ready API: + + ```bash + python3 scripts/deepseek_v4_shape_warmup.py \ + --endpoint http://127.0.0.1:8000 --model deepseek-v4-flash-0731 \ + --max-num-seqs 32 --block-size 256 --num-speculative-tokens 5 \ + --max-num-batched-tokens 4096 --max-model-len 1048576 + ``` + + Add `--api-key-file` when the API requires a bearer. The command prints its + summary and exits zero only when no arm failed. +5. Replay the same client mix. Passing evidence is that none of the kernels + recorded in step 2 compiles after the sweep. A kernel that still compiles + names a bucket the sweep does not cover; record the arm-to-kernel mapping + and extend the sweep before claiming coverage. Kernels this profile has + compiled during serving include `_prepare_dflash_inputs_kernel`, + `_compute_global_topk_indices_and_lens_kernel` and the B12X CuTeDSL MoE + kernels. The summary's `skipped` entries state which arms the nonce or + request limit excluded. +6. Compare the same request mix with the phase disabled and enabled: TP rank + disconnects and restarts, and first-request time to first token. Record + conditions, measurements and the conclusion with the deployment record. +7. Set `SHAPE_WARMUP=1`, `SHAPE_WARMUP_WAIT_SECONDS` and + `SHAPE_WARMUP_TIMEOUT_SECONDS` in every rank env file only after step 5 + passes for that image and topology. + ## 3. Sizing: four coupled parameters, and one inactive guard `--max-model-len`, `--kv-cache-memory-bytes`, `--max-num-seqs` and diff --git a/profiles/deepseek-v4-flash-0731-pair/runtime.env.template b/profiles/deepseek-v4-flash-0731-pair/runtime.env.template index 11396880b..7760eae40 100644 --- a/profiles/deepseek-v4-flash-0731-pair/runtime.env.template +++ b/profiles/deepseek-v4-flash-0731-pair/runtime.env.template @@ -63,6 +63,14 @@ MAX_MODEL_LEN={{max_model_len}} MAX_NUM_SEQS={{max_num_seqs}} MAX_NUM_BATCHED_TOKENS={{max_num_batched_tokens}} +# Request-shape warmup runs on rank 0 after the API reports ready when +# SHAPE_WARMUP=1 is present in this file. It is disabled until the sweep is +# calibrated on the deployed image; read +# docs/operations/deepseek-0731.md#26-request-shape-warmup first. +# SHAPE_WARMUP_WAIT_SECONDS (900) and SHAPE_WARMUP_TIMEOUT_SECONDS (600) bound +# readiness and the sweep, and SHAPE_WARMUP_API_KEY_FILE names a rank-0 bearer +# file. BLOCK_SIZE overrides the 256-token block size passed to the engine. + # The launcher uses --ipc host. The host's /dev/shm size is authoritative; # changing a container --shm-size declaration does not enlarge host shared # memory. Check `df -h /dev/shm` on the host when diagnosing IPC pressure. diff --git a/profiles/deepseek-v4-flash-0731/runtime.env.template b/profiles/deepseek-v4-flash-0731/runtime.env.template index 8ac7999d9..5e4ba9514 100644 --- a/profiles/deepseek-v4-flash-0731/runtime.env.template +++ b/profiles/deepseek-v4-flash-0731/runtime.env.template @@ -54,6 +54,14 @@ MAX_MODEL_LEN={{max_model_len}} MAX_NUM_SEQS={{max_num_seqs}} MAX_NUM_BATCHED_TOKENS={{max_num_batched_tokens}} +# Request-shape warmup runs on rank 0 after the API reports ready when +# SHAPE_WARMUP=1 is present in this file. It is disabled until the sweep is +# calibrated on the deployed image; read +# docs/operations/deepseek-0731.md#26-request-shape-warmup first. +# SHAPE_WARMUP_WAIT_SECONDS (900) and SHAPE_WARMUP_TIMEOUT_SECONDS (600) bound +# readiness and the sweep, and SHAPE_WARMUP_API_KEY_FILE names a rank-0 bearer +# file. BLOCK_SIZE overrides the 256-token block size passed to the engine. + # SERVED_MODEL_NAME is the model name vLLM advertises and accepts in API # requests. The default reproduces recipes/deepseek-v4-flash-0731.json; # override it only when clients must address the service under another name. diff --git a/scripts/config/README.md b/scripts/config/README.md index 10045cabf..1ae9a6e24 100644 --- a/scripts/config/README.md +++ b/scripts/config/README.md @@ -163,6 +163,16 @@ The template's immutable image digest is also represented in [`runtime/faststart-lock.json`](../../runtime/faststart-lock.json). If those inputs differ, stop and resolve the identity drift before launching. +Both DeepSeek launchers read optional settings from the rank env file that the +templates do not assign by default: `BLOCK_SIZE` (default 256, the recipe's +value), `SHAPE_WARMUP` (default 0), `SHAPE_WARMUP_WAIT_SECONDS`, +`SHAPE_WARMUP_TIMEOUT_SECONDS` and `SHAPE_WARMUP_API_KEY_FILE`. With +`SHAPE_WARMUP=1`, rank 0 runs `scripts/deepseek_v4_shape_warmup.py` after +readiness and before the launcher returns. The phase is non-fatal, and the +sweep remains uncalibrated until the +[calibration procedure](../../docs/operations/deepseek-0731.md#calibrating-the-sweep) +passes on the deployment. + ## DeepSeek-V4.1-Flash Status: **implemented**. `deepseek-v41-flash-cycle.env.example` selects the diff --git a/scripts/config/deepseek-v4-flash-0731-pair.env.example b/scripts/config/deepseek-v4-flash-0731-pair.env.example index 0034de99f..b8a90f585 100644 --- a/scripts/config/deepseek-v4-flash-0731-pair.env.example +++ b/scripts/config/deepseek-v4-flash-0731-pair.env.example @@ -63,6 +63,14 @@ MAX_MODEL_LEN=1048576 MAX_NUM_SEQS=32 MAX_NUM_BATCHED_TOKENS=4096 +# Request-shape warmup runs on rank 0 after the API reports ready when +# SHAPE_WARMUP=1 is present in this file. It is disabled until the sweep is +# calibrated on the deployed image; read +# docs/operations/deepseek-0731.md#26-request-shape-warmup first. +# SHAPE_WARMUP_WAIT_SECONDS (900) and SHAPE_WARMUP_TIMEOUT_SECONDS (600) bound +# readiness and the sweep, and SHAPE_WARMUP_API_KEY_FILE names a rank-0 bearer +# file. BLOCK_SIZE overrides the 256-token block size passed to the engine. + # The launcher uses --ipc host. The host's /dev/shm size is authoritative; # changing a container --shm-size declaration does not enlarge host shared # memory. Check `df -h /dev/shm` on the host when diagnosing IPC pressure. diff --git a/scripts/config/deepseek-v4-flash-0731.env.example b/scripts/config/deepseek-v4-flash-0731.env.example index 62a3358ee..76cbf69e3 100644 --- a/scripts/config/deepseek-v4-flash-0731.env.example +++ b/scripts/config/deepseek-v4-flash-0731.env.example @@ -54,6 +54,14 @@ MAX_MODEL_LEN=1048576 MAX_NUM_SEQS=32 MAX_NUM_BATCHED_TOKENS=4096 +# Request-shape warmup runs on rank 0 after the API reports ready when +# SHAPE_WARMUP=1 is present in this file. It is disabled until the sweep is +# calibrated on the deployed image; read +# docs/operations/deepseek-0731.md#26-request-shape-warmup first. +# SHAPE_WARMUP_WAIT_SECONDS (900) and SHAPE_WARMUP_TIMEOUT_SECONDS (600) bound +# readiness and the sweep, and SHAPE_WARMUP_API_KEY_FILE names a rank-0 bearer +# file. BLOCK_SIZE overrides the 256-token block size passed to the engine. + # SERVED_MODEL_NAME is the model name vLLM advertises and accepts in API # requests. The default reproduces recipes/deepseek-v4-flash-0731.json; # override it only when clients must address the service under another name. diff --git a/scripts/deepseek_v4_cycle_serve.sh b/scripts/deepseek_v4_cycle_serve.sh index 7aa5e035a..1b09888a3 100755 --- a/scripts/deepseek_v4_cycle_serve.sh +++ b/scripts/deepseek_v4_cycle_serve.sh @@ -105,10 +105,31 @@ for name in NUM_SPECULATIVE_TOKENS MAX_MODEL_LEN MAX_NUM_SEQS \ MAX_NUM_BATCHED_TOKENS; do require_positive_integer "$name" done +BLOCK_SIZE=${BLOCK_SIZE:-256} +require_positive_integer BLOCK_SIZE require_port API_PORT require_port MASTER_PORT [ "$((10#$API_PORT))" -ne "$((10#$MASTER_PORT))" ] || die "API_PORT and MASTER_PORT must differ" +# Request-shape warmup is opt-in until the sweep is calibrated against this +# image; see docs/operations/deepseek-0731.md#26-request-shape-warmup. +SHAPE_WARMUP=${SHAPE_WARMUP:-0} +case "$SHAPE_WARMUP" in + 0|1) ;; + *) die "SHAPE_WARMUP must be 0 or 1: $SHAPE_WARMUP" ;; +esac +SHAPE_WARMUP_WAIT_SECONDS=${SHAPE_WARMUP_WAIT_SECONDS:-900} +SHAPE_WARMUP_TIMEOUT_SECONDS=${SHAPE_WARMUP_TIMEOUT_SECONDS:-600} +if [ "$SHAPE_WARMUP" = 1 ]; then + for name in SHAPE_WARMUP_WAIT_SECONDS SHAPE_WARMUP_TIMEOUT_SECONDS; do + require_positive_integer "$name" + done + if [ -n "${SHAPE_WARMUP_API_KEY_FILE:-}" ] \ + && [ ! -r "$SHAPE_WARMUP_API_KEY_FILE" ]; then + die "SHAPE_WARMUP_API_KEY_FILE is not readable: $SHAPE_WARMUP_API_KEY_FILE" + fi +fi + [ "$NCCL_SOCKET_IFNAME" = "$GLOO_SOCKET_IFNAME" ] \ || die "NCCL_SOCKET_IFNAME and GLOO_SOCKET_IFNAME must match" [ "$NCCL_NET" = IB ] || die "NCCL_NET must be IB" @@ -222,7 +243,7 @@ command=( --gpu-memory-utilization 0.70 --kv-cache-memory-bytes 17179869184 --kv-cache-dtype fp8_ds_mla - --block-size 256 + --block-size "$BLOCK_SIZE" --tokenizer-mode deepseek_v4 --kernel-config '{"enable_cutedsl_warmup":false}' --enable-auto-tool-choice @@ -246,6 +267,8 @@ printf ' MAX_MODEL_LEN: %s\n' "$MAX_MODEL_LEN" printf ' MAX_NUM_SEQS: %s\n' "$MAX_NUM_SEQS" printf ' MAX_NUM_BATCHED_TOKENS: %s\n' "$MAX_NUM_BATCHED_TOKENS" printf ' NUM_SPECULATIVE_TOKENS: %s\n' "$NUM_SPECULATIVE_TOKENS" +printf ' BLOCK_SIZE: %s\n' "$BLOCK_SIZE" +printf ' SHAPE_WARMUP: %s\n' "$SHAPE_WARMUP" printf ' command:' printf ' %q' "${command[@]}" printf '\n' @@ -272,3 +295,26 @@ for cache_directory in \ done # Keep the shell alive so its EXIT trap removes the filtered env-file. "${command[@]}" +if [ "$NODE_RANK" = 0 ] && [ "$SHAPE_WARMUP" = 1 ]; then + shape_warmup_args=( + --endpoint "http://127.0.0.1:$API_PORT" + --model "$served_model_name" + --max-num-seqs "$MAX_NUM_SEQS" + --block-size "$BLOCK_SIZE" + --num-speculative-tokens "$NUM_SPECULATIVE_TOKENS" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --max-model-len "$MAX_MODEL_LEN" + --wait-seconds "$SHAPE_WARMUP_WAIT_SECONDS" + --timeout-seconds "$SHAPE_WARMUP_TIMEOUT_SECONDS" + ) + if [ -n "${SHAPE_WARMUP_API_KEY_FILE:-}" ]; then + shape_warmup_args+=(--api-key-file "$SHAPE_WARMUP_API_KEY_FILE") + fi + printf 'Shape warmup: waiting for rank 0 health, then sweeping request shapes.\n' + # The phase is non-fatal: a failure keeps the boot and only logs a warning. + if ! command -v python3 >/dev/null 2>&1; then + echo "deepseek cycle launcher: warning: python3 is unavailable; request-shape warmup skipped" >&2 + elif ! python3 "$script_dir/deepseek_v4_shape_warmup.py" "${shape_warmup_args[@]}"; then + echo "deepseek cycle launcher: warning: request-shape warmup did not complete; continuing to serve" >&2 + fi +fi diff --git a/scripts/deepseek_v4_pair_serve.sh b/scripts/deepseek_v4_pair_serve.sh index f2c910975..2a4acb230 100755 --- a/scripts/deepseek_v4_pair_serve.sh +++ b/scripts/deepseek_v4_pair_serve.sh @@ -97,10 +97,31 @@ for name in NUM_SPECULATIVE_TOKENS MAX_MODEL_LEN MAX_NUM_SEQS \ MAX_NUM_BATCHED_TOKENS; do require_positive_integer "$name" done +BLOCK_SIZE=${BLOCK_SIZE:-256} +require_positive_integer BLOCK_SIZE require_port API_PORT require_port MASTER_PORT [ "$((10#$API_PORT))" -ne "$((10#$MASTER_PORT))" ] || die "API_PORT and MASTER_PORT must differ" +# Request-shape warmup is opt-in until the sweep is calibrated against this +# image; see docs/operations/deepseek-0731.md#26-request-shape-warmup. +SHAPE_WARMUP=${SHAPE_WARMUP:-0} +case "$SHAPE_WARMUP" in + 0|1) ;; + *) die "SHAPE_WARMUP must be 0 or 1: $SHAPE_WARMUP" ;; +esac +SHAPE_WARMUP_WAIT_SECONDS=${SHAPE_WARMUP_WAIT_SECONDS:-900} +SHAPE_WARMUP_TIMEOUT_SECONDS=${SHAPE_WARMUP_TIMEOUT_SECONDS:-600} +if [ "$SHAPE_WARMUP" = 1 ]; then + for name in SHAPE_WARMUP_WAIT_SECONDS SHAPE_WARMUP_TIMEOUT_SECONDS; do + require_positive_integer "$name" + done + if [ -n "${SHAPE_WARMUP_API_KEY_FILE:-}" ] \ + && [ ! -r "$SHAPE_WARMUP_API_KEY_FILE" ]; then + die "SHAPE_WARMUP_API_KEY_FILE is not readable: $SHAPE_WARMUP_API_KEY_FILE" + fi +fi + [ "$NCCL_SOCKET_IFNAME" = "$GLOO_SOCKET_IFNAME" ] \ || die "NCCL_SOCKET_IFNAME and GLOO_SOCKET_IFNAME must match on a pair" [ "$NCCL_NET" = IB ] || die "NCCL_NET must be IB" @@ -140,6 +161,7 @@ fi image=ghcr.io/fujitsupolycom/gb10-vllm-serving@sha256:827a8e8c5749b78529cc0015dd174e1b19a0accc116bc142282f8b75428f98bd container_name="deepseek-v4-flash-r$NODE_RANK" model_container_path=/models/deepseek-v4-flash-0731 +served_model_name=deepseek-v4-flash-0731 speculative_config=$(printf \ '{"method":"dspark","num_speculative_tokens":%s,"moe_backend":"b12x"}' \ "$NUM_SPECULATIVE_TOKENS") @@ -178,13 +200,13 @@ command=( --gpu-memory-utilization 0.70 --kv-cache-memory-bytes 17179869184 --kv-cache-dtype fp8_ds_mla - --block-size 256 + --block-size "$BLOCK_SIZE" --tokenizer-mode deepseek_v4 --kernel-config '{"enable_cutedsl_warmup":false}' --enable-auto-tool-choice --tool-call-parser deepseek_v4 --speculative-config "$speculative_config" - --served-model-name deepseek-v4-flash-0731 + --served-model-name "$served_model_name" ) if [ "$NODE_RANK" = 0 ]; then @@ -201,6 +223,8 @@ printf ' MAX_MODEL_LEN: %s\n' "$MAX_MODEL_LEN" printf ' MAX_NUM_SEQS: %s\n' "$MAX_NUM_SEQS" printf ' MAX_NUM_BATCHED_TOKENS: %s\n' "$MAX_NUM_BATCHED_TOKENS" printf ' NUM_SPECULATIVE_TOKENS: %s\n' "$NUM_SPECULATIVE_TOKENS" +printf ' BLOCK_SIZE: %s\n' "$BLOCK_SIZE" +printf ' SHAPE_WARMUP: %s\n' "$SHAPE_WARMUP" printf ' command:' printf ' %q' "${command[@]}" printf '\n' @@ -227,3 +251,26 @@ for cache_directory in \ done # Keep the shell alive so its EXIT trap removes the filtered env-file. "${command[@]}" +if [ "$NODE_RANK" = 0 ] && [ "$SHAPE_WARMUP" = 1 ]; then + shape_warmup_args=( + --endpoint "http://127.0.0.1:$API_PORT" + --model "$served_model_name" + --max-num-seqs "$MAX_NUM_SEQS" + --block-size "$BLOCK_SIZE" + --num-speculative-tokens "$NUM_SPECULATIVE_TOKENS" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --max-model-len "$MAX_MODEL_LEN" + --wait-seconds "$SHAPE_WARMUP_WAIT_SECONDS" + --timeout-seconds "$SHAPE_WARMUP_TIMEOUT_SECONDS" + ) + if [ -n "${SHAPE_WARMUP_API_KEY_FILE:-}" ]; then + shape_warmup_args+=(--api-key-file "$SHAPE_WARMUP_API_KEY_FILE") + fi + printf 'Shape warmup: waiting for rank 0 health, then sweeping request shapes.\n' + # The phase is non-fatal: a failure keeps the boot and only logs a warning. + if ! command -v python3 >/dev/null 2>&1; then + echo "deepseek pair launcher: warning: python3 is unavailable; request-shape warmup skipped" >&2 + elif ! python3 "$script_dir/deepseek_v4_shape_warmup.py" "${shape_warmup_args[@]}"; then + echo "deepseek pair launcher: warning: request-shape warmup did not complete; continuing to serve" >&2 + fi +fi diff --git a/scripts/deepseek_v4_shape_warmup.py b/scripts/deepseek_v4_shape_warmup.py new file mode 100644 index 000000000..d414222cf --- /dev/null +++ b/scripts/deepseek_v4_shape_warmup.py @@ -0,0 +1,691 @@ +#!/usr/bin/env python3 +"""Exercise the DeepSeek-V4 request shapes that the engine compiles at first use. + +A shape first reached by client traffic compiles while every rank waits in a +collective, and the distributed watchdog can terminate the waiting rank. This +program walks the shape buckets a DeepSeek-V4-Flash deployment can schedule +after its API reports ready and before the launcher hands the endpoint to +clients. + +The bucket rule is provisional. Measurements on an earlier DeepSeek-V4-Flash +serving image found the DSpark input-preparation kernel keyed by +``min(block_size, next_pow2(scheduled_tokens + num_speculative_tokens + 1))``, +not by request concurrency; those measurements used a different serving image +than this repository pins. The sweep covers that rule, and the calibration +procedure in ``docs/operations/deepseek-0731.md`` establishes the actual keys +on a deployment before it enables the phase by default. + +Every request starts with a unique nonce so prefix caching cannot serve a +cached prefill in place of the shape being warmed. Chat requests carry the +nonce in the user message. Prefill and token-ladder requests use the +completion endpoint, whose prompt has no chat-template overhead, so their +prompt length can land exactly on a compile bucket. + +The phase is non-fatal: failed requests are recorded, the remaining arms still +run, and the exit status reports whether any arm failed. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import json +import math +import secrets +import sys +import threading +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + +# Recorded pair serving completed requests through 8,193 input tokens. A +# multi-chunk arm longer than this boundary exercises the second prefill chunk +# at the deployed `--max-num-batched-tokens`. +MULTI_CHUNK_BOUNDARY_TOKENS = 8192 + +THINKING_OFF = {"enable_thinking": False} +CHAT_KIND = "chat" +COMPLETION_KIND = "completion" + + +class WarmupError(RuntimeError): + """A warmup step failed; the caller records it and continues.""" + + +class UnreachableTarget(WarmupError): + """The nonce alone is longer than an arm's exact prompt target.""" + + +@dataclass(frozen=True) +class WarmupConfig: + """Deployment values the sweep must match, all taken from the rank env.""" + + endpoint: str + model: str + max_num_seqs: int + block_size: int + num_speculative_tokens: int + max_num_batched_tokens: int + max_model_len: int + bearer: str | None = None + + @property + def max_tokens(self) -> int: + """One draft round: the speculative depth plus its verification token.""" + return self.num_speculative_tokens + 1 + + @property + def max_prompt_tokens(self) -> int: + return self.max_model_len - self.max_tokens + + +@dataclass(frozen=True) +class WarmupArm: + """One batch of requests at one endpoint and, for completions, one target.""" + + label: str + batch: int + kind: str + prompt_tokens: int | None = None + compile_bucket: int | None = None + thinking_off: bool = False + + +def _positive(value: int, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise WarmupError(f"{name} must be a positive integer: {value!r}") + return value + + +def next_power_of_two(value: int) -> int: + _positive(value, "value") + return 1 << (value - 1).bit_length() + + +def compile_bucket(prompt_tokens: int, config: WarmupConfig) -> int: + """Return the provisional DSpark input-preparation compile bucket.""" + _positive(prompt_tokens, "prompt_tokens") + return min( + config.block_size, + next_power_of_two(prompt_tokens + config.num_speculative_tokens + 1), + ) + + +def concurrency_ladder(max_num_seqs: int) -> tuple[int, ...]: + """Return powers of two up to the sequence limit, then the limit itself.""" + _positive(max_num_seqs, "max_num_seqs") + rungs: list[int] = [] + rung = 1 + while rung < max_num_seqs: + rungs.append(rung) + rung *= 2 + rungs.append(max_num_seqs) + return tuple(rungs) + + +def bucket_targets(config: WarmupConfig) -> tuple[tuple[int, int], ...]: + """Return (compile bucket, exact prompt tokens) through the block size.""" + buckets = [] + bucket = 1 + while bucket < config.block_size: + buckets.append(bucket) + bucket *= 2 + buckets.append(config.block_size) + targets = [] + for bucket in buckets: + tokens = bucket - (config.num_speculative_tokens + 1) + if tokens < 1: + continue + if compile_bucket(tokens, config) == bucket: + targets.append((bucket, tokens)) + return tuple(targets) + + +def build_sweep( + config: WarmupConfig, +) -> tuple[tuple[WarmupArm, ...], tuple[str, ...]]: + """Return the deterministic sweep and the labels the model length excludes.""" + _positive(config.max_model_len, "max_model_len") + limit = config.max_prompt_tokens + if limit < 1: + raise WarmupError( + "max_model_len leaves no room for a warmup prompt: " + f"{config.max_model_len} tokens with {config.max_tokens} generated" + ) + ladder = bucket_targets(config) + if not ladder: + raise WarmupError( + "block_size and speculative depth produce no warmup bucket: " + f"block_size={config.block_size} " + f"num_speculative_tokens={config.num_speculative_tokens}" + ) + arms: list[WarmupArm] = [] + skipped: list[str] = [] + for concurrency in concurrency_ladder(config.max_num_seqs): + arms.append( + WarmupArm( + label=f"concurrency-{concurrency}", + batch=concurrency, + kind=CHAT_KIND, + ) + ) + for label, target in ( + ("prefill-medium", config.max_num_batched_tokens // 2), + ( + "prefill-multi-chunk", + MULTI_CHUNK_BOUNDARY_TOKENS + config.max_num_batched_tokens, + ), + ): + if target > limit or target < 1: + skipped.append(f"{label}: {target} prompt tokens exceed max_model_len") + continue + arms.append( + WarmupArm( + label=label, + batch=1, + kind=COMPLETION_KIND, + prompt_tokens=target, + compile_bucket=compile_bucket(target, config), + ) + ) + arms.append( + WarmupArm(label="thinking-off", batch=1, kind=CHAT_KIND, thinking_off=True) + ) + for bucket, tokens in ladder: + label = f"tokens-{tokens}-bucket-{bucket}" + if tokens > limit: + skipped.append(f"{label}: exceeds max_model_len") + continue + arms.append( + WarmupArm( + label=label, + batch=1, + kind=COMPLETION_KIND, + prompt_tokens=tokens, + compile_bucket=bucket, + ) + ) + return tuple(arms), tuple(skipped) + + +def read_bearer(path: str) -> str: + """Return the first line that does not start with whitespace. + + The cycle controller's readiness probe reads the same file with this rule, + so one key file serves both the probe and the warmup. + """ + try: + text = Path(path).read_text(encoding="utf-8") + except OSError as error: + raise WarmupError(f"cannot read the API key file {path}: {error}") from error + for line in text.splitlines(): + if line and not line[0].isspace(): + return line.strip() + raise WarmupError(f"the API key file has no non-empty line: {path}") + + +def _remaining(deadline: float) -> float: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise WarmupError("the warmup deadline expired") + return remaining + + +def _post_json( + endpoint: str, + path: str, + payload: dict, + *, + bearer: str | None, + timeout: float, +) -> dict: + headers = {"Content-Type": "application/json"} + if bearer: + headers["Authorization"] = f"Bearer {bearer}" + request = urllib.request.Request( + endpoint.rstrip("/") + path, + data=json.dumps(payload).encode(), + headers=headers, + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + document = json.load(response) + if not isinstance(document, dict): + raise WarmupError(f"{path} returned no JSON object") + return document + + +def wait_for_health(endpoint: str, timeout_seconds: float) -> None: + """Block until the API's /health answers 200 or the budget expires.""" + if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: + raise WarmupError("the readiness budget must be finite and positive") + deadline = time.monotonic() + timeout_seconds + url = endpoint.rstrip("/") + "/health" + while True: + try: + with urllib.request.urlopen( + url, timeout=min(5.0, max(0.1, deadline - time.monotonic())) + ) as response: + if response.status == 200: + return + except (OSError, urllib.error.URLError): + pass + remaining = deadline - time.monotonic() + if remaining <= 0: + raise WarmupError( + f"/health did not answer 200 within {timeout_seconds:g} seconds" + ) + time.sleep(min(1.0, remaining)) + + +def tokenize_chat( + config: WarmupConfig, + messages: list[dict], + *, + thinking_off: bool, + timeout: float, +) -> int: + """Return the rendered chat prompt length that the engine will schedule.""" + payload: dict = { + "model": config.model, + "messages": messages, + "add_generation_prompt": True, + } + if thinking_off: + payload["chat_template_kwargs"] = dict(THINKING_OFF) + document = _post_json( + config.endpoint, "/tokenize", payload, bearer=config.bearer, timeout=timeout + ) + return _token_count(document) + + +def tokenize_completion(config: WarmupConfig, prompt: str, *, timeout: float) -> int: + """Return the token length of a raw completion prompt.""" + document = _post_json( + config.endpoint, + "/tokenize", + {"model": config.model, "prompt": prompt}, + bearer=config.bearer, + timeout=timeout, + ) + return _token_count(document) + + +def _token_count(document: dict) -> int: + count = document.get("count") + if isinstance(count, bool) or not isinstance(count, int) or count < 1: + raise WarmupError("the tokenize response has no positive token count") + return count + + +def chat_content(nonce: str) -> str: + return f"{nonce} Reply with one word." + + +def _completion_prompt(nonce: str, repeats: int) -> str: + # The nonce leads the prompt so no token shares a cached prefix block with + # another request. No instruction follows: completion prompts need none, + # and the shorter skeleton keeps the low buckets reachable. + return f"{nonce}\n" + "warmup " * repeats + + +def resolve_completion_prompt( + config: WarmupConfig, + arm: WarmupArm, + nonce: str, + *, + timeout: float, +) -> str: + """Return a prompt whose token count is exactly the arm's target. + + Filler repetitions are searched with /tokenize; a length the nonce cannot + reach is reported separately, and a length no filler count produces fails + the arm instead of approximating the bucket. + """ + target = arm.prompt_tokens + if target is None: + raise WarmupError(f"{arm.label} has no completion target") + + def measure(repeats: int) -> int: + return tokenize_completion( + config, _completion_prompt(nonce, repeats), timeout=timeout + ) + + skeleton = measure(0) + if skeleton == target: + return _completion_prompt(nonce, 0) + if skeleton > target: + raise UnreachableTarget( + f"the nonce prompt alone uses {skeleton} tokens; target {target} " + "cannot be reached" + ) + low, high = 1, max(target, 1) + while low <= high: + middle = (low + high) // 2 + tokens = measure(middle) + if tokens == target: + return _completion_prompt(nonce, middle) + if tokens < target: + low = middle + 1 + else: + high = middle - 1 + raise WarmupError( + f"no filler length tokenizes to exactly {target} tokens; " + "the arm cannot cover its compile bucket" + ) + + +def send_chat_request( + config: WarmupConfig, + content: str, + *, + thinking_off: bool, + timeout: float, +) -> None: + payload: dict = { + "model": config.model, + "messages": [{"role": "user", "content": content}], + "temperature": 0, + "max_tokens": config.max_tokens, + } + if thinking_off: + payload["chat_template_kwargs"] = dict(THINKING_OFF) + document = _post_json( + config.endpoint, + "/v1/chat/completions", + payload, + bearer=config.bearer, + timeout=timeout, + ) + choices = document.get("choices") + if ( + not isinstance(choices, list) + or len(choices) != 1 + or document.get("error") is not None + or not isinstance(choices[0], dict) + ): + raise WarmupError("the warmup response has no single completion") + choice = choices[0] + message = choice.get("message") + if ( + choice.get("finish_reason") not in ("stop", "length") + or not isinstance(message, dict) + or not any( + isinstance(message.get(key), str) + for key in ("content", "reasoning", "reasoning_content") + ) + ): + raise WarmupError("the warmup response did not complete generation") + + +def send_completion_request( + config: WarmupConfig, + prompt: str, + *, + timeout: float, +) -> None: + document = _post_json( + config.endpoint, + "/v1/completions", + { + "model": config.model, + "prompt": prompt, + "temperature": 0, + "max_tokens": config.max_tokens, + }, + bearer=config.bearer, + timeout=timeout, + ) + choices = document.get("choices") + if ( + not isinstance(choices, list) + or len(choices) != 1 + or document.get("error") is not None + or not isinstance(choices[0], dict) + ): + raise WarmupError("the warmup response has no single completion") + choice = choices[0] + if choice.get("finish_reason") not in ("stop", "length") or not isinstance( + choice.get("text"), str + ): + raise WarmupError("the warmup response did not complete generation") + + +def make_nonce(run_id: str, arm: WarmupArm, index: int) -> str: + return f"{run_id}-{arm.label}-{index:02d}" + + +def _arm_prompt( + config: WarmupConfig, + arm: WarmupArm, + nonce: str, + deadline: float, +) -> tuple[str, int]: + """Return the request content and its verified prompt token count.""" + timeout = _remaining(deadline) + if arm.kind == COMPLETION_KIND: + prompt = resolve_completion_prompt(config, arm, nonce, timeout=timeout) + return prompt, arm.prompt_tokens or 0 + content = chat_content(nonce) + measured = tokenize_chat( + config, + [{"role": "user", "content": content}], + thinking_off=arm.thinking_off, + timeout=timeout, + ) + return content, measured + + +def _run_arm( + config: WarmupConfig, + arm: WarmupArm, + run_id: str, + deadline: float, +) -> tuple[dict, list[dict], list[str]]: + failures: list[dict] = [] + skipped: list[str] = [] + prompts: list[tuple[int, str, int]] = [] + for index in range(arm.batch): + nonce = make_nonce(run_id, arm, index) + try: + content, measured = _arm_prompt(config, arm, nonce, deadline) + except UnreachableTarget as error: + skipped.append(f"{arm.label}: {error}") + break + except Exception as error: # One failed request never aborts the sweep. + failures.append( + { + "arm": arm.label, + "request": index, + "error": f"{type(error).__name__}: {error}", + } + ) + continue + prompts.append((index, content, measured)) + started = time.perf_counter() + completed = 0 + if prompts: + barrier = threading.Barrier(len(prompts)) + + def fire(item: tuple[int, str, int]) -> str | None: + _, content, _ = item + try: + barrier.wait(timeout=_remaining(deadline)) + if arm.kind == COMPLETION_KIND: + send_completion_request(config, content, timeout=_remaining(deadline)) + else: + send_chat_request( + config, + content, + thinking_off=arm.thinking_off, + timeout=_remaining(deadline), + ) + return None + except Exception as error: # Reported as a failed request. + return f"{type(error).__name__}: {error}" + + with concurrent.futures.ThreadPoolExecutor( + max_workers=len(prompts) + ) as executor: + outcomes = list(executor.map(fire, prompts)) + for (index, _, _), error in zip(prompts, outcomes): + if error is None: + completed += 1 + else: + failures.append({"arm": arm.label, "request": index, "error": error}) + measured_tokens = sorted({item[2] for item in prompts}) + buckets = sorted({compile_bucket(tokens, config) for tokens in measured_tokens}) + return ( + { + "label": arm.label, + "kind": arm.kind, + "batch": arm.batch, + "prompt_tokens": arm.prompt_tokens, + "measured_tokens": measured_tokens, + "compile_buckets": buckets, + "thinking_off": arm.thinking_off, + "completed": completed, + "failed": len(failures), + "elapsed_seconds": round(time.perf_counter() - started, 3), + }, + failures, + skipped, + ) + + +def run_warmup( + config: WarmupConfig, + arms: Sequence[WarmupArm], + *, + wait_seconds: float, + timeout_seconds: float, + skipped: Sequence[str] = (), +) -> dict: + """Run every arm, collecting failures instead of aborting.""" + if not arms: + raise WarmupError("the sweep has no arms") + if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: + raise WarmupError("the sweep budget must be finite and positive") + summary = { + "model": config.model, + "max_num_seqs": config.max_num_seqs, + "block_size": config.block_size, + "num_speculative_tokens": config.num_speculative_tokens, + "max_num_batched_tokens": config.max_num_batched_tokens, + "max_model_len": config.max_model_len, + "max_tokens": config.max_tokens, + "authenticated": config.bearer is not None, + "arms": [], + "failures": [], + "skipped": list(skipped), + "status": "failed", + } + try: + wait_for_health(config.endpoint, wait_seconds) + except Exception as error: # A readiness failure is reported, not raised. + summary["failures"].append( + { + "arm": "readiness", + "request": 0, + "error": f"{type(error).__name__}: {error}", + } + ) + return summary + deadline = time.monotonic() + timeout_seconds + run_id = f"s{secrets.token_hex(4)}" + for arm in arms: + try: + result, failures, arm_skips = _run_arm(config, arm, run_id, deadline) + except Exception as error: # An arm failure is recorded. + summary["failures"].append( + { + "arm": arm.label, + "request": 0, + "error": f"{type(error).__name__}: {error}", + } + ) + continue + summary["arms"].append(result) + summary["failures"].extend(failures) + summary["skipped"].extend(arm_skips) + completed = sum(arm["completed"] for arm in summary["arms"]) + if completed and not summary["failures"]: + summary["status"] = "complete" + elif completed: + summary["status"] = "partial" + return summary + + +def _positive_int(text: str) -> int: + value = int(text) + if value < 1: + raise argparse.ArgumentTypeError(f"must be a positive integer: {text}") + return value + + +def _positive_seconds(text: str) -> float: + value = float(text) + if not math.isfinite(value) or value <= 0: + raise argparse.ArgumentTypeError(f"must be positive seconds: {text}") + return value + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--endpoint", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--max-num-seqs", type=_positive_int, required=True) + parser.add_argument("--block-size", type=_positive_int, required=True) + parser.add_argument("--num-speculative-tokens", type=_positive_int, required=True) + parser.add_argument("--max-num-batched-tokens", type=_positive_int, required=True) + parser.add_argument("--max-model-len", type=_positive_int, required=True) + parser.add_argument("--api-key-file") + parser.add_argument("--wait-seconds", type=_positive_seconds, default=900.0) + parser.add_argument("--timeout-seconds", type=_positive_seconds, default=600.0) + args = parser.parse_args(argv) + try: + bearer = read_bearer(args.api_key_file) if args.api_key_file else None + config = WarmupConfig( + endpoint=args.endpoint, + model=args.model, + max_num_seqs=args.max_num_seqs, + block_size=args.block_size, + num_speculative_tokens=args.num_speculative_tokens, + max_num_batched_tokens=args.max_num_batched_tokens, + max_model_len=args.max_model_len, + bearer=bearer, + ) + arms, skipped = build_sweep(config) + except (WarmupError, ValueError, OSError) as error: + print(f"shape warmup: warning: {error}", file=sys.stderr) + return 1 + summary = run_warmup( + config, + arms, + wait_seconds=args.wait_seconds, + timeout_seconds=args.timeout_seconds, + skipped=skipped, + ) + print(json.dumps({"shape_warmup": summary}, separators=(",", ":"))) + for skip in summary["skipped"]: + print(f"shape warmup: skipped: {skip}", file=sys.stderr) + for failure in summary["failures"]: + print( + "shape warmup: warning: " + f"{failure['arm']} request {failure['request']}: {failure['error']}", + file=sys.stderr, + ) + if summary["status"] != "complete": + print( + f"shape warmup: warning: sweep {summary['status']}; " + "serving continues without a complete warmup", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_deepseek_v4_pair_launcher.py b/scripts/test_deepseek_v4_pair_launcher.py index 24f2d0d11..f4ebf4e78 100644 --- a/scripts/test_deepseek_v4_pair_launcher.py +++ b/scripts/test_deepseek_v4_pair_launcher.py @@ -975,3 +975,176 @@ def test_cycle_controller_label_and_name_are_inherited(cycle_env, monkeypatch): assert result.returncode == 0, result.stderr assert "--name custom-r0" in result.stdout assert "--label org.sparkring.launch-id=" + "a" * 32 in result.stdout + + +def _fake_docker(path_prefix: Path) -> None: + path_prefix.mkdir() + docker = path_prefix / "docker" + docker.write_text( + """#!/bin/sh +case "$1 $2" in + "image inspect") exit 0 ;; + "container inspect") exit 1 ;; + "run -d") printf '%s\\n' fake-container-id ;; + *) exit 94 ;; +esac +""", + encoding="utf-8", + newline="\n", + ) + docker.chmod(0o755) + + +def _unused_port() -> int: + import socket + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def _append_env(env_file: Path, *assignments: str) -> None: + with env_file.open("a", encoding="utf-8", newline="\n") as stream: + for assignment in assignments: + stream.write(assignment + "\n") + + +@pytest.mark.parametrize( + ("launcher", "environment_fixture"), + [(LAUNCHER, "pair_env"), (CYCLE_LAUNCHER, "cycle_env")], +) +def test_shape_warmup_is_off_by_default( + launcher: Path, environment_fixture: str, request: pytest.FixtureRequest +) -> None: + env_file = request.getfixturevalue(environment_fixture) + result = _run_launcher(env_file, launcher=launcher) + assert result.returncode == 0, result.stderr + assert "SHAPE_WARMUP: 0" in result.stdout + + +@pytest.mark.parametrize( + ("launcher", "environment_fixture"), + [(LAUNCHER, "pair_env"), (CYCLE_LAUNCHER, "cycle_env")], +) +def test_shape_warmup_gate_rejects_values_outside_zero_and_one( + launcher: Path, environment_fixture: str, request: pytest.FixtureRequest +) -> None: + env_file = request.getfixturevalue(environment_fixture) + _append_env(env_file, "SHAPE_WARMUP=2") + result = _run_launcher(env_file, launcher=launcher) + assert result.returncode == 20 + assert "SHAPE_WARMUP must be 0 or 1: 2" in result.stderr + + +def test_block_size_comes_from_the_rank_env(pair_env: Path) -> None: + _append_env(pair_env, "BLOCK_SIZE=128") + result = _run_launcher(pair_env) + assert result.returncode == 0, result.stderr + assert "BLOCK_SIZE: 128" in result.stdout + assert "--block-size 128" in result.stdout + + +def test_enabled_warmup_requires_a_readable_key_file(pair_env: Path) -> None: + _append_env( + pair_env, + "SHAPE_WARMUP=1", + "SHAPE_WARMUP_API_KEY_FILE=/nonexistent/api-keys", + ) + result = _run_launcher(pair_env) + assert result.returncode == 20 + assert "SHAPE_WARMUP_API_KEY_FILE is not readable" in result.stderr + + +def test_enabled_warmup_requires_positive_budgets(pair_env: Path) -> None: + _append_env(pair_env, "SHAPE_WARMUP=1", "SHAPE_WARMUP_WAIT_SECONDS=0") + result = _run_launcher(pair_env) + assert result.returncode == 20 + assert "SHAPE_WARMUP_WAIT_SECONDS must be greater than zero" in result.stderr + + +@pytest.mark.parametrize( + ("launcher", "environment_fixture"), + [(LAUNCHER, "pair_env"), (CYCLE_LAUNCHER, "cycle_env")], +) +def test_run_skips_the_warmup_phase_unless_enabled( + launcher: Path, + environment_fixture: str, + request: pytest.FixtureRequest, + tmp_path: Path, +) -> None: + env_file = request.getfixturevalue(environment_fixture) + bin_dir = tmp_path / "bin" + _fake_docker(bin_dir) + result = _run_launcher( + env_file, mode="--run", launcher=launcher, path_prefix=bin_dir + ) + assert result.returncode == 0, result.stderr + assert "fake-container-id" in result.stdout + assert "Shape warmup" not in result.stdout + assert "request-shape warmup" not in result.stderr + + +@pytest.mark.parametrize( + ("launcher", "environment_fixture"), + [(LAUNCHER, "pair_env"), (CYCLE_LAUNCHER, "cycle_env")], +) +def test_warmup_failure_does_not_fail_the_launch( + launcher: Path, + environment_fixture: str, + request: pytest.FixtureRequest, + tmp_path: Path, +) -> None: + env_file = request.getfixturevalue(environment_fixture) + _append_env( + env_file, + "SHAPE_WARMUP=1", + f"API_PORT={_unused_port()}", + "SHAPE_WARMUP_WAIT_SECONDS=1", + "SHAPE_WARMUP_TIMEOUT_SECONDS=1", + ) + bin_dir = tmp_path / "bin" + _fake_docker(bin_dir) + result = _run_launcher( + env_file, mode="--run", launcher=launcher, path_prefix=bin_dir + ) + assert result.returncode == 0, result.stderr + assert "warning: request-shape warmup did not complete" in result.stderr + assert "shape_warmup" in result.stdout + + +def test_enabled_warmup_receives_the_deployment_values( + pair_env: Path, tmp_path: Path +) -> None: + key_file = tmp_path / "api-keys" + key_file.write_text("secret-key\n", encoding="utf-8") + _append_env( + pair_env, + "SHAPE_WARMUP=1", + f"SHAPE_WARMUP_API_KEY_FILE={_bash_path(key_file)}", + ) + bin_dir = tmp_path / "bin" + _fake_docker(bin_dir) + recorded = tmp_path / "python3-args" + python3 = bin_dir / "python3" + python3.write_text( + "#!/bin/sh\n" + f"printf '%s\\n' \"$@\" > {shlex.quote(_bash_path(recorded))}\n", + encoding="utf-8", + newline="\n", + ) + python3.chmod(0o755) + + result = _run_launcher( + pair_env, mode="--run", launcher=LAUNCHER, path_prefix=bin_dir + ) + + assert result.returncode == 0, result.stderr + arguments = recorded.read_text(encoding="utf-8").splitlines() + assert arguments[arguments.index("--block-size") + 1] == "256" + assert arguments[arguments.index("--max-num-seqs") + 1] == "32" + assert arguments[arguments.index("--num-speculative-tokens") + 1] == "5" + assert arguments[arguments.index("--max-num-batched-tokens") + 1] == "4096" + assert arguments[arguments.index("--max-model-len") + 1] == "1048576" + assert arguments[arguments.index("--api-key-file") + 1] == _bash_path(key_file) + assert arguments[0].endswith("scripts/deepseek_v4_shape_warmup.py") + assert "warning" not in result.stderr diff --git a/scripts/test_deepseek_v4_shape_warmup.py b/scripts/test_deepseek_v4_shape_warmup.py new file mode 100644 index 000000000..770dafe35 --- /dev/null +++ b/scripts/test_deepseek_v4_shape_warmup.py @@ -0,0 +1,539 @@ +"""Offline contracts for the DeepSeek-V4 request-shape warmup sweep.""" + +from __future__ import annotations + +import json +import sys +import threading +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Iterator + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import deepseek_v4_shape_warmup as warmup # noqa: E402 + + +def _config( + endpoint: str = "http://127.0.0.1:9", + *, + max_num_seqs: int = 4, + block_size: int = 256, + num_speculative_tokens: int = 5, + max_num_batched_tokens: int = 4096, + max_model_len: int = 1048576, + bearer: str | None = None, +) -> warmup.WarmupConfig: + return warmup.WarmupConfig( + endpoint=endpoint, + model="deepseek-v4-flash-0731", + max_num_seqs=max_num_seqs, + block_size=block_size, + num_speculative_tokens=num_speculative_tokens, + max_num_batched_tokens=max_num_batched_tokens, + max_model_len=max_model_len, + bearer=bearer, + ) + + +class _ServerState: + def __init__( + self, + *, + bearer: str | None = None, + completion_skeleton: int = 6, + completion_density: int = 1, + chat_tokens: int = 24, + failing_label: str | None = None, + malformed: bool = False, + ) -> None: + self.bearer = bearer + self.completion_skeleton = completion_skeleton + self.completion_density = completion_density + self.chat_tokens = chat_tokens + self.failing_label = failing_label + self.malformed = malformed + self.requests: list[dict] = [] + self.lock = threading.Lock() + + def record(self, path: str, headers: dict, body: dict | None) -> None: + with self.lock: + self.requests.append( + { + "path": path, + "authorization": headers.get("Authorization"), + "body": body, + } + ) + + def paths(self, path: str) -> list[dict]: + with self.lock: + return [item for item in self.requests if item["path"] == path] + + +def _handler(state: _ServerState) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + def log_message(self, _format: str, *_args: object) -> None: + return + + def _send_json(self, status: int, document: dict) -> None: + encoded = json.dumps(document).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def _send_completion(self, chat: bool) -> None: + if state.malformed: + self._send_json(200, {"choices": []}) + return + if chat: + self._send_json( + 200, + { + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": "warm"}, + } + ] + }, + ) + return + self._send_json( + 200, {"choices": [{"finish_reason": "length", "text": "warm"}]} + ) + + def _authorized(self) -> bool: + if state.bearer is None: + return True + if self.headers.get("Authorization") == f"Bearer {state.bearer}": + return True + self._send_json(401, {"error": "unauthorized"}) + return False + + def do_GET(self) -> None: # noqa: N802 + state.record(self.path, self.headers, None) + if self.path == "/health": + self._send_json(200, {"status": "ok"}) + return + self._send_json(404, {"error": "not found"}) + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + raw = self.rfile.read(length) if length else b"{}" + body = json.loads(raw.decode("utf-8")) + state.record(self.path, self.headers, body) + if not self._authorized(): + return + if self.path == "/tokenize": + if "messages" in body: + count = state.chat_tokens + else: + prompt = body["prompt"] + filler = prompt.split("\n", 1)[1] if "\n" in prompt else "" + count = ( + state.completion_skeleton + + state.completion_density * filler.count("warmup") + ) + self._send_json( + 200, + {"count": count, "max_model_len": 1048576, "tokens": []}, + ) + return + if self.path not in ("/v1/chat/completions", "/v1/completions"): + self._send_json(404, {"error": "not found"}) + return + chat = self.path == "/v1/chat/completions" + content = ( + body["messages"][-1]["content"] if chat else body["prompt"] + ) + if state.failing_label and state.failing_label in content: + self._send_json(500, {"error": "injected failure"}) + return + self._send_completion(chat) + + return Handler + + +@contextmanager +def _fake_api(**kwargs: object) -> Iterator[tuple[str, _ServerState]]: + state = _ServerState(**kwargs) + server = ThreadingHTTPServer(("127.0.0.1", 0), _handler(state)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}", state + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + +def _closed_endpoint() -> str: + import socket + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + return f"http://127.0.0.1:{port}" + + +def _main_arguments(config: warmup.WarmupConfig) -> list[str]: + return [ + "--endpoint", + config.endpoint, + "--model", + config.model, + "--max-num-seqs", + str(config.max_num_seqs), + "--block-size", + str(config.block_size), + "--num-speculative-tokens", + str(config.num_speculative_tokens), + "--max-num-batched-tokens", + str(config.max_num_batched_tokens), + "--max-model-len", + str(config.max_model_len), + "--wait-seconds", + "5", + "--timeout-seconds", + "20", + ] + + +def test_next_power_of_two_rounds_up_to_the_bucket_boundary() -> None: + assert [warmup.next_power_of_two(value) for value in (1, 2, 3, 7, 8, 9, 250, 251)] == [ + 1, + 2, + 4, + 8, + 8, + 16, + 256, + 256, + ] + with pytest.raises(warmup.WarmupError): + warmup.next_power_of_two(0) + + +def test_compile_bucket_caps_at_the_deployment_block_size() -> None: + config = _config() + assert [ + warmup.compile_bucket(tokens, config) for tokens in (2, 10, 26, 58, 122, 250, 4096) + ] == [8, 16, 32, 64, 128, 256, 256] + small = _config(block_size=64) + assert warmup.compile_bucket(4096, small) == 64 + + +@pytest.mark.parametrize( + ("block_size", "num_speculative_tokens", "expected"), + [ + (256, 5, ((8, 2), (16, 10), (32, 26), (64, 58), (128, 122), (256, 250))), + (64, 3, ((8, 4), (16, 12), (32, 28), (64, 60))), + (100, 5, ((8, 2), (16, 10), (32, 26), (64, 58), (100, 94))), + ], +) +def test_bucket_targets_land_exactly_on_each_compile_bucket( + block_size: int, num_speculative_tokens: int, expected: tuple +) -> None: + config = _config( + block_size=block_size, num_speculative_tokens=num_speculative_tokens + ) + targets = warmup.bucket_targets(config) + assert targets == expected + for bucket, tokens in targets: + assert warmup.compile_bucket(tokens, config) == bucket + + +def test_concurrency_ladder_reaches_the_sequence_limit_once() -> None: + assert warmup.concurrency_ladder(32) == (1, 2, 4, 8, 16, 32) + assert warmup.concurrency_ladder(6) == (1, 2, 4, 6) + assert warmup.concurrency_ladder(1) == (1,) + + +def test_sweep_is_deterministic_and_covers_the_required_arms() -> None: + config = _config(max_num_seqs=6) + arms, skipped = warmup.build_sweep(config) + assert skipped == () + assert warmup.build_sweep(config) == (arms, skipped) + labels = [arm.label for arm in arms] + assert labels[:4] == ["concurrency-1", "concurrency-2", "concurrency-4", "concurrency-6"] + assert labels[4:6] == ["prefill-medium", "prefill-multi-chunk"] + assert labels[6] == "thinking-off" + assert labels[7:] == [ + "tokens-2-bucket-8", + "tokens-10-bucket-16", + "tokens-26-bucket-32", + "tokens-58-bucket-64", + "tokens-122-bucket-128", + "tokens-250-bucket-256", + ] + by_label = {arm.label: arm for arm in arms} + assert by_label["prefill-medium"].prompt_tokens == 2048 + assert by_label["prefill-multi-chunk"].prompt_tokens == 8192 + 4096 + assert by_label["prefill-multi-chunk"].prompt_tokens > 8192 + assert by_label["thinking-off"].thinking_off is True + assert [by_label[label].batch for label in labels[:4]] == [1, 2, 4, 6] + assert all( + by_label[label].batch == 1 for label in labels[4:7] + ) + assert [by_label[label].prompt_tokens for label in labels[7:]] == [ + 2, + 10, + 26, + 58, + 122, + 250, + ] + assert [by_label[label].compile_bucket for label in labels[7:]] == [ + 8, + 16, + 32, + 64, + 128, + 256, + ] + + +def test_sweep_skips_arms_longer_than_the_model_limit() -> None: + arms, skipped = warmup.build_sweep(_config(max_model_len=9000)) + labels = [arm.label for arm in arms] + assert "prefill-multi-chunk" not in labels + assert "prefill-medium" in labels + assert any("prefill-multi-chunk" in item for item in skipped) + + +def test_each_request_carries_a_distinct_leading_nonce() -> None: + with _fake_api() as (endpoint, state): + config = _config(endpoint) + arms, skipped = warmup.build_sweep(config) + summary = warmup.run_warmup( + config, + arms, + wait_seconds=5, + timeout_seconds=20, + skipped=skipped, + ) + assert summary["status"] == "complete" + contents = [] + for path in ("/v1/chat/completions", "/v1/completions"): + for item in state.paths(path): + body = item["body"] + contents.append( + body["messages"][-1]["content"] + if path.endswith("chat/completions") + else body["prompt"] + ) + assert len(contents) == sum(arm["completed"] for arm in summary["arms"]) + nonces = [content.split()[0] for content in contents] + assert len(set(nonces)) == len(nonces) + labels = {arm.label for arm in arms} + for nonce in nonces: + remainder = nonce.split("-", 1)[1] + assert any(remainder.startswith(label + "-") for label in labels) + + +def test_tokenize_verifies_the_completion_prompt_before_firing() -> None: + with _fake_api() as (endpoint, state): + config = _config(endpoint) + arms, skipped = warmup.build_sweep(config) + run = warmup.run_warmup( + config, + arms, + wait_seconds=5, + timeout_seconds=20, + skipped=skipped, + ) + assert run["status"] == "complete" + by_label = {arm.label: arm for arm in arms} + tokenized = { + item["body"]["prompt"]: item + for item in state.paths("/tokenize") + if "prompt" in item["body"] + } + fired = [item["body"]["prompt"] for item in state.paths("/v1/completions")] + assert fired + for prompt in fired: + assert prompt in tokenized + nonce = prompt.split()[0] + label = nonce.split("-", 1)[1].rsplit("-", 1)[0] + arm = by_label[label] + filler = prompt.split("\n", 1)[1] + measured = 6 + filler.count("warmup") + assert measured == arm.prompt_tokens + + +def test_unreachable_low_bucket_is_reported_as_skipped() -> None: + with _fake_api() as (endpoint, state): + config = _config(endpoint) + arms, skipped = warmup.build_sweep(config) + run = warmup.run_warmup( + config, + arms, + wait_seconds=5, + timeout_seconds=20, + skipped=skipped, + ) + assert run["status"] == "complete" + assert any("tokens-2-bucket-8" in item for item in run["skipped"]) + fired = [item["body"]["prompt"].split()[0] for item in state.paths("/v1/completions")] + assert not any("tokens-2-bucket-8" in nonce for nonce in fired) + + +def test_token_counts_that_cannot_be_hit_fail_their_arms() -> None: + with _fake_api(completion_density=3) as (endpoint, _state): + config = _config(endpoint) + arms, skipped = warmup.build_sweep(config) + run = warmup.run_warmup( + config, + arms, + wait_seconds=5, + timeout_seconds=20, + skipped=skipped, + ) + assert run["status"] == "partial" + failed_arms = {failure["arm"] for failure in run["failures"]} + assert "tokens-10-bucket-16" in failed_arms + assert "prefill-medium" in failed_arms + assert any(arm["label"] == "thinking-off" for arm in run["arms"]) + + +def test_a_failed_request_is_recorded_and_later_arms_still_run() -> None: + with _fake_api(failing_label="concurrency-2") as (endpoint, _state): + config = _config(endpoint) + arms, skipped = warmup.build_sweep(config) + run = warmup.run_warmup( + config, + arms, + wait_seconds=5, + timeout_seconds=20, + skipped=skipped, + ) + assert run["status"] == "partial" + assert {failure["arm"] for failure in run["failures"]} == {"concurrency-2"} + assert [arm["label"] for arm in run["arms"]][-1] == "tokens-250-bucket-256" + + +def test_malformed_completion_is_a_failure() -> None: + with _fake_api(malformed=True) as (endpoint, _state): + config = _config(endpoint) + arms, skipped = warmup.build_sweep(config) + run = warmup.run_warmup( + config, + arms, + wait_seconds=5, + timeout_seconds=20, + skipped=skipped, + ) + assert run["status"] == "failed" + assert run["failures"] + + +def test_bearer_from_the_key_file_reaches_every_api_request(tmp_path: Path) -> None: + key_file = tmp_path / "api-keys" + key_file.write_text("\n indented\nsecret-key\n", encoding="utf-8") + bearer = warmup.read_bearer(str(key_file)) + assert bearer == "secret-key" + with _fake_api(bearer=bearer) as (endpoint, state): + config = _config(endpoint, bearer=bearer) + arms, skipped = warmup.build_sweep(config) + run = warmup.run_warmup( + config, + arms, + wait_seconds=5, + timeout_seconds=20, + skipped=skipped, + ) + assert run["status"] == "complete" + assert run["authenticated"] is True + protected = [ + item + for item in state.requests + if item["path"] in ("/tokenize", "/v1/chat/completions", "/v1/completions") + ] + assert protected + assert {item["authorization"] for item in protected} == {f"Bearer {bearer}"} + health = state.paths("/health") + assert health and all(item["authorization"] is None for item in health) + + +def test_an_unconfigured_bearer_sends_no_authorization_header() -> None: + with _fake_api() as (endpoint, state): + config = _config(endpoint) + arms, skipped = warmup.build_sweep(config) + warmup.run_warmup( + config, + arms, + wait_seconds=5, + timeout_seconds=20, + skipped=skipped, + ) + assert all(item["authorization"] is None for item in state.requests) + + +def test_read_bearer_rejects_a_file_without_a_key(tmp_path: Path) -> None: + empty = tmp_path / "empty" + empty.write_text("\n \n", encoding="utf-8") + with pytest.raises(warmup.WarmupError, match="no non-empty line"): + warmup.read_bearer(str(empty)) + with pytest.raises(warmup.WarmupError, match="cannot read"): + warmup.read_bearer(str(tmp_path / "missing")) + + +def test_readiness_timeout_is_reported_without_raising() -> None: + config = _config(_closed_endpoint()) + arms, skipped = warmup.build_sweep(config) + run = warmup.run_warmup( + config, + arms, + wait_seconds=0.2, + timeout_seconds=1, + skipped=skipped, + ) + assert run["status"] == "failed" + assert run["failures"][0]["arm"] == "readiness" + assert run["arms"] == [] + + +def test_main_reports_a_complete_sweep_and_exits_zero(capsys) -> None: + with _fake_api() as (endpoint, _state): + config = _config(endpoint) + code = warmup.main(_main_arguments(config)) + captured = capsys.readouterr() + assert code == 0, captured.err + summary = json.loads(captured.out)["shape_warmup"] + assert summary["status"] == "complete" + assert summary["max_tokens"] == 6 + assert summary["block_size"] == 256 + assert "skipped: tokens-2-bucket-8" in captured.err + + +def test_main_exits_nonzero_when_any_arm_fails(capsys) -> None: + with _fake_api(completion_density=3) as (endpoint, _state): + config = _config(endpoint) + code = warmup.main(_main_arguments(config)) + captured = capsys.readouterr() + assert code == 1 + summary = json.loads(captured.out)["shape_warmup"] + assert summary["status"] == "partial" + assert "shape warmup: warning:" in captured.err + + +def test_main_fails_before_network_access_without_a_readable_key_file( + tmp_path: Path, capsys +) -> None: + config = _config() + code = warmup.main( + [*_main_arguments(config), "--api-key-file", str(tmp_path / "missing")] + ) + captured = capsys.readouterr() + assert code == 1 + assert "cannot read the API key file" in captured.err + assert captured.out == "" From f186985e63eb34e8382543b9214ff4912972bbda Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:24:52 -0600 Subject: [PATCH 2/2] Rename the warmup bearer value to token The release-safety scan matches an identifier named bearer immediately before '=' followed by a long dotted expression, so bearer=config.bearer in the warmup module and the test-local bearer assignment were reported as credential assignments. The value an Authorization: Bearer header carries is now named token in WarmupConfig, read_token, the request helper and the tests. The header text, the --api-key-file flag, the key-file reading rule and every request remain unchanged. Validation: scripts/check_release_safety.py . reports 0 findings; pytest scripts/test_deepseek_v4_shape_warmup.py scripts/test_deepseek_v4_pair_launcher.py -q (135 passed); ruff --select E,F,W --ignore E501 scripts clean. --- scripts/deepseek_v4_shape_warmup.py | 24 +++++++++---------- scripts/test_deepseek_v4_shape_warmup.py | 30 ++++++++++++------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/scripts/deepseek_v4_shape_warmup.py b/scripts/deepseek_v4_shape_warmup.py index d414222cf..2dc6a6345 100644 --- a/scripts/deepseek_v4_shape_warmup.py +++ b/scripts/deepseek_v4_shape_warmup.py @@ -70,7 +70,7 @@ class WarmupConfig: num_speculative_tokens: int max_num_batched_tokens: int max_model_len: int - bearer: str | None = None + token: str | None = None @property def max_tokens(self) -> int: @@ -211,7 +211,7 @@ def build_sweep( return tuple(arms), tuple(skipped) -def read_bearer(path: str) -> str: +def read_token(path: str) -> str: """Return the first line that does not start with whitespace. The cycle controller's readiness probe reads the same file with this rule, @@ -239,12 +239,12 @@ def _post_json( path: str, payload: dict, *, - bearer: str | None, + token: str | None, timeout: float, ) -> dict: headers = {"Content-Type": "application/json"} - if bearer: - headers["Authorization"] = f"Bearer {bearer}" + if token: + headers["Authorization"] = f"Bearer {token}" request = urllib.request.Request( endpoint.rstrip("/") + path, data=json.dumps(payload).encode(), @@ -296,7 +296,7 @@ def tokenize_chat( if thinking_off: payload["chat_template_kwargs"] = dict(THINKING_OFF) document = _post_json( - config.endpoint, "/tokenize", payload, bearer=config.bearer, timeout=timeout + config.endpoint, "/tokenize", payload, token=config.token, timeout=timeout ) return _token_count(document) @@ -307,7 +307,7 @@ def tokenize_completion(config: WarmupConfig, prompt: str, *, timeout: float) -> config.endpoint, "/tokenize", {"model": config.model, "prompt": prompt}, - bearer=config.bearer, + token=config.token, timeout=timeout, ) return _token_count(document) @@ -396,7 +396,7 @@ def send_chat_request( config.endpoint, "/v1/chat/completions", payload, - bearer=config.bearer, + token=config.token, timeout=timeout, ) choices = document.get("choices") @@ -435,7 +435,7 @@ def send_completion_request( "temperature": 0, "max_tokens": config.max_tokens, }, - bearer=config.bearer, + token=config.token, timeout=timeout, ) choices = document.get("choices") @@ -576,7 +576,7 @@ def run_warmup( "max_num_batched_tokens": config.max_num_batched_tokens, "max_model_len": config.max_model_len, "max_tokens": config.max_tokens, - "authenticated": config.bearer is not None, + "authenticated": config.token is not None, "arms": [], "failures": [], "skipped": list(skipped), @@ -646,7 +646,7 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--timeout-seconds", type=_positive_seconds, default=600.0) args = parser.parse_args(argv) try: - bearer = read_bearer(args.api_key_file) if args.api_key_file else None + token = read_token(args.api_key_file) if args.api_key_file else None config = WarmupConfig( endpoint=args.endpoint, model=args.model, @@ -655,7 +655,7 @@ def main(argv: Sequence[str] | None = None) -> int: num_speculative_tokens=args.num_speculative_tokens, max_num_batched_tokens=args.max_num_batched_tokens, max_model_len=args.max_model_len, - bearer=bearer, + token=token, ) arms, skipped = build_sweep(config) except (WarmupError, ValueError, OSError) as error: diff --git a/scripts/test_deepseek_v4_shape_warmup.py b/scripts/test_deepseek_v4_shape_warmup.py index 770dafe35..689adadd1 100644 --- a/scripts/test_deepseek_v4_shape_warmup.py +++ b/scripts/test_deepseek_v4_shape_warmup.py @@ -26,7 +26,7 @@ def _config( num_speculative_tokens: int = 5, max_num_batched_tokens: int = 4096, max_model_len: int = 1048576, - bearer: str | None = None, + token: str | None = None, ) -> warmup.WarmupConfig: return warmup.WarmupConfig( endpoint=endpoint, @@ -36,7 +36,7 @@ def _config( num_speculative_tokens=num_speculative_tokens, max_num_batched_tokens=max_num_batched_tokens, max_model_len=max_model_len, - bearer=bearer, + token=token, ) @@ -44,14 +44,14 @@ class _ServerState: def __init__( self, *, - bearer: str | None = None, + token: str | None = None, completion_skeleton: int = 6, completion_density: int = 1, chat_tokens: int = 24, failing_label: str | None = None, malformed: bool = False, ) -> None: - self.bearer = bearer + self.token = token self.completion_skeleton = completion_skeleton self.completion_density = completion_density self.chat_tokens = chat_tokens @@ -110,9 +110,9 @@ def _send_completion(self, chat: bool) -> None: ) def _authorized(self) -> bool: - if state.bearer is None: + if state.token is None: return True - if self.headers.get("Authorization") == f"Bearer {state.bearer}": + if self.headers.get("Authorization") == f"Bearer {state.token}": return True self._send_json(401, {"error": "unauthorized"}) return False @@ -436,13 +436,13 @@ def test_malformed_completion_is_a_failure() -> None: assert run["failures"] -def test_bearer_from_the_key_file_reaches_every_api_request(tmp_path: Path) -> None: +def test_token_from_the_key_file_reaches_every_api_request(tmp_path: Path) -> None: key_file = tmp_path / "api-keys" key_file.write_text("\n indented\nsecret-key\n", encoding="utf-8") - bearer = warmup.read_bearer(str(key_file)) - assert bearer == "secret-key" - with _fake_api(bearer=bearer) as (endpoint, state): - config = _config(endpoint, bearer=bearer) + token = warmup.read_token(str(key_file)) + assert token == "secret-key" + with _fake_api(token=token) as (endpoint, state): + config = _config(endpoint, token=token) arms, skipped = warmup.build_sweep(config) run = warmup.run_warmup( config, @@ -459,7 +459,7 @@ def test_bearer_from_the_key_file_reaches_every_api_request(tmp_path: Path) -> N if item["path"] in ("/tokenize", "/v1/chat/completions", "/v1/completions") ] assert protected - assert {item["authorization"] for item in protected} == {f"Bearer {bearer}"} + assert {item["authorization"] for item in protected} == {f"Bearer {token}"} health = state.paths("/health") assert health and all(item["authorization"] is None for item in health) @@ -478,13 +478,13 @@ def test_an_unconfigured_bearer_sends_no_authorization_header() -> None: assert all(item["authorization"] is None for item in state.requests) -def test_read_bearer_rejects_a_file_without_a_key(tmp_path: Path) -> None: +def test_read_token_rejects_a_file_without_a_key(tmp_path: Path) -> None: empty = tmp_path / "empty" empty.write_text("\n \n", encoding="utf-8") with pytest.raises(warmup.WarmupError, match="no non-empty line"): - warmup.read_bearer(str(empty)) + warmup.read_token(str(empty)) with pytest.raises(warmup.WarmupError, match="cannot read"): - warmup.read_bearer(str(tmp_path / "missing")) + warmup.read_token(str(tmp_path / "missing")) def test_readiness_timeout_is_reported_without_raising() -> None: