From 31c08be798a91a542f4aca833083ab488e16f3ae Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:02:37 -0500 Subject: [PATCH 1/5] Bound opt-in subset ring sessions with isolated relay queues Admit cycle arcs leaving at most two adjacent non-members, validate universal lanes, and budget payload plus bidirectional control traffic. Keep disabled plans and setup records byte-identical for fixed build provenance. Disable delivery-proof credit release and exclude additional relayed P2P traffic in subset mode. Validate the subset matrix, native progress with concurrent pair traffic, baseline byte hashes, and six-rank GPU chain/ring capture and replay. Hardware packet accounting and concurrent decode performance remain unqualified. --- spark_transport/sircl/SUBSET_RING.md | 149 +++++++++++++ .../sircl/sparkring_sircl/agreement.py | 4 + spark_transport/sircl/sparkring_sircl/env.py | 2 + .../sircl/sparkring_sircl/oneshot/runtime.py | 30 ++- .../sircl/sparkring_sircl/p2p/budget.py | 10 +- .../sircl/sparkring_sircl/ring/plan.py | 34 ++- .../sircl/sparkring_sircl/routes.py | 33 ++- .../sircl/sparkring_sircl/subset.py | 84 +++++++ .../sparkring_sircl/testing/ring_links.py | 7 +- .../sparkring_sircl/vllm/serve/bundle.py | 9 +- .../sircl/sparkring_sircl/vllm/serve/cli.py | 2 +- .../sircl/sparkring_sircl/vllm/serve/plan.py | 38 +++- .../sircl/sparkring_sircl/vllm/settings.py | 1 + .../sircl/tests/data/subset-off-baseline.json | 148 ++++++++++++ .../sircl/tests/subset_compat_driver.py | 58 +++++ .../sircl/tests/test_subset_ring.py | 211 ++++++++++++++++++ .../sircl/tests/test_vllm_serve.py | 2 +- 17 files changed, 783 insertions(+), 39 deletions(-) create mode 100644 spark_transport/sircl/SUBSET_RING.md create mode 100644 spark_transport/sircl/sparkring_sircl/subset.py create mode 100644 spark_transport/sircl/tests/data/subset-off-baseline.json create mode 100644 spark_transport/sircl/tests/subset_compat_driver.py create mode 100644 spark_transport/sircl/tests/test_subset_ring.py diff --git a/spark_transport/sircl/SUBSET_RING.md b/spark_transport/sircl/SUBSET_RING.md new file mode 100644 index 00000000..d8aab2a6 --- /dev/null +++ b/spark_transport/sircl/SUBSET_RING.md @@ -0,0 +1,149 @@ +# Subset chain and ring sessions + +**Experimental.** Set `SIRCL_SUBSET_RING=1` on the tensor-parallel session, +with `SIRCL_ROUTE_SCOPE=fabric` in the vLLM adapter. The serve and bundle +commands expose `--subset-ring`, which sets both variables. Choose +`--large-schedule ring --gather-schedule ring` to request ring schedules. +The setting is off by default. It does not change the fabric's relay rules. + +Admission is limited to a cable-connected arc of at least three ranks on a +cycle of four to eight Sparks, leaving one or two adjacent non-members. +Wrapping arcs and reordered ranks are supported. Larger complements are +refused, even if idle: they can carry independent traffic through the same +queues and need a reservation protocol that this implementation does not +provide. This is not general support for every contiguous subset. + +Use the universal, same-function relay table of the complete cycle. Lane +selection remains the universal shortest-path rule, including opposite-pair +ties. Setup checks each lane's physical port and function against that rule. +Extending a closing edge around the long side of a cycle would require relay +routes that the universal table does not necessarily carry; this setting +does not do so. Chain edges remain direct cables. Only the ring closes the +chain through relays. + +## Relay capacity model + +The queue key is `(relay position, function class, egress port)`. Host-originated +traffic does not enter this ingress-to-egress hairpin queue. For TP6 on +positions 0–5, the closing lanes from 5 to 0 pass through 6 and 7. TP2 and DCP2 +traffic confined to 6–7 uses their direct cable in both directions, including +ACKs. Its contribution to every hairpin queue is zero, regardless of the +number of sessions, message size or forward-window setting. It can still +compete for egress bandwidth, so queue safety does not establish decode +performance isolation. + +The following is a conservative **inferred packet envelope**, not a hardware +measurement. Its assumptions are a queue of 524,288 bytes, RC path MTU at +least 256 bytes, at most 128 bytes of framing and queue accounting per packet, +and no additional copy retained by retransmission. The driver and NIC's +internal cell accounting are not established by CPU or GPU emulation. Confirm +those assumptions and the drop counters before treating the bound as a +physical queue guarantee. + +For queue `q`, let `n(q)` count all directed session lanes through `q` and its +reverse queue. The native sender permits at most 64 completion credits per +QP; allow two short packets per credit. Reserve +`C(q) = n(q) × 64 × 2 × 128` bytes for flags, credits, final partial packets +and their ACKs. With 128 bytes of overhead per 256 payload bytes, framing is +at most half the payload plus the partial-packet allowance. Reverse ACKs +are bounded the same way. A control write and its ACK belong to opposite +directions; both directions' lane counts appear in `n(q)`. + +Let `Cmax` be the largest `C(q)` over the session's queues. Each operation +family gets a per-queue payload budget + +``` +B = min(floor(0.75 × Q), floor((Q − Cmax) / 4)) +``` + +Arena traffic has at most `B` payload bytes per queue. Ring traffic has at +most `B` per queue. Allow both families to overlap, with the same allowances +in the reverse direction. Payload, framing and reverse ACKs then occupy at +most `2B + B + B + C(q) = 4B + C(q) ≤ Q`. No external relay share is assumed. +The admitted non-members contribute zero hairpin bytes by topology. + +For the six-of-eight layout with two lanes, `Cmax = 163,840`, so `B = 90,112`. +Ring windows round down to 65,536 bytes (two 32-KiB chunks), instead of the +legacy 393,216-byte window. The two ring links share the QP's unacknowledged +byte counter; their windows do not add. Arena windows divide `B` among the +lanes sharing each queue and round down to 4-KiB chunks. The conservative +envelope on relays 6 and 7 is: + +| Function and egress | Arena payload | Ring payload | Control allowance | Total including reverse ACKs | +|---|---:|---:|---:|---:| +| Primary, port 0 | 90,112 | 65,536 | 32,768 | 311,296 | +| Primary, port 1 | 90,112 | 0 | 32,768 | 245,760 | +| Secondary, port 0 | 86,016 | 65,536 | 98,304 | 368,640 | +| Secondary, port 1 | 86,016 | 0 | 98,304 | 303,104 | + +All values are bytes per queue, and each row applies independently on both +relays. These are saturation bounds, not expected occupancies. + +## Enforcement and progress + +[subset.py](sparkring_sircl/subset.py) owns admission and the budget. +[routes.py](sparkring_sircl/routes.py) enforces the budget for ring and arena +windows. A disabled forward window, oversized queue declaration, non-universal +lane, or chunk that cannot fit its share is refused. A window is never rounded +up past its allowance. The 4-KiB forward chunk is a subset default; legacy +sessions keep their existing settings. + +[runtime.py](sparkring_sircl/oneshot/runtime.py) admits the chain and link +areas, configures the bounded windows before starting the progress thread, +and disables delivery-proof credit release in the native constructor. Bytes +therefore stay charged until their RC completion, including the returning +ACK. A rank mismatch is rejected by setup agreement. The built-in tuning +table still requires full fabric coverage; no full-cycle measurement is +implicitly applied to a subset. + +[budget.py](sparkring_sircl/p2p/budget.py) reserves the complete collective +share of every used relay queue, preventing additional relayed P2P QPs in +the same deployment. Direct P2P channels remain available. DCP2 sessions on +adjacent member pairs and on the two non-members use no hairpin queues; DCP +sessions do not inherit the tensor-parallel subset flag. The proof does not +cover another overlapping deployment on the member positions. + +The native link pump polls completions and releases credits while payload +posting waits for window space. Every admitted ring has at least one chunk +of credit. Existing slot/stagger and thread-count checks remain in force. +These invariants remove a zero-credit cycle introduced by budgeting; they +are not a proof of progress after a failed rank or transport fault. The +native simulator tests repeated all-reduce and all-gather operations with a +delayed rank. CUDA graph progress also requires all emulated ranks' blocks +to be resident; production uses one GPU per rank. + +## Harness comparison + +The harness accepts `--session-env SIRCL_ROUTE_SCOPE=fabric` to keep the same +universal routes in both arms. An operator can plan the comparison offline: + +```bash +python -m sparkring_sircl.ring plan --site "$SITE" --groups 0-5 --name subset6 \ + --large --large-sizes 1048576,8388608,33554432,67108864 \ + --chain-gather-sizes 131072,1048576,8388608 \ + --large-schedules ring --gather-schedules ring \ + --session-env SIRCL_ROUTE_SCOPE=fabric --session-env SIRCL_SUBSET_RING=1 +``` + +For the off arm, keep `SIRCL_ROUTE_SCOPE=fabric`, set `SIRCL_SUBSET_RING=0`, +and select `pieces` for both schedules. Capture counters on **all eight** +Sparks independently: the six-rank harness does not collect the two +non-members' counters. Run only after a coordinating operator has arranged +the concurrent DCP2 workload and a relay-drop watchdog. Do not promote the +setting on emulation results alone. + +## Compatibility tests + +[test_subset_ring.py](tests/test_subset_ring.py) enumerates every subset of +size 3–7 on cycles of size 4–8, with one and two lanes and with the complement +active or idle. Admitted layouts must satisfy the packet envelope; others +must fail closed. This does not claim ring execution for rejected layouts. +Nine admitted size combinations run the production C link pump through the +CPU verbs stand-in. The stand-in does not model physical hairpin occupancy. + +The off-setting fixture records ordered JSON hashes from commit `df86c93d` +for 20 layouts, four schedule requests per layout, setup records, lane maps, +forward windows and six complete bundle plans. Both an unset flag and `0` +must match those bytes. Build provenance is fixed for that comparison: +source digests necessarily change when implementation files change and must +not be falsified to preserve a receipt's build hash. diff --git a/spark_transport/sircl/sparkring_sircl/agreement.py b/spark_transport/sircl/sparkring_sircl/agreement.py index c832e106..8b0403e9 100644 --- a/spark_transport/sircl/sparkring_sircl/agreement.py +++ b/spark_transport/sircl/sparkring_sircl/agreement.py @@ -32,6 +32,10 @@ def agreement_failures(statuses: Sequence[tuple[Optional[str], bytes, Mapping[st if failures: return failures reference = statuses[0][2] + subset_ring = reference.get("subset_ring", False) + for index, status in enumerate(statuses): + if status[2].get("subset_ring", False) != subset_ring: + failures.append(f"rank {index} subset ring setting differs from rank 0") for index, status in enumerate(statuses): differing = {key: (reference[key], status[2].get(key)) for key in reference if key not in RANK_LOCAL_FIELDS and status[2].get(key) != reference[key]} diff --git a/spark_transport/sircl/sparkring_sircl/env.py b/spark_transport/sircl/sparkring_sircl/env.py index 5a97fee2..71f5c2a0 100644 --- a/spark_transport/sircl/sparkring_sircl/env.py +++ b/spark_transport/sircl/sparkring_sircl/env.py @@ -127,6 +127,8 @@ class Variable: Variable("SIRCL_FORWARD_PROOF", "1: bytes of a relayed lane that the op order proves delivered leave its " "forward window at once; 0: only their completions free the window", "1"), Variable("SIRCL_FORWARD_CHUNK_BYTES", "chunk of a windowed lane's stripe (multiple of 16)", "32768"), + Variable("SIRCL_SUBSET_RING", "1: experimental chain/ring on a contiguous cycle subset leaving at most two " + "adjacent non-members, with conservative relay windows", "0"), Variable("SIRCL_HAIRPIN_QUEUE_BYTES", "bytes of one relay hairpin queue (windows keep 75 % of it)", "524288"), Variable("SIRCL_PACKS_PER_THREAD", "16-byte packs per thread when sizing the grid, 1-64", "2"), Variable("SIRCL_POST_ORDER", "posting order of the progress thread's lanes: rank, ring-farthest, farthest " diff --git a/spark_transport/sircl/sparkring_sircl/oneshot/runtime.py b/spark_transport/sircl/sparkring_sircl/oneshot/runtime.py index 0a439a0b..2b7d5be4 100644 --- a/spark_transport/sircl/sparkring_sircl/oneshot/runtime.py +++ b/spark_transport/sircl/sparkring_sircl/oneshot/runtime.py @@ -99,6 +99,8 @@ from __future__ import annotations +from .. import subset as subset_mod + import contextlib import functools import logging @@ -506,7 +508,8 @@ def __init__( if error is None: try: self._allocate() - with _environment(SIRCL_POST_ORDER=self._post_order_text): + with _environment(SIRCL_POST_ORDER=self._post_order_text, + **({"SIRCL_FORWARD_PROOF": "0"} if self.subset_ring else {})): self._proxy = Proxy( world_size=self.world_size, rank=self.rank, hca_names=self.hca_names, peer_lane_devices=self._lane_devices, lane_count=self.lane_count, @@ -767,9 +770,17 @@ def _configure(self, max_size, max_gather_bytes, hca_names, peer_routes, gid_ind raise ValueError("SIRCL_SWING_ABOVE_BYTES needs the Swing all-reduce, which is unavailable") self._slot_bytes = slot_bytes + # Subset admission precedes arena allocation and joins setup agreement only when enabled. + self.subset_ring = subset_mod.enabled() and not own_fabric + if self.subset_ring: + if layout is None: + raise ValueError("SIRCL_SUBSET_RING needs a layout") + subset_mod.order(layout) + # Large messages and forward windows of relayed lanes. self.forward_window_bytes = _env_int("SIRCL_FORWARD_WINDOW_BYTES", default=DEFAULT_FORWARD_WINDOW_BYTES) - self.forward_chunk_bytes = _env_int("SIRCL_FORWARD_CHUNK_BYTES", default=DEFAULT_FORWARD_CHUNK_BYTES) + self.forward_chunk_bytes = _env_int("SIRCL_FORWARD_CHUNK_BYTES", default=( + subset_mod.FORWARD_CHUNK if self.subset_ring else DEFAULT_FORWARD_CHUNK_BYTES)) self.hairpin_queue_bytes = _env_int("SIRCL_HAIRPIN_QUEUE_BYTES", default=DEFAULT_HAIRPIN_QUEUE_BYTES) if self.forward_window_bytes < 0: raise ValueError("SIRCL_FORWARD_WINDOW_BYTES must not be negative (0 turns forward windows off)") @@ -807,6 +818,7 @@ def _configure(self, max_size, max_gather_bytes, hca_names, peer_routes, gid_ind self._check_chain_chunk(chunk) self.chain_chunk_bytes = chunk fabric_full = layout is not None and sorted(layout.positions) == list(layout.fabric.positions) + fabric_full = fabric_full or self.subset_ring # The chain kernel runs its send-slot wait in a warp apart from the lane waits. chain_threads = self._threads >= 64 self._chain_region = self.large_schedule != "pieces" and fabric_full and chain_threads @@ -1359,6 +1371,7 @@ def _setup_record(self) -> dict[str, Any]: "forward_window_bytes": self.forward_window_bytes, "forward_chunk_bytes": self.forward_chunk_bytes, "hairpin_queue_bytes": self.hairpin_queue_bytes, + **({"subset_ring": True} if getattr(self, "subset_ring", False) else {}), "large_schedule": self.large_schedule, "chain_region": self._chain_region, "chain_slots": self.chain_slots, @@ -1400,10 +1413,11 @@ def _plan_relays(self, route_maps: Sequence[Mapping[int, Sequence[str]]]) -> Non self.relay_safe_bytes = None if layout is None: return + subset_ring = getattr(self, "subset_ring", False) self._forward_table = routes_mod.forward_windows( layout, route_maps, self.rank, max_window=self.forward_window_bytes, chunk=self.forward_chunk_bytes or DEFAULT_FORWARD_CHUNK_BYTES, - queue_bytes=self.hairpin_queue_bytes) + queue_bytes=self.hairpin_queue_bytes, subset_ring=subset_ring) if self.forward_window_bytes: return queues = routes_mod.relay_queues(layout, route_maps) @@ -1425,7 +1439,8 @@ def _plan_chain(self, route_maps: Sequence[Mapping[int, Sequence[str]]]) -> None self.link_available = False if not (self._chain_region or self._link_region): return - order = routes_mod.chain_order(self._layout_identity_object, route_maps) + order = routes_mod.chain_order(self._layout_identity_object, route_maps, + subset_ring=getattr(self, "subset_ring", False)) if order is None: for name, schedule in (("SIRCL_LARGE_SCHEDULE", self.large_schedule), ("SIRCL_GATHER_SCHEDULE", self.gather_schedule)): @@ -1459,14 +1474,16 @@ def _plan_ring(self, order: Sequence[int], index: int, relays. During a ring op the ring's lanes are the only relayed traffic, and relays forward each function's lanes through that function's own hairpin queue, so the ring runs when no relay queue carries two ring lanes (``routes.ring_window``); each relayed ring lane then keeps - at most 75 % of a queue unacknowledged. Every rank decides the same: the decision follows the + at most 75 % of a queue unacknowledged; subset mode uses its smaller + packet envelope and excludes external relayed traffic by admission. Every rank decides the same: the decision follows the agreed layout and route maps. A ring schedule named at construction makes a ring that cannot run a setup error. """ world = len(order) window, problems = routes_mod.ring_window(self._layout_identity_object, route_maps, order, chunk=proto.LINK_WINDOW_CHUNK, - queue_bytes=self.hairpin_queue_bytes) + queue_bytes=self.hairpin_queue_bytes, + subset_ring=getattr(self, "subset_ring", False)) if problems: self.ring_problem = "; ".join(problems) for name, schedule in (("SIRCL_LARGE_SCHEDULE", self.large_schedule), @@ -2772,6 +2789,7 @@ def stats(self) -> dict[str, Any]: "link_available": self.link_available, "ring_available": self.ring_available, "ring_window_bytes": self.ring_window_bytes, + **({"subset_ring": True} if getattr(self, "subset_ring", False) else {}), "ring_problem": self.ring_problem, "link_slots": self.link_slots, "link_slot_bytes": self.link_slot_bytes, diff --git a/spark_transport/sircl/sparkring_sircl/p2p/budget.py b/spark_transport/sircl/sparkring_sircl/p2p/budget.py index d00a7b0a..17979649 100644 --- a/spark_transport/sircl/sparkring_sircl/p2p/budget.py +++ b/spark_transport/sircl/sparkring_sircl/p2p/budget.py @@ -77,6 +77,8 @@ class SessionSettings: forward_chunk: int = routes_mod.DEFAULT_FORWARD_CHUNK ring_schedule: bool = False # a large, gather or scatter schedule is ``ring`` + subset_ring: bool = False + @classmethod def from_env(cls, environ: Mapping[str, str]) -> "SessionSettings": def number(name: str, default: int) -> int: @@ -85,8 +87,9 @@ def number(name: str, default: int) -> int: ring = any(environ.get(name, "").strip() == "ring" for name in ("SIRCL_LARGE_SCHEDULE", "SIRCL_GATHER_SCHEDULE", "SIRCL_SCATTER_SCHEDULE")) + from .. import subset return cls(number("SIRCL_FORWARD_WINDOW_BYTES", routes_mod.DEFAULT_FORWARD_WINDOW), - number("SIRCL_FORWARD_CHUNK_BYTES", routes_mod.DEFAULT_FORWARD_CHUNK), ring) + number("SIRCL_FORWARD_CHUNK_BYTES", routes_mod.DEFAULT_FORWARD_CHUNK), ring, subset.enabled(environ)) def queue_share(queue_bytes: int) -> int: @@ -99,6 +102,11 @@ def session_reservation(lanes: LaneSet, settings: SessionSettings, *, queues = lanes.queues() if not queues: return {} + if settings.subset_ring and sorted(lanes.layout.positions) != list(lanes.layout.fabric.positions): + # Relayed P2P sessions have independent QPs and are outside the subset session proof. + from .. import subset + subset.validate(lanes.layout, lanes.filtered_maps(), queue_bytes=queue_bytes) + return {key: queue_share(queue_bytes) for key in queues} if settings.forward_window <= 0: return {key: queue_share(queue_bytes) for key in queues} maps = lanes.filtered_maps() diff --git a/spark_transport/sircl/sparkring_sircl/ring/plan.py b/spark_transport/sircl/sparkring_sircl/ring/plan.py index 76d23533..fd7d9065 100644 --- a/spark_transport/sircl/sparkring_sircl/ring/plan.py +++ b/spark_transport/sircl/sparkring_sircl/ring/plan.py @@ -352,13 +352,13 @@ def site_layout(site: Site, members: Sequence[int]) -> routes_mod.Layout: return group_layout(site.size, members, None if site.cabling == "ring" else site.fabric()) -def ring_text(layout: routes_mod.Layout, maps) -> str: +def ring_text(layout: routes_mod.Layout, maps, *, subset_ring: bool = False) -> str: """The ring that closes a group's chain: closed by cables, or through relays with one ring lane per relay hairpin queue (the sessions' check, ``routes.ring_window``), or why it cannot run.""" - order = routes_mod.chain_order(layout, maps) + order = routes_mod.chain_order(layout, maps, subset_ring=subset_ring) if order is None: return "no ring: the ranks do not form a chain of cable neighbors" - window, problems = routes_mod.ring_window(layout, maps, order) + window, problems = routes_mod.ring_window(layout, maps, order, subset_ring=subset_ring) if problems: return "no ring: " + "; ".join(problems) queues = routes_mod.ring_queues(layout, maps, order) @@ -508,7 +508,7 @@ def __post_init__(self) -> None: raise PlanError("wait limits must satisfy 0 < serving <= startup <= 4294 seconds") if not self.large_schedules or any(name not in LARGE_SCHEDULES for name in self.large_schedules): raise PlanError(f"large-message schedules are drawn from {', '.join(LARGE_SCHEDULES)}") - known = {variable.name for variable in env_mod.VARIABLES} + known = {variable.name for variable in env_mod.VARIABLES} | {"SIRCL_ROUTE_SCOPE"} for name, value in self.session_env: if name in HARNESS_SESSION_VARIABLES: raise PlanError(f"{name} is set by {HARNESS_SESSION_VARIABLES[name]}, not --session-env") @@ -805,7 +805,16 @@ def build_plan(site: Site, name: str, run_id: str, *, groups: Sequence[Sequence[ used = [p for group in chosen for p in group] if len(used) != len(set(used)): raise PlanError(f"configuration {name}: a Spark belongs to two groups") - layouts = [site_layout(site, members) for members in chosen] + from .. import subset + settings = dict(options.session_env) + scope = settings.get("SIRCL_ROUTE_SCOPE", "group") + if scope not in ("group", "fabric"): + raise PlanError("SIRCL_ROUTE_SCOPE must be group or fabric") + subset_ring = subset.enabled(settings) + if subset_ring and scope != "fabric": + raise PlanError("subset ring harness needs --session-env SIRCL_ROUTE_SCOPE=fabric") + layouts = ([routes_mod.Layout(site.fabric(), tuple(members)) for members in chosen] if scope == "fabric" + else [site_layout(site, members) for members in chosen]) derived = [routes_mod.derive_routes(layout, 2) for layout in layouts] problems = routes_mod.isolation_problems(derived) if problems: @@ -828,7 +837,12 @@ def build_plan(site: Site, name: str, run_id: str, *, groups: Sequence[Sequence[ limit = RELAY_QUEUE_SHARE * HAIRPIN_QUEUE_BYTES window = routes_mod.DEFAULT_FORWARD_WINDOW if options.forward_window is None else options.forward_window maps = [routes.route_map(rank) for rank in range(layout.world)] - windows = [routes_mod.forward_windows(layout, maps, rank, max_window=window) for rank in range(layout.world)] + active_subset = subset_ring and layout.world < len(layout.fabric.positions) + chunk = int(settings.get("SIRCL_FORWARD_CHUNK_BYTES", subset.FORWARD_CHUNK if active_subset + else routes_mod.DEFAULT_FORWARD_CHUNK)) + windows = [routes_mod.forward_windows(layout, maps, rank, max_window=window, + chunk=chunk, subset_ring=active_subset) + for rank in range(layout.world)] paced = sorted({w for table in windows for row in table for w in row if w}) for label, sizes in (("all-reduce", options.allreduce_sizes), ("all-gather", options.allgather_sizes)): over = [size for size in sizes if load * size > limit] @@ -846,7 +860,7 @@ def build_plan(site: Site, name: str, run_id: str, *, groups: Sequence[Sequence[ ranks_of_group = tuple(range(global_rank, global_rank + layout.world)) group_plans.append(GroupPlan(index, tuple(members), ranks_of_group, layout_text(layout), 2, relays, busiest, load, tuple(routes.route_text(r) for r in range(layout.world)), - tuple(details), tuple(warnings), ring_text(layout, maps))) + tuple(details), tuple(warnings), ring_text(layout, maps, subset_ring=active_subset))) for group_rank, position in enumerate(members): host = site.host(position) rank_plans.append(RankPlan( @@ -950,8 +964,10 @@ def tune_families_text(plan: ConfigurationPlan) -> list[str]: for group in plan.groups: layout = routes_mod.Layout.parse(group.layout) maps = [routes_mod.derive_routes(layout, group.lanes).route_map(rank) for rank in range(layout.world)] - order = routes_mod.chain_order(layout, maps) - problems = routes_mod.ring_window(layout, maps, order)[1] if order is not None else ["no chain"] + from .. import subset + active_subset = subset.enabled(dict(options.session_env)) and layout.world < len(layout.fabric.positions) + order = routes_mod.chain_order(layout, maps, subset_ring=active_subset) + problems = routes_mod.ring_window(layout, maps, order, subset_ring=active_subset)[1] if order is not None else ["no chain"] links = (", chain" if order is not None else "") + (", ring" if order is not None and not problems else "") lines.append(f" tune families, group {group.index}: all_reduce: one-shot, two-shot, pieces{links}{nccl}; " f"all_gather: tiles{links}{nccl}; reduce_scatter: scatter ops{links}; all_to_all: scatter ops") diff --git a/spark_transport/sircl/sparkring_sircl/routes.py b/spark_transport/sircl/sparkring_sircl/routes.py index c6461731..b6b3d1e2 100644 --- a/spark_transport/sircl/sparkring_sircl/routes.py +++ b/spark_transport/sircl/sparkring_sircl/routes.py @@ -727,6 +727,7 @@ def ring_window( chunk: int = DEFAULT_FORWARD_CHUNK, queue_bytes: int = DEFAULT_HAIRPIN_QUEUE, roles: Mapping[str, Role] | None = None, + subset_ring: bool = False, ) -> tuple[int, list[str]]: """``(window, problems)`` of the ring over ``order``: the bytes each ring lane through relays keeps unacknowledged (0 when every ring edge is a cable), and why the ring cannot run. @@ -735,14 +736,19 @@ def ring_window( holds the bytes of the ring lanes through it; the window is the largest multiple of ``chunk`` within ``RELAY_QUEUE_SHARE`` of a queue when every queue carries at most one ring lane (relays forward each function's lanes through that function's own queue). A queue that carries two - ring lanes is a problem. + ring lanes is a problem. Subset mode uses the smaller shared packet envelope + from subset.payload_budget instead of the legacy payload share. """ queues = ring_queues(layout, route_maps, order, roles=roles) + allowance = int(RELAY_QUEUE_SHARE * queue_bytes) + if subset_ring: + from . import subset + allowance = subset.payload_budget(layout, route_maps, queue_bytes=queue_bytes) problems = [f"relay {key[0]} {'secondary' if key[1] else 'primary'} function toward port {key[2]} carries " f"ring lanes {members}" for key, members in sorted(queues.items()) if len(members) > 1] if not queues: return 0, problems - window = int(RELAY_QUEUE_SHARE * queue_bytes) // chunk * chunk + window = allowance // chunk * chunk if window < chunk: problems.append(f"a relay queue of {queue_bytes} bytes holds less than one {chunk}-byte chunk") return window, problems @@ -757,17 +763,25 @@ def forward_windows( chunk: int = DEFAULT_FORWARD_CHUNK, queue_bytes: int = DEFAULT_HAIRPIN_QUEUE, roles: Mapping[str, Role] | None = None, + subset_ring: bool = False, ) -> list[list[int]]: """Forward window in bytes of every lane of ``rank`` (``[peer][lane]``; 0 for direct lanes). A lane through relays gets the largest multiple of ``chunk`` that is at most ``max_window`` and, for every relay queue it crosses, at most ``RELAY_QUEUE_SHARE * queue_bytes / n``, where ``n`` lanes share that queue; - never less than one chunk. A ``max_window`` of 0 disables windows. + never less than one chunk. A ``max_window`` of 0 disables windows. Subset + mode instead rejects disabled windows and shares smaller than a chunk. """ world = layout.world lanes = max((len(devices) for devices in route_maps[rank].values()), default=0) table = [[0] * lanes for _ in range(world)] + allowance = int(RELAY_QUEUE_SHARE * queue_bytes) + if subset_ring: + from . import subset + allowance = subset.payload_budget(layout, route_maps, queue_bytes=queue_bytes) + if max_window <= 0: + raise RouteError("subset ring requires forward windows") if max_window <= 0: return table if chunk <= 0 or chunk % 16 or max_window < chunk: @@ -782,7 +796,9 @@ def forward_windows( sharing = load.get((rank, peer, lane), 0) if sharing == 0: continue - limit = min(max_window, int(RELAY_QUEUE_SHARE * queue_bytes / sharing)) + limit = min(max_window, allowance // sharing) + if subset_ring and limit < chunk: + raise RouteError(f"subset relay allowance {allowance} cannot hold {sharing} chunks of {chunk} bytes") table[peer][lane] = max(chunk, limit // chunk * chunk) return table @@ -792,10 +808,12 @@ def chain_order( route_maps: Sequence[Mapping[int, Sequence[str]]], *, roles: Mapping[str, Role] | None = None, + subset_ring: bool = False, ) -> tuple[int, ...] | None: """The ranks in chain order when the group's ranks form a chain of cable neighbors, else None. - Every Spark of the fabric must host a rank. On a path the chain starts at the + Every Spark of the fabric must host a rank unless subset_ring admits a + bounded, isolated arc through subset.validate. On a path the chain starts at the end with the smaller position; on a cycle it starts at rank 0 and follows port 0, leaving the cable that closes the cycle unused. Every lane between consecutive ranks of the chain, in both directions, must be a direct lane @@ -803,7 +821,10 @@ def chain_order( """ fabric = layout.fabric if sorted(layout.positions) != list(fabric.positions): - return None + if not subset_ring: + return None + from . import subset + return subset.validate(layout, route_maps) rank_at = {position: rank for rank, position in enumerate(layout.positions)} if fabric.kind == "path": degree: dict[int, int] = {} diff --git a/spark_transport/sircl/sparkring_sircl/subset.py b/spark_transport/sircl/sparkring_sircl/subset.py new file mode 100644 index 00000000..042e50a1 --- /dev/null +++ b/spark_transport/sircl/sparkring_sircl/subset.py @@ -0,0 +1,84 @@ +"""Admission and conservative queue budgets for independent subset sessions. + +Only a cable-connected arc leaving at most two adjacent non-members is admitted. +Those non-members have no relayed traffic in the universal shortest-path table, +including when TP and DCP sessions run concurrently. No shared-queue reservation +protocol is implemented; larger complements are refused even when idle. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping + +from . import routes + +VARIABLE = "SIRCL_SUBSET_RING" +FORWARD_CHUNK = 4096 +# RC's smallest path MTU is 256. Allow 128 bytes per packet for framing, +# transport headers, CRC and queue accounting. A signaled credit may cover +# two WRs; each QP has at most 64 such credits (_roce_proxy.c). +MIN_MTU = 256 +FRAME_ALLOWANCE = 128 +CONTROL_BYTES_PER_LANE = 64 * 2 * FRAME_ALLOWANCE + + +def enabled(environ: Mapping[str, str] | None = None) -> bool: + raw = (os.environ if environ is None else environ).get(VARIABLE, "0").strip() or "0" + if raw not in ("0", "1"): + raise routes.RouteError(f"{VARIABLE} must be 0 or 1, got {raw!r}") + return raw == "1" + + +def order(layout: routes.Layout) -> tuple[int, ...]: + """Admit a physical arc, including wrapping arcs and permuted rank order.""" + fabric = layout.fabric + members = set(layout.positions) + outside = set(fabric.positions) - members + if (fabric.kind != "cycle" or not 4 <= len(fabric.positions) <= 8 + or len(members) < 3 or not 1 <= len(outside) <= 2): + raise routes.RouteError("subset ring needs 3 or more ranks on a cycle of 4 to 8, leaving one or two " + "adjacent non-members; other subsets need coordinated relay reservations") + starts = [p for p in members if fabric.cross(p, 1).dst not in members] + if len(starts) != 1: + raise routes.RouteError("subset ring ranks must form one cable-connected arc") + positions = [starts[0]] + while len(positions) < len(members): + p = fabric.cross(positions[-1], 0).dst + if p not in members or p in positions: + raise routes.RouteError("subset ring ranks must form one cable-connected arc") + positions.append(p) + return tuple(layout.positions.index(p) for p in positions) + + +def validate(layout, maps, *, queue_bytes=routes.DEFAULT_HAIRPIN_QUEUE): + """Reject custom lanes and queue capacities outside the proved model.""" + result = order(layout) + if not 4096 <= queue_bytes <= routes.DEFAULT_HAIRPIN_QUEUE: + raise routes.RouteError("subset ring hairpin capacity must be between 4096 and 524288 bytes") + lanes = len(next(iter(maps[0].values()))) + derived = routes.derive_routes(layout, lanes) + def roles(route_map): + return {peer: tuple(routes.role_of(device) for device in devices) + for peer, devices in route_map.items()} + + if [roles(m) for m in maps] != [roles(derived.route_map(rank)) for rank in range(layout.world)]: + raise routes.RouteError("subset ring requires the universal shortest-path lane maps") + return result + + +def payload_budget(layout, maps, *, queue_bytes=routes.DEFAULT_HAIRPIN_QUEUE) -> int: + """One common payload allowance for each direction of every relay queue. + + Let n count all session lanes through q and its reverse. Reserve n*16384 + bytes for short writes, short final packets and their ACKs. The remaining + bytes hold arena and ring payload simultaneously in both directions: + 2*B payload + 2*B/2 framing + 2*B/2 reverse ACKs = 4*B. + This is deliberately below the legacy 75-percent payload ceiling. + """ + validate(layout, maps, queue_bytes=queue_bytes) + queues = routes.relay_queues(layout, maps) + n = max((len(v) + len(queues.get((p, c, 1 - e), ())) + for (p, c, e), v in queues.items()), default=0) + return min(int(routes.RELAY_QUEUE_SHARE * queue_bytes), + max(0, (queue_bytes - n * CONTROL_BYTES_PER_LANE) // 4)) diff --git a/spark_transport/sircl/sparkring_sircl/testing/ring_links.py b/spark_transport/sircl/sparkring_sircl/testing/ring_links.py index 6a73bba4..bbeec002 100644 --- a/spark_transport/sircl/sparkring_sircl/testing/ring_links.py +++ b/spark_transport/sircl/sparkring_sircl/testing/ring_links.py @@ -34,17 +34,18 @@ class RingLinks(LocalSession): """Every rank of one session with its link area, ring links set as a session sets them.""" def __init__(self, library_path: str, layout: routes_mod.Layout, *, lanes: int = 2, slots: int = 8, - slot_bytes: int = 4096, timeout: float = 20.0) -> None: + slot_bytes: int = 4096, timeout: float = 20.0, subset_ring: bool = False) -> None: arena = proto.ArenaLayout(layout.world, 16384) self.link_offset = proto.chain_offset(arena.total_bytes) self.links = proto.LinkLayout(lanes, slots, slot_bytes) super().__init__(library_path, layout, lanes=lanes, slot_bytes=16384, region_bytes=self.link_offset + self.links.total_bytes) maps = [self.routes.route_map(rank) for rank in range(self.world)] - order = routes_mod.chain_order(layout, maps) + order = routes_mod.chain_order(layout, maps, subset_ring=subset_ring) if order is None: raise ValueError(f"{layout.identity()} is not a chain of cable neighbors") - window, problems = routes_mod.ring_window(layout, maps, order, chunk=proto.LINK_WINDOW_CHUNK) + window, problems = routes_mod.ring_window(layout, maps, order, chunk=proto.LINK_WINDOW_CHUNK, + subset_ring=subset_ring) if problems: raise ValueError("the ring cannot run: " + "; ".join(problems)) relayed = {rank for members in routes_mod.ring_queues(layout, maps, order).values() diff --git a/spark_transport/sircl/sparkring_sircl/vllm/serve/bundle.py b/spark_transport/sircl/sparkring_sircl/vllm/serve/bundle.py index a62f0b89..fe47cb60 100644 --- a/spark_transport/sircl/sparkring_sircl/vllm/serve/bundle.py +++ b/spark_transport/sircl/sparkring_sircl/vllm/serve/bundle.py @@ -129,6 +129,7 @@ class BundleOptions: ring_min: int | None = None # SIRCL_RING_MIN_BYTES; None: the session's defaults link_sizes: Mapping[str, int] = dataclasses.field(default_factory=dict) # plan.LinkSize.attribute -> bytes link_slots: int | None = None # SIRCL_LINK_SLOTS; None: the session's default + subset_ring: bool = False ring_gather_stagger: int | None = None # SIRCL_RING_GATHER_STAGGER; None: the session's default tuning_tables: tuple[str, ...] = () # --tuning-table PATH (SIRCL_TUNING_TABLE) reasoning_effort: str | None = None # --reasoning-effort, checked against the checkpoint below @@ -358,7 +359,8 @@ def bundle_sessions(group: fabric.GroupTopology, positions: Sequence[int], optio if getattr(options, attribute) is not None} tables = tuning or plan_mod.TuningPlan() rows = [plan_mod.session_settings(group, name="tp", groups=f"one group of ranks 0-{len(positions) - 1}", - scoped=True, schedules=schedules, link_sizes=options.link_sizes, + scoped=True, subset_ring=options.subset_ring, schedules=schedules, + link_sizes=options.link_sizes, link_slots=options.link_slots, chain_min=options.chain_min, ring_min=options.ring_min, ring_gather_stagger=options.ring_gather_stagger, @@ -401,7 +403,8 @@ def build_bundle(site: ServeSite | Site, options: BundleOptions, *, staged_diges raise ServePlanError(f"positions {list(positions)} must be distinct Sparks of the {ring.size}-Spark ring") layout = fabric.Layout.ring(ring.size) try: - group = fabric.describe_group(layout, positions) + group = fabric.describe_group(layout, positions, + route_scope="fabric" if options.subset_ring else "group") except fabric.FabricError as error: raise ServePlanError(str(error)) from None run_id = options.run_id or default_run_id(positions) @@ -824,7 +827,7 @@ def plan_from_args(args: argparse.Namespace) -> BundlePlan: gid_index=args.gid_index, fused_norm=None if args.fused_norm is None else args.fused_norm == "on", column_gather=None if args.column_gather is None else args.column_gather == "on", large_schedule=args.large_schedule, gather_schedule=args.gather_schedule, - scatter_schedule=args.scatter_schedule, ring_min=args.ring_min, chain_min=args.chain_min, + subset_ring=args.subset_ring, scatter_schedule=args.scatter_schedule, ring_min=args.ring_min, chain_min=args.chain_min, link_sizes=plan_mod.link_values(args), reasoning_effort=args.reasoning_effort, repository=args.repository, checkpoint=args.checkpoint_id, thinking_behaviour=args.thinking_behaviour, extra_env=plan_mod.parse_env(args.env or []), vllm_edits=plan_mod.edits_from_args(args), diff --git a/spark_transport/sircl/sparkring_sircl/vllm/serve/cli.py b/spark_transport/sircl/sparkring_sircl/vllm/serve/cli.py index 1bdb9638..7affa073 100644 --- a/spark_transport/sircl/sparkring_sircl/vllm/serve/cli.py +++ b/spark_transport/sircl/sparkring_sircl/vllm/serve/cli.py @@ -851,7 +851,7 @@ def contexts_from_args(args: argparse.Namespace, *, run: Runner = default_runner gid_index=args.gid_index, nccl_mode=args.nccl, large_allreduce=args.large_allreduce, dcp_size=args.dcp_size, mhc_prefill_shard=args.mhc_prefill_shard, extra_env=plan_mod.parse_env(args.env or []), large_schedule=args.large_schedule, gather_schedule=args.gather_schedule, - scatter_schedule=args.scatter_schedule, ring_min=args.ring_min, chain_min=args.chain_min, + subset_ring=args.subset_ring, scatter_schedule=args.scatter_schedule, ring_min=args.ring_min, chain_min=args.chain_min, link_sizes=plan_mod.link_values(args), link_slots=args.link_slots, ring_gather_stagger=args.ring_gather_stagger, tuning_tables=tuple(args.tuning_table or ()), diff --git a/spark_transport/sircl/sparkring_sircl/vllm/serve/plan.py b/spark_transport/sircl/sparkring_sircl/vllm/serve/plan.py index ca72e48d..0631fb28 100644 --- a/spark_transport/sircl/sparkring_sircl/vllm/serve/plan.py +++ b/spark_transport/sircl/sparkring_sircl/vllm/serve/plan.py @@ -360,6 +360,7 @@ def flag(self) -> str: "SIRCL_NCCL": "--nccl", "SIRCL_LARGE_ALLREDUCE": "--large-allreduce", "SIRCL_GID_INDEX": "--gid-index", + "SIRCL_SUBSET_RING": "--subset-ring", **{variable: "--" + attribute.replace("_", "-") for attribute, variable in SCHEDULE_VARIABLES}, CHAIN_MIN_VARIABLE: "--chain-min", RING_MIN_VARIABLE: "--ring-min", @@ -413,6 +414,7 @@ class Options: ring_min: int | None = None # SIRCL_RING_MIN_BYTES; None: the session's defaults link_sizes: Mapping[str, int] = dataclasses.field(default_factory=dict) # LinkSize.attribute -> bytes link_slots: int | None = None # SIRCL_LINK_SLOTS; None: the session's default + subset_ring: bool = False ring_gather_stagger: int | None = None # SIRCL_RING_GATHER_STAGGER; None: the session's default tuning_tables: tuple[str, ...] = () # --tuning-table PATH (SIRCL_TUNING_TABLE) extra_env: Mapping[str, str] = dataclasses.field(default_factory=dict) # --env KEY=VALUE, every rank @@ -562,6 +564,7 @@ class ServePlan: serving_wait: float = DEFAULT_SERVING_WAIT_S link_sizes: Mapping[str, int] = dataclasses.field(default_factory=dict) # the LinkSize values set link_slots: int | None = None # --link-slots; None: the session's default + subset_ring: bool = False ring_gather_stagger: int | None = None # --ring-gather-stagger; None: the session's default schedules: Mapping[str, str] = dataclasses.field(default_factory=dict) # the schedules set, by attribute tuning: TuningPlan = dataclasses.field(default_factory=lambda: TuningPlan()) # --tuning-table @@ -597,7 +600,8 @@ def sessions(self) -> list["SessionSettings"]: above 1, one per decode-context-parallel group (with the session defaults, as SIRCL's adapter builds them).""" rows = [session_settings(self.group, name="tp", groups=f"one group of ranks 0-{len(self.ranks) - 1}", - scoped=True, schedules=self.schedules, link_sizes=self.link_sizes, + scoped=True, subset_ring=self.subset_ring, schedules=self.schedules, + link_sizes=self.link_sizes, link_slots=self.link_slots, chain_min=self.chain_min, ring_min=self.ring_min, ring_gather_stagger=self.ring_gather_stagger, table_settings=self.tuning.settings_of("tp"))] @@ -967,6 +971,8 @@ def relay_conflicts(arguments: Sequence[str], environment: Mapping[str, str]) -> def add_schedule_arguments(command: object) -> None: """``--large-schedule``, ``--gather-schedule`` and ``--scatter-schedule`` on an argparse (sub)parser.""" + command.add_argument("--subset-ring", action="store_true", + help="experimental subset chain/ring with isolated relay queues") for flag, what in (("--large-schedule", "all-reduces above the dispatch ceiling (all_reduce_large)"), ("--gather-schedule", "all-gathers above the gather op size (all_gather_large)"), ("--scatter-schedule", "reduce-scatters")): @@ -977,7 +983,7 @@ def add_schedule_arguments(command: object) -> None: def schedule_environment(options: object) -> dict[str, str]: """``SIRCL_*_SCHEDULE`` settings for the schedules ``options`` name (attributes of :data:`SCHEDULE_VARIABLES`).""" - result = {} + result = {"SIRCL_SUBSET_RING": "1", "SIRCL_ROUTE_SCOPE": "fabric"} if getattr(options, "subset_ring", False) else {} for attribute, variable in SCHEDULE_VARIABLES: value = getattr(options, attribute, None) if value is None: @@ -2091,6 +2097,7 @@ def to_json(self) -> dict[str, Any]: def session_settings(group: fabric.GroupTopology, *, name: str, groups: str, scoped: bool, + subset_ring: bool = False, schedules: Mapping[str, str] | None = None, link_sizes: Mapping[str, int] | None = None, link_slots: int | None = None, chain_min: int | None = None, ring_min: int | None = None, ring_gather_stagger: int | None = None, @@ -2099,7 +2106,7 @@ def session_settings(group: fabric.GroupTopology, *, name: str, groups: str, sco The rules of oneshot/runtime.py for a session built with the layout the adapter passes (``GroupTopology.session_layout``), the default thread count and hairpin queue: chain and ring ops need - every Spark of the session's fabric to host one of its ranks; a chain schedule needs the ranks to form a + full coverage or an explicitly admitted subset; a chain schedule needs the ranks to form a chain of cable neighbors (``routes.chain_order``); a ring schedule needs a ring no relay queue carries twice (``routes.ring_window``); the link geometry must be valid (``protocol.LinkLayout``) whether or not the session reserves a link area, and a ring stagger the launcher sets needs ``stagger x (ranks - 1) + @@ -2115,20 +2122,31 @@ def session_settings(group: fabric.GroupTopology, *, name: str, groups: str, sco maps = [group.route_map(rank) for rank in range(len(group.members))] hosted = len(set(layout.positions)), len(layout.fabric.positions) covers = sorted(layout.positions) == list(layout.fabric.positions) - order = routes_mod.chain_order(layout, maps) if covers else None + subset_ring = subset_ring and not covers + subset_problems = [] + if subset_ring: + from ... import subset + try: + subset.validate(layout, maps) + routes_mod.forward_windows(layout, maps, 0, chunk=subset.FORWARD_CHUNK, subset_ring=True) + covers = True + except routes_mod.RouteError as error: + subset_problems.append(str(error)) + order = routes_mod.chain_order(layout, maps, subset_ring=subset_ring) if covers else None large, gather, scatter = (effective[key] for key in DEFAULT_SCHEDULES) chain_area = covers and large != "pieces" # A tuning table that chooses a link schedule gives the session a link area; a table the tune command # built records the link slots exactly when it chooses one (tuning.table_settings). link_area = covers and (gather != "pieces" or scatter != "pieces" or large == "ring" or LINK_SLOTS_VARIABLE in table) - problems = [] + problems = subset_problems flags = {key: "--" + key.replace("_", "-") for key in DEFAULT_SCHEDULES} ring_problems: list[str] = [] window = 0 if link_area and order is not None: window, ring_problems = routes_mod.ring_window(layout, maps, order, chunk=protocol.LINK_WINDOW_CHUNK, - queue_bytes=routes_mod.DEFAULT_HAIRPIN_QUEUE) + queue_bytes=routes_mod.DEFAULT_HAIRPIN_QUEUE, + subset_ring=subset_ring) for key, value in effective.items(): if value not in ("chain", "ring"): continue @@ -2497,7 +2515,8 @@ def build_plan(site: ServeSite | Site, profile: ServingProfile, options: Options raise ServePlanError(f"positions {list(positions)} must be distinct Sparks of the {ring.size}-Spark ring") layout = fabric.Layout.ring(ring.size) try: - group = fabric.describe_group(layout, positions) + group = fabric.describe_group(layout, positions, + route_scope="fabric" if options.subset_ring else "group") except fabric.FabricError as error: raise ServePlanError(str(error)) from None run_id = _run_id(options.run_id, positions) @@ -2727,7 +2746,8 @@ def build_plan(site: ServeSite | Site, profile: ServingProfile, options: Options raise ServePlanError(f"the profile's ranks disagree on {HC_PREFILL}") given_schedules = {attribute: getattr(options, attribute) for attribute, _ in SCHEDULE_VARIABLES if getattr(options, attribute) is not None} - rows = [session_settings(group, name="tp", groups="", scoped=True, schedules=given_schedules, + rows = [session_settings(group, name="tp", groups="", scoped=True, subset_ring=options.subset_ring, + schedules=given_schedules, link_sizes=options.link_sizes, link_slots=options.link_slots, ring_gather_stagger=options.ring_gather_stagger, table_settings=tuning.settings_of("tp"))] if dcp > 1: @@ -2743,7 +2763,7 @@ def build_plan(site: ServeSite | Site, profile: ServingProfile, options: Options large_allreduce=options.large_allreduce, mhc_prefill_shard=shards.pop(), startup_wait=float(options.startup_wait), serving_wait=float(options.serving_wait), link_sizes=dict(options.link_sizes), link_slots=options.link_slots, schedules=given_schedules, - ring_gather_stagger=options.ring_gather_stagger, + ring_gather_stagger=options.ring_gather_stagger, subset_ring=options.subset_ring, tuning=tuning, oneshot_max=options.oneshot_max, ring_min=options.ring_min, chain_min=options.chain_min, large_blocks=options.large_blocks, reasoning_effort=options.reasoning_effort, diff --git a/spark_transport/sircl/sparkring_sircl/vllm/settings.py b/spark_transport/sircl/sparkring_sircl/vllm/settings.py index e17eb958..27f9d846 100644 --- a/spark_transport/sircl/sparkring_sircl/vllm/settings.py +++ b/spark_transport/sircl/sparkring_sircl/vllm/settings.py @@ -53,6 +53,7 @@ # runtime runs chain and ring ops only where every Spark of a session's fabric hosts one of its ranks, so # a DCP session keeps the session defaults. TP_SESSION_VARIABLES = ( + "SIRCL_SUBSET_RING", "SIRCL_LARGE_SCHEDULE", "SIRCL_GATHER_SCHEDULE", "SIRCL_SCATTER_SCHEDULE", "SIRCL_CHAIN_MIN_BYTES", "SIRCL_RING_MIN_BYTES", "SIRCL_CHAIN_SLOTS", "SIRCL_CHAIN_SLOT_BYTES", "SIRCL_CHAIN_BLOCKS", "SIRCL_CHAIN_UNROLL", diff --git a/spark_transport/sircl/tests/data/subset-off-baseline.json b/spark_transport/sircl/tests/data/subset-off-baseline.json new file mode 100644 index 00000000..63d6086d --- /dev/null +++ b/spark_transport/sircl/tests/data/subset-off-baseline.json @@ -0,0 +1,148 @@ +{ + "ring:4:0,1,2/auto": "810d60cfc0f61980e43ad0303634a843cc7a2bc6fb1e0a78d871333441916709", + "ring:4:0,1,2/pieces": "82ccc05086fd63da2d8a09252fc5cd63f309e6f74e96ec986fa26639820ce8f9", + "ring:4:0,1,2/chain": "1c795a133d9c65a0a31f127c6aa8ff4d9312b5aedc75fb3d8a15b42dcabf22a2", + "ring:4:0,1,2/ring": "99441ba4cc9ca1d5630726c41909b84ef4a130d9f507784cb4a7c4b24f705018", + "ring:4:0,1,2/record": "75ec32b7a79ae74db8d0fc6bdbe005e0e30bb62a008e659a6046f41ef9f2b373", + "ring:4:0,1,2/lanes": "c50c40c76bdcbb2e6847c195785fd8ff7b28f73fcf0ef942ed4ab5ef3fd5f620", + "ring:4:0,1,2/windows": "63324589ed4348df276d5b17b264630bbd765027191cadf82fb85767e03d3d3f", + "ring:4:0,1,2,3/auto": "2234574a5ebffba34a1975527264ac2a3efa19f0b367f567b18900e8b98753d7", + "ring:4:0,1,2,3/pieces": "5bdace72afb4c26887491bfb7ee625f9f0f0811d6744a41ce6fb36acd513fa64", + "ring:4:0,1,2,3/chain": "ee52c798449ccade03b71094dfdf2216c200a9358cd18a689d1ab04e57217642", + "ring:4:0,1,2,3/ring": "ddaa0f85580ada021574f3e75242090fa9e9e2ce5921600010fa1875efd86bd9", + "ring:4:0,1,2,3/record": "ad8608a053a014fd885c60093a734b386c7637b3824dc5caf5916b585344d651", + "ring:4:0,1,2,3/lanes": "0fcac1d480c26a746720a860c5296f8214795b87c14e0e1b9e003fd5dab4bca5", + "ring:4:0,1,2,3/windows": "984652d42716f93542334bc8adb7a895bdb4030b86a388da668c0fb809364a7d", + "ring:5:0,1,2/auto": "810d60cfc0f61980e43ad0303634a843cc7a2bc6fb1e0a78d871333441916709", + "ring:5:0,1,2/pieces": "82ccc05086fd63da2d8a09252fc5cd63f309e6f74e96ec986fa26639820ce8f9", + "ring:5:0,1,2/chain": "1c795a133d9c65a0a31f127c6aa8ff4d9312b5aedc75fb3d8a15b42dcabf22a2", + "ring:5:0,1,2/ring": "99441ba4cc9ca1d5630726c41909b84ef4a130d9f507784cb4a7c4b24f705018", + "ring:5:0,1,2/record": "723870b271ee810d72ea64a7f87784a67fc49849e429fda0b51b3d254de4d32f", + "ring:5:0,1,2/lanes": "e87f0303eeaee10a4a006ebee327050b279a20148febf17621a0da8db9ed9d1c", + "ring:5:0,1,2/windows": "63324589ed4348df276d5b17b264630bbd765027191cadf82fb85767e03d3d3f", + "ring:5:0,1,2,3/auto": "810d60cfc0f61980e43ad0303634a843cc7a2bc6fb1e0a78d871333441916709", + "ring:5:0,1,2,3/pieces": "82ccc05086fd63da2d8a09252fc5cd63f309e6f74e96ec986fa26639820ce8f9", + "ring:5:0,1,2,3/chain": "1c795a133d9c65a0a31f127c6aa8ff4d9312b5aedc75fb3d8a15b42dcabf22a2", + "ring:5:0,1,2,3/ring": "99441ba4cc9ca1d5630726c41909b84ef4a130d9f507784cb4a7c4b24f705018", + "ring:5:0,1,2,3/record": "7b9ad6036f9b626019015a92309ab2d9b73c3635234dd7b6ecad9790879377ce", + "ring:5:0,1,2,3/lanes": "2c3dedba446b5f2b116e8b1da6095eea05e64221ab13e65cf252fc1c9781ec88", + "ring:5:0,1,2,3/windows": "29232d5894326deddf609202e345100b08d0d3b898b7d50e4069cd5d71004b8e", + "ring:5:0,1,2,3,4/auto": "e86e985014a6cd97444d82258a9b2c96ec07b3ae9a0118d0130c988f41a8223c", + "ring:5:0,1,2,3,4/pieces": "dad9b954d7921aaf35921f5690207b8a7aa6156222e7ebf33a838d0a40080f41", + "ring:5:0,1,2,3,4/chain": "c684e650119bfd2eb37edd4d1b0e8dc08ec91f4309633e92f6ab20a78bc5bd07", + "ring:5:0,1,2,3,4/ring": "21237fd9f9eaf46bbcf2d6b91c9d35dbe85b4111267ede8bed763d005fcc3c90", + "ring:5:0,1,2,3,4/record": "4fb8a9a167cdabc9e47f7fb1576fd9bb6bca549cf474b9dcc5fb4e736a3ff1a4", + "ring:5:0,1,2,3,4/lanes": "8b132945d31fd0a71f8bf3136da10d260a481486ffaca69a162c730fa03b5b01", + "ring:5:0,1,2,3,4/windows": "139e0874915e69bbe2c2b7570c3967cfee1505a8166e65c930ed52f50b792f4d", + "ring:6:0,1,2/auto": "810d60cfc0f61980e43ad0303634a843cc7a2bc6fb1e0a78d871333441916709", + "ring:6:0,1,2/pieces": "82ccc05086fd63da2d8a09252fc5cd63f309e6f74e96ec986fa26639820ce8f9", + "ring:6:0,1,2/chain": "1c795a133d9c65a0a31f127c6aa8ff4d9312b5aedc75fb3d8a15b42dcabf22a2", + "ring:6:0,1,2/ring": "99441ba4cc9ca1d5630726c41909b84ef4a130d9f507784cb4a7c4b24f705018", + "ring:6:0,1,2/record": "4d883cdbeb392f2c8fa0ed1a6b364f77d18dd1476b5689b53ad7c9d6abed08f2", + "ring:6:0,1,2/lanes": "e87f0303eeaee10a4a006ebee327050b279a20148febf17621a0da8db9ed9d1c", + "ring:6:0,1,2/windows": "63324589ed4348df276d5b17b264630bbd765027191cadf82fb85767e03d3d3f", + "ring:6:0,1,2,3/auto": "810d60cfc0f61980e43ad0303634a843cc7a2bc6fb1e0a78d871333441916709", + "ring:6:0,1,2,3/pieces": "82ccc05086fd63da2d8a09252fc5cd63f309e6f74e96ec986fa26639820ce8f9", + "ring:6:0,1,2,3/chain": "1c795a133d9c65a0a31f127c6aa8ff4d9312b5aedc75fb3d8a15b42dcabf22a2", + "ring:6:0,1,2,3/ring": "99441ba4cc9ca1d5630726c41909b84ef4a130d9f507784cb4a7c4b24f705018", + "ring:6:0,1,2,3/record": "b9265ab84ece6c06e72c04c5f60c7f23744e64e5b706b585b545c08510875aca", + "ring:6:0,1,2,3/lanes": "8568d0db28236b28273edbf6315ad54c0dae90eb4756ae66e0dbaca1a2c467cb", + "ring:6:0,1,2,3/windows": "29232d5894326deddf609202e345100b08d0d3b898b7d50e4069cd5d71004b8e", + "ring:6:0,1,2,3,4/auto": "112b7fa3be5f225acc70c85fd943146b805455bb3eacd9178cffa38f2e10c307", + "ring:6:0,1,2,3,4/pieces": "00859413c43538ba2c4d043687b78ea020a58fc07e6f48d5e0a74fe128090349", + "ring:6:0,1,2,3,4/chain": "27842d9709e5486b09740559a0394703e5f21ae93aaf40b3e54862ae8df2bae8", + "ring:6:0,1,2,3,4/ring": "0ef5d324b3647f086d987b33c963b86443337754e133042fc94d90c1223cab77", + "ring:6:0,1,2,3,4/record": "dbb766b3cc59d32c19c0956556ffd535ea981e2ace6577bfcfb52625bedd6ee7", + "ring:6:0,1,2,3,4/lanes": "49b2e0d6f2e08e9480ddb11fb2035dfa767fe8586c7cd16da33cd77580fb0b49", + "ring:6:0,1,2,3,4/windows": "c46664621814ca2e9d1b1bba0ab205cf77b13d6f930366abd90f32165cf77f04", + "ring:6:0,1,2,3,4,5/auto": "1f24bec01bc156f33a3a072b1cfd21fa8634ac441ebc1cb3091b51398c0a4ec6", + "ring:6:0,1,2,3,4,5/pieces": "827cccb19fc9152b3f21785a36bdf6a08adaf265c0c65928d48ebf0a9d9b44f5", + "ring:6:0,1,2,3,4,5/chain": "7e18fa1bf231f8f06fe59b6757fbb591ee144dc3158018f7bc1c1db0e2f08fe3", + "ring:6:0,1,2,3,4,5/ring": "042f7e30e8a94790e580e7cd5927e006b9e78ad8c8e3ec3d2c5195dddca95f96", + "ring:6:0,1,2,3,4,5/record": "5dd9b314aaadd9d446cf4f8ffc82a46ba9aae1ef8dbaa1ceef7447011cc52cba", + "ring:6:0,1,2,3,4,5/lanes": "03676fc71a80134ad22fed8cb45a6164cf8ac27a3e1cebeddd4b034103f6464b", + "ring:6:0,1,2,3,4,5/windows": "3624452a05bc64ff75805af3e9558ba05fd7c46772807bb34b8638bc4d41545b", + "ring:7:0,1,2/auto": "810d60cfc0f61980e43ad0303634a843cc7a2bc6fb1e0a78d871333441916709", + "ring:7:0,1,2/pieces": "82ccc05086fd63da2d8a09252fc5cd63f309e6f74e96ec986fa26639820ce8f9", + "ring:7:0,1,2/chain": "1c795a133d9c65a0a31f127c6aa8ff4d9312b5aedc75fb3d8a15b42dcabf22a2", + "ring:7:0,1,2/ring": "99441ba4cc9ca1d5630726c41909b84ef4a130d9f507784cb4a7c4b24f705018", + "ring:7:0,1,2/record": "f32d4c7dbd1dd8ef64c46d5290f7512c8f045d4411d0aae16ad62231b3af6536", + "ring:7:0,1,2/lanes": "e87f0303eeaee10a4a006ebee327050b279a20148febf17621a0da8db9ed9d1c", + "ring:7:0,1,2/windows": "63324589ed4348df276d5b17b264630bbd765027191cadf82fb85767e03d3d3f", + "ring:7:0,1,2,3/auto": "810d60cfc0f61980e43ad0303634a843cc7a2bc6fb1e0a78d871333441916709", + "ring:7:0,1,2,3/pieces": "82ccc05086fd63da2d8a09252fc5cd63f309e6f74e96ec986fa26639820ce8f9", + "ring:7:0,1,2,3/chain": "1c795a133d9c65a0a31f127c6aa8ff4d9312b5aedc75fb3d8a15b42dcabf22a2", + "ring:7:0,1,2,3/ring": "99441ba4cc9ca1d5630726c41909b84ef4a130d9f507784cb4a7c4b24f705018", + "ring:7:0,1,2,3/record": "036490ce7ccab7e73c9d0adfb28c8391fa84cdf2989ad7dd0adac1615a77ac65", + "ring:7:0,1,2,3/lanes": "52546528c92048da334a6b8e12ab1ca3944fd215ea51efc0eeaeafea1b39d9bc", + "ring:7:0,1,2,3/windows": "29232d5894326deddf609202e345100b08d0d3b898b7d50e4069cd5d71004b8e", + "ring:7:0,1,2,3,4/auto": "112b7fa3be5f225acc70c85fd943146b805455bb3eacd9178cffa38f2e10c307", + "ring:7:0,1,2,3,4/pieces": "00859413c43538ba2c4d043687b78ea020a58fc07e6f48d5e0a74fe128090349", + "ring:7:0,1,2,3,4/chain": "27842d9709e5486b09740559a0394703e5f21ae93aaf40b3e54862ae8df2bae8", + "ring:7:0,1,2,3,4/ring": "0ef5d324b3647f086d987b33c963b86443337754e133042fc94d90c1223cab77", + "ring:7:0,1,2,3,4/record": "0fc96029d5481ff37418a4362bdedefdd5682dbc95a42f2b27e4e46de11578ba", + "ring:7:0,1,2,3,4/lanes": "7c23ab18ae1b4e066c5d443aced53ebc0d8945371bde0c67e15a38d988446887", + "ring:7:0,1,2,3,4/windows": "c46664621814ca2e9d1b1bba0ab205cf77b13d6f930366abd90f32165cf77f04", + "ring:7:0,1,2,3,4,5/auto": "d22c74dbf4f42fbb9637e4ba64f084a7cd25123b526e221390356a6912880cfe", + "ring:7:0,1,2,3,4,5/pieces": "6bfedf17fc057dd66564e8d7d6bd7dd1b867268f5f1ae3343dbec8b17316f4c2", + "ring:7:0,1,2,3,4,5/chain": "06cca343589485b5a8144e5720873ebb2a9499e7ea9a6e519d97f43330fe2cfd", + "ring:7:0,1,2,3,4,5/ring": "a9e23463d8c0e8185f8385111fa42f33dc8e00c3399bcfb9de9af38ef177ddd9", + "ring:7:0,1,2,3,4,5/record": "207d64e7c8bed880ab47031eece01bd88b7706ebc270513451a51862fd9c2cfa", + "ring:7:0,1,2,3,4,5/lanes": "0a3c671f0b10ce576489515f872a90a483256835b678f3224dbb2f5ee7e00a9d", + "ring:7:0,1,2,3,4,5/windows": "d3a049c787ed0b523a1743a2c6cdbbfffeec6239a0de1367d228b1c2781b3602", + "ring:7:0,1,2,3,4,5,6/auto": "1ce105be9b4770d16f020c9ff63c64f3027b38a18a4477e6b2d6b9f41d51d6b0", + "ring:7:0,1,2,3,4,5,6/pieces": "2265b6cd8073f609b296266cd74129f2e787553e7fa67bf330ef98fee68b6862", + "ring:7:0,1,2,3,4,5,6/chain": "f60fbd58b7bda4c7e313de875467b9ce69172c1032193bf5c7ec04ee34397ec2", + "ring:7:0,1,2,3,4,5,6/ring": "f9fba9f40fdf2fe7e62070b6c737854c6613cfec8753801b7f9a3852dcf1191f", + "ring:7:0,1,2,3,4,5,6/record": "dea69a72b60eca7fcc15d5cdd98f73c27750a4978ca4a0412473d8ebea9c1138", + "ring:7:0,1,2,3,4,5,6/lanes": "264cc3db72712218ad17b539a7cbe75d9fda318ee841c3bba939dbf639e90b2b", + "ring:7:0,1,2,3,4,5,6/windows": "99fe2c91b615ddf70aba9ff19d58555fcd18d1149448654af74a32ec9cff862b", + "ring:8:0,1,2/auto": "810d60cfc0f61980e43ad0303634a843cc7a2bc6fb1e0a78d871333441916709", + "ring:8:0,1,2/pieces": "82ccc05086fd63da2d8a09252fc5cd63f309e6f74e96ec986fa26639820ce8f9", + "ring:8:0,1,2/chain": "1c795a133d9c65a0a31f127c6aa8ff4d9312b5aedc75fb3d8a15b42dcabf22a2", + "ring:8:0,1,2/ring": "99441ba4cc9ca1d5630726c41909b84ef4a130d9f507784cb4a7c4b24f705018", + "ring:8:0,1,2/record": "c4f918c03b69e3bcc97380b9e08191467d8e3919d9090368d9e6ae51ec8e251a", + "ring:8:0,1,2/lanes": "e87f0303eeaee10a4a006ebee327050b279a20148febf17621a0da8db9ed9d1c", + "ring:8:0,1,2/windows": "63324589ed4348df276d5b17b264630bbd765027191cadf82fb85767e03d3d3f", + "ring:8:0,1,2,3/auto": "810d60cfc0f61980e43ad0303634a843cc7a2bc6fb1e0a78d871333441916709", + "ring:8:0,1,2,3/pieces": "82ccc05086fd63da2d8a09252fc5cd63f309e6f74e96ec986fa26639820ce8f9", + "ring:8:0,1,2,3/chain": "1c795a133d9c65a0a31f127c6aa8ff4d9312b5aedc75fb3d8a15b42dcabf22a2", + "ring:8:0,1,2,3/ring": "99441ba4cc9ca1d5630726c41909b84ef4a130d9f507784cb4a7c4b24f705018", + "ring:8:0,1,2,3/record": "57775f5bd0e97b44f482e0a92327c1ebaef5be89a5909aaf3a8e5bdd650ad1e3", + "ring:8:0,1,2,3/lanes": "52546528c92048da334a6b8e12ab1ca3944fd215ea51efc0eeaeafea1b39d9bc", + "ring:8:0,1,2,3/windows": "29232d5894326deddf609202e345100b08d0d3b898b7d50e4069cd5d71004b8e", + "ring:8:0,1,2,3,4/auto": "112b7fa3be5f225acc70c85fd943146b805455bb3eacd9178cffa38f2e10c307", + "ring:8:0,1,2,3,4/pieces": "00859413c43538ba2c4d043687b78ea020a58fc07e6f48d5e0a74fe128090349", + "ring:8:0,1,2,3,4/chain": "27842d9709e5486b09740559a0394703e5f21ae93aaf40b3e54862ae8df2bae8", + "ring:8:0,1,2,3,4/ring": "0ef5d324b3647f086d987b33c963b86443337754e133042fc94d90c1223cab77", + "ring:8:0,1,2,3,4/record": "ec9d073b613a65e22ad16221cff05e5509b2666154f1834344df952fdf94a0f0", + "ring:8:0,1,2,3,4/lanes": "b5cf3aaaf9e586a7c15cbd316625da108c38512be26c300ca9c849fced128b67", + "ring:8:0,1,2,3,4/windows": "1a0f2589bcb2b3883ed2f2b4ea519bc6fd30299598d2dd7b9ca6b73c4e84a7e7", + "ring:8:0,1,2,3,4,5/auto": "d22c74dbf4f42fbb9637e4ba64f084a7cd25123b526e221390356a6912880cfe", + "ring:8:0,1,2,3,4,5/pieces": "6bfedf17fc057dd66564e8d7d6bd7dd1b867268f5f1ae3343dbec8b17316f4c2", + "ring:8:0,1,2,3,4,5/chain": "06cca343589485b5a8144e5720873ebb2a9499e7ea9a6e519d97f43330fe2cfd", + "ring:8:0,1,2,3,4,5/ring": "a9e23463d8c0e8185f8385111fa42f33dc8e00c3399bcfb9de9af38ef177ddd9", + "ring:8:0,1,2,3,4,5/record": "420f43c7c311305e6b5416c71715f6175030b2b9387e3bc8d3d7a135c14e946d", + "ring:8:0,1,2,3,4,5/lanes": "636b5519bca6e0e839059052325d657b23a8d2f18d85463fbd0882de09cdb61b", + "ring:8:0,1,2,3,4,5/windows": "464892e20cebd9d50d89caa6e9b9ba1784764bfd445b44057d104371fc94e7e6", + "ring:8:0,1,2,3,4,5,6/auto": "017c10fcc1fd1df348388340ee567a38247ac62d54b1f16f0481abc9a0c338d5", + "ring:8:0,1,2,3,4,5,6/pieces": "85375ac2dc5f5f39230538f54938cd2c04a79930ca3788125f928151aa59d15d", + "ring:8:0,1,2,3,4,5,6/chain": "065d258cede0912f02a598f805d60441b1f03abe61f16916aa2ba364e035ad8a", + "ring:8:0,1,2,3,4,5,6/ring": "7d0072fab9c785cf106c8f9f77a201d78e00308f4126a2195786faee3ad8371f", + "ring:8:0,1,2,3,4,5,6/record": "20049f86684dd4920ecfaca115f8cdce462c3115d6fc84c9072d2ed095854ee4", + "ring:8:0,1,2,3,4,5,6/lanes": "898f5b77993f2faf6e9f7bbfbe0bbcea9b1c0d336e1d0d36c3ff59511f7684b1", + "ring:8:0,1,2,3,4,5,6/windows": "d98b93e322b8e12df48ebfc140a4adad26433388b51c537e2705b3a193b14aae", + "ring:8:0,1,2,3,4,5,6,7/auto": "a1e7f14dd14c35deef7a589f5d949fa53cf60740a4391874eb86ca1062a67ede", + "ring:8:0,1,2,3,4,5,6,7/pieces": "0e1a9de3d55cd0f8efa2da8f4e766bd96e0ee24d6095231b9a5e9a78a91cf490", + "ring:8:0,1,2,3,4,5,6,7/chain": "0a4b073a06392d581722c1464631865dc68d6769f4a7878827fdcb68d377a75b", + "ring:8:0,1,2,3,4,5,6,7/ring": "5e18d4a58102a158feb5f5abf5e6f78a6a274bde59eb0b4960ec31717046074f", + "ring:8:0,1,2,3,4,5,6,7/record": "361653b57f9552328222c93299f4c32f051c19f4e94e44de51cc9efa67443fdb", + "ring:8:0,1,2,3,4,5,6,7/lanes": "02e07644aa9251505ce41179dbb96f60ec3f0ef3951c85facf5f42026288fe20", + "ring:8:0,1,2,3,4,5,6,7/windows": "3048127cbdeab89b25d57c98e48e7251c031288b6cbf9775e0d01a55adccb9e3", + "bundle/(0, 1)/None": "2162ca8e9e2ca14ded029ab83f2d54267e4cbf2dd47d4d121b5c274eaf581fc0", + "bundle/(0, 1)/pieces": "d7b404c5a76895b10b1072fa1ffdeab0754f277ab971031a208a82ff563b71c1", + "bundle/(0, 1, 2, 3)/None": "3404ab44c63e0dbf0973e91c42bc6cb5738fb15b03b36f19eb6d202edadd74c8", + "bundle/(0, 1, 2, 3)/pieces": "d475eb09f079c1edb2e9b227cb4b4addb3523ce709ad78bdf9fd62ef1583302e", + "bundle/(0, 1, 2, 3, 4, 5, 6, 7)/None": "3aaab615e77ae8f7d769b08228c9055f0ff4bbd168fdb2b7f6adfc8c7e186279", + "bundle/(0, 1, 2, 3, 4, 5, 6, 7)/pieces": "fdd083109f42868f3228d3d9b0a9fd46918f060e6cb214012794c2411a0e8f3c" +} diff --git a/spark_transport/sircl/tests/subset_compat_driver.py b/spark_transport/sircl/tests/subset_compat_driver.py new file mode 100644 index 00000000..1bb469b2 --- /dev/null +++ b/spark_transport/sircl/tests/subset_compat_driver.py @@ -0,0 +1,58 @@ +"""Hash ordered JSON setup records and plans with fixed build provenance. + +The fixture was generated with this driver against df86c93d, before subset +support. CUDA is mocked; the real configuration, route and plan code runs. +""" + +import hashlib +import json +import sys +from types import SimpleNamespace + +from teardown_driver import _CudaStandIn + +sys.meta_path.insert(0, _CudaStandIn()) + +from plan_driver import configured +from sparkring_sircl import routes +from sparkring_sircl.oneshot import runtime +from sparkring_sircl.vllm import fabric +from sparkring_sircl.vllm.serve import bundle, plan +from sparkring_sircl.vllm.serve.sitefile import ServeSite +from test_vllm_serve import SITE + + +def digest(value): + return hashlib.sha256(json.dumps(value, ensure_ascii=True).encode()).hexdigest() + + +def records(): + result = {} + for n in range(4, 9): + for k in range(3, n + 1): + text = f"ring:{n}:" + ",".join(map(str, range(k))) + layout = routes.Layout.parse(text) + derived = routes.derive_routes(layout) + maps = [derived.route_map(i) for i in range(k)] + session = configured(runtime, routes, text, 0) + session._proxy = None + session._layout = SimpleNamespace(slots=runtime.proto.SLOTS, flag_stride=runtime.proto.FLAG_STRIDE) + group = fabric.describe_group(fabric.Layout.ring(n), list(range(k)), parent=list(range(n))) + for schedule in ("auto", "pieces", "chain", "ring"): + settings = plan.session_settings(group, name="tp", groups="fixed", scoped=True, + schedules={"large_schedule": schedule}) + result[f"{text}/{schedule}"] = digest(settings.to_json()) + result[f"{text}/record"] = digest(session._setup_record()) + result[f"{text}/lanes"] = digest(maps) + result[f"{text}/windows"] = digest([routes.forward_windows(layout, maps, r) for r in range(k)]) + for positions in ((0, 1), (0, 1, 2, 3), tuple(range(8))): + for schedule in (None, "pieces"): + built = bundle.build_bundle(ServeSite.from_json(SITE), + bundle.BundleOptions(positions=positions, large_schedule=schedule), + staged_digest="fixed-source-digest", library="fixed-library.so") + result[f"bundle/{positions}/{schedule}"] = digest(built.to_json("/etc/sircl/site.json")) + return result + + +if __name__ == "__main__": + print(json.dumps(records(), indent=2)) diff --git a/spark_transport/sircl/tests/test_subset_ring.py b/spark_transport/sircl/tests/test_subset_ring.py new file mode 100644 index 00000000..255fc07a --- /dev/null +++ b/spark_transport/sircl/tests/test_subset_ring.py @@ -0,0 +1,211 @@ +"""Subset admission, universal lanes, queue envelopes and native progress.""" + +import json +import os +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from itertools import combinations +from pathlib import Path + +import pytest +from sparkring_sircl import protocol, routes, subset +from sparkring_sircl.p2p import budget +from sparkring_sircl.testing import ring_links +from sparkring_sircl.vllm import fabric +from sparkring_sircl.vllm.serve import plan + + +def layout_maps(n, positions, lanes=2): + layout = routes.Layout.parse(f"ring:{n}:" + ",".join(map(str, positions))) + derived = routes.derive_routes(layout, lanes) + return layout, [derived.route_map(rank) for rank in range(layout.world)] + + +@pytest.mark.parametrize("n", range(4, 9)) +@pytest.mark.parametrize("active", (False, True)) +@pytest.mark.parametrize("lanes", (1, 2)) +def test_all_subsets_admit_a_bounded_envelope_or_refuse(n, active, lanes): + for k in range(3, min(n, 8)): + for positions in combinations(range(n), k): + layout, maps = layout_maps(n, positions, lanes) + outside = sorted(set(range(n)) - set(positions)) + arc = sum((p - 1) % n not in positions for p in positions) == 1 + admitted = arc and len(outside) <= 2 + if not admitted: + with pytest.raises(routes.RouteError): + routes.chain_order(layout, maps, subset_ring=True) + with pytest.raises(routes.RouteError): + routes.forward_windows(layout, maps, 0, chunk=4096, subset_ring=True) + continue + order = routes.chain_order(layout, maps, subset_ring=True) + window, problems = routes.ring_window(layout, maps, order, subset_ring=True) + assert not problems and window >= protocol.LINK_WINDOW_CHUNK + b = subset.payload_budget(layout, maps) + queues = routes.relay_queues(layout, maps) + rings = routes.ring_queues(layout, maps, order) + tables = [routes.forward_windows(layout, maps, rank, chunk=4096, subset_ring=True) + for rank in range(k)] + other = {} + if active and len(outside) > 1: + other_layout, other_maps = layout_maps(n, outside) + other = routes.relay_queues(other_layout, other_maps) + # TP, DCP and any number of sessions confined to this pair use no hairpin queue. + assert other == {} + for (p, c, e), members in queues.items(): + reverse = queues.get((p, c, 1 - e), ()) + forward = sum(tables[r][peer][lane] for r, peer, lane in members) + back = sum(tables[r][peer][lane] for r, peer, lane in reverse) + ring = len(rings.get((p, c, e), ())) * window + ring_back = len(rings.get((p, c, 1 - e), ())) * window + assert max(forward, back, ring, ring_back) <= b + # Saturate both operation families, both directions and every control credit. + data = forward + ring + ack_payload = back + ring_back + controls = (len(members) + len(reverse)) * subset.CONTROL_BYTES_PER_LANE + wire = data + (data + ack_payload) // 2 + controls + assert wire <= routes.DEFAULT_HAIRPIN_QUEUE + assert (p, c, e) not in other + + +def test_six_of_eight_lanes_are_carried_by_the_universal_table(): + from sparkring_sircl.fabric import plan as fabric_plan + from test_fabric_plan import FACTS, _plan + + universal = _plan(layout="ring8").groups[0] + layout, maps = layout_maps(8, range(6)) + derived = routes.derive_routes(layout) + for rank, peers in enumerate(derived.ranks): + for peer, lanes in peers.items(): + for lane in lanes: + if lane.hops == 1: + assert layout.fabric.cross(rank, lane.local.port).dst == peer + continue + destination = FACTS.port(peer, lane.remote.netdev).address + actual = universal.spark(rank).route_to(destination) + assert actual.netdev == lane.local.netdev and actual.cables == lane.cables + trace = fabric_plan.trace({s.position: s for s in universal.sparks}, FACTS, 8, rank, destination) + assert trace.outcome == "delivered" + assert derived.lanes_to(5, 0)[0].relays == (6, 7) + assert routes.ring_window(layout, maps, tuple(range(6)), subset_ring=True) == (65536, []) + + +def test_wrapped_and_permuted_rank_order(): + layout, maps = layout_maps(8, (1, 7, 2, 0, 5, 6)) + assert routes.chain_order(layout, maps, subset_ring=True) == (4, 5, 1, 3, 0, 2) + + +def test_unsafe_overrides_fail_closed(): + layout, maps = layout_maps(8, range(6)) + with pytest.raises(routes.RouteError, match="forward windows"): + routes.forward_windows(layout, maps, 0, max_window=0, subset_ring=True) + with pytest.raises(routes.RouteError, match="cannot hold"): + routes.forward_windows(layout, maps, 0, chunk=32768, subset_ring=True) + with pytest.raises(routes.RouteError, match="capacity"): + routes.ring_window(layout, maps, tuple(range(6)), queue_bytes=1048576, subset_ring=True) + maps[0][5] = tuple(reversed(maps[0][5])) + with pytest.raises(routes.RouteError, match="universal"): + routes.chain_order(layout, maps, subset_ring=True) + with pytest.raises(routes.RouteError, match="0 or 1"): + subset.enabled({subset.VARIABLE: "yes"}) + + +def test_planner_and_p2p_reservation(): + group = fabric.describe_group(fabric.Layout.ring(8), list(range(6)), parent=list(range(8))) + off = plan.session_settings(group, name="tp", groups="", scoped=True) + on = plan.session_settings(group, name="tp", groups="", scoped=True, subset_ring=True, + schedules={"large_schedule": "ring", "gather_schedule": "ring"}) + assert not off.chain_area and not off.link_area + assert on.chain_area and on.link_area and not on.problems + assert on.chain_order == tuple(range(6)) and "65,536" in on.ring + layout, _ = layout_maps(8, range(6)) + lanes = budget.LaneSet.of("tp", layout) + reserved = budget.session_reservation(lanes, budget.SessionSettings.from_env({subset.VARIABLE: "1"})) + allocation = budget.allocate([[lanes]], reserved=reserved, max_window=131072, chunk=4096) + assert not allocation.windows and allocation.unavailable + + +def test_bundle_explicit_option_reaches_the_rank_environment(): + from test_vllm_serve import make_bundle + + bundle = make_bundle(positions=tuple(range(6)), subset_ring=True, + large_schedule="ring", gather_schedule="ring") + assert not bundle.sessions()[0].problems + assert bundle.sessions()[0].ranks_on_fabric == (6, 8) + assert all(r.environment["SIRCL_SUBSET_RING"] == "1" and + r.environment["SIRCL_ROUTE_SCOPE"] == "fabric" for r in bundle.ranks) + + +def test_harness_on_and_off_share_the_universal_fabric(): + from sparkring_sircl.ring import plan as harness + from test_ring_harness import _site + + records = [] + for enabled in (False, True): + options = harness.Options(session_env=(("SIRCL_ROUTE_SCOPE", "fabric"), + ("SIRCL_SUBSET_RING", str(int(enabled))))) + built = harness.build_plan(_site(), "subset", "test", groups=[tuple(range(6))], + options=options, digest="fixed") + records.append(built) + assert records[0].groups[0].layout == records[1].groups[0].layout + assert records[0].groups[0].route_texts == records[1].groups[0].route_texts + assert "no ring" in records[0].groups[0].ring + assert "65536" in records[1].groups[0].ring + + +def test_subset_setting_must_agree_even_when_rank_zero_omits_it(): + from sparkring_sircl.agreement import agreement_failures + + records = [(None, b"", {"lane_count": 1, "lane_counts": [0, 1], "route_map": {1: ("a",)}}), + (None, b"", {"lane_count": 1, "lane_counts": [1, 0], "route_map": {0: ("a",)}, + "subset_ring": True})] + assert "rank 1 subset ring setting differs from rank 0" in agreement_failures(records, None) + + +@pytest.mark.parametrize("value", (None, "0")) +def test_off_plan_and_setup_record_bytes_match_origin_main(value): + pytest.importorskip("torch") + tests = Path(__file__).parent + env = {k: v for k, v in os.environ.items() if not k.startswith("SIRCL_")} + env["PYTHONPATH"] = str(tests.parent) + if value is not None: + env[subset.VARIABLE] = value + result = subprocess.run([sys.executable, str(tests / "subset_compat_driver.py")], + env=env, capture_output=True, text=True, timeout=120, check=False) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == json.loads((tests / "data/subset-off-baseline.json").read_text()) + + +@pytest.mark.parametrize("active", (False, True)) +@pytest.mark.parametrize("n,k", [(n, k) for n in range(4, 9) for k in range(max(3, n - 2), n)]) +def test_admitted_subset_native_progress(simulator_library, n, k, active): + from sparkring_sircl.testing import fabric as sim + + layout, _ = layout_maps(n, range(k)) + links = ring_links.RingLinks(str(simulator_library), layout, subset_ring=True, + slots=12, slot_bytes=65536, timeout=30) + pair = None + try: + if active and n - k == 2: + other, _ = layout_maps(n, range(k, n)) + pair = sim.LocalSession(str(simulator_library), other, node_base=k, reset=False) + pair.connect() + links.connect() + links.delay_link_ops(k - 1, 2000) + # More than one ring window, changing operations and repeated sequence slots. + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit(sim.run_oneshot_ops, pair, [16384] * 100) if pair else None + ring_links.run_ring_ops(links, [("gather", 262160, 65536, 0, 1), + ("reduce", 262176, 65536, 1, 1), + ("gather", 131088, 65536, 0, 0), + ("reduce", 196640, 65536, 0, 0)]) + if pending: + pending.result(timeout=30) + assert not any(proxy.failed() for proxy in pair.proxies) + assert not any(proxy.failed() for proxy in links.proxies) + assert all(proxy.stats()["forward_max_unacked_bytes"] <= links._ring_window + 256 + for proxy in links.proxies) + finally: + if pair: + pair.close() + links.close() diff --git a/spark_transport/sircl/tests/test_vllm_serve.py b/spark_transport/sircl/tests/test_vllm_serve.py index 1e886883..1de61908 100644 --- a/spark_transport/sircl/tests/test_vllm_serve.py +++ b/spark_transport/sircl/tests/test_vllm_serve.py @@ -3586,7 +3586,7 @@ def test_the_tensor_parallel_session_variables_are_every_schedule_and_link_varia scoped = {name for name in names if re.fullmatch(r"SIRCL_(LARGE|GATHER|SCATTER)_SCHEDULE|SIRCL_(CHAIN|LINK|RING)_\w+", name)} | set(constants["LINK_COLLECTIVES"].values()) \ | set(constants["LINK_BLOCK_VARIABLES"].values()) - assert scoped == set(adapter_settings.TP_SESSION_VARIABLES) + assert scoped | {"SIRCL_SUBSET_RING"} == set(adapter_settings.TP_SESSION_VARIABLES) assert len(adapter_settings.TP_SESSION_VARIABLES) == len(set(adapter_settings.TP_SESSION_VARIABLES)) # The plan's copies of the session's defaults and rules. assert constants["DEFAULT_LINK_SLOTS"] == plan_mod.DEFAULT_LINK_SLOTS From 05a0ed3bcc778dff0bd211f4169ff660c852e9dd Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:13:27 -0500 Subject: [PATCH 2/5] Limit subset acceleration to direct flat all-gather chain --- spark_transport/sircl/SUBSET_RING.md | 251 +++++++----------- spark_transport/sircl/sparkring_sircl/env.py | 4 +- .../sircl/sparkring_sircl/oneshot/runtime.py | 35 ++- .../sircl/sparkring_sircl/ring/plan.py | 14 +- .../sircl/sparkring_sircl/ring/worker.py | 6 +- .../sircl/sparkring_sircl/subset.py | 19 +- .../sircl/sparkring_sircl/vllm/serve/plan.py | 12 +- .../sircl/tests/subset_chain_driver.py | 79 ++++++ .../sircl/tests/test_subset_ring.py | 74 +++++- 9 files changed, 316 insertions(+), 178 deletions(-) create mode 100644 spark_transport/sircl/tests/subset_chain_driver.py diff --git a/spark_transport/sircl/SUBSET_RING.md b/spark_transport/sircl/SUBSET_RING.md index d8aab2a6..15c7c8d1 100644 --- a/spark_transport/sircl/SUBSET_RING.md +++ b/spark_transport/sircl/SUBSET_RING.md @@ -1,149 +1,102 @@ -# Subset chain and ring sessions - -**Experimental.** Set `SIRCL_SUBSET_RING=1` on the tensor-parallel session, -with `SIRCL_ROUTE_SCOPE=fabric` in the vLLM adapter. The serve and bundle -commands expose `--subset-ring`, which sets both variables. Choose -`--large-schedule ring --gather-schedule ring` to request ring schedules. -The setting is off by default. It does not change the fabric's relay rules. - -Admission is limited to a cable-connected arc of at least three ranks on a -cycle of four to eight Sparks, leaving one or two adjacent non-members. -Wrapping arcs and reordered ranks are supported. Larger complements are -refused, even if idle: they can carry independent traffic through the same -queues and need a reservation protocol that this implementation does not -provide. This is not general support for every contiguous subset. - -Use the universal, same-function relay table of the complete cycle. Lane -selection remains the universal shortest-path rule, including opposite-pair -ties. Setup checks each lane's physical port and function against that rule. -Extending a closing edge around the long side of a cycle would require relay -routes that the universal table does not necessarily carry; this setting -does not do so. Chain edges remain direct cables. Only the ring closes the -chain through relays. - -## Relay capacity model - -The queue key is `(relay position, function class, egress port)`. Host-originated -traffic does not enter this ingress-to-egress hairpin queue. For TP6 on -positions 0–5, the closing lanes from 5 to 0 pass through 6 and 7. TP2 and DCP2 -traffic confined to 6–7 uses their direct cable in both directions, including -ACKs. Its contribution to every hairpin queue is zero, regardless of the -number of sessions, message size or forward-window setting. It can still -compete for egress bandwidth, so queue safety does not establish decode -performance isolation. - -The following is a conservative **inferred packet envelope**, not a hardware -measurement. Its assumptions are a queue of 524,288 bytes, RC path MTU at -least 256 bytes, at most 128 bytes of framing and queue accounting per packet, -and no additional copy retained by retransmission. The driver and NIC's -internal cell accounting are not established by CPU or GPU emulation. Confirm -those assumptions and the drop counters before treating the bound as a -physical queue guarantee. - -For queue `q`, let `n(q)` count all directed session lanes through `q` and its -reverse queue. The native sender permits at most 64 completion credits per -QP; allow two short packets per credit. Reserve -`C(q) = n(q) × 64 × 2 × 128` bytes for flags, credits, final partial packets -and their ACKs. With 128 bytes of overhead per 256 payload bytes, framing is -at most half the payload plus the partial-packet allowance. Reverse ACKs -are bounded the same way. A control write and its ACK belong to opposite -directions; both directions' lane counts appear in `n(q)`. - -Let `Cmax` be the largest `C(q)` over the session's queues. Each operation -family gets a per-queue payload budget - -``` -B = min(floor(0.75 × Q), floor((Q − Cmax) / 4)) -``` - -Arena traffic has at most `B` payload bytes per queue. Ring traffic has at -most `B` per queue. Allow both families to overlap, with the same allowances -in the reverse direction. Payload, framing and reverse ACKs then occupy at -most `2B + B + B + C(q) = 4B + C(q) ≤ Q`. No external relay share is assumed. -The admitted non-members contribute zero hairpin bytes by topology. - -For the six-of-eight layout with two lanes, `Cmax = 163,840`, so `B = 90,112`. -Ring windows round down to 65,536 bytes (two 32-KiB chunks), instead of the -legacy 393,216-byte window. The two ring links share the QP's unacknowledged -byte counter; their windows do not add. Arena windows divide `B` among the -lanes sharing each queue and round down to 4-KiB chunks. The conservative -envelope on relays 6 and 7 is: - -| Function and egress | Arena payload | Ring payload | Control allowance | Total including reverse ACKs | -|---|---:|---:|---:|---:| -| Primary, port 0 | 90,112 | 65,536 | 32,768 | 311,296 | -| Primary, port 1 | 90,112 | 0 | 32,768 | 245,760 | -| Secondary, port 0 | 86,016 | 65,536 | 98,304 | 368,640 | -| Secondary, port 1 | 86,016 | 0 | 98,304 | 303,104 | - -All values are bytes per queue, and each row applies independently on both -relays. These are saturation bounds, not expected occupancies. - -## Enforcement and progress - -[subset.py](sparkring_sircl/subset.py) owns admission and the budget. -[routes.py](sparkring_sircl/routes.py) enforces the budget for ring and arena -windows. A disabled forward window, oversized queue declaration, non-universal -lane, or chunk that cannot fit its share is refused. A window is never rounded -up past its allowance. The 4-KiB forward chunk is a subset default; legacy -sessions keep their existing settings. - -[runtime.py](sparkring_sircl/oneshot/runtime.py) admits the chain and link -areas, configures the bounded windows before starting the progress thread, -and disables delivery-proof credit release in the native constructor. Bytes -therefore stay charged until their RC completion, including the returning -ACK. A rank mismatch is rejected by setup agreement. The built-in tuning -table still requires full fabric coverage; no full-cycle measurement is -implicitly applied to a subset. - -[budget.py](sparkring_sircl/p2p/budget.py) reserves the complete collective -share of every used relay queue, preventing additional relayed P2P QPs in -the same deployment. Direct P2P channels remain available. DCP2 sessions on -adjacent member pairs and on the two non-members use no hairpin queues; DCP -sessions do not inherit the tensor-parallel subset flag. The proof does not -cover another overlapping deployment on the member positions. - -The native link pump polls completions and releases credits while payload -posting waits for window space. Every admitted ring has at least one chunk -of credit. Existing slot/stagger and thread-count checks remain in force. -These invariants remove a zero-credit cycle introduced by budgeting; they -are not a proof of progress after a failed rank or transport fault. The -native simulator tests repeated all-reduce and all-gather operations with a -delayed rank. CUDA graph progress also requires all emulated ranks' blocks -to be resident; production uses one GPU per rank. - -## Harness comparison - -The harness accepts `--session-env SIRCL_ROUTE_SCOPE=fabric` to keep the same -universal routes in both arms. An operator can plan the comparison offline: - -```bash -python -m sparkring_sircl.ring plan --site "$SITE" --groups 0-5 --name subset6 \ - --large --large-sizes 1048576,8388608,33554432,67108864 \ - --chain-gather-sizes 131072,1048576,8388608 \ - --large-schedules ring --gather-schedules ring \ - --session-env SIRCL_ROUTE_SCOPE=fabric --session-env SIRCL_SUBSET_RING=1 -``` - -For the off arm, keep `SIRCL_ROUTE_SCOPE=fabric`, set `SIRCL_SUBSET_RING=0`, -and select `pieces` for both schedules. Capture counters on **all eight** -Sparks independently: the six-rank harness does not collect the two -non-members' counters. Run only after a coordinating operator has arranged -the concurrent DCP2 workload and a relay-drop watchdog. Do not promote the -setting on emulation results alone. - -## Compatibility tests - -[test_subset_ring.py](tests/test_subset_ring.py) enumerates every subset of -size 3–7 on cycles of size 4–8, with one and two lanes and with the complement -active or idle. Admitted layouts must satisfy the packet envelope; others -must fail closed. This does not claim ring execution for rejected layouts. -Nine admitted size combinations run the production C link pump through the -CPU verbs stand-in. The stand-in does not model physical hairpin occupancy. - -The off-setting fixture records ordered JSON hashes from commit `df86c93d` -for 20 layouts, four schedule requests per layout, setup records, lane maps, -forward windows and six complete bundle plans. Both an unset flag and `0` -must match those bytes. Build provenance is fixed for that comparison: -source digests necessarily change when implementation files change and must -not be falsified to preserve a receipt's build hash. +# Subset chain all-gather + +**Experimental.** Set `SIRCL_SUBSET_RING=1` and `SIRCL_ROUTE_SCOPE=fabric` +on the tensor-parallel session, or use the serve/bundle option `--subset-ring`. +The public option retains its name for compatibility. It is off by default. + +On an admitted subset, only flat (1-D) `all_gather_large` can use chain. +The input shard must be at least 4 MiB, satisfy the existing alignment/shape +checks, and meet the configured chain minimum for automatic selection. +Explicit `pieces` disables it. A `ring` gather request becomes `auto`. +All-reduce, reduce-scatter, row, padded and 2-D gathers use the pieces path; +small collectives retain their existing one-shot/two-shot behavior. Runtime +schedule overrides and tuning choices cannot enable subset ring operations. +Every native ring neighbor is disabled and its window is zero. + +Admission requires a cable-connected arc of at least three ranks on a cycle +of four to eight Sparks, leaving one or two adjacent non-members. Wrapping +arcs and reordered ranks are supported. Larger complements are refused, +including when idle. Use the universal same-function relay table of the +complete cycle; setup verifies every lane against that table. The mode +does not install or modify fabric routes. Full-fabric sessions retain their +existing behavior even when the option is set. + +## Selection evidence + +The six-rank measurement of `cb519293` on ring8 positions 0-5, with DCP2 +on 6-7, found flat all-gather chain throughput at 1.20x OFF for 4 MiB +shards and 1.32x for 16 MiB shards. Those are hardware measurements of that +revision, not qualification of this policy. The 4 MiB floor avoids enabling +chain for the small and 1 MiB cases. Other sizes at or above that floor and +other admitted placements require hardware confirmation. + +Reduce-scatter can traverse the same direct chain, but its two partial +reductions, scratch storage and rendezvous differ from forwarding gather +shards. All-reduce also performs reduction and return traffic. Direct cables +alone do not establish a speedup for either collective; both remain on pieces. + +The measured padded/2-D gather regression occurred despite unchanged 2 MiB +pieces. The former opt-in changed the underlying transport: chunks shrank +from 32 KiB to 4 KiB, the common per-queue allowance shrank from 393,216 to +90,112 bytes, and delivery-proof credit release was disabled. On the busiest +five-lane queues, per-lane windows shrank from 65,536 to 16,384 bytes. +More chunk posts and longer credit waits can slow the same GPU pieces. +The measurement changed these settings together and does not isolate their +individual costs. This mode now uses exactly the OFF pieces sizes, forward +windows, transport chunks and native ACK-release policy, including explicit +user overrides. Recovery of throughput still needs hardware verification. + +## Capacity and isolation + +The active chain consists solely of direct neighbor lanes in both directions. +It adds **zero data or ACK bytes** to every relay hairpin queue. There is no +closing edge, ring-window reservation or relayed link operation. Sessions +confined to the one/two adjacent non-members likewise add zero relay bytes. +These facts hold independently of traffic rate and are checked across the +admitted layouts. Direct traffic still competes for cables and NIC resources; +DCP2 decode isolation must be measured with no unrelated clients. + +Pieces keeps the OFF capacity model: default forward windows share 75% of +the declared queue, rounded to 32 KiB chunks; disabling windows uses the +existing relay-load piece limit. On the admitted default lane maps the sum +of windows is at most 393,216 bytes per 524,288-byte queue. Chain adds zero +to that bound. The mode continues reserving the full collective share from +the same-instance P2P allocator, preventing extra relayed P2P sessions. +Overlapping independent deployments on the member positions are not covered. + +This is a payload/delivery bound inherited from OFF, **not** a proof of +physical packet occupancy including NIC cell accounting, pending ACKs and +retransmissions. In particular, restoring early credit release invalidates +using the former conservative combined ring/arena packet envelope as an +active-session guarantee. No such guarantee is claimed here. + +The conservative helpers remain unused by the active subset runtime and +planner: `subset.payload_budget`, and `routes.ring_window` / +`routes.forward_windows` with `subset_ring=True`. Their tests still verify +`4B + C(q) <= Q`, with `B = min(floor(.75Q), floor((Q-Cmax)/4))` and +`C(q) = n(q)*64*2*128`, and exercise bounded native ring progress. +That conditional model assumes MTU >= 256, at most 128 bytes of framing +and queue accounting per packet, no retained retransmission copies and +credit release only on returning completion. It applies only to those +explicitly bounded helper configurations, not the restored OFF pieces path. + +## Validation gates + +Run `python -m pytest spark_transport/sircl -q`. The subset tests cover +admission, direct chain lanes, active payload bounds, retained conservative +helper bounds, runtime schedule selection, ON/OFF pieces equality, native +progress and OFF byte hashes generated from `df86c93d`. +GPU emulation must cover exact eager and captured/replayed flat and padded +gathers and all-reduces, with delayed link progress and matching native +ACK-release settings. Emulation does not measure hairpin drops or serving +performance. + +Before promotion, coordinate a maximum 25-minute TP6 outage on positions +0-5, preserving DCP2 on 6-7 with no other client. Pre-stage artifacts and +budget restoration time before taking TP6 down. Measure three ABBA blocks +(12 arms) of large flat/row/padded gathers and all-reduces, plus small +messages, using identical configurations except the opt-in. Require exact +results, zero drops on all eight Sparks, DCP2 decode within predeclared A/A +noise, and every case at least 0.97x OFF throughput (OFF/ON latency for +small messages). An incomplete block, noisy baseline or missed deadline +is inconclusive; restore TP6 within the reserved time and keep the mode off. diff --git a/spark_transport/sircl/sparkring_sircl/env.py b/spark_transport/sircl/sparkring_sircl/env.py index 71f5c2a0..7d631d2b 100644 --- a/spark_transport/sircl/sparkring_sircl/env.py +++ b/spark_transport/sircl/sparkring_sircl/env.py @@ -127,8 +127,8 @@ class Variable: Variable("SIRCL_FORWARD_PROOF", "1: bytes of a relayed lane that the op order proves delivered leave its " "forward window at once; 0: only their completions free the window", "1"), Variable("SIRCL_FORWARD_CHUNK_BYTES", "chunk of a windowed lane's stripe (multiple of 16)", "32768"), - Variable("SIRCL_SUBSET_RING", "1: experimental chain/ring on a contiguous cycle subset leaving at most two " - "adjacent non-members, with conservative relay windows", "0"), + Variable("SIRCL_SUBSET_RING", "1: experimental direct chain for flat all-gather shards >= 4 MiB on a " + "contiguous cycle subset leaving at most two adjacent non-members; other ops keep pieces", "0"), Variable("SIRCL_HAIRPIN_QUEUE_BYTES", "bytes of one relay hairpin queue (windows keep 75 % of it)", "524288"), Variable("SIRCL_PACKS_PER_THREAD", "16-byte packs per thread when sizing the grid, 1-64", "2"), Variable("SIRCL_POST_ORDER", "posting order of the progress thread's lanes: rank, ring-farthest, farthest " diff --git a/spark_transport/sircl/sparkring_sircl/oneshot/runtime.py b/spark_transport/sircl/sparkring_sircl/oneshot/runtime.py index 2b7d5be4..99a0ea0f 100644 --- a/spark_transport/sircl/sparkring_sircl/oneshot/runtime.py +++ b/spark_transport/sircl/sparkring_sircl/oneshot/runtime.py @@ -508,8 +508,7 @@ def __init__( if error is None: try: self._allocate() - with _environment(SIRCL_POST_ORDER=self._post_order_text, - **({"SIRCL_FORWARD_PROOF": "0"} if self.subset_ring else {})): + with _environment(SIRCL_POST_ORDER=self._post_order_text): self._proxy = Proxy( world_size=self.world_size, rank=self.rank, hca_names=self.hca_names, peer_lane_devices=self._lane_devices, lane_count=self.lane_count, @@ -779,8 +778,7 @@ def _configure(self, max_size, max_gather_bytes, hca_names, peer_routes, gid_ind # Large messages and forward windows of relayed lanes. self.forward_window_bytes = _env_int("SIRCL_FORWARD_WINDOW_BYTES", default=DEFAULT_FORWARD_WINDOW_BYTES) - self.forward_chunk_bytes = _env_int("SIRCL_FORWARD_CHUNK_BYTES", default=( - subset_mod.FORWARD_CHUNK if self.subset_ring else DEFAULT_FORWARD_CHUNK_BYTES)) + self.forward_chunk_bytes = _env_int("SIRCL_FORWARD_CHUNK_BYTES", default=DEFAULT_FORWARD_CHUNK_BYTES) self.hairpin_queue_bytes = _env_int("SIRCL_HAIRPIN_QUEUE_BYTES", default=DEFAULT_HAIRPIN_QUEUE_BYTES) if self.forward_window_bytes < 0: raise ValueError("SIRCL_FORWARD_WINDOW_BYTES must not be negative (0 turns forward windows off)") @@ -800,6 +798,8 @@ def _configure(self, max_size, max_gather_bytes, hca_names, peer_routes, gid_ind self.large_schedule = _env_text("SIRCL_LARGE_SCHEDULE", default="auto") if self.large_schedule not in LARGE_SCHEDULES: raise ValueError(f"SIRCL_LARGE_SCHEDULE {self.large_schedule!r} is not one of {LARGE_SCHEDULES}") + if self.subset_ring: + self.large_schedule = subset_mod.schedule(self.large_schedule, "reduce") self.chain_slots = _env_int("SIRCL_CHAIN_SLOTS", default=DEFAULT_CHAIN_SLOTS) self.chain_slot_bytes = _env_int("SIRCL_CHAIN_SLOT_BYTES", default=self._table_settings.get( "SIRCL_CHAIN_SLOT_BYTES", DEFAULT_CHAIN_SLOT_BYTES)) @@ -833,6 +833,8 @@ def _configure(self, max_size, max_gather_bytes, hca_names, peer_routes, gid_ind self.gather_schedule = _env_text("SIRCL_GATHER_SCHEDULE", default="auto") if self.gather_schedule not in GATHER_SCHEDULES: raise ValueError(f"SIRCL_GATHER_SCHEDULE {self.gather_schedule!r} is not one of {GATHER_SCHEDULES}") + if self.subset_ring: + self.gather_schedule = subset_mod.schedule(self.gather_schedule, "gather") self.link_slots = _env_int("SIRCL_LINK_SLOTS", default=self._table_settings.get( "SIRCL_LINK_SLOTS", proto.default_link_slots(self.world_size))) configured = {name: _env_int(name, default=0) @@ -872,6 +874,8 @@ def _configure(self, max_size, max_gather_bytes, hca_names, peer_routes, gid_ind self.scatter_schedule = _env_text("SIRCL_SCATTER_SCHEDULE", default="pieces") if self.scatter_schedule not in SCATTER_SCHEDULES: raise ValueError(f"SIRCL_SCATTER_SCHEDULE {self.scatter_schedule!r} is not one of {SCATTER_SCHEDULES}") + if self.subset_ring: + self.scatter_schedule = subset_mod.schedule(self.scatter_schedule, "scatter") # The schedules the session was configured with: a built-in plan decides a collective's ops only # while it keeps this schedule, so a schedule a caller sets at run time wins over the plan. self._configured_schedules = {"all_reduce": self.large_schedule, "all_gather": self.gather_schedule, @@ -1120,6 +1124,9 @@ def _apply_choice(self, collective: str, choice: tuning_mod.Choice, nbytes: int) hold, a grid above its counters, an all-reduce algorithm for a message above the capacity, which those algorithms run in one op). Every rank has the same settings, so every rank decides alike.""" kind = {"all_reduce": "reduce", "all_gather": "gather", "reduce_scatter": "scatter"}.get(collective) + if getattr(self, "subset_ring", False) and choice.schedule in ("chain", "ring"): + if choice.schedule == "ring" or kind != "gather" or nbytes < subset_mod.GATHER_MIN_BYTES: + return False if choice.grid is not None and choice.grid > self._counter_layout.blocks: return False if collective == "all_reduce" and choice.algorithm is not None and nbytes > self.max_size: @@ -1413,11 +1420,12 @@ def _plan_relays(self, route_maps: Sequence[Mapping[int, Sequence[str]]]) -> Non self.relay_safe_bytes = None if layout is None: return - subset_ring = getattr(self, "subset_ring", False) + if getattr(self, "subset_ring", False): + subset_mod.validate(layout, route_maps, queue_bytes=self.hairpin_queue_bytes) self._forward_table = routes_mod.forward_windows( layout, route_maps, self.rank, max_window=self.forward_window_bytes, chunk=self.forward_chunk_bytes or DEFAULT_FORWARD_CHUNK_BYTES, - queue_bytes=self.hairpin_queue_bytes, subset_ring=subset_ring) + queue_bytes=self.hairpin_queue_bytes) if self.forward_window_bytes: return queues = routes_mod.relay_queues(layout, route_maps) @@ -1474,16 +1482,18 @@ def _plan_ring(self, order: Sequence[int], index: int, relays. During a ring op the ring's lanes are the only relayed traffic, and relays forward each function's lanes through that function's own hairpin queue, so the ring runs when no relay queue carries two ring lanes (``routes.ring_window``); each relayed ring lane then keeps - at most 75 % of a queue unacknowledged; subset mode uses its smaller - packet envelope and excludes external relayed traffic by admission. Every rank decides the same: the decision follows the + at most 75 % of a queue unacknowledged. Subsets never configure a ring. + Every rank decides the same: the decision follows the agreed layout and route maps. A ring schedule named at construction makes a ring that cannot run a setup error. """ + if getattr(self, "subset_ring", False): + self.ring_problem = "subset mode permits only direct chain all-gather links" + return -1, -1, 0 world = len(order) window, problems = routes_mod.ring_window(self._layout_identity_object, route_maps, order, chunk=proto.LINK_WINDOW_CHUNK, - queue_bytes=self.hairpin_queue_bytes, - subset_ring=getattr(self, "subset_ring", False)) + queue_bytes=self.hairpin_queue_bytes) if problems: self.ring_problem = "; ".join(problems) for name, schedule in (("SIRCL_LARGE_SCHEDULE", self.large_schedule), @@ -2043,6 +2053,9 @@ def gather_uses_chain(self, inp: torch.Tensor, dim: int = -1, *, mode: Optional[ return self._gather_uses_chain(inp, dim) def _gather_uses_chain(self, inp: torch.Tensor, dim: int) -> bool: + if getattr(self, "subset_ring", False) and ( + inp.dim() != 1 or inp.numel() * inp.element_size() < subset_mod.GATHER_MIN_BYTES): + return False if not self.link_available or not self._gather_shape(inp, dim): return False shard = inp.numel() * inp.element_size() @@ -2118,6 +2131,8 @@ def _schedule(self, schedule: str, nbytes: int, collective: str) -> str: """A schedule as it runs for ``collective`` of ``nbytes`` (the all-reduce's message, the all-gather's output, the reduce-scatter's input): ``ring`` runs as ``auto`` without a ring and below ``ring_min_for(collective)``. The same on every rank for the same sizes.""" + if getattr(self, "subset_ring", False): + return subset_mod.schedule(schedule, collective) if schedule == "ring" and (not self.ring_available or nbytes < self.ring_min_for(collective)): return "auto" return schedule diff --git a/spark_transport/sircl/sparkring_sircl/ring/plan.py b/spark_transport/sircl/sparkring_sircl/ring/plan.py index fd7d9065..5c1d97f8 100644 --- a/spark_transport/sircl/sparkring_sircl/ring/plan.py +++ b/spark_transport/sircl/sparkring_sircl/ring/plan.py @@ -356,6 +356,8 @@ def ring_text(layout: routes_mod.Layout, maps, *, subset_ring: bool = False) -> """The ring that closes a group's chain: closed by cables, or through relays with one ring lane per relay hairpin queue (the sessions' check, ``routes.ring_window``), or why it cannot run.""" order = routes_mod.chain_order(layout, maps, subset_ring=subset_ring) + if subset_ring: + return f"no ring: subset mode permits only flat all-gather on direct chain {list(order)}" if order is None: return "no ring: the ranks do not form a chain of cable neighbors" window, problems = routes_mod.ring_window(layout, maps, order, subset_ring=subset_ring) @@ -838,10 +840,11 @@ def build_plan(site: Site, name: str, run_id: str, *, groups: Sequence[Sequence[ window = routes_mod.DEFAULT_FORWARD_WINDOW if options.forward_window is None else options.forward_window maps = [routes.route_map(rank) for rank in range(layout.world)] active_subset = subset_ring and layout.world < len(layout.fabric.positions) - chunk = int(settings.get("SIRCL_FORWARD_CHUNK_BYTES", subset.FORWARD_CHUNK if active_subset - else routes_mod.DEFAULT_FORWARD_CHUNK)) + if active_subset: + subset.validate(layout, maps) + chunk = int(settings.get("SIRCL_FORWARD_CHUNK_BYTES", routes_mod.DEFAULT_FORWARD_CHUNK)) windows = [routes_mod.forward_windows(layout, maps, rank, max_window=window, - chunk=chunk, subset_ring=active_subset) + chunk=chunk) for rank in range(layout.world)] paced = sorted({w for table in windows for row in table for w in row if w}) for label, sizes in (("all-reduce", options.allreduce_sizes), ("all-gather", options.allgather_sizes)): @@ -967,6 +970,11 @@ def tune_families_text(plan: ConfigurationPlan) -> list[str]: from .. import subset active_subset = subset.enabled(dict(options.session_env)) and layout.world < len(layout.fabric.positions) order = routes_mod.chain_order(layout, maps, subset_ring=active_subset) + if active_subset: + lines.append(f" tune families, group {group.index}: all_reduce: one-shot, two-shot, pieces{nccl}; " + f"all_gather: tiles, flat chain (shards >= 4 MiB){nccl}; " + "reduce_scatter: scatter ops; all_to_all: scatter ops; no ring candidates") + continue problems = routes_mod.ring_window(layout, maps, order, subset_ring=active_subset)[1] if order is not None else ["no chain"] links = (", chain" if order is not None else "") + (", ring" if order is not None and not problems else "") lines.append(f" tune families, group {group.index}: all_reduce: one-shot, two-shot, pieces{links}{nccl}; " diff --git a/spark_transport/sircl/sparkring_sircl/ring/worker.py b/spark_transport/sircl/sparkring_sircl/ring/worker.py index 3e7be4b5..307b6e6f 100644 --- a/spark_transport/sircl/sparkring_sircl/ring/worker.py +++ b/spark_transport/sircl/sparkring_sircl/ring/worker.py @@ -1260,6 +1260,7 @@ def with_blocks(choice: dict, variant: dict, blocks, key: str) -> tuple[dict, di return ({**choice, "blocks": blocks}, {**variant, key: blocks}) if blocks else (choice, variant) ring = bool(getattr(session, "ring_available", False)) + subset_chain = bool(getattr(session, "subset_ring", False)) if collective == "all_reduce": if size <= session.max_size: for algorithm in ("oneshot", "twoshot"): @@ -1298,7 +1299,8 @@ def with_blocks(choice: dict, variant: dict, blocks, key: str) -> tuple[dict, di for grid in grids: found.append((grid_choice({"schedule": "pieces"}, grid), "all_gather_large", shape, {"gather_schedule": "pieces", **({"grid": grid} if grid else {})})) - if large and session.link_available: + from .. import subset + if large and session.link_available and (not subset_chain or size >= subset.GATHER_MIN_BYTES): for piece in pieces: for blocks in link_blocks: choice, variant = with_blocks({"schedule": "chain", "piece": piece}, @@ -1321,7 +1323,7 @@ def with_blocks(choice: dict, variant: dict, blocks, key: str) -> tuple[dict, di for grid in grids: found.append((grid_choice({"schedule": "pieces"}, grid), "reduce_scatter", shape, {"scatter_schedule": "pieces", **({"grid": grid} if grid else {})})) - if large and session.link_available and self.scatter_through == "session": + if large and session.link_available and self.scatter_through == "session" and not subset_chain: for piece in pieces: for blocks in link_blocks: choice, variant = with_blocks({"schedule": "chain", "piece": piece}, diff --git a/spark_transport/sircl/sparkring_sircl/subset.py b/spark_transport/sircl/sparkring_sircl/subset.py index 042e50a1..a40d638a 100644 --- a/spark_transport/sircl/sparkring_sircl/subset.py +++ b/spark_transport/sircl/sparkring_sircl/subset.py @@ -1,9 +1,11 @@ -"""Admission and conservative queue budgets for independent subset sessions. +"""Admission and schedule policy for independent subset chain sessions. Only a cable-connected arc leaving at most two adjacent non-members is admitted. Those non-members have no relayed traffic in the universal shortest-path table, including when TP and DCP sessions run concurrently. No shared-queue reservation protocol is implemented; larger complements are refused even when idle. +The conservative ring/arena budget helper is retained for isolated tests; +the active chain policy adds no relayed links and preserves OFF arena pacing. """ from __future__ import annotations @@ -15,6 +17,8 @@ VARIABLE = "SIRCL_SUBSET_RING" FORWARD_CHUNK = 4096 +# Small flat gathers have no measured gain. Sizes are per-rank input bytes. +GATHER_MIN_BYTES = 4 << 20 # RC's smallest path MTU is 256. Allow 128 bytes per packet for framing, # transport headers, CRC and queue accounting. A signaled credit may cover # two WRs; each QP has at most 64 such credits (_roce_proxy.c). @@ -30,6 +34,17 @@ def enabled(environ: Mapping[str, str] | None = None) -> bool: return raw == "1" +def schedule(requested: str, collective: str) -> str: + """Subset links serve only flat all-gather; shape and size are checked by the runtime. + + A ring request becomes automatic chain selection, retaining the small-message + pieces fallback. Explicit pieces remains a way to disable chain all-gather. + """ + if collective != "gather": + return "pieces" + return "auto" if requested == "ring" else requested + + def order(layout: routes.Layout) -> tuple[int, ...]: """Admit a physical arc, including wrapping arcs and permuted rank order.""" fabric = layout.fabric @@ -68,7 +83,7 @@ def roles(route_map): def payload_budget(layout, maps, *, queue_bytes=routes.DEFAULT_HAIRPIN_QUEUE) -> int: - """One common payload allowance for each direction of every relay queue. + """Retained, inactive ring/arena allowance for each direction of every relay queue. Let n count all session lanes through q and its reverse. Reserve n*16384 bytes for short writes, short final packets and their ACKs. The remaining diff --git a/spark_transport/sircl/sparkring_sircl/vllm/serve/plan.py b/spark_transport/sircl/sparkring_sircl/vllm/serve/plan.py index 0631fb28..ee3ffebb 100644 --- a/spark_transport/sircl/sparkring_sircl/vllm/serve/plan.py +++ b/spark_transport/sircl/sparkring_sircl/vllm/serve/plan.py @@ -972,7 +972,7 @@ def relay_conflicts(arguments: Sequence[str], environment: Mapping[str, str]) -> def add_schedule_arguments(command: object) -> None: """``--large-schedule``, ``--gather-schedule`` and ``--scatter-schedule`` on an argparse (sub)parser.""" command.add_argument("--subset-ring", action="store_true", - help="experimental subset chain/ring with isolated relay queues") + help="experimental subset chain for flat all-gather; pieces for other operations") for flag, what in (("--large-schedule", "all-reduces above the dispatch ceiling (all_reduce_large)"), ("--gather-schedule", "all-gathers above the gather op size (all_gather_large)"), ("--scatter-schedule", "reduce-scatters")): @@ -2128,10 +2128,12 @@ def session_settings(group: fabric.GroupTopology, *, name: str, groups: str, sco from ... import subset try: subset.validate(layout, maps) - routes_mod.forward_windows(layout, maps, 0, chunk=subset.FORWARD_CHUNK, subset_ring=True) covers = True except routes_mod.RouteError as error: subset_problems.append(str(error)) + for key, collective in (("large_schedule", "reduce"), ("gather_schedule", "gather"), + ("scatter_schedule", "scatter")): + effective[key] = subset.schedule(effective[key], collective) order = routes_mod.chain_order(layout, maps, subset_ring=subset_ring) if covers else None large, gather, scatter = (effective[key] for key in DEFAULT_SCHEDULES) chain_area = covers and large != "pieces" @@ -2143,7 +2145,7 @@ def session_settings(group: fabric.GroupTopology, *, name: str, groups: str, sco flags = {key: "--" + key.replace("_", "-") for key in DEFAULT_SCHEDULES} ring_problems: list[str] = [] window = 0 - if link_area and order is not None: + if link_area and order is not None and not subset_ring: window, ring_problems = routes_mod.ring_window(layout, maps, order, chunk=protocol.LINK_WINDOW_CHUNK, queue_bytes=routes_mod.DEFAULT_HAIRPIN_QUEUE, subset_ring=subset_ring) @@ -2181,7 +2183,9 @@ def held(stagger: int) -> bool: problems.append(f"--ring-gather-stagger {ring_gather_stagger}: the {name} session's {world} ranks need " f"{protocol.ring_stagger_slots(world, ring_gather_stagger)} link slots for it, and it " f"has {slots} (--link-slots)") - if not link_area: + if subset_ring: + ring = "disabled (subset mode uses direct chain all-gather only)" + elif not link_area: ring = "none (no link area)" elif order is None or ring_problems: ring = "cannot run: " + ("; ".join(ring_problems) or "the ranks form no chain of cable neighbors") diff --git a/spark_transport/sircl/tests/subset_chain_driver.py b/spark_transport/sircl/tests/subset_chain_driver.py new file mode 100644 index 00000000..bf16bc82 --- /dev/null +++ b/spark_transport/sircl/tests/subset_chain_driver.py @@ -0,0 +1,79 @@ +"""Exercise subset policy through real runtime setup, with CUDA imports mocked.""" + +# CUDA stand-ins must be installed before importing the runtime. +# ruff: noqa: E402 + +import os +import sys + +from teardown_driver import _CudaStandIn + +sys.meta_path.insert(0, _CudaStandIn()) + +import torch + +from plan_driver import configured +from sparkring_sircl import routes, tuning +from sparkring_sircl.oneshot import runtime + + +def main(): + layout = "ring:8:0,1,2,3,4,5" + for windows in ("0", "131072"): + os.environ["SIRCL_FORWARD_WINDOW_BYTES"] = windows + os.environ["SIRCL_SUBSET_RING"] = "0" + off = configured(runtime, routes, layout, 0) + os.environ["SIRCL_SUBSET_RING"] = "1" + on = configured(runtime, routes, layout, 0) + assert on._forward_table == off._forward_table, "pieces windows changed" + assert on.forward_chunk_bytes == off.forward_chunk_bytes, "transport chunk changed" + assert on.gather_piece_bytes == off.gather_piece_bytes, "gather pieces changed" + assert on.large_piece_bytes == off.large_piece_bytes, "reduce pieces changed" + assert on.relay_safe_bytes == off.relay_safe_bytes + assert on.link_available and not on.chain_available and not on.ring_available + assert on.ring_window_bytes == 0 + for collective in ("all_reduce", "reduce_scatter", "all_gather"): + assert not on._apply_choice(collective, tuning.Choice(schedule="ring"), 8 << 20) + if collective != "all_gather": + assert not on._apply_choice(collective, tuning.Choice(schedule="chain"), 8 << 20) + assert not on._apply_choice("all_gather", tuning.Choice(schedule="chain"), 1 << 20) + assert on._apply_choice("all_gather", tuning.Choice(schedule="chain"), 4 << 20) + for schedule in ("auto", "pieces", "chain", "ring"): + on.large_schedule = on.scatter_schedule = on.gather_schedule = schedule + for size in (16, 1 << 20, 8 << 20, 256 << 20): + assert on.large_reduce_plan(size) == off.large_reduce_plan(size) + assert on._schedule(schedule, size, "scatter") == "pieces" + for shape, dim, expected in (((2 << 20,), 0, schedule != "pieces"), + ((8 << 20,), -1, schedule != "pieces"), + ((524288,), 0, False), ((1024, 2048), 1, False), + ((1, 2 << 20), 0, False), ((2048, 1024), 0, False), + ((3, 7), 1, False)): + inp = torch.empty(shape, dtype=torch.bfloat16) + assert on._gather_uses_chain(inp, dim) == expected, (schedule, shape) + assert not on._gather_uses_ring(inp, dim) + # Constructor requests and full-fabric opt-ins use the same policy as per-op overrides. + os.environ["SIRCL_FORWARD_WINDOW_BYTES"] = "131072" + for schedule in ("auto", "pieces", "chain", "ring"): + for key in ("SIRCL_LARGE_SCHEDULE", "SIRCL_GATHER_SCHEDULE", "SIRCL_SCATTER_SCHEDULE"): + os.environ[key] = schedule + for rank in range(6): + s = configured(runtime, routes, layout, rank) + assert not s.ring_available and not s.chain_available + assert s.link_available == (schedule != "pieces") + assert s.large_schedule == s.scatter_schedule == "pieces" + assert s.gather_schedule == ("auto" if schedule == "ring" else schedule) + full = "ring:8:0,1,2,3,4,5,6,7" + os.environ["SIRCL_SUBSET_RING"] = "0" + off = configured(runtime, routes, full, 0) + os.environ["SIRCL_SUBSET_RING"] = "1" + on = configured(runtime, routes, full, 0) + assert not on.subset_ring + assert (off.large_schedule, off.gather_schedule, off.scatter_schedule, off._forward_table, + off.ring_window_bytes, off.chain_available, off.ring_available) == ( + on.large_schedule, on.gather_schedule, on.scatter_schedule, on._forward_table, + on.ring_window_bytes, on.chain_available, on.ring_available) + print("subset schedule and OFF pieces parity passed") + + +if __name__ == "__main__": + main() diff --git a/spark_transport/sircl/tests/test_subset_ring.py b/spark_transport/sircl/tests/test_subset_ring.py index 255fc07a..436dccf3 100644 --- a/spark_transport/sircl/tests/test_subset_ring.py +++ b/spark_transport/sircl/tests/test_subset_ring.py @@ -1,4 +1,4 @@ -"""Subset admission, universal lanes, queue envelopes and native progress.""" +"""Active subset chain policy plus retained, inactive ring budget/progress helpers.""" import json import os @@ -95,7 +95,7 @@ def test_wrapped_and_permuted_rank_order(): assert routes.chain_order(layout, maps, subset_ring=True) == (4, 5, 1, 3, 0, 2) -def test_unsafe_overrides_fail_closed(): +def test_retained_budget_helpers_reject_unsafe_overrides(): layout, maps = layout_maps(8, range(6)) with pytest.raises(routes.RouteError, match="forward windows"): routes.forward_windows(layout, maps, 0, max_window=0, subset_ring=True) @@ -116,8 +116,9 @@ def test_planner_and_p2p_reservation(): on = plan.session_settings(group, name="tp", groups="", scoped=True, subset_ring=True, schedules={"large_schedule": "ring", "gather_schedule": "ring"}) assert not off.chain_area and not off.link_area - assert on.chain_area and on.link_area and not on.problems - assert on.chain_order == tuple(range(6)) and "65,536" in on.ring + assert not on.chain_area and on.link_area and not on.problems + assert on.chain_order == tuple(range(6)) and "disabled" in on.ring + assert on.schedules == {"large_schedule": "pieces", "gather_schedule": "auto", "scatter_schedule": "pieces"} layout, _ = layout_maps(8, range(6)) lanes = budget.LaneSet.of("tp", layout) reserved = budget.session_reservation(lanes, budget.SessionSettings.from_env({subset.VARIABLE: "1"})) @@ -150,7 +151,68 @@ def test_harness_on_and_off_share_the_universal_fabric(): assert records[0].groups[0].layout == records[1].groups[0].layout assert records[0].groups[0].route_texts == records[1].groups[0].route_texts assert "no ring" in records[0].groups[0].ring - assert "65536" in records[1].groups[0].ring + assert "no ring" in records[1].groups[0].ring + assert "direct chain" in records[1].groups[0].ring + assert records[0].groups[0].lanes_detail == records[1].groups[0].lanes_detail + + +def test_subset_chain_runtime_selection_and_pieces_parity(): + pytest.importorskip("torch") + env = {k: v for k, v in os.environ.items() if not k.startswith("SIRCL_")} + env["PYTHONPATH"] = str(Path(__file__).parents[1]) + result = subprocess.run([sys.executable, str(Path(__file__).with_name("subset_chain_driver.py"))], + env=env, capture_output=True, text=True, timeout=120, check=False) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("large", ("auto", "pieces", "chain", "ring")) +@pytest.mark.parametrize("gather", ("auto", "pieces", "chain", "ring")) +def test_subset_planner_limits_explicit_schedules(large, gather): + group = fabric.describe_group(fabric.Layout.ring(8), list(range(6)), parent=list(range(8))) + settings = plan.session_settings(group, name="tp", groups="", scoped=True, subset_ring=True, + schedules={"large_schedule": large, "gather_schedule": gather, + "scatter_schedule": large}) + assert not settings.problems and not settings.chain_area + assert settings.link_area == (gather != "pieces") + assert settings.schedules == {"large_schedule": "pieces", "scatter_schedule": "pieces", + "gather_schedule": "auto" if gather == "ring" else gather} + assert "disabled" in settings.ring + + +def test_subset_tuner_only_offers_measurable_chain_gathers(): + from test_op_blocks import harness + + built = harness() + built.world = 6 + built.session.subset_ring = True + built.session.chain_available = built.session.ring_available = False + for collective, size, expected in (("all_reduce", 8 << 20, {"pieces"}), + ("reduce_scatter", 6 << 20, {"pieces"}), + ("all_gather", 1 << 20, {"pieces"}), + ("all_gather", 4 << 20, {"pieces", "chain"})): + assert {choice.get("schedule") for choice, *_ in built.tune_candidates(collective, size)} == expected + + +@pytest.mark.parametrize("n", range(4, 9)) +@pytest.mark.parametrize("lanes", (1, 2)) +def test_active_chain_adds_zero_relay_bytes_to_legacy_pieces_bound(n, lanes): + for k in range(max(3, n - 2), n): + for start in range(n): + positions = tuple((start + i) % n for i in range(k)) + layout, maps = layout_maps(n, positions, lanes) + order = subset.validate(layout, maps) + derived = routes.derive_routes(layout, lanes) + # Both directions include data and returning ACKs. No closing link exists. + for a, b in zip(order, order[1:]): + assert all(not lane.relays for lane in (*derived.lanes_to(a, b), *derived.lanes_to(b, a))) + outside = sorted(set(range(n)) - set(positions)) + if len(outside) == 2: + other_layout, other_maps = layout_maps(n, outside, lanes) + assert not routes.relay_queues(other_layout, other_maps) + tables = [routes.forward_windows(layout, maps, r) for r in range(k)] + for members in routes.relay_queues(layout, maps).values(): + payload = sum(tables[r][peer][lane] for r, peer, lane in members) + assert payload <= int(routes.RELAY_QUEUE_SHARE * routes.DEFAULT_HAIRPIN_QUEUE) def test_subset_setting_must_agree_even_when_rank_zero_omits_it(): @@ -178,7 +240,7 @@ def test_off_plan_and_setup_record_bytes_match_origin_main(value): @pytest.mark.parametrize("active", (False, True)) @pytest.mark.parametrize("n,k", [(n, k) for n in range(4, 9) for k in range(max(3, n - 2), n)]) -def test_admitted_subset_native_progress(simulator_library, n, k, active): +def test_retained_subset_ring_helper_native_progress(simulator_library, n, k, active): from sparkring_sircl.testing import fabric as sim layout, _ = layout_maps(n, range(k)) From 667f002c922b3646e13f8c6569a095ce8895842c Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:08:27 -0500 Subject: [PATCH 3/5] Add advisory SIRCL placement selection and shared-QP window proof Estimate schedules, lane paths and conditional queue windows offline, with placement-qualified measurement overrides. Derive a 192 KiB subset ring bound while retaining the direct-chain runtime policy and all existing defaults. Document model misses, proof assumptions and hardware validation plans. Validate with the SIRCL suite, native progress and placement tests. --- docs/architecture/sircl-schedule-selection.md | 295 ++++++++++++++ spark_transport/sircl/SUBSET_RING.md | 74 ++++ .../sparkring_sircl/schedule_selection.py | 370 ++++++++++++++++++ .../sircl/sparkring_sircl/subset.py | 54 +++ .../sircl/tests/test_schedule_selection.py | 201 ++++++++++ 5 files changed, 994 insertions(+) create mode 100644 docs/architecture/sircl-schedule-selection.md create mode 100644 spark_transport/sircl/sparkring_sircl/schedule_selection.py create mode 100644 spark_transport/sircl/tests/test_schedule_selection.py diff --git a/docs/architecture/sircl-schedule-selection.md b/docs/architecture/sircl-schedule-selection.md new file mode 100644 index 00000000..92bf9048 --- /dev/null +++ b/docs/architecture/sircl-schedule-selection.md @@ -0,0 +1,295 @@ +# SIRCL placement-aware schedule estimates + +**Experimental, research-only.** The offline selector estimates schedules and +per-lane windows for two to eight Sparks. It prints conditional queue certificates, +paths, rejected candidates and predictions. It does not configure a session, +contact a Spark, install routes, or replace any runtime tuning decision. + +From the repository root, with `spark_transport/sircl` on `PYTHONPATH`: + +```bash +python -m sparkring_sircl.schedule_selection --fabric ring8 --placement 0-5 --co-resident 6-7 +python -m sparkring_sircl.schedule_selection --fabric ring8 --placement 0-3 --co-resident 4-7 --collective all_gather +python -m sparkring_sircl.schedule_selection --fabric-document fabric.json --placement 0-7 --sizes 4096,65536,4194304,33554432 +``` + +The implementation is [schedule_selection.py](../../spark_transport/sircl/sparkring_sircl/schedule_selection.py). +The separate [subset policy](../../spark_transport/sircl/SUBSET_RING.md) remains +direct chain for large flat all-gather and pieces for large all-reduce. +The selector's estimates are hypotheses for measurement, including when a +candidate uses an operation or reservation policy that runtime does not support. + +## Inputs and identity + +The inputs are the fabric, ordered rank placement, disjoint co-resident placements, +collective, per-rank input bytes, eager/graph mode and flat/non-flat gather shape. +All-reduce bytes mean the full input; gather bytes mean one rank's shard; +reduce-scatter bytes mean the full input before scattering. Sizes are positive +multiples of 16. The CLI accepts inclusive ranges and explicit rank permutations. +Pair means two cables between two Sparks; `path2` means one cable. + +For a setup document, the selector uses the existing `sparkring-fabric/v1` +validator. It preserves the cabling, fabric identity, complete document digest +and installed relay-plan reference. It validates uniform function names. The +document records a relay-plan digest, not its forwarding rules: path estimates +assume the universal same-function shortest-path table that SIRCL derives from +those cables. A relayed candidate is refused when the document has no SIRCL +relay plan. Verification of the live table against that reference is outside +this offline program. Synthetic `ringN`/`pathN` inputs model that table without +asserting that any fabric has been configured. + +The output key includes physical cables, the document/reference when supplied, +ordered placement, every co-resident placement, collective, mode, gather shape, +model revision, native and kernel source hashes, and SIRCL version/ABI. Positions +are not reduced to a shape label: 0-5 beside 6-7 is distinct from an isolated +six-Spark cycle. Overlapping deployments, duplicate ranks, invalid cables and +out-of-range positions are rejected. Unlisted positions are assumed idle. +Multiple independent sessions on the same positions are outside this model; +their QPs need explicit additional reservations before admission. + +## Candidate paths and schedules + +Each candidate chooses one or two lanes. Paths come from `routes.derive_lanes`: +the shortest physical route, or the existing primary/secondary split over tied +shortest routes. It never invents a forwarding route absent from the modeled +table. The physical direct-chain order is derived independently of rank order. +The first implementation does not search longer alternate paths, all Hamiltonian +orders, kernel grids, pieces, or channel counts. Those require additional +measurements and, for alternate routes, a matching installed relay plan. + +| Candidate | Modeled bytes per directed peer/edge | Admission | +|---|---|---| +| One-shot all-reduce | S to each other rank | S <= 128 KiB | +| Two-shot all-reduce | 2S/N to each other rank | S <= 2 MiB | +| Pieces all-reduce | 2S/N to each other rank | Any modeled size | +| Pieces gather / scatter | S / S/N to each other rank | Any modeled size | +| Ring reduce / gather / scatter | 2(N-1)S/N / (N-1)S / (N-1)S/N per successor edge | A direct chain order exists; closing route bounded; at most one ring lane per queue | +| Chain all-reduce | S in each direction on each chain edge | Every chain edge direct | +| Chain gather / scatter | At cut i, iS and (N-i)S in opposite directions, divided by N for scatter | Direct chain; gather must be flat | + +The size ceilings are model capacities, not a claim about an installed session's +allocation. A route with more than the supported three relays is refused. On a +path of eight, for example, a direct chain remains modelable while a ring's +closing path is refused. Some sparse placements have no admitted candidate; +the JSON records the reasons and the CLI exits 1. An unsafe fallback is never +substituted. One-shot and two-shot are distinct candidates for all-reduce only; +the gather/scatter pieces models include their existing small-operation paths. + +## Cost model and calibration + +For each candidate, the selector accumulates traffic on each directed cable and +on each endpoint NIC's send and receive resources. The two PCIe functions share +their cable's bound. Cable directions and NIC sends allow 24 GB/s; NIC receives +allow 26.8 GB/s. Pure hardware forwarding is charged to cable resources and +hairpin queues, not to endpoint DMA. Rates use decimal GB/s. + +The predicted period is a launch/stage term plus the maximum of resource time +and each relayed lane's `lane_volume * RTT(window, relays) / window`. The launch +term is 5 us plus 1.5 us per dependency stage. Pieces adds 1.5 us per additional +2 MiB piece. These constants are assumptions, not measured kernel timings. +Eager and graph have separate measurement keys but share this provisional +latency estimate. Memory/reduction throughput, overlapping kernels and receiver +backpressure are not fitted. Ties prefer fewer lanes; this is an explicit model +tie-break, not evidence that one lane outperforms two on hardware. + +Co-resident groups reserve all-pairs traffic at two lanes per peer. The model +divides a shared resource's bandwidth equally among groups using it. Placement +alone cannot determine the other deployment's offered load. This equal-share +scenario is neither a bandwidth guarantee nor a DCP2 performance prediction. +Queue safety is independent of that fairness assumption. + +Calibration uses the retained `ring8-hw-20261013` records: + +| Record | Observation | Use | +|---|---|---| +| `sircl-subset-ring/RESULTS.md`, cb519293 | TP6 ring, 65,536 B/lane, about 9.4 GB/s; pieces about 11.5 GB/s | Effective two-lane RTT inferred as 13.94 us | +| `net-sircl-0.1.1-profile2/RESULTS.md`, SIRCL section | Path4 closing edge, 393,216 B/lane, 21.11 GB/s at 256 MiB eager | Effective RTT inferred as 37.25 us | +| `net-sircl-0.1.1-profile2/RESULTS.md`, net-sircl C | 163,840-byte allowance, measured post-to-CQE 40.3 us | Profiled completion-latency reference | +| `net-sircl-0.1.1-route/RESULTS.md`, DP | 245,760-byte allowance, 46.4 us | RTT grows with flight size | +| `net-sircl-0.1.1-smallsizes/RESULTS.md`, DP | 57.0 us, changed posting policy/load | Congestion sensitivity, not an independent byte-size anchor | +| `net-sircl-0.1.1-round9/RESULTS.md` | Cap 8 beat 16/24/64 on the relayed path | Avoid assuming more channels create bandwidth; channel search is deferred | + +For two relays, `RTT_us = 13.94 + (W-65536)*(37.25-13.94)/327680`. +The code retains the unrounded anchor values. For another relay count h it +adds `4*(h-2)` us and floors at 4 us. Hop scaling is an extrapolation. +The congested estimate multiplies RTT by `57/46.4`. Neither estimate is a +confidence interval. The raw net-sircl profile is perturbed by instrumentation +and a different transport; using 40 us as SIRCL's RTT would predict only +3.28 GB/s at 64 KiB, contradicting the measured 9.4 GB/s. The inferred SIRCL +calibration reconciles that discrepancy explicitly; it is not a direct RTT +measurement. Two fitted anchors are not independent validation. + +## Conditional queue proof + +A queue is `(relay position, PCIe function class, egress port)`. With minimum +MTU 256 B and maximum framing/queue-accounting allowance 128 B per packet, +each byte of full-packet data consumes at most 1.5 bytes in its data direction +and generates at most 0.5 bytes of ACK traffic in the reverse direction. +Short tails, flags, software credits and their ACKs reserve +`C(q) = 16384 * (number of QPs through q + number through reverse(q))`. +The 16,384 B comes from 64 native credits, at most two WRs per credit and +128 B of control/tail allowance per WR, following the subset helper model. + +For summed windows D(q), every candidate must satisfy: + +```text +1.5 * D(q) + 0.5 * D(reverse(q)) + C(q) <= 524288 +``` + +ACK-only queues are included. `packet_windows` divides the residual capacity +among the forward and reverse lanes, rounds down to chunks, then constrains +each lane by every data and ACK queue it traverses. It recomputes each final +inequality and refuses zero-chunk relayed lanes. Windows are also limited to +60 chunks. Ring candidates use 32 KiB chunks; others use 4 KiB chunks. The +output includes reservations for co-resident groups as well as the target. +These are joint requirements, not reservations that existing deployments have +accepted. A measurement never bypasses the queue proof. + +The proof assumes completion-only release (`SIRCL_FORWARD_PROOF=0`), no retained +retransmission copies, the stated packet-accounting bound, and no unknown +relayed sessions. Hardware qualification must check those assumptions. It is +not an unconditional statement about physical NIC occupancy. It also assumes +all arena/link payload and ACKs are drained before switching candidate paths +or windows. The runtime does not implement this placement-wide transition +protocol. Consequently **no selector output is an executable session policy**. +In particular, copying a selected ring window into an existing session is unsafe. + +The active subset runtime preserves OFF arena pacing and delivery-proof release. +Its simultaneous arena/ring bound is different from a drained offline candidate. +[The subset bound](../../spark_transport/sircl/SUBSET_RING.md#larger-window-analysis) +accounts for that overlap: at TP6 on 0-5 the maximum common 32 KiB-aligned ring +window in the conservative fixed-arena shared-QP envelope is 196,608 B, not 393,216 B. + +## Measurements override the model + +`--measurements FILE` accepts `sircl-placement-measurements/v1` with the exact +`key` printed by the selector and a `rows` array. Each row names `bytes`, the +candidate's full `id`, positive finite `period_us`, `exact: true`, `drops: 0`, +`counters_valid: true`, `isolation_passed: true`, and a nonempty `evidence` +reference. A candidate ID includes a hash of its paths and joint windows. +The fastest admitted measured candidate wins over all model predictions at +that exact size. Unmeasured sizes use the model; no measured interval is +extrapolated. Wrong keys and failed gate records are rejected. Unknown candidate +IDs cannot win. This small research schema is separate from runtime v2 tables; +existing measured defaults are comparison evidence, not silently imported as +placement-qualified measurements. + +Measured selection also requires `--conditions FILE`, a JSON object recording +`image`, `drivers`, `dtype`, `shape`, `rotate_buffers` and `co_workload`. These +values become part of the exact key. Use explicit `none` descriptions for +absent co-resident load; do not leave measurement conditions implicit. Model-only +queries need no conditions file. A row/padded shape description must include +its row geometry, not merely say `non-flat`. + +### Comparison with retained choices + +These are model outputs at graph-mode input sizes, not timing measurements. +Runtime rows come from `runtime/common/sircl-tuning-defaults.json`, the cycle-4 +v2 table it references, and SIRCL's built-in cycle-8 decisions. + +| Fabric / placement | Collective and bytes | Model schedule | Retained choice/evidence | Agreement | +|---|---|---|---|---| +| Pair | Reduce 4/32 MiB | Chain, one lane | Pair row: ring | Disagrees; model does not price chain kernel overhead | +| Path4 | Reduce 4/32 MiB | Ring, two lanes | Path-4 row: ring | Schedule agrees; modeled packet window differs | +| Cycle4 | Reduce 64 KiB / 4 MiB | Pieces / pieces | Measured v2: one-shot / ring | Disagrees at both sizes | +| Cycle4 | Reduce 32 MiB | Ring | Measured v2: ring | Schedule agrees | +| Cycle8 | Reduce 4/32 MiB | Ring, one lane | Built-in decisions: ring | Schedule agrees; lane tie-break is unmeasured | +| Ring8 0-5 beside 6-7 | Reduce 32 MiB | Chain, one lane | Runtime subset: pieces; measured 64 KiB ring window lost to pieces | Unvalidated chain hypothesis, not a runtime recommendation | +| Ring8 0-5 beside 6-7 | Flat gather 4 MiB / 16 MiB | Pieces / chain | Chain measured 1.20 / 1.32 times pieces | Misses the 4 MiB winner; agrees at 16 MiB | +| Cycle7 | Reduce 32 MiB | Ring, one lane | Generic rules, no measured row | Requires measurement | + +The 4 MiB gather candidates are close in this model (about 1,233 us pieces +versus 1,236 us chain). A narrow synthetic tie must not override the observed +chain gain. At 16 MiB, predicted chain bus bandwidth is 17.10 GB/s versus +16.33-16.34 measured. Predicted TP6 pieces at 64 MiB is 17.00 GB/s versus +11.45-11.46 measured: the missing kernel/transport costs materially matter. +This disagreement is a reason to measure placements, not to publish predictions +as defaults. The fitted path4 endpoint is 21.11 GB/s at 384 KiB, while the +selector's stricter 320 KiB packet envelope predicts about 20.09 GB/s at +256 MiB; the hardware record at 384 KiB measured 21.11-21.13 GB/s. + +`sparkring fabric tune` currently measures configured group shapes and stores +SIRCL v2 timing tables through the ring harness. The existing promotion process +is specified in [releases](../development/releases.md). It does not yet produce +the placement schema or coordinate joint reservations. A placement measurement +requires a harness run with explicit `--groups`, universal fabric scope, every +co-resident session and its workload recorded, exact eager/graph results, +per-queue counters and the candidate's actual lane/window receipt. A converter +must join those receipts with the exact selector key and candidate ID; this +converter and the reservation/transition runtime are not implemented here. + +An eventual install-time decision can enumerate paths, prove joint queue +reservations, prepare kernels, measure candidate schedules and bind a table +to fabric/build/co-residency. Runtime can then look up a size/collective/mode +decision within that admitted reservation. A topology, group or window change +requires drain and re-admission at install/session setup time. Runtime must +never independently enlarge a reservation based on a fast timing row. Today, +runtime retains its existing explicit overrides, matching tables and generic +rules; it does not import this selector. + +## Hardware validation plans + +### TP6 with DCP2, maximum 25-minute outage + +The owner frees positions 0-5 for the harness and reserves DCP2 on 6-7 for a +single fixed decode probe. No other DCP2 client is allowed. Keep the existing +NIC settings and universal relay table. Pre-stage source, image, commands, +receipt collection and rollback without changing the deployment. + +The retained hardware run required about 12 minutes to restore TP6. Reserve +13 minutes for restoration and 2 minutes for stop/start and idle/A/A baselines. +That leaves 10 minutes for three ABBA blocks (12 arms): **at most 50 seconds per +arm, including session setup and counters**. The prior 124-case arms took +3-4 minutes each and cannot fit. Use a reduced matrix: small all-reduce at +16 B/64 KiB, all-reduce at 8/64 MiB, flat gather at 4/16 MiB, padded/row gather +at 4 MiB, eager and graph, exactness before timing and eight rotated buffers. +Pilot the arm duration within the budget; if 50 seconds is infeasible, abort +and restore rather than calling a partial block a completed 25-minute test. + +A is pieces with the original pacing; B is the opt-in direct-chain policy. +There is no larger-window all-reduce B executable in this revision. A future +ring B requires a proved transition/arena policy and matching emulation first. +Do not substitute the drained advisory ring window. Each block is A-B-B-A; +pair adjacent A/B arms and compare each case, not only an aggregate mean. + +Snapshot all eight Sparks' drops, `rx_out_of_buffer`, relay-rule drops, sequence +errors, ACK timeouts and retries before/after each arm. Watch those counters +during execution. A reset or missing counter invalidates the arm. Require exact +results, zero counter increases, every case at least 0.97 times pieces +throughput (inverse latency ratio for small cases), and DCP2 decode within an +A/A noise band declared before B. If A/A is too noisy to assess a 3% change, +the test is inconclusive. Abort on a safety/error gate or at minute 12; restore +TP6 and verify both APIs by minute 25. The retained run's external DCP2 client +made its isolation conclusion inconclusive; it is not an isolation baseline. + +### Placement validation for fabric tune + +These are separate owner-approved reservations. No hardware action is part of +the offline implementation. Estimates include setup, exactness, candidate +timing, repeated winner/control arms and restoration. They assume a reduced +candidate set and warm build caches; expand the reservation before a full sweep. + +| Placement on ring8c | Owner must free or reserve | Estimated reservation | +|---|---|---| +| 0-5 plus 6-7 | Free TP6; DCP2 exclusive fixed workload, no external clients | 25 min reduced ABBA above; 60-75 min for a full placement sweep | +| 0-3 plus 4-7 | Free all eight; two controlled four-rank groups, idle and concurrent arms | 60-90 min | +| 0-7 | Free all eight; no serving co-residents | 45-60 min | +| Pairs | Free tested pair and every relay used by a nonadjacent pair; controlled complements | 20-30 min per distinct pair/overlap class; 60-90 min for adjacent, distance-2 and opposite classes | + +At each placement, measure 4 KiB, 28/64/128 KiB, 1/4/16/64 MiB for +all-reduce, flat gather and reduce-scatter; add padded gather controls and +boundary-adjacent sizes. Measure the model winner, pieces, the matching runtime +row and any competing candidate within 10% of predicted time. Record one/two +lanes separately. Use eager and graph, eight rotated buffers and at least +100 timed calls for large cases, with an A/A noise sample and three ABBA +blocks for each proposed promoted setting. First admit only settings executable +under the existing runtime proof. Unsupported advisory candidates are deferred, +not forced into `fabric tune` by an override. + +Save the setup document, relay-plan digest, ordered groups, workload descriptions, +image/driver/build receipts, actual windows, per-arm counters, exactness and +per-rank periods. Require the same safety, 0.97 per-case and co-resident noise +gates as above. Report a model miss even when another candidate passes. Do not +promote subset evidence to `path-4` or `cycle-8`: a placement-qualified table must +retain its fabric and co-resident key. Default behavior changes only through a +separate measured promotion after runtime admission exists. diff --git a/spark_transport/sircl/SUBSET_RING.md b/spark_transport/sircl/SUBSET_RING.md index 15c7c8d1..e169fe5e 100644 --- a/spark_transport/sircl/SUBSET_RING.md +++ b/spark_transport/sircl/SUBSET_RING.md @@ -80,6 +80,71 @@ and queue accounting per packet, no retained retransmission copies and credit release only on returning completion. It applies only to those explicitly bounded helper configurations, not the restored OFF pieces path. +## Larger-window analysis + +The offline helper `subset.ring_packet_bound` fixes the conservative arena +windows from `forward_windows(..., chunk=4096, subset_ring=True)` and solves +each queue separately for a common ring window R. It does not configure the +runtime. For each directed QP i, let A_i be its arena window. Its shared +native `unacked` counter bounds arena plus ring payload by `max(A_i, R)` +when it carries the ring, and by A_i otherwise. Let D(q,R) sum those bounds +for every QP traversing q. The inequality is: + +```text +1.5*D(q,R) + 0.5*D(reverse(q),R) + C(q) <= 524288 +``` + +This uses `_roce_proxy.c`'s `post_stream_write`, `link_pump_windowed` and +`ack_push`: both pumps test the same device/peer byte counter. With delivery +proof disabled, only returning completion releases those bytes. Payload on +other QPs can remain live. The helper includes ACK-only queues and finds the +largest common 32 KiB-aligned window satisfying every inequality. This is the +maximum **within this fixed-arena, shared-QP envelope**. Trading arena capacity +for ring capacity changes the optimization problem. Control credits are also +shared by arena and ring on each native QP and are counted once. + +For TP6 on positions 0-5 of ring8, the closing route is 5-6-7-0. A deployment +confined to 6-7 sends both its payload and ACKs over their direct cable, using +zero hairpin bytes. The limiting queues are the secondary function toward +port 0 on positions 6 and 7: + +| Quantity | Bytes | +|---|---:| +| Conservative arena windows in each direction (three lanes) | 86,016 | +| Control reserve (six directed QPs times 16,384) | 98,304 | +| Arena window on the QP that also carries the ring | 28,672 | +| Other forward arena QPs, still independently live | 57,344 | +| Fixed other-QP framing, reverse arena ACKs and controls | 227,328 | +| Residual for shared ring-QP payload and framing | 296,960 | +| Unrounded ring payload allowance, floor(296960 / 1.5) | 197,973 | +| Largest common 32 KiB-aligned window | **196,608** | +| Occupancy bound at that window | 522,240 | +| Occupancy at the next 32 KiB window (229,376) | 571,392: refused | + +This improves the retained helper's 65,536-byte common budget by three times. +It is not close to the normal 393,216-byte ring window. Even an isolated +393,216-byte data window would require 589,824 bytes with the minimum-MTU +framing assumption, before controls or reverse traffic. An isolated single +ring lane with no arena traffic can fit 327,680 bytes under this envelope; +that requires an arena/ACK drain, which the active subset runtime does not +establish. Larger MTU assumptions require negotiated-MTU enforcement and a +separate reviewed proof; measured absence of drops does not supply that proof. + +The active arena windows are larger than the conservative helper's and retain +delivery-proof release. Its proof assumptions therefore do not hold merely by +setting `SIRCL_SUBSET_RING=1`. Subset ring all-reduce stays disabled, and flat +all-gather keeps chain. No default or opt-in transport setting changes here. + +Using the inferred SIRCL window/RTT model in +[schedule selection](../../docs/architecture/sircl-schedule-selection.md), +196,608 bytes over two relays predicts 16.90 GB/s before kernel and cable/NIC +contention, or about 1.47 times 11.5 GB/s pieces. Applying the measured +57/46.4 RTT-growth sensitivity gives 13.76 GB/s, about 1.20 times pieces. +These are conditional steady-state predictions, not measured gains. A +co-resident DCP2 workload can further reduce available bandwidth. Using the +profiled net-sircl 40.3 us directly instead predicts 9.76 GB/s (0.85 times +pieces); the different transport/profiling conditions are material. + ## Validation gates Run `python -m pytest spark_transport/sircl -q`. The subset tests cover @@ -91,6 +156,15 @@ gathers and all-reduces, with delayed link progress and matching native ACK-release settings. Emulation does not measure hairpin drops or serving performance. +The offline selector tests cover every path/cycle size from two to eight, +representative subset and co-resident placements, odd cycles, ACK-only queues, +measurement precedence and rejection of unbounded candidates. The fixed-arena +ring bound is checked for every admitted subset, including failure of the +next larger common window. The selector's drained candidates do not change +runtime admission. Both hardware plans, including the restoration budget and +the 25-minute feasibility limit, are in the +[selection design](../../docs/architecture/sircl-schedule-selection.md#hardware-validation-plans). + Before promotion, coordinate a maximum 25-minute TP6 outage on positions 0-5, preserving DCP2 on 6-7 with no other client. Pre-stage artifacts and budget restoration time before taking TP6 down. Measure three ABBA blocks diff --git a/spark_transport/sircl/sparkring_sircl/schedule_selection.py b/spark_transport/sircl/sparkring_sircl/schedule_selection.py new file mode 100644 index 00000000..110feb24 --- /dev/null +++ b/spark_transport/sircl/sparkring_sircl/schedule_selection.py @@ -0,0 +1,370 @@ +"""Offline, conditional packet bounds and cost estimates; never imported by dispatch. + +Run ``python -m sparkring_sircl.schedule_selection --fabric ring8 --placement 0-5 +--co-resident 6-7``. Output is research advice, not a runtime configuration. +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import hashlib +import json +import math +from pathlib import Path +import re + +from . import routes, subset, tuning + +MODEL = "sircl-placement-model/v1" +MEASUREMENTS = "sircl-placement-measurements/v1" +COLLECTIVES = ("all_reduce", "all_gather", "reduce_scatter") +QUEUE = routes.DEFAULT_HAIRPIN_QUEUE +CHUNK = 4096 +LINK_CHUNK = 32768 + + +def positions(text): + """Ordered comma-separated positions and inclusive ascending ranges.""" + result = [] + for item in text.split(","): + if not re.fullmatch(r"\d+(?:-\d+)?", item): + raise ValueError(f"invalid placement {text!r}") + ends = [int(v) for v in item.split("-")] + if len(ends) == 2 and ends[1] < ends[0]: + raise ValueError("placement ranges must ascend") + result.extend(range(ends[0], ends[-1] + 1)) + if len(set(result)) != len(result): + raise ValueError("placement positions must be distinct") + return tuple(result) + + +def fabric_named(name): + if name == "pair": + return routes.Fabric.ring(2) + match = re.fullmatch(r"(ring|cycle|path)-?([2-8])", name) + if not match: + raise ValueError("fabric must be pair, ring2..ring8, cycle2..cycle8, or path2..path8") + kind, count = match.groups() + return routes.Fabric.path(range(int(count))) if kind == "path" else routes.Fabric.ring(int(count)) + + +def document_fabric(path): + """Read setup's validated document; no host access or environment mutation. + + Setup records the installed relay plan's digest, not its forwarding rules. + Predictions assume that plan implements SIRCL's universal shortest paths. + """ + from runtime.common import fabric_document + + document = fabric_document.load(path) + routes.roles_from_fabric_document(document) + cables = [] + for cable in document["cables"]: + a, b = cable["ends"] + cables.append(routes.Cable(a["position"], a["port"], b["position"], b["port"])) + return routes.Fabric(tuple(cables)), { + "id": document["id"], "document_sha256": fabric_document.digest(document), + "relay_plan": document["relays"], "transports": document["transports"], + } + + +def rtt_us(window, relays, *, congested=False): + """Effective SIRCL RTT inferred from bus bandwidth, not a measured latency. + + Two-relay anchors: 2*65536/9.4 GB/s and 2*393216/21.11 GB/s. + Other hop counts use an unmeasured 4 us per relay. The 57/46.4 factor + exposes the measured net-sircl queueing sensitivity, not a third flight-size + anchor: those observations also changed posting policy and channel load. + """ + low = 2 * 65536 / (9.4 * 1000) + high = 2 * 393216 / (21.11 * 1000) + value = max(4.0, low + (window - 65536) * (high - low) / (393216 - 65536) + + 4.0 * (relays - 2)) + return value * (57.0 / 46.4 if congested else 1.0) + + +def _reverse(key): + return (*key[:2], 1 - key[2]) + + +def _queue_keys(lane): + return tuple((s.dst, lane.local.secondary, 1 - s.dst_port) for s in lane.steps[:-1]) + + +def _all_lanes(fabric, members, lanes, group): + return {(group, r, p, lane.lane): lane + for r, a in enumerate(members) for p, b in enumerate(members) if r != p + for lane in routes.derive_lanes(fabric, a, b, lanes)} + + +def _order(fabric, members): + """A direct physical chain, independent of rank numbering.""" + for start in members: + for port in (0, 1): + order, here, out = [start], start, port + while len(order) < len(members): + step = fabric.cross(here, out) + if step is None or step.dst not in members or step.dst in order: + break + order.append(step.dst) + here, out = step.dst, 1 - step.dst_port + if len(order) == len(members): + return tuple(members.index(p) for p in order) + return None + + +def _traffic(schedule, collective, size, world, order): + traffic = {} + if schedule in ("oneshot", "twoshot", "pieces"): + volume = size if collective == "all_gather" or schedule == "oneshot" else size / world + if collective == "all_reduce" and schedule != "oneshot": + volume *= 2 + traffic = {(a, b): volume for a in range(world) for b in range(world) if a != b} + elif schedule == "ring": + volume = size * (world - 1) * (1 if collective == "all_gather" else 1 / world) + if collective == "all_reduce": + volume *= 2 + traffic = {(a, order[(i + 1) % world]): volume for i, a in enumerate(order)} + else: + for i, (a, b) in enumerate(zip(order, order[1:])): + traffic[a, b] = size if collective == "all_reduce" else size * (i + 1) + traffic[b, a] = size if collective == "all_reduce" else size * (world - i - 1) + if collective == "reduce_scatter": + traffic[a, b] /= world + traffic[b, a] /= world + return traffic + + +def packet_windows(lanes, *, queue_bytes=QUEUE, chunk=CHUNK): + """Equal-share, completion-only windows with data and reverse ACK bounds. + + All supplied lanes can be live together. Each queue reserves control + bytes for every QP through it or its reverse. A lane is constrained by + both its data queues and the reverse queues carrying its ACKs. + """ + if type(queue_bytes) is not int or not 4096 <= queue_bytes <= QUEUE: + raise ValueError("queue capacity must be between 4096 and 524288 bytes") + if type(chunk) is not int or chunk <= 0 or chunk % 16: + raise ValueError("chunk must be a positive multiple of 16 bytes") + queues = defaultdict(list) + for identity, lane in lanes.items(): + for key in _queue_keys(lane): + queues[key].append(identity) + keys = set(queues) | {_reverse(q) for q in queues} + limits = {} + for key in keys: + forward, reverse = len(queues[key]), len(queues[_reverse(key)]) + control = (forward + reverse) * subset.CONTROL_BYTES_PER_LANE + coefficient = 3 * forward + reverse + limits[key] = max(0, (2 * (queue_bytes - control)) // coefficient // chunk * chunk) + windows = {} + for identity, lane in lanes.items(): + keys_for_lane = _queue_keys(lane) + windows[identity] = min((limits[k] for q in keys_for_lane for k in (q, _reverse(q))), + default=0) + if keys_for_lane and windows[identity] < chunk: + raise ValueError("packet envelope cannot hold one chunk on every relayed lane") + # The native credit ring must hold all chunks plus control entries. + windows[identity] = min(windows[identity], 60 * chunk) + proof = [] + for key in sorted(keys): + reverse = _reverse(key) + forward_bytes = sum(windows[k] for k in queues[key]) + reverse_bytes = sum(windows[k] for k in queues[reverse]) + control = (len(queues[key]) + len(queues[reverse])) * subset.CONTROL_BYTES_PER_LANE + occupied = (3 * forward_bytes + reverse_bytes + 1) // 2 + control + if occupied > queue_bytes: + raise ValueError("packet envelope exceeds queue capacity") + proof.append({"queue": list(key), "data_bytes": forward_bytes, + "reverse_data_bytes": reverse_bytes, "control_bytes": control, + "bound_bytes": occupied, "capacity_bytes": queue_bytes}) + return windows, proof + + +def _resources(lane): + # Functions share a cable and a NIC; do not give each function 24 GB/s. + return ([('cable', s.src, s.src_port, s.dst, s.dst_port) for s in lane.steps] + + [('send', lane.steps[0].src), ('receive', lane.steps[-1].dst)]) + + +def _estimate(traffic, active, windows, co_lanes, schedule, collective, size, world, congested=False): + load = defaultdict(float) + users = defaultdict(set) + for identity, lane in co_lanes.items(): + for resource in _resources(lane): + users[resource].add(identity[0]) + lane_count = max(k[3] for k in active) + 1 + window_time = 0.0 + for identity, lane in active.items(): + volume = traffic[identity[1:3]] / lane_count + for resource in _resources(lane): + load[resource] += volume + if lane.relays: + window_time = max(window_time, volume / windows[identity] * + rtt_us(windows[identity], len(lane.relays), congested=congested)) + resource_time = max((volume * (1 + len(users[resource])) / + (26800 if resource[0] == 'receive' else 24000) + for resource, volume in load.items()), default=0) + stages = {"oneshot": 1, "twoshot": 2, "pieces": 2, + "chain": world - 1, "ring": world - 1}[schedule] + if collective == "all_reduce" and schedule in ("chain", "ring"): + stages *= 2 + # A deliberately simple GPU/launch term. No claim of calibrated small-message latency. + startup = 5.0 + stages * 1.5 + if schedule == "pieces": + startup += max(0, math.ceil(size / (2 << 20)) - 1) * 1.5 + return round(startup + max(resource_time, window_time), 3) + + +def plan(fabric, placement, *, co_resident=(), sizes=(4096, 65536, 4 << 20, 32 << 20), + collective="all_reduce", mode="graph", flat=True, fabric_record=None, measurements=None, + conditions=None): + """Return deterministic per-size choices, rejected candidates and queue certificates. + + The modeled sessions use only the candidate's data paths, with drained + transitions between schedules. Co-resident groups reserve all-pairs lanes + at two lanes per peer. These requirements are not installed by this tool. + """ + placement = tuple(placement) + co_resident = tuple(sorted(tuple(p) for p in co_resident)) + if collective not in COLLECTIVES or mode not in tuning.MODES: + raise ValueError("unsupported collective or execution mode") + if not 2 <= len(fabric.positions) <= 8 or not 2 <= len(placement) <= 8: + raise ValueError("fabric and group need 2 to 8 Sparks") + # Derivation also rejects disconnected cable descriptions. + routes.derive_routes(routes.Layout(fabric, fabric.positions)) + groups = (placement, *co_resident) + claimed = set() + for group in groups: + if not group: + raise ValueError("co-resident placement must not be empty") + routes.Layout(fabric, group) + if claimed.intersection(group): + raise ValueError("deployments must occupy disjoint positions") + claimed.update(group) + sizes = tuple(sorted(set(sizes))) + if not sizes or any(type(size) is not int or size <= 0 or size % 16 for size in sizes): + raise ValueError("sizes must be positive multiples of 16 bytes") + key = {"model": MODEL, "fabric": [c.text for c in sorted(fabric.cables, key=lambda c: c.text)], + "fabric_record": fabric_record, "placement": list(placement), + "co_resident": [list(p) for p in co_resident], "collective": collective, + "mode": mode, "flat": flat, "conditions": conditions, "native": tuning.native_hash(), + "kernels": tuning.kernels_hash(), "sircl": tuning.sircl_version()} + co_lanes = {} + for index, group in enumerate(co_resident, 1): + co_lanes.update(_all_lanes(fabric, group, 2, index)) + order = _order(fabric, placement) + result = {"schema": MODEL, "advisory_only": True, "runtime_configuration": False, "key": key, + "requirements": ["universal shortest-path relay table", + "all listed sessions use the returned joint reservations", + "completion-only credit release; MTU >= 256; overhead <= 128 B/packet", + "no retained retransmission copies; no unlisted relayed clients", + "drain data and ACKs before changing schedules or windows"], "sizes": []} + measured_rows = {} + if measurements is not None: + required = ("image", "drivers", "dtype", "shape", "rotate_buffers", "co_workload") + if not isinstance(conditions, dict) or any(not conditions.get(k) for k in required): + raise ValueError("measured selection requires image, drivers, dtype, shape, rotate_buffers and co_workload conditions") + if type(conditions["rotate_buffers"]) is not int or conditions["rotate_buffers"] < 1: + raise ValueError("rotate_buffers must be a positive integer") + if not isinstance(measurements, dict) or measurements.get("schema") != MEASUREMENTS or measurements.get("key") != key: + raise ValueError("measurement schema or placement/build/conditions key does not match") + rows = measurements.get("rows", []) + if not isinstance(rows, list): + raise ValueError("measurement rows must be a list") + for row in rows: + if (not isinstance(row, dict) or type(row.get("bytes")) is not int + or row["bytes"] <= 0 or row["bytes"] % 16 or not isinstance(row.get("candidate"), str)): + raise ValueError("measurement case needs aligned positive bytes and a candidate ID") + value = row.get("period_us") + if (row.get("exact") is not True or row.get("drops") != 0 or row.get("counters_valid") is not True + or row.get("isolation_passed") is not True or not row.get("evidence") + or type(value) not in (int, float) or not math.isfinite(value) or value <= 0): + raise ValueError("measurement needs positive time, evidence, exactness, clean valid counters and isolation") + identity = (row.get("bytes"), row.get("candidate")) + if identity in measured_rows: + raise ValueError("duplicate measurement case") + measured_rows[identity] = value + for size in sizes: + candidates, rejected = [], [] + for schedule in ("oneshot", "twoshot", "pieces", "chain", "ring"): + for count in (1, 2): + name = f"{schedule}/lanes={count}" + try: + if schedule in ("oneshot", "twoshot") and collective != "all_reduce": + raise ValueError("one-shot and two-shot candidates model all-reduce only") + if schedule == "oneshot" and size > 131072 or schedule == "twoshot" and size > 2 << 20: + raise ValueError("exceeds modeled single-op capacity") + if schedule in ("chain", "ring") and order is None: + raise ValueError("no cable-connected chain through the placement") + if schedule == "chain" and collective == "all_gather" and not flat: + raise ValueError("chain gather requires flat input") + traffic = _traffic(schedule, collective, size, len(placement), order) + all_lanes = _all_lanes(fabric, placement, count, 0) + active = {k: v for k, v in all_lanes.items() if k[1:3] in traffic} + if any(len(v.relays) > routes.DEFAULT_MAX_RELAYS for v in (*active.values(), *co_lanes.values())): + raise ValueError("route exceeds the supported relay-hop limit") + if fabric_record is not None and any(v.relays for v in (*active.values(), *co_lanes.values())): + if not fabric_record.get("relay_plan") or "sircl" not in fabric_record.get("transports", ()): + raise ValueError("fabric document has no SIRCL relay plan") + if schedule == "ring": + queue_count = defaultdict(int) + for lane in active.values(): + for q in _queue_keys(lane): + queue_count[q] += 1 + if max(queue_count.values(), default=0) > 1: + raise ValueError("more than one ring lane uses a relay queue") + windows, proof = packet_windows({**active, **co_lanes}, + chunk=LINK_CHUNK if schedule == "ring" else CHUNK) + predicted = _estimate(traffic, active, windows, co_lanes, schedule, + collective, size, len(placement)) + slow = _estimate(traffic, active, windows, co_lanes, schedule, + collective, size, len(placement), congested=True) + paths = [{"group": k[0], "rank": k[1], "peer": k[2], "lane": k[3], + "path": [v.steps[0].src, *(s.dst for s in v.steps)], + "function": "secondary" if v.local.secondary else "primary", + "window_bytes": windows[k]} for k, v in sorted({**active, **co_lanes}.items())] + candidate_id = name + "/" + hashlib.sha256(tuning.canonical_bytes(paths)).hexdigest()[:16] + candidates.append({"id": candidate_id, "schedule": schedule, "lanes": count, + "predicted_us": predicted, "congested_us": slow, + "paths": paths, "queues": proof, + "measured_us": measured_rows.get((size, candidate_id))}) + except ValueError as error: + rejected.append({"candidate": name, "reason": str(error)}) + measured = [c for c in candidates if c["measured_us"] is not None] + chosen = min(measured or candidates, key=lambda c: (c["measured_us"] if measured else c["predicted_us"], + c["lanes"], c["id"]), default=None) + result["sizes"].append({"bytes": size, "chosen": chosen, "source": "measured" if measured else "model", + "candidates": candidates, "rejected": rejected}) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--fabric") + source.add_argument("--fabric-document", type=Path) + parser.add_argument("--placement", required=True) + parser.add_argument("--co-resident", action="append", default=[]) + parser.add_argument("--sizes", default="4096,65536,4194304,33554432") + parser.add_argument("--collective", choices=COLLECTIVES, default="all_reduce") + parser.add_argument("--mode", choices=tuning.MODES, default="graph") + parser.add_argument("--non-flat", action="store_true") + parser.add_argument("--measurements", type=Path) + parser.add_argument("--conditions", type=Path, help="JSON measurement conditions; required with --measurements") + args = parser.parse_args(argv) + try: + fabric, record = document_fabric(args.fabric_document) if args.fabric_document else (fabric_named(args.fabric), None) + measured = json.loads(args.measurements.read_text(encoding="utf-8")) if args.measurements else None + conditions = json.loads(args.conditions.read_text(encoding="utf-8")) if args.conditions else None + result = plan(fabric, positions(args.placement), co_resident=[positions(p) for p in args.co_resident], + sizes=[int(v) for v in args.sizes.split(",")], collective=args.collective, mode=args.mode, + flat=not args.non_flat, fabric_record=record, measurements=measured, conditions=conditions) + except (ValueError, OSError) as error: + parser.error(str(error)) + print(json.dumps(result, indent=2, allow_nan=False)) + return 0 if all(row["chosen"] is not None for row in result["sizes"]) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/spark_transport/sircl/sparkring_sircl/subset.py b/spark_transport/sircl/sparkring_sircl/subset.py index a40d638a..40a3e8e4 100644 --- a/spark_transport/sircl/sparkring_sircl/subset.py +++ b/spark_transport/sircl/sparkring_sircl/subset.py @@ -97,3 +97,57 @@ def payload_budget(layout, maps, *, queue_bytes=routes.DEFAULT_HAIRPIN_QUEUE) -> for (p, c, e), v in queues.items()), default=0) return min(int(routes.RELAY_QUEUE_SHARE * queue_bytes), max(0, (queue_bytes - n * CONTROL_BYTES_PER_LANE) // 4)) + + +def ring_packet_bound(layout, maps, *, chunk=32768, queue_bytes=routes.DEFAULT_HAIRPIN_QUEUE): + """Offline maximum common ring window with the conservative arena windows fixed. + + This is not an active runtime allowance. Completion-only credit release, + MTU >= 256, <= 128 bytes of overhead per packet, and no retained retry + copies are required. Arena and ring payload may overlap, but a native QP + shares one unacked counter: its payload is bounded by max(arena, ring), + not their sum. The reverse + queue's payload generates ACKs in this queue, including ACK-only queues. + Control credits are shared by arena and ring on each native QP. + """ + order = validate(layout, maps, queue_bytes=queue_bytes) + if chunk <= 0 or chunk % 16: + raise routes.RouteError("ring bound chunk must be a positive multiple of 16") + arena = routes.relay_queues(layout, maps) + ring = routes.ring_queues(layout, maps, order) + tables = [routes.forward_windows(layout, maps, rank, chunk=FORWARD_CHUNK, + queue_bytes=queue_bytes, subset_ring=True) + for rank in range(layout.world)] + keys = set(arena) | set(ring) + keys |= {(p, c, 1 - e) for p, c, e in keys.copy()} + rows = [] + for key in sorted(keys): + reverse = (*key[:2], 1 - key[2]) + a = sum(tables[r][p][l] for r, p, l in arena.get(key, ())) + b = sum(tables[r][p][l] for r, p, l in arena.get(reverse, ())) + controls = (len(arena.get(key, ())) + len(arena.get(reverse, ()))) * CONTROL_BYTES_PER_LANE + shared = [tables[r][p][l] for r, p, l in ring.get(key, ())] + back_shared = [tables[r][p][l] for r, p, l in ring.get(reverse, ())] + # Payload on QPs that do not carry the ring remains independently live. + fixed2 = 3 * (a - sum(shared)) + b - sum(back_shared) + 2 * controls + if 3 * a + b + 2 * controls > 2 * queue_bytes: + raise routes.RouteError("arena exceeds the packet envelope") + rows.append({"queue": list(key), "arena_bytes": a, "reverse_arena_bytes": b, + "control_bytes": controls, "fixed_twice_bytes": fixed2, + "ring_arena_windows": shared, "reverse_ring_arena_windows": back_shared}) + + def occupied2(row, window): + return (row["fixed_twice_bytes"] + 3 * sum(max(w, window) for w in row["ring_arena_windows"]) + + sum(max(w, window) for w in row["reverse_ring_arena_windows"])) + + # A monotone piecewise-linear bound; enumerate at most Q/chunk aligned values. + window = 0 + if ring: + for candidate in range(chunk, queue_bytes + 1, chunk): + if any(occupied2(row, candidate) > 2 * queue_bytes for row in rows): + break + window = candidate + for row in rows: + row["bound_bytes"] = (occupied2(row, window) + 1) // 2 + row["next_chunk_bound_bytes"] = (occupied2(row, window + chunk) + 1) // 2 + return window, rows diff --git a/spark_transport/sircl/tests/test_schedule_selection.py b/spark_transport/sircl/tests/test_schedule_selection.py new file mode 100644 index 00000000..72622bbe --- /dev/null +++ b/spark_transport/sircl/tests/test_schedule_selection.py @@ -0,0 +1,201 @@ +"""Offline selection must not turn an unbounded candidate into a recommendation.""" +import copy +import json +import subprocess +import sys +from itertools import combinations +from pathlib import Path + +import pytest + +from sparkring_sircl import routes, schedule_selection as select, subset + + +@pytest.mark.parametrize("kind", ("path", "ring")) +@pytest.mark.parametrize("n", range(2, 9)) +@pytest.mark.parametrize("collective", select.COLLECTIVES) +def test_layouts_and_representative_placements(kind, n, collective): + fabric = select.fabric_named(f"{kind}{n}") + placements = {tuple(range(n)), tuple(range(max(2, n - 2))), tuple(range(max(2, n // 2))), + (0, n - 1), tuple(range(0, n, 2))} + for members in placements: + if len(members) < 2: + continue + outside = tuple(p for p in range(n) if p not in members) + result = select.plan(fabric, members, co_resident=[outside] if outside else [], + sizes=[4096, 4 << 20], collective=collective) + assert result["advisory_only"] + for row in result["sizes"]: + if row["chosen"] is None: + assert len(row["rejected"]) == 10 + continue + assert row["chosen"] in row["candidates"] + for candidate in row["candidates"]: + assert candidate["congested_us"] >= candidate["predicted_us"] > 0 + for queue in candidate["queues"]: + assert (3 * queue["data_bytes"] + queue["reverse_data_bytes"]) / 2 + queue["control_bytes"] <= queue["capacity_bytes"] + for path in candidate["paths"]: + assert bool(path["window_bytes"]) == (len(path["path"]) > 2) + assert len(path["path"]) <= 5 + + +def test_subsets_fixed_arena_ring_bound_is_maximal_in_its_envelope(): + for n in range(4, 9): + for count in range(max(3, n - 2), n): + for members in combinations(range(n), count): + layout = routes.Layout(routes.Fabric.ring(n), members) + derived = routes.derive_routes(layout) + maps = [derived.route_map(r) for r in range(count)] + try: + subset.order(layout) + except routes.RouteError: + continue + window, rows = subset.ring_packet_bound(layout, maps) + assert all(row["bound_bytes"] <= select.QUEUE for row in rows) + if window: + assert any(row["next_chunk_bound_bytes"] > select.QUEUE for row in rows) + outside = tuple(p for p in range(n) if p not in members) + if len(outside) == 2: + other = routes.Layout(layout.fabric, outside) + other_routes = routes.derive_routes(other) + assert routes.relay_queues(other, [other_routes.route_map(r) for r in range(2)]) == {} + layout = routes.Layout.parse("ring:8:0,1,2,3,4,5") + derived = routes.derive_routes(layout) + window, rows = subset.ring_packet_bound(layout, [derived.route_map(r) for r in range(6)]) + assert window == 196608 + closing = [row for row in rows if row["queue"] == [6, True, 0]][0] + assert closing["arena_bytes"] == closing["reverse_arena_bytes"] == 86016 + assert closing["control_bytes"] == 98304 + assert closing["bound_bytes"] == 522240 + + +def test_ack_only_queues_and_joint_reservations(): + fabric = routes.Fabric.path(range(4)) + lane = routes.derive_lanes(fabric, 3, 0, 1)[0] + windows, proof = select.packet_windows({(0, 0, 1, 0): lane}, chunk=32768) + assert windows[(0, 0, 1, 0)] == 327680 + assert any(q["data_bytes"] == 0 and q["reverse_data_bytes"] == 327680 for q in proof) + crowded = {(g, 0, 1, 0): lane for g in range(40)} + with pytest.raises(ValueError, match="packet envelope"): + select.packet_windows(crowded) + for options in ({"queue_bytes": 1048576}, {"chunk": 0}, {"chunk": 15}): + with pytest.raises(ValueError): + select.packet_windows({(0, 0, 1, 0): lane}, **options) + + +def test_measurement_precedence_is_exact_and_cannot_bypass_safety(): + fabric = select.fabric_named("ring8") + kwargs = dict(co_resident=[(6, 7)], sizes=[4 << 20], conditions={ + "image": "fixture", "drivers": "fixture", "dtype": "bfloat16", "shape": "flat", + "rotate_buffers": 8, "co_workload": "fixed decode fixture"}) + original = select.plan(fabric, tuple(range(6)), **kwargs) + slower = max(original["sizes"][0]["candidates"], key=lambda c: c["predicted_us"]) + measured = {"schema": select.MEASUREMENTS, "key": original["key"], "rows": [ + {"bytes": 4 << 20, "candidate": slower["id"], "period_us": 123.0, "exact": True, + "drops": 0, "counters_valid": True, "isolation_passed": True, "evidence": "offline test fixture"}]} + answer = select.plan(fabric, tuple(range(6)), measurements=measured, **kwargs) + assert answer["sizes"][0]["chosen"]["id"] == slower["id"] + assert answer["sizes"][0]["source"] == "measured" + for field, value in (("exact", False), ("drops", 1), ("period_us", float("nan")), + ("counters_valid", False), ("isolation_passed", False)): + bad = copy.deepcopy(measured) + bad["rows"][0][field] = value + with pytest.raises(ValueError, match="measurement needs"): + select.plan(fabric, tuple(range(6)), measurements=bad, **kwargs) + bad = copy.deepcopy(measured) + bad["key"]["co_resident"] = [] + with pytest.raises(ValueError, match="does not match"): + select.plan(fabric, tuple(range(6)), measurements=bad, **kwargs) + for rows in ({}, [None], [{"bytes": -16, "candidate": slower["id"]}]): + bad = {**measured, "rows": rows} + with pytest.raises(ValueError, match="measurement"): + select.plan(fabric, tuple(range(6)), measurements=bad, **kwargs) + measured["rows"][0]["candidate"] = "unsafe/nonexistent" + answer = select.plan(fabric, tuple(range(6)), measurements=measured, **kwargs) + assert answer["sizes"][0]["source"] == "model" + + +def test_nonflat_and_missing_relay_plan(): + result = select.plan(select.fabric_named("ring8"), tuple(range(6)), collective="all_gather", + sizes=[4 << 20], flat=False, fabric_record={"relay_plan": None}) + assert not result["sizes"][0]["candidates"] + assert all("chain" not in c["schedule"] for c in result["sizes"][0]["candidates"]) + + +@pytest.mark.parametrize("members,other", [((0, 0), []), ((0, 9), []), ((0, 1), [(1, 2)]), ((0,), [])]) +def test_bad_placements_refused(members, other): + with pytest.raises(ValueError): + select.plan(select.fabric_named("ring8"), members, co_resident=other) + + +def test_cli_is_offline_and_deterministic(): + root = Path(__file__).resolve().parents[1] + command = [sys.executable, "-m", "sparkring_sircl.schedule_selection", "--fabric", "ring7", + "--placement", "6,0-3", "--co-resident", "4-5", "--sizes", "4096"] + runs = [subprocess.run(command, cwd=root, capture_output=True, text=True, check=True) for _ in range(2)] + assert runs[0].stdout == runs[1].stdout + assert json.loads(runs[0].stdout)["advisory_only"] + + +def test_rtt_calibration_and_sensitivity(): + assert select.rtt_us(65536, 2) == pytest.approx(2 * 65536 / 9400) + assert select.rtt_us(393216, 2) == pytest.approx(2 * 393216 / 21110) + assert select.rtt_us(245760, 2) > select.rtt_us(163840, 2) + assert select.rtt_us(163840, 3) > select.rtt_us(163840, 2) + assert select.rtt_us(163840, 2, congested=True) > select.rtt_us(163840, 2) + + +def test_setup_fabric_document_and_invalid_identity(tmp_path): + from runtime.common import fabric_document, fabric_layout + from runtime.host.test_fabric import document_of + + _, document, _ = document_of(fabric_layout.layout("cycle", 8)) + path = tmp_path / "fabric.json" + path.write_text(json.dumps(document), encoding="utf-8") + fabric, record = select.document_fabric(path) + assert fabric == select.fabric_named("ring8") + assert record["relay_plan"] == document["relays"] + assert record["document_sha256"] == fabric_document.digest(document) + result = select.plan(fabric, tuple(range(6)), co_resident=[(6, 7)], + sizes=[4096], fabric_record=record) + assert result["key"]["fabric_record"] == record + document["cables"][0]["ends"][0]["port"] = 1 + path.write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(ValueError): + select.document_fabric(path) + + +@pytest.mark.parametrize("active", (False, True)) +def test_larger_bound_native_progress_with_direct_complement(simulator_library, monkeypatch, active): + from concurrent.futures import ThreadPoolExecutor + from sparkring_sircl.testing import fabric as sim, ring_links + + monkeypatch.setenv("SIRCL_FORWARD_PROOF", "0") + layout = routes.Layout.parse("ring:8:0,1,2,3,4,5") + derived = routes.derive_routes(layout) + window, _ = subset.ring_packet_bound(layout, [derived.route_map(r) for r in range(6)]) + links = ring_links.RingLinks(str(simulator_library), layout, subset_ring=True, + slots=12, slot_bytes=65536, timeout=30) + # Only the in-memory link harness uses the offline bound; runtime admission stays unchanged. + links._ring_window = window + pair = None + try: + if active: + pair = sim.LocalSession(str(simulator_library), routes.Layout.parse("ring:8:6,7"), + node_base=6, reset=False) + pair.connect() + links.connect() + links.delay_link_ops(5, 2000) + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit(sim.run_oneshot_ops, pair, [16384] * 100) if pair else None + ring_links.run_ring_ops(links, [("gather", 524304, 65536, 0, 1), + ("reduce", 524352, 65536, 1, 1)]) + if pending: + pending.result(timeout=30) + assert not any(proxy.failed() for proxy in pair.proxies) + assert not any(proxy.failed() for proxy in links.proxies) + assert all(proxy.stats()["forward_max_unacked_bytes"] <= window + 256 for proxy in links.proxies) + finally: + if pair: + pair.close() + links.close() From 8ff32cef11f079e9c60d1b0a3d462ba426440fee Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:56:42 -0500 Subject: [PATCH 4/5] Satisfy lint in subset placement and the compatibility driver Rename ambiguous lane variables in the subset arena budget and exempt the compatibility driver's post-stand-in imports from E402, matching the subset chain driver. Validation: ruff E,F,W; pytest spark_transport/sircl (1087 passed, 126 skipped). Co-Authored-By: Claude Opus 5.5 --- spark_transport/sircl/sparkring_sircl/subset.py | 8 ++++---- spark_transport/sircl/tests/subset_compat_driver.py | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/spark_transport/sircl/sparkring_sircl/subset.py b/spark_transport/sircl/sparkring_sircl/subset.py index 40a3e8e4..799030a3 100644 --- a/spark_transport/sircl/sparkring_sircl/subset.py +++ b/spark_transport/sircl/sparkring_sircl/subset.py @@ -123,11 +123,11 @@ def ring_packet_bound(layout, maps, *, chunk=32768, queue_bytes=routes.DEFAULT_H rows = [] for key in sorted(keys): reverse = (*key[:2], 1 - key[2]) - a = sum(tables[r][p][l] for r, p, l in arena.get(key, ())) - b = sum(tables[r][p][l] for r, p, l in arena.get(reverse, ())) + a = sum(tables[r][p][lane] for r, p, lane in arena.get(key, ())) + b = sum(tables[r][p][lane] for r, p, lane in arena.get(reverse, ())) controls = (len(arena.get(key, ())) + len(arena.get(reverse, ()))) * CONTROL_BYTES_PER_LANE - shared = [tables[r][p][l] for r, p, l in ring.get(key, ())] - back_shared = [tables[r][p][l] for r, p, l in ring.get(reverse, ())] + shared = [tables[r][p][lane] for r, p, lane in ring.get(key, ())] + back_shared = [tables[r][p][lane] for r, p, lane in ring.get(reverse, ())] # Payload on QPs that do not carry the ring remains independently live. fixed2 = 3 * (a - sum(shared)) + b - sum(back_shared) + 2 * controls if 3 * a + b + 2 * controls > 2 * queue_bytes: diff --git a/spark_transport/sircl/tests/subset_compat_driver.py b/spark_transport/sircl/tests/subset_compat_driver.py index 1bb469b2..109ed035 100644 --- a/spark_transport/sircl/tests/subset_compat_driver.py +++ b/spark_transport/sircl/tests/subset_compat_driver.py @@ -4,6 +4,9 @@ support. CUDA is mocked; the real configuration, route and plan code runs. """ +# CUDA stand-ins must be installed before importing the runtime. +# ruff: noqa: E402 + import hashlib import json import sys From 2dfd2a4046c1eb5a998667f286b3965d459e0cfd Mon Sep 17 00:00:00 2001 From: Fixture Date: Sun, 1 Nov 2026 19:13:52 -0600 Subject: [PATCH 5/5] Re-pin the GLM DCP plugin against the subset-change SIRCL tree sparkring_sircl/oneshot/runtime.py is one of the files integrations/vllm/glm_dcp_decode_comm pins by SHA-256. The subset change edits that file, so test_every_sircl_pin_matches_the_sircl_tree and test_the_sircl_only_refresh_finds_no_drift_and_needs_no_image_tree failed. Behavior: tools/refresh_pins.py --sircl-only re-pinned that one file (631cc652... -> 308e278b...); the other 16 SIRCL pins are unchanged. The FileCheck reason still holds: all_gather(out=), the arena view, _counter_addresses, _order_stream, _mark_stream, _tuned_op, _lock, check_health and the launch geometry keep their definition, signature and call order, and every statement the subset change adds runs only when SIRCL_SUBSET_RING=1, which defaults to 0. The plugin is version 2.0.3 (PLUGIN_VERSION, dist-info directory and METADATA, the GLM-5.3 plugin layer's DCP_DIST_INFO, and the tests that expect the layer's versions). The SIRCL package version stays 0.3.2; the README's Pins section records this tree as the build 2.0.3 pins. Compatibility: published images and their release locks keep the plugin version they carry. With every item flag 0, registration patches nothing and compares no pin, so a profile that loads the plugin with its items off runs on any build; with an item flag 1, the plugin refuses a SIRCL tree other than this one. An image derived from this tree carries 2.0.3. Validation: refresh_pins --check reports every pin matches; the plugin suite passes (235 passed, 16 skipped without the image's vLLM tree); the SIRCL suite passes (1210 passed, 3 skipped); ruff E,F,W, markdown links and repository layout pass; runtime/images, runtime/host/test_images.py and runtime/common/test_image_lock.py give 5 failed, 1007 passed, 2 skipped, the same five pinned-source-byte failures the unchanged tree gives in this environment. GPU checks and audit-mode qualification are pending; both need Spark hardware. --- .../vllm/glm_dcp_decode_comm/README.md | 30 +++++++++++++------ .../METADATA | 2 +- .../entry_points.txt | 0 .../top_level.txt | 0 .../glm_dcp_decode_comm/__init__.py | 4 +-- runtime/common/test_image_lock.py | 2 +- runtime/images/derive_glm53_plugins.py | 4 +-- runtime/images/installer-images.md | 2 +- runtime/images/test_derive_glm53_plugins.py | 4 +-- 9 files changed, 30 insertions(+), 18 deletions(-) rename integrations/vllm/glm_dcp_decode_comm/dist-info/{glm_dcp_decode_comm-2.0.2.dist-info => glm_dcp_decode_comm-2.0.3.dist-info}/METADATA (97%) rename integrations/vllm/glm_dcp_decode_comm/dist-info/{glm_dcp_decode_comm-2.0.2.dist-info => glm_dcp_decode_comm-2.0.3.dist-info}/entry_points.txt (100%) rename integrations/vllm/glm_dcp_decode_comm/dist-info/{glm_dcp_decode_comm-2.0.2.dist-info => glm_dcp_decode_comm-2.0.3.dist-info}/top_level.txt (100%) diff --git a/integrations/vllm/glm_dcp_decode_comm/README.md b/integrations/vllm/glm_dcp_decode_comm/README.md index 5a899053..fd194112 100644 --- a/integrations/vllm/glm_dcp_decode_comm/README.md +++ b/integrations/vllm/glm_dcp_decode_comm/README.md @@ -1,6 +1,6 @@ # glm_dcp_decode_comm -A vLLM general plugin (distribution `glm-dcp-decode-comm` 2.0.2, entry point +A vLLM general plugin (distribution `glm-dcp-decode-comm` 2.0.3, entry point `glm_dcp_decode_comm = glm_dcp_decode_comm:register`) that changes how GLM-5.3's DSA attention layers run their decode context parallel (DCP) collectives when those collectives run on a SIRCL DCP session. Every item is @@ -89,7 +89,7 @@ GLM_DCP_DECODE_A2A_FUSED=1 GLM_DCP_DECODE_AUDIT=1 ``` -It passes when every rank logs the startup line `glm_dcp_decode_comm 2.0.2: +It passes when every rank logs the startup line `glm_dcp_decode_comm 2.0.3: enabled ...` naming the five items and audit mode, no `PatchRefused`, and every `glm_dcp_decode_comm audit cuda:N: CHECK: ...` line of every rank reports `0 differ` with a nonzero call count for each of the five checks @@ -140,8 +140,8 @@ The pins describe two builds: - the image's vLLM and b12x files: serving image `816c6d6a7e96`; - SIRCL: package version 0.3.2, SparkRing's [`spark_transport/sircl/sparkring_sircl`](../../../spark_transport/sircl/sparkring_sircl) - as commit `32c41e7f` has it, the source of the SIRCL layer of release - 2026.10.4's image (Git tree `a385bb72b629` of `sparkring_sircl`). + as commit `ab9e2888` has it, with the opt-in `SIRCL_SUBSET_RING` subset + mode (Git tree `462e000619e1` of `sparkring_sircl`). Version 2.0.1, which the images of releases 2026.10.2, 2026.10.4 and 2026.10.5 carry, pins the same files except `vllm/adapter.py`: it pins that file as release @@ -149,10 +149,22 @@ pins the same files except `vllm/adapter.py`: it pins that file as release of `sparkring_sircl`), and 2.0.2 pins it as commit `32c41e7f` has it (SHA-256 `ebe9b323…`), which adds the `max_relays` and `route_scope` fields to `AdapterConfig` and `GroupPlacement` and leaves `GroupAdapter`'s `.dcp`, -`.session` and `live_adapters` unchanged. Each version therefore refuses an -item flag on the other release's image. With every item flag `0`, -registration patches nothing and compares no pin, so a profile that loads -the plugin with its items off starts on either image. +`.session` and `live_adapters` unchanged. + +Version 2.0.3 pins the same files as 2.0.2 except `oneshot/runtime.py`: 2.0.2 +pins that file as commit `32c41e7f` has it (SHA-256 `631cc652…`), and 2.0.3 +pins it as commit `ab9e2888` has it (SHA-256 `308e278b…`), which adds the +subset admission, the subset schedule notices and the subset early returns. +Every member the file's reason names (`all_gather(out=)`, the arena view, +`_counter_addresses`, `_order_stream`, `_mark_stream`, `_tuned_op`, `_lock`, +`check_health` and the launch geometry) keeps its definition, its signature +and its call order, and every subset statement runs only when +`SIRCL_SUBSET_RING=1`, so a session that leaves the variable unset takes the +ops and settings 2.0.2's build takes. + +Each version therefore refuses an item flag on the other release's image. +With every item flag `0`, registration patches nothing and compares no pin, +so a profile that loads the plugin with its items off starts on either image. Against SIRCL 0.3.0 (image `816c6d6a7e96`), 0.3.1 changes five of the 17 pinned files: the version in `__init__.py`; aligned working buffers around @@ -207,7 +219,7 @@ and the qualification run again. | `glm_dcp_decode_comm/_scatter_pack_cute.py` | the CuTe DSL packed all-to-all, the pinned SIRCL build's scatter op with the pack in its staging | | `glm_dcp_decode_comm/layout.py` | the query and wire-format geometry and the kernel's index helpers (pure Python) | | `glm_dcp_decode_comm/reference.py` | torch references of the image's computations and of the plugin's replacements | -| `dist-info/glm_dcp_decode_comm-2.0.2.dist-info/` | the installed distribution's metadata and its `vllm.general_plugins` entry point | +| `dist-info/glm_dcp_decode_comm-2.0.3.dist-info/` | the installed distribution's metadata and its `vllm.general_plugins` entry point | | `tools/refresh_pins.py` | records or checks every pin against vLLM, b12x and SIRCL trees | | `tests/` | the CPU suite (`test_dcp_decode_*.py`), the subprocess driver over SIRCL's CUDA stand-in (`scatter_driver.py`) and the GPU checks (`gpu_checks.py`) | diff --git a/integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.2.dist-info/METADATA b/integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.3.dist-info/METADATA similarity index 97% rename from integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.2.dist-info/METADATA rename to integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.3.dist-info/METADATA index 8774143d..0a12ad4f 100644 --- a/integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.2.dist-info/METADATA +++ b/integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.3.dist-info/METADATA @@ -1,5 +1,5 @@ Metadata-Version: 2.1 Name: glm-dcp-decode-comm -Version: 2.0.2 +Version: 2.0.3 Summary: vLLM general plugin that makes the GLM-5.3 DSA decode collectives of decode context parallelism cheaper on a SIRCL DCP session of image 816c6d6a7e96, with exact results: the query built by one kernel and gathered into the attention workspace, the DCP collectives on their own stream, the indexer wk GEMM beside the latent projection, one top-k slot conversion per indexer layer and the all-to-all combine packed into the scatter op; each item off until its GLM_DCP_DECODE_* flag is 1 Requires-Python: >=3.12 diff --git a/integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.2.dist-info/entry_points.txt b/integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.3.dist-info/entry_points.txt similarity index 100% rename from integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.2.dist-info/entry_points.txt rename to integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.3.dist-info/entry_points.txt diff --git a/integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.2.dist-info/top_level.txt b/integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.3.dist-info/top_level.txt similarity index 100% rename from integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.2.dist-info/top_level.txt rename to integrations/vllm/glm_dcp_decode_comm/dist-info/glm_dcp_decode_comm-2.0.3.dist-info/top_level.txt diff --git a/integrations/vllm/glm_dcp_decode_comm/glm_dcp_decode_comm/__init__.py b/integrations/vllm/glm_dcp_decode_comm/glm_dcp_decode_comm/__init__.py index c38b1e2c..f06dfeb1 100644 --- a/integrations/vllm/glm_dcp_decode_comm/glm_dcp_decode_comm/__init__.py +++ b/integrations/vllm/glm_dcp_decode_comm/glm_dcp_decode_comm/__init__.py @@ -131,7 +131,7 @@ from types import CodeType, FunctionType, ModuleType PLUGIN_NAME = "glm_dcp_decode_comm" -PLUGIN_VERSION = "2.0.2" +PLUGIN_VERSION = "2.0.3" MARKER = "__glm_dcp_decode_comm__" HELPER_PREFIX = "_glm_dcp_decode_comm_" IMAGE = "816c6d6a7e96" @@ -314,7 +314,7 @@ class FileCheck: "bebd8e7117ae5ec462b1871a12ebde30eadbebf89a45dc4bf645fa076f2b9411", "the scatter piece rules that decide when an all-to-all is one op"), FileCheck("sparkring_sircl", "oneshot/runtime.py", - "631cc65241508e60ed48d30b9f6dbedbef831f025661f8d355350834d7ea5645", + "308e278b87cb352d71efeb647907156d600d289c8b3fca757bf3a145df6b73e9", "RoceOneshotAllReduce: all_gather(out=), the arena view, _counter_addresses, _order_stream, " "_mark_stream, _tuned_op, _lock, check_health and the launch geometry"), FileCheck("sparkring_sircl", "oneshot/_scatter_ops.py", diff --git a/runtime/common/test_image_lock.py b/runtime/common/test_image_lock.py index 26da4c29..a64a0162 100644 --- a/runtime/common/test_image_lock.py +++ b/runtime/common/test_image_lock.py @@ -463,7 +463,7 @@ def test_the_glm53_tp8_profile_runs_only_on_an_image_that_carries_the_glm53_plug profile = "glm53-nvfp4-tp8" assert image_lock.required_plugins(profile) == ["glm_dsa_indexer_split", "glm53full_speedups", "glm_dcp_decode_comm"] - assert derive_glm53_plugins.PLUGINS == {**PLUGINS, "glm_dcp_decode_comm": "2.0.2"} + assert derive_glm53_plugins.PLUGINS == {**PLUGINS, "glm_dcp_decode_comm": "2.0.3"} with pytest.raises(ValueError, match=f"{profile} loads the vLLM plugins glm_dsa_indexer_split, " "glm53full_speedups, glm_dcp_decode_comm"): image_lock.for_profile(profile, with_eight_spark_profiles()) diff --git a/runtime/images/derive_glm53_plugins.py b/runtime/images/derive_glm53_plugins.py index 8c3d1ba5..4ca7c8d8 100644 --- a/runtime/images/derive_glm53_plugins.py +++ b/runtime/images/derive_glm53_plugins.py @@ -18,7 +18,7 @@ the fused ``q_a``/``kv_a`` latent projection is column-parallel over TP8 with one all-gather (``GLM53FULL_LATENT_SHARD=1``) and the MTP ``eh_proj`` is row-parallel (``GLM53FULL_EH_PROJ_TP=1``); -- ``glm_dcp_decode_comm`` 2.0.2 (``integrations/vllm/glm_dcp_decode_comm``): +- ``glm_dcp_decode_comm`` 2.0.3 (``integrations/vllm/glm_dcp_decode_comm``): exact changes to the DSA attention's decode context parallel collectives on a SIRCL DCP session, five items behind the ``GLM_DCP_DECODE_*`` flags (the query pack, the communication-stream overlap, the indexer ``wk`` overlap, @@ -84,7 +84,7 @@ SPEEDUPS = "integrations/vllm/glm53full_speedups/" DCP = "integrations/vllm/glm_dcp_decode_comm/" DCP_MODULES = ("__init__.py", "_scatter_pack_cute.py", "kernels.py", "layout.py", "reference.py", "runtime.py") -DCP_DIST_INFO = "glm_dcp_decode_comm-2.0.2.dist-info/" +DCP_DIST_INFO = "glm_dcp_decode_comm-2.0.3.dist-info/" # Every added file: the site-packages path, its repository source and the # SHA-256 of the repository bytes (the layer pins each result; additions have diff --git a/runtime/images/installer-images.md b/runtime/images/installer-images.md index 06786e9f..f8a9e674 100644 --- a/runtime/images/installer-images.md +++ b/runtime/images/installer-images.md @@ -151,7 +151,7 @@ inherited SHA-256 `None`, and the parent receipt must not record it. tool-result policy as a vLLM module and pins the `serving.py` that installs it. [derive_glm53_plugins.py](derive_glm53_plugins.py) adds the GLM-5.3 vLLM general plugins `glm_dsa_indexer_split` and `glm53full_speedups` 1.1.0 and -`glm_dcp_decode_comm` 2.0.2 with their dist-info directories and names all +`glm_dcp_decode_comm` 2.0.3 with their dist-info directories and names all three in `Layer.plugins`. The derived lock has its parent lock's schema. A parent with a diff --git a/runtime/images/test_derive_glm53_plugins.py b/runtime/images/test_derive_glm53_plugins.py index 67760a2a..c0a545fa 100644 --- a/runtime/images/test_derive_glm53_plugins.py +++ b/runtime/images/test_derive_glm53_plugins.py @@ -37,7 +37,7 @@ def test_replace_returns_only_the_pinned_files(): @pytest.mark.parametrize("plugin,package,version", [("glm_dsa_indexer_split", "glm-dsa-indexer-split", "1.1.0"), ("glm53full_speedups", "glm53full-speedups", "1.1.0"), - ("glm_dcp_decode_comm", "glm-dcp-decode-comm", "2.0.2")]) + ("glm_dcp_decode_comm", "glm-dcp-decode-comm", "2.0.3")]) def test_dist_info_registers_the_plugin_in_vllm_general_plugins(plugin, package, version): root = Path(derive_glm53_plugins.ROOT) metadata = (root / "integrations/vllm" / plugin / "dist-info" @@ -72,7 +72,7 @@ def re_fullmatch(value: str) -> bool: def test_the_layer_declares_each_plugin_at_its_dist_info_version(): """The derived v3 lock lists exactly the entry points and versions the layer's dist-info files register.""" - assert derive_glm53_plugins.PLUGINS == {"glm53full_speedups": "1.1.0", "glm_dcp_decode_comm": "2.0.2", + assert derive_glm53_plugins.PLUGINS == {"glm53full_speedups": "1.1.0", "glm_dcp_decode_comm": "2.0.3", "glm_dsa_indexer_split": "1.1.0"} assert derive_glm53_plugins.LAYER.plugins == derive_glm53_plugins.PLUGINS for name, version in derive_glm53_plugins.PLUGINS.items():