diff --git a/research/p104-toolkit-v3/NOTICE b/research/p104-toolkit-v3/NOTICE new file mode 100644 index 0000000000..3552f648bb --- /dev/null +++ b/research/p104-toolkit-v3/NOTICE @@ -0,0 +1,19 @@ +Apache-2.0. DreamingOfClouds with Claude Opus 5.5 (Anthropic), via Claude Code. + +All code in this package is our own (DreamingOfClouds lanes PRICEKIT, TOKDEBT, SEEDBUILD, OWNRETIME, DUALWORD, DUALSCAN, +NETCAUX, REBUILDKIT, NETC4, KFORMULA, CALLEROUT and CALLEROPEN). CALLEROPEN (layout_price.py, cap_check.py) is an +outside-session lane of DreamingOfClouds'. No code from any other pull request or author is included or run; +validate.py and tests/test_checkers.py call the repository's own invoice tools scripts/compare_recursive_invoices.py and +scripts/evaluate_recursive_power.py, pinned in SOURCES.json. + +Data, used as data only: +- data/base-7546-word.json.gz is the #7546 word by ToikY (bytes of expected/role902-WORD.json.gz at #7546 head + 99438c43df7aac1e5dbcc43c1578fcd886c119ec, SHA256 pinned in tests/common.py); credit ToikY. #7546 builds on our #7542 + and Clear1nsight's #7334. +- data/caller-7113.json.gz is the signed 17-stage #7113 caller by Rohan Arun in the #7113 caller format (gzip of our + extracted JSON, content SHA256 pinned in tests/common.py); credit Rohan Arun. +- data/edits-7601.json is our own #7601 edit list. + +Credit Clear1nsight, ToikY and the other contributors of the P104 price and replay lineage for the word and invoice +conventions these tools follow. Our earlier toolkit research/p104-designer-toolkit-v2 (#6170) targets the older #6030 +word lineage; this package targets the hosted-word format of #7546 and later. diff --git a/research/p104-toolkit-v3/README.md b/research/p104-toolkit-v3/README.md new file mode 100644 index 0000000000..75342af0f9 --- /dev/null +++ b/research/p104-toolkit-v3/README.md @@ -0,0 +1,380 @@ +# P104 toolkit v3: token-settled drops, frame-perp duality, fast exact replay, κ(k) caller law (no κ) + +This package ships our stable, verified P104 tools for the hosted-word format of ToikY's +[#7546](https://github.com/CrocSwap/integer-mult-bounds/pull/7546) and later words. Each tool has a usage section +below, a runnable command-line example and tests with positive and negative controls. The tests run in minutes on +vendored data (the #7546 word, the #7113 caller and our #7601 edit list). + +It makes **no κ claim** and ships no frontier word. At the time of writing the frontier is aliromman's #7605 +(κ 0.0036504442446). The one new word the tools produce here, #7601 + the main-902 drop (§C.3), prices below it. + +| | tool | what it does | needs | +|---|---|---|---| +| A | `pricekit.py` | exact caller + native invoice, certified κ enclosure, D0, ledger law | stdlib | +| B | `dualword.py` | frame-perp transpose: word ↦ dual word, an exact involution that preserves the price | stdlib | +| C | `tokdrop.py` (+ `modp.py`) | token-settled drop engine: stay-drop repair in clean mode, premix settlement rank, exact build | numpy (python-flint optional) | +| D1 | `accx.py` (+ `fastreplay.py`) | admissibility of an edited word against its base; replay mod 2^61−1 with exact re-checks | stdlib | +| D2 | `lossless.py` | law DUAL: finds every lossy owned CLEAR in one forward pass | stdlib | +| D3 | `nativeplan.py` | helper-plan extractor (routes, stays, typed events, requirement blocks, exact π-states) | stdlib | +| D4 | `validate.py` | one command: accx + lossless + the repository's invoice tools | stdlib | +| E1 | `kformula.py` | closed-form caller C(k) and the κ(k) law | stdlib | +| E2 | `layout_price.py`, `cap_check.py` (+ `sl_lib.py`) | parametric mask-layout pricer; crossing caps and the Floor lemma on a real caller | stdlib | + +Tags: + +- **[C]** exact computation, re-run by the tests in this package (`tests/`, listed in [VERIFY.json](VERIFY.json)). +- **[C-param]** exact price of a stated parametric histogram, not of a word. +- **[P]** proved (short argument given here). +- **[S]** judgement. + +Nothing tagged [S] is claimed as a result. + +## How to verify + +``` +python3 -I -B research/p104-toolkit-v3/tests/test_pricing.py # A, E PASS_TOOLKIT_PRICING +python3 -I -B research/p104-toolkit-v3/tests/test_checkers.py # D PASS_TOOLKIT_CHECKERS +python3 -I -B research/p104-toolkit-v3/tests/test_dual.py # B PASS_TOOLKIT_DUAL +python3 -I -B research/p104-toolkit-v3/tests/test_drops.py # C PASS_TOOLKIT_DROPS (numpy) +python3 -I -B research/p104-toolkit-v3/tests/test_dualdrop.py # B + C PASS_TOOLKIT_DUAL_DROP (numpy) +python3 scripts/research_package.py --package research/p104-toolkit-v3 +``` + +Run them from the repository root; `validate.py` needs the checkout for the repository's invoice tools and skips +that step elsewhere. Every tool also runs alone (`python3 TOOL.py` prints its usage). Words are JSON or gzipped JSON +in the hosted native format (`public`, `owned_fields`, `frames`, `initial`, `final`, `live`, `hosted`, `program`). + +--- + +## A. `pricekit.py`: the exact pricer + +Our single exact pricer for P104 caller + native invoices (PRICEKIT lane, vendored unchanged). The module docstring +states the full accounting contract: charging rules for caller events, the native word's paid and helper histograms, +the ledger law, and the parametric native. + +``` +python3 pricekit.py data/caller-7113.json.gz data/base-7546-word.json.gz +# [PRICEKIT pricekit/1 (2026-11-11)] ...: S=1424 kappa=0.00364841129479084532481164 D0=393242.1350 ... +``` + +- API: `native_from_word(W)`, `caller_invoice(C)`, `invoice(caller, native)`, `price(inv)` → S, κ enclosure, D0, Dk; + `check_slots(C, native)` (the ledger law d = m); `replay(W)` (exact Fractions); `drop_registers` / `verify_drop`. +- κ is the root of F(t) = Σ (T_r − P_r) r^(1−t). It is enclosed by bisection with error-bounded Decimal signs. + Uniqueness is certified by interval monotonicity. + +Controls [C]: + +- #7546 prices to S = 1,424, κ = 0.00364841129479084532481164, D0 = 393,242.1350 and 110,468 paid jobs. The + enclosure is narrower than 1e−50 and uniqueness is proved. +- Our #7601 word, rebuilt from its edit list, prices to κ = 0.0036490213053944204892137903671. +- The ledger law d = m holds for both port types. +- Negative controls: an invoice with S ≤ 0 returns `NO_CROSSING`, and 104 extra rank-1 paid jobs lower κ. + +## B. `dualword.py`: frame-perp duality + +`dual(W)` reverses time, transposes every local op and replaces every frame F by F^⊥ (standard dot on F₂⁹). It also +swaps the X and Y register of each port (live r ↔ r + 112, hosted X2 ↔ Y2). The op rules: + +- `ADD r += q·s` becomes `ADD s += q·r`, and `SCALE` is unchanged. +- `MOVE a→b` becomes `MOVE b^⊥→a^⊥`. Perp reverses inclusion, so an ascent stays an ascent with the same gap. +- `INIT` and `CLEAR` trade places, and `COPY` becomes `ADD`. + +``` +python3 dualword.py data/base-7546-word.json.gz /tmp/dual.json.gz +python3 accx.py data/base-7546-word.json.gz DUAL_WITH_SCALES.json.gz --allow-owned-reframe +``` + +Results: + +- **[P] The price is invariant.** Every charge of the native invoice is a function of dimension gaps: a MOVE pays + |dim b − dim a|, and a helper is credited dim final − dim initial. Perp maps dim d to 9 − d, so every gap is kept. + Hence the dual has the same T and P histograms and the same κ. The cost table is not symmetric in dim, but only + gaps are priced. +- **[C]** dual(#7546) has identical native histograms and the same S, κ enclosure and D0 bit for bit. + dual(#7601) prices to #7601's κ. +- **[C] dual is an involution.** dual(dual(#7546)) differs from #7546 in exactly one event: the single COPY comes + back as an ADD, and `restore_copy` puts it back. The frame basis lists differ; their spans agree. +- **[C] B ports.** The 8 native-B ports (the ports outside `hosted.Dports`) need free contract SCALEs (x·2 at the + start, Y·½ at the end; 16 events, `fix_b_ports`). After them, all 1,501 public rows equal #7546's exactly. +- **[C] A second admissible base.** The dual passes every accx check except our edit lanes' rule "owned MOVEs + identical to the base": its 47 owned moves use different frames, with the same 24 descents and 23 ascents. It has + 0 lossy CLEARs. +- **[C] Negative control.** The transpose without the X ↔ Y port swap fails accx. + +**[S] Use.** Duality gives a second admissible word with the same κ and different owned frames and co-locations. +Scans that are blind on the primal can see edits on the dual (§C.3). + +## C. `tokdrop.py`: the token-settled drop engine + +This is the engine behind our [#7601](https://github.com/CrocSwap/integer-mult-bounds/pull/7601). A **stay** is one +interior stop of a public register, `(r, F, t_arr, t_dep)`. Dropping it deletes the MOVE into F and every event of r +at F, and lets r climb straight to its next frame. The engine then decides whether the damage repairs for free, in +three steps. + +1. **Clean-mode repair** (`Context.test`, `Engine.solve`). + - Unknown corrections `a += k·b` are placed wherever a and b share a frame after a became dirty, with b still + clean there. This keeps the final rows linear in k. + - Every main, and every helper not ending at V, must end with its base row. The comparison uses the main (π) + columns only: this is "clean mode", where the helper-token columns are deleted. + - Helpers ending at V must land in the row span of the V group, because V post-ops are free. + - Owned rows at every owned→public write and at every CLEAR must equal the base. + - The system is solved mod 2^31 − 1 by random rank-1 tensor projections (Schwartz–Zippel), a rank test and the + canonical rref solution. +2. **Settlement rank** (`Context.settle_rank`). Clean mode ignored the helper-token columns. They are settled by a + premix at t = 0: an invertible block-diagonal map on registers that share a birth frame. + rank = Σ_f rank[X_new,f | X_base,f] − rank X_new,f, and rank 0 means such a premix exists. Tier B keeps every row + exact; tier A relaxes helpers that end at V. +3. **Exact build** (`Context.build_exact`). + - Rational reconstruction of the corrections. + - An exact Fraction solve for the premix, as ADDs at the birth frame at t = 0. + - Exact V post-ops. + - An exact replay against the base. The build refuses (`BuildError`) unless every step is exact and no new owned + observation appears. + +``` +python3 tokdrop.py data/base-7546-word.json.gz scan 134:595 100:783 # repair + settlement ranks +python3 tokdrop.py data/base-7546-word.json.gz build 100:783 134:595 OUT.json.gz +``` + +`numpy` is required. If `python-flint` is installed, `modp.py` uses it; otherwise a pure-numpy backend runs. Both +return the same canonical rank, rref and null space. The rebuild below is byte-identical under either backend: we ran +it with flint in the lane and with numpy here. + +### C.1 Controls on #7546 [C] + +- The census lists 4,728 interior public stays with their stop price m. The stop price is the D0 per call saved by + merging the two MOVEs: f(gap) − f(a) − f(b), with f(x) = x ln x. +- **134@595** repairs in clean mode with K = 355 candidates (r1 = r2 = 135), settlement rank 0 in tier A and tier B, + and π exact. +- **100@783** repairs with K = 118 candidates (r1 = r2 = 46), settlement rank 0/0. +- **Negative control.** With correction candidates switched off (`frames_mode='none'`), both drops leave violated + rows. The repair is what makes them free. +- **Negative control for the build.** On the dual (§C.3), stay 1289@117 settles in tier A but not in tier B, and + `build_exact` refuses it because its premix is not exactly solvable. + +### C.2 #7601 rebuilt byte-for-byte [C] + +Running `build_exact` on both stays gives **exactly the #7601 word**: 76,642 events, 4 corrections, 152 premix ADDs at +{0} and 0 post-ops. Its stop price is 3.8191 D0 per call. One owned observation is removed (`134 += −2·owned1503 @595`) +and none is added. accx gives ALL CHECKS PASS and κ = 0.0036490213053944204892137903671. + +### C.3 The main-902 drop, found on the dual [C] + +This is the dual scan of our DUALSCAN lane: + +1. Take the dual of #7601 and swap frames 3 ↔ 127, since {0} and V trade places under perp. The engine then runs + unchanged on the dual. +2. On the dual, register 794 (the image of hosted main 902) drops its stop at chart 595 with **K = 0**: no + correction, no premix, settlement rank 0/0. +3. Map the result back with `swap_frames`, `dual` and `restore_copy`. Main 902 then climbs 3 → 618 in one MOVE + instead of 3 → 595 → 618, and the 5 owned ADDs it made at chart 595 vanish. + +Results: + +- accx against #7546 (strict, owned MOVEs identical): ALL CHECKS PASS, with 0 lossy CLEARs. +- The word **#7601 + 902@595** prices to κ = 0.00364932638757719639692487. That is ×1.0000836 over #7601 and + below the frontier #7605. +- **Negative control.** The primal engine cannot find this drop: on #7601, dropping 902@595 directly has no clean + repair (K = 211, rank test fails). It is a V-side-origin edit. + +Caveats: + +- The drop removes one owned observation, the public→owned write 1532 += 2·w902, as 134@595 did in #7601. +- This word has not had an independent audit of owned-observation legality, so we make **no κ claim** for it. + +**[S] Scale.** Single token-settled drops are worth about +0.008% each. In our census only a handful have rank 0, so +this is a small layer of about 10 D0 per call, not a mass mechanism. + +## D. Checkers + +### D1. `accx.py` + `fastreplay.py`: admissibility against a base + +``` +python3 accx.py BASE NEW [--caller CALLER] [--allow-owned-reframe] +``` + +The checks: + +1. The owned, live, ports and hosted blocks are the same. Extra helpers may be appended. +2. The replay is clean: frames, exits and owned lifetimes all check. +3. Every base final row is unchanged. +4. Mains keep their initial and final frames. +5. Helpers end as the identity row, owned rows end empty, and every helper strictly ascends. +6. MOVE legality holds. +7. The owned MOVEs are the same as the base's. + +Then it prints both prices and the κ ratio. `fastreplay.py` replays the N × 4 projection row·R mod 2^61 − 1 instead +of the full row matrix. A nonzero row difference vanishes with probability 2^−244, and every flagged register is +re-checked by an exact Fraction replay of its backward slice. So **FAIL is exact-confirmed and PASS is +randomised-exact**. + +Results [C]: + +- #7546 against itself passes in about 1 s; the exact accx2 took 14 s. +- #7601 passes with GAIN. +- One corrupted ADD coefficient fails, and the exact re-check confirms the differing row. +- One reversed public MOVE fails. + +### D2. `lossless.py`: law DUAL + +**[P] The law.** While the live rows span Qⁿ, the relation space Λ = {λ : Σ λ_r row_r = 0} over live registers has +dimension equal to the number of live owned registers. Ops act on Λ contragrediently: + +- `ADD d += q·s` acts as λ_s −= q·λ_d. +- `INIT o` adds e_o to Λ. +- `CLEAR o` is lossless iff some λ ∈ Λ has λ_o ≠ 0; then Λ shrinks to {λ_o = 0}. + +So one forward pass mod p finds every lossy CLEAR. `culprit_chain` names the read that killed the coordinate. + +``` +python3 lossless.py WORD.json.gz +``` + +Results [C]: + +- #7546, #7601 and the dual each have 0 lossy CLEARs. +- Deleting one owned→public write-back in #7546 makes a CLEAR lossy, and it is found. +- On 400 random programs the law agrees with exact Fraction ranks (141 of them have a lossy CLEAR). + +### D3. `nativeplan.py`: the helper plan + +``` +python3 nativeplan.py WORD.json.gz +``` + +It extracts the helper layer as a plan: routes and stays; every helper event typed as pickup, delivery, relay or +scale; meetings; requirement blocks linked to their deliveries; exact π-states at every stay entry and exit; and the +requirement-space basis. `recost(plan)` re-costs the helper layer from routes alone, and `reemit_check` regenerates +every helper event from the plan. + +Results on #7546 [C]: + +- 1,101 helpers, 3,427 interior stops, 2,682 requirement blocks, 36,299 meetings, and state basis rank 253. +- The recost is **9,975.447669**, the same from routes and from stays. +- All 70,714 helper events are re-emitted identically. +- Negative control: one tampered plan coefficient breaks the re-emission check. + +### D4. `validate.py`: one command + +``` +python3 validate.py BASE NEW +``` + +It runs accx, then lossless, then the repository's `compare_recursive_invoices.py` (BASE → NEW) and +`evaluate_recursive_power.py` (NEW against the empty invoice) at C just below and just above NEW's certified κ. +It prints a summary table, and the exit code is 0 iff every step passes. + +Results [C]: + +- On #7601: OVERALL PASS. The repository tools give `uniformly_better` and place the crossing in + (0.0036490213053, 0.0036490213054), as in #7601's own certificate. +- A corrupted word gives OVERALL FAIL. + +## E. Pricing models + +### E1. `kformula.py`: κ as a function of the stage count k + +The closed-form caller C(k) has two arms ⌊k/2⌋ | ⌈k/2⌉, 2 hubs and grow 2, with g = h − 1 = 8. Per port: + +- **Targets.** 2·T[gj+1] for each arm role, and 2·T[gk+1] for the hubs. +- **Paid.** 2·P[gj] for the interior arm idles. +- **Type A (96 ports).** 4·P[g] + 4·P[1]. +- **Type B (8 ports).** 2k·P[g] + 4·P[g+1]. + +The invoice is Invoice(k) = C(k) + k·N, where N is a word's native invoice. + +``` +python3 kformula.py 12 16 17 18 24 +``` + +Results: + +- **[C] C(17) is the #7113 caller.** C(17) equals the #7113 caller invoice (T and P), and C(17) × the #7546 native + prices to κ(#7546). C(16) differs. +- **[P, C] S(k) = 96k − 208 = 2k(v − c) − ρv** (v = 104, c = 56, ρ = 2), checked for k = 4..64. Per port, arm field j + and idle j cancel their gj parts: an A port gives 34k − 2, a B port 18k − 2, and the native gives −3,312 per call. +- **[P, C] The D0 slope.** D0(k)/104 = β k ln k + α k + O(ln k), with β = 2(2h − 1) = 34. Measured from + k = 1000, 2000: 33.976. The ln k comes from the hubs (degree k) and arm tops (degree k/2): they carry each stage's + 2(h − 1) main mass at rank about (h − 1)k. +- **[C] The exact sweep.** κ(k) peaks at k = 17 and is strictly unimodal on 6..40: κ(16) = 0.0036481937 and + κ(18) = 0.0036477620. +- **[P, C] The first-order law.** κ(k) ≈ S/D0 = (ε − ρ/k)/(β ln k + α) with ε = 2(1 − c/v) = 12/13. At k = 17, + κ/(S/D0) = 1.0075. With α = 124.913 the formula is within 1% of the exact κ. +- **[P, C] The optimum.** Stationarity gives **κ* = ρ/(βk*) = 1/((2h − 1)k*)**. Here γ = 1 + α/β + ln(ρ/ε) = 5.4471 + and u* = −W₋₁(−e^−γ), so k* = 16.15 and κ* = 0.003641. The exact family optimum is k = 17. +- **[P] No positive limit.** κ(k) ~ ε/(β ln k) → 0: every nested two-arm root decays like 1/ln k. + +### E2. `layout_price.py` and `cap_check.py`: caller layouts + +These tools come from CALLEROPEN, an outside-session lane of DreamingOfClouds'. + +**`layout_price.py` [C-param]** prices a ρ = 2 mask layout from role masks and per-boundary line caps. + +- Ledger: T = Σ T[8·deg + 1]. Idle incidences at stage k are |Z_k| − 2. There are n + 2 − 2L thin QUERYs, absorbed + as IDLE_ODD. +- Lines continue the oldest first, up to the caps. +- It prices against three what-if readings of the #7546 native: + - *twins deleted*: the hosted main moves removed from the paid histogram. This is a ceiling. + - *shadows*: the hosted block priced as helpers. + - *shadows + 2 relays*. +- These are prices of hypothetical callers with hypothetical natives, not words. + +Results: + +- **[C-param]** Model B reproduces the built B caller's readings: 1.0950 / 1.0031 / 0.9773 (κ ratios against #7546). +- **[C-param]** Uncapped lines give the unreachable reference 1.8950 / 1.6368 / 1.5695. + +**`cap_check.py` [C]** works on the built #7113 B program. It computes end-referred x and y spans per call mod p, at +q₀ = 1 and q₀ = i, and checks: + +- **Lemma C.** The crossing cap at boundary k|k+1 is |S| − rank(Y_k, Y_{k+1} | S), with S = Z_k ∩ Z_{k+1}. +- **The Floor lemma.** rank(X of the arm up to k, plus the hub vectors) = |Z_k|. + +Results [C]: + +- rank = 4 at all 15 non-centre boundaries, at both q₀ = 1 and q₀ = i. +- The Floor lemma holds with equality at every non-centre call. + +--- + +## What is not included, and overlap with earlier packages + +- **Excluded as work in progress.** The network-compiler route (NETC2–NETC6 and the REBUILDKIT netc planners), and + `floor_report.py`, which reads NETC2 plans. Nothing from that route is claimed. +- **Excluded as not self-contained.** + - The RES-LAYER sensitivity sweep, which executes another lane's script on a composite W6805 model. + - The TOY5 Pocket-Lemma checker, which needs CP-SAT, unavailable in generic CI. +- **Already shipped in #7606** (`research/p104-odd-unit-per-grading`): + - The Cut Lemma checker on real words (WEBS `cutcheck.py`, shipped there as `cuts.py`). + - The CLONEAUDIT Rank-lemma toy, in `walls.py`. + - The TOY4 F₂³ question, recorded there as in progress. +- **#6170** (`research/p104-designer-toolkit-v2`) is our earlier toolkit for the #6030 word lineage and its verifier + profiles. This package targets the hosted #7546 format and does not replace it. +- **#7601**'s `check.py` certifies one word. `tokdrop.py` is the engine that found and built that word. + +## Open next questions + +1. **More dual drops.** + - *Question.* Do the untried non-owned stays of the dual (about 4,500), or the dual of the current frontier word, + hold more rank-0 drops? + - *First step.* Run `tokdrop.py scan` over the dual census in shards, then `build_exact` each hit and map it back. +2. **Exact tier-A settlements.** + - *Question.* Can the tier-A-only settlements be made exact? + - *The candidates.* 624@117, 627@117 and 490@594 on the primal; 438@245, 271@306 and 1289@117 on the dual. + Together they are worth up to about 7.7 D0 per call. + - *First step.* Give the exact premix solve the V post-op freedom. +3. **Bounded headers.** + - *Question.* Is there a caller whose stages meet in bounded headers stitched by tree nodes, so that + D_caller = O(k) and κ(k) has a positive limit? + - *First step.* Price a toy 8-stage version with `layout_price.py` and check its crossings with `cap_check.py`. + +## Credits + +The tools are ours (DreamingOfClouds with Claude Opus 5.5, via Claude Code); CALLEROPEN is an outside-session lane +of DreamingOfClouds'. Credits for the data: + +- ToikY for the #7546 word. +- Rohan Arun for the #7113 caller. +- Clear1nsight, ToikY and the P104 price and replay lineage for the conventions these tools follow. + +See [NOTICE](NOTICE). diff --git a/research/p104-toolkit-v3/SOURCES.json b/research/p104-toolkit-v3/SOURCES.json new file mode 100644 index 0000000000..fcea174185 --- /dev/null +++ b/research/p104-toolkit-v3/SOURCES.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "base_commit": "e54523517d0f03e58352e647c403a5af3e525a57", + "files": [ + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + }, + { + "path": "scripts/compare_recursive_invoices.py", + "usage": "input", + "git_blob": "fda2a852e9937e42fe40df0707694465ca7c1b48", + "sha256": "3930c22c11da7bc6042cfeb6e459ecb06e4478430037bee3469d034353855fc6" + }, + { + "path": "scripts/evaluate_recursive_power.py", + "usage": "input", + "git_blob": "ca50f8021fc3238a58fd420383f4619a8d008ace", + "sha256": "52b20f41fd0bb55fa88dccd6015af2ce358917817b9506720f3e60ce681cc3ad" + } + ] +} diff --git a/research/p104-toolkit-v3/TASK.json b/research/p104-toolkit-v3/TASK.json new file mode 100644 index 0000000000..46236df1a8 --- /dev/null +++ b/research/p104-toolkit-v3/TASK.json @@ -0,0 +1,129 @@ +{ + "schema_version": 1, + "task_id": "p104-side-correction", + "priority_revision": 8, + "base_commit": "e54523517d0f03e58352e647c403a5af3e525a57", + "original_card": { + "id": "p104-side-correction", + "question": "Can joint helper bases and a legal causal split improve the complete P104 productive/paid invoice?", + "starting_points": [ + "docs/research/current-priorities.md", + "docs/research/research-family-status.md", + "certificates/selected-result.json", + "docs/research/pr-intake-evidence/pr5599-completion/PROOF-REVIEW.md", + "research/p104-helper-subspace-splits/README.md", + "research/dyadic-helper-channel-capacity/README.md" + ], + "methods": [ + "Construct and independently check one source-bound instance of the stated first experiment.", + "Seek an exact counterexample or target-specific complete-budget obstruction to that same proposed change." + ], + "acceptance_gate": "An exact arbitrary-input identity, source/target frame availability and restoration at the chosen cut, plus the complete channel+correction+idle/hosting bill or an explicit favorable bound. The native channel alone is not a supplier. A scoped local result may be retained without a complete supplier. Expanding it into further construction rounds requires the family-level budget and obligation gate in research-family-status.md; an optimistic passing relaxation alone is not acceptance.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Read revision 8, the selected review and live exact-source handoffs. Then follow this card's bounded first_experiment.", + "scope": "Source-bound orientation only, not a supplier replay or theorem proof." + }, + "first_experiment": "Start with the selected #5599 emitted word and review pending #5718 distinct input/output bases plus #5717/#5719 coverage before claiming work. Pick one uncovered source word, helper group and cut region. Prove the arbitrary-input relation and legal entry/exit corrections, then compare full target and paid profiles. Reject if completion, owned lifetimes or consumer costs repay the gain. A static pairing rank alone is not a physical split.", + "bottleneck_link": "Judge the actual current supplier contract and complete signed target/paid bill; frozen-profile ceilings require an applicability proof.", + "allowed_variations": [ + "One exact source head, helper/cut region or quantified proposition and changed hypothesis; check live handoffs first. Different seeds are not different scopes." + ], + "scope_change_triggers": [ + "Changing source head, helper/cut domain, operation alphabet or caller boundary without recording the new scope.", + "Treating an archived proposal or local numerical pass as complete supplier admission." + ], + "overlap_terms": [ + "P104", + "helper basis", + "helper split", + "5718", + "dyadic capacity" + ], + "existing_checks": [ + { + "scope": "PR #4133: literal-slot defect", + "evidence": "research/p104-literal-slot-defect/README.md", + "remaining": "Independently validate the stated scope or change its producer/consumer premise and pay the complete boundary.", + "status": "author-checked", + "source_commit": "88666f1dee93fbd85b31fd5d6f91a219e111b5f4", + "selected_record_sha256": "bb2eaa58647b353f88d0aec2eafb9d30df1be53589231b23e12ceef3e4bad129" + }, + { + "scope": "PR #4136: uniform commutation gap", + "evidence": "research/p104-uniform-commutation-gap/README.md", + "remaining": "Independently validate the stated scope or change its producer/consumer premise and pay the complete boundary.", + "status": "author-checked", + "source_commit": "281ce601a8be9db470de630759179c23e1dfaa88", + "selected_record_sha256": "bb2eaa58647b353f88d0aec2eafb9d30df1be53589231b23e12ceef3e4bad129" + }, + { + "scope": "PR #4140: uniform encoding gap", + "evidence": "research/p104-uniform-encoding-gap/README.md", + "remaining": "Independently validate the stated scope or change its producer/consumer premise and pay the complete boundary.", + "status": "author-checked", + "source_commit": "de25c6525aa866962309df5d107b32d55e6e5628", + "selected_record_sha256": "bb2eaa58647b353f88d0aec2eafb9d30df1be53589231b23e12ceef3e4bad129" + }, + { + "scope": "PR #4153: source-chart shortcut exhausts its optimistic allowance", + "evidence": "research/p104-source-chart-shortcut/README.md", + "remaining": "Independently validate the stated scope or change its producer/consumer premise and pay the complete boundary.", + "status": "author-checked", + "source_commit": "9123241fea53df77e51ffe30d9aca1625d4bf3a8", + "selected_record_sha256": "bb2eaa58647b353f88d0aec2eafb9d30df1be53589231b23e12ceef3e4bad129" + }, + { + "scope": "PR #4154: fiber-tag seam mismatch", + "evidence": "research/p104-fiber-tag-seam/README.md", + "remaining": "Independently validate the stated scope or change its producer/consumer premise and pay the complete boundary.", + "status": "author-checked", + "source_commit": "4ffcdaee5d6255a9cea1fe53a5424435068089d0", + "selected_record_sha256": "bb2eaa58647b353f88d0aec2eafb9d30df1be53589231b23e12ceef3e4bad129" + }, + { + "scope": "Selected PR5599: complete helper-subspace construction and independent caller replay; five framework assumptions remain.", + "evidence": "docs/research/pr-intake-evidence/pr5599-completion/PROOF-REVIEW.md", + "remaining": "The next experiment must identify uncovered source-specific work. Successor archive intake does not transfer this admission.", + "status": "reviewed", + "source_commit": "09dd0965711ca7eeb169057a420abab1c9052137", + "selected_record_sha256": "2a0220e866417774606e0f991e41b9c1d26b1f43b97857238cf478dbc7d330b5" + } + ] + }, + "scope_status": "changed", + "actual_scope": "A reusable toolkit of our stable, verified P104 tools for the hosted-word format of #7546 and later: (A) the exact pricer pricekit.py; (B) the frame-perp duality dualword.py (word -> dual word, a price-preserving involution); (C) the token-settled drop engine tokdrop.py (clean-mode stay-drop repair, premix settlement rank, exact build), which rebuilds our #7601 word byte-for-byte and, run on the dual, finds the main-902 drop; (D) checkers accx.py (fast randomised-exact replay with exact re-checks), lossless.py (law DUAL for lossy CLEARs), nativeplan.py (helper-plan extractor) and validate.py (accx + lossless + the repository's invoice tools); (E) pricing models kformula.py (closed-form caller C(k) and the kappa(k) law), layout_price.py and cap_check.py. Every tool ships with a runnable example and tests with positive and negative controls on vendored #7546 data.", + "scope_change_reason": "The card asks for a constructive helper-basis or causal-split improvement. This package instead ships the tools we used for such edits (drop repair and settlement, duality, exact checking and pricing), so others can run and extend them; it makes no kappa claim.", + "bottleneck_link": "Every candidate P104 edit has to be repaired, settled, checked and priced exactly before it can be judged against the complete invoice; these tools do each step exactly and quickly on the current word format.", + "budget_minutes": 240, + "stop_rule": "Every tool has docs, a CLI or API example and a test with positive and negative controls that runs in minutes on vendored data; claims are tagged; WIP tools are excluded; validator PASS.", + "outcome": "completed", + "evidence": "README.md, pricekit.py, dualword.py, tokdrop.py, modp.py, accx.py, fastreplay.py, lossless.py, nativeplan.py, validate.py, kformula.py, layout_price.py, cap_check.py, sl_lib.py, tests/, data/, VERIFY.json.", + "author": "DreamingOfClouds", + "ai_assistance": "Claude Opus 5.5 (Anthropic), via Claude Code.", + "open_questions": [ + { + "question": "Do the remaining non-owned stays of the dual of #7546 (about 4,500, untried) or of the current frontier word hold more rank-0 drops?", + "why_it_matters": "The dual scan found a free drop (main 902 at chart 595) that no primal scan could see; each such drop is worth about +0.008% on its own.", + "first_step": "Run tokdrop.py scan over the dual's census (Context.census(), kinds without OW/OR), in shards; build every rank-0/0 hit with build_exact and map it back with dualword.py.", + "blocked_by": "CPU time only (about 10-30 s per stay)." + }, + { + "question": "Can tier-A-only settlements (rank 0 with V-helper rows relaxed, rank >= 1 exactly) be made exact?", + "why_it_matters": "About ten such stays exist on #7546 and its dual (624@117, 627@117, 490@594 primal; 438@245, 271@306, 1289@117 dual); together worth up to ~7.7 D0 per call.", + "first_step": "For one stay, add the V-helper post-op freedom to the exact premix solve in build_exact and test whether the failing column becomes solvable.", + "blocked_by": "Nothing." + }, + { + "question": "Is there a caller whose stages meet in bounded headers stitched by tree nodes, so that D_caller = O(k)?", + "why_it_matters": "kformula.py shows kappa(k) ~ eps/(beta ln k) -> 0 for every nested two-arm root; bounded top-field rank is the only route to a positive limit.", + "first_step": "Price a toy 8-stage caller with two bounded headers joined by one tree node with layout_price.py and check its crossings with cap_check.py.", + "blocked_by": "Nothing." + } + ] +} diff --git a/research/p104-toolkit-v3/VERIFY.json b/research/p104-toolkit-v3/VERIFY.json new file mode 100644 index 0000000000..300ed634d6 --- /dev/null +++ b/research/p104-toolkit-v3/VERIFY.json @@ -0,0 +1,64 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [ + "numpy==2.5.4" + ], + "inputs": [ + "scripts/compare_recursive_invoices.py", + "scripts/evaluate_recursive_power.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-B", + "{package}/tests/test_pricing.py" + ], + "timeout_seconds": 120, + "checks": "Data hashes (#7546 word bytes, #7113 caller JSON, #7601 edit list). pricekit: #7546 S = 1424, kappa 0.00364841129479084532481164, D0 393242.1350, 110468 paid jobs, enclosure < 1e-50, uniqueness proved; #7601 word rebuilt from its edit list kappa 0.0036490213053944204892137903671; ledger law d = m; NEG: S <= 0 gives NO_CROSSING, 104 extra rank-1 jobs lower kappa. kformula: C(17) == #7113 caller invoice, C(17) x #7546 native = kappa(#7546), NEG C(16) differs, S(k) = 96k - 208 for k = 4..64, exact sweep peaks at k = 17 and is unimodal on 6..40, D0 slope beta = 34, kappa(17)/(S/D0) = 1.0075, first-order formula within 1%, Lambert optimum k* = 16.15, kappa* = .003641. layout_price: model B = built B readings 1.0950/1.0031/0.9773 [C-param], uncapped reference 1.8950/1.6368/1.5695. cap_check on the built #7113 B: rank 4 at all 15 non-centre boundaries (q0 = 1, i), Floor lemma equality at every non-centre call. Prints PASS_TOOLKIT_PRICING." + }, + { + "argv": [ + "{python}", + "-I", + "-B", + "{package}/tests/test_checkers.py" + ], + "timeout_seconds": 240, + "checks": "accx: #7546 vs itself PASS; #7601 vs #7546 PASS with GAIN and kappa 0.0036490213053944...; NEG corrupted ADD coefficient FAILS with exact re-check; NEG reversed public MOVE FAILS. validate: #7601 OVERALL PASS with the repository tools (compare_recursive_invoices uniformly_better, evaluate_recursive_power crossing in (0.0036490213053, 0.0036490213054)); NEG corrupted word OVERALL FAIL. fastreplay exact backward slice. lossless (law DUAL): 0 lossy CLEARs on #7546 and #7601; NEG deleting an owned->public write-back makes a CLEAR lossy; agreement with exact Fraction ranks on 400/400 random programs (141 lossy). nativeplan: recost 9975.447669 (routes = stays), 1101 helpers, 3427 stops, 2682 blocks, 36299 meetings, basis rank 253, 70714 helper events re-emitted identically; NEG tampered plan breaks re-emission. Prints PASS_TOOLKIT_CHECKERS." + }, + { + "argv": [ + "{python}", + "-I", + "-B", + "{package}/tests/test_dual.py" + ], + "timeout_seconds": 180, + "checks": "dualword: dual(#7546) has identical native T/P histograms and identical S, kappa enclosure, D0; frames map to perp (dim d -> 9 - d); every MOVE keeps its signed gap; dual(dual) = #7546 up to frame bases with the single COPY restored (exactly one event differs); exactly the 8 native-B ports need the free contract SCALEs, after which all 1501 public rows equal #7546 (Fraction replay); accx strict fails only the owned-MOVE identity rule, --allow-owned-reframe ALL PASS with kappa ratio 1; 0 lossy CLEARs; dual(#7601) priced = #7601; NEG transpose without the X/Y port swap fails accx. Prints PASS_TOOLKIT_DUAL." + }, + { + "argv": [ + "{python}", + "-I", + "-B", + "{package}/tests/test_drops.py" + ], + "timeout_seconds": 360, + "checks": "tokdrop on #7546 (numpy backend unless python-flint is present): census 4728 stays, {0} = 3, V = 127, stop price 1.9095 for both #7601 stays; 134@595 repairs with K = 355, 100@783 with K = 118, settlement rank 0 in tiers A and B, pi exact; NEG both drops with no correction candidates are not repairable; exact build of both = the #7601 word byte-for-byte (4 corrections, 152 premix ADDs, 0 post-ops, 1 owned observation removed, none added); accx ALL PASS, kappa 0.0036490213053944.... Prints PASS_TOOLKIT_DROPS." + }, + { + "argv": [ + "{python}", + "-I", + "-B", + "{package}/tests/test_dualdrop.py" + ], + "timeout_seconds": 300, + "checks": "Dual scan: dual of #7601 with frames 3 <-> 127 swapped, main 902 = dual register 794; NEG primal 902@595 on #7601 has no clean repair; dual 794@595 repairs with K = 0, settlement rank 0/0; exact build (0 corrections, 0 premix, 1 owned observation removed); mapped back (swap, dual, COPY restore) main 902 climbs 3 -> 618 in one MOVE and its 5 owned ADDs at 595 vanish; accx strict vs #7546 ALL PASS, kappa 0.00364932638757719639692487 (above #7601); 0 lossy CLEARs; NEG dual 1289@117 has settlement rank A 0 / B >= 1 and the exact build is refused. Prints PASS_TOOLKIT_DUAL_DROP." + } + ] +} diff --git a/research/p104-toolkit-v3/accx.py b/research/p104-toolkit-v3/accx.py new file mode 100644 index 0000000000..29b6af5535 --- /dev/null +++ b/research/p104-toolkit-v3/accx.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""accx: admissibility check of an edited P104 native word against its base (our ABSLOCAL accx -> REBUILDKIT accx2 -> +NETCAUX accxfast lineage). Standard library only. + +The two exact Fraction replays of accx2 are replaced by fastreplay.py projections (mod 2^61-1, k = 4 random columns); +every projected mismatch is re-checked by an exact Fraction replay of the backward slice, so FAIL is exact-confirmed and +PASS is randomised-exact (a nonzero row difference survives with probability 2^-244 per row). On #7546: 14 s -> ~2 s. + +Checks (each prints PASS/FAIL; exit 1 at the first FAIL): + 1. same owned/live/ports/hosted block; extra public registers may be appended after the base publics + 2. replay: every MOVE leaves the frame it claims, every ADD/SCALE/COPY happens with both registers at its frame, + COPY targets an empty register, every owned lifetime is closed, exit frames match 'final' + 3. every base final row (public and owned) is unchanged (through the index map) + 4. mains (live X/Y, hosted X2/Y2) keep their initial and final frames + 5. helpers end as the identity row; owned rows end empty; every helper strictly ascends + 6. MOVE legality: public moves strictly ascend, owned moves are nested, ownership flags are right + 7. owned MOVEs identical to the base as a multiset of (a, b) frame pairs (the rule of our edit lanes; pass + --allow-owned-reframe for words, such as a frame-perp dual, that legally re-frame the owned moves) +Then it prints the price of both words with pricekit.py (#7113 caller by default) and the kappa ratio. +Coefficients outside the dyadic alphabet are reported (ODD-DENOMINATOR FLAG), not failed. + +usage: python3 accx.py BASE NEW [--caller CALLER.json[.gz]] [--allow-owned-reframe] +""" +import collections +import sys +import time +from fractions import Fraction as Q +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import fastreplay as fastx # noqa: E402 +import pricekit as pk # noqa: E402 + +DEFAULT_CALLER = HERE / 'data' / 'caller-7113.json.gz' + + +def span(b): + s = {0} + for v in b: + if not (isinstance(v, int) and 0 < v < 512 and v not in s): + raise ValueError('frame basis vector %r is not independent in F2^9' % (v,)) + s |= {x ^ v for x in s} + return frozenset(s) + + +class Fail(Exception): + pass + + +def check(B, W, caller=DEFAULT_CALLER, allow_owned_reframe=False, out=print, price=True): + t0 = time.time() + + def need(ok, msg): + out(('PASS ' if ok else 'FAIL ') + msg) + if not ok: + raise Fail(msg) + spB = [span(b) for b in B['frames']]; spW = [span(b) for b in W['frames']] + n = W['public']; NO = W['owned_fields']; NR = n + NO + n0 = B['public']; EXT = n - n0 + need(EXT >= 0 and NO == B['owned_fields'] and W['live'] == B['live'] and W['ports'] == B['ports'] + and W['hosted'] == B['hosted'], 'same owned/live/ports/hosted block; public %d -> %d (+%d appended helpers)' % (n0, n, EXT)) + mp = lambda r: r if r < n0 else r + EXT + + def sig(e, m): + if e[0] in ('ADD', 'COPY'): + return (e[0], m(e[1]), m(e[2]), e[3]) + if e[0] == 'SCALE': + return (e[0], m(e[1]), e[2]) + return (e[0], m(e[1])) + cB = collections.Counter(sig(e, mp) for e in B['program'] if e[0] != 'MOVE') + cW = collections.Counter(sig(e, lambda r: r) for e in W['program'] if e[0] != 'MOVE') + out(' declared event diff: +%d / -%d non-MOVE events' % (sum((cW - cB).values()), sum((cB - cW).values()))) + K = 4 + Rb = fastx.proj_matrix(n0, K, 1) + Rn = Rb + fastx.proj_matrix(n - n0, K, 7) + fB = fastx.FastReplay(B, Rb, K); fW = fastx.FastReplay(W, Rn, K) + need(fW.ok and fW.nproblems == 0, 'randomised-exact replay (mod 2^61-1, k=%d): 0 frame/exit/owned problems %s [%.1fs]' + % (K, fW.bad[:3], time.time() - t0)) + inv = {mp(r): r for r in range(n0 + NO)} + + def exact_flag(regs_new): + regs_new = sorted(regs_new)[:3] + eN = fastx.exact_rows(W, regs_new); eB = fastx.exact_rows(B, [inv[r] for r in regs_new if r in inv]) + confirmed = 0 + for r in regs_new: + if r not in inv: + b = eN[r]; a = {r: Q(1)} + else: + a = {mp(k): v for k, v in eB[inv[r]].items()}; b = eN[r] + confirmed += a != b + out(' exact re-check: reg %d row differs: %s' % (r, 'YES' if a != b else 'NO (projection false alarm)')) + return confirmed + bad_rows = [r for r in range(n0 + NO) if fB.proj[r] != fW.proj[mp(r)]] + if bad_rows: + exact_flag([mp(r) for r in bad_rows]) + need(not bad_rows, 'all %d base final rows equal (projected; flags re-checked exactly)' % (n0 + NO)) + live = set(W['live']); H = W['hosted']; mains = live | {r + 112 for r in live} | set(H['X2']) | set(H['Y2']) + need(all(spW[W['initial'][r]] == spB[B['initial'][r]] and spW[W['final'][r]] == spB[B['final'][r]] for r in mains), + 'all mains (live X/Y, hosted X2/Y2): initial and final frames unchanged') + badh = [r for r in range(n) if r not in mains and fW.proj[r] != Rn[r]] + if badh: + exact_flag(badh) + need(not badh, 'helpers end identity') + need(all(fW.is_zero(r) for r in range(n, NR)), 'owned rows empty at end') + need(all(spW[W['initial'][r]] < spW[W['final'][r]] for r in range(n) if r not in mains), 'every helper strictly ascends') + + def scan(prog, sp): + v = collections.Counter(); om = collections.Counter(); owned = set() + for e in prog: + if e[0] == 'INIT': + owned.add(e[1]) + elif e[0] == 'CLEAR': + owned.discard(e[1]) + elif e[0] == 'MOVE': + a, b = sp[e[2]], sp[e[3]] + if bool(e[4]) != (e[1] in owned): + v['ownership flag'] += 1 + if e[4]: + om[(tuple(sorted(a)), tuple(sorted(b)))] += 1 + if not (a < b or b < a): + v['owned not nested'] += 1 + elif not a < b: + v['public not strictly ascending'] += 1 + else: + q = Q(e[2] if e[0] == 'SCALE' else e[3]) + if q == 0 or abs(q) + 1 >= 2 ** 16 or (q.denominator & (q.denominator - 1)): + v['alphabet'] += 1 + return v, om + vW, omW = scan(W['program'], spW); vB, omB = scan(B['program'], spB) + alW = vW.pop('alphabet', 0); alB = vB.pop('alphabet', 0) + need(not vW and not vB, 'MOVE legality: public strictly ascending, owned nested, ownership flags (violations %s)' % dict(vW)) + if alW or alB: + out(' ODD-DENOMINATOR FLAG: alphabet violations %d (base %d)' % (alW, alB)) + same_owned = omW == omB + if allow_owned_reframe: + out('INFO owned MOVEs %s base (%d moves; --allow-owned-reframe)' % ('identical to' if same_owned else 'differ from', + sum(omW.values()))) + else: + need(same_owned, 'owned MOVEs identical to base (%d, same (a,b) multiset)' % sum(omW.values())) + res = dict(ok=True, owned_moves_same=same_owned, seconds=time.time() - t0) + if price: + R0 = pk.run(caller, pk.native_from_word(B)); R1 = pk.run(caller, pk.native_from_word(W)) + out(pk.cite('BASE', R0['price'])); out(pk.cite('NEW', R1['price'])) + k0 = R0['price']['kappa']; k1 = R1['price']['kappa'] + out('kappa ratio %.8f %s' % (k1['lo'] / k0['lo'], 'GAIN' if k1['lo'] > k0['hi'] else 'no gain')) + res.update(kappa_base=k0, kappa_new=k1) + out('ALL CHECKS PASS') + return res + + +def main(argv): + args = list(argv); caller = DEFAULT_CALLER; reframe = False + if '--allow-owned-reframe' in args: + args.remove('--allow-owned-reframe'); reframe = True + if '--caller' in args: + i = args.index('--caller'); caller = args[i + 1]; del args[i:i + 2] + if len(args) != 2: + print(__doc__); return 2 + try: + check(pk.load(args[0]), pk.load(args[1]), caller, reframe) + except Fail: + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/research/p104-toolkit-v3/cap_check.py b/research/p104-toolkit-v3/cap_check.py new file mode 100644 index 0000000000..2bdd96f53b --- /dev/null +++ b/research/p104-toolkit-v3/cap_check.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Crossing caps and the Floor lemma on a real caller (CALLEROPEN, an outside-session lane of DreamingOfClouds) +[C exact mod p]. Standard library only. On the BUILT #7113 B program (default; any caller in #7113 format via argv) compute, per call k, +the end-referred x/y spans (sl_lib = copy of CALLEROUT/sl_check.py) and check: + (1) Lemma C: rank(Y_k, Y_k+1 restricted to S = Z_k cap Z_k+1) and rank(X_k, X_k+1 | S) -> crossing cap |S| - rank; + (2) Floor lemma: rank(X of left calls <= k, plus the hub coordinate vectors) = 2(k+1) + 2 <= |Z_k| (and right arm mirrored); + (3) the B idle crossings actually used (from the IDLE runs) vs the cap. Random D, q0 = 1 and i.""" +import gzip, json, random, sys +from pathlib import Path +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from sl_lib import * # noqa: E402,F403 +def spans(C, part, q0, seed=7): + n = C['roles']; rnd = random.Random(seed); Dv = {s: rnd.randrange(2, P) for s in range(1, 18)} + R = ident(n); snaps = {}; order = [] + for e in C['events']: + if e[0] == 'J': + apply_native(R, C, e, q0, Dv, 'A'); apply_native(R, C, e, q0, Dv, 'B'); snaps[e[1]] = (e, [r[:] for r in R]); order.append(e[1]) + else: apply_op(R, e, q0, Dv) + F = R; T = C['types'][part]; off = T['offset']; m = len(T['diagonal']) + masks = {k: {off + i for i, dg in enumerate(T['diagonal']) if dg[1 + k]} for k in range(1, 18)} + XS = {}; YS = {} + for k, (e, Rk) in snaps.items(): + Jp = ident(n); apply_native(Jp, C, e, q0, Dv, part); Jm = ident(n); apply_native(Jm, C, e, (P - q0) % P, Dv, part) + s = matmul(Jp, inv(Jm)); S = matmul(F, inv(Rk)); E = [[(s[i][j] - (i == j)) % P for j in range(n)] for i in range(n)] + Ee = matmul(matmul(S, E), inv(S)) + XS[k] = basis([list(c) for c in zip(*Ee)]); YS[k] = basis(Ee) + return order, masks, XS, YS, off, m +def restrict(vecs, S): return [[v[i] for i in sorted(S)] for v in vecs] +ARGS = sys.argv[1:] or [str(HERE / 'data' / 'caller-7113.json.gz'), 'B'] +for fn, part in zip(ARGS[::2], ARGS[1::2]): + with (gzip.open(fn, 'rt') if fn.endswith('.gz') else open(fn)) as fh: + C = json.load(fh) + for q0 in (1, IM): + order, Z, XS, YS, off, m = spans(C, part, q0) + hubs = [r for r in range(off, off + m) if all(r in Z[k] for k in range(1, 18))] + print('%s part %s q0=%s: call order %s; hubs %s; |Z| %s' % (fn.split('/')[-1], part, 'i' if q0 == IM else q0, order == sorted(order), [h - off for h in hubs], [len(Z[k]) for k in order])) + line = [] + for a, b in zip(order, order[1:]): + S = Z[a] & Z[b] + ry = rank(restrict(YS[a] + YS[b], S)); rx = rank(restrict(XS[a] + XS[b], S)) + line.append('%d|%d:S%d ry%d rx%d cap%d' % (a, b, len(S), ry, rx, len(S) - max(rx, ry))) + print(' boundaries:', ', '.join(line)) + # floor lemma: left arm = calls whose mask grows; use first half where |Z| increases + sizes = [len(Z[k]) for k in order]; peak = sizes.index(max(sizes)) + fl = [] + for i in range(len(order)): + k = order[i]; A = order[:i + 1] if i <= peak else order[i:] + vec = sum((XS[j] for j in A), []) if i <= peak else sum((YS[j] for j in A), []) + hv = [[int(c == h) for c in range(len(vec[0]))] for h in hubs] + fl.append('%d:%d/%d/%d' % (k, rank(vec), rank(vec + hv), len(Z[k]))) + print(' floor (rank X_arm<=k / +hubs / |Z_k|):', ' '.join(fl), flush=True) diff --git a/research/p104-toolkit-v3/data/base-7546-word.json.gz b/research/p104-toolkit-v3/data/base-7546-word.json.gz new file mode 100644 index 0000000000..d63f6b195e Binary files /dev/null and b/research/p104-toolkit-v3/data/base-7546-word.json.gz differ diff --git a/research/p104-toolkit-v3/data/caller-7113.json.gz b/research/p104-toolkit-v3/data/caller-7113.json.gz new file mode 100644 index 0000000000..8a6cac5f96 Binary files /dev/null and b/research/p104-toolkit-v3/data/caller-7113.json.gz differ diff --git a/research/p104-toolkit-v3/data/edits-7601.json b/research/p104-toolkit-v3/data/edits-7601.json new file mode 100644 index 0000000000..a80a021db0 --- /dev/null +++ b/research/p104-toolkit-v3/data/edits-7601.json @@ -0,0 +1 @@ +{"base_events":76490,"delete":[[39879,["MOVE",134,3,595,false]],[39880,["ADD",134,1503,"-2",595]],[40889,["MOVE",134,595,618,false]],[41827,["MOVE",100,716,783,false]],[41828,["ADD",166,100,"-1/2",783]],[42006,["MOVE",100,783,807,false]]],"insert_before":[[0,[["ADD",134,146,"-1/2",3],["ADD",221,146,"-1/2",3],["ADD",134,230,"-1/2",3],["ADD",221,230,"-1/2",3],["ADD",134,367,"-1/2",3],["ADD",221,367,"-1/2",3],["ADD",134,372,"-5/32",3],["ADD",221,372,"-5/32",3],["ADD",134,471,"1/2",3],["ADD",221,471,"1/2",3],["ADD",134,497,"-5/8",3],["ADD",221,497,"-5/8",3],["ADD",134,521,"1",3],["ADD",221,521,"1",3],["ADD",134,537,"-1/2",3],["ADD",221,537,"-1/2",3],["ADD",134,541,"1/4",3],["ADD",221,541,"1/4",3],["ADD",134,542,"-1/4",3],["ADD",221,542,"-1/4",3],["ADD",134,545,"-1",3],["ADD",221,545,"-1",3],["ADD",134,550,"3",3],["ADD",166,550,"-2",3],["ADD",214,550,"-2",3],["ADD",221,550,"3",3],["ADD",134,553,"-3/4",3],["ADD",221,553,"-3/4",3],["ADD",134,568,"-1/2",3],["ADD",221,568,"-1/2",3],["ADD",134,569,"-1/2",3],["ADD",221,569,"-1/2",3],["ADD",134,575,"-1/2",3],["ADD",221,575,"-1/2",3],["ADD",134,576,"-1/2",3],["ADD",221,576,"-1/2",3],["ADD",134,599,"-1/2",3],["ADD",221,599,"-1/2",3],["ADD",134,655,"-1/8",3],["ADD",221,655,"-1/8",3],["ADD",134,671,"1/2",3],["ADD",166,671,"-1/2",3],["ADD",214,671,"-1/2",3],["ADD",221,671,"1/2",3],["ADD",134,711,"-1/4",3],["ADD",221,711,"-1/4",3],["ADD",134,775,"-1/2",3],["ADD",221,775,"-1/2",3],["ADD",134,917,"1/8",3],["ADD",221,917,"1/8",3],["ADD",166,990,"-1/2",3],["ADD",214,990,"-1/2",3],["ADD",134,999,"-1/4",3],["ADD",221,999,"-1/4",3],["ADD",134,1006,"1/4",3],["ADD",221,1006,"1/4",3],["ADD",134,1008,"-1/2",3],["ADD",221,1008,"-1/2",3],["ADD",134,1013,"-5/2",3],["ADD",221,1013,"-5/2",3],["ADD",134,1016,"-5/4",3],["ADD",221,1016,"-5/4",3],["ADD",134,1030,"-11/2",3],["ADD",221,1030,"-11/2",3],["ADD",134,1032,"-2",3],["ADD",221,1032,"-2",3],["ADD",134,1037,"-9/2",3],["ADD",166,1037,"4",3],["ADD",214,1037,"4",3],["ADD",221,1037,"-9/2",3],["ADD",134,1053,"-3",3],["ADD",166,1053,"2",3],["ADD",214,1053,"2",3],["ADD",221,1053,"-3",3],["ADD",134,1058,"1",3],["ADD",221,1058,"1",3],["ADD",134,1075,"-3/4",3],["ADD",166,1075,"1/2",3],["ADD",214,1075,"1/2",3],["ADD",221,1075,"-3/4",3],["ADD",134,1077,"-1",3],["ADD",166,1077,"2",3],["ADD",214,1077,"2",3],["ADD",221,1077,"-1",3],["ADD",134,1078,"-1/2",3],["ADD",221,1078,"-1/2",3],["ADD",134,1080,"1/2",3],["ADD",221,1080,"1/2",3],["ADD",134,1091,"1",3],["ADD",166,1091,"-1/2",3],["ADD",214,1091,"-1/2",3],["ADD",221,1091,"1",3],["ADD",134,1095,"-6",3],["ADD",166,1095,"4",3],["ADD",214,1095,"4",3],["ADD",221,1095,"-6",3],["ADD",134,1097,"2",3],["ADD",166,1097,"-1",3],["ADD",214,1097,"-1",3],["ADD",221,1097,"2",3],["ADD",134,1110,"-6",3],["ADD",166,1110,"4",3],["ADD",214,1110,"4",3],["ADD",221,1110,"-6",3],["ADD",134,1111,"3",3],["ADD",166,1111,"-2",3],["ADD",214,1111,"-2",3],["ADD",221,1111,"3",3],["ADD",134,1113,"-1",3],["ADD",166,1113,"6",3],["ADD",214,1113,"6",3],["ADD",221,1113,"-1",3],["ADD",134,1116,"-5/4",3],["ADD",166,1116,"-1/2",3],["ADD",214,1116,"-1/2",3],["ADD",221,1116,"-5/4",3],["ADD",134,1127,"1/4",3],["ADD",221,1127,"1/4",3],["ADD",134,1129,"5",3],["ADD",166,1129,"2",3],["ADD",214,1129,"2",3],["ADD",221,1129,"5",3],["ADD",134,1132,"-3/2",3],["ADD",166,1132,"1",3],["ADD",214,1132,"1",3],["ADD",221,1132,"-3/2",3],["ADD",134,1136,"-6",3],["ADD",166,1136,"4",3],["ADD",214,1136,"4",3],["ADD",221,1136,"-6",3],["ADD",166,1143,"-1/2",3],["ADD",214,1143,"-1/2",3],["ADD",134,1171,"-1",3],["ADD",221,1171,"-1",3],["ADD",134,1172,"1/2",3],["ADD",221,1172,"1/2",3],["ADD",134,1207,"1/4",3],["ADD",221,1207,"1/4",3],["ADD",134,1224,"1/4",3],["ADD",221,1224,"1/4",3],["ADD",134,1234,"1",3],["ADD",221,1234,"1",3],["ADD",134,1284,"-5/2",3],["ADD",221,1284,"-5/2",3],["ADD",134,1287,"1/8",3],["ADD",221,1287,"1/8",3],["ADD",134,1466,"1/2",3],["ADD",221,1466,"1/2",3],["ADD",166,1477,"1/2",3],["ADD",214,1477,"1/2",3],["ADD",134,1479,"1/2",3],["ADD",221,1479,"1/2",3]]],[40889,[["MOVE",134,3,618,false]]],[41930,[["ADD",134,315,"1/2",789]]],[42006,[["MOVE",100,716,807,false]]],[43334,[["ADD",166,990,"1/2",907]]],[43373,[["ADD",134,994,"1/2",913],["ADD",134,230,"1/2",913]]]],"append":[]} \ No newline at end of file diff --git a/research/p104-toolkit-v3/dualword.py b/research/p104-toolkit-v3/dualword.py new file mode 100644 index 0000000000..7d6aaec64a --- /dev/null +++ b/research/p104-toolkit-v3/dualword.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Frame-perp transpose ("duality") of a P104 native word (our DUALWORD / DUALSCAN lanes). Standard library only. + +dual(W): reverse time, transpose every local op, replace every frame F by F^perp (standard dot on F2^9), and swap the +X/Y register of each port (live r <-> r + 112, hosted X2 <-> Y2). Op rules: + ADD r += q*s -> ADD s += q*r SCALE r *= q -> SCALE r *= q + MOVE r a->b -> MOVE r b->a (on perp frames; perp reverses inclusion, so an ascent stays an ascent) + INIT <-> CLEAR COPY -> ADD (COPY targets an empty register; ADD is the same map there) +Facts (README Part B): perp preserves every dimension gap, and every charge of the native invoice is a function of +dimension gaps, so the dual is priced bit-identically; dual(dual(W)) = W up to the frame basis lists (same spans) and the +one COPY. The 8 native-B ports need free SCALEs to restore the contract (fix_b_ports). + +API + dual(W) -> W* raw dual (exact, no arithmetic) + fix_b_ports(base, D) -> (D', ports) free SCALEs (x*2 at start, Y*1/2 at end) on ports whose contract row differs + restore_copy(D, base) -> D' put the base word's single COPY back (dual maps COPY to ADD) + swap_frames(W, a, b) -> W' relabel two frame indices (an involution; used to run tokdrop.py on a dual, + where {0} and V trade places: swap_frames(dual(W), Z, V)) + same_word(A, B) -> (bool, why) equality up to frame basis lists (compares spans) +CLI + python3 dualword.py IN OUT write dual(IN) +""" +import copy +import gzip +import json +import sys + + +def load(path): + op = gzip.open if str(path).endswith('.gz') else open + with op(path, 'rt') as f: + return json.load(f) + + +def save(W, path): + with gzip.open(path, 'wt') as f: + json.dump(W, f) + + +def span(b): + s = {0} + for v in b: + s |= {x ^ v for x in s} + return frozenset(s) + + +def perp_basis(b): + """ascending greedy basis of b^perp in F2^9 (standard dot).""" + Pv = [v for v in range(512) if all(bin(v & u).count('1') % 2 == 0 for u in b)] + basis = []; s = {0} + for v in Pv: + if v not in s: + basis.append(v); s |= {x ^ v for x in s} + if len(span(b)) == 1 << len(b): + assert len(basis) == 9 - len(b) + return basis + + +def port_swap(W): + n = W['public']; H = W['hosted'] + pi = list(range(n)) + for r in W['live']: + pi[r], pi[r + 112] = r + 112, r + for x, y in zip(H['X2'], H['Y2']): + pi[x], pi[y] = y, x + return pi + + +def dual(W): + n = W['public']; pi = port_swap(W) + D = copy.deepcopy(W) + cache = {} + + def pp(b): + t = tuple(b) + if t not in cache: + cache[t] = perp_basis(b) + return cache[t] + D['frames'] = [pp(b) for b in W['frames']] + new_init = [None] * n; new_fin = [None] * n + for k in range(n): + new_init[pi[k]] = W['final'][k]; new_fin[pi[k]] = W['initial'][k] + D['initial'] = new_init; D['final'] = new_fin + m = lambda r: pi[r] if r < n else r + prog = [] + for e in reversed(W['program']): + op = e[0] + if op in ('ADD', 'COPY'): + prog.append(['ADD', m(e[2]), m(e[1]), e[3], e[4]]) + elif op == 'SCALE': + prog.append(['SCALE', m(e[1]), e[2], e[3]]) + elif op == 'MOVE': + prog.append(['MOVE', m(e[1]), e[3], e[2], e[4]]) + elif op == 'INIT': + prog.append(['CLEAR', e[1], e[2]]) + elif op == 'CLEAR': + prog.append(['INIT', e[1], e[2]]) + else: + raise ValueError('unknown op %r' % (op,)) + D['program'] = prog + return D + + +def restore_copy(D, base): + cps = [e for e in base['program'] if e[0] == 'COPY'] + out = copy.deepcopy(D) + for c in cps: + k = [i for i, e in enumerate(out['program']) if e == ['ADD'] + list(c[1:])] + if len(k) != 1: + raise ValueError('COPY %r matches %d ADD events' % (c, len(k))) + out['program'][k[0]] = list(c) + return out + + +def fix_b_ports(base, D, rows_base, rows_dual): + """rows_*: exact final public rows (e.g. pricekit.replay(W)['rows']). Adds the free SCALEs on every live port + whose X row differs (the native-B ports, contract [[1,-1],[1/2,1/2]]).""" + bad = [r for r in base['live'] if rows_base[r] != rows_dual[r]] + out = copy.deepcopy(D) + pre = [['SCALE', r, '2', out['initial'][r]] for r in bad] + post = [['SCALE', r + 112, '1/2', out['final'][r + 112]] for r in bad] + out['program'] = pre + out['program'] + post + return out, bad + + +def swap_frames(W, a, b): + sw = lambda f: b if f == a else a if f == b else f + out = copy.deepcopy(W) + fr = out['frames']; fr[a], fr[b] = fr[b], fr[a] + out['initial'] = [None if f is None else sw(f) for f in out['initial']] + out['final'] = [None if f is None else sw(f) for f in out['final']] + for e in out['program']: + op = e[0] + if op == 'MOVE': + e[2] = sw(e[2]); e[3] = sw(e[3]) + elif op in ('ADD', 'COPY'): + e[4] = sw(e[4]) + elif op == 'SCALE': + e[3] = sw(e[3]) + elif op in ('INIT', 'CLEAR'): + e[2] = sw(e[2]) + return out + + +def same_word(A, B): + for k in A: + if k not in ('frames',) and A[k] != B.get(k): + return False, 'field %s differs' % k + if len(A['frames']) != len(B['frames']): + return False, 'frame count' + if any(span(x) != span(y) for x, y in zip(A['frames'], B['frames'])): + return False, 'frame spans differ' + return True, 'equal up to frame bases' + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(__doc__); sys.exit(2) + save(dual(load(sys.argv[1])), sys.argv[2]) diff --git a/research/p104-toolkit-v3/fastreplay.py b/research/p104-toolkit-v3/fastreplay.py new file mode 100644 index 0000000000..8d35d50592 --- /dev/null +++ b/research/p104-toolkit-v3/fastreplay.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Fast randomised-exact replay of a P104 word (our NETCAUX lane; idea from our SHADOWCORE lane: replay mod 2^61-1). Stdlib only. +Rows are LINEAR in the program, so instead of the full N x N row matrix we replay the N x k PROJECTION row . R where R is a fixed random +N x k matrix mod p = 2^61-1 (pure python ints, no numpy). A nonzero row difference projects to zero with probability p^-k = 2^-244 at k=4. +Any FLAG (a projected mismatch) is re-checked by an EXACT Fraction replay of the backward slice that influences the flagged registers. +API + FastReplay(W, R=None, k=4, seed=1) -> .proj (N,2,k), .bad (structural problems, same kinds as pricekit.replay), .ok, .nproblems + proj_matrix(n, N, k, seed) -> R for a word with n publics and N-n owned (owned rows start at 0) + exact_rows(W, regs) -> {reg: Fraction row} by exact replay of the backward slice only + slice_events(W, regs) -> sorted event indices that can influence the final rows of `regs` +""" +import random, collections +from fractions import Fraction as Q +P = 2 ** 61 - 1 + +def proj_matrix(n, k=4, seed=1): + """R for publics 0..n-1: list of n random k-vectors mod 2^61-1.""" + rng = random.Random(seed) + return [[rng.randrange(1, P) for _ in range(k)] for _ in range(n)] + +class FastReplay: + def __init__(self, W, R=None, k=4, seed=1): + n = W['public']; NO = W['owned_fields']; N = n + NO; self.n = n; self.N = N; self.k = k + if R is None: R = proj_matrix(n, k, seed) + assert len(R) >= n + zero = [0] * k + M = [list(R[r]) for r in range(n)] + [list(zero) for _ in range(NO)] + cur = list(W['initial']) + [None] * NO; bad = []; owned = set() + cache = {} + def qm(s): + c = cache.get(s) + if c is None: + q = Q(s) + if q.denominator % P == 0: bad.append(('denominator-divisible-by-modulus', s)) + c = cache[s] = q.numerator % P * pow(q.denominator % P, -1, P) % P + return c + for i, e in enumerate(W['program']): + op, r = e[0], e[1] + if op == 'MOVE': + if cur[r] != e[2]: bad.append(('move-from', i)) + cur[r] = e[3] + elif op == 'INIT': owned.add(r); M[r] = list(zero); cur[r] = e[2] + elif op == 'CLEAR': owned.discard(r); M[r] = list(zero); cur[r] = None + elif op == 'SCALE': + if cur[r] != e[3]: bad.append(('scale-frame', i)) + q = qm(e[2]); M[r] = [x * q % P for x in M[r]] + elif op in ('ADD', 'COPY'): + s = e[2]; f = e[4] + if cur[r] != f or cur[s] != f: bad.append(('add-frame', i)) + if op == 'COPY': + if any(M[r]): bad.append(('copy-nonzero', i)) + M[r] = list(zero) + q = qm(e[3]); M[r] = [(x + q * y) % P for x, y in zip(M[r], M[s])] + else: bad.append(('op', i, op)) + if owned: bad.append(('owned-alive', len(owned))) + if cur[:n] != list(W['final']): bad.append(('exit-frames',)) + self.proj = M; self.bad = bad; self.ok = not bad; self.nproblems = len(bad); self.cur = cur + self.W = W + def is_zero(self, r): return not any(self.proj[r]) + +def slice_events(W, regs): + """Backward slice: indices of events that can influence the final rows of regs.""" + need = set(regs); keep = [] + P = W['program'] + for i in range(len(P) - 1, -1, -1): + e = P[i]; op = e[0] + if op == 'MOVE': continue + r = e[1] + if r not in need: continue + keep.append(i) + if op in ('INIT', 'CLEAR'): need.discard(r) # content before is dropped + elif op == 'COPY': need.discard(r); need.add(e[2]) + elif op == 'ADD': need.add(e[2]) + return sorted(keep) + +def exact_rows(W, regs): + n = W['public']; N = n + W['owned_fields']; regs = list(regs) + ev = slice_events(W, regs); P = W['program'] + rows = {} + def row(r): + if r not in rows: rows[r] = {r: Q(1)} if r < n else {} + return rows[r] + for i in ev: + e = P[i]; op = e[0]; r = e[1] + if op in ('INIT', 'CLEAR'): rows[r] = {}; continue + if op == 'SCALE': + q = Q(e[2]); rows[r] = {k: v * q for k, v in row(r).items()}; continue + s = e[2]; q = Q(e[3]); R = row(r) + if op == 'COPY': R.clear() + for k, v in list(row(s).items()): + nv = R.get(k, 0) + q * v + if nv: R[k] = nv + else: R.pop(k, None) + return {r: row(r) for r in regs} + +def replay(W, R=None, k=4, seed=1): + """returns (rows, cur, bad) where rows[r] is the k-vector PROJECTION of row r + (list of k ints mod 2^61-1), not a Fraction dict. Use proj_matrix(...) once and pass R to compare two words' rows + (equal projections <=> equal rows except with probability 2^-244 per row). bad entries are structural, same shapes as pricekit.replay.""" + F = FastReplay(W, R, k, seed) + return F.proj, F.cur, F.bad + +def unit_projection(W, R, r): + """projection of the identity row e_r (what 'helper ends identity' compares against).""" + return list(R[r]) + +def rows_equal(rowsA, rowsB, mapA=lambda r: r): + """indices r (of A) whose projections differ between A and B (B indexed through mapA).""" + return [r for r in range(len(rowsA)) if rowsA[r] != rowsB[mapA(r)]] diff --git a/research/p104-toolkit-v3/kformula.py b/research/p104-toolkit-v3/kformula.py new file mode 100644 index 0000000000..87f8de0aac --- /dev/null +++ b/research/p104-toolkit-v3/kformula.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""kappa as a function of the stage count k (our KFORMULA lane). Standard library only. + +Closed-form two-arm rho = 2 caller C(k) (arms floor(k/2) | ceil(k/2), 2 hubs, grow 2; the CALLERMAP closed formula), +with g = h - 1 = 8, for 96 type-A and 8 type-B ports: + per port T: 2*T[g j + 1] for j = 1..a and j = 1..b (arm roles), 2*T[g k + 1] (hubs) + P: 2*P[g j] for j = 2..a-1 and j = 2..b-1 (grouped idles) + type A: 4*P[g] + 4*P[1] type B: 2k*P[g] + 4*P[g+1] +Invoice(k) = C(k) + k * N, N = the per-call native invoice of a word (default: #7546's). +Laws [P, derivation in README Part D]: + S(k) = 96k - 208 = 2k(v - c) - rho*v (v = 104, c = 56, rho = 2) + D0(k)/104 = beta k ln k + alpha k + O(ln k), beta = 2(2h - 1) = 34 + kappa(k) ~ (2(1 - c/v) - rho/k) / (2(2h - 1) ln k + alpha) (first order) + stationarity: kappa* = rho/(beta k*) = 1/((2h - 1) k*) +API: caller(k) -> dict(T, P, nJ); price(k, native) -> pricekit price dict; first_order(k, alpha) +CLI: python3 kformula.py [k ...] (control: C(17) == the #7113 caller invoice and prices #7546) +""" +import math +import sys +from collections import Counter +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import pricekit as pk # noqa: E402 + +NA, NB = 96, 8 +V, C_OWNED, RHO, H = 104, 56, 2, 9 + + +def caller(k, g=8, nA=NA, nB=NB): + a, b = k // 2, k - k // 2 + T = Counter(); P = Counter() + for arm in (a, b): + for j in range(1, arm + 1): + T[g * j + 1] += 2 + for j in range(2, arm): + P[g * j] += 2 + T[g * k + 1] += 2 + TA = Counter({r: v * (nA + nB) for r, v in T.items()}) + PA = Counter({r: v * (nA + nB) for r, v in P.items()}) + PA[g] += 4 * nA; PA[1] += 4 * nA + PA[g] += 2 * k * nB; PA[g + 1] += 4 * nB + return dict(T=+TA, P=+PA, nJ=k) + + +def price(k, native, digits=30): + return pk.price(pk.invoice(caller(k), native), digits) + + +def S_closed(k): + return 96 * k - 208 + + +def first_order(k, alpha, h=H, v=V, c=C_OWNED, rho=RHO): + eps = 2 * (1 - c / v) + return (eps - rho / k) / (2 * (2 * h - 1) * math.log(k) + alpha) + + +def main(argv): + W = pk.load(HERE / 'data' / 'base-7546-word.json.gz'); nat = pk.native_from_word(W) + C7 = pk.caller_invoice(pk.load(HERE / 'data' / 'caller-7113.json.gz')) + c17 = caller(17) + print('closed form C(17) == #7113 caller invoice: T', c17['T'] == C7['T'], 'P', c17['P'] == C7['P'], 'nJ', C7['nJ']) + print(pk.cite('C(17) x #7546 native', price(17, nat))) + ks = [int(x) for x in argv] or list(range(6, 41)) + [48, 64, 96, 128] + for k in ks: + p = price(k, nat, 20) + print(k, p['S'], '%.1f' % float(p['D0']), pk.kstr(p['kappa'], 15)) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/research/p104-toolkit-v3/layout_price.py b/research/p104-toolkit-v3/layout_price.py new file mode 100644 index 0000000000..5a8b996ca7 --- /dev/null +++ b/research/p104-toolkit-v3/layout_price.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Mask-layout pricer for rho = 2 callers (CALLEROPEN, an outside-session lane of DreamingOfClouds) [C-param]. +Standard library only. PARAMETRIC: prices hypothetical caller layouts with hypothetical natives; no word is built. + price a rho=2 mask layout from (i) role masks (stage sets), (ii) per-boundary +line caps. Ledger [P]: T = sum_roles T[8 deg+1] (odd phase), idle incidences at stage k = |Z_k| - 2 (Lemma Det), thin = n+2-2L +QUERY P[1]; lines = intervals of consecutive stages, #lines at k = |Z_k|-2, #crossing k|k+1 <= cap_k. Line lengths: at each +boundary continue the OLDEST lines up to min(m_k, m_k+1, cap_k) (most unequal arrangement for convex f) -> P[8 len]. +Natives (three what-if readings of the #7546 native, 104 ports, 17 calls): + 'twins deleted' the hosted X2/Y2 main moves removed from the paid histogram (a ceiling: assumes they vanish for free) + 'shadows' the word with the hosted block dropped from the main set (its X2/Y2 registers priced as helpers) + 'shadows+2 relays' 'shadows' plus 2 relays per type-A port (T[9] += 2, P[8] += 2, P[1] += 2 per port) +Ratios printed are kappa(layout, native) / kappa(#7546).""" +import sys, copy, math +from pathlib import Path +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)); import pricekit as pk +from collections import Counter +W = pk.load(HERE / 'data' / 'base-7546-word.json.gz'); C7 = pk.load(HERE / 'data' / 'caller-7113.json.gz') +n0 = pk.native_from_word(W); base = pk.price(pk.invoice(pk.caller_invoice(C7), n0), 30); K = base['kappa']['lo'] +W2 = copy.deepcopy(W); W2['hosted'] = {}; nt = pk.native_from_word(W2) +nm = dict(nt); P_ = Counter(n0['P']) +for c in ('X2', 'Y2'): P_.subtract(n0['main_moves'][c]) +nm['P'] = +P_; nm['T'] = Counter(n0['T']); nm['source'] = 'param' +nr = dict(nt); nr['T'] = Counter(nt['T']); nr['P'] = Counter(nt['P']); nr['source'] = 'param' +nr['T'][9] += 2 * 96; nr['P'][1] += 96; nr['P'][8] += 2 * 96; nr['P'][1] += 96 +NAT = (('twins deleted', nm), ('shadows', nt), ('shadows+2 relays', nr)) +f = lambda r: r * math.log(r) if r > 1 else 0.0 +def lines(m, cap): + """m[k] lines needed at stage k (k=0..L-1); cap[k] max crossing k|k+1. Returns Counter of line lengths.""" + alive = [] # start stages, oldest first + out = Counter() + for k in range(len(m)): + if k > 0: + keep = min(m[k - 1], m[k], cap[k - 1], len(alive)) + for s in alive[keep:]: out[k - s] += 1 + alive = alive[:keep] + alive += [k] * (m[k] - len(alive)) + for s in alive: out[len(m) - s] += 1 + return out +def invoice(masks, L, cap, cov=2): + """masks: list of role stage-sets; cap: list of L-1 caps.""" + Z = [sum(1 for ms in masks if k in ms) for k in range(L)] + m = [z - cov for z in Z]; assert min(m) >= 0, Z + T = Counter(8 * len(ms) + 1 for ms in masks) + P = Counter({8 * l: v for l, v in lines(m, cap).items()}) + n = len(masks); thin = n + 2 - 2 * L; assert thin >= 0, (n, L) + # thin QUERYs absorbed as IDLE_ODD into the shortest lines (as the built B does: 4 x r9) + for _ in range(thin): + r = min((r for r in P if r % 8 == 0 and P[r] > 0), default=None) + if r is None: P[1] += 1; continue + P[r] -= 1; P[r + 1] += 1 + return T, +P, Z +def price(T, P, lab, nJ=17, quiet=False): + Ta = Counter({r: v * 104 for r, v in T.items()}); Pa = Counter({r: v * 104 for r, v in P.items()}) + res = [] + for nn, nat in NAT: + pr = pk.price(pk.invoice(dict(T=Ta, P=Pa, nJ=nJ), nat), 30); k = pr['kappa'] + res.append(float(k['lo'] / K) if isinstance(k, dict) and k.get('lo') else 0.0) + D = sum(v * f(r) for r, v in T.items()) - sum(v * f(r) for r, v in P.items()) + if not quiet: + print('%-58s S/port %4d D0/port %7.1f header %3d | td %.4f sh %.4f rel %.4f' % (lab, pk.S_of(T, P), D, sum((r - 1) // 8 * v for r, v in T.items()), *res)) + return res +def two_arm(L=17, nl=8, hubs=2, grow=2, extra=0, deg1=True): + """B-type: hubs on all stages; left arm stages 0..nl-1 (centre nl-1), right nl..L-1 (centre nl). At left stage k (0-based) + 'grow' own roles enter (k>=1 or always), mask k..nl-1; 'extra' own roles enter at stage 0 additionally.""" + masks = [set(range(L)) for _ in range(hubs)] + for k in range(nl): + cnt = grow + (extra if k == 0 else 0) + masks += [set(range(k, nl)) for _ in range(cnt)] + for k in range(L - 1, nl - 1, -1): + cnt = grow + (extra if k == L - 1 else 0) + masks += [set(range(nl, k + 1)) for _ in range(cnt)] + return masks +def caps_B(Z, L, nl, bonus=0): + """interior caps |S|-4 ; centre boundary: S = hubs only -> crossing <= hubs-? use |S|-4 floored at 0; outer boundaries |S|-4+bonus.""" + cap = [] + for k in range(L - 1): + S = Z[k] if k < nl - 1 else (Z[k + 1] if k >= nl else None) + if S is None: cap.append(0); continue # centre (thin block between): no line crosses (S = hubs, covered) + c = S - 4 + if k == 0 or k == L - 2: c += bonus + cap.append(max(c, 0)) + return cap +if __name__ == '__main__': + print(pk.cite('control #7546 on #7113', base)) + Zof = lambda ms: [sum(1 for m in ms if k in m) for k in range(17)] + ms = two_arm(); T, P, Z = invoice(ms, 17, caps_B(Zof(ms), 17, 8)) + print('model B: Z', Z, 'P', sorted(P.items())) + price(T, P, 'model B (built B: 1.0950/1.0031/0.9773)') + T, P, Z = invoice(ms, 17, caps_B(Zof(ms), 17, 8, bonus=2)); price(T, P, 'B + outer bonus (2 lines cross each outer boundary)') + T, P, Z = invoice(ms, 17, caps_B(Zof(ms), 17, 8, bonus=1)); price(T, P, 'B + outer bonus 1') + ms2 = two_arm(extra=-2); Z2 = Zof(ms2); print('tight Z', Z2, 'n', len(ms2)) + T, P, Z = invoice(ms2, 17, caps_B(Z2, 17, 8)); price(T, P, 'tight two-hub |Z_k|=2k (no own role at outer stages)') + for nl in (7, 9): + ms3 = two_arm(nl=nl); T, P, Z = invoice(ms3, 17, caps_B(Zof(ms3), 17, nl)); price(T, P, 'B with centre at %d|%d' % (nl, nl + 1)) + # uncapped ceiling (all lines run full) for reference + T, P, Z = invoice(ms, 17, [99] * 16); price(T, P, 'B masks, NO caps (unreachable reference)') diff --git a/research/p104-toolkit-v3/lossless.py b/research/p104-toolkit-v3/lossless.py new file mode 100644 index 0000000000..8e9a6e6735 --- /dev/null +++ b/research/p104-toolkit-v3/lossless.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Lossless-CLEAR checker, law DUAL (our NETC4 lane; only this checker is shipped, not the NETC compiler). Stdlib only. + Live rows always span Q^n (n publics) until a lossy CLEAR. The relation space +Lam(t) = {lam : sum_r lam_r row_r = 0} over live registers has dim = #live owned. Ops act on it contragrediently: + ADD dst += q src : lam_src -= q lam_dst SCALE r *= q : lam_r /= q + INIT o : Lam += e_o CLEAR o : lossless iff some lam has lam_o != 0; + then Lam := {lam in Lam : lam_o = 0} +So one forward pass mod p = 2^61-1 finds every lossy CLEAR. A lossless CLEAR is never missed; a reported loss could, +with negligible probability, be a mod-p false zero of every lam_o, so re-check reported losses exactly (as +tests/test_checkers.py does with a Fraction rank). dual_scan(W) -> list of (t, o, blame) where blame lists the +ops that wrote coordinate o of some relation (the reads of o by relation-carrying registers).""" +import collections +from fractions import Fraction as Q +P = (1 << 61) - 1 +def m(q): + q = Q(q); return q.numerator % P * pow(q.denominator % P, P - 2, P) % P + +def dual_scan(W, track=None, stop_at=None): + n = W['public']; prog = W['program'] + lam = {}; occ = collections.defaultdict(set); nid = [0]; home = {} + loss = []; hist = collections.defaultdict(list) + def setc(i, r, v): + L = lam[i] + if v: L[r] = v; occ[r].add(i) + else: L.pop(r, None); occ[r].discard(i) + for t, e in enumerate(prog): + if stop_at is not None and t >= stop_at: break + op = e[0] + if op in ('ADD', 'COPY'): + d, s = e[1], e[2] + if occ.get(d): + q = m(e[3]) + for i in list(occ[d]): + nv = (lam[i].get(s, 0) - q * lam[i][d]) % P + setc(i, s, nv) + if s >= n: hist[s].append(t) + elif op == 'SCALE': + r = e[1] + if occ.get(r): + iq = pow(m(e[2]), P - 2, P) + for i in list(occ[r]): setc(i, r, lam[i][r] * iq % P) + elif op == 'INIT': + o = e[1]; i = nid[0]; nid[0] += 1; lam[i] = {}; setc(i, o, 1); home[o] = i; hist[o] = [] + elif op == 'CLEAR': + o = e[1]; c = list(occ.get(o, ())) + if not c: + loss.append((t, o, list(hist[o]))) + else: + piv = min(c, key=lambda i: len(lam[i])); pv = lam[piv]; ip = pow(pv[o], P - 2, P) + for i in c: + if i == piv: continue + f = lam[i][o] * ip % P + for r, v in pv.items(): setc(i, r, (lam[i].get(r, 0) - f * v) % P) + for r in list(pv): occ[r].discard(piv) + del lam[piv] + return loss, lam, occ + +def prank(vecs, cols): + """rank mod P of the projections of vecs onto cols.""" + piv = {}; rk = 0 + for v in vecs: + x = {c: v[c] for c in cols if v.get(c)} + while x: + p = min(x) + if p not in piv: break + f = x[p]; pv = piv[p] + for k, y in pv.items(): + nv = (x.get(k, 0) - f * y) % P + if nv: x[k] = nv + else: x.pop(k, None) + if x: + p = min(x); ip = pow(x[p], P - 2, P); piv[p] = {k: y * ip % P for k, y in x.items()}; rk += 1 + return rk + +def culprit_ops(W, tc, C, t_from): + """re-scan; between t_from and tc report every op after which rank(pi_C(Lam)) changes (C = owned set cleared at tc).""" + n = W['public']; prog = W['program'] + lam = {}; occ = collections.defaultdict(set); nid = [0]; out = []; last = None + def setc(i, r, v): + L = lam[i] + if v: L[r] = v; occ[r].add(i) + else: L.pop(r, None); occ[r].discard(i) + Cs = set(C) + for t, e in enumerate(prog[:tc]): + op = e[0]; touched = False + if op in ('ADD', 'COPY'): + d, s = e[1], e[2] + if occ.get(d): + q = m(e[3]) + for i in list(occ[d]): setc(i, s, (lam[i].get(s, 0) - q * lam[i][d]) % P) + touched = s in Cs + elif op == 'SCALE': + r = e[1] + if occ.get(r): + iq = pow(m(e[2]), P - 2, P) + for i in list(occ[r]): setc(i, r, lam[i][r] * iq % P) + elif op == 'INIT': + o = e[1]; i = nid[0]; nid[0] += 1; lam[i] = {}; setc(i, o, 1); touched = o in Cs + elif op == 'CLEAR': + o = e[1]; c = list(occ.get(o, ())) + if c: + piv = min(c, key=lambda i: len(lam[i])); pv = lam[piv]; ip = pow(pv[o], P - 2, P) + for i in c: + if i == piv: continue + f = lam[i][o] * ip % P + for r, v in pv.items(): setc(i, r, (lam[i].get(r, 0) - f * v) % P) + for r in list(pv): occ[r].discard(piv) + del lam[piv] + touched = True + if t >= t_from and touched: + rk = prank(list(lam.values()), sorted(Cs)) + if rk != last: out.append((t, e, rk)); last = rk + return out + +def culprit_chain(W, tc, C, t_from): + """like culprit_ops, with per-(relation, coordinate) provenance: for each rank-dropping read d <- o, follow the + ops that made the relation coordinates nonzero back from d: returns list of (t_drop, op, chain=[(t, op), ...]).""" + n = W['public']; prog = W['program'] + lam = {}; occ = collections.defaultdict(set); nid = [0]; who = {}; out = []; last = None + def setc(i, r, v, t): + L = lam[i] + if v: L[r] = v; occ[r].add(i); who[(i, r)] = t + else: L.pop(r, None); occ[r].discard(i) + Cs = set(C) + for t, e in enumerate(prog[:tc]): + op = e[0]; touched = False; snap = None + if op in ('ADD', 'COPY'): + d, s = e[1], e[2] + if occ.get(d): + if t >= t_from and s in Cs: snap = [(i, dict(lam[i])) for i in occ[d]] + q = m(e[3]) + for i in list(occ[d]): setc(i, s, (lam[i].get(s, 0) - q * lam[i][d]) % P, t) + touched = s in Cs + elif op == 'SCALE': + r = e[1] + if occ.get(r): + iq = pow(m(e[2]), P - 2, P) + for i in list(occ[r]): setc(i, r, lam[i][r] * iq % P, who.get((i, r), t)) + elif op == 'INIT': + o = e[1]; i = nid[0]; nid[0] += 1; lam[i] = {}; setc(i, o, 1, t); touched = o in Cs + elif op == 'CLEAR': + o = e[1]; c = list(occ.get(o, ())) + if c: + piv = min(c, key=lambda i: len(lam[i])); pv = lam[piv]; ip = pow(pv[o], P - 2, P) + for i in c: + if i == piv: continue + f = lam[i][o] * ip % P + for r, v in pv.items(): setc(i, r, (lam[i].get(r, 0) - f * v) % P, max(who.get((i, r), -1), who.get((piv, r), -1))) + for r in list(pv): occ[r].discard(piv) + del lam[piv] + touched = True + if t >= t_from and touched: + rk = prank(list(lam.values()), sorted(Cs)) + if last is not None and rk < last and op in ('ADD', 'COPY'): + # chain back from d through the relations that held d + chain = []; seen = set() + for i, L in (snap or []): + x = e[1]; tt = t + while (i, x) in who and (i, x) not in seen: + seen.add((i, x)); ts = who[(i, x)] + if ts >= tt or ts < 0: break + ee = prog[ts]; chain.append((ts, ee)) + if ee[0] not in ('ADD', 'COPY') or ee[2] != x: break + x = ee[1]; tt = ts + out.append((t, e, chain)) + last = rk + return out + + +def main(argv): + import gzip, json + if len(argv) != 1: + print('usage: python3 lossless.py WORD.json[.gz] -> every lossy CLEAR (t, owned register, reads of it)'); return 2 + op = gzip.open if argv[0].endswith('.gz') else open + with op(argv[0], 'rt') as f: + W = json.load(f) + loss, lam, occ = dual_scan(W) + print('lossy CLEARs: %d' % len(loss)) + for t, o, reads in loss[:50]: + print(' t=%d owned=%d reads-of-o=%s' % (t, o, reads[-5:])) + print('relations live at end: %d' % len(lam)) + return 0 if not loss else 1 + + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv[1:])) diff --git a/research/p104-toolkit-v3/modp.py b/research/p104-toolkit-v3/modp.py new file mode 100644 index 0000000000..5d02497d6e --- /dev/null +++ b/research/p104-toolkit-v3/modp.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Dense linear algebra mod P = 2^31 - 1 on numpy int64 arrays (backend of tokdrop.py). + +If python-flint is importable it is used (fast); otherwise a pure-numpy fallback runs. +Both backends return the same canonical objects (rank, reduced row echelon form, a +null-space basis spanning the same space), so engine verdicts do not depend on the backend. + + rank(A) rank of A mod P + rref(A) (R, rank): canonical reduced row echelon form mod P + nullspace(A) n x k int64 basis of {x : A x = 0} (columns) + matmul(A, B) A @ B mod P without int64 overflow (float64 limbs, exact) +""" +import numpy as np + +P = 2 ** 31 - 1 +try: # optional accelerator + from flint import nmod_mat as _nm +except Exception: # pragma: no cover - depends on environment + _nm = None +BACKEND = 'flint' if _nm is not None else 'numpy' + + +def _to_flint(A): + A = np.asarray(A, dtype=np.int64) % P + return _nm(A.shape[0], A.shape[1], [int(x) for x in A.ravel()], P) + + +def _from_flint(M): + return np.array([[int(M[i, j]) for j in range(M.ncols())] for i in range(M.nrows())], dtype=np.int64).reshape( + M.nrows(), M.ncols()) + + +def _rref_np(A): + A = np.array(A, dtype=np.int64) % P + m, n = A.shape + r = 0 + pivots = [] + for c in range(n): + if r == m: + break + nz = np.nonzero(A[r:, c])[0] + if len(nz) == 0: + continue + p = r + int(nz[0]) + if p != r: + A[[r, p]] = A[[p, r]] + inv = pow(int(A[r, c]), P - 2, P) + A[r] = A[r] * inv % P + col = A[:, c].copy() + col[r] = 0 + rows = np.nonzero(col)[0] + if len(rows): + A[rows] = (A[rows] - (col[rows, None] * A[r][None, :]) % P) % P + pivots.append(c) + r += 1 + return A, r, pivots + + +def rank(A): + A = np.asarray(A) + if A.size == 0: + return 0 + if _nm is not None: + return _to_flint(A).rank() + return _rref_np(A)[1] + + +def rref(A): + A = np.asarray(A) + if _nm is not None: + R, rk = _to_flint(A).rref() + return _from_flint(R), rk + R, rk, _ = _rref_np(A) + return R, rk + + +def nullspace(A): + A = np.asarray(A, dtype=np.int64) + n = A.shape[1] + if _nm is not None: + M = _to_flint(A) + N, k = M.nullspace() + return _from_flint(N)[:, :k].reshape(n, k) + R, rk, piv = _rref_np(A) + free = [c for c in range(n) if c not in set(piv)] + out = np.zeros((n, len(free)), dtype=np.int64) + for j, f in enumerate(free): + out[f, j] = 1 + for i, c in enumerate(piv): + out[c, j] = (-R[i, f]) % P + return out + + +def matmul(A, B): + """Exact A @ B mod P. Splits entries into 16/15-bit limbs so every float64 dot product is exact.""" + A = np.asarray(A, dtype=np.int64) % P + B = np.asarray(B, dtype=np.int64) % P + if A.shape[1] == 0: + return np.zeros((A.shape[0], B.shape[1]), dtype=np.int64) + assert A.shape[1] < (1 << 21), 'inner dimension too large for exact float64 limbs' + a0 = (A & 0xFFFF).astype(np.float64) + a1 = (A >> 16).astype(np.float64) + b0 = (B & 0xFFFF).astype(np.float64) + b1 = (B >> 16).astype(np.float64) + m = lambda x, y: np.rint(x @ y).astype(np.int64) % P + c00 = m(a0, b0) + c01 = (m(a0, b1) + m(a1, b0)) % P + c11 = m(a1, b1) + s16 = pow(2, 16, P) + s32 = pow(2, 32, P) + return (c00 + c01 * s16 % P + c11 * s32 % P) % P diff --git a/research/p104-toolkit-v3/nativeplan.py b/research/p104-toolkit-v3/nativeplan.py new file mode 100644 index 0000000000..66fc8cb410 --- /dev/null +++ b/research/p104-toolkit-v3/nativeplan.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Native helper-plan extractor (our NETCAUX lane; loader and requirement extraction from our REBUILDKIT lane, inlined). +Standard library only. Reads a word and returns its helper layer as a plan: helper routes and stays, every +helper-involving event typed pickup / delivery / relay / scale, meetings (helper<->helper exchanges grouped per stay +pair), requirement blocks (maximal runs of helper->r deliveries on a non-helper r) linked to their deliveries, exact +pi-states (non-helper coordinates) at every stay entry and exit, and the requirement-space state basis (mod 2^61-1). +recost(plan) re-costs the helper layer from the routes alone (sum over helpers of f(end - birth) - sum f(steps), +f(x) = x ln x); reemit_check(plan, path) regenerates every helper-touching event from the plan and compares. +On #7546: recost 9,975.447669, 1,101 helpers, 3,427 interior stops, 70,714 helper events re-emitted identically, +state basis rank 253. +usage: python3 nativeplan.py WORD.json.gz (prints the summary; the API returns the plan dict) +""" +import sys, os, json, gzip, collections, time, math, bisect +from fractions import Fraction as Q +P = 2 ** 61 - 1 + + +def tomod(d): + return {k: (v.numerator % P) * pow(v.denominator % P, P - 2, P) % P for k, v in d.items()} + + +def fcost(r): + return r * math.log(r) if r > 0 else 0.0 + + +def span(b): + s = {0} + for v in b: + s |= {x ^ v for x in s} + return frozenset(s) + +class Word: + def __init__(self, path): + W=json.load(gzip.open(path,'rt')) if str(path).endswith('.gz') else json.load(open(path)) + self.W=W; n=W['public']; self.n=n; self.N=n+W['owned_fields'] + self.dim=[len(b) for b in W['frames']]; self.sp=[span(b) for b in W['frames']] + live=W['live']; H=W['hosted'] + cls={} + for r in live: cls[r]='X'; cls[r+112]='Y' + for r in H['X2']: cls[r]='X2' + for r in H['Y2']: cls[r]='Y2' + for r in range(n): cls.setdefault(r,'h') + for r in range(n,self.N): cls[r]='o' + self.cls=cls + # routes: list of (frame, t_enter) for public + cur=list(W['initial'])+[None]*W['owned_fields'] + route={r:[(W['initial'][r],-1)] for r in range(n)} + ops=[] # (t, op, r, s, q, frame) + olife=collections.defaultdict(list) + for t,e in enumerate(W['program']): + op=e[0] + if op=='MOVE': + r=e[1]; cur[r]=e[3] + if rc mod P, residual vector dict). u = sum coeffs*basis + residual.""" + x, cx, p = self._red(dict(u), {}) + # u - sum(c_k b_k): _red subtracts multiples; cx accumulates -(...) of comb; sign: x = u + sum cx*basis(k) + co = {k: (-v) % P for k, v in cx.items()} + return co, x + +def extract_plan(path, log=print): + t0 = time.time() + w = Word(path); W = w.W; n = w.n; cls = w.cls; dim = w.dim; N = w.N + H = [r for r in range(n) if cls[r] == 'h']; Hs = set(H) + Vd = max(dim); + blocks, traj = extract(w) + log('reqx blocks', len(blocks), '%.0fs' % (time.time() - t0)) + prog = W['program'] + # exact replay tracking helper stays + rows = [{r: Q(1)} if r < n else {} for r in range(N)] + cur = list(W['initial']) + [None] * W['owned_fields'] + stay = {h: 0 for h in H} # current stay index of helper + ish = [r in Hs for r in range(n)] + [False] * (N - n) + def pi(R): return {k: v for k, v in R.items() if not ish[k]} + stays = {h: [dict(frame=W['initial'][h], t_in=-1, t_out=None, dim=dim[W['initial'][h]], pi_in={}, pi_out=None)] for h in H} + for h in H: stays[h][0]['pi_in'] = {} # born holding only own token (pi = 0) + events = [] # helper-involving ADD/COPY/SCALE: dict + for t, e in enumerate(prog): + op = e[0]; r = e[1] + if op == 'MOVE': + if r in Hs: + s = stays[r][-1]; s['t_out'] = t; s['pi_out'] = pi(rows[r]) + stays[r].append(dict(frame=e[3], t_in=t, t_out=None, dim=dim[e[3]], pi_in=pi(rows[r]), pi_out=None)) + cur[r] = e[3]; continue + if op == 'INIT': rows[r] = {}; cur[r] = e[2]; continue + if op == 'CLEAR': rows[r] = {}; cur[r] = None; continue + if op == 'SCALE': + q = Q(e[2]); rows[r] = {k: v * q for k, v in rows[r].items()} + if r in Hs: events.append(dict(t=t, op='SCALE', dst=r, src=None, q=e[2], frame=e[3], kind='scale', dst_stay=len(stays[r]) - 1, src_stay=None)) + continue + s = e[2]; q = Q(e[3]) + dh, sh = r in Hs, s in Hs + if dh or sh: + kind = 'relay' if (dh and sh) else 'pickup' if dh else 'delivery' + events.append(dict(t=t, op=op, dst=r, src=s, q=e[3], frame=e[4], kind=kind, + dst_stay=len(stays[r]) - 1 if dh else None, src_stay=len(stays[s]) - 1 if sh else None)) + R = rows[r] + for k, v in rows[s].items(): + nv = R.get(k, 0) + q * v + if nv: R[k] = nv + else: R.pop(k, None) + for h in H: + s = stays[h][-1]; s['t_out'] = None; s['pi_out'] = pi(rows[h]) + log('replay done %.0fs' % (time.time() - t0)) + # helper routes + helpers = {} + for h in H: + rt = w.route[h]; d = [dim[f] for f, _ in rt] + helpers[h] = dict(route=[(f, t) for f, t in rt], dims=d, birth_frame=rt[0][0], end_frame=rt[-1][0], + birth_dim=d[0], end_dim=d[-1], loss=w.loss(h), nstops=len(rt) - 2) + # stay-level role + event index + ev_by_stay = collections.defaultdict(list) + for i, ev in enumerate(events): + if ev['dst'] in Hs: ev_by_stay[(ev['dst'], ev['dst_stay'])].append(i) + if ev['src'] in Hs and ev['kind'] != 'scale': ev_by_stay[(ev['src'], ev['src_stay'])].append(i) + # meetings: helper<->helper exchanges grouped by unordered stay pair + meet = collections.defaultdict(list) + for i, ev in enumerate(events): + if ev['kind'] == 'relay': + a = (ev['dst'], ev['dst_stay']); b = (ev['src'], ev['src_stay']); meet[tuple(sorted((a, b)))].append(i) + meetings = [dict(a=k[0], b=k[1], interior=any(0 < x[1] < len(stays[x[0]]) - 1 for x in k), frame=events[v[0]]['frame'], t0=events[v[0]]['t'], t1=events[v[-1]]['t'], events=v) for k, v in sorted(meet.items(), key=lambda kv: events[kv[1][0]]['t'])] + # blocks <-> deliveries: for each reqx block (r, t0..t1) the helper->r delivery event indices + dl = collections.defaultdict(list) + for i, ev in enumerate(events): + if ev['kind'] == 'delivery': dl[ev['dst']].append(i) + bl = [] + for b in blocks: + lst = dl[b['r']]; ts = [events[i]['t'] for i in lst] + lo = bisect.bisect_left(ts, b['t0']); hi = bisect.bisect_right(ts, b['t1']) + ids = [i for i in lst[lo:hi]] + bl.append(dict(r=b['r'], frame=b['frame'], dim=dim[b['frame']], t0=b['t0'], t1=b['t1'], nops=b['nops'], v=b['v'], + deliveries=ids, helpers=sorted({(events[i]['src'], events[i]['src_stay']) for i in ids}))) + # state basis: independent interior requirement vectors (frames dim not in {0, V}) + E = Echelon() + for i, b in enumerate(bl): + if b['v'] and b['dim'] not in (0, Vd): + E.add(tomod(b['v']), tag=i) + log('state basis rank %d' % E.n) + nbasis_hits = 0 + for h in H: + for si, s in enumerate(stays[h]): + for key in ('pi_in', 'pi_out'): + v = s[key] + if v is None: s[key + '_c'] = None; continue + co, res = E.coords(tomod(v)) if v else ({}, {}) + s[key + '_c'] = dict(coef=co, in_span=not res, residual_support=len(res)) + inspan = sum(1 for h in H for s in stays[h] for key in ('pi_out_c',) if s[key] and s[key]['in_span']) + log('stay-exit states in requirement span: %d / %d' % (inspan, sum(len(stays[h]) for h in H))) + plan = dict(fmt='netcaux-plan/1', word=os.path.abspath(str(path)), n=n, N=N, V_dim=Vd, + frames_dim=dim, helpers=helpers, stays=stays, events=events, meetings=meetings, blocks=bl, + basis=dict(rank=E.n, block_ids=E.src), cls={r: cls[r] for r in range(n)}, owned=N - n) + return plan + +def recost(plan): + """Re-cost from the plan's helper routes only (cost function f(x) = x ln x).""" + tot = 0.0 + for h, H in plan['helpers'].items(): + d = H['dims']; tot += fcost(d[-1] - d[0]) - sum(fcost(d[i + 1] - d[i]) for i in range(len(d) - 1)) + return tot + +def recost_from_stays(plan): + tot = 0.0 + for h, ss in plan['stays'].items(): + d = [s['dim'] for s in ss]; tot += fcost(d[-1] - d[0]) - sum(fcost(d[i + 1] - d[i]) for i in range(len(d) - 1)) + return tot + +def reemit_helper_events(plan): + """Regenerate every helper-touching program event (MOVE of helpers, ADD/COPY/SCALE with a helper end) from the plan.""" + out = {} + for h, H in plan['helpers'].items(): + rt = H['route'] + for i in range(1, len(rt)): + out[rt[i][1]] = ['MOVE', h, rt[i - 1][0], rt[i][0], False] + for ev in plan['events']: + out[ev['t']] = ['SCALE', ev['dst'], ev['q'], ev['frame']] if ev['kind'] == 'scale' else [ev['op'], ev['dst'], ev['src'], ev['q'], ev['frame']] + return out + +def reemit_check(plan, path=None): + w = Word(path or plan['word']); Hs = set(plan['helpers']); prog = w.W['program']; n = w.n + want = {} + for t, e in enumerate(prog): + if e[0] == 'MOVE' and e[1] in Hs: want[t] = list(e) + elif e[0] in ('ADD', 'COPY') and (e[1] in Hs or e[2] in Hs): want[t] = list(e) + elif e[0] == 'SCALE' and e[1] in Hs: want[t] = list(e) + got = reemit_helper_events(plan) + return got == want, len(want) + + + +def summary(plan): + ev = collections.Counter(e['kind'] for e in plan['events']) + return dict(recost=recost(plan), helpers=len(plan['helpers']), stops=sum(h['nstops'] for h in plan['helpers'].values()), + events=dict(ev), meetings=len(plan['meetings']), blocks=len(plan['blocks']), basis_rank=plan['basis']['rank']) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print(__doc__); sys.exit(2) + plan = extract_plan(sys.argv[1], log=lambda *a: None) + print(json.dumps(summary(plan))) + ok, m = reemit_check(plan); print('re-emitted helper events identical:', ok, m) diff --git a/research/p104-toolkit-v3/pricekit.py b/research/p104-toolkit-v3/pricekit.py new file mode 100644 index 0000000000..10cd0ddf9c --- /dev/null +++ b/research/p104-toolkit-v3/pricekit.py @@ -0,0 +1,490 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""PRICEKIT (our lane, vendored unchanged apart from this header): the one canonical exact pricer for P104 caller+native invoices. Standard library only. + +Accounting contract (README.md has the one-page version). Units: + rank r job a paid job of rank r (P[r] += 1) or a productive target of rank r (T[r] += 1). + S rank-mass surplus sum_r r (T_r - P_r) (= F(0)). + F(t) the bill sum_r (T_r - P_r) r^(1-t). + kappa the unique root of F on (0, hi]; certified by an interval enclosure (Decimal, error-bounded signs). + D0 sum (T-P) r ln r (= -F'(0); the HOSTCRIB/MAINRETIME/CALLERMAP "D", e.g. #6934 422589.15263423) + Dk sum (T-P) r^(1-kappa) ln r (= -F'(kappa); the DSPLIT/ARCHEQ dk.py/TRADELAW "D(kappa)") + +Charging rules (caller events, '#7113 format' = schema mixed-root-caller/1): + port count every caller event of type A/B is charged once per port of that type (len(types[t]['ports'])). + QUERY P[1] per port (thin unit / sign query). + IDLE_ONE s P[8|s|] per port (one idle over the stage set s). + IDLE_ODD s P[8|s|+1] per port. + LAMBDA ordinary work, 1000 units per port-use (#6913). lambda_mode='paid_rank1' charges P[1] per port + instead (counterfactual only). + ADD/SWAP/SCALE not invoiced (ordinary only). + diagonal row T[(q mod 2) + 8*sum(stage bits)] per port: a field of degree d is a target of rank 8d+1 (odd phase) + or 8d (even phase). Hubs are fields with d = #stages (r137 at 17 stages). + J (one stage) ONE native word per J event: stage = call = J event = one copy of the native word. The native word + serves all 104 ports internally, so it is NOT multiplied by ports. +Native word (schema p104-accumulated-block-composition/2): + paid P[|dim b - dim a|] per MOVE. + helpers T[dim final - dim initial] per public register that is not a main. + mains live X (r), live Y (r+112), hosted X2, Y2. A port with a hosted plane (r in hosted.Dports) has 4 + mains, otherwise 2. Mains are never in T: their climbs are paid (MOVEs) and their credit is the + caller diagonal. Ledger law: native mains per port per stage == caller slots covered per stage + (= mask rank - idle incidences); check_slots() tests it. +Parametric native (native_param): start from a word or a histogram native and + drop_mains_A/B k mains per A/B port per call removed, priced by main_unit: + 'rank1x8' (8 rank-1 jobs, the ARMS/MONOCALLER/SLOT3 unit), 'rank8' (one rank-8 job), or exact via + drop_registers() on a real word (preferred; this is not parametric). + c_A/c_B extra owned rank-1 paid jobs per A/B port per call (may be a Fraction). + dT/dP helper/paid histogram deltas per call. +Every switch is an explicit argument; nothing reads the environment. +""" +import gzip, json, math +from collections import Counter +from decimal import Decimal, localcontext +from fractions import Fraction as Q + +VERSION = 'pricekit/1 (2026-11-11)' +NPORTS = 104 + +# ------------------------------------------------------------------------------------------- io +def load(path): + op = gzip.open if str(path).endswith('.gz') else open + with op(path, 'rt') as f: + return json.load(f) + +def hist(d): + """dict with str/int keys -> Counter{int: int|Fraction}; zero entries dropped.""" + return Counter({int(k): (Q(v) if isinstance(v, str) else v) for k, v in d.items() if v}) + +def _add(dst, src, mult=1): + for r, v in src.items(): + dst[r] += mult * v + return dst + +# ------------------------------------------------------------------------------------------- native +def native_from_word(W, verify=False): + """Exact native invoice of one word (one call). verify=True also replays the word exactly (slow).""" + n = len(W['initial']); live = set(W['live']); H = W.get('hosted') or {} + X2, Y2 = list(H.get('X2', [])), list(H.get('Y2', [])) + main = live | {r + 112 for r in live} | set(X2) | set(Y2) + dim = [len(b) for b in W['frames']] + paid = Counter(); main_paid = Counter(); cls = {} + for r in live: cls[r] = 'X'; cls[r + 112] = 'Y' + for r in X2: cls[r] = 'X2' + for r in Y2: cls[r] = 'Y2' + by_class = {k: Counter() for k in ('X', 'Y', 'X2', 'Y2')} + for e in W['program']: + if e[0] == 'MOVE': + rk = abs(dim[e[3]] - dim[e[2]]); paid[rk] += 1 + if e[1] in cls: by_class[cls[e[1]]][rk] += 1 + helpers = Counter(dim[W['final'][r]] - dim[W['initial'][r]] for r in range(n) if r not in main) + dports = set(H.get('Dports', [])) + mains_per_port = {p: 2 + 2 * (p in dports and bool(X2)) for p in range(NPORTS)} + out = dict(T=+helpers, P=+paid, public=n, events=len(W['program']), mains=len(main), main_moves=by_class, + mains_per_port=mains_per_port, Dports=sorted(dports), source='word') + if verify: + out['replay'] = replay(W)['ok'] + return out + +def native_from_hist(paid, helpers, Dports=None, mains_A=4, mains_B=2): + dp = set(Dports) if Dports is not None else None + mpp = None if dp is None else {p: (mains_A if p in dp else mains_B) for p in range(NPORTS)} + return dict(T=hist(helpers), P=hist(paid), mains_per_port=mpp, Dports=sorted(dp) if dp else None, source='hist') + +def native_param(base, nA, nB, drop_mains_A=0, drop_mains_B=0, main_unit='rank1x8', c_A=0, c_B=0, dT=None, dP=None, + allow_negative=False): + """Parametric native per call. base = native dict. Returns a new native dict (source='param').""" + T = Counter(base['T']); P = Counter(base['P']) + unit = {'rank1x8': {1: 8}, 'rank8': {8: 1}}[main_unit] + for ports, k in ((nA, drop_mains_A), (nB, drop_mains_B)): + for r, v in unit.items(): + P[r] -= ports * k * v + P[1] += nA * Q(c_A) + nB * Q(c_B) + _add(T, dT or {}); _add(P, dP or {}) + mpp = base.get('mains_per_port') + if mpp is not None: + dp = set(base['Dports'] or []) + mpp = {p: m - (drop_mains_A if p in dp else drop_mains_B) for p, m in mpp.items()} + for r in list(P): # a negative native count is legal only if the caller absorbs it (invoice() re-checks) + if P[r] < 0 and not allow_negative: raise ValueError('native_param: negative paid count at rank %d' % r) + return dict(T=T, P=P, mains_per_port=mpp, Dports=base.get('Dports'), source='param', + param=dict(drop_mains_A=drop_mains_A, drop_mains_B=drop_mains_B, main_unit=main_unit, c_A=str(c_A), c_B=str(c_B))) + +# ------------------------------------------------------------------------------------------- exact word replay / pruning +def _span(b): + return frozenset(b) + +def replay(W): + """Exact replay (Fractions, sparse rows). Returns dict(ok, rows, frames, problems).""" + n = W['public']; N = n + W['owned_fields']; frames = W['frames'] + fs = [set(f) for f in frames] + rows = [{r: Q(1)} if r < n else {} for r in range(N)] + cur = list(W['initial']) + [None] * W['owned_fields']; owned = set(); bad = [] + for i, e in enumerate(W['program']): + op, r = e[0], e[1] + if op == 'MOVE': + a, b, priv = e[2:] + if cur[r] != a: bad.append(('move-from', i)); + cur[r] = b + elif op == 'INIT': + owned.add(r); rows[r] = {}; cur[r] = e[2] + elif op == 'CLEAR': + owned.discard(r); rows[r] = {}; cur[r] = None + elif op == 'SCALE': + q = Q(e[2]) + if cur[r] != e[3]: bad.append(('scale-frame', i)) + rows[r] = {k: v * q for k, v in rows[r].items()} + elif op in ('ADD', 'COPY'): + s = e[2]; q = Q(e[3]); f = e[4] + if cur[r] != f or cur[s] != f: bad.append(('add-frame', i)) + if op == 'COPY' and rows[r]: bad.append(('copy-nonzero', i)) + R = rows[r] + for k, v in rows[s].items(): + nv = R.get(k, 0) + q * v + if nv: R[k] = nv + else: R.pop(k, None) + else: + bad.append(('op', i, op)) + if owned: bad.append(('owned-alive', len(owned))) + if cur[:n] != list(W['final']): bad.append(('exit-frames',)) + return dict(ok=not bad, rows=rows, cur=cur, problems=bad[:20], nproblems=len(bad)) + +def drop_registers(W, drop): + """Exact dead-code deletion of the public registers in `drop` (e.g. hosted X2/Y2). + Keeps every event that some kept public output depends on; removes whole owned lifetimes with no live use. + Refuses (ValueError) if a dropped register feeds a kept output. Returns (W2, report).""" + drop = set(drop); n = W['public']; N = n + W['owned_fields']; PR = W['program'] + need_mask = 0 + dem = [0] * N + for r in range(n): + if r not in drop: dem[r] = 1 << r + keep = [False] * len(PR); feeds_dropped = 0 + for i in range(len(PR) - 1, -1, -1): + e = PR[i]; op, t = e[0], e[1] + if op in ('INIT', 'CLEAR'): + dem[t] = 0; continue + if op == 'MOVE': + continue + if dem[t]: + keep[i] = True + if op in ('ADD', 'COPY'): + s = e[2] + if s in drop: feeds_dropped += 1 + dem[s] |= dem[t] + if op == 'COPY': dem[t] = 0 + if feeds_dropped: + raise ValueError('drop_registers: %d kept events read a dropped register' % feeds_dropped) + # lifetime decisions for owned registers + life_keep = {}; open_life = {} + for i, e in enumerate(PR): + op, t = e[0], e[1] + if t >= n: + if op == 'INIT': open_life[t] = i; life_keep[i] = False + elif op in ('ADD', 'COPY', 'SCALE') and keep[i]: life_keep[open_life[t]] = True + if op in ('ADD', 'COPY') and keep[i] and e[2] >= n: life_keep[open_life[e[2]]] = True + life_of = {}; open_life = {} + for i, e in enumerate(PR): + op, t = e[0], e[1] + if t >= n: + if op == 'INIT': open_life[t] = i + life_of[i] = open_life[t] + out = []; dropped_moves = Counter(); dropped_lifetimes = 0; dim = [len(b) for b in W['frames']] + for i, e in enumerate(PR): + op, t = e[0], e[1] + if t in drop: + if op == 'MOVE': dropped_moves[abs(dim[e[3]] - dim[e[2]])] += 1 + continue + if t >= n: + if not life_keep[life_of[i]]: + if op == 'MOVE': dropped_moves[abs(dim[e[3]] - dim[e[2]])] += 1 + if op == 'INIT': dropped_lifetimes += 1 + continue + if op in ('INIT', 'CLEAR', 'MOVE') or keep[i]: out.append(e) + continue + if op == 'MOVE' or keep[i]: out.append(e) + W2 = dict(W); W2['program'] = out + H = dict(W.get('hosted') or {}) + if drop >= set(H.get('X2', [])) | set(H.get('Y2', [])) and H.get('X2'): + H['X2'] = []; H['Y2'] = []; H['dropped_hosted_plane'] = True + W2['hosted'] = H + rep = dict(events_before=len(PR), events_after=len(out), dropped_moves=dict(dropped_moves), + dropped_owned_lifetimes=dropped_lifetimes) + return W2, rep + +def zero_registers(W, drop): + """Hard deletion: registers in `drop` cease to exist (rows = 0). Every event that targets OR reads them is + removed, then dead code is pruned with drop_registers(). Only valid if verify_drop() shows every kept public row + unchanged; otherwise the word genuinely uses the dropped registers and a rebuild is needed.""" + drop = set(drop) + W1 = dict(W); W1['program'] = [e for e in W['program'] if not (e[1] in drop and e[0] != 'MOVE') + and not (e[0] in ('ADD', 'COPY') and e[2] in drop)] + removed_reads = sum(1 for e in W['program'] if e[0] in ('ADD', 'COPY') and e[2] in drop and e[1] not in drop) + W2, rep = drop_registers(W1, drop); rep['removed_reads'] = removed_reads + return W2, rep + +def verify_drop(W, W2, drop): + """Replay both words exactly; every kept public row and exit frame must be unchanged.""" + a = replay(W); b = replay(W2); n = W['public'] + same = all(a['rows'][r] == b['rows'][r] for r in range(n) if r not in set(drop)) + frames = all(b['cur'][r] == W['final'][r] for r in range(n)) + return dict(original_ok=a['ok'], pruned_ok=b['ok'], pruned_problems=b['problems'], kept_rows_equal=same, exit_frames=frames) + +# ------------------------------------------------------------------------------------------- caller +def caller_ports(C): + return {t: len(v['ports']) for t, v in C['types'].items()} + +def caller_invoice(C, lambda_mode='ordinary', only=None): + """Caller part of the invoice (no native). only='A'/'B' restricts to one port type. + Returns dict(T, P, nJ, ports, lambda_uses, ops).""" + assert lambda_mode in ('ordinary', 'paid_rank1') + ports = caller_ports(C); T = Counter(); P = Counter(); nJ = 0; lam = 0; ops = Counter() + for e in C['events']: + if e[0] == 'J': + nJ += 1; continue + ty, op = e[0], e[1] + if only and ty != only: continue + k = ports[ty]; ops[(ty, op)] += 1 + if op == 'QUERY': P[1] += k + elif op == 'IDLE_ONE': P[8 * len(e[3])] += k + elif op == 'IDLE_ODD': P[8 * len(e[3]) + 1] += k + elif op == 'LAMBDA': + lam += k + if lambda_mode == 'paid_rank1': P[1] += k + elif op in ('ADD', 'SWAP', 'SCALE', 'HOST'): pass + else: raise ValueError('caller op not in the alphabet: %r' % (op,)) + for ty, t in C['types'].items(): + if only and ty != only: continue + for dg in t['diagonal']: + T[int(dg[0]) % 2 + 8 * sum(dg[2:])] += ports[ty] + T[0] = 0; T = +T + return dict(T=T, P=+P, nJ=nJ, ports=ports, lambda_uses=lam, ops=ops) + +def check_slots(C, native=None): + """Per type and stage: mask rank, idle incidences, covered = mask - idles. With a native, compare covered with + the native's mains per port (the ledger law d = m).""" + nst = C['stages']; res = {} + for ty, t in C['types'].items(): + mask = [0] * (nst + 1); idle = [0] * (nst + 1) + for dg in t['diagonal']: + for k, b in enumerate(dg[2:], 1): + if b: mask[k] += 1 + for e in C['events']: + if e[0] == ty and e[1] in ('IDLE_ONE', 'IDLE_ODD'): + for k in e[3]: idle[k] += 1 + cov = [mask[k] - idle[k] for k in range(1, nst + 1)] + res[ty] = dict(mask=mask[1:], idle=idle[1:], covered=cov, header=sum(mask), idle_incidences=sum(idle)) + if native is not None and native.get('mains_per_port'): + mpp = native['mains_per_port'] + for ty, t in C['types'].items(): + ms = {mpp[p] for p in t['ports']} + res[ty]['native_mains'] = sorted(ms) + res[ty]['law_d_eq_m'] = (len(ms) == 1 and all(c == next(iter(ms)) for c in res[ty]['covered'])) + return res + +def caller_from_roots(fields, hosts_per_stage_cap, thin, ports, nst): + """Parametric caller (rule of our #6934 package, check.py outer()). fields = list of (mask stages, host stages); + a field of degree d is a target 8d+1; its unhosted mask stages are idled in maximal consecutive runs (P[8*run]); + a stage hosting more than hosts_per_stage_cap[k] fields pays one rank-8 idle per excess; thin rank-1 queries.""" + T = Counter(); P = Counter(); hc = Counter() + for ms, host in fields: + T[8 * len(ms) + 1] += ports + hs = set(host) + for k in hs: hc[k] += 1 + run = 0; prev = None + for k in sorted(ms): + if k in hs: + if run: P[8 * run] += ports + run = 0; prev = None; continue + if prev is not None and k == prev + 1: run += 1 + else: + if run: P[8 * run] += ports + run = 1 + prev = k + if run: P[8 * run] += ports + for k in range(nst): + if hc[k] > hosts_per_stage_cap[k]: P[8] += ports * (hc[k] - hosts_per_stage_cap[k]) + P[1] += ports * thin + return dict(T=+T, P=+P) + +# ------------------------------------------------------------------------------------------- invoice and price +def invoice(caller, native, calls=None): + """caller: dict from caller_invoice/caller_from_roots; native: native dict. calls defaults to caller nJ.""" + L = caller.get('nJ') if calls is None else calls + if not L: raise ValueError('invoice: number of native calls unknown (no J events and calls=None)') + T = Counter(caller['T']); P = Counter(caller['P']) + _add(T, native['T'], L); _add(P, native['P'], L) + for C_ in (T, P): + for r in list(C_): + if C_[r] == 0: del C_[r] + elif C_[r] < 0: raise ValueError('invoice: negative count at rank %d' % r) + return dict(T=T, P=P, calls=L) + +def S_of(T, P): + return sum(r * (T.get(r, 0) - P.get(r, 0)) for r in set(T) | set(P)) + +def _dec(x): + x = Q(x); return Decimal(x.numerator) / Decimal(x.denominator) + +def _coef(T, P): + return sorted((r, Q(T.get(r, 0)) - Q(P.get(r, 0))) for r in set(T) | set(P) if Q(T.get(r, 0)) != Q(P.get(r, 0))) + +def _F(cs, t, prec): + """F(t) and a rigorous bound on its evaluation error at this precision.""" + with localcontext() as c: + c.prec = prec; t = Decimal(t) + val = Decimal(0); mag = Decimal(0) + for r, k in cs: + p = (Decimal(r).ln() * (1 - t)).exp(); kd = _dec(k) + val += kd * p; mag += abs(kd) * p + err = mag * Decimal(10) ** (-(prec - 6)) + return val, err + +def _sign(cs, t, prec=60): + while prec <= 1200: + v, e = _F(cs, t, prec) + if abs(v) > e: return 1 if v > 0 else -1 + prec *= 2 + raise ArithmeticError('sign of F undetermined at t=%s' % t) + +def _unique(cs, hi, depth=14): + """Certify at most one root of F on [0, hi] by interval bounds (F excluded from 0, or F' < 0 on the piece).""" + with localcontext() as c: + c.prec = 40 + def pw(r, t): return (Decimal(r).ln() * (1 - Decimal(t))).exp() + def piece(a, b): + Fl = sum(_dec(k) * (pw(r, b) if k > 0 else pw(r, a)) for r, k in cs) + Fh = sum(_dec(k) * (pw(r, a) if k > 0 else pw(r, b)) for r, k in cs) + if Fl > 0 or Fh < 0: return 'noroot' + Dl = sum(_dec(k) * Decimal(r).ln() * (pw(r, b) if k > 0 else pw(r, a)) for r, k in cs if r > 1) + return 'mono' if Dl > 0 else 'unknown' + stack = [(Decimal(0), Decimal(hi), 0)]; mono = [] + while stack: + a, b, d = stack.pop(); s = piece(a, b) + if s == 'noroot': continue + if s == 'mono': mono.append((a, b)); continue + if d >= depth: return False + m = (a + b) / 2; stack += [(a, m, d + 1), (m, b, d + 1)] + mono.sort() + return all(mono[i][1] == mono[i + 1][0] for i in range(len(mono) - 1)) # one contiguous decreasing block + +def kappa(T, P, digits=50): + """Certified enclosure of the crossing. Returns dict(lo, hi, S, status). status NO_CROSSING if S <= 0.""" + cs = _coef(T, P); S = S_of(T, P) + if S <= 0: return dict(status='NO_CROSSING', S=S, lo=None, hi=None) + hi = Decimal('0.001') + while _sign(cs, hi) > 0: + hi *= 2 + if hi > 64: return dict(status='NO_SIGN_CHANGE', S=S, lo=None, hi=None) + lo = Decimal(0); uniq = _unique(cs, hi) + width = Decimal(10) ** (-digits - 2) + with localcontext() as c: + c.prec = digits + 30 + while hi - lo > width: + m = (lo + hi) / 2 + if _sign(cs, m, digits + 30) > 0: lo = m + else: hi = m + return dict(status='OK' if uniq else 'OK_UNIQUENESS_UNPROVED', S=S, lo=lo, hi=hi, unique=uniq) + +def Dsum(T, P, t=0, prec=60): + with localcontext() as c: + c.prec = prec; t = Decimal(t) + return sum(_dec(k) * Decimal(r).ln() * (Decimal(r).ln() * (1 - t)).exp() for r, k in _coef(T, P) if r > 1) + +def F_at(T, P, t, prec=60): + return _F(_coef(T, P), t, prec)[0] + +def price(inv, digits=50): + T, P = inv['T'], inv['P']; k = kappa(T, P, digits) + out = dict(S=S_of(T, P), paid_jobs=sum(P.values()), paid_positions=sum(r * v for r, v in P.items()), + targets=sum(T.values()), target_positions=sum(r * v for r, v in T.items()), kappa=k, + D0=Dsum(T, P, 0)) + if k.get('lo') is not None: out['Dk'] = Dsum(T, P, k['lo']) + return out + +def kstr(k, n=30): + if k.get('lo') is None: return k['status'] + return str(k['lo'])[:n + 2] + +def break_even_c(T, P, kappa_ref, ports_calls): + """Linear break-even: extra rank-1 paid per (port x call) at which kappa == kappa_ref. F(kref; c) = F0 - c*ports_calls.""" + return F_at(T, P, kappa_ref) / ports_calls + +# ------------------------------------------------------------------------------------------- ordinary units (#6934 package reading) +def word_ordinary(W): + """Per-call ordinary terms of a native word (allowance, reviewed upper, routes): our #6934 jclib.word_terms.""" + H = W['hosted']; nD = len(H['Dports']); n = len(W['initial']); live = set(W['live']) + main = live | {r + 112 for r in live} | set(H.get('X2', [])) | set(H.get('Y2', [])) + dim = [len(b) for b in W['frames']] + paid = Counter(abs(dim[e[3]] - dim[e[2]]) for e in W['program'] if e[0] == 'MOVE') + hel = Counter(dim[W['final'][r]] - dim[W['initial'][r]] for r in range(n) if r not in main) + life = sum(e[0] == 'INIT' for e in W['program']); allow = upper = 0 + for e in W['program']: + if e[0] in ('INIT', 'CLEAR'): allow += 2 + elif e[0] == 'SCALE': + q = Q(e[2]); a, d = abs(q.numerator), q.denominator + allow += 2 * (abs(a.bit_length() - d.bit_length()) + int(q < 0)); upper += 4 * (a.bit_length() + d.bit_length() + 1) + elif e[0] in ('ADD', 'COPY'): + q = Q(e[3]); a, d = abs(q.numerator), q.denominator + allow += 2 * (3 + a.bit_length() - 1 + bin(a).count('1') + d.bit_length() - 1); upper += 4 * (a.bit_length() + bin(a).count('1') + d.bit_length() + 1) + routes = (sum(paid.values()) + 2 * sum(hel.values()) + 208 + 2 * nD + 4 * 104 + 4 * nD + 4 * life) * 1000 + return dict(allow=allow, upper=upper, routes=routes, n=n, nD=nD) + +def _swaps(sel): + wanted = set(range(len(sel))); return len(list(zip(sorted(set(sel) - wanted), sorted(wanted - set(sel))))) + +def _shim(C, ty): + t = C['types'][ty]; off = t['offset']; ev = [] + for e in C['events']: + if e[0] == 'J': ev.append(['HOST', e[1]]) + elif e[0] == ty: + e = list(e[1:]) + if e[0] in ('ADD', 'SWAP', 'QUERY'): e[1] -= off; e[2] -= off + else: e[1] -= off + ev.append(e) + return dict(events=ev, roles=t['roles'], diagonal=t['diagonal']) + +def _rest(C, ports): + ev = C['events']; roles = C['roles']; mx = mn = mp = 0 + for role, (qp, zero, *sel) in enumerate(C['diagonal']): + axes = ([0] if qp else []) + [a for st, y in enumerate(sel, 1) if y for a in range(9 + 8 * (st - 1), 9 + 8 * st)] + k = _swaps(axes); mx += 2 * k * ports; mn += 2 * bool(qp) * ports; mp += int(qp == -1) * ports + ie = qn = qph = 0 + for e in ev: + if e[0] in ('IDLE_ONE',): + ie += 2 * _swaps([a for st in e[2] for a in range(9 + 8 * (st - 1), 9 + 8 * st)]) * ports + elif e[0] == 'QUERY': + qn += 2 * ports; qph += int(e[3] == -1) * ports + return (32 * roles * ports * len(ev) + sum(e[0] == 'QUERY' for e in ev) * 16 * ports + roles * ports * 26 + + (mx + mn + mp) * 1000 + ie * 1000 + (qn + qph) * 1000) + +def ordinary(C, W, odd_scale=1125): + """Raw ordinary readings of the #6934 package (literal/actual/upper/recharge/per_input). IDLE_ODD is not in that + formula (#7113-era callers): such callers get a flag, not a number for the IDLE_ODD routes.""" + t = word_ordinary(W); ports = caller_ports(C); L = sum(1 for e in C['events'] if e[0] == 'J') + rest = _rest(_shim(C, 'A'), ports['A']) + _rest(_shim(C, 'B'), ports['B']) + lam = sum(ports[e[0]] for e in C['events'] if e[0] != 'J' and e[1] == 'LAMBDA'); debit = 1000 * lam + sc = 4 * (odd_scale.bit_length() + 2) + 4 * (odd_scale.bit_length() + 2) + per_in = L * t['n'] * sc + lit = L * 10 ** 7 + rest; act = L * (t['allow'] + t['routes']) + rest; up = L * (t['upper'] + t['routes']) + rest + return dict(literal=lit, actual=act, upper=up, recharge=up + debit, per_input=up + per_in, per_input_recharge=up + per_in + debit, + lambda_uses=lam, lambda_debit=debit, idle_odd_present=any(e[0] != 'J' and e[1] == 'IDLE_ODD' for e in C['events'])) + +# ------------------------------------------------------------------------------------------- one-call convenience +def run(caller_path, native, lambda_mode='ordinary', digits=50): + C = load(caller_path); ci = caller_invoice(C, lambda_mode) + inv = invoice(ci, native); pr = price(inv, digits) + return dict(caller=ci, invoice=inv, price=pr, slots=check_slots(C, native)) + +def cite(label, pr): + """One-line citation string for RESULT.md files.""" + k = pr['kappa'] + return '[PRICEKIT %s] %s: S=%s kappa=%s D0=%.4f Dk=%s paid=%d' % ( + VERSION, label, pr['S'], kstr(k, 26), float(pr['D0']), ('%.8f' % float(pr['Dk'])) if 'Dk' in pr else '-', pr['paid_jobs']) + +if __name__ == '__main__': + import sys + if len(sys.argv) < 3: + print('usage: pricekit.py CALLER.json WORD.json[.gz] | NATIVE-RESULT.json'); sys.exit(1) + X = load(sys.argv[2]) + nat = native_from_word(X) if 'program' in X else native_from_hist(X['native']['paid'], X['native']['helpers']) + R = run(sys.argv[1], nat) + print(cite(sys.argv[2], R['price'])) diff --git a/research/p104-toolkit-v3/sl_lib.py b/research/p104-toolkit-v3/sl_lib.py new file mode 100644 index 0000000000..384ef7c02b --- /dev/null +++ b/research/p104-toolkit-v3/sl_lib.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""sl_lib.py (copy of our CALLEROUT sl_check.py library part, used by cap_check.py) [C exact mod p]: on a real exact caller (default: the #7113 caller in data/) compute, for every +native call k and each role block (A, B), the END-REFERRED sign-factor perturbation E_k = S (s(J_k) - I) S^-1 at q0 (S = suffix, +s(X) = X(q0) X(-q0)^-1), with random stage scalars D. Report: rank E_k, pair invariant t_k = (tr s(J_k) - n)/4, whether the row and +column spaces of E_k are supported inside the mask Z_k = {roles whose target contains D_k} (the Support Law SL), rank K = rho of the +whole block, and for every pair of calls the cross Gram rank rank(Y_k^T X_m) = rank(E_k E_m) vs |Z_k cap Z_m| (Theorem 1'). +Replay semantics copied from MONOCALLER/check.py (own re-implementation).""" +import json, random, sys +P = 2147483629 +IM = next(pow(g, (P - 1) // 4, P) for g in range(2, 99) if pow(g, (P - 1) // 2, P) == P - 1); HALF = (P + 1) // 2 +from fractions import Fraction as Q +fm = lambda x: (lambda f: f.numerator % P * pow(f.denominator, P - 2, P) % P)(Q(x)) +def ident(n): return [[int(i == j) for j in range(n)] for i in range(n)] +def matmul(A, B): + n = len(A); m = len(B[0]); Bt = list(zip(*B)) + return [[sum(a * b for a, b in zip(A[i], Bt[j])) % P for j in range(m)] for i in range(n)] +def inv(A): + n = len(A); M = [r[:] + [int(i == j) for j in range(n)] for i, r in enumerate(A)] + for c in range(n): + p = next(i for i in range(c, n) if M[i][c]); M[c], M[p] = M[p], M[c] + iv = pow(M[c][c], P - 2, P); M[c] = [v * iv % P for v in M[c]] + for i in range(n): + if i != c and M[i][c]: + f = M[i][c]; M[i] = [(a - f * b) % P for a, b in zip(M[i], M[c])] + return [r[n:] for r in M] +def rank(A): + A = [r[:] for r in A]; rk = 0 + if not A: return 0 + for c in range(len(A[0])): + p = next((i for i in range(rk, len(A)) if A[i][c]), None) + if p is None: continue + A[rk], A[p] = A[p], A[rk]; iv = pow(A[rk][c], P - 2, P); A[rk] = [v * iv % P for v in A[rk]] + for i in range(len(A)): + if i != rk and A[i][c]: + f = A[i][c]; A[i] = [(a - f * b) % P for a, b in zip(A[i], A[rk])] + rk += 1 + return rk +def basis(A): + A=[r[:] for r in A if any(r)];out=[]; + if not A:return out + rk=0 + for c in range(len(A[0])): + p=next((i for i in range(rk,len(A)) if A[i][c]),None) + if p is None:continue + A[rk],A[p]=A[p],A[rk];iv=pow(A[rk][c],P-2,P);A[rk]=[v*iv%P for v in A[rk]] + for i in range(len(A)): + if i!=rk and A[i][c]: + f=A[i][c];A[i]=[(a-f*b)%P for a,b in zip(A[i],A[rk])] + rk+=1 + return A[:rk] +def host_rows(R, src, dst, al, be, qv, d): + qi = pow(qv, P - 2, P); tt = al * be % P + ns = [(x + al * qv % P * y) % P * d % P for x, y in zip(R[src], R[dst])] + nd = [(be * qi % P * x + (1 + tt) * y) % P * d % P for x, y in zip(R[src], R[dst])] + R[src], R[dst] = ns, nd +def apply_native(R, C, e, qv, Dv, part): + """apply the A part (part='A') or B part (part='B') of J event e to row matrix R.""" + d = Dv[e[1]]; qi = pow(qv, P - 2, P) + if part == 'A': + form = C['types']['A']['form'] + if form == 'two-chain-HOST2': + src, dst, x2, y2 = e[2]; a2 = fm(C['types']['A'].get('alpha2', '-1')) + R[dst] = [(a + qi * b) % P for a, b in zip(R[dst], R[src])] + R[x2] = [(a + a2 * qv % P * b) % P for a, b in zip(R[x2], R[y2])] + for r in (src, dst, x2, y2): R[r] = [a * d % P for a in R[r]] + else: + src, dst, al, be = e[2]; host_rows(R, src, dst, fm(al), fm(be), qv, d) + else: + src, dst, al, be = e[3]; host_rows(R, src, dst, fm(al), fm(be), qv, d) +def apply_op(R, e, qv, Dv): + op = e[1] + if op == 'ADD': R[e[2]] = [(a + fm(e[4]) * b) % P for a, b in zip(R[e[2]], R[e[3]])] + elif op == 'SCALE': c = fm(e[3]); R[e[2]] = [a * c % P for a in R[e[2]]] + elif op == 'SWAP': R[e[2]], R[e[3]] = R[e[3]], R[e[2]] + elif op == 'QUERY': c = pow(qv, e[4] % 4, P); R[e[2]] = [(a + c * b) % P for a, b in zip(R[e[2]], R[e[3]])] + elif op == 'LAMBDA': + s = e[3]; c = ((1 + s * IM) * HALF + (1 - s * IM) * HALF * qv * qv) % P; R[e[2]] = [a * c % P for a in R[e[2]]] + elif op in ('IDLE_ONE', 'IDLE_ODD'): + c = 1 if op == 'IDLE_ONE' else pow(qv, e[4] % 4, P) + for s in e[3]: c = c * Dv[s] % P + R[e[2]] = [a * c % P for a in R[e[2]]] + else: raise SystemExit(op) +def analyse(C, q0, seed=7, label=''): + n = C['roles']; rnd = random.Random(seed); Dv = {s: rnd.randrange(2, P) for s in range(1, 18)} + R = ident(n); snaps = {} + for e in C['events']: + if e[0] == 'J': + apply_native(R, C, e, q0, Dv, 'A'); apply_native(R, C, e, q0, Dv, 'B'); snaps[e[1]] = (e, [r[:] for r in R]) + else: apply_op(R, e, q0, Dv) + F = R + out = {} + for part in ('A', 'B'): + T = C['types'][part]; off = T['offset']; m = len(T['diagonal']); blk = range(off, off + m) + masks = {k: {off + i for i, dg in enumerate(T['diagonal']) if dg[1 + k]} for k in range(1, 18)} + Fi = inv(F); Es = {}; info = {} + for k, (e, Rk) in snaps.items(): + # local sign factor of this part: J(q0) J(-q0)^-1 on the identity frame + Jp = ident(n); apply_native(Jp, C, e, q0, Dv, part) + Jm = ident(n); apply_native(Jm, C, e, (P - q0) % P, Dv, part) + s = matmul(Jp, inv(Jm)) + # row-op convention: new R = J R, J acts on the left; suffix S with F = S Rk => S = F Rk^-1 + S = matmul(F, inv(Rk)); E = [[(s[i][j] - (i == j)) % P for j in range(n)] for i in range(n)] + Ee = matmul(matmul(S, E), inv(S)) + tr = sum(s[i][i] for i in range(n)) % P; t = (tr - n) * pow(4, P - 2, P) % P + rows_sup = {i for i in range(n) if any(Ee[i])}; cols_sup = {j for j in range(n) if any(Ee[i][j] for i in range(n))} + Es[k] = (basis([list(c) for c in zip(*Ee)]), basis(Ee)) # (x basis = column space, y basis = row space) + tf = {fm(x): x for x in ('-1', '0', '-1/2', '-2', '1/2', '-1/4')}.get(t, str(t)) + info[k] = dict(rank=rank(Ee), t=tf, x_in_mask=rows_sup <= masks[k], y_in_mask=cols_sup <= masks[k], mask=len(masks[k]), + x_sup=len(rows_sup), y_sup=len(cols_sup)) + # whole-block K rank via product of all end-referred sign factors = s(F) restricted + pairs = [] + for k in Es: + for mm in Es: + if k < mm: + gram = lambda Y, X: [[sum(a * b for a, b in zip(y, x)) % P for x in X] for y in Y] + r = rank(gram(Es[k][1], Es[mm][0])); r2 = rank(gram(Es[mm][1], Es[k][0])) + pairs.append((k, mm, r, r2, len(masks[k] & masks[mm]))) + out[part] = (info, pairs) + bad_sl = [k for k, v in info.items() if not (v['x_in_mask'] and v['y_in_mask'])] + ts = sorted({v['t'] for v in info.values()}); rks = sorted({v['rank'] for v in info.values()}) + nont = [k for k, v in info.items() if v['t'] != '-1'] + viol = [(a, b, r, r2, z) for (a, b, r, r2, z) in pairs if a in nont and b in nont and (r < 2 or r2 < 2 or z < 2)] + sh = [z for (a, b, r, r2, z) in pairs] + mincr = min(min(r, r2) for (a, b, r, r2, z) in pairs if a in nont and b in nont) if len(nont) > 1 else None + print('%s q0=%s part %s: SL violations %s | call ranks %s | t values %s | calls with t!=-1: %d | min cross rank among them %s | ' + 'Thm1\' violations %d | shared-mask roles over all pairs min %d max %d | rank(E_k E_m) <= shared always: %s' + % (label, 'i' if q0 == IM else q0, part, bad_sl, rks, ts, len(nont), mincr, len(viol), min(sh), max(sh), + all(min(r, r2) <= z for (a, b, r, r2, z) in pairs)), flush=True) + return out +if __name__ == '__main__': + import gzip + from pathlib import Path + path = sys.argv[1] if len(sys.argv) > 1 else str(Path(__file__).resolve().parent / 'data' / 'caller-7113.json.gz') + with (gzip.open(path, 'rt') if path.endswith('.gz') else open(path)) as fh: + C = json.load(fh) + for q0 in (1, IM): analyse(C, q0, label=path.split('/')[-1]) diff --git a/research/p104-toolkit-v3/tests/common.py b/research/p104-toolkit-v3/tests/common.py new file mode 100644 index 0000000000..3f637d387e --- /dev/null +++ b/research/p104-toolkit-v3/tests/common.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Shared helpers for the toolkit tests: pinned data, #7601 word from its edit list, PASS/FAIL reporting.""" +import collections +import copy +import gzip +import hashlib +import json +import sys +import time +from pathlib import Path + +PKG = Path(__file__).resolve().parents[1] +DATA = PKG / 'data' +sys.path.insert(0, str(PKG)) + +SHA = { + 'base-7546-word.json.gz': '6e03624b173b90356e96f845d05fc93153846b863734dc8163ab6c44c602cd32', # ToikY #7546 bytes + 'caller-7113.json.gz': None, # gzip container; the JSON inside is pinned below + 'edits-7601.json': '05535155f82eb3e906b390fd2734ac42bd06720de2c9a4c62defc9ec9e977ed5', # our #7601 edit list +} +CALLER_JSON_SHA = '255d89f38cd45e38e5770f559bc43dc217e989032359768cdc7e8cdfd64b47bc' # Rohan Arun #7113 caller + +KAPPA_7546 = '0.00364841129479084532481164' +KAPPA_7601 = '0.0036490213053944204892137903671' +T0 = time.time() + + +def ok(cond, label): + if not cond: + print('FAIL ' + label, flush=True) + sys.exit(1) + print('ok %s [%.1fs]' % (label, time.time() - T0), flush=True) + + +def check_data(): + for name, h in SHA.items(): + if h: + ok(hashlib.sha256((DATA / name).read_bytes()).hexdigest() == h, 'sha256 pinned: data/' + name) + raw = gzip.decompress((DATA / 'caller-7113.json.gz').read_bytes()) + ok(hashlib.sha256(raw).hexdigest() == CALLER_JSON_SHA, 'sha256 pinned: data/caller-7113.json.gz content (#7113 caller)') + + +def load_gz(path): + return json.loads(gzip.decompress(Path(path).read_bytes())) + + +def base_word(): + return load_gz(DATA / 'base-7546-word.json.gz') + + +def caller_path(): + return DATA / 'caller-7113.json.gz' + + +def apply_edits(W, E): + """our #7601 edit-list format: delete [(index, event)], insert_before [(index, [events])], append [events].""" + prog = W['program'] + assert len(prog) == E['base_events'] + dele = {} + for i, ev in E['delete']: + assert prog[i] == ev, ('deleted event mismatch', i) + dele[i] = ev + ins = collections.defaultdict(list) + for i, evs in E['insert_before']: + ins[i].extend(evs) + out = [] + for i, ev in enumerate(prog): + out.extend(ins.get(i, [])) + if i not in dele: + out.append(ev) + out.extend(ins.get(len(prog), [])) + W2 = copy.deepcopy(W) + W2['program'] = out + E['append'] + return W2 + + +def word_7601(base=None): + return apply_edits(base or base_word(), json.loads((DATA / 'edits-7601.json').read_text())) diff --git a/research/p104-toolkit-v3/tests/test_checkers.py b/research/p104-toolkit-v3/tests/test_checkers.py new file mode 100644 index 0000000000..9c22871766 --- /dev/null +++ b/research/p104-toolkit-v3/tests/test_checkers.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +"""accx.py + fastreplay.py, lossless.py, nativeplan.py: positive and negative controls (standard library only).""" +import copy +import random +import sys +import time +from fractions import Fraction as Q +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from common import ok, check_data, base_word, word_7601, caller_path, KAPPA_7601 # noqa: E402 +import accx # noqa: E402 +import fastreplay # noqa: E402 +import lossless # noqa: E402 +import nativeplan # noqa: E402 +import validate # noqa: E402 +import contextlib, gzip, io, json, tempfile # noqa: E402,E401 + +check_data() +B = base_word(); W1 = word_7601(B) +quiet = [] + +# ---------------------------------------------------------------- accx +t = time.time(); r = accx.check(B, B, caller_path(), out=quiet.append, price=False) +ok(r['ok'] and 'ALL CHECKS PASS' in quiet, 'accx: #7546 vs itself ALL CHECKS PASS (%.1fs for the replay checks)' % (time.time() - t)) +quiet.clear(); r = accx.check(B, W1, caller_path(), out=quiet.append) +ok(r['ok'] and str(r['kappa_new']['lo']).startswith(KAPPA_7601) and any(l.endswith('GAIN') for l in quiet), + 'accx: #7601 word vs #7546 ALL CHECKS PASS, kappa %s, GAIN' % KAPPA_7601) +bad = copy.deepcopy(B) +for i, e in enumerate(bad['program']): + if i > 30000 and e[0] == 'ADD' and e[3] == '1': + e[3] = '2'; break +quiet.clear() +try: + accx.check(B, bad, caller_path(), out=quiet.append, price=False); failed = False +except accx.Fail: + failed = True +ok(failed and any('row differs: YES' in l for l in quiet), 'accx NEG: one corrupted ADD coefficient (event %d) FAILS, exact re-check confirms' % i) +bad2 = copy.deepcopy(B) +j = next(k for k, e in enumerate(bad2['program']) if e[0] == 'MOVE' and not e[4] and k > 1000) +bad2['program'][j] = list(bad2['program'][j]); bad2['program'][j][2], bad2['program'][j][3] = bad2['program'][j][3], bad2['program'][j][2] +quiet.clear() +try: + accx.check(B, bad2, caller_path(), out=quiet.append, price=False); failed = False +except accx.Fail: + failed = True +ok(failed, 'accx NEG: one reversed public MOVE FAILS (replay frame check)') +F = fastreplay.FastReplay(B) +ok(F.ok and fastreplay.exact_rows(B, [5])[5] == {5: Q(1)}, 'fastreplay: #7546 replays clean; exact backward slice of helper 5 is its identity row') + +# ---------------------------------------------------------------- validate (accx + lossless + the repository's invoice tools) +with tempfile.TemporaryDirectory() as d: + for name, Wx, want in (('#7601', W1, 0), ('corrupted', bad, 1)): + pth = Path(d) / 'w.json.gz' + with gzip.open(pth, 'wt') as fh: + json.dump(Wx, fh) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc = validate.main([str(Path(__file__).resolve().parents[1] / 'data' / 'base-7546-word.json.gz'), str(pth)]) + txt = buf.getvalue() + if want == 0: + ok(rc == 0 and 'official PASS' in txt and 'uniformly_better' in txt and 'crossing in (0.0036490213053, 0.0036490213054)' in txt, + 'validate: #7601 word OVERALL PASS; repository tools: uniformly_better, crossing in (0.0036490213053, 0.0036490213054)') + else: + ok(rc == 1 and 'accx FAIL' in txt, 'validate NEG: corrupted word OVERALL FAIL (accx)') + +# ---------------------------------------------------------------- lossless (law DUAL) +for name, Wx in (('#7546', B), ('#7601', W1)): + loss, lam, occ = lossless.dual_scan(Wx) + ok(not loss and not lam, 'lossless: %s has 0 lossy CLEARs and no relation left at the end' % name) +# negative: delete owned->public write-backs until a CLEAR becomes lossy +n = B['public']; wb = [k for k, e in enumerate(B['program']) if e[0] == 'ADD' and e[1] < n and e[2] >= n] +rng = random.Random(3); hit = None +for k in rng.sample(wb, 40): + Wd = copy.deepcopy(B); del Wd['program'][k] + loss = lossless.dual_scan(Wd)[0] + if loss: + hit = (k, loss); break +ok(hit is not None, 'lossless NEG: deleting the owned->public write-back at event %d makes %d CLEAR(s) lossy' % (hit[0], len(hit[1]))) + + +def exact_scan(prog, n): + """reference: Fraction state replay; a CLEAR of o is lossless iff row(o) is in the span of the other live rows.""" + rows = {r: {r: Q(1)} for r in range(n)}; out = [] + + def rank(vs): + piv = {}; rk = 0 + for v in vs: + x = dict(v) + while x: + p = min(x) + if p not in piv: + break + c = x[p] + for kk, y in piv[p].items(): + nv = x.get(kk, 0) - c * y + if nv: + x[kk] = nv + else: + x.pop(kk, None) + if x: + p = min(x); c = x[p]; piv[p] = {kk: y / c for kk, y in x.items()}; rk += 1 + return rk + for t_, e in enumerate(prog): + if e[0] == 'INIT': + rows[e[1]] = {} + elif e[0] == 'CLEAR': + o = e[1]; others = [v for r_, v in rows.items() if r_ != o] + if rank(others) < rank(others + [rows[o]]): + out.append(t_) + del rows[o] + elif e[0] == 'SCALE': + rows[e[1]] = {kk: v * Q(e[2]) for kk, v in rows[e[1]].items()} + else: + R = rows[e[1]] + for kk, v in list(rows[e[2]].items()): + nv = R.get(kk, 0) + Q(e[3]) * v + if nv: + R[kk] = nv + else: + R.pop(kk, None) + return out + + +rng = random.Random(11); agree = 0; nloss = 0; trials = 400 +for _ in range(trials): + n_, no = rng.randint(2, 4), rng.randint(1, 3); prog = []; live = [] + for _ in range(rng.randint(6, 24)): + c = rng.random() + free = [o for o in range(n_, n_ + no) if o not in live] + if c < 0.15 and free: + o = rng.choice(free); live.append(o); prog.append(['INIT', o, 0]) + elif c < 0.27 and live: + o = rng.choice(live); live.remove(o); prog.append(['CLEAR', o, 0]) + elif c < 0.42 and live: # park a public row in an owned register: o += a; a -= o + o = rng.choice(live); a = rng.randrange(n_); prog += [['ADD', o, a, '1', 0], ['ADD', a, o, '-1', 0]] + elif c < 0.52 and live: # write back + o = rng.choice(live); a = rng.randrange(n_); prog.append(['ADD', a, o, '1', 0]) + else: + regs = list(range(n_)) + live; a, b = rng.choice(regs), rng.choice(regs) + if a == b: + prog.append(['SCALE', a, rng.choice(['2', '-1', '1/2']), 0]) + else: + prog.append(['ADD', a, b, rng.choice(['1', '-1', '2', '1/2', '-3']), 0]) + for o in live: + prog.append(['CLEAR', o, 0]) + Wt = dict(public=n_, owned_fields=no, program=prog) + got = [t_ for t_, o, _ in lossless.dual_scan(Wt)[0]]; ref = exact_scan(prog, n_) + agree += got == ref; nloss += bool(ref) +ok(agree == trials and 0 < nloss < trials, 'lossless: law DUAL agrees with exact Fraction ranks on %d/%d random programs (%d with a lossy CLEAR)' + % (agree, trials, nloss)) + +# ---------------------------------------------------------------- nativeplan +plan = nativeplan.extract_plan(str(Path(__file__).resolve().parents[1] / 'data' / 'base-7546-word.json.gz'), log=lambda *a: None) +s = nativeplan.summary(plan) +ok(round(s['recost'], 6) == 9975.447669 and abs(nativeplan.recost_from_stays(plan) - s['recost']) < 1e-9, + 'nativeplan: #7546 helper layer re-costs to 9,975.447669 from routes alone (= from stays)') +ok(s['helpers'] == 1101 and s['stops'] == 3427 and s['basis_rank'] == 253 and s['blocks'] == 2682 and s['meetings'] == 36299, + 'nativeplan: 1,101 helpers, 3,427 interior stops, 2,682 requirement blocks, 36,299 meetings, state basis rank 253') +path = plan['word'] +okr, m = nativeplan.reemit_check(plan, path) +ok(okr and m == 70714, 'nativeplan: all 70,714 helper-touching events re-emitted identically from the plan') +plan['events'][100]['q'] = '7' +okr2, _ = nativeplan.reemit_check(plan, path) +ok(not okr2, 'nativeplan NEG: one tampered plan coefficient breaks the re-emission check') +print('PASS_TOOLKIT_CHECKERS') diff --git a/research/p104-toolkit-v3/tests/test_drops.py b/research/p104-toolkit-v3/tests/test_drops.py new file mode 100644 index 0000000000..82209ed61a --- /dev/null +++ b/research/p104-toolkit-v3/tests/test_drops.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +"""tokdrop.py on #7546 (needs numpy): the engine re-finds and rebuilds our #7601 drops byte-for-byte.""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from common import ok, check_data, base_word, word_7601, caller_path, KAPPA_7601 # noqa: E402 +import accx # noqa: E402 +import modp # noqa: E402 +import tokdrop as T # noqa: E402 + +check_data() +print('modp backend:', modp.BACKEND) +B = base_word() +ctx = T.Context(B) +recs = ctx.census() +ok(len(recs) > 3000 and ctx.Z == 3 and ctx.VF == 127, 'census: %d interior public stays on #7546; {0} = frame 3, V = frame 127' % len(recs)) +m = {(x['r'], x['f']): x['m'] for x in recs} +ok(abs(m[(134, 595)] - 1.9095425) < 1e-6 and abs(m[(100, 783)] - 1.9095425) < 1e-6, + 'census: stop price m = f(3) - f(1) - f(2) = 1.9095 D0/call for both #7601 stays') + +# controls: the two stays of #7601 repair in clean mode with settlement rank 0 (tiers A and B) +for r_, F in ((134, 595), (100, 783)): + st = ctx.stay(r_, F) + res = ctx.test_best([st]) + a = ctx.settle_rank(res, 'A'); b = ctx.settle_rank(res, 'B') + ok(res['ok'] and res['nviol'] == 0 and a[0] == 0 and b[0] == 0 and a[2], + 'scan %d@%d %s: clean repair (K = %d candidates, r1 = r2 = %d), settlement rank A 0 / B 0, pi exact' % (r_, F, st, res['K'], res['r1'])) +# negative controls: without correction candidates the same drops are damaging (the repair is what makes them free) +for r_, F in ((134, 595), (100, 783)): + res = ctx.test([ctx.stay(r_, F)], frames_mode='none') + ok(not res['ok'] and res['nviol'] > 0, 'scan NEG %d@%d with no correction candidates: not repairable (%d violated rows)' % (r_, F, res['nviol'])) + +# exact build of both drops = our #7601 word, byte-for-byte +W4, rep = ctx.build_exact([ctx.stay(100, 783), ctx.stay(134, 595)]) +W7601 = word_7601(B) +ok(W4 == W7601, 'build: both drops -> exactly the #7601 word (%d events; %d corrections, %d premix ADDs at {0}, %d post-ops)' + % (len(W4['program']), rep['corrections'], rep['premix'], rep['postops'])) +ok(rep['owned_obs_lost'] == 1 and rep['owned_obs_new_only'] == 0 and abs(rep['saved_D0_per_call'] - 3.8190850) < 1e-6, + 'build: stop price 3.8191 D0/call; 1 owned observation removed (134 += -2*owned1503 at 595), none added') +out = []; r = accx.check(B, W4, caller_path(), out=out.append) +ok(r['ok'] and str(r['kappa_new']['lo']).startswith(KAPPA_7601), 'accx: rebuilt word ALL CHECKS PASS, kappa %s' % KAPPA_7601) +print('PASS_TOOLKIT_DROPS') diff --git a/research/p104-toolkit-v3/tests/test_dual.py b/research/p104-toolkit-v3/tests/test_dual.py new file mode 100644 index 0000000000..738a695328 --- /dev/null +++ b/research/p104-toolkit-v3/tests/test_dual.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +"""dualword.py: the frame-perp transpose is an exact, price-preserving involution (standard library only).""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from common import ok, check_data, base_word, word_7601, caller_path, KAPPA_7546, KAPPA_7601 # noqa: E402 +import accx # noqa: E402 +import dualword as DW # noqa: E402 +import lossless # noqa: E402 +import pricekit as pk # noqa: E402 + +check_data() +B = base_word() + + +def price(W): + nat = pk.native_from_word(W) + return nat, pk.run(caller_path(), nat)['price'] + + +nB, pB = price(B) +D1 = DW.dual(B) +nD, pD = price(D1) +ok(nD['T'] == nB['T'] and nD['P'] == nB['P'], 'dual(#7546): native target and paid histograms identical (every charge is a dim gap)') +ok(pD['S'] == pB['S'] == 1424 and pD['kappa']['lo'] == pB['kappa']['lo'] and pD['kappa']['hi'] == pB['kappa']['hi'] + and pD['D0'] == pB['D0'] and pk.kstr(pD['kappa'], 26) == KAPPA_7546, 'dual(#7546): priced bit-identically, kappa %s' % KAPPA_7546) +ok(all(len(a) + len(b) == 9 for a, b in zip(B['frames'], D1['frames'])), 'dual: every frame F maps to F^perp (dim d -> 9 - d)') +mv = [(e, f) for e, f in zip(B['program'], reversed(D1['program'])) if e[0] == 'MOVE'] +ok(all((len(B['frames'][e[3]]) - len(B['frames'][e[2]])) == (len(D1['frames'][f[3]]) - len(D1['frames'][f[2]])) for e, f in mv), + 'dual: every MOVE keeps its signed dimension gap (ascents stay ascents, %d MOVEs)' % len(mv)) +DD = DW.restore_copy(DW.dual(D1), B) +same, why = DW.same_word(DD, B) +ok(same, 'dual(dual(#7546)) = #7546 (%s; the single COPY restored)' % why) +DDraw = DW.dual(D1) +diff = [k for k, (a, b) in enumerate(zip(DDraw['program'], B['program'])) if a != b] +ok(len(diff) == 1 and B['program'][diff[0]][0] == 'COPY', 'dual(dual) differs from #7546 in exactly one event: COPY -> ADD at %d' % diff[0]) + +rB = pk.replay(B)['rows']; rD = pk.replay(D1)['rows'] +D1s, ports = DW.fix_b_ports(B, D1, rB, rD) +ok(sorted(B['live'].index(p) for p in ports) == sorted(set(range(104)) - set(B['hosted']['Dports'])), + 'dual: exactly the 8 native-B ports (the ports outside hosted.Dports) need the free contract SCALEs (16 SCALE events)') +r1 = pk.replay(D1s) +ok(r1['ok'] and r1['rows'][:B['public']] == rB[:B['public']], 'dual + SCALEs: all 1,501 public rows equal #7546 exactly (Fraction replay)') +out = [] +try: + accx.check(B, D1s, caller_path(), out=out.append, price=False); strict = True +except accx.Fail: + strict = False +ok(not strict and out[-1].startswith('FAIL owned MOVEs identical'), 'accx strict on the dual fails ONLY the owned-MOVE identity rule (new owned frames)') +out = []; res = accx.check(B, D1s, caller_path(), allow_owned_reframe=True, out=out.append) +ok(res['ok'] and res['kappa_new']['lo'] == res['kappa_base']['lo'] and res['kappa_new']['hi'] == res['kappa_base']['hi'], + 'accx --allow-owned-reframe: dual + SCALEs ALL CHECKS PASS, kappa ratio 1') +ok(not lossless.dual_scan(D1s)[0], 'lossless: the dual has 0 lossy CLEARs') +W1 = word_7601(B); n1, p1 = price(DW.dual(W1)) +ok(str(p1['kappa']['lo']).startswith(KAPPA_7601), 'dual(#7601) priced bit-identically to #7601: kappa %s' % KAPPA_7601) +# negative control: transpose without the X/Y port swap is not a valid native +orig = DW.port_swap +DW.port_swap = lambda W: list(range(W['public'])) +bad = DW.dual(B) +DW.port_swap = orig +try: + accx.check(B, bad, caller_path(), out=[].append, price=False, allow_owned_reframe=True); passed = True +except (accx.Fail, ValueError): + passed = False +ok(not passed, 'dual NEG: the transpose WITHOUT the X <-> Y port swap fails accx') +print('PASS_TOOLKIT_DUAL') diff --git a/research/p104-toolkit-v3/tests/test_dualdrop.py b/research/p104-toolkit-v3/tests/test_dualdrop.py new file mode 100644 index 0000000000..ebb22ce0e2 --- /dev/null +++ b/research/p104-toolkit-v3/tests/test_dualdrop.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +"""tokdrop.py on the frame-perp dual (needs numpy): the dual-side scan finds the main-902 drop that the primal engine +cannot see; mapped back it stacks on our #7601 word.""" +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from common import ok, check_data, base_word, word_7601, caller_path, KAPPA_7601 # noqa: E402 +import accx # noqa: E402 +import dualword as DW # noqa: E402 +import lossless # noqa: E402 +import tokdrop as T # noqa: E402 + +KAPPA_902 = '0.00364932638757719639692487' +check_data() +B = base_word(); W1 = word_7601(B) + + +def dual_ctx(W): + D = DW.dual(W) + Z = [i for i, f in enumerate(D['frames']) if not f][0]; V = [i for i, f in enumerate(D['frames']) if len(f) == 9][0] + DS = DW.swap_frames(D, Z, V) # {0} and V trade places under perp; swap back so the engine runs unchanged + return T.Context(DS), Z, V + + +ctx, Z, V = dual_ctx(W1) +pi = DW.port_swap(W1) +ok(pi[902] == 794 and (Z, V) == (127, 3), 'dual of #7601: hosted main 902 is register 794 on the dual side; {0}/V frames swapped (3 <-> 127)') +# the primal engine cannot drop 902's stop: 902 is a main whose stop at 595 carries 5 owned writes +pctx = T.Context(W1) +pres = pctx.test_best([pctx.stay(902, 595)]) +ok(not pres['ok'], 'primal NEG: dropping 902@595 on #7601 directly has no clean repair (K = %d, violations %d)' % (pres['K'], pres['nviol'])) +st = ctx.stay(794, 595) +res = ctx.test_best([st]) +a = ctx.settle_rank(res, 'A'); b = ctx.settle_rank(res, 'B') +ok(res['ok'] and res['K'] == 0 and a[0] == 0 and b[0] == 0, 'dual scan 794@595 %s: repaired with K = 0, settlement rank A 0 / B 0' % (st,)) +Wd, rep = ctx.build_exact([st]) +ok(rep['corrections'] == 0 and rep['premix'] == 0 and rep['owned_obs_lost'] == 1, 'dual build: 0 corrections, 0 premix, 1 owned observation removed') +W5 = DW.restore_copy(DW.dual(DW.swap_frames(Wd, Z, V)), W1) +import collections # noqa: E402 +key = lambda e: json.dumps(e) +dele = list((collections.Counter(map(key, W1['program'])) - collections.Counter(map(key, W5['program']))).elements()) +dele = [json.loads(x) for x in dele] +ok(len(W5['program']) == len(W1['program']) - 6 and sum(e[0] == 'ADD' and 902 in (e[1], e[2]) for e in dele) == 5, + 'mapped back: main 902 climbs 3 -> 618 in one MOVE (2 MOVEs -> 1) and its 5 owned ADDs at chart 595 vanish') +out = []; r = accx.check(B, W5, caller_path(), out=out.append) +ok(r['ok'] and r['owned_moves_same'] and str(r['kappa_new']['lo']).startswith(KAPPA_902), + 'accx vs #7546 (strict): ALL CHECKS PASS; #7601 + 902@595 prices kappa %s' % KAPPA_902) +ok(r['kappa_new']['lo'] > r['kappa_base']['hi'], 'pricekit: above #7601 (%s) on the #7113 caller' % KAPPA_7601) +ok(not lossless.dual_scan(W5)[0], 'lossless: 0 lossy CLEARs') +# negative control: a tier-A-only stay on the dual (rank 0 with V-helper rows relaxed) is refused by the exact build +st = ctx.stay(1289, 117) +res = ctx.test_best([st]) +a = ctx.settle_rank(res, 'A')[0] if res['ok'] else None +b = ctx.settle_rank(res, 'B')[0] if res['ok'] else None +try: + ctx.build_exact([st]); built = True +except T.BuildError as ex: + built = False; why = str(ex) +ok(res['ok'] and a == 0 and b >= 1 and not built, 'build NEG dual 1289@117: settlement rank A 0 / B %s, exact build refused (%s)' % (b, why)) +print('PASS_TOOLKIT_DUAL_DROP') diff --git a/research/p104-toolkit-v3/tests/test_pricing.py b/research/p104-toolkit-v3/tests/test_pricing.py new file mode 100644 index 0000000000..3d2649ab26 --- /dev/null +++ b/research/p104-toolkit-v3/tests/test_pricing.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +"""pricekit.py, kformula.py, layout_price.py, cap_check.py: controls on the vendored #7546 word and #7113 caller.""" +import contextlib +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import io +import math +import runpy +from collections import Counter + +from common import ok, check_data, base_word, caller_path, word_7601, KAPPA_7546, KAPPA_7601, PKG +import pricekit as pk +import kformula as kf + +check_data() +W = base_word(); C = pk.load(caller_path()) + +# ---------------------------------------------------------------- pricekit +nat = pk.native_from_word(W) +R = pk.run(caller_path(), nat) +pr = R['price'] +ok(pr['S'] == 1424 and pk.kstr(pr['kappa'], 26) == KAPPA_7546 and pr['kappa']['status'] == 'OK', + 'pricekit: #7546 = S 1424, kappa %s (certified enclosure, uniqueness proved)' % KAPPA_7546) +ok('%.4f' % float(pr['D0']) == '393242.1350' and pr['paid_jobs'] == 110468, 'pricekit: #7546 D0 393242.1350, 110468 paid jobs') +ok(pr['kappa']['hi'] - pr['kappa']['lo'] < 1e-50, 'pricekit: enclosure width < 1e-50') +p1 = pk.run(caller_path(), pk.native_from_word(word_7601(W)))['price'] +ok(str(p1['kappa']['lo']).startswith(KAPPA_7601) and p1['S'] == 1424, + 'pricekit: #7601 word (rebuilt from our edit list) kappa %s' % KAPPA_7601) +sl = pk.check_slots(C, nat) +ok(all(sl[t]['law_d_eq_m'] for t in ('A', 'B')), 'pricekit: ledger law d = m holds on #7113 x #7546 (both port types)') +# negative controls +bad = dict(T=Counter({1: 1}), P=Counter({8: 1})) +ok(pk.kappa(bad['T'], bad['P'])['status'] == 'NO_CROSSING', 'pricekit NEG: S <= 0 gives NO_CROSSING, no kappa') +inv = pk.invoice(pk.caller_invoice(C), nat) +inv2 = dict(T=Counter(inv['T']), P=Counter(inv['P'])); inv2['P'][1] += 104 +ok(pk.price(inv2, 30)['kappa']['lo'] < pr['kappa']['lo'], 'pricekit NEG: 104 extra rank-1 paid jobs (one per port) lower kappa') + +# ---------------------------------------------------------------- kformula +C7 = pk.caller_invoice(C); c17 = kf.caller(17) +ok(c17['T'] == C7['T'] and c17['P'] == C7['P'] and C7['nJ'] == 17, 'kformula: closed-form C(17) == the #7113 caller invoice (T and P)') +ok(pk.kstr(kf.price(17, nat)['kappa'], 26) == KAPPA_7546, 'kformula: C(17) x #7546 native prices kappa(#7546)') +ok(kf.caller(16)['T'] != C7['T'], 'kformula NEG: C(16) differs from the #7113 caller') +ok(all(pk.S_of(*[pk.invoice(kf.caller(k), nat)[x] for x in ('T', 'P')]) == kf.S_closed(k) for k in range(4, 65)), + 'kformula: S(k) = 96k - 208 exactly for k = 4..64') +kap = {k: kf.price(k, nat, 20)['kappa']['lo'] for k in range(6, 41)} +kbest = max(kap, key=kap.get) +ok(kbest == 17, 'kformula: exact sweep k = 6..40 peaks at k = 17 (kappa(16) %.10f, kappa(18) %.10f)' % (kap[16], kap[18])) +ok(all(kap[k] > kap[k + 1] for k in range(17, 40)) and all(kap[k] < kap[k + 1] for k in range(6, 17)), + 'kformula: kappa(k) strictly unimodal on 6..40') +def D0_of(k): + iv = pk.invoice(kf.caller(k), nat) + return float(pk.Dsum(iv['T'], iv['P'], 0)) +D0 = {k: D0_of(k) for k in (1000, 2000)} +beta = (D0[2000] / 2 - D0[1000]) / (1000 * math.log(2)) / 104 +ok(abs(beta - 34) < 0.2, 'kformula: D0(k)/104 slope beta = 2(2h-1) = 34 (measured %.3f from k = 1000, 2000)' % beta) +s_over_d = 1424 / float(pr['D0']) +ok(abs(float(kap[17]) / s_over_d - 1.0075) < 0.0005, 'kformula: kappa(17) / (S/D0) = %.4f (first-order law kappa ~ S/D0)' % (float(kap[17]) / s_over_d)) +fo = kf.first_order(17, 124.913) +ok(abs(fo / float(kap[17]) - 1) < 0.01, 'kformula: asymptotic first-order formula at k = 17 within 1%% of exact (%.7f vs %.7f)' % (fo, float(kap[17]))) +eps, rho, beta_ = 2 * (1 - 56 / 104), 2, 34 +gam = 1 + 124.913 / beta_ + math.log(rho / eps) +u = gam +for _ in range(60): + u = gam + math.log(u) # u - ln u = gamma, larger root (= -W_{-1}(-e^-gamma)) +kst, kapst = rho / eps * u, eps / (beta_ * u) +ok(abs(gam - 5.4471) < 1e-3 and abs(kst - 16.15) < 0.01 and abs(kapst - 0.003641) < 2e-6 and abs(17 * kst * kapst - 1) < 1e-12, + 'kformula: Lambert optimum gamma %.4f, k* = %.2f, kappa* = %.6f, (2h-1) k* kappa* = 1 (exact family optimum k = 17)' % (gam, kst, kapst)) + +# ---------------------------------------------------------------- layout_price / cap_check +buf = io.StringIO() +with contextlib.redirect_stdout(buf): + runpy.run_path(str(PKG / 'layout_price.py'), run_name='__main__') +txt = buf.getvalue() +ok('td 1.0950 sh 1.0031 rel 0.9773' in txt.split('model B (built B')[1].splitlines()[0], + 'layout_price: model B reproduces the built B caller readings 1.0950 / 1.0031 / 0.9773 [C-param]') +ok('td 1.8950 sh 1.6368 rel 1.5695' in txt, 'layout_price NEG/reference: uncapped lines give the unreachable 1.8950 / 1.6368 / 1.5695') +buf = io.StringIO() +with contextlib.redirect_stdout(buf): + runpy.run_path(str(PKG / 'cap_check.py'), run_name='__main__') +txt = buf.getvalue() +blines = [l for l in txt.splitlines() if l.strip().startswith('boundaries:')] +ok(len(blines) == 2 and all(l.count('ry4 rx4') == 15 and '8|9:S2 ry2 rx2 cap0' in l for l in blines), + 'cap_check: on the built #7113 B, rank(Y_k,Y_k+1|S) = rank(X_k,X_k+1|S) = 4 at all 15 non-centre boundaries (q0 = 1 and i)') +fl = [l for l in txt.splitlines() if 'floor (' in l] +eq = sum(1 for l in fl for tok in l.split(': ', 1)[1].split() if tok.split(':')[1].split('/')[1] == tok.split(':')[1].split('/')[2]) +ok(len(fl) == 2 and eq == 2 * 15, 'cap_check: Floor lemma rank(X_arm, hubs) = |Z_k| at every non-centre call (15 x 2)') +print('PASS_TOOLKIT_PRICING') diff --git a/research/p104-toolkit-v3/tokdrop.py b/research/p104-toolkit-v3/tokdrop.py new file mode 100644 index 0000000000..367f34366c --- /dev/null +++ b/research/p104-toolkit-v3/tokdrop.py @@ -0,0 +1,825 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Token-settled drop engine for P104 native words (our TOKDEBT / SEEDBUILD / OWNRETIME lanes, consolidated). + +A *stay* is one interior stop of a public register: (r, F, t_arr, t_dep) = register r sits at frame F from the MOVE at +program index t_arr to the MOVE at t_dep. Dropping a stay deletes the two MOVEs around it (the register climbs +straight to its next frame) and every event that touched r at F. The engine then asks whether the damage can be +repaired for free: + + 1. clean-mode repair (Engine.solve). Unknown corrections a += k*b are inserted wherever a and b sit at a common + frame after a became dirty (b clean there, so final rows stay LINEAR in k). Constraints: every main and every + helper not ending at V has its base final row, compared on the main (pi) columns only ("clean mode": helper token + columns are deleted); helpers ending at V land in the row span of the V group (free V post-ops); owned rows at + every owned->public write and CLEAR equal the base. Solved mod 2^31-1 by random rank-1 tensor projections + (Schwartz-Zippel), rank test, canonical rref solution. + 2. settlement rank (settle). The helper-token columns the clean repair ignored are settled by a premix at t = 0: an + invertible block-diagonal map on registers sharing a birth frame. rank = sum_f rank[Xnew_f | Xbase_f] - rank Xnew_f. + Tier B keeps every row exact; tier A relaxes helpers ending at V. rank 0 <=> a premix exists. + 3. exact build (build_exact). Rational reconstruction of the corrections, exact Fraction solve of the premix + (ADDs at the birth frame at t = 0), exact V post-ops, exact replay against the base. + +Results from this engine (see README): the two drops of our #7601 (main 134 at chart 595, main 100 at chart 783), +rebuilt here byte-for-byte; and, run on the frame-perp dual (dualword.py), the main-902 drop at chart 595. + +API + ctx = Context(word) base word (dict); ctx.census() -> interior stays with stop price m + ctx.stay(r, F) -> (r, F, t_arr, t_dep) locate a stay + ctx.test(stays, owned=True, frames_mode='down'|'all'|'none', want=True) -> dict(ok, K, nviol, r1, r2, x, ...) + ctx.settle_rank(res, tier) -> (rank, per_frame, pi_ok, nobs) + ctx.build_exact(stays, owned=False) -> (word, report) raises BuildError if any step is not exact +CLI + python3 tokdrop.py WORD scan r:F [r:F ...] repair + settlement rank of single-stay drops + python3 tokdrop.py WORD build r:F [r:F ...] OUT exact word with all listed stays dropped +numpy is required (stdlib + numpy); python-flint is used automatically if installed (faster, same verdicts). +""" +import collections +import gzip +import json +import math +import random +import sys +import time +from fractions import Fraction as Q +from pathlib import Path + +try: + import numpy as np +except ImportError: # pragma: no cover + sys.exit('tokdrop.py needs numpy (pip install numpy); stdlib-only tools: pricekit.py, accx.py, dualword.py') +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import modp # noqa: E402 + +P = modp.P + + +class BuildError(ValueError): + pass + + +def load(path): + op = gzip.open if str(path).endswith('.gz') else open + with op(path, 'rt') as f: + return json.load(f) + + +def save(W, path): + with gzip.open(path, 'wt') as f: + json.dump(W, f) + + +def qm(s): + q = Q(s) + return q.numerator % P * pow(q.denominator % P, P - 2, P) % P + + +def mains_of(W): + live = set(W['live']); H = W['hosted'] + return live | {r + 112 for r in live} | set(H['X2']) | set(H['Y2']) + + +def spanmask(basis): + s = {0} + for v in basis: + s |= {x ^ v for x in s} + m = 0 + for x in s: + m |= 1 << x + return m + + +def xlogx(j): + return j * math.log(j) if j > 0 else 0.0 + + +def ratrec(a, p=P): + a %= p; r0, r1 = p, a; s0, s1 = 0, 1; B = int((p // 2) ** 0.5) + while r1 > B: + q = r0 // r1; r0, r1 = r1, r0 - q * r1; s0, s1 = s1, s0 - q * s1 + if s1 == 0 or abs(s1) > B: + return None + v = Q(r1, s1) + return v if (v.numerator - a * v.denominator) % p == 0 else None + + +# ----------------------------------------------------------------------------------------------- word structure +def stays(W): + """per register: list of [frame, t_arr, t_dep] (t_arr = -1 initial; t_dep = len(program) final).""" + L = collections.defaultdict(list) + for r in range(W['public']): + L[r].append([W['initial'][r], -1, None]) + for i, e in enumerate(W['program']): + if e[0] == 'MOVE': + L[e[1]][-1][2] = i; L[e[1]].append([e[3], i, None]) + elif e[0] == 'INIT': + L[e[1]].append([e[2], i, None]) + elif e[0] == 'CLEAR': + L[e[1]][-1][2] = i + for r in L: + if L[r][-1][2] is None: + L[r][-1][2] = len(W['program']) + return L + + +def replay_mod(W, snaps=None): + """dense mod-P replay; snaps {pos: regs} records rows BEFORE event pos. Returns (X, snapshots, owned-write rows, + CLEAR rows).""" + n = W['public']; N = n + W['owned_fields'] + X = np.zeros((N, n), dtype=np.int64); X[np.arange(n), np.arange(n)] = 1 + out = {}; ow = {}; cl = {}; snaps = snaps or {} + prog = W['program'] + for i, e in enumerate(prog + [None]): + if i in snaps: + for r in snaps[i]: + out[(i, r)] = X[r].copy() + if e is None: + break + op = e[0] + if op == 'MOVE': + continue + r = e[1] + if op == 'INIT': + X[r] = 0 + elif op == 'CLEAR': + cl[i] = X[r].copy(); X[r] = 0 + elif op == 'SCALE': + X[r] = X[r] * qm(e[2]) % P + else: + s_ = e[2] + if r < n and s_ >= n: + ow[i] = X[s_].copy() + if op == 'COPY': + X[r] = qm(e[3]) * X[s_] % P + else: + X[r] = (X[r] + qm(e[3]) * X[s_]) % P + return X, out, ow, cl + + +def frac_replay(W): + n = W['public']; N = n + W['owned_fields'] + rows = [{r: Q(1)} if r < n else {} for r in range(N)] + for e in W['program']: + op = e[0] + if op == 'MOVE': + continue + r = e[1] + if op in ('INIT', 'CLEAR'): + rows[r] = {}; continue + if op == 'SCALE': + q = Q(e[2]); rows[r] = {k: v * q for k, v in rows[r].items()}; continue + s_, q = e[2], Q(e[3]); R = rows[r] + if op == 'COPY': + R.clear() + for k, v in list(rows[s_].items()): + nv = R.get(k, 0) + q * v + if nv: + R[k] = nv + else: + R.pop(k, None) + return rows + + +def owned_observations(W, with_rows=False): + """every owned->public write and every CLEAR, with the exact owned row at that moment (and the final rows).""" + n = W['public'] + rows = [{r: Q(1)} if r < n else {} for r in range(n + W['owned_fields'])] + out = [] + for e in W['program']: + op = e[0] + if op == 'MOVE': + continue + r = e[1] + if op == 'INIT': + rows[r] = {}; continue + if op == 'CLEAR': + out.append(('cl', r, dict(rows[r]))); rows[r] = {}; continue + if op == 'SCALE': + q = Q(e[2]); rows[r] = {k: v * q for k, v in rows[r].items()}; continue + s_, q = e[2], Q(e[3]) + if r < n and s_ >= n: + out.append(('ow', r, s_, dict(rows[s_]))) + R = rows[r] + if op == 'COPY': + R.clear() + for k, v in list(rows[s_].items()): + nv = R.get(k, 0) + q * v + if nv: + R[k] = nv + else: + R.pop(k, None) + return (out, rows) if with_rows else out + + +def insert(W2, evs): + """evs: list of (pos, event) inserted before W2 position pos (stable order).""" + ins = collections.defaultdict(list) + for pos, ev in evs: + ins[pos].append(ev) + prog = [] + for j, e in enumerate(W2['program'] + [None]): + prog.extend(ins.get(j, [])) + if e is not None: + prog.append(e) + W3 = dict(W2); W3['program'] = prog + return W3 + + +def postops(W, VF, rows=None): + """exact V post-ops (ADD/SCALE at V) so every helper ending at V is the identity row. Returns (W', nops). + rows: exact final rows of W if already known (they are copied, not modified).""" + rows = frac_replay(W) if rows is None else [dict(x) for x in rows] + n = W['public']; mains = mains_of(W) + Vg = [r for r in range(n) if W['final'][r] == VF] + Vh = [r for r in Vg if r not in mains]; Vm = [r for r in Vg if r in mains]; Vhs = set(Vh) + ops = [] + bad = [h for h in Vh if rows[h] != {h: Q(1)}] + + def axpy(dst, src, c): + for k, v in src.items(): + nv = dst.get(k, 0) + c * v + if nv: + dst[k] = nv + else: + dst.pop(k, None) + for h in bad: + R = dict(rows[h]) + for j in Vm: + c = R.get(j) + if c: + ops.append(['ADD', h, j, str(-c), VF]); axpy(R, rows[j], -c) + for j in list(R): + if j not in Vhs: + raise BuildError('helper %d keeps non-V coordinate %d' % (h, j)) + rows[h] = R + badset = set(bad) + for h in bad: + R = rows[h] + for j in [j for j in R if j not in badset]: + c = R[j]; ops.append(['ADD', h, j, str(-c), VF]); R.pop(j) + order = list(bad) + for i, h in enumerate(order): + if not rows[h].get(h): + for g in order[i + 1:]: + if rows[g].get(h): + ops.append(['ADD', h, g, '1', VF]); axpy(rows[h], rows[g], 1) + break + else: + raise BuildError('singular V block at %d' % h) + c = rows[h][h] + if c != 1: + ops.append(['SCALE', h, str(1 / c), VF]); rows[h] = {k: v / c for k, v in rows[h].items()} + for g in order: + if g == h: + continue + c = rows[g].get(h) + if c: + ops.append(['ADD', g, h, str(-c), VF]); axpy(rows[g], rows[h], -c) + W2 = dict(W); W2['program'] = W['program'] + ops + return W2, len(ops) + + +# ----------------------------------------------------------------------------------------------- edits +class Edit: + """build W2 from base W. ins[i] = events inserted before base event i; dele = deleted base indices; + mv[i] = replacement event for base index i.""" + + def __init__(s, W): + s.W = W; s.n = W['public']; s.n2 = s.n + s.ins = collections.defaultdict(list); s.dele = set(); s.mv = {} + + def build(s): + W = s.W; prog = []; origin = [] + for i, e in enumerate(W['program'] + [None]): + for ev in s.ins.get(i, []): + prog.append(list(ev)); origin.append(None) + if e is None: + break + if i in s.dele: + continue + prog.append(s.mv[i] if i in s.mv else list(e)); origin.append(i) + W2 = dict(W); W2['program'] = prog + return W2, origin + + +# ----------------------------------------------------------------------------------------------- the engine +class Engine: + def __init__(s, ctx, W2, origin): + s.ctx = ctx; s.W2 = W2; s.origin = origin; s.n2 = W2['public']; s.N2 = s.n2 + W2['owned_fields'] + s.B = ctx.W; s.n = s.B['public'] + Xb, _, owb, clb = ctx.brep + T = np.zeros((s.n2, s.n2), dtype=np.int64); T[:s.n, :s.n] = Xb[:s.n] + s.cm = np.zeros(s.n2, dtype=np.int64) + s.owb = owb; s.clb = clb + s.mains = mains_of(W2) + for r in s.mains: + if r < s.n2: + s.cm[r] = 1 + s.T = T * s.cm[None, :] if ctx.clean else T + s.Vgrp = [r for r in range(s.n2) if W2['final'][r] == ctx.VF] + s.Vh = set(r for r in s.Vgrp if r not in s.mains) + s.L = stays(W2) + + def ext(s, row): + o = np.zeros(s.n2, dtype=np.int64); o[:s.n] = row[:s.n] + return o * s.cm if s.ctx.clean else o + + def dirt(s, cands): + d = {} + for a, b, pos, F in cands: + if a not in d or d[a] > pos: + d[a] = pos + for j, e in enumerate(s.W2['program']): + if e[0] in ('ADD', 'COPY'): + r, s_ = e[1], e[2] + if s_ in d and d[s_] <= j and (r not in d or d[r] > j): + d[r] = j + 1 + return d + + def filter_linear(s, cands): + while True: + d = s.dirt(cands) + keep = [c for c in cands if not (c[1] in d and d[c[1]] <= c[2])] + if len(keep) == len(cands): + return keep, d + cands = keep + + def solve(s, cands, seed=1, extra_M=8, want_solution=True): + t0 = time.time(); clean = s.ctx.clean + cands, d = s.filter_linear(list(cands)) + K = len(cands) + W2 = s.W2; n2 = s.n2; N2 = s.N2 + snaps = collections.defaultdict(set) + for a, b, pos, F in cands: + snaps[pos].add(b) + X2, sn, ow2, cl2 = replay_mod(W2, snaps) + if clean: + cm = s.cm + X2 = X2 * cm[None, :]; sn = {k: v * cm for k, v in sn.items()} + ow2 = {k: v * cm for k, v in ow2.items()}; cl2 = {k: v * cm for k, v in cl2.items()} + Rb = (np.array([sn[(pos, b)] for a, b, pos, F in cands], dtype=np.int64).reshape(K, n2) if K + else np.zeros((0, n2), dtype=np.int64)) + inj = collections.defaultdict(list) + for i, (a, b, pos, F) in enumerate(cands): + inj[pos].append((a, i)) + Sym = np.zeros((N2, max(K, 1)), dtype=np.int64); live = np.zeros(N2, dtype=bool) + ownc = [] + prog = W2['program'] + for j, e in enumerate(prog + [None]): + for a, i in inj.get(j, []): + Sym[a, i] = (Sym[a, i] + 1) % P; live[a] = True + if e is None: + break + op = e[0] + if op == 'MOVE': + continue + r = e[1] + if op == 'INIT': + Sym[r] = 0; live[r] = False + elif op == 'CLEAR': + if live[r]: + ownc.append((Sym[r].copy(), j, 'clear')) + Sym[r] = 0; live[r] = False + elif op == 'SCALE': + if live[r]: + Sym[r] = Sym[r] * qm(e[2]) % P + else: + s_ = e[2] + if r < n2 and s_ >= n2 and live[s_]: + ownc.append((Sym[s_].copy(), j, 'write')) + if op == 'COPY': + Sym[r] = qm(e[3]) * Sym[s_] % P if live[s_] else 0; live[r] = live[s_] + elif live[s_]: + Sym[r] = (Sym[r] + qm(e[3]) * Sym[s_]) % P; live[r] = True + Nv = s.ctx.v_null(s.T, s.Vgrp) + E_rows = []; E_rhs = []; V_rows = []; V_rhs = []; viol = [] + Vlist = sorted(s.Vh) + VC = modp.matmul(X2[Vlist][:, :n2], Nv) if Vlist else None + Vpos = {R: i for i, R in enumerate(Vlist)} + for R in range(n2): + if R in s.Vh: + cur = VC[Vpos[R]] + if live[R]: + V_rows.append(Sym[R]); V_rhs.append((-cur) % P) + elif cur.any(): + viol.append(('Vspan', R)) + else: + diff = (s.T[R] - X2[R][:n2]) % P + if live[R]: + E_rows.append(Sym[R]); E_rhs.append(diff) + elif diff.any(): + viol.append(('exact', R)) + + def borow(j, kind): + oi = s.origin[j] + if oi is None: + o = W2['program'][j][2] if kind == 'write' else W2['program'][j][1] + for jj in range(j + 1, len(W2['program'])): + ee = W2['program'][jj] + if ee[0] in ('ADD', 'COPY') and ee[1] == o: + return None + if s.origin[jj] is None: + continue + if ee[0] in ('ADD', 'COPY') and ee[2] == o and ee[1] < n2: + return s.owb.get(s.origin[jj]) + if ee[0] == 'CLEAR' and ee[1] == o: + return s.clb.get(s.origin[jj]) + return None + return (s.owb if kind == 'write' else s.clb).get(oi) + livepos = {(j, kind) for _, j, kind in ownc} + for j, kind in [(j, 'write') for j in ow2] + [(j, 'clear') for j in cl2]: + if (j, kind) in livepos: + continue + b = borow(j, kind); c = (ow2 if kind == 'write' else cl2)[j] + if b is None: + continue + if not np.array_equal(s.ext(b) % P, c % P): + viol.append(('owned', j, kind)) + for sv, j, kind in ownc: + b = borow(j, kind); c = (ow2 if kind == 'write' else cl2)[j] + if b is None: + raise BuildError('dirty owned register at a new event %d' % j) + E_rows.append(sv); E_rhs.append((s.ext(b) - c) % P) + res = dict(K=K, ndirty=len(d), nE=len(E_rows), nV=len(V_rows), viol=viol[:20], nviol=len(viol), cands=cands) + if viol: + res['ok'] = False; res['why'] = 'clean-violation'; return res + if K == 0: + res['ok'] = all(not x.any() for x in E_rhs + V_rhs); res['x'] = []; res['r1'] = res['r2'] = 0 + return res + rng = random.Random(seed); M = K + extra_M + + def rmat(a, b): + return np.array([rng.randrange(P) for _ in range(a * b)], dtype=np.int64).reshape(a, b) + Atot = np.zeros((M, K), dtype=np.int64); btot = np.zeros(M, dtype=np.int64) + blocks = [] + if E_rows: + blocks.append((np.array(E_rows) % P, np.array(E_rhs) % P, None)) + if V_rows: + blocks.append((np.array(V_rows) % P, np.array(V_rhs) % P, Nv)) + for S, Rhs, Nm in blocks: + nr = S.shape[0] + al = rmat(M, nr) + if Nm is None: + be = rmat(n2, M) + Bproj = modp.matmul(Rb, be); Rp = modp.matmul(Rhs, be) + else: + ga = rmat(Nm.shape[1], M) + Bproj = modp.matmul(Rb, modp.matmul(Nm, ga)); Rp = modp.matmul(Rhs, ga) + A1 = modp.matmul(al, S); B1 = Bproj.T + Atot = (Atot + (A1 * B1) % P) % P + btot = (btot + ((al * Rp.T) % P).sum(axis=1) % P) % P + Ab = np.hstack([Atot, btot.reshape(M, 1)]) + r1 = modp.rank(Atot); r2 = modp.rank(Ab) + res.update(r1=r1, r2=r2, ok=(r1 == r2), secs=time.time() - t0) + if r1 == r2 and want_solution: + R_, rk = modp.rref(Ab); x = [0] * K + for i in range(rk): + nz = np.nonzero(R_[i])[0] + piv = int(nz[0]) if len(nz) else None + if piv is None or piv == K: + res['ok'] = False; break + x[piv] = int(R_[i, K]) + res['x'] = x + return res + + +def closure(E, seeds): + d = dict(seeds) + for j, e in enumerate(E.W2['program']): + if e[0] in ('ADD', 'COPY'): + r, s_ = e[1], e[2] + if s_ in d and d[s_] <= j and (r not in d or d[r] > j + 1): + d[r] = j + 1 + return d + + +def byframe(E): + if not hasattr(E, '_bf'): + bf = collections.defaultdict(list) + for r, L in E.L.items(): + for F, s, e in L: + bf[F].append((s, e, r)) + E._bf = bf + return E._bf + + +# ----------------------------------------------------------------------------------------------- context +class Context: + def __init__(s, W, clean=True): + s.W = W; s.clean = clean; s.n = W['public']; s.NO = W['owned_fields'] + s.dim = [len(b) for b in W['frames']] + s.mask = [spanmask(b) for b in W['frames']] + zs = [i for i, d in enumerate(s.dim) if d == 0]; vs = [i for i, d in enumerate(s.dim) if d == 9] + if len(zs) != 1 or len(vs) != 1: + raise BuildError('need exactly one {0} frame and one V frame') + s.Z, s.VF = zs[0], vs[0] + s.stays = stays(W) + s.brep = replay_mod(W) + s._vnull = {} + s._census = None + s._rows = None + s._obs = None + + def base_rows(s): + if s._rows is None: + s._obs, s._rows = owned_observations(s.W, with_rows=True) + return s._rows + + def base_obs(s): + if s._obs is None: + s._obs, s._rows = owned_observations(s.W, with_rows=True) + return s._obs + + def v_null(s, T, Vgrp): + key = (tuple(Vgrp), T.shape[0]) + if key not in s._vnull: + s._vnull[key] = modp.nullspace(T[Vgrp]) + return s._vnull[key] + + def sub(s, a, b): + return (s.mask[a] & ~s.mask[b]) == 0 + + def downset(s, G): + return {F for F in range(len(s.W['frames'])) if F != s.Z and s.sub(F, G)} + + def census(s): + """interior public stays with their stop price m (D0/call saved by dropping the stop) and event kinds.""" + if s._census is None: + W = s.W; n = s.n + st = [[dict(f=W['initial'][r], t=-1, ev=[])] for r in range(n)] + for i, e in enumerate(W['program']): + op = e[0]; r = e[1] + if op == 'MOVE': + if r < n: + st[r].append(dict(f=e[3], t=i, ev=[])) + continue + if op in ('ADD', 'COPY'): + s_ = e[2] + if r < n: + st[r][-1]['ev'].append(('OW' if s_ >= n else 'PI', i, s_)) + if s_ < n: + st[s_][-1]['ev'].append(('OR' if r >= n else 'PO', i, r)) + recs = [] + for r in range(n): + S = st[r]; d = [s.dim[x['f']] for x in S] + for k in range(1, len(S) - 1): + m = xlogx(d[k + 1] - d[k - 1]) - xlogx(d[k] - d[k - 1]) - xlogx(d[k + 1] - d[k]) + kinds = frozenset(x[0] for x in S[k]['ev']) + t_dep = S[k + 1]['t'] + recs.append(dict(r=r, k=k, f=S[k]['f'], dim=d[k], m=m, kinds=kinds, + stay=(r, S[k]['f'], S[k]['t'], t_dep))) + s._census = recs + return s._census + + def stay(s, r, F): + hits = [x for x in s.stays[r] if x[0] == F and x[1] >= 0 and x[2] < len(s.W['program'])] + if len(hits) != 1: + raise BuildError('register %d has %d interior stays at frame %d' % (r, len(hits), F)) + return (r, F, hits[0][1], hits[0][2]) + + def route_cost(s, fr): + return sum(xlogx(abs(s.dim[b] - s.dim[a])) for a, b in zip(fr, fr[1:])) + + def make(s, drops): + B0 = s.W; ed = Edit(B0); prog = B0['program']; seeds_base = {}; saved = 0.0 + byreg = collections.defaultdict(list) + for x in drops: + byreg[x[0]].append(x) + for h, L in byreg.items(): + full = s.stays[h]; fr0 = [x[0] for x in full] + dset = {(F, a) for _, F, a, _ in L} + keep = [x for x in full if (x[0], x[1]) not in dset] + saved += s.route_cost([x[0] for x in keep]) - s.route_cost(fr0) + for _, F, a, d in L: + for i in range(a + 1, d): + e = prog[i] + if e[0] == 'MOVE': + continue + regs = [e[1]] if e[0] in ('SCALE', 'INIT', 'CLEAR') else [e[1], e[2]] + if h in regs: + ed.dele.add(i); seeds_base.setdefault(e[1], i) + if seeds_base[e[1]] > i: + seeds_base[e[1]] = i + if e[0] in ('ADD', 'COPY'): + if h not in seeds_base or seeds_base[h] > i: + seeds_base[h] = i + ed.dele.add(a) + for j in range(1, len(full)): + if (full[j][0], full[j][1]) in dset: + continue + if (full[j - 1][0], full[j - 1][1]) in dset: + k = j - 1 + while (full[k][0], full[k][1]) in dset: + k -= 1 + a = full[j][1]; e = list(prog[a]); e[2] = full[k][0]; ed.mv[a] = e + if (full[-1][0], full[-1][1]) in dset: + raise BuildError('cannot drop the final stay of %d' % h) + W2, org = ed.build() + pos_of = {o: j for j, o in enumerate(org) if o is not None} + + def wpos(i): + while i not in pos_of: + i += 1 + return pos_of[i] + seeds = {r: wpos(i) for r, i in seeds_base.items()} + return ed, W2, org, seeds, saved + + def cands_all(s, E, d, frames=None, owned_src=True, owned_tgt=True): + bf = byframe(E); out = set() + for a, da in d.items(): + if a >= E.n2 and not owned_tgt: + continue + for F, sa, ea in E.L[a]: + if ea is None or ea < da: + continue + if frames is not None and F not in frames: + continue + if F == s.VF and a < E.n2: + continue + for sb, eb, b in bf[F]: + if b == a: + continue + if b >= E.n2 and not owned_src: + continue + lo = max(sa, sb) + 1; hi = min(ea, eb); lo = max(lo, da) + if lo > hi: + continue + if b in d and d[b] <= lo: + continue + out.add((a, b, lo, F)) + hb = min(hi, d[b]) if b in d else hi + if hb > lo: + out.add((a, b, hb, F)) + return sorted(out, key=lambda c: c[2]) + + def test(s, drops, frames_mode='down', K=4000, owned=True, seed=1, want=False): + ed, W2, org, seeds, sv = s.make(drops) + E = Engine(s, W2, org) + dd = closure(E, seeds) + S = set() + for h, F, a, d_ in drops: + L = s.stays[h]; i = [x[1] for x in L].index(a); nf = L[i + 1][0] + S |= s.downset(nf) | {F} + if frames_mode == 'all': + S = None + elif frames_mode == 'none': # no correction candidates at all (control: is the drop damaging?) + S = set() + cc = s.cands_all(E, dd, frames=S, owned_src=owned, owned_tgt=owned) + if len(cc) > K: + t0 = min(x[2] for x in cc) + cc = [c for c in cc if c[2] <= t0 + 3000][:K] + r = E.solve(cc, seed=seed, want_solution=want) + r['saved'] = sv; r['W2'] = W2; r['org'] = org + return r + + def test_best(s, drops, owned=True): + """frames 'down' first, then 'all' if it does better (the scan rule of TOKDEBT/DUALSCAN).""" + r = s.test(drops, owned=owned, want=True, frames_mode='down') + if r['nviol'] or r['K'] < 30 or not r.get('ok'): + r2 = s.test(drops, owned=owned, want=True, frames_mode='all') + if r2['nviol'] < r['nviol'] or r2.get('ok'): + r = r2 + return r + + # ------------------------------------------------------------------------------------------- settlement + def corrected(s, r): + W2 = r['W2']; evs = collections.defaultdict(list) + for (a, b, pos, F), v in zip(r.get('cands', []), r.get('x', [])): + if v: + evs[pos].append(['ADD', a, b, str(int(v)), F]) + prog = []; org = list(r['org']); org2 = [] + for j, e in enumerate(W2['program'] + [None]): + for ev in evs.get(j, []): + prog.append(ev); org2.append(None) + if e is None: + break + prog.append(e); org2.append(org[j]) + W3 = dict(W2); W3['program'] = prog + return W3, org2 + + def settle_rank(s, r, tier='A'): + """premix settlement rank of a repaired schedule: (rank, {birth frame: rank}, pi_ok, #observations).""" + B0 = s.W; n = s.n + W3, org2 = s.corrected(r) + n2 = W3['public']; X, _, ow, cl = replay_mod(W3) + Xb, _, owb, clb = s.brep + new = [X[q][:n2] for q in range(n2)]; base = [Xb[q][:n] for q in range(n)]; kind = [('pub', q) for q in range(n2)] + for j, row in ow.items(): + oi = org2[j] + if oi is not None and oi in owb: + new.append(row); base.append(owb[oi]); kind.append(('ow', j)) + for j, row in cl.items(): + oi = org2[j] + if oi is not None and oi in clb: + new.append(row); base.append(clb[oi]); kind.append(('cl', j)) + Xn = np.array(new) % P; Xbb = np.array(base) % P + mains = mains_of(B0); pc = sorted(mains) + Vh = {q for q in range(n) if B0['final'][q] == s.VF and q not in mains} + keep = [i for i, k in enumerate(kind) if not (tier == 'A' and k[0] == 'pub' and k[1] in Vh)] + Xn = Xn[keep]; Xbb = Xbb[keep] + pi_ok = not ((Xn[:, pc] - Xbb[:, pc]) % P).any() if tier == 'A' else None + fr = collections.defaultdict(list) + for q in range(n): + fr[B0['initial'][q]].append(q) + tot = 0; per = {} + for f_, R in fr.items(): + An = Xn[:, R]; Ab = Xbb[:, R] + if not (An != Ab).any(): + continue + rn = modp.rank(An); rj = modp.rank(np.hstack([An, Ab])) + if rj > rn: + per[f_] = rj - rn; tot += rj - rn + return tot, per, pi_ok, len(keep) + + # ------------------------------------------------------------------------------------------- exact build + def build_exact(s, drops, owned=False, log=print): + """exact word with every stay in `drops` removed; raises BuildError unless every step is exact.""" + B0 = s.W; n = s.n + r = s.test(drops, owned=owned, want=True) + if not r.get('ok'): + r = s.test(drops, owned=owned, want=True, frames_mode='all') + if not r.get('ok'): + raise BuildError('engine found no clean repair (K=%d, violations %d)' % (r['K'], r['nviol'])) + W2 = r['W2'] + evs = [] + for (a, b, pos, F), v in zip(r['cands'], r['x']): + if v: + q = ratrec(v) + if q is None: + raise BuildError('rational reconstruction failed') + evs.append((pos, ['ADD', a, b, str(q), F])) + W3 = insert(W2, evs) + obs3, rows = owned_observations(W3, with_rows=True); rows_b = s.base_rows() + mains = mains_of(B0) + Vh = {R for R in range(n) if B0['final'][R] == s.VF and R not in mains} + bad = [R for R in range(n) if rows[R] != rows_b[R] and R not in Vh] + cols = sorted({c for R in bad for c in set(rows[R]) | set(rows_b[R]) if rows[R].get(c, 0) != rows_b[R].get(c, 0)}) + fr = collections.defaultdict(list) + for c in range(n): + fr[B0['initial'][c]].append(c) + pre = [] + for j in cols: + R = [x for x in fr[B0['initial'][j]] if x != j] + idx = sorted(i for i in ({i for i in range(n) for x in R if x in rows[i]} + | {i for i in range(n) if j in rows[i] or j in rows_b[i]}) if i not in Vh) + A = [[rows[i].get(x, Q(0)) for x in R] + [rows_b[i].get(j, Q(0)) - rows[i].get(j, Q(0))] for i in idx] + m = len(R); piv = []; rr = 0 + for c in range(m): + p = next((i for i in range(rr, len(A)) if A[i][c] != 0), None) + if p is None: + continue + A[rr], A[p] = A[p], A[rr]; inv = 1 / A[rr][c]; A[rr] = [x * inv for x in A[rr]] + for i in range(len(A)): + if i != rr and A[i][c] != 0: + f_ = A[i][c]; A[i] = [x - f_ * y for x, y in zip(A[i], A[rr])] + piv.append(c); rr += 1 + if any(all(x == 0 for x in row[:m]) and row[m] != 0 for row in A): + raise BuildError('premix for column %d is not solvable exactly' % j) + pre.append((j, {R[piv[i]]: A[i][m] for i in range(len(piv)) if A[i][m] != 0})) + ops = [['ADD', x, j, str(c), B0['initial'][j]] for j, y in pre for x, c in y.items()] + W4 = insert(W3, [(0, e) for e in ops]) + obs4, rows4 = owned_observations(W4, with_rows=True) if ops else (obs3, rows) + W4, npo = postops(W4, s.VF, rows4) + if npo: + obs4, rows4 = owned_observations(W4, with_rows=True) + bad4 = [R for R in range(n) if rows4[R] != rows_b[R]] + if bad4: + raise BuildError('%d public rows differ from the base after premix' % len(bad4)) + key = lambda o: (o[0], o[1], o[2] if o[0] == 'ow' else None, tuple(sorted(o[-1].items()))) + cb = collections.Counter(key(o) for o in s.base_obs()) + c4 = collections.Counter(key(o) for o in obs4) + rep = dict(K=r['K'], corrections=len(evs), premix=len(ops), postops=npo, saved_D0_per_call=r['saved'], + owned_obs_base=sum(cb.values()), owned_obs_new=sum(c4.values()), + owned_obs_lost=sum((cb - c4).values()), owned_obs_new_only=sum((c4 - cb).values())) + if rep['owned_obs_new_only']: + raise BuildError('new owned observations appeared: %d' % rep['owned_obs_new_only']) + return W4, rep + + +def parse_stays(ctx, args): + out = [] + for a in args: + r, F = a.split(':') + out.append(ctx.stay(int(r), int(F))) + return out + + +def main(argv): + if len(argv) < 3 or argv[1] not in ('scan', 'build'): + print(__doc__); return 2 + W = load(argv[0]); ctx = Context(W) + print('backend', modp.BACKEND, flush=True) + if argv[1] == 'scan': + for st in parse_stays(ctx, argv[2:]): + r = ctx.test_best([st]) + line = dict(stay=st, ok=bool(r.get('ok')), K=r['K'], nviol=r['nviol'], saved=round(r['saved'], 4)) + if r.get('ok'): + line['rankA'] = ctx.settle_rank(r, 'A')[0]; line['rankB'] = ctx.settle_rank(r, 'B')[0] + print(json.dumps(line), flush=True) + return 0 + *sts, out = argv[2:] + W4, rep = ctx.build_exact(parse_stays(ctx, sts)) + save(W4, out); print(json.dumps(rep)); return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/research/p104-toolkit-v3/validate.py b/research/p104-toolkit-v3/validate.py new file mode 100644 index 0000000000..b206be44cf --- /dev/null +++ b/research/p104-toolkit-v3/validate.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""One-command validation of an edited P104 native word against its base (our NETCAUX validate wrapper, made +self-contained). Standard library only. + +Steps (each PASS/FAIL, summary table at the end; exit 0 iff every step passed): + 1 accx accx.py BASE NEW (randomised-exact replay with exact re-checks, legality, owned rules, pricekit kappa) + 2 lossless lossless.py law DUAL on NEW: no CLEAR destroys information + 3 official when run inside an integer-mult-bounds checkout: the repository's own invoice tools + scripts/compare_recursive_invoices.py (BASE invoice -> NEW invoice) and + scripts/evaluate_recursive_power.py (NEW vs the empty invoice at r^(1-C)) at C just below and just above + NEW's certified kappa enclosure; requires new_cost_lower below and new_cost_higher above. + Skipped (not failed) outside a checkout. +usage: python3 validate.py BASE NEW [--caller CALLER.json[.gz]] [--allow-owned-reframe] [--repo ROOT] +""" +import json +import os +import subprocess +import sys +import tempfile +import time +from decimal import Decimal +from fractions import Fraction +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import accx # noqa: E402 +import lossless # noqa: E402 +import pricekit as pk # noqa: E402 + + +def invoice_json(inv, boundary='complete signed #7113 caller application with 17 native calls'): + def num(v): + v = Fraction(v) + return int(v) if v.denominator == 1 else str(v) + return {'schema': 'integer-mult-recursive-invoice/v1', + 'contract': {'model': 'mixed-target-unnormalized-power-v1', + 'rank_semantics': 'H_r costs r^(1-C); debit surrendered productive fields', + 'ambient_rank': 145, 'boundary': boundary}, + 'paid': {str(k): num(inv['P'][k]) for k in sorted(inv['P']) if inv['P'][k]}, + 'targets': {str(k): num(inv['T'][k]) for k in sorted(inv['T']) if inv['T'][k]}} + + +def official(root, caller, B, W, kappa, out=print): + root = Path(root) + if not (root / 'scripts' / 'compare_recursive_invoices.py').exists(): + return None, 'skipped (no repository checkout at %s)' % root + C = pk.load(caller); ci = pk.caller_invoice(C) + invB = pk.invoice(ci, pk.native_from_word(B)); invW = pk.invoice(ci, pk.native_from_word(W)) + with tempfile.TemporaryDirectory() as d: + pb, pw, pz = (Path(d) / x for x in ('base.json', 'new.json', 'zero.json')) + pb.write_text(json.dumps(invoice_json(invB))); pw.write_text(json.dumps(invoice_json(invW))) + pz.write_text(json.dumps(invoice_json(dict(T={}, P={})))) + + def tool(name, *args): + p = subprocess.run([sys.executable, '-I', '-B', str(root / 'scripts' / name), *map(str, args)], + capture_output=True, text=True, timeout=300, cwd=root) + if p.returncode: + raise RuntimeError('%s failed: %s' % (name, (p.stderr or p.stdout)[-400:])) + return json.loads(p.stdout) + rep = tool('compare_recursive_invoices.py', pb, pw) + out(' compare_recursive_invoices base -> new: %s, paid delta %s, target delta %s' + % (rep['classification'], rep.get('delta_paid'), rep.get('delta_targets'))) + lo = Decimal(str(kappa['lo'])[:15]); hi = lo + Decimal('1e-13') + res = {} + for c in (lo, hi): + r = tool('evaluate_recursive_power.py', pz, pw, '--exponent', str(1 - c), '--bits', 256) + res[c] = r['classification'] + out(' evaluate_recursive_power at C = %s: %s' % (c, r['classification'])) + good = res[lo] == 'new_cost_lower' and res[hi] == 'new_cost_higher' + return good, 'compare: %s; crossing in (%s, %s)' % (rep['classification'], lo, hi) + + +def main(argv): + args = list(argv); caller = accx.DEFAULT_CALLER; reframe = False; root = HERE.parents[1] + if '--allow-owned-reframe' in args: + args.remove('--allow-owned-reframe'); reframe = True + for flag in ('--caller', '--repo'): + if flag in args: + i = args.index(flag); val = args[i + 1]; del args[i:i + 2] + if flag == '--caller': + caller = val + else: + root = Path(val) + if len(args) != 2: + print(__doc__); return 2 + B = pk.load(args[0]); W = pk.load(args[1]); rows = [] + t = time.time() + try: + r = accx.check(B, W, caller, reframe); rows.append(('accx', True, '%.1fs' % (time.time() - t), 'ALL CHECKS PASS')) + except accx.Fail as ex: + rows.append(('accx', False, '%.1fs' % (time.time() - t), str(ex))); r = None + t = time.time(); loss = lossless.dual_scan(W)[0] + rows.append(('lossless', not loss, '%.1fs' % (time.time() - t), '%d lossy CLEARs' % len(loss))) + if r is not None: + t = time.time() + try: + good, note = official(root, caller, B, W, r['kappa_new']) + except Exception as ex: # a tool error is a failure, not a skip + good, note = False, str(ex) + if good is not None: + rows.append(('official', good, '%.1fs' % (time.time() - t), note)) + else: + print(' official tools ' + note) + print('\n=== validate summary for %s' % args[1]) + for name, good, dt, note in rows: + print(' %-10s %-5s %8s %s' % (name, 'PASS' if good else 'FAIL', dt, note)) + allok = all(x[1] for x in rows) + print(' OVERALL', 'PASS' if allok else 'FAIL') + return 0 if allok else 1 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))