test(closure): define portable UTF-16 tuple ordering and semantic digest tests

codegen/closure-semantic.mjs

@@ -0,0 +1,37 @@
+import {createHash} from 'node:crypto';
+const tupleKeys=['ecosystem','registry_uri','package_name','package_version','resolved_revision','artifact_digest','source_digest','outer_checksum'];
+const scalar=(v)=>v===undefined||v===null?'':v;
+export const utf16Compare=(a,b)=>a<b?-1:a>b?1:0;
+function canonical(value) {
+  if(Array.isArray(value))return '['+value.map(canonical).join(',')+']';
+  if(value && typeof value==='object')return '{'+Object.keys(value).sort(utf16Compare).map(k=>JSON.stringify(k)+':'+canonical(value[k])).join(',')+'}';
+  if(typeof value==='number' && !Number.isFinite(value))throw Error('nonfinite');
+  return JSON.stringify(value);
+}
+const key=(d)=>tupleKeys.map(k=>scalar(d[k]));
+export function normalizeClosure(closure) {
+  if(!closure||!Array.isArray(closure.dependencies)||!closure.root)throw Error('invalid_closure');
+  const dependencies=closure.dependencies.map(d=>structuredClone(d));
+  for(const o of [closure.root,...dependencies]) {
+    for(const k of Object.keys(o))if(k.endsWith('_digest')||k==='outer_checksum') {
+      if(typeof o[k]!=='string')throw Error('invalid_digest');
+      o[k]=o[k].toLowerCase();
+    }
+  }
+  const root=structuredClone(closure.root);
+  for(const k of Object.keys(root))if(k.endsWith('_digest')||k==='outer_checksum')root[k]=root[k].toLowerCase();
+  const compare=(a,b)=>{
+    const ka=key(a),kb=key(b);
+    for(let i=0;i<ka.length;i++){const c=utf16Compare(ka[i],kb[i]);if(c)return c;}
+    return 0;
+  };
+  dependencies.sort(compare);
+  for(let i=1;i<dependencies.length;i++)if(compare(dependencies[i-1],dependencies[i])===0)throw Error('duplicate_dependency');
+  if(dependencies.some(d=>compare(d,root)===0))throw Error('root_repeated');
+  const out={...closure,root,dependencies};
+  delete out.closure_digest;
+  if(typeof out.dependency_lock_digest==='string')out.dependency_lock_digest=out.dependency_lock_digest.toLowerCase();
+  const preimage=canonical(out);
+  const digest=createHash('sha256').update(Buffer.from(preimage,'utf8')).digest('hex');
+  return {preimage,digest,normalized:{...out,closure_digest:digest}};
+}

codegen/closure-semantic.test.mjs

@@ -0,0 +1,37 @@
+import assert from 'node:assert/strict';
+import {test} from 'node:test';
+import {readFileSync} from 'node:fs';
+import {normalizeClosure,utf16Compare} from './closure-semantic.mjs';
+const fixture=JSON.parse(readFileSync(new URL('../validation/package-security/instances/PackageDependencyClosure/valid/hex-closure.json',import.meta.url),'utf8'));
+test('UTF-16 collation differs from UTF-8 for astral versus BMP private-use',()=>{
+ assert.equal(utf16Compare('\u{10000}','\uE000'),-1);
+ const a=structuredClone(fixture);
+ a.dependencies[0].package_name='\uE000';
+ a.dependencies[1].package_name='\u{10000}';
+ const one=normalizeClosure(a);
+ const reverse=normalizeClosure({...a,dependencies:[...a.dependencies].reverse()});
+ assert.equal(one.digest,reverse.digest);
+ assert.equal(one.preimage,reverse.preimage);
+ assert.equal(one.normalized.dependencies[0].package_name,'\u{10000}');
+ assert.equal(one.digest,'84efe3277da20763fbc5134d9568390493d6b4f7faffd02e4f47e66302af68a4');
+});
+test('duplicate and root identities fail instead of disappearing',()=>{
+ const duplicate={...fixture,dependencies:[fixture.dependencies[0],structuredClone(fixture.dependencies[0])]};
+ assert.throws(()=>normalizeClosure(duplicate),/duplicate_dependency/);
+ const root={...fixture,dependencies:[fixture.root]};
+ assert.throws(()=>normalizeClosure(root),/root_repeated/);
+});
+test('lock and resolver metadata are in the semantic digest',()=>{
+ const a=normalizeClosure(fixture);
+ assert.notEqual(a.digest,normalizeClosure({...fixture,resolver_version:'9.9.9'}).digest);
+ assert.notEqual(a.digest,normalizeClosure({...fixture,dependency_lock_digest:'f'.repeat(64)}).digest);
+});
+
+test('published Hex golden preimage is byte-for-byte JCS and SHA-256 matched',()=>{
+ const expected=readFileSync(new URL('../validation/package-security/golden/hex-closure.canonical-preimage.json',import.meta.url),'utf8').trimEnd();
+ const receipt=normalizeClosure(fixture);
+ assert.equal(receipt.digest,'2f704cfa7c9f59894d97d4b2552e7dec0e6bdee6852a70b1dca8cd50b9e78684');
+ assert.equal(receipt.digest,fixture.closure_digest);
+ assert.equal(receipt.preimage,expected);
+ assert.equal(Buffer.byteLength(receipt.preimage,'utf8'),1550);
+});

validation/package-security/README.md

@@ -30,7 +30,7 @@ This lane models evidence and admission receipts; it does not make a scanner or
 Producers and consumers must recompute `closure_digest`; it is not trusted merely because it is present. The v1 digest is SHA-256 over RFC 8785 JCS serialization of the closure object **without** the `closure_digest` member. Before serialization:
 
 1. normalize hexadecimal digests to lowercase;
-2. sort `dependencies` by the tuple `(ecosystem, registry_uri-or-empty, package_name, package_version, resolved_revision, artifact_digest, source_digest, outer_checksum-or-empty)`;
+2. sort `dependencies` by the tuple `(ecosystem, registry_uri-or-empty, package_name, package_version, resolved_revision, artifact_digest, source_digest, outer_checksum-or-empty)` using **lexicographic UTF-16 code-unit order** for each string, ascending, with no locale collation or Unicode normalization; for example U+10000 sorts before U+E000 (unlike UTF-8 byte order);
 3. reject duplicate tuples rather than silently deduplicating them;
 4. require the root artifact to be separate from, and not repeated in, `dependencies`;
 5. retain the exact `dependency_lock_digest`, `resolver_id`, and `resolver_version` in the signed/hashed payload.
@@ -39,6 +39,19 @@ A consumer must treat a closure as changed if the lock digest, resolver identity
 
 For Hex closures, each member carries the registry `outer_checksum` plus independent Zed artifact/source digests. A closure made only from package names and versions is insufficient for safety reuse.
 
+
+The authoritative semantic golden vector is
+`instances/PackageDependencyClosure/valid/hex-closure.json`: its
+`closure_digest` equals SHA-256
+`2f704cfa7c9f59894d97d4b2552e7dec0e6bdee6852a70b1dca8cd50b9e78684`.
+The **exact UTF-8 canonical preimage**, without `closure_digest`, is
+`golden/hex-closure.canonical-preimage.json` (1550 bytes excluding the
+file's one trailing newline). Tests exercise reversed input, duplicate/root
+repetition, resolver and lock drift, and an astral-versus-BMP Unicode vector.
+These are behavior/conformance fixtures and **not** an additional JSON Schema
+or TypeSpec authority. Downstream Rust, Dart and TypeScript implementations
+must verify the same bytes and digests independently before promotion.
+
 ## Repository boundary
 
 These contracts are portable public validation shapes. They do not contain registry credentials, network policy, deployment topology, database connection details, or executable ORM models.

validation/package-security/golden/hex-closure.canonical-preimage.json

@@ -0,0 +1 @@
+{"dependencies":[{"artifact_digest":"2222222222222222222222222222222222222222222222222222222222222222","ecosystem":"hex","outer_checksum":"3333333333333333333333333333333333333333333333333333333333333333","package_name":"gleam_json","package_version":"3.0.2","registry_uri":"https://repo.hex.pm","resolved_revision":"3.0.2","source_digest":"1111111111111111111111111111111111111111111111111111111111111111","source_uri":"https://github.com/gleam-lang/json"},{"artifact_digest":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","ecosystem":"hex","outer_checksum":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","package_name":"gleam_stdlib","package_version":"0.62.1","registry_uri":"https://repo.hex.pm","resolved_revision":"0.62.1","source_digest":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","source_uri":"https://github.com/gleam-lang/stdlib"}],"dependency_lock_digest":"4444444444444444444444444444444444444444444444444444444444444444","format":"zed-package-dependency-closure-v1","resolver_id":"zed-hex-resolver","resolver_version":"1.0.0","root":{"artifact_digest":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","ecosystem":"hex","outer_checksum":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","package_name":"beam_app","package_version":"1.0.0","registry_uri":"https://repo.hex.pm","resolved_revision":"1.0.0","source_digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","source_uri":"https://github.com/example/beam_app"}}

validation/package-security/instances/PackageDependencyClosure/valid/hex-closure.json

@@ -36,7 +36,7 @@
     }
   ],
   "dependency_lock_digest": "4444444444444444444444444444444444444444444444444444444444444444",
-  "closure_digest": "5555555555555555555555555555555555555555555555555555555555555555",
+  "closure_digest": "2f704cfa7c9f59894d97d4b2552e7dec0e6bdee6852a70b1dca8cd50b9e78684",
   "resolver_id": "zed-hex-resolver",
   "resolver_version": "1.0.0"
 }