Optimize native exact-tree materialization

native/Fixtures/Git/README.md

@@ -5,3 +5,60 @@ rename/copy/delete/add/binary files, hostile newline/tab/HTML names, dirty
 working trees, symlink roots, SHA-256 repositories, and the 1 MiB/8 MiB/2,000
 file caps. Tests construct repositories under temporary directories so no
 fixture writes occur in a reviewed repository.
+
+## Exact-tree materialization benchmark
+
+Run `native/Fixtures/Git/run-benchmark.sh` from any directory. It creates the
+RTC-301 101-file/5,000-line repository in a private temporary directory, builds
+RTCGit with release optimization, performs two warm-ups and 20 measured
+materializations by default, and prints one schema-versioned JSON record. The
+measurement starts after exact revision resolution and ends with the complete
+diff manifest. Environment variables can change the iteration counts:
+
+- `RTC_GIT_BENCHMARK_WARMUPS`
+- `RTC_GIT_BENCHMARK_ITERATIONS`
+- `RTC_GIT_ENFORCE_REFERENCE_BUDGET=1` to enforce the 2,000 ms p95 wall budget
+  on a machine explicitly standing in for the reference host
+
+The 2,000 ms wall gate applies automatically only on the stated reference
+hardware, an Apple M1 Mac with 16 GiB RAM. Other hosts report but do not enforce
+wall time. Every host enforces at most 110 Git processes per iteration, the
+stable structural regression gate for the measured cause.
+
+### RTC-301 causal record (2026-09-03)
+
+Measurements used an Apple M5 MacBook Pro with 24 GiB RAM, macOS 26.1,
+Swift 6.2.1, Apple Git 2.50.1, release builds, two warm-ups, and 20 iterations.
+Variance is population variance in ms².
+
+| implementation                                    |   p50 ms |   p95 ms | variance ms² | Git processes/iteration |
+| ------------------------------------------------- | -------: | -------: | -----------: | ----------------------: |
+| original per-file reads                           | 7,226.99 | 8,926.56 |   326,962.87 |                     611 |
+| batched metadata/blob counterfactual              | 1,516.96 | 1,555.46 |       590.26 |                     109 |
+| rebased batched reads plus four-patch concurrency | 1,299.24 | 1,428.05 |     5,466.70 |                     109 |
+
+The earliest representative-path divergence was process amplification after
+the single name-status read: each of 101 files launched two size reads, one
+numstat, one patch, and two content reads. Each original iteration contained
+404 `cat-file`, 102 metadata `diff`, 101 patch `diff`, and four `rev-parse`
+processes. Across the original 20 measured runs, those 12,220 launches
+accumulated about 140.3 seconds inside Git while wall time totaled about 148.1
+seconds. The roughly 5.3% residual covered Swift parsing, allocation, line
+counting, and context hashing, so those were not the first-order divergence.
+The source audit also found no working-tree reads or repository-content
+execution: patch parsing and hashes consume only bounded Git output. The new
+blob reader caps each retained batch at 4 MB, and patch concurrency is fixed at
+four.
+
+Batching size/content reads and numstats was the smallest counterfactual; its
+83% p95 reduction confirms launch volume as the leading cause. The hypothesis
+would have been disproved if reducing launches from 611 to 109 had left latency
+materially unchanged.
+
+The same final build on the ordinary six-file fixture measured p50 111.18 ms,
+p95 116.70 ms, and variance 7.53 ms². The existing Node baseline method was
+also run five times on this checkout and representative fixture: p50 3,680 ms,
+p95 3,788 ms, variance 4,970.56 ms² (Node 22.23.1). The shipped historical Node
+reference remains 1,685 ms on its recorded Apple Silicon host; build cache,
+host generation, fixture path depth, and current-checkout changes make these
+cross-run values characterization rather than a replacement baseline.

native/Fixtures/Git/RTCGitBenchmark.swift

@@ -0,0 +1,185 @@
+import Darwin
+import Foundation
+import RTCContracts
+import RTCGit
+
+private let referenceBudgetMilliseconds = 2_000.0
+private let maximumProcessesPerIteration = 110
+
+private actor MeasuringRunner: GitBatchProcessRunning {
+    private let system = SystemGitProcessRunner()
+    private var counts: [String: Int] = [:]
+
+    func run(repository: String, arguments: [String], environment: [String: String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult {
+        try await measured(arguments) {
+            try await system.run(repository: repository, arguments: arguments, environment: environment, outputLimit: outputLimit, timeout: timeout)
+        }
+    }
+
+    func runBatch(repository: String, arguments: [String], standardInput: Data, environment: [String: String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult {
+        try await measured(arguments) {
+            try await system.runBatch(repository: repository, arguments: arguments, standardInput: standardInput, environment: environment, outputLimit: outputLimit, timeout: timeout)
+        }
+    }
+
+    private func measured(_ arguments: [String], operation: () async throws -> GitProcessResult) async throws -> GitProcessResult {
+        let command = arguments.first == "-c" ? arguments.dropFirst(2).first ?? "unknown" : arguments.first ?? "unknown"
+        let result = try await operation()
+        counts[command, default: 0] += 1
+        return result
+    }
+
+    func snapshot() -> [String: Int] { counts }
+}
+
+@main
+private enum RTCGitBenchmark {
+    static func main() async throws {
+        let warmups = integerEnvironment("RTC_GIT_BENCHMARK_WARMUPS", default: 2)
+        let iterations = integerEnvironment("RTC_GIT_BENCHMARK_ITERATIONS", default: 20)
+        let fixture = try RepresentativeFixture()
+        defer { fixture.remove() }
+
+        let runner = MeasuringRunner()
+        let engine = ExactGitEngine(runner: runner)
+        let revision = try await engine.resolveRevision(repositoryPath: fixture.repository.path, base: fixture.base, head: fixture.head)
+        for _ in 0..<warmups { _ = try await engine.materialize(revision) }
+        let countsBefore = await runner.snapshot()
+
+        var samples: [Double] = []
+        for _ in 0..<iterations {
+            let started = ContinuousClock.now
+            let manifest = try await engine.materialize(revision)
+            precondition(manifest.files.count == 101)
+            samples.append(milliseconds(started.duration(to: .now)))
+        }
+        let countsAfter = await runner.snapshot()
+        let measuredCounts = Dictionary(uniqueKeysWithValues: countsAfter.map { key, value in
+            (key, value - (countsBefore[key] ?? 0))
+        })
+        let processCount = measuredCounts.values.reduce(0, +)
+        let processRegressionPassed = processCount <= maximumProcessesPerIteration * iterations
+
+        let sorted = samples.sorted()
+        let p50 = percentile(0.50, sorted)
+        let p95 = percentile(0.95, sorted)
+        let mean = samples.reduce(0, +) / Double(samples.count)
+        let variance = samples.reduce(0) { $0 + pow($1 - mean, 2) } / Double(samples.count)
+        let chip = sysctl("machdep.cpu.brand_string")
+        let memoryBytes = Int64(sysctl("hw.memsize")) ?? 0
+        let referenceHardwareMatched = chip.contains("Apple M1") && memoryBytes == 16 * 1_024 * 1_024 * 1_024
+        let explicitlyEnforced = ProcessInfo.processInfo.environment["RTC_GIT_ENFORCE_REFERENCE_BUDGET"] == "1"
+        let wallBudgetEnforced = referenceHardwareMatched || explicitlyEnforced
+        let wallBudgetPassed = !wallBudgetEnforced || p95 <= referenceBudgetMilliseconds
+
+        let output: [String: Any] = [
+            "schemaVersion": 1,
+            "workload": ["changedFiles": 101, "largeFileChangedLines": 5_000],
+            "measurementBoundary": "resolved exact revision through materialized diff artifacts",
+            "buildMode": "release (-O)",
+            "warmups": warmups,
+            "iterations": iterations,
+            "samplesMs": samples,
+            "p50Ms": p50,
+            "p95Ms": p95,
+            "meanMs": mean,
+            "varianceMs2": variance,
+            "processCounts": measuredCounts,
+            "machine": [
+                "chip": chip,
+                "memoryBytes": memoryBytes,
+                "os": ProcessInfo.processInfo.operatingSystemVersionString,
+            ],
+            "gate": [
+                "referenceHardware": "baseline Apple M1 with 16 GiB RAM",
+                "referenceP95BudgetMs": referenceBudgetMilliseconds,
+                "referenceHardwareMatched": referenceHardwareMatched,
+                "wallBudgetEnforced": wallBudgetEnforced,
+                "wallBudgetPassed": wallBudgetPassed,
+                "maximumProcessesPerIteration": maximumProcessesPerIteration,
+                "processRegressionPassed": processRegressionPassed,
+            ],
+        ]
+        print(String(decoding: try JSONSerialization.data(withJSONObject: output, options: [.sortedKeys]), as: UTF8.self))
+        if !processRegressionPassed || !wallBudgetPassed { exit(1) }
+    }
+
+    private static func integerEnvironment(_ name: String, default defaultValue: Int) -> Int {
+        guard let raw = ProcessInfo.processInfo.environment[name], let value = Int(raw), value > 0 else { return defaultValue }
+        return value
+    }
+
+    private static func milliseconds(_ duration: Duration) -> Double {
+        let components = duration.components
+        return Double(components.seconds) * 1_000 + Double(components.attoseconds) / 1e15
+    }
+
+    private static func percentile(_ value: Double, _ sorted: [Double]) -> Double {
+        sorted[min(sorted.count - 1, Int(ceil(Double(sorted.count) * value)) - 1)]
+    }
+
+    private static func sysctl(_ name: String) -> String {
+        let process = Process(), output = Pipe()
+        process.executableURL = URL(fileURLWithPath: "/usr/sbin/sysctl")
+        process.arguments = ["-n", name]
+        process.standardOutput = output
+        process.standardError = FileHandle.nullDevice
+        do { try process.run(); process.waitUntilExit() } catch { return "unknown" }
+        guard process.terminationStatus == 0 else { return "unknown" }
+        return String(decoding: output.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self).trimmingCharacters(in: .whitespacesAndNewlines)
+    }
+}
+
+private final class RepresentativeFixture {
+    let repository: URL
+    let base: String
+    let head: String
+
+    init() throws {
+        repository = FileManager.default.temporaryDirectory.appendingPathComponent("RTCGitBenchmark-\(UUID().uuidString)", isDirectory: true)
+        try FileManager.default.createDirectory(at: repository, withIntermediateDirectories: false)
+        do {
+            try Self.git(repository, ["init", "-q", "--initial-branch=main"])
+            let deep = (0..<20).reduce(repository) { partial, index in
+                partial.appendingPathComponent("level-\(String(format: "%02d", index))", isDirectory: true)
+            }
+            try FileManager.default.createDirectory(at: deep, withIntermediateDirectories: true)
+            try Self.writeFiles(repository: repository, in: deep, increment: 0)
+            try Self.git(repository, ["add", "--all"])
+            try Self.git(repository, ["-c", "user.name=Read the Code Performance Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "representative base"])
+            base = try Self.git(repository, ["rev-parse", "HEAD"])
+            try Self.writeFiles(repository: repository, in: deep, increment: 1)
+            try Self.git(repository, ["add", "--all"])
+            try Self.git(repository, ["-c", "user.name=Read the Code Performance Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "representative head"])
+            head = try Self.git(repository, ["rev-parse", "HEAD"])
+        } catch {
+            try? FileManager.default.removeItem(at: repository)
+            throw error
+        }
+    }
+
+    func remove() { try? FileManager.default.removeItem(at: repository) }
+
+    private static func writeFiles(repository: URL, in deep: URL, increment: Int) throws {
+        for index in 0..<100 {
+            let path = deep.appendingPathComponent("component-with-a-stable-long-name-\(String(format: "%04d", index)).ts")
+            try Data("export const value\(index) = \(index + increment);\n".utf8).write(to: path)
+        }
+        let lines = (0..<5_000).map { "export const line\($0) = \($0 + increment);" }.joined(separator: "\n") + "\n"
+        try Data(lines.utf8).write(to: repository.appendingPathComponent("large-diff.ts"))
+    }
+
+    @discardableResult
+    private static func git(_ repository: URL, _ arguments: [String]) throws -> String {
+        let process = Process(), output = Pipe(), error = Pipe()
+        process.executableURL = URL(fileURLWithPath: SystemGitProcessRunner.executable)
+        process.arguments = ["-C", repository.path] + arguments
+        process.standardOutput = output
+        process.standardError = error
+        try process.run(); process.waitUntilExit()
+        guard process.terminationStatus == 0 else {
+            throw NSError(domain: "RTCGitBenchmark", code: Int(process.terminationStatus), userInfo: [NSLocalizedDescriptionKey: String(decoding: error.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self)])
+        }
+        return String(decoding: output.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self).trimmingCharacters(in: .whitespacesAndNewlines)
+    }
+}

native/Fixtures/Git/run-benchmark.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -eu
+
+repository_root=$(CDPATH= cd -- "$(dirname "$0")/../../.." && pwd -P)
+swift build --package-path "$repository_root/native" -c release --target RTCGit >&2
+build_dir=$(swift build --package-path "$repository_root/native" -c release --show-bin-path)
+benchmark_binary=$(mktemp "${TMPDIR:-/tmp}/rtc-git-benchmark.XXXXXX")
+trap 'rm -f "$benchmark_binary"' EXIT HUP INT TERM
+
+swiftc -O -parse-as-library \
+    "$repository_root/native/Fixtures/Git/RTCGitBenchmark.swift" \
+    "$build_dir/RTCGit.build/GitEngine.swift.o" \
+    "$build_dir/RTCContracts.build/Contracts.swift.o" \
+    "$build_dir/RTCContracts.build/JSONPreflight.swift.o" \
+    -I "$build_dir/Modules" \
+    -o "$benchmark_binary"
+"$benchmark_binary"