diff --git a/native/Fixtures/Git/README.md b/native/Fixtures/Git/README.md index fd6cb94..0db8870 100644 --- a/native/Fixtures/Git/README.md +++ b/native/Fixtures/Git/README.md @@ -5,3 +5,60 @@ rename/copy/delete/add/binary files, hostile newline/tab/HTML names, dirty working trees, symlink roots, SHA-256 repositories, and the 1 MiB/8 MiB/2,000 file caps. Tests construct repositories under temporary directories so no fixture writes occur in a reviewed repository. + +## Exact-tree materialization benchmark + +Run `native/Fixtures/Git/run-benchmark.sh` from any directory. It creates the +RTC-301 101-file/5,000-line repository in a private temporary directory, builds +RTCGit with release optimization, performs two warm-ups and 20 measured +materializations by default, and prints one schema-versioned JSON record. The +measurement starts after exact revision resolution and ends with the complete +diff manifest. Environment variables can change the iteration counts: + +- `RTC_GIT_BENCHMARK_WARMUPS` +- `RTC_GIT_BENCHMARK_ITERATIONS` +- `RTC_GIT_ENFORCE_REFERENCE_BUDGET=1` to enforce the 2,000 ms p95 wall budget + on a machine explicitly standing in for the reference host + +The 2,000 ms wall gate applies automatically only on the stated reference +hardware, an Apple M1 Mac with 16 GiB RAM. Other hosts report but do not enforce +wall time. Every host enforces at most 110 Git processes per iteration, the +stable structural regression gate for the measured cause. + +### RTC-301 causal record (2026-09-03) + +Measurements used an Apple M5 MacBook Pro with 24 GiB RAM, macOS 26.1, +Swift 6.2.1, Apple Git 2.50.1, release builds, two warm-ups, and 20 iterations. +Variance is population variance in ms². + +| implementation | p50 ms | p95 ms | variance ms² | Git processes/iteration | +| ------------------------------------------------- | -------: | -------: | -----------: | ----------------------: | +| original per-file reads | 7,226.99 | 8,926.56 | 326,962.87 | 611 | +| batched metadata/blob counterfactual | 1,516.96 | 1,555.46 | 590.26 | 109 | +| rebased batched reads plus four-patch concurrency | 1,299.24 | 1,428.05 | 5,466.70 | 109 | + +The earliest representative-path divergence was process amplification after +the single name-status read: each of 101 files launched two size reads, one +numstat, one patch, and two content reads. Each original iteration contained +404 `cat-file`, 102 metadata `diff`, 101 patch `diff`, and four `rev-parse` +processes. Across the original 20 measured runs, those 12,220 launches +accumulated about 140.3 seconds inside Git while wall time totaled about 148.1 +seconds. The roughly 5.3% residual covered Swift parsing, allocation, line +counting, and context hashing, so those were not the first-order divergence. +The source audit also found no working-tree reads or repository-content +execution: patch parsing and hashes consume only bounded Git output. The new +blob reader caps each retained batch at 4 MB, and patch concurrency is fixed at +four. + +Batching size/content reads and numstats was the smallest counterfactual; its +83% p95 reduction confirms launch volume as the leading cause. The hypothesis +would have been disproved if reducing launches from 611 to 109 had left latency +materially unchanged. + +The same final build on the ordinary six-file fixture measured p50 111.18 ms, +p95 116.70 ms, and variance 7.53 ms². The existing Node baseline method was +also run five times on this checkout and representative fixture: p50 3,680 ms, +p95 3,788 ms, variance 4,970.56 ms² (Node 22.23.1). The shipped historical Node +reference remains 1,685 ms on its recorded Apple Silicon host; build cache, +host generation, fixture path depth, and current-checkout changes make these +cross-run values characterization rather than a replacement baseline. diff --git a/native/Fixtures/Git/RTCGitBenchmark.swift b/native/Fixtures/Git/RTCGitBenchmark.swift new file mode 100644 index 0000000..c59f43b --- /dev/null +++ b/native/Fixtures/Git/RTCGitBenchmark.swift @@ -0,0 +1,185 @@ +import Darwin +import Foundation +import RTCContracts +import RTCGit + +private let referenceBudgetMilliseconds = 2_000.0 +private let maximumProcessesPerIteration = 110 + +private actor MeasuringRunner: GitBatchProcessRunning { + private let system = SystemGitProcessRunner() + private var counts: [String: Int] = [:] + + func run(repository: String, arguments: [String], environment: [String: String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult { + try await measured(arguments) { + try await system.run(repository: repository, arguments: arguments, environment: environment, outputLimit: outputLimit, timeout: timeout) + } + } + + func runBatch(repository: String, arguments: [String], standardInput: Data, environment: [String: String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult { + try await measured(arguments) { + try await system.runBatch(repository: repository, arguments: arguments, standardInput: standardInput, environment: environment, outputLimit: outputLimit, timeout: timeout) + } + } + + private func measured(_ arguments: [String], operation: () async throws -> GitProcessResult) async throws -> GitProcessResult { + let command = arguments.first == "-c" ? arguments.dropFirst(2).first ?? "unknown" : arguments.first ?? "unknown" + let result = try await operation() + counts[command, default: 0] += 1 + return result + } + + func snapshot() -> [String: Int] { counts } +} + +@main +private enum RTCGitBenchmark { + static func main() async throws { + let warmups = integerEnvironment("RTC_GIT_BENCHMARK_WARMUPS", default: 2) + let iterations = integerEnvironment("RTC_GIT_BENCHMARK_ITERATIONS", default: 20) + let fixture = try RepresentativeFixture() + defer { fixture.remove() } + + let runner = MeasuringRunner() + let engine = ExactGitEngine(runner: runner) + let revision = try await engine.resolveRevision(repositoryPath: fixture.repository.path, base: fixture.base, head: fixture.head) + for _ in 0.. Int { + guard let raw = ProcessInfo.processInfo.environment[name], let value = Int(raw), value > 0 else { return defaultValue } + return value + } + + private static func milliseconds(_ duration: Duration) -> Double { + let components = duration.components + return Double(components.seconds) * 1_000 + Double(components.attoseconds) / 1e15 + } + + private static func percentile(_ value: Double, _ sorted: [Double]) -> Double { + sorted[min(sorted.count - 1, Int(ceil(Double(sorted.count) * value)) - 1)] + } + + private static func sysctl(_ name: String) -> String { + let process = Process(), output = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/sbin/sysctl") + process.arguments = ["-n", name] + process.standardOutput = output + process.standardError = FileHandle.nullDevice + do { try process.run(); process.waitUntilExit() } catch { return "unknown" } + guard process.terminationStatus == 0 else { return "unknown" } + return String(decoding: output.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self).trimmingCharacters(in: .whitespacesAndNewlines) + } +} + +private final class RepresentativeFixture { + let repository: URL + let base: String + let head: String + + init() throws { + repository = FileManager.default.temporaryDirectory.appendingPathComponent("RTCGitBenchmark-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: repository, withIntermediateDirectories: false) + do { + try Self.git(repository, ["init", "-q", "--initial-branch=main"]) + let deep = (0..<20).reduce(repository) { partial, index in + partial.appendingPathComponent("level-\(String(format: "%02d", index))", isDirectory: true) + } + try FileManager.default.createDirectory(at: deep, withIntermediateDirectories: true) + try Self.writeFiles(repository: repository, in: deep, increment: 0) + try Self.git(repository, ["add", "--all"]) + try Self.git(repository, ["-c", "user.name=Read the Code Performance Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "representative base"]) + base = try Self.git(repository, ["rev-parse", "HEAD"]) + try Self.writeFiles(repository: repository, in: deep, increment: 1) + try Self.git(repository, ["add", "--all"]) + try Self.git(repository, ["-c", "user.name=Read the Code Performance Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "representative head"]) + head = try Self.git(repository, ["rev-parse", "HEAD"]) + } catch { + try? FileManager.default.removeItem(at: repository) + throw error + } + } + + func remove() { try? FileManager.default.removeItem(at: repository) } + + private static func writeFiles(repository: URL, in deep: URL, increment: Int) throws { + for index in 0..<100 { + let path = deep.appendingPathComponent("component-with-a-stable-long-name-\(String(format: "%04d", index)).ts") + try Data("export const value\(index) = \(index + increment);\n".utf8).write(to: path) + } + let lines = (0..<5_000).map { "export const line\($0) = \($0 + increment);" }.joined(separator: "\n") + "\n" + try Data(lines.utf8).write(to: repository.appendingPathComponent("large-diff.ts")) + } + + @discardableResult + private static func git(_ repository: URL, _ arguments: [String]) throws -> String { + let process = Process(), output = Pipe(), error = Pipe() + process.executableURL = URL(fileURLWithPath: SystemGitProcessRunner.executable) + process.arguments = ["-C", repository.path] + arguments + process.standardOutput = output + process.standardError = error + try process.run(); process.waitUntilExit() + guard process.terminationStatus == 0 else { + throw NSError(domain: "RTCGitBenchmark", code: Int(process.terminationStatus), userInfo: [NSLocalizedDescriptionKey: String(decoding: error.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self)]) + } + return String(decoding: output.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self).trimmingCharacters(in: .whitespacesAndNewlines) + } +} diff --git a/native/Fixtures/Git/run-benchmark.sh b/native/Fixtures/Git/run-benchmark.sh new file mode 100755 index 0000000..d5d163c --- /dev/null +++ b/native/Fixtures/Git/run-benchmark.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname "$0")/../../.." && pwd -P) +swift build --package-path "$repository_root/native" -c release --target RTCGit >&2 +build_dir=$(swift build --package-path "$repository_root/native" -c release --show-bin-path) +benchmark_binary=$(mktemp "${TMPDIR:-/tmp}/rtc-git-benchmark.XXXXXX") +trap 'rm -f "$benchmark_binary"' EXIT HUP INT TERM + +swiftc -O -parse-as-library \ + "$repository_root/native/Fixtures/Git/RTCGitBenchmark.swift" \ + "$build_dir/RTCGit.build/GitEngine.swift.o" \ + "$build_dir/RTCContracts.build/Contracts.swift.o" \ + "$build_dir/RTCContracts.build/JSONPreflight.swift.o" \ + -I "$build_dir/Modules" \ + -o "$benchmark_binary" +"$benchmark_binary" diff --git a/native/Sources/RTCGit/GitEngine.swift b/native/Sources/RTCGit/GitEngine.swift index e38054f..4fad78e 100644 --- a/native/Sources/RTCGit/GitEngine.swift +++ b/native/Sources/RTCGit/GitEngine.swift @@ -28,6 +28,13 @@ public protocol GitProcessRunning: Sendable { func run(repository: String, arguments: [String], environment: [String: String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult } +/// An opt-in extension for Git modes that consume a bounded request on standard input. +/// Keeping this separate preserves simple process doubles while production can use +/// `cat-file --batch` without weakening the fixed executable and environment policy. +public protocol GitBatchProcessRunning: GitProcessRunning { + func runBatch(repository: String, arguments: [String], standardInput: Data, environment: [String: String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult +} + public extension GitProcessRunning { func run(repository: String, arguments: [String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult { try await run(repository: repository, arguments: arguments, environment: [:], outputLimit: outputLimit, timeout: timeout) @@ -42,11 +49,19 @@ private final class ProcessBox: @unchecked Sendable { func terminate() { lock.lock(); if let pid { kill(pid, SIGTERM); self.pid = nil }; lock.unlock() } } -public struct SystemGitProcessRunner: GitProcessRunning { +public struct SystemGitProcessRunner: GitBatchProcessRunning { public static let executable = "/usr/bin/git" public init() {} public func run(repository: String, arguments: [String], environment: [String: String] = [:], outputLimit: Int = RTCConstants.maxPatchBytesTotal + 128_000, timeout: Duration = .seconds(30)) async throws -> GitProcessResult { + try await run(repository: repository, arguments: arguments, standardInput: nil, environment: environment, outputLimit: outputLimit, timeout: timeout) + } + + public func runBatch(repository: String, arguments: [String], standardInput: Data, environment: [String: String] = [:], outputLimit: Int = RTCConstants.maxPatchBytesTotal + 128_000, timeout: Duration = .seconds(30)) async throws -> GitProcessResult { + try await run(repository: repository, arguments: arguments, standardInput: standardInput, environment: environment, outputLimit: outputLimit, timeout: timeout) + } + + private func run(repository: String, arguments: [String], standardInput: Data?, environment: [String: String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult { let box = ProcessBox() return try await withTaskCancellationHandler(operation: { try await withCheckedThrowingContinuation { continuation in @@ -65,12 +80,19 @@ public struct SystemGitProcessRunner: GitProcessRunning { // Git tree paths are opaque bytes, so launch their UTF-8 bytes directly. let argv = [Self.executable, "-C", repository] + arguments let envp = policyEnvironment.sorted { $0.key < $1.key }.map { "\($0.key)=\($0.value)" } - let stdoutPipe = Pipe(), stderrPipe = Pipe() + let stdoutPipe = Pipe(), stderrPipe = Pipe(), stdinPipe = standardInput.map { _ in Pipe() } do { - let pid = try spawn(arguments: argv, environment: envp, stdout: stdoutPipe.fileHandleForWriting.fileDescriptor, stderr: stderrPipe.fileHandleForWriting.fileDescriptor) + let pid = try spawn( + arguments: argv, + environment: envp, + standardInput: stdinPipe.map { ($0.fileHandleForReading.fileDescriptor, $0.fileHandleForWriting.fileDescriptor) }, + stdout: stdoutPipe.fileHandleForWriting.fileDescriptor, + stderr: stderrPipe.fileHandleForWriting.fileDescriptor + ) box.set(pid) try? stdoutPipe.fileHandleForWriting.close() try? stderrPipe.fileHandleForWriting.close() + try? stdinPipe?.fileHandleForReading.close() let state = OutputState(limit: outputLimit, continuation: continuation, process: box) // A pipe has one reader. Mixing a readability callback with a final // drain after waitpid races those reads and can lose Git output under @@ -84,12 +106,23 @@ public struct SystemGitProcessRunner: GitProcessRunning { state.processExited(status: waited == pid ? status : -1) } Task { - do { try await Task.sleep(for: timeout); if !Task.isCancelled, state.timeout() { box.terminate() } } + do { try await Task.sleep(for: timeout); if !Task.isCancelled { _ = state.timeout() } } catch { } } + if let standardInput, let stdinPipe { + do { + try stdinPipe.fileHandleForWriting.write(contentsOf: standardInput) + try stdinPipe.fileHandleForWriting.close() + } catch { + try? stdinPipe.fileHandleForWriting.close() + box.terminate() + } + } } catch { try? stdoutPipe.fileHandleForWriting.close() try? stderrPipe.fileHandleForWriting.close() + try? stdinPipe?.fileHandleForReading.close() + try? stdinPipe?.fileHandleForWriting.close() continuation.resume(throwing: GitEngineError.gitFailed) return } @@ -97,7 +130,7 @@ public struct SystemGitProcessRunner: GitProcessRunning { }, onCancel: { box.terminate() }) } - private func spawn(arguments: [String], environment: [String], stdout: Int32, stderr: Int32) throws -> pid_t { + private func spawn(arguments: [String], environment: [String], standardInput: (read: Int32, write: Int32)?, stdout: Int32, stderr: Int32) throws -> pid_t { guard arguments.allSatisfy({ !$0.utf8.contains(0) }), environment.allSatisfy({ !$0.utf8.contains(0) }) else { throw GitEngineError.gitFailed } @@ -120,6 +153,12 @@ public struct SystemGitProcessRunner: GitProcessRunning { posix_spawn_file_actions_addclose(&actions, stdout) == 0, posix_spawn_file_actions_addclose(&actions, stderr) == 0 else { throw GitEngineError.gitFailed } + if let standardInput { + guard posix_spawn_file_actions_adddup2(&actions, standardInput.read, STDIN_FILENO) == 0, + posix_spawn_file_actions_addclose(&actions, standardInput.read) == 0, + posix_spawn_file_actions_addclose(&actions, standardInput.write) == 0 + else { throw GitEngineError.gitFailed } + } var pid: pid_t = 0 let result = argv.withUnsafeMutableBufferPointer { argvBuffer in envp.withUnsafeMutableBufferPointer { envpBuffer in @@ -167,7 +206,14 @@ private final class OutputState: @unchecked Sendable { if stream == .stdout { stdout.append(data) } else { stderr.append(data) } if stdout.count + stderr.count > limit { completed=true; process.terminate(); continuation.resume(throwing: GitEngineError.outputLimit) } } - func timeout() -> Bool { lock.lock(); defer { lock.unlock() }; guard !completed else { return false }; timedOut=true; return true } + func timeout() -> Bool { + lock.lock(); defer { lock.unlock() } + guard !completed else { return false } + timedOut = true; completed = true + process.terminate() + continuation.resume(throwing: GitEngineError.timedOut) + return true + } func processExited(status: Int32) { lock.lock(); defer { lock.unlock() } processStatus = status @@ -227,28 +273,54 @@ public actor ExactGitEngine: GitService { let evidenceRepository = isolatedRepository.path let names = try await nameStatus(evidenceRepository, base: base, head: head) guard names.count <= RTCConstants.maxFiles else { throw GitEngineError.tooManyFiles } + let statsByChange = try await numstats(evidenceRepository, base: base, head: head, expectedCount: names.count) + let blobs = try await blobMetadata(evidenceRepository, base: base, head: head, changes: names) var files = [DiffArtifact](); var totalPatch = 0 - for change in names { + let work = zip(names, statsByChange).enumerated().map { (index: $0.offset, change: $0.element.0, stats: $0.element.1) } + for start in stride(from: 0, to: work.count, by: 4) { try checkCancellation() - let oldPath = change.oldPath ?? change.path - let oldSize = change.status == .added ? 0 : try await blobSize(evidenceRepository, sha: base, path: oldPath) - let newSize = change.status == .deleted ? 0 : try await blobSize(evidenceRepository, sha: head, path: change.path) - let stats = try await numstat(evidenceRepository, base: base, head: head, paths: [oldPath, change.path]) - let truncated = max(oldSize, newSize) > RTCConstants.maxPatchBytesPerFile - if truncated || stats.binary { - let oldLines = oldSize > RTCConstants.maxPatchBytesPerFile ? nil : try await lineCount(evidenceRepository, sha: base, path: change.status == .added ? nil : oldPath) - let newLines = newSize > RTCConstants.maxPatchBytesPerFile ? nil : try await lineCount(evidenceRepository, sha: head, path: change.status == .deleted ? nil : change.path) - files.append(DiffArtifact(path: change.path, oldPath: change.oldPath, status: stats.binary ? .binary : change.status, additions: stats.additions, deletions: stats.deletions, binary: stats.binary, truncated: truncated, oldLineCount: oldLines, newLineCount: newLines, hunks: [])); continue + let end = min(start + 4, work.count) + let materialized = try await withThrowingTaskGroup(of: MaterializedChange.self) { group in + for item in work[start.. MaterializedChange { + try checkCancellation() + let oldPath = change.oldPath ?? change.path + let oldBlob = change.status == .added ? nil : blobs[BlobRequest(sha: base, path: oldPath)] + let newBlob = change.status == .deleted ? nil : blobs[BlobRequest(sha: head, path: change.path)] + let oldSize = oldBlob?.size ?? 0 + let newSize = newBlob?.size ?? 0 + let truncated = max(oldSize, newSize) > RTCConstants.maxPatchBytesPerFile + if truncated || stats.binary { + let oldLines = oldSize > RTCConstants.maxPatchBytesPerFile ? nil : oldBlob?.lineCount + let newLines = newSize > RTCConstants.maxPatchBytesPerFile ? nil : newBlob?.lineCount + let artifact = DiffArtifact(path: change.path, oldPath: change.oldPath, status: stats.binary ? .binary : change.status, additions: stats.additions, deletions: stats.deletions, binary: stats.binary, truncated: truncated, oldLineCount: oldLines, newLineCount: newLines, hunks: []) + return MaterializedChange(index: index, artifact: artifact, patchBytes: 0) + } + let patch = try await run(repository, ["-c", "core.quotePath=true", "diff", "--find-renames", "--no-ext-diff", "--no-textconv", "--no-color", "--unified=4", base, head, "--", oldPath, change.path], limit: RTCConstants.maxPatchBytesPerFile + 100_000) + let hunks = try parsePatch(String(decoding: patch, as: UTF8.self), path: change.path) + let artifact = DiffArtifact(path: change.path, oldPath: change.oldPath, status: change.status, additions: stats.additions, deletions: stats.deletions, binary: false, truncated: false, oldLineCount: oldBlob?.lineCount, newLineCount: newBlob?.lineCount, hunks: hunks) + return MaterializedChange(index: index, artifact: artifact, patchBytes: patch.count) + } + public func context(_ request: GitContextRequest) async throws -> GitContext { try checkCancellation() let repo = try await resolveRepository(request.revision.repositoryPath) @@ -277,6 +349,10 @@ public actor ExactGitEngine: GitService { private func run(_ repo: String, _ args: [String], limit: Int = RTCConstants.maxPatchBytesTotal + 128_000) async throws -> Data { try await runner.run(repository: repo, arguments: args, environment: [:], outputLimit: limit, timeout: .seconds(30)).stdout } + private func runBatch(_ repo: String, _ args: [String], input: Data, limit: Int) async throws -> Data { + guard let batchRunner = runner as? any GitBatchProcessRunning else { throw GitEngineError.gitFailed } + return try await batchRunner.runBatch(repository: repo, arguments: args, standardInput: input, environment: [:], outputLimit: limit, timeout: .seconds(30)).stdout + } private func resolveRepository(_ path: String) async throws -> String { do { let value = try await run(path, ["rev-parse", "--show-toplevel"], limit: 4_096) @@ -339,13 +415,148 @@ public actor ExactGitEngine: GitService { return sha } catch GitEngineError.gitFailed { throw GitEngineError.invalidRef } } - private struct Change { let status: ChangeStatus; let path: String; let oldPath: String? } + private struct Change: Sendable { let status: ChangeStatus; let path: String; let oldPath: String? } private func nameStatus(_ repo: String, base: String, head: String) async throws -> [Change] { let data = try await run(repo, ["diff", "--name-status", "-z", "--find-renames", "--no-ext-diff", "--no-textconv", base, head, "--"], limit: 2 * 1024 * 1024); let f = String(decoding: data, as: UTF8.self).split(separator: "\0", omittingEmptySubsequences: true).map(String.init); var result=[Change](); var i=0; while i < f.count { let code=f[i]; i += 1; guard i < f.count else { throw GitEngineError.invalidDiff }; if code.first == "R" || code.first == "C" { guard i + 1 < f.count else { throw GitEngineError.invalidDiff }; result.append(Change(status: .renamed, path: f[i+1], oldPath: f[i])); i += 2 } else { let status: ChangeStatus = code.first == "A" ? .added : code.first == "D" ? .deleted : .modified; result.append(Change(status: status, path: f[i], oldPath: nil)); i += 1 } }; return result } - private struct Stats { let additions: Int; let deletions: Int; let binary: Bool } - private func numstat(_ repo: String, base: String, head: String, paths: [String]) async throws -> Stats { let d=try await run(repo,["diff","--numstat","-z","--find-renames","--no-ext-diff","--no-textconv",base,head,"--"]+paths,limit:4096); let p=String(decoding:d,as:UTF8.self).split(separator:"\t",omittingEmptySubsequences:false); guard p.count >= 2 else { return Stats(additions: 0, deletions: 0, binary: false) }; return Stats(additions: Int(p[0]) ?? 0, deletions: Int(p[1]) ?? 0, binary: p[0] == "-" || p[1] == "-") } + private struct Stats: Sendable { let additions: Int; let deletions: Int; let binary: Bool } + private func numstats(_ repo: String, base: String, head: String, expectedCount: Int) async throws -> [Stats] { + let data = try await run(repo, ["diff", "--numstat", "-z", "--find-renames", "--no-ext-diff", "--no-textconv", base, head, "--"], limit: RTCConstants.maxPatchBytesTotal + 128_000) + let parsed = try parseNumstats(data) + guard parsed.count == expectedCount else { throw GitEngineError.invalidDiff } + return parsed + } + + private struct BlobRequest: Hashable, Sendable { let sha: String; let path: String } + private struct BlobMetadata: Sendable { let size: Int; let lineCount: Int? } + private func blobMetadata(_ repo: String, base: String, head: String, changes: [Change]) async throws -> [BlobRequest: BlobMetadata] { + var requests: [BlobRequest] = [] + var seen = Set() + for change in changes { + let oldPath = change.oldPath ?? change.path + if change.status != .added { + let request = BlobRequest(sha: base, path: oldPath) + if seen.insert(request).inserted { requests.append(request) } + } + if change.status != .deleted { + let request = BlobRequest(sha: head, path: change.path) + if seen.insert(request).inserted { requests.append(request) } + } + } + guard !requests.isEmpty else { return [:] } + guard runner is any GitBatchProcessRunning else { + var result: [BlobRequest: BlobMetadata] = [:] + for request in requests { + let size = try await blobSize(repo, sha: request.sha, path: request.path) + let lines = size <= RTCConstants.maxPatchBytesPerFile ? try await lineCount(repo, sha: request.sha, path: request.path) : nil + result[request] = BlobMetadata(size: size, lineCount: lines) + } + return result + } + + let sizeInput = batchInput(requests) + let sizeOutput = try await runBatch(repo, ["cat-file", "--batch-check=%(objecttype) %(objectsize)", "-z"], input: sizeInput, limit: requests.count * 32 + 4_096) + let sizes = try parseBatchSizes(sizeOutput, expectedCount: requests.count) + var result = Dictionary(uniqueKeysWithValues: zip(requests, sizes).map { pair in + (pair.0, BlobMetadata(size: pair.1, lineCount: nil)) + }) + + let eligible = zip(requests, sizes).filter { $0.1 <= RTCConstants.maxPatchBytesPerFile } + var chunk: [(BlobRequest, Int)] = [] + var chunkBytes = 0 + func outputLimit(for values: [(BlobRequest, Int)]) -> Int { + values.reduce(4_096) { $0 + $1.1 + 32 } + } + for value in eligible { + if !chunk.isEmpty && chunkBytes + value.1 > 4_000_000 { + try await fillLineCounts(repo, values: chunk, result: &result, outputLimit: outputLimit(for: chunk)) + chunk.removeAll(keepingCapacity: true); chunkBytes = 0 + try checkCancellation() + } + chunk.append(value); chunkBytes += value.1 + } + if !chunk.isEmpty { try await fillLineCounts(repo, values: chunk, result: &result, outputLimit: outputLimit(for: chunk)) } + return result + } + + private func fillLineCounts(_ repo: String, values: [(BlobRequest, Int)], result: inout [BlobRequest: BlobMetadata], outputLimit: Int) async throws { + let output = try await runBatch(repo, ["cat-file", "--batch=%(objecttype) %(objectsize)", "-z"], input: batchInput(values.map(\.0)), limit: outputLimit) + let blobs = try parseBatchBlobs(output, expectedSizes: values.map(\.1)) + for (value, blob) in zip(values, blobs) { + let lineFeeds = blob.reduce(into: 0) { if $1 == 10 { $0 += 1 } } + let lines = blob.isEmpty ? 0 : lineFeeds + (blob.last == 10 ? 0 : 1) + result[value.0] = BlobMetadata(size: value.1, lineCount: lines) + } + } + + private func batchInput(_ requests: [BlobRequest]) -> Data { + var input = Data() + for request in requests { + input.append(contentsOf: request.sha.utf8); input.append(58) + input.append(contentsOf: request.path.utf8); input.append(0) + } + return input + } private func blobSize(_ repo: String, sha: String, path: String) async throws -> Int { Int(String(decoding: try await run(repo,["cat-file","-s","\(sha):\(path)"],limit:1024),as:UTF8.self).trimmingCharacters(in:.whitespacesAndNewlines)) ?? 0 } private func lineCount(_ repo: String, sha: String, path: String?) async throws -> Int? { guard let path else { return nil }; let d=try await run(repo,["cat-file","blob","\(sha):\(path)"],limit:RTCConstants.maxPatchBytesPerFile+1); guard d.count <= RTCConstants.maxPatchBytesPerFile else { return nil }; let s=String(decoding:d,as:UTF8.self); return s.isEmpty ? 0 : s.split(separator:"\n",omittingEmptySubsequences:false).count - (s.hasSuffix("\n") ? 1 : 0) } private func blobLines(_ repo: String, sha: String, path: String) async throws -> [String] { let d=try await run(repo,["cat-file","blob","\(sha):\(path)"],limit:RTCConstants.maxPatchBytesPerFile+1); var lines=String(decoding:d,as:UTF8.self).components(separatedBy:"\n"); if lines.last == "" { lines.removeLast() }; return lines } + + private func parseNumstats(_ data: Data) throws -> [Stats] { + var index = data.startIndex, result: [Stats] = [] + while index < data.endIndex { + let additions = try readField(data, index: &index, delimiter: 9) + let deletions = try readField(data, index: &index, delimiter: 9) + guard index < data.endIndex else { throw GitEngineError.invalidDiff } + if data[index] == 0 { + index += 1 + _ = try readField(data, index: &index, delimiter: 0) + _ = try readField(data, index: &index, delimiter: 0) + } else { + _ = try readField(data, index: &index, delimiter: 0) + } + let addText = String(decoding: additions, as: UTF8.self) + let deleteText = String(decoding: deletions, as: UTF8.self) + let binary = addText == "-" || deleteText == "-" + guard binary || (Int(addText) != nil && Int(deleteText) != nil) else { throw GitEngineError.invalidDiff } + result.append(Stats(additions: Int(addText) ?? 0, deletions: Int(deleteText) ?? 0, binary: binary)) + } + return result + } + + private func parseBatchSizes(_ data: Data, expectedCount: Int) throws -> [Int] { + var index = data.startIndex, result: [Int] = [] + while index < data.endIndex { + let field = try readField(data, index: &index, delimiter: 10) + let parts = String(decoding: field, as: UTF8.self).split(separator: " ") + guard parts.first == "blob" else { throw GitEngineError.gitFailed } + guard parts.count == 2, let size = Int(parts[1]), size >= 0 else { throw GitEngineError.invalidDiff } + result.append(size) + } + guard result.count == expectedCount else { throw GitEngineError.invalidDiff } + return result + } + + private func parseBatchBlobs(_ data: Data, expectedSizes: [Int]) throws -> [Data] { + var index = data.startIndex, result: [Data] = [] + for expectedSize in expectedSizes { + let field = try readField(data, index: &index, delimiter: 10) + let parts = String(decoding: field, as: UTF8.self).split(separator: " ") + guard parts.first == "blob" else { throw GitEngineError.gitFailed } + guard parts.count == 2, Int(parts[1]) == expectedSize, + expectedSize <= data.distance(from: index, to: data.endIndex) + else { throw GitEngineError.invalidDiff } + let end = data.index(index, offsetBy: expectedSize) + result.append(data[index.. Data { + guard let end = data[index...].firstIndex(of: delimiter) else { throw GitEngineError.invalidDiff } + let field = Data(data[index.. SHA256Digest { diff --git a/native/Tests/RTCGitTests/GitEngineTests.swift b/native/Tests/RTCGitTests/GitEngineTests.swift index e43752c..0c4f3be 100644 --- a/native/Tests/RTCGitTests/GitEngineTests.swift +++ b/native/Tests/RTCGitTests/GitEngineTests.swift @@ -66,6 +66,29 @@ final class GitEngineTests: XCTestCase { } } + func testRealBatchRunnerClosesAfterCheckAndContentRequests() async throws { + let repository = try LargeOutputRepository() + let request = Data("\(repository.head):payload.txt\0".utf8) + let runner = SystemGitProcessRunner() + let check = try await runner.runBatch( + repository: repository.url.path, + arguments: ["cat-file", "--batch-check=%(objecttype) %(objectsize)", "-z"], + standardInput: request, + outputLimit: 1_024, + timeout: .seconds(3) + ) + XCTAssertTrue(check.stdout.starts(with: Data("blob ".utf8))) + let contents = try await runner.runBatch( + repository: repository.url.path, + arguments: ["cat-file", "--batch=%(objecttype) %(objectsize)", "-z"], + standardInput: request, + outputLimit: repository.payload.count + 128, + timeout: .seconds(3) + ) + XCTAssertEqual(contents.stdout.last, 10) + XCTAssertNotNil(contents.stdout.range(of: repository.payload)) + } + func testMalformedNameStatusRemainsInvalidDiff() async throws { let repository = try HostileFilenameRepository() let engine = ExactGitEngine(runner: NameStatusFaultRunner(fault: .malformed)) @@ -106,6 +129,38 @@ final class GitEngineTests: XCTestCase { let digest = SHA256Digest(data: Data("committed tree".utf8)) XCTAssertEqual(digest.hex.count, 64) } + + func testBatchedReadsPreserveHostilePathAndIgnoreDirtyWorktree() async throws { + let fixture = try GitFixture.hostilePath() + defer { fixture.remove() } + let runner = CountingSystemRunner() + let engine = ExactGitEngine(runner: runner) + let revision = try await engine.resolveRevision(repositoryPath: fixture.repository.path, base: fixture.base, head: fixture.head) + await runner.reset() + + let manifest = try await engine.materialize(revision) + XCTAssertEqual(manifest.files.map(\.path), [fixture.hostilePath]) + XCTAssertTrue(manifest.files[0].hunks.flatMap(\.lines).contains { $0.text == "committed object" }) + XCTAssertFalse(manifest.files[0].hunks.flatMap(\.lines).contains { $0.text == "dirty working tree" }) + let counts = await runner.counts() + XCTAssertEqual(counts.batch, 2) + XCTAssertLessThanOrEqual(counts.total, 9) + } + + func testRepresentativeFixtureHasStableProcessCeiling() async throws { + let fixture = try GitFixture.representative() + defer { fixture.remove() } + let runner = CountingSystemRunner() + let engine = ExactGitEngine(runner: runner) + let revision = try await engine.resolveRevision(repositoryPath: fixture.repository.path, base: fixture.base, head: fixture.head) + await runner.reset() + + let manifest = try await engine.materialize(revision) + XCTAssertEqual(manifest.files.count, 101) + let counts = await runner.counts() + XCTAssertEqual(counts.batch, 2) + XCTAssertLessThanOrEqual(counts.total, 110, "process amplification is the stable cross-host performance regression gate") + } } private final class LargeOutputRepository { @@ -255,3 +310,101 @@ private actor RecordingRunner: GitProcessRunning { throw GitEngineError.invalidRef } } + +private actor CountingSystemRunner: GitBatchProcessRunning { + private let runner = SystemGitProcessRunner() + private var processCount = 0 + private var batchCount = 0 + + func run(repository: String, arguments: [String], environment: [String: String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult { + let result = try await runner.run(repository: repository, arguments: arguments, environment: environment, outputLimit: outputLimit, timeout: timeout) + processCount += 1 + return result + } + + func runBatch(repository: String, arguments: [String], standardInput: Data, environment: [String: String], outputLimit: Int, timeout: Duration) async throws -> GitProcessResult { + let result = try await runner.runBatch(repository: repository, arguments: arguments, standardInput: standardInput, environment: environment, outputLimit: outputLimit, timeout: timeout) + processCount += 1; batchCount += 1 + return result + } + + func reset() { processCount = 0; batchCount = 0 } + func counts() -> (total: Int, batch: Int) { (processCount, batchCount) } +} + +private final class GitFixture { + let repository: URL + let base: String + let head: String + let hostilePath: String + + private init(repository: URL, base: String, head: String, hostilePath: String = "") { + self.repository = repository; self.base = base; self.head = head; self.hostilePath = hostilePath + } + + static func hostilePath() throws -> GitFixture { + let repository = try makeRepository() + let path = "tab\tline\n