diff --git a/.oss-scanner/swift/Package.swift b/.oss-scanner/swift/Package.swift index 49240392bd..620fc2365b 100644 --- a/.oss-scanner/swift/Package.swift +++ b/.oss-scanner/swift/Package.swift @@ -15,8 +15,12 @@ let package = Package( .target(name: "TableProDocumentPath"), .target(name: "TableProJavaScriptText"), .target(name: "TableProDamengCore", dependencies: ["TableProSQLGrammar"]), + .target(name: "TableProWireIO"), + .target(name: "TableProCassandraCore"), .testTarget(name: "TableProSQLGrammarTests", dependencies: ["TableProSQLGrammar"]), .testTarget(name: "TableProLogRedactionTests", dependencies: ["TableProLogRedaction"]), - .testTarget(name: "TableProDamengCoreTests", dependencies: ["TableProDamengCore", "TableProSQLGrammar"]) + .testTarget(name: "TableProDamengCoreTests", dependencies: ["TableProDamengCore", "TableProSQLGrammar"]), + .testTarget(name: "TableProWireIOTests", dependencies: ["TableProWireIO"]), + .testTarget(name: "TableProCassandraCoreTests", dependencies: ["TableProCassandraCore"]) ] ) diff --git a/apple/Packages/TableProCore/Package.resolved b/apple/Packages/TableProCore/Package.resolved new file mode 100644 index 0000000000..004f562d0a --- /dev/null +++ b/apple/Packages/TableProCore/Package.resolved @@ -0,0 +1,15 @@ +{ + "originHash" : "cbc6f6a1c7f19cf35f585286c0442eb44227fc47eac1f3525abe32f3c629b3a7", + "pins" : [ + { + "identity" : "zstd", + "kind" : "remoteSourceControl", + "location" : "https://github.com/facebook/zstd", + "state" : { + "revision" : "f8745da6ff1ad1e7bab384bd1f9d742439278e99", + "version" : "1.5.7" + } + } + ], + "version" : 3 +} diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 4369056b17..23c933ac27 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -60,7 +60,15 @@ let package = Package( .library(name: "TableProPostgreSQLCore", targets: ["TableProPostgreSQLCore"]), .library(name: "TableProRedisCore", targets: ["TableProRedisCore"]), .library(name: "TableProMemcachedCore", targets: ["TableProMemcachedCore"]), - .library(name: "TableProTestSupport", targets: ["TableProTestSupport"]) + .library(name: "TableProTestSupport", targets: ["TableProTestSupport"]), + .library(name: "TableProWireIO", targets: ["TableProWireIO"]), + .library(name: "TableProGzip", targets: ["TableProGzip"]), + .library(name: "TableProKafkaCore", targets: ["TableProKafkaCore"]), + .library(name: "TableProCassandraCore", targets: ["TableProCassandraCore"]) + ], + // Kafka brokers hand back whatever codec the producer stored, and macOS has no zstd. + dependencies: [ + .package(url: "https://github.com/facebook/zstd", from: "1.5.0") ], targets: [ .target( @@ -110,7 +118,7 @@ let package = Package( ), .target( name: "TableProImport", - dependencies: ["TableProConnectionLibrary", "TableProSSHTransport"], + dependencies: ["TableProConnectionLibrary", "TableProCoreTypes", "TableProSSHTransport"], path: "Sources/TableProImport" ), .target( @@ -152,7 +160,7 @@ let package = Package( ), .target( name: "TableProTeradataCore", - dependencies: [], + dependencies: ["TableProWireIO"], path: "Sources/TableProTeradataCore" ), .target( @@ -263,7 +271,7 @@ let package = Package( ), .target( name: "TableProTabularIO", - dependencies: [], + dependencies: ["TableProWireIO"], path: "Sources/TableProTabularIO" ), .target( @@ -301,6 +309,33 @@ let package = Package( path: "Sources/TableProRedisCore", swiftSettings: pluginCoreSettings ), + .target( + name: "TableProWireIO", + dependencies: [], + path: "Sources/TableProWireIO" + ), + .target( + name: "TableProGzip", + dependencies: ["TableProWireIO"], + path: "Sources/TableProGzip" + ), + // Linked by a plugin bundle, so never TableProPluginKit. + .target( + name: "TableProKafkaCore", + dependencies: [ + "TableProWireIO", + "TableProGzip", + .product(name: "libzstd", package: "zstd") + ], + path: "Sources/TableProKafkaCore", + swiftSettings: pluginCoreSettings + ), + .target( + name: "TableProCassandraCore", + dependencies: [], + path: "Sources/TableProCassandraCore", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProBigtableCoreTests", dependencies: ["TableProBigtableCore", "TableProGoogleCloud"], @@ -323,7 +358,7 @@ let package = Package( ), .testTarget( name: "TableProTeradataCoreTests", - dependencies: ["TableProTeradataCore"], + dependencies: ["TableProTeradataCore", "TableProWireIO"], path: "Tests/TableProTeradataCoreTests" ), .testTarget( @@ -493,6 +528,28 @@ let package = Package( dependencies: ["TableProRedisCore"], path: "Tests/TableProRedisCoreTests", swiftSettings: pluginCoreSettings + ), + .testTarget( + name: "TableProWireIOTests", + dependencies: ["TableProWireIO"], + path: "Tests/TableProWireIOTests" + ), + .testTarget( + name: "TableProGzipTests", + dependencies: ["TableProGzip", "TableProWireIO"], + path: "Tests/TableProGzipTests" + ), + .testTarget( + name: "TableProKafkaCoreTests", + dependencies: ["TableProKafkaCore", "TableProWireIO"], + path: "Tests/TableProKafkaCoreTests", + swiftSettings: pluginCoreSettings + ), + .testTarget( + name: "TableProCassandraCoreTests", + dependencies: ["TableProCassandraCore"], + path: "Tests/TableProCassandraCoreTests", + swiftSettings: pluginCoreSettings ) ] ) diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraCellText.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellText.swift similarity index 60% rename from apple/Plugins/CassandraDriverPlugin/CassandraCellText.swift rename to apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellText.swift index cda0de00f1..53b4f0092e 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraCellText.swift +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellText.swift @@ -1,17 +1,18 @@ // // CassandraCellText.swift -// CassandraDriverPlugin +// TableProCassandraCore // import Foundation -/// The text a cell shows for the CQL types the C driver hands over as raw parts rather than as a string. -enum CassandraCellText { +// The text a cell shows for the CQL types the C driver hands over as raw parts rather than as a string. +public enum CassandraCellText { private static let nanosecondsPerSecond: Int64 = 1_000_000_000 + private static let hexDigits = Array("0123456789abcdef".unicodeScalars) - /// Cassandra's own spelling of a duration, the form cqlsh prints and CQL accepts back: `1y2mo3d4h5m6s7ms`. - /// A duration's parts all carry the same sign, so a negative one is the positive spelling with a leading `-`. - static func durationText(months: Int32, days: Int32, nanoseconds: Int64) -> String { + // Cassandra's own spelling of a duration, the form cqlsh prints and CQL accepts back: `1y2mo3d4h5m6s7ms`. + // A duration's parts all carry the same sign, so a negative one is the positive spelling with a leading `-`. + public static func durationText(months: Int32, days: Int32, nanoseconds: Int64) -> String { let isNegative = months < 0 || days < 0 || nanoseconds < 0 let months = Int64(months).magnitude let days = Int64(days).magnitude @@ -27,9 +28,9 @@ enum CassandraCellText { return isNegative ? "-" + text : text } - /// A `vector` arrives as a custom type whose class name carries the element type, with the values - /// packed as big-endian 32-bit floats. Any other custom value is shown as its bytes. - static func customText(className: String?, bytes: Data) -> String { + // A `vector` arrives as a custom type whose class name carries the element type, with the values + // packed as big-endian 32-bit floats. Any other custom value is shown as its bytes. + public static func customText(className: String?, bytes: Data) -> String { guard let className, isFloatVector(className), bytes.count.isMultiple(of: 4) else { return hex(bytes) } @@ -41,8 +42,14 @@ enum CassandraCellText { return "[" + values.joined(separator: ", ") + "]" } - static func hex(_ bytes: Data) -> String { - "0x" + bytes.map { String(format: "%02x", $0) }.joined() + public static func hex(_ bytes: Data) -> String { + var text = "0x" + text.reserveCapacity(2 + bytes.count * 2) + for byte in bytes { + text.unicodeScalars.append(hexDigits[Int(byte >> 4)]) + text.unicodeScalars.append(hexDigits[Int(byte & 0x0F)]) + } + return text } private static func isFloatVector(_ className: String) -> Bool { diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraVarint.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraVarint.swift similarity index 61% rename from apple/Plugins/CassandraDriverPlugin/CassandraVarint.swift rename to apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraVarint.swift index e61d6da1db..7832e4bed9 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraVarint.swift +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraVarint.swift @@ -1,16 +1,21 @@ // // CassandraVarint.swift -// CassandraDriverPlugin +// TableProCassandraCore // import Foundation -/// CQL's `varint` is a big-endian two's complement integer of any length, and `decimal` is that integer unscaled -/// with a 32-bit scale beside it. The C driver hands both over as raw bytes, so reading and writing them as numbers -/// is done here. -enum CassandraVarint { - static func decimalString(fromTwosComplement bytes: Data) -> String { +// CQL's `varint` is a big-endian two's complement integer of any length, and `decimal` is that integer unscaled +// with a 32-bit scale beside it. The C driver hands both over as raw bytes, so reading and writing them as numbers +// is done here. +public enum CassandraVarint { + // Each pass over the number yields nine digits, so the cost grows with the square of its length and a longer + // value is shown as hex. 4 KiB is about 9,900 digits, near the 10,000 characters the grid shows of a cell. + public static let maximumDecimalBytes = 4_096 + + public static func decimalString(fromTwosComplement bytes: Data) -> String { guard let first = bytes.first else { return "0" } + guard bytes.count <= maximumDecimalBytes else { return CassandraCellText.hex(bytes) } let isNegative = first & 0x80 != 0 var magnitude = [UInt8](bytes) if isNegative { @@ -20,11 +25,14 @@ enum CassandraVarint { return isNegative ? "-" + digits : digits } - /// A scale the plain form would pad with more zeros than this is written with an exponent instead. The wire - /// scale is any 32-bit value, and expanding one near its limits would allocate billions of characters. + // A scale the plain form would pad with more zeros than this is written with an exponent instead. The wire + // scale is any 32-bit value, and expanding one near its limits would allocate billions of characters. static let maximumExpandedZeros = 64 - static func decimalString(unscaled: Data, scale: Int32) -> String { + public static func decimalString(unscaled: Data, scale: Int32) -> String { + guard unscaled.count <= maximumDecimalBytes else { + return CassandraCellText.hex(unscaled) + "E" + String(-Int(scale)) + } let integer = decimalString(fromTwosComplement: unscaled) let isNegative = integer.hasPrefix("-") var digits = isNegative ? String(integer.dropFirst()) : integer @@ -46,7 +54,7 @@ enum CassandraVarint { return isNegative ? "-" + rendered : String(rendered) } - static func bytes(fromDecimalInteger text: String) -> Data? { + public static func bytes(fromDecimalInteger text: String) -> Data? { var body = Substring(text) var isNegative = false if body.first == "-" || body.first == "+" { @@ -72,7 +80,7 @@ enum CassandraVarint { return Data(twosComplement(magnitude: trimmedMagnitude(magnitude), isNegative: isNegative)) } - static func decimal(fromText text: String) -> (unscaled: Data, scale: Int32)? { + public static func decimal(fromText text: String) -> (unscaled: Data, scale: Int32)? { var mantissa = Substring(text) var exponent = 0 if let marker = mantissa.firstIndex(where: { $0 == "e" || $0 == "E" }) { @@ -96,25 +104,48 @@ enum CassandraVarint { return (unscaled, scale32) } + private static let chunkDivisor: UInt64 = 1_000_000_000 + private static let chunkDigits = 9 + + // Long division of 32-bit limbs by 10^9, nine digits a pass. private static func decimalDigits(ofMagnitude magnitude: [UInt8]) -> String { - var remaining = trimmedMagnitude(magnitude) - guard remaining != [0] else { return "0" } - var digits: [Character] = [] - while !(remaining.count == 1 && remaining[0] == 0) { - var remainder = 0 - var quotient: [UInt8] = [] - for byte in remaining { - let value = remainder << 8 | Int(byte) - let digit = value / 10 - remainder = value % 10 - if !(quotient.isEmpty && digit == 0) { - quotient.append(UInt8(digit)) - } + var limbs = Self.limbs(ofMagnitude: magnitude) + guard !limbs.isEmpty else { return "0" } + var chunks: [UInt64] = [] + while !limbs.isEmpty { + var remainder: UInt64 = 0 + for index in limbs.indices { + let value = remainder << 32 | UInt64(limbs[index]) + limbs[index] = UInt32(value / chunkDivisor) + remainder = value % chunkDivisor + } + chunks.append(remainder) + limbs.removeFirst(limbs.prefix { $0 == 0 }.count) + } + var text = String(chunks.removeLast()) + for chunk in chunks.reversed() { + let digits = String(chunk) + text += String(repeating: "0", count: chunkDigits - digits.count) + digits + } + return text + } + + private static func limbs(ofMagnitude magnitude: [UInt8]) -> [UInt32] { + let significant = magnitude.drop { $0 == 0 } + var limbs: [UInt32] = [] + limbs.reserveCapacity((significant.count + 3) / 4) + var limb: UInt32 = 0 + var filled = (4 - significant.count % 4) % 4 + for byte in significant { + limb = limb << 8 | UInt32(byte) + filled += 1 + if filled == 4 { + limbs.append(limb) + limb = 0 + filled = 0 } - digits.append(Character(String(remainder))) - remaining = quotient.isEmpty ? [0] : quotient } - return String(digits.reversed()) + return limbs } private static func trimmedMagnitude(_ magnitude: [UInt8]) -> [UInt8] { diff --git a/apple/Packages/TableProCore/Sources/TableProCoreTypes/UniqueNameAllocator.swift b/apple/Packages/TableProCore/Sources/TableProCoreTypes/UniqueNameAllocator.swift new file mode 100644 index 0000000000..b3e1d3b3de --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCoreTypes/UniqueNameAllocator.swift @@ -0,0 +1,40 @@ +// Gives each base name the lowest free ordinal, starting at 1. Remembering where each base +// stopped keeps a run of repeated names linear: the ordinals below the mark are taken for good, +// because a name, once taken, stays taken. +public struct UniqueNameAllocator { + private var taken: Set + private var nextOrdinal: [String: Int] = [:] + private let key: (String) -> String + private let candidate: (String, Int) -> String + + public init( + taken: [String] = [], + key: @escaping (String) -> String = { $0 }, + candidate: @escaping (_ base: String, _ ordinal: Int) -> String + ) { + self.key = key + self.candidate = candidate + self.taken = Set(taken.map(key)) + } + + public func contains(_ name: String) -> Bool { + taken.contains(key(name)) + } + + @discardableResult + public mutating func reserve(_ name: String) -> Bool { + taken.insert(key(name)).inserted + } + + public mutating func allocate(_ base: String) -> String { + var ordinal = nextOrdinal[base, default: 1] + while true { + let name = candidate(base, ordinal) + ordinal += 1 + if taken.insert(key(name)).inserted { + nextOrdinal[base] = ordinal + return name + } + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift b/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift new file mode 100644 index 0000000000..19a4dcad91 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift @@ -0,0 +1,62 @@ +import Foundation +import TableProWireIO + +public enum GzipFile { + private static let readSize = 256 * 1_024 + + // What a decompressed file leaves free on its volume, or half the free space on a fuller one, so a small file + // cannot fill the disk. + public static let reservedSpace = 2 << 30 + + // A file the user picked gets no expansion ratio, since a dump of zero-filled BLOBs packs past + // 500:1. Its output stops short of the volume's free space instead. + public static func budget(writingTo destination: URL) -> DecompressionBudget { + let values = try? destination.deletingLastPathComponent().resourceValues(forKeys: [ + .volumeAvailableCapacityForImportantUsageKey, + .volumeAvailableCapacityKey + ]) + let available = values?.volumeAvailableCapacityForImportantUsage.map { Int(clamping: $0) } + ?? values?.volumeAvailableCapacity + return budget(availableSpace: available) + } + + public static func budget(availableSpace: Int?) -> DecompressionBudget { + sizeLimit(availableSpace.map { max($0, 0) - min(reservedSpace, max($0, 0) / 2) } ?? .max) + } + + // The destination is removed on any failure, so a refused or cancelled file leaves nothing behind. + @concurrent + public static func decompress( + source: URL, + destination: URL, + outputLimit: Int? = nil + ) async throws { + let budget = outputLimit.map(Self.sizeLimit) ?? Self.budget(writingTo: destination) + guard FileManager.default.createFile(atPath: destination.path(percentEncoded: false), contents: nil) else { + throw CocoaError(.fileWriteUnknown, userInfo: [NSFilePathErrorKey: destination.path(percentEncoded: false)]) + } + do { + let input = try FileHandle(forReadingFrom: source) + defer { try? input.close() } + let output = try FileHandle(forWritingTo: destination) + defer { try? output.close() } + let inflater = try GzipInflater(budget: budget) + while let block = try input.read(upToCount: readSize), !block.isEmpty { + try Task.checkCancellation() + try block.withUnsafeBytes { raw in + try inflater.inflate(raw) { try output.write(contentsOf: $0) } + } + } + try inflater.finish() + } catch { + try? FileManager.default.removeItem(at: destination) + // With no ratio in the budget, only the size limit refuses a file. + if (error as? GzipError) == .expandsTooFar { throw GzipError.needsMoreDiskSpace } + throw error + } + } + + private static func sizeLimit(_ limit: Int) -> DecompressionBudget { + DecompressionBudget(limit: limit, ratioLimit: .max, grace: .max) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProGzip/GzipInflater.swift b/apple/Packages/TableProCore/Sources/TableProGzip/GzipInflater.swift new file mode 100644 index 0000000000..89f0237669 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProGzip/GzipInflater.swift @@ -0,0 +1,130 @@ +import Foundation +import TableProWireIO +import zlib + +public enum GzipError: Error, Equatable, Sendable, LocalizedError { + case corrupt + case truncated + case expandsTooFar + case needsMoreDiskSpace + + public var errorDescription: String? { + switch self { + case .corrupt: + return String(localized: "The gzip data is corrupt.") + case .truncated: + return String(localized: "The gzip data ends before its last block.") + case .expandsTooFar: + return String(localized: "The compressed data expands to far more than it holds, so it was not read.") + case .needsMoreDiskSpace: + return String(localized: "The decompressed file needs more free disk space than this Mac has.") + } + } +} + +// zlib's own gzip mode (window bits + 16) reads the header, checks the CRC and the length, and +// handles every member of a concatenated file the way gzip -d does. +public final class GzipInflater { + private let stream: UnsafeMutablePointer + private var budget: DecompressionBudget + private var memberEnded = false + private var ignoringTrailer = false + private let chunk: UnsafeMutableBufferPointer + + private static let chunkSize = 256 * 1_024 + private static let gzipMagic: UInt8 = 0x1F + + public init(budget: DecompressionBudget = .standard) throws(GzipError) { + self.budget = budget + stream = .allocate(capacity: 1) + stream.initialize(to: z_stream()) + chunk = .allocate(capacity: Self.chunkSize) + guard inflateInit2_(stream, 16 + MAX_WBITS, ZLIB_VERSION, Int32(MemoryLayout.size)) == Z_OK else { + chunk.deallocate() + stream.deinitialize(count: 1) + stream.deallocate() + throw .corrupt + } + } + + deinit { + inflateEnd(stream) + stream.deinitialize(count: 1) + stream.deallocate() + chunk.deallocate() + } + + public var bytesProduced: Int { budget.produced } + + public func inflate( + _ input: UnsafeRawBufferPointer, + into sink: (UnsafeRawBufferPointer) throws -> Void + ) throws { + guard !ignoringTrailer, let base = input.bindMemory(to: UInt8.self).baseAddress, !input.isEmpty else { return } + var cursor = 0 + var outputPending = false + while cursor < input.count || outputPending { + if memberEnded { + // Padding after the last member is ignored, as gzip -d ignores trailing garbage. + guard base[cursor] == Self.gzipMagic else { + ignoringTrailer = true + return + } + guard inflateReset(stream) == Z_OK else { throw GzipError.corrupt } + memberEnded = false + } + stream.pointee.next_in = UnsafeMutablePointer(mutating: base + cursor) + stream.pointee.avail_in = uInt(min(input.count - cursor, Int(UInt32.max))) + let availableBefore = Int(stream.pointee.avail_in) + stream.pointee.next_out = chunk.baseAddress + stream.pointee.avail_out = uInt(chunk.count) + let status = zlib.inflate(stream, Z_NO_FLUSH) + let consumed = availableBefore - Int(stream.pointee.avail_in) + let produced = chunk.count - Int(stream.pointee.avail_out) + cursor += consumed + switch status { + case Z_OK, Z_BUF_ERROR, Z_STREAM_END: + break + default: + throw GzipError.corrupt + } + do { + try budget.record(produced: produced, consumed: consumed) + } catch { + throw GzipError.expandsTooFar + } + if produced > 0 { + try sink(UnsafeRawBufferPointer(start: chunk.baseAddress, count: produced)) + } + // A full output chunk can leave inflated bytes inside zlib after the input is used up. + outputPending = status != Z_STREAM_END && stream.pointee.avail_out == 0 + if status == Z_STREAM_END { + memberEnded = true + if cursor >= input.count { return } + } else if consumed == 0, produced == 0 { + guard cursor < input.count else { return } + throw GzipError.corrupt + } + } + } + + public func finish() throws(GzipError) { + guard memberEnded || ignoringTrailer else { throw .truncated } + } + + public static func decompress(_ data: Data, budget: DecompressionBudget = .standard) throws -> Data { + var output = Data() + try decompress(data, budget: budget) { output.append($0.assumingMemoryBound(to: UInt8.self)) } + return output + } + + public static func decompress( + _ data: Data, + budget: DecompressionBudget = .standard, + into sink: (UnsafeRawBufferPointer) throws -> Void + ) throws { + let inflater = try GzipInflater(budget: budget) + try data.withUnsafeBytes { try inflater.inflate($0, into: sink) } + try inflater.finish() + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift b/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift index aa939d542c..f2110e17af 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift @@ -1,4 +1,5 @@ import Foundation +import TableProCoreTypes public enum ImportPlanner { public static func plan( @@ -36,7 +37,13 @@ extension ImportPlanner { let bundle = preview.collected.bundle let rules = preview.environment.rules let tagColors = firstTagColors(bundle.tags) - var takenNames = Set(preview.library.connections.map { normalized($0.name) }) + var takenNames = UniqueNameAllocator( + taken: preview.library.connections.map(\.name), + key: normalized, + candidate: { base, ordinal in + ordinal == 1 ? "\(base) (Imported)" : "\(base) (Imported \(ordinal))" + } + ) var replacedTargets: Set = [] var tagKeys: Set = [] var profileRefs: Set = [] @@ -61,13 +68,13 @@ extension ImportPlanner { case .addCopy: id = makeId() write = .add - name = uniqueCopyName(for: name, taken: takenNames) + name = takenNames.allocate(name) case .replace(let existingId): guard replacedTargets.insert(existingId).inserted else { continue } id = existingId write = .replace } - takenNames.insert(normalized(name)) + takenNames.reserve(name) plan.targets[row.ref] = id var settings = selection.keepsCommands @@ -109,21 +116,6 @@ extension ImportPlanner { return plan } - static func uniqueCopyName(for baseName: String, taken: Set) -> String { - let firstCandidate = "\(baseName) (Imported)" - if !taken.contains(normalized(firstCandidate)) { - return firstCandidate - } - var suffix = 2 - while true { - let candidate = "\(baseName) (Imported \(suffix))" - if !taken.contains(normalized(candidate)) { - return candidate - } - suffix += 1 - } - } - private static func groupPath( for connection: BundleConnection, in bundle: ConnectionBundle, diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift index cf73e40e3f..4d1eb53d11 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift @@ -2,9 +2,10 @@ import Foundation public enum InfluxResponseDecoder { // 3.x CSV keeps the statement's column order; its JSON drops null columns. CSV has no null, so an empty cell is one. - public static func decodeV3CSV(_ data: Data) -> InfluxResultTable { + public static func decodeV3CSV(_ data: Data) throws -> InfluxResultTable { var records = InfluxCSV.records(data)[...] guard let columns = records.popFirst() else { return .empty } + try CellBudget.perByte(of: data).check(rows: records.count, columns: columns.count) let rows: [[String?]] = records.map { record in columns.indices.map { index in guard index < record.count, !record[index].isEmpty else { return nil } @@ -66,14 +67,16 @@ public enum InfluxResponseDecoder { uniquingKeysWith: { first, _ in first } ) let tagRange = offset ..< offset + tagKeys.count + let seriesValues = series.map { $0["values"] as? [[Any]] ?? [] } + try CellBudget.sparse(of: data).check(rows: seriesValues.reduce(0) { $0 + $1.count }, columns: columns.count) var rows: [[String?]] = [] var jsonTypes = [String?](repeating: nil, count: columns.count) - for item in series { + for (item, itemValues) in zip(series, seriesValues) { let targets = (item["columns"] as? [String] ?? []).map { position[$0] } let tags = item["tags"] as? [String: Any] ?? [:] let tagValues = tagKeys.map { text(tags[$0]) } - for values in item["values"] as? [[Any]] ?? [] { + for values in itemValues { var row = [String?](repeating: nil, count: columns.count) if includesName { row[0] = item["name"] as? String } for (index, value) in tagValues.enumerated() { @@ -102,10 +105,12 @@ public enum InfluxResponseDecoder { var typeNames: [String] = [] var position: [String: Int] = [:] var rows: [[String?]] = [] + let budget = CellBudget.sparse(of: data) var datatypes: [String] = [] var header: [String]? var targets: [Int?] = [] + var width = 0 for record in InfluxCSV.records(data) { if record == [""] { datatypes = [] @@ -131,24 +136,52 @@ public enum InfluxResponseDecoder { typeNames.append(fluxTypeName(index < datatypes.count ? datatypes[index] : "")) return columns.count - 1 } + // A row is only as wide as its own table; the grid pads it once every header is known. + width = (targets.compactMap { $0 }.max() ?? -1) + 1 continue } if isFluxErrorHeader(currentHeader) { throw InfluxError.query(record.count > 1 ? record[1] : "Flux error") } - var row = [String?](repeating: nil, count: columns.count) + try budget.check(rows: rows.count + 1, columns: columns.count) + var row = [String?](repeating: nil, count: width) for (index, target) in targets.enumerated() where index < record.count { guard let target else { continue } row[target] = record[index].isEmpty ? nil : record[index] } rows.append(row) } + try budget.check(rows: rows.count, columns: columns.count) for index in rows.indices where rows[index].count < columns.count { rows[index] += [String?](repeating: nil, count: columns.count - rows[index].count) } return InfluxResultTable(columns: columns, typeNames: typeNames, rows: rows) } + // Rows are as wide as every header together, so a short record costs more cells than bytes. + private struct CellBudget { + let limit: Int + + // Every v3 CSV row carries every column, a byte per cell at least: 16 cells per byte, or a million at any size. + static func perByte(of data: Data) -> CellBudget { + let (scaled, overflow) = data.count.multipliedReportingOverflow(by: 16) + return CellBudget(limit: overflow ? .max : max(scaled, 1 << 20)) + } + + // v1 JSON and Flux widen each series to every series' columns, so a real result can fill far more cells than + // bytes: 300 measurements of 10 fields fill 9 million from 460 KB. 32 million empty cells take about 512 MB, + // and a large result whose rows are full still gets 16 cells per byte. + static func sparse(of data: Data) -> CellBudget { + CellBudget(limit: max(perByte(of: data).limit, 32 << 20)) + } + + func check(rows: Int, columns: Int) throws { + let (cells, overflow) = rows.multipliedReportingOverflow(by: columns) + guard overflow || cells > limit else { return } + throw InfluxError.malformedResponse(String(localized: "the result has far more columns than its rows fill")) + } + } + private static func isFluxErrorHeader(_ header: [String]) -> Bool { header.dropFirst().elementsEqual(["error", "reference"]) } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaCRC32C.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaCRC32C.swift similarity index 100% rename from apple/Plugins/KafkaDriverPlugin/KafkaCRC32C.swift rename to apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaCRC32C.swift diff --git a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaCompression.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaCompression.swift new file mode 100644 index 0000000000..540adfa6d3 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaCompression.swift @@ -0,0 +1,261 @@ +import Compression +import Foundation +import libzstd +import TableProGzip +import TableProWireIO + +/// A Kafka broker never transcodes a batch for the consumer: it hands back exactly the bytes +/// the producer stored, in whatever codec the producer chose. Measured against a live broker +/// by producing one topic per codec and reading the stored attribute plus the payload's magic +/// bytes back: gzip `1f8b0800`, snappy `82 53 4e 41 50 50 59 00` (xerial framing, not raw +/// snappy), lz4 `04224d18` (the LZ4 frame format, not a raw block), zstd `28b52ffd`. +/// +/// So all four have to be decoded here. macOS supplies only some of that: zlib is in the SDK, +/// Compression.framework has a raw LZ4 block decoder, and there is no snappy and no zstd +/// anywhere on the system (verified by reading compression.h and by dlopen'ing libarchive, +/// which exports no ZSTD_ symbol). zstd therefore comes from the upstream SPM package and +/// snappy is implemented here. +public enum KafkaCompression { + // Every codec charges `budget` as its output grows, so no declared or undeclared size gets + // past it, and one budget spans every batch of a fetch. + public static func decompress( + _ payload: Data, + codec: KafkaCompressionCodec, + budget: inout DecompressionBudget + ) throws -> Data { + switch codec { + case .none: return payload + case .gzip: return try gunzip(payload, budget: &budget) + case .snappy: return try snappyXerial(payload, budget: &budget) + case .lz4: return try lz4Frame(payload, budget: &budget) + case .zstd: return try zstd(payload, budget: &budget) + } + } + + // The standard ratio on a 64 MiB request would let one reply expand to 16 GiB. + static let fetchOutputCeiling = 1 << 30 + + // A Fetch reply may expand to what its byte limit could hold at the standard ratio, up to the + // ceiling. The totals carry over from the earlier pages of the same read, so paging never + // resets the grace, while each page gets its own allowance. + public static func fetchBudget( + requesting requestedBytes: Int32, + after spent: DecompressionBudget? = nil + ) throws(WireIOError) -> DecompressionBudget { + let standard = DecompressionBudget.standard + let (allowance, allowanceOverflows) = Int(max(requestedBytes, 0)) + .multipliedReportingOverflow(by: standard.ratioLimit) + let pageAllowance = allowanceOverflows ? fetchOutputCeiling : min(allowance, fetchOutputCeiling) + let (limit, limitOverflows) = (spent?.produced ?? 0).addingReportingOverflow(pageAllowance) + var budget = DecompressionBudget( + limit: limitOverflows ? .max : max(limit, standard.grace), + ratioLimit: standard.ratioLimit, + grace: standard.grace + ) + if let spent { + try budget.record(produced: spent.produced, consumed: spent.consumed) + } + return budget + } + + static func charge( + _ budget: inout DecompressionBudget, + produced: Int, + consumed: Int, + codec: KafkaCompressionCodec + ) throws { + do { + try budget.record(produced: produced, consumed: consumed) + } catch { + throw KafkaError.decompressionExceeded(codec: codec.name) + } + } + + // MARK: - gzip + + private static func gunzip(_ payload: Data, budget: inout DecompressionBudget) throws -> Data { + guard !payload.isEmpty else { return Data() } + let output: Data + do { + output = try GzipInflater.decompress(payload, budget: budget) + } catch GzipError.expandsTooFar { + throw KafkaError.decompressionExceeded(codec: "gzip") + } catch { + throw KafkaError.decompressionFailed( + codec: "gzip", + reason: String(localized: "the compressed data is corrupt or was cut short") + ) + } + // The inflater checked a copy that started from these totals, so this carries them forward. + try charge(&budget, produced: output.count, consumed: payload.count, codec: .gzip) + return output + } + + // MARK: - snappy + + private static let xerialMagic: [UInt8] = [0x82, 0x53, 0x4E, 0x41, 0x50, 0x50, 0x59, 0x00] + + // The Java producer frames snappy as xerial: the magic, two version ints, then big-endian + // length-prefixed blocks. Some non-Java clients write one bare block instead. + private static func snappyXerial(_ payload: Data, budget: inout DecompressionBudget) throws -> Data { + let bytes = [UInt8](payload) + guard bytes.count >= xerialMagic.count, Array(bytes.prefix(8)) == xerialMagic else { + return Data(try SnappyDecoder.decode(bytes, budget: &budget)) + } + var cursor = 16 // magic + version + compatibleVersion + var output = Data() + while cursor + 4 <= bytes.count { + let length = Int(bytes[cursor]) << 24 | Int(bytes[cursor + 1]) << 16 + | Int(bytes[cursor + 2]) << 8 | Int(bytes[cursor + 3]) + cursor += 4 + guard length >= 0, cursor + length <= bytes.count else { + throw KafkaError.decompressionFailed(codec: "snappy", reason: "block runs past the payload") + } + output.append(contentsOf: try SnappyDecoder.decode(Array(bytes[cursor ..< cursor + length]), budget: &budget)) + cursor += length + } + return output + } + + // MARK: - lz4 + + // Kafka writes the LZ4 frame format. COMPRESSION_LZ4 is Apple's own `bv41` container and + // decodes it to garbage, but the raw blocks inside the frame are COMPRESSION_LZ4_RAW. + private static func lz4Frame(_ payload: Data, budget: inout DecompressionBudget) throws -> Data { + let bytes = [UInt8](payload) + guard bytes.count > 7, + bytes[0] == 0x04, bytes[1] == 0x22, bytes[2] == 0x4D, bytes[3] == 0x18 else { + throw KafkaError.decompressionFailed(codec: "lz4", reason: "not an LZ4 frame") + } + let flags = bytes[4] + let blockDescriptor = bytes[5] + let hasContentSize = flags & 0x08 != 0 + let hasDictionaryId = flags & 0x01 != 0 + let hasBlockChecksum = flags & 0x10 != 0 + let maxBlockSize = lz4MaxBlockSize(blockDescriptor) + + var cursor = 7 // magic + FLG + BD + header checksum + if hasContentSize { cursor += 8 } + if hasDictionaryId { cursor += 4 } + + // One buffer for every block of the frame, left uninitialized, so a block costs what it + // decodes. Sized by the frame's declared maximum block size, not LZ4's 255x worst case. + var scratch = UnsafeMutableBufferPointer.allocate(capacity: maxBlockSize) + defer { scratch.deallocate() } + + var output = Data() + while cursor + 4 <= bytes.count { + let rawSize = UInt32(bytes[cursor]) | UInt32(bytes[cursor + 1]) << 8 + | UInt32(bytes[cursor + 2]) << 16 | UInt32(bytes[cursor + 3]) << 24 + cursor += 4 + if rawSize == 0 { break } // EndMark + let isUncompressed = rawSize & 0x8000_0000 != 0 + let blockSize = Int(rawSize & 0x7FFF_FFFF) + guard blockSize >= 0, cursor + blockSize <= bytes.count else { + throw KafkaError.decompressionFailed(codec: "lz4", reason: "block runs past the frame") + } + let block = bytes[cursor ..< cursor + blockSize] + cursor += blockSize + if hasBlockChecksum { cursor += 4 } + + if isUncompressed { + try charge(&budget, produced: blockSize, consumed: blockSize + 4, codec: .lz4) + output.append(contentsOf: block) + continue + } + let capacity = max(maxBlockSize, blockSize + 64) + if scratch.count < capacity { + scratch.deallocate() + scratch = .allocate(capacity: capacity) + } + let produced = try lz4RawBlock(block, into: UnsafeMutableBufferPointer(rebasing: scratch[.. Int { + switch (descriptor >> 4) & 0x07 { + case 4: return 64 * 1_024 + case 5: return 256 * 1_024 + case 6: return 1_024 * 1_024 + case 7: return 4 * 1_024 * 1_024 + default: return 4 * 1_024 * 1_024 + } + } + + private static func lz4RawBlock( + _ block: ArraySlice, + into destination: UnsafeMutableBufferPointer + ) throws -> Int { + let produced = block.withUnsafeBufferPointer { source -> Int in + guard let sourceBase = source.baseAddress, let targetBase = destination.baseAddress else { return 0 } + return compression_decode_buffer( + targetBase, destination.count, + sourceBase, source.count, + nil, COMPRESSION_LZ4_RAW + ) + } + guard produced > 0 else { + throw KafkaError.decompressionFailed(codec: "lz4", reason: "the block did not decode") + } + return produced + } + + // MARK: - zstd + + // Streaming in fixed chunks sizes nothing by the frame's declared size, and handles a frame + // that declares none the same way. + private static func zstd(_ payload: Data, budget: inout DecompressionBudget) throws -> Data { + guard !payload.isEmpty else { return Data() } + let bytes = [UInt8](payload) + let declared = bytes.withUnsafeBufferPointer { buffer -> UInt64 in + guard let base = buffer.baseAddress else { return 0 } + return ZSTD_getFrameContentSize(base, buffer.count) + } + guard declared != UInt64(bitPattern: -2) else { + throw KafkaError.decompressionFailed(codec: "zstd", reason: "not a zstd frame") + } + guard let context = ZSTD_createDCtx() else { + throw KafkaError.decompressionFailed(codec: "zstd", reason: "could not start zstd") + } + defer { ZSTD_freeDCtx(context) } + + var chunk = [UInt8](repeating: 0, count: ZSTD_DStreamOutSize()) + var output = Data() + try bytes.withUnsafeBufferPointer { source in + var input = ZSTD_inBuffer(src: UnsafeRawPointer(source.baseAddress), size: source.count, pos: 0) + var frameRemainder = 0 + while input.pos < input.size { + let consumedBefore = input.pos + let (result, produced) = chunk.withUnsafeMutableBytes { target -> (Int, Int) in + var out = ZSTD_outBuffer(dst: target.baseAddress, size: target.count, pos: 0) + let code = ZSTD_decompressStream(context, &out, &input) + return (code, out.pos) + } + guard ZSTD_isError(result) == 0 else { + throw KafkaError.decompressionFailed(codec: "zstd", reason: zstdMessage(result)) + } + let consumed = input.pos - consumedBefore + guard consumed > 0 || produced > 0 else { + throw KafkaError.decompressionFailed(codec: "zstd", reason: "the frame stopped decoding") + } + try charge(&budget, produced: produced, consumed: consumed, codec: .zstd) + output.append(contentsOf: chunk[0 ..< produced]) + frameRemainder = result + } + // zstd keeps a frame's last input byte until the frame is flushed, so a nonzero + // remainder once the input is used up means the frame was cut short. + guard frameRemainder == 0 else { + throw KafkaError.decompressionFailed(codec: "zstd", reason: "the frame was cut short") + } + } + return output + } + + private static func zstdMessage(_ code: Int) -> String { + guard let raw = ZSTD_getErrorName(code) else { return "unknown zstd error" } + return String(cString: raw) + } +} diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaError.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaError.swift similarity index 85% rename from apple/Plugins/KafkaDriverPlugin/KafkaError.swift rename to apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaError.swift index 26788e3a00..8714627f1d 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaError.swift +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaError.swift @@ -1,7 +1,6 @@ import Foundation -import TableProPluginKit -enum KafkaError: LocalizedError { +public enum KafkaError: LocalizedError { case notConnected case connectionFailed(String) case truncatedResponse(needed: Int, available: Int) @@ -11,6 +10,7 @@ enum KafkaError: LocalizedError { case authenticationFailed(String) case unsupportedCompression(String) case decompressionFailed(codec: String, reason: String) + case decompressionExceeded(codec: String) case syntax(String) case unknownTopic(String) case unknownGroup(String) @@ -23,7 +23,7 @@ enum KafkaError: LocalizedError { case partitionsHaveNoLeader(topic: String, partitions: [Int32]) case verifyCaNeedsCertificate - var errorDescription: String? { + public var errorDescription: String? { switch self { case .notConnected: return String(localized: "Not connected to the Kafka cluster.") @@ -70,6 +70,11 @@ enum KafkaError: LocalizedError { codec, reason ) + case .decompressionExceeded(let codec): + return String( + format: String(localized: "A %@ batch decompresses to far more data than the broker sent, so it was not read."), + codec + ) case .syntax(let detail): return detail case .unknownTopic(let name): @@ -144,7 +149,7 @@ enum KafkaPartitionList { } /// What to do about an error code before giving it to the user. -enum KafkaRetryAction: Sendable { +public enum KafkaRetryAction: Sendable { /// A real answer. Report it. case report /// The client's view of the cluster is stale. Re-read Metadata, re-resolve the leader, retry. @@ -157,37 +162,37 @@ enum KafkaRetryAction: Sendable { /// The subset of Kafka's error codes this client can meet, with what to do about each. /// A code that is not listed still reports its number rather than being swallowed. -enum KafkaErrorCode { - static let none: Int16 = 0 - static let offsetOutOfRange: Int16 = 1 - static let unknownTopicOrPartition: Int16 = 3 - static let leaderNotAvailable: Int16 = 5 - static let notLeaderOrFollower: Int16 = 6 - static let requestTimedOut: Int16 = 7 - static let brokerNotAvailable: Int16 = 8 - static let replicaNotAvailable: Int16 = 9 - static let messageTooLarge: Int16 = 10 - static let networkException: Int16 = 13 - static let coordinatorLoadInProgress: Int16 = 14 - static let coordinatorNotAvailable: Int16 = 15 - static let notCoordinator: Int16 = 16 - static let illegalSaslState: Int16 = 34 - static let unsupportedVersion: Int16 = 35 - static let notController: Int16 = 41 - static let topicAuthorizationFailed: Int16 = 29 - static let groupAuthorizationFailed: Int16 = 30 - static let clusterAuthorizationFailed: Int16 = 31 - static let kafkaStorageError: Int16 = 56 - static let saslAuthenticationFailed: Int16 = 58 +public enum KafkaErrorCode { + public static let none: Int16 = 0 + public static let offsetOutOfRange: Int16 = 1 + public static let unknownTopicOrPartition: Int16 = 3 + public static let leaderNotAvailable: Int16 = 5 + public static let notLeaderOrFollower: Int16 = 6 + public static let requestTimedOut: Int16 = 7 + public static let brokerNotAvailable: Int16 = 8 + public static let replicaNotAvailable: Int16 = 9 + public static let messageTooLarge: Int16 = 10 + public static let networkException: Int16 = 13 + public static let coordinatorLoadInProgress: Int16 = 14 + public static let coordinatorNotAvailable: Int16 = 15 + public static let notCoordinator: Int16 = 16 + public static let illegalSaslState: Int16 = 34 + public static let unsupportedVersion: Int16 = 35 + public static let notController: Int16 = 41 + public static let topicAuthorizationFailed: Int16 = 29 + public static let groupAuthorizationFailed: Int16 = 30 + public static let clusterAuthorizationFailed: Int16 = 31 + public static let kafkaStorageError: Int16 = 56 + public static let saslAuthenticationFailed: Int16 = 58 /// 72 is LISTENER_NOT_FOUND and 73 is TOPIC_DELETION_DISABLED, which is the opposite of what /// this file said until #2993. Nothing read the constant, so nothing broke; the two are /// spelled out here because a hand-transcribed code table is exactly the thing that drifts. - static let listenerNotFound: Int16 = 72 - static let topicDeletionDisabled: Int16 = 73 - static let fencedLeaderEpoch: Int16 = 74 - static let unknownLeaderEpoch: Int16 = 75 - static let offsetNotAvailable: Int16 = 78 - static let unknownTopicId: Int16 = 100 + public static let listenerNotFound: Int16 = 72 + public static let topicDeletionDisabled: Int16 = 73 + public static let fencedLeaderEpoch: Int16 = 74 + public static let unknownLeaderEpoch: Int16 = 75 + public static let offsetNotAvailable: Int16 = 78 + public static let unknownTopicId: Int16 = 100 /// What a code means for the request that drew it. /// @@ -197,7 +202,7 @@ enum KafkaErrorCode { /// and would only be delayed by them. 8 BROKER_NOT_AVAILABLE is deliberately not retried: /// Kafka does not classify it retriable, and a client that retries it hides a broker that /// is genuinely down. - static func retryAction(for code: Int16) -> KafkaRetryAction { + public static func retryAction(for code: Int16) -> KafkaRetryAction { switch code { case unknownTopicOrPartition, leaderNotAvailable, notLeaderOrFollower, replicaNotAvailable, networkException, kafkaStorageError, listenerNotFound, fencedLeaderEpoch, @@ -221,7 +226,7 @@ enum KafkaErrorCode { /// broke and of a log directory that failed mid-append. This client sends no producer id, /// so Kafka cannot deduplicate a replay and the message is appended twice. The codes below /// all mean the broker refused the request outright, which is safe to send elsewhere. - static func provesRequestWasNotApplied(_ code: Int16) -> Bool { + public static func provesRequestWasNotApplied(_ code: Int16) -> Bool { switch code { case unknownTopicOrPartition, leaderNotAvailable, notLeaderOrFollower, replicaNotAvailable, listenerNotFound, fencedLeaderEpoch, unknownLeaderEpoch, unknownTopicId: @@ -231,7 +236,7 @@ enum KafkaErrorCode { } } - static func describe(_ code: Int16) -> String { + public static func describe(_ code: Int16) -> String { switch code { case offsetOutOfRange: return String(localized: "the requested offset is outside the partition's range") @@ -290,7 +295,7 @@ enum KafkaErrorCode { } } - static func check(_ code: Int16, api: String) throws { + public static func check(_ code: Int16, api: String) throws { guard code != none else { return } if code == saslAuthenticationFailed || code == illegalSaslState { throw KafkaError.authenticationFailed(describe(code)) diff --git a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift new file mode 100644 index 0000000000..cb7d49a806 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift @@ -0,0 +1,26 @@ +// Offsets come from the broker's replies, so arithmetic on them saturates or stops instead of trapping. +public enum KafkaOffsetMath { + // Nothing follows Int64.max, so a read that reaches it is over. + public static func offset(after offset: Int64) -> Int64? { + offset < .max ? offset + 1 : nil + } + + public static func messageCount(earliest: Int64, latest: Int64) -> Int64 { + let (count, overflows) = latest.subtractingReportingOverflow(earliest) + guard !overflows else { return latest < earliest ? 0 : .max } + return max(0, count) + } + + public static func total(_ counts: some Sequence) -> Int64 { + counts.reduce(0) { sum, count in + let (next, overflows) = sum.addingReportingOverflow(count) + return overflows ? (count < 0 ? .min : .max) : next + } + } + + // A start that falls below Int64 is below any floor there is. + public static func stepBack(from end: Int64, by step: Int64, floor: Int64) -> Int64 { + let (start, overflows) = end.subtractingReportingOverflow(step) + return overflows ? floor : max(floor, start) + } +} diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaProtocolReader.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaProtocolReader.swift similarity index 74% rename from apple/Plugins/KafkaDriverPlugin/KafkaProtocolReader.swift rename to apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaProtocolReader.swift index cb6c92b660..ac7be198c0 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaProtocolReader.swift +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaProtocolReader.swift @@ -1,60 +1,60 @@ import Foundation -struct KafkaProtocolReader { +public struct KafkaProtocolReader: Sendable { private let bytes: [UInt8] - private(set) var offset: Int + public private(set) var offset: Int - init(_ data: Data) { + public init(_ data: Data) { bytes = [UInt8](data) offset = 0 } - init(_ bytes: [UInt8]) { + public init(_ bytes: [UInt8]) { self.bytes = bytes offset = 0 } - var remaining: Int { bytes.count - offset } - var isAtEnd: Bool { offset >= bytes.count } + public var remaining: Int { bytes.count - offset } + public var isAtEnd: Bool { offset >= bytes.count } - mutating func skip(_ count: Int) throws { + public mutating func skip(_ count: Int) throws { try require(count) offset += count } - mutating func int8() throws -> Int8 { + public mutating func int8() throws -> Int8 { try require(1) defer { offset += 1 } return Int8(bitPattern: bytes[offset]) } - mutating func uint8() throws -> UInt8 { + public mutating func uint8() throws -> UInt8 { try require(1) defer { offset += 1 } return bytes[offset] } - mutating func boolean() throws -> Bool { + public mutating func boolean() throws -> Bool { try uint8() != 0 } - mutating func int16() throws -> Int16 { + public mutating func int16() throws -> Int16 { Int16(bitPattern: try unsigned(UInt16.self)) } - mutating func int32() throws -> Int32 { + public mutating func int32() throws -> Int32 { Int32(bitPattern: try unsigned(UInt32.self)) } - mutating func int64() throws -> Int64 { + public mutating func int64() throws -> Int64 { Int64(bitPattern: try unsigned(UInt64.self)) } - mutating func uint32() throws -> UInt32 { + public mutating func uint32() throws -> UInt32 { try unsigned(UInt32.self) } - mutating func uuid() throws -> KafkaUUID { + public mutating func uuid() throws -> KafkaUUID { try require(16) defer { offset += 16 } return KafkaUUID(bytes: Array(bytes[offset ..< offset + 16])) @@ -62,7 +62,7 @@ struct KafkaProtocolReader { /// Unsigned LEB128, the framing varint. Distinct from `varint`, which is zig-zag signed /// and appears only inside a record batch's records. - mutating func unsignedVarint() throws -> UInt32 { + public mutating func unsignedVarint() throws -> UInt32 { var result: UInt32 = 0 var shift: UInt32 = 0 while true { @@ -74,7 +74,7 @@ struct KafkaProtocolReader { } } - mutating func unsignedVarlong() throws -> UInt64 { + public mutating func unsignedVarlong() throws -> UInt64 { var result: UInt64 = 0 var shift: UInt64 = 0 while true { @@ -86,29 +86,29 @@ struct KafkaProtocolReader { } } - mutating func varint() throws -> Int32 { + public mutating func varint() throws -> Int32 { let raw = try unsignedVarint() return Int32(bitPattern: (raw >> 1) ^ (0 &- (raw & 1))) } - mutating func varlong() throws -> Int64 { + public mutating func varlong() throws -> Int64 { let raw = try unsignedVarlong() return Int64(bitPattern: (raw >> 1) ^ (0 &- (raw & 1))) } - mutating func legacyString() throws -> String { + public mutating func legacyString() throws -> String { let length = Int(try int16()) guard length >= 0 else { throw KafkaError.malformedResponse("non-nullable string was null") } return try string(length: length) } - mutating func nullableLegacyString() throws -> String? { + public mutating func nullableLegacyString() throws -> String? { let length = Int(try int16()) if length < 0 { return nil } return try string(length: length) } - mutating func compactString() throws -> String { + public mutating func compactString() throws -> String { guard let value = try nullableCompactString() else { throw KafkaError.malformedResponse("non-nullable compact string was null") } @@ -117,44 +117,44 @@ struct KafkaProtocolReader { /// A compact string is length+1, and 0 means null. `0xff` is the legacy null and is not /// valid here; treating it as one silently consumes the following bytes. - mutating func nullableCompactString() throws -> String? { + public mutating func nullableCompactString() throws -> String? { let encoded = Int(try unsignedVarint()) if encoded == 0 { return nil } return try string(length: encoded - 1) } - mutating func nullableLegacyBytes() throws -> Data? { + public mutating func nullableLegacyBytes() throws -> Data? { let length = Int(try int32()) if length < 0 { return nil } return try data(length: length) } - mutating func nullableCompactBytes() throws -> Data? { + public mutating func nullableCompactBytes() throws -> Data? { let encoded = Int(try unsignedVarint()) if encoded == 0 { return nil } return try data(length: encoded - 1) } - mutating func compactArrayCount() throws -> Int? { + public mutating func compactArrayCount() throws -> Int? { let encoded = Int(try unsignedVarint()) if encoded == 0 { return nil } return encoded - 1 } - mutating func legacyArrayCount() throws -> Int? { + public mutating func legacyArrayCount() throws -> Int? { let count = Int(try int32()) if count < 0 { return nil } return count } - mutating func array(compact: Bool, _ element: (inout KafkaProtocolReader) throws -> T) throws -> [T] { + public mutating func array(compact: Bool, _ element: (inout KafkaProtocolReader) throws -> T) throws -> [T] { let count = compact ? try compactArrayCount() : try legacyArrayCount() guard let count else { return [] } guard count <= remaining + 1 else { throw KafkaError.malformedResponse("array claims \(count) elements, \(remaining) bytes left") } var items: [T] = [] - items.reserveCapacity(min(count, 1024)) + items.reserveCapacity(min(count, 1_024)) for _ in 0 ..< count { items.append(try element(&self)) } @@ -163,7 +163,7 @@ struct KafkaProtocolReader { /// Tagged fields close every struct in a flexible message. Unknown tags are skipped by /// size, which is what lets a newer broker add a field without breaking this client. - mutating func taggedFields() throws { + public mutating func taggedFields() throws { let count = Int(try unsignedVarint()) for _ in 0 ..< count { _ = try unsignedVarint() @@ -172,7 +172,7 @@ struct KafkaProtocolReader { } } - mutating func rest() -> Data { + public mutating func rest() -> Data { defer { offset = bytes.count } return Data(bytes[offset...]) } @@ -204,7 +204,7 @@ struct KafkaProtocolReader { } private func require(_ count: Int) throws { - guard count >= 0, offset + count <= bytes.count else { + guard count >= 0, count <= remaining else { throw KafkaError.truncatedResponse(needed: count, available: remaining) } } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaProtocolWriter.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaProtocolWriter.swift similarity index 66% rename from apple/Plugins/KafkaDriverPlugin/KafkaProtocolWriter.swift rename to apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaProtocolWriter.swift index 41e4bd4274..940e169722 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaProtocolWriter.swift +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaProtocolWriter.swift @@ -1,56 +1,56 @@ import Foundation -struct KafkaProtocolWriter { - private(set) var bytes: [UInt8] = [] +public struct KafkaProtocolWriter: Sendable { + public private(set) var bytes: [UInt8] = [] - init() { + public init() { bytes.reserveCapacity(256) } - var data: Data { Data(bytes) } - var count: Int { bytes.count } + public var data: Data { Data(bytes) } + public var count: Int { bytes.count } - mutating func int8(_ value: Int8) { + public mutating func int8(_ value: Int8) { bytes.append(UInt8(bitPattern: value)) } - mutating func uint8(_ value: UInt8) { + public mutating func uint8(_ value: UInt8) { bytes.append(value) } - mutating func boolean(_ value: Bool) { + public mutating func boolean(_ value: Bool) { bytes.append(value ? 1 : 0) } - mutating func int16(_ value: Int16) { + public mutating func int16(_ value: Int16) { unsigned(UInt16(bitPattern: value)) } - mutating func int32(_ value: Int32) { + public mutating func int32(_ value: Int32) { unsigned(UInt32(bitPattern: value)) } - mutating func int64(_ value: Int64) { + public mutating func int64(_ value: Int64) { unsigned(UInt64(bitPattern: value)) } - mutating func uint32(_ value: UInt32) { + public mutating func uint32(_ value: UInt32) { unsigned(value) } - mutating func uuid(_ value: KafkaUUID) { + public mutating func uuid(_ value: KafkaUUID) { bytes.append(contentsOf: value.bytes) } - mutating func raw(_ value: Data) { + public mutating func raw(_ value: Data) { bytes.append(contentsOf: value) } - mutating func raw(_ value: [UInt8]) { + public mutating func raw(_ value: [UInt8]) { bytes.append(contentsOf: value) } - mutating func unsignedVarint(_ value: UInt32) { + public mutating func unsignedVarint(_ value: UInt32) { var remaining = value while remaining >= 0x80 { bytes.append(UInt8((remaining & 0x7F) | 0x80)) @@ -59,7 +59,7 @@ struct KafkaProtocolWriter { bytes.append(UInt8(remaining)) } - mutating func unsignedVarlong(_ value: UInt64) { + public mutating func unsignedVarlong(_ value: UInt64) { var remaining = value while remaining >= 0x80 { bytes.append(UInt8((remaining & 0x7F) | 0x80)) @@ -68,23 +68,23 @@ struct KafkaProtocolWriter { bytes.append(UInt8(remaining)) } - mutating func varint(_ value: Int32) { + public mutating func varint(_ value: Int32) { let zigzag = (UInt32(bitPattern: value) << 1) ^ UInt32(bitPattern: value >> 31) unsignedVarint(zigzag) } - mutating func varlong(_ value: Int64) { + public mutating func varlong(_ value: Int64) { let zigzag = (UInt64(bitPattern: value) << 1) ^ UInt64(bitPattern: value >> 63) unsignedVarlong(zigzag) } - mutating func legacyString(_ value: String) { + public mutating func legacyString(_ value: String) { let encoded = Array(value.utf8) int16(Int16(encoded.count)) bytes.append(contentsOf: encoded) } - mutating func nullableLegacyString(_ value: String?) { + public mutating func nullableLegacyString(_ value: String?) { guard let value else { int16(-1) return @@ -92,7 +92,7 @@ struct KafkaProtocolWriter { legacyString(value) } - mutating func compactString(_ value: String) { + public mutating func compactString(_ value: String) { let encoded = Array(value.utf8) unsignedVarint(UInt32(encoded.count + 1)) bytes.append(contentsOf: encoded) @@ -100,7 +100,7 @@ struct KafkaProtocolWriter { /// Null is uvarint 0 and an empty string is uvarint 1. The legacy -1 form does not apply /// to compact encoding, and a broker that reads one keeps consuming and then hangs up. - mutating func nullableCompactString(_ value: String?) { + public mutating func nullableCompactString(_ value: String?) { guard let value else { unsignedVarint(0) return @@ -108,7 +108,7 @@ struct KafkaProtocolWriter { compactString(value) } - mutating func nullableLegacyBytes(_ value: Data?) { + public mutating func nullableLegacyBytes(_ value: Data?) { guard let value else { int32(-1) return @@ -117,7 +117,7 @@ struct KafkaProtocolWriter { bytes.append(contentsOf: value) } - mutating func nullableCompactBytes(_ value: Data?) { + public mutating func nullableCompactBytes(_ value: Data?) { guard let value else { unsignedVarint(0) return @@ -126,7 +126,7 @@ struct KafkaProtocolWriter { bytes.append(contentsOf: value) } - mutating func compactArrayCount(_ count: Int?) { + public mutating func compactArrayCount(_ count: Int?) { guard let count else { unsignedVarint(0) return @@ -134,11 +134,11 @@ struct KafkaProtocolWriter { unsignedVarint(UInt32(count + 1)) } - mutating func legacyArrayCount(_ count: Int?) { + public mutating func legacyArrayCount(_ count: Int?) { int32(Int32(count ?? -1)) } - mutating func array(_ items: [T], compact: Bool, _ element: (inout KafkaProtocolWriter, T) -> Void) { + public mutating func array(_ items: [T], compact: Bool, _ element: (inout KafkaProtocolWriter, T) -> Void) { if compact { compactArrayCount(items.count) } else { @@ -151,7 +151,7 @@ struct KafkaProtocolWriter { /// This client never sends a tagged field, so the buffer is always the empty marker. It /// is still mandatory: a flexible struct that omits it shifts everything after it. - mutating func emptyTaggedFields() { + public mutating func emptyTaggedFields() { unsignedVarint(0) } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaRecordBatch.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift similarity index 81% rename from apple/Plugins/KafkaDriverPlugin/KafkaRecordBatch.swift rename to apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift index 086330f135..173e024685 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaRecordBatch.swift +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift @@ -1,35 +1,46 @@ import Foundation +import TableProWireIO /// One entry from a Fetch response's aborted-transaction list. Both halves matter: the /// producer owns the run and the offset says where it starts. -struct KafkaAbortedTransaction: Sendable { - let producerId: Int64 - let firstOffset: Int64 +public struct KafkaAbortedTransaction: Sendable { + public let producerId: Int64 + public let firstOffset: Int64 + + public init(producerId: Int64, firstOffset: Int64) { + self.producerId = producerId + self.firstOffset = firstOffset + } } -struct KafkaRecordHeader: Sendable { - let key: String - let value: Data? +public struct KafkaRecordHeader: Sendable { + public let key: String + public let value: Data? + + public init(key: String, value: Data?) { + self.key = key + self.value = value + } } -struct KafkaRecord: Sendable { - let offset: Int64 - let timestamp: Int64 - let timestampIsLogAppendTime: Bool - let key: Data? - let value: Data? - let headers: [KafkaRecordHeader] - let partition: Int32 +public struct KafkaRecord: Sendable { + public let offset: Int64 + public let timestamp: Int64 + public let timestampIsLogAppendTime: Bool + public let key: Data? + public let value: Data? + public let headers: [KafkaRecordHeader] + public let partition: Int32 } -enum KafkaCompressionCodec: Int16, Sendable { +public enum KafkaCompressionCodec: Int16, Sendable { case none = 0 case gzip = 1 case snappy = 2 case lz4 = 3 case zstd = 4 - var name: String { + public var name: String { switch self { case .none: return "none" case .gzip: return "gzip" @@ -43,7 +54,7 @@ enum KafkaCompressionCodec: Int16, Sendable { /// Decoder for the v2 record batch (magic byte 2), which is the only format Kafka has written /// since 0.11. The older v0/v1 message sets are not read: a broker still storing them is more /// than eight years old, and guessing at a half-supported legacy path is worse than saying so. -enum KafkaRecordBatchDecoder { +public enum KafkaRecordBatchDecoder { /// Fixed bytes from `baseOffset` to the record count, inclusive. Measured against a real /// broker's batches, and it is also what Kafka's DefaultRecordBatch documents. static let headerLength = 61 @@ -69,18 +80,15 @@ enum KafkaRecordBatchDecoder { private static let logAppendTimeMask: Int16 = 0x08 private static let controlBatchMask: Int16 = 0x20 - /// Decodes every batch in one partition's records blob. - /// - /// Two shapes here are not optional. The blob is a SEQUENCE of batches, not one batch, and - /// the broker truncates it at `partition_max_bytes` mid-batch, so a trailing partial batch - /// is normal and must be dropped rather than treated as corruption. And a control batch - /// (the transaction commit and abort markers) carries records whose payload is internal - /// bookkeeping: surfacing them would put marker rows in the user's grid. - static func decode( + // The blob is a sequence of batches cut at partition_max_bytes, so a trailing partial batch + // is normal. Control batches hold transaction markers, never rows for the grid. + // Every batch charges `budget`, so the records kept across the whole blob stay within it. + public static func decode( blob: Data, partition: Int32, abortedTransactions: [KafkaAbortedTransaction] = [], - skipAborted: Bool + skipAborted: Bool, + budget: inout DecompressionBudget ) throws -> (records: [KafkaRecord], truncated: Bool) { let bytes = [UInt8](blob) var cursor = 0 @@ -90,6 +98,17 @@ enum KafkaRecordBatchDecoder { // producer. Keying this on the offset alone let one producer's commit marker end // another's aborted run, and its rolled-back records then reached the grid. var abortedProducers = Set() + // The lowest first offset per producer answers the abort check in one lookup per batch, + // where scanning the whole list per batch grew with both counts. + var firstAbortedOffset: [Int64: Int64] = [:] + if skipAborted { + for transaction in abortedTransactions { + firstAbortedOffset[transaction.producerId] = min( + transaction.firstOffset, + firstAbortedOffset[transaction.producerId] ?? .max + ) + } + } while cursor + headerLength <= bytes.count { // Read the header in place. Re-slicing the remaining blob per batch made decoding @@ -123,7 +142,7 @@ enum KafkaRecordBatchDecoder { } if skipAborted, isTransactional, - abortedTransactions.contains(where: { $0.producerId == producerId && $0.firstOffset <= baseOffset }) { + let firstOffset = firstAbortedOffset[producerId], firstOffset <= baseOffset { abortedProducers.insert(producerId) } @@ -136,7 +155,7 @@ enum KafkaRecordBatchDecoder { } let payload = Data(bytes[(cursor + headerLength) ..< end]) - let decoded = try KafkaCompression.decompress(payload, codec: codec) + let decoded = try KafkaCompression.decompress(payload, codec: codec, budget: &budget) let shouldSkip = skipAborted && abortedProducers.contains(producerId) if !shouldSkip { records.append(contentsOf: try decodeRecords( @@ -202,9 +221,15 @@ enum KafkaRecordBatchDecoder { "record length \(length) disagrees with \(reader.offset - start) decoded bytes" ) } + // Paging steps from the last offset, so one past Int64 is a corrupt reply. Timestamps + // wrap the way Kafka's own client adds them, so a batch other clients read still reads. + let (offset, offsetOverflows) = baseOffset.addingReportingOverflow(Int64(offsetDelta)) + guard !offsetOverflows else { + throw KafkaError.malformedResponse("record offset does not fit in 64 bits") + } records.append(KafkaRecord( - offset: baseOffset + Int64(offsetDelta), - timestamp: baseTimestamp + timestampDelta, + offset: offset, + timestamp: baseTimestamp &+ timestampDelta, timestampIsLogAppendTime: logAppendTime, key: key, value: value, @@ -236,8 +261,8 @@ enum KafkaRecordBatchDecoder { /// Builds the single-record v2 batch that PRODUCE sends. The CRC is mandatory and covers the /// batch from the attributes field to the end, not the whole batch. -enum KafkaRecordBatchEncoder { - static func singleRecordBatch( +public enum KafkaRecordBatchEncoder { + public static func singleRecordBatch( key: Data?, value: Data?, headers: [KafkaRecordHeader], @@ -296,7 +321,7 @@ enum KafkaRecordBatchEncoder { } } -extension Data { +public extension Data { var hexString: String { map { String(format: "%02X", $0) }.joined() } diff --git a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaUUID.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaUUID.swift new file mode 100644 index 0000000000..6a98ba6969 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaUUID.swift @@ -0,0 +1,21 @@ +import Foundation + +public struct KafkaUUID: Hashable, Sendable { + public let bytes: [UInt8] + + public static let zero = KafkaUUID(bytes: [UInt8](repeating: 0, count: 16)) + + public init(bytes: [UInt8]) { + self.bytes = bytes.count == 16 ? bytes : [UInt8](repeating: 0, count: 16) + } + + public var isZero: Bool { bytes.allSatisfy { $0 == 0 } } + + public var uuidString: String { + let hex = bytes.map { String(format: "%02x", $0) }.joined() + let groups = [0 ..< 8, 8 ..< 12, 12 ..< 16, 16 ..< 20, 20 ..< 32] + return groups + .map { String(hex[hex.index(hex.startIndex, offsetBy: $0.lowerBound) ..< hex.index(hex.startIndex, offsetBy: $0.upperBound)]) } + .joined(separator: "-") + } +} diff --git a/apple/Plugins/KafkaDriverPlugin/SnappyDecoder.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/SnappyDecoder.swift similarity index 83% rename from apple/Plugins/KafkaDriverPlugin/SnappyDecoder.swift rename to apple/Packages/TableProCore/Sources/TableProKafkaCore/SnappyDecoder.swift index 89f2322db3..106cb0d847 100644 --- a/apple/Plugins/KafkaDriverPlugin/SnappyDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/SnappyDecoder.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWireIO /// Snappy block-format decompression. macOS ships no snappy at any layer, and Kafka's Java /// producer still writes it, so a topic produced by a Java service is unreadable without this. @@ -7,15 +8,21 @@ import Foundation /// either a literal or a back-reference copy. Only decompression is implemented, because this /// client never produces a compressed batch. enum SnappyDecoder { - static func decode(_ input: [UInt8]) throws -> [UInt8] { + // A 3-byte copy element writes at most 64 bytes, so no block holds more than this per input byte. + private static let maximumExpansion = 22 + + // The declared length is charged before anything is allocated, and decoding stops the moment + // the output passes it, so the charge always covers what the block really writes. + static func decode(_ input: [UInt8], budget: inout DecompressionBudget) throws -> [UInt8] { var cursor = 0 let expectedLength = try varint(input, &cursor) - guard expectedLength <= 512 * 1024 * 1024 else { + guard expectedLength <= 512 * 1_024 * 1_024 else { throw KafkaError.decompressionFailed(codec: "snappy", reason: "declared length is implausible") } + try KafkaCompression.charge(&budget, produced: expectedLength, consumed: input.count, codec: .snappy) var output = [UInt8]() - output.reserveCapacity(expectedLength) + output.reserveCapacity(min(expectedLength, input.count * maximumExpansion)) while cursor < input.count { let tag = input[cursor] @@ -37,6 +44,12 @@ enum SnappyDecoder { default: try copyWithOffset(width: 4, tag: tag, input: input, cursor: &cursor, output: &output) } + guard output.count <= expectedLength else { + throw KafkaError.decompressionFailed( + codec: "snappy", + reason: "decoded more than the \(expectedLength) bytes the block declared" + ) + } } guard output.count == expectedLength else { diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift index cd44dc0707..21fb9221ce 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift @@ -12,9 +12,11 @@ public struct MemcachedKeyEntry: Equatable, Sendable { self.flags = flags } - /// Seconds left at `serverTime`, nil for no expiry. An item past its time reads as gone. + // Nil for no expiry, and for a clock too far from the expiry to subtract, so the item stays listed. public func secondsLeft(at serverTime: Int64) -> Int64? { - expiresAt.map { $0 - serverTime } + guard let expiresAt else { return nil } + let (left, overflow) = expiresAt.subtractingReportingOverflow(serverTime) + return overflow ? nil : left } public func isExpired(at serverTime: Int64) -> Bool { @@ -22,6 +24,19 @@ public struct MemcachedKeyEntry: Equatable, Sendable { } } +// A server clock too far off to subtract reads as the local one, as a missing `STAT time` does. +public enum MemcachedServerClock { + public static func offset(serverTime: Int64, localTime: Int64) -> Int64 { + let (offset, overflow) = serverTime.subtractingReportingOverflow(localTime) + return overflow ? 0 : offset + } + + public static func serverTime(localTime: Int64, offset: Int64) -> Int64 { + let (time, overflow) = localTime.addingReportingOverflow(offset) + return overflow ? localTime : time + } +} + /// How the key list was read. A cachedump is a per-slab sample capped by the server, never the whole cache. public enum MemcachedKeySource: String, Sendable, Equatable { case metadump @@ -56,13 +71,12 @@ public enum MemcachedKeyListing { return MemcachedKeyEntry(key: key, expiresAt: expiry.flatMap { $0 > 0 ? $0 : nil }, flags: nil) } - /// The slab classes holding items, read from `stats items` lines such as `STAT items:1:number 3`. public static func slabClasses(fromItemStats lines: [String]) -> [Int] { - var classes: [Int] = [] + var classes = Set() for (name, value) in statPairs(lines) where name.hasSuffix(":number") && (Int(value) ?? 0) > 0 { let parts = name.split(separator: ":") - if parts.count == 3, parts[0] == "items", let id = Int(parts[1]), !classes.contains(id) { - classes.append(id) + if parts.count == 3, parts[0] == "items", let id = Int(parts[1]) { + classes.insert(id) } } return classes.sorted() diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift index 619c215945..0a02390564 100644 --- a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift @@ -32,7 +32,10 @@ public struct Neo4jUpdateCounts: Equatable, Sendable { // A property or label counter counts values, not entities. public var entitiesChanged: Int { - [.nodesCreated, .nodesDeleted, .relationshipsCreated, .relationshipsDeleted].reduce(0) { $0 + count($1) } + [Counter.nodesCreated, .nodesDeleted, .relationshipsCreated, .relationshipsDeleted].reduce(0) { total, counter in + let (sum, overflow) = total.addingReportingOverflow(count(counter)) + return overflow ? .max : sum + } } public static func == (lhs: Self, rhs: Self) -> Bool { diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift index 4285727595..594ba9d93b 100644 --- a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift @@ -103,7 +103,7 @@ public enum Neo4jValueText { var orderedRelationships: [Any] = [] var current = start for step in stride(from: 0, to: sequence.count, by: 2) { - let relationshipIndex = Int(sequence[step].magnitude) - 1 + let relationshipIndex = Int(clamping: sequence[step].magnitude) - 1 let nodeIndex = Int(sequence[step + 1]) guard sequence[step] != 0, relationships.indices.contains(relationshipIndex), nodes.indices.contains(nodeIndex), @@ -161,27 +161,48 @@ public enum Neo4jValueText { // MARK: - Temporal and spatial + // java.time's range, which bounds what Neo4j stores: LocalDate.MIN to LocalDate.MAX and ZoneOffset's + // 18 hours. A value outside it reads like any other structure that does not decode. + private static let epochDays: ClosedRange = -365_243_219_162...365_241_780_471 + private static let epochSeconds: ClosedRange = (epochDays.lowerBound * 86_400)...(epochDays.upperBound * 86_400 + 86_399) + private static let nanosOfSecond: Range = 0..<1_000_000_000 + private static let nanosOfDay: Range = 0..<86_400_000_000_000 + private static let offsetSeconds: ClosedRange = -64_800...64_800 + private static let utcSeconds: ClosedRange = + (epochSeconds.lowerBound - offsetSeconds.upperBound)...(epochSeconds.upperBound - offsetSeconds.lowerBound) + + // Tags 0x49 and 0x69 carry UTC seconds, so the range bounds them after the offset is added. + private static func localSeconds(_ seconds: Int64, plus offset: Int64) -> Int64? { + let (local, overflow) = seconds.addingReportingOverflow(offset) + return overflow || !epochSeconds.contains(local) ? nil : local + } + static func structureText(tag: UInt8, fields: [PackStreamValue]) -> String? { let ints = fields.map { $0.intValue } switch tag { case Neo4jStructureTag.date: - guard let days = ints.first ?? nil else { return nil } + guard let days = ints.first ?? nil, epochDays.contains(days) else { return nil } return dateText(days: days) case Neo4jStructureTag.localTime: - guard let nanos = ints.first ?? nil else { return nil } + guard let nanos = ints.first ?? nil, nanosOfDay.contains(nanos) else { return nil } return timeText(nanosOfDay: nanos) case Neo4jStructureTag.time: - guard ints.count == 2, let nanos = ints[0], let offset = ints[1] else { return nil } + guard ints.count == 2, let nanos = ints[0], let offset = ints[1], + nanosOfDay.contains(nanos), offsetSeconds.contains(offset) else { return nil } return timeText(nanosOfDay: nanos) + offsetText(offset) case Neo4jStructureTag.localDateTime: - guard ints.count == 2, let seconds = ints[0], let nanos = ints[1] else { return nil } + guard ints.count == 2, let seconds = ints[0], let nanos = ints[1], + epochSeconds.contains(seconds), nanosOfSecond.contains(nanos) else { return nil } return dateTimeText(localSeconds: seconds, nanos: nanos) case Neo4jStructureTag.dateTime, Neo4jStructureTag.legacyDateTime: - guard ints.count == 3, let seconds = ints[0], let nanos = ints[1], let offset = ints[2] else { return nil } - let local = tag == Neo4jStructureTag.dateTime ? seconds + offset : seconds + guard ints.count == 3, let seconds = ints[0], let nanos = ints[1], let offset = ints[2], + nanosOfSecond.contains(nanos), offsetSeconds.contains(offset), + let local = localSeconds(seconds, plus: tag == Neo4jStructureTag.dateTime ? offset : 0) + else { return nil } return dateTimeText(localSeconds: local, nanos: nanos) + offsetText(offset) case Neo4jStructureTag.dateTimeZoneId, Neo4jStructureTag.legacyDateTimeZoneId: guard fields.count == 3, let seconds = ints[0], let nanos = ints[1], + utcSeconds.contains(seconds), nanosOfSecond.contains(nanos), let zoneId = fields[2].stringValue else { return nil } let zone = TimeZone(identifier: zoneId) let offset: Int64 @@ -191,7 +212,9 @@ public enum Neo4jValueText { let guess = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds))) ?? 0) offset = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds - guess))) ?? 0) } - let local = tag == Neo4jStructureTag.dateTimeZoneId ? seconds + offset : seconds + guard let local = localSeconds(seconds, plus: tag == Neo4jStructureTag.dateTimeZoneId ? offset : 0) else { + return nil + } return dateTimeText(localSeconds: local, nanos: nanos) + offsetText(offset) + "[\(zoneId)]" case Neo4jStructureTag.duration: guard ints.count == 4, let months = ints[0], let days = ints[1], @@ -257,7 +280,8 @@ public enum Neo4jValueText { let carry = floorDivide(nanos, 1_000_000_000) let (total, overflow) = seconds.addingReportingOverflow(carry) let signedSeconds = overflow ? seconds : total - var fraction = nanos - carry * 1_000_000_000 + var fraction = nanos % 1_000_000_000 + if fraction < 0 { fraction += 1_000_000_000 } if signedSeconds != 0 || fraction != 0 { let negative = signedSeconds < 0 var wholeSeconds = signedSeconds.magnitude diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift index 0894a0ae07..962faf11df 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -180,6 +180,8 @@ public struct QdrantCollection: Sendable, Equatable { public let indexes: [QdrantPayloadIndex] public let pointsCount: Int? public let info: String + private let fieldPositions: [String: Int] + private let vectorPositions: [String: Int] public init( name: String, @@ -195,6 +197,12 @@ public struct QdrantCollection: Sendable, Equatable { self.indexes = indexes self.pointsCount = pointsCount self.info = info + fieldPositions = Self.firstPositions(of: payloadFields.map(\.name)) + vectorPositions = Self.firstPositions(of: vectors.map(\.column)) + } + + private static func firstPositions(of names: [String]) -> [String: Int] { + Dictionary(names.enumerated().map { ($1, $0) }, uniquingKeysWith: { first, _ in first }) } // Only an index states a field's type; a sample guesses it from 100 points. @@ -205,15 +213,18 @@ public struct QdrantCollection: Sendable, Equatable { // `id`, `score` and the vectors are the point's own; a payload key of that name is `payload.`. private func field(ofColumn column: String) -> QdrantPayloadField? { guard !QdrantSchema.isPointColumn(column) else { return nil } - let key = QdrantSchema.payloadKey(forColumn: column) - return payloadFields.first { $0.name == key } + return fieldPositions[QdrantSchema.payloadKey(forColumn: column)].map { payloadFields[$0] } + } + + func vector(ofColumn column: String) -> QdrantVectorSpec? { + vectorPositions[column].map { vectors[$0] } } // A page's points widen the types a sample guessed, so a list first seen on page 2 is not saved as // text. `addingKeys` is false for a grid page, whose columns stay the ones the table opened with. public func including(_ points: [QdrantPoint], addingKeys: Bool = true) -> QdrantCollection { var fields = payloadFields - var position = Dictionary(uniqueKeysWithValues: fields.enumerated().map { ($1.name, $0) }) + var position = fieldPositions for point in points { for key in point.payloadKeys { let observed = point.payload[key].flatMap(QdrantFieldType.observed) @@ -269,10 +280,11 @@ public struct QdrantCollection: Sendable, Equatable { let schema = result["payload_schema"] as? [String: Any] ?? [:] var indexes: [QdrantPayloadIndex] = [] var fields: [QdrantPayloadField] = [] + var fieldNames: Set = [] for path in schema.keys.sorted() { guard let type = QdrantFieldType.indexed(schema[path]) else { continue } indexes.append(QdrantPayloadIndex(path: path, type: type.rawValue)) - if let key = QdrantKeyPath.topLevelKey(ofIndexPath: path), !fields.contains(where: { $0.name == key }) { + if let key = QdrantKeyPath.topLevelKey(ofIndexPath: path), fieldNames.insert(key).inserted { fields.append(QdrantPayloadField(name: key, type: type, isIndexed: true)) } } @@ -282,7 +294,7 @@ public struct QdrantCollection: Sendable, Equatable { vectors: vectors, payloadFields: fields, indexes: indexes, - pointsCount: result["points_count"] as? Int, + pointsCount: (result["points_count"] as? Int).flatMap { $0 >= 0 ? $0 : nil }, info: (try? QdrantJSON.text(result, pretty: true)) ?? "{}" ).including(sample) } @@ -353,7 +365,7 @@ public enum QdrantSchema { public static func knownTypeName(of column: String, in collection: QdrantCollection) -> String? { if column == idColumn { return "id" } - if let spec = collection.vectors.first(where: { $0.column == column }) { + if let spec = collection.vector(ofColumn: column) { return spec.typeName } return collection.cellTypeName(ofColumn: column) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift index 28e1f0c544..98a69f3420 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift @@ -86,7 +86,7 @@ public final class QdrantClient: @unchecked Sendable { } let response = try await post("\(collectionPath(collection))/points/count", body) let result = (response.json as? [String: Any])?["result"] as? [String: Any] - guard let count = result?["count"] as? Int else { + guard let count = result?["count"] as? Int, count >= 0 else { throw QdrantError.malformedResponse(String(localized: "Qdrant answered a count without a number.")) } return count diff --git a/apple/Packages/TableProCore/Sources/TableProTabular/TabularTable.swift b/apple/Packages/TableProCore/Sources/TableProTabular/TabularTable.swift index 262c69b37e..d371f2b696 100644 --- a/apple/Packages/TableProCore/Sources/TableProTabular/TabularTable.swift +++ b/apple/Packages/TableProCore/Sources/TableProTabular/TabularTable.swift @@ -19,6 +19,7 @@ public struct TabularTable: Sendable { public private(set) var rowOrder: TabularRowOrder public private(set) var headerRowKey: Int? public private(set) var generation: Int + private var columnIndexByID: [TabularColumnID: Int] private var nextRowKey: Int private var nextColumnID: Int @@ -41,6 +42,7 @@ public struct TabularTable: Sendable { built.append(TabularColumn(id: TabularColumnID(rawValue: column), name: name, values: ColumnValues(base: .source(column)))) } columns = built + columnIndexByID = Self.indexByID(of: built) headerRowKey = hasHeader ? 0 : nil rowOrder = .range((hasHeader ? 1 : 0).. Int? { - columns.firstIndex { $0.id == id } + columnIndexByID[id] } public func column(_ id: TabularColumnID) -> TabularColumn? { - columns.first { $0.id == id } + columnIndexByID[id].map { columns[$0] } } public func key(atRow logicalRow: Int) -> Int { @@ -117,12 +119,8 @@ public struct TabularTable: Sendable { public mutating func setCells(_ updates: [(key: Int, columnID: TabularColumnID, cell: TabularCell)]) { guard !updates.isEmpty else { return } - var indexByID: [TabularColumnID: Int] = [:] - for (index, column) in columns.enumerated() { - indexByID[column.id] = index - } for update in updates { - guard let index = indexByID[update.columnID] else { continue } + guard let index = columnIndexByID[update.columnID] else { continue } columns[index].values.edits[update.key] = update.cell } generation += 1 @@ -172,6 +170,7 @@ public struct TabularTable: Sendable { nextColumnID += 1 let column = TabularColumn(id: id, name: name, values: ColumnValues(base: .constant(fill ?? source.absentCell))) columns.insert(column, at: min(max(0, index), columns.count)) + reindexColumns() generation += 1 return id } @@ -181,6 +180,7 @@ public struct TabularTable: Sendable { let id = TabularColumnID(rawValue: nextColumnID) nextColumnID += 1 columns.insert(TabularColumn(id: id, name: name, values: values), at: min(max(0, index), columns.count)) + reindexColumns() generation += 1 return id } @@ -188,6 +188,7 @@ public struct TabularTable: Sendable { public mutating func deleteColumns(_ ids: Set) { guard !ids.isEmpty else { return } columns.removeAll { ids.contains($0.id) } + reindexColumns() generation += 1 } @@ -201,6 +202,7 @@ public struct TabularTable: Sendable { guard let index = columnIndex(of: id) else { return } let column = columns.remove(at: index) columns.insert(column, at: min(max(0, destination), columns.count)) + reindexColumns() generation += 1 } @@ -234,4 +236,17 @@ public struct TabularTable: Sendable { } generation += 1 } + + private mutating func reindexColumns() { + columnIndexByID = Self.indexByID(of: columns) + } + + private static func indexByID(of columns: [TabularColumn]) -> [TabularColumnID: Int] { + var index: [TabularColumnID: Int] = [:] + index.reserveCapacity(columns.count) + for (position, column) in columns.enumerated() { + index[column.id] = position + } + return index + } } diff --git a/apple/Packages/TableProCore/Sources/TableProTabular/TabularTypeInference.swift b/apple/Packages/TableProCore/Sources/TableProTabular/TabularTypeInference.swift index 33523b8e47..299e28610e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTabular/TabularTypeInference.swift +++ b/apple/Packages/TableProCore/Sources/TableProTabular/TabularTypeInference.swift @@ -23,40 +23,72 @@ public enum TabularInferredKind: String, Sendable, Equatable, CaseIterable { public enum TabularTypeInference { public static let sampleSize = 200 + // Columns still short of a full sample when this many cells have been read are typed from what they have. + public static let cellBudget = 10_000_000 private static let booleanWords: Set = ["true", "false", "yes", "no", "t", "f", "y", "n"] - public static func inferKinds(of table: TabularTable) -> [TabularColumnID: TabularInferredKind] { + public static func inferKinds( + of table: TabularTable, + cellBudget: Int = TabularTypeInference.cellBudget + ) throws -> [TabularColumnID: TabularInferredKind] { var samples: [TabularColumnID: [(TabularCellKind, String)]] = [:] - let ids = table.columnIDs - for id in ids { - samples[id] = [] - } - var open = Set(ids) + var open = table.columnIDs var row = 0 - let rows = table.rowCount - while row < rows, !open.isEmpty { - let end = min(rows, row + 256) - table.scan(columns: ids, rows: row..= sampleSize { - open.remove(id) - } - } - return !open.isEmpty - } - row = end + var budget = cellBudget + while row < table.rowCount, !open.isEmpty, budget >= open.count { + let pass = try samplePass(table, columns: open, from: row, budget: budget, into: &samples) + guard pass.rows > 0 else { break } + row += pass.rows + budget -= pass.rows * open.count + open = pass.open } var result: [TabularColumnID: TabularInferredKind] = [:] - for id in ids { + for id in table.columnIDs { result[id] = infer(samples[id] ?? []) } return result } + // A pass ends once half its columns have a full sample, so the next scan plan is built for the rest only. + private static func samplePass( + _ table: TabularTable, + columns: [TabularColumnID], + from start: Int, + budget: Int, + into samples: inout [TabularColumnID: [(TabularCellKind, String)]] + ) throws -> (rows: Int, open: [TabularColumnID]) { + var isOpen = [Bool](repeating: true, count: columns.count) + var openCount = columns.count + var visited = 0 + var cellsSinceCheck = 0 + var cancelled = false + let end = start + min(table.rowCount - start, budget / columns.count) + table.scan(columns: columns, rows: start..= sampleSize { + isOpen[slot] = false + openCount -= 1 + } + } + cellsSinceCheck += columns.count + if cellsSinceCheck >= TabularScanEngine.cancellationStride { + cellsSinceCheck = 0 + if Task.isCancelled { + cancelled = true + return false + } + } + return openCount * 2 > columns.count + } + if cancelled { throw CancellationError() } + return (visited, columns.indices.filter { isOpen[$0] }.map { columns[$0] }) + } + public static func infer(_ sample: [(TabularCellKind, String)]) -> TabularInferredKind { guard !sample.isEmpty else { return .text } if sample.allSatisfy({ $0.0 == .number }) { diff --git a/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXColumnStore.swift b/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXColumnStore.swift index 9244007a78..eccff08928 100644 --- a/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXColumnStore.swift +++ b/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXColumnStore.swift @@ -53,11 +53,21 @@ struct XLSXColumnStore: Sendable { hint = slot return kinds[slot] == XLSXStoredCell.empty ? nil : slot } + + func forEachOccupiedRow(_ body: (Int) -> Void) { + for slot in 0..() @@ -66,12 +76,15 @@ struct XLSXColumnBuilder { var isEmpty: Bool { kinds.isEmpty } + var lastRowIndex: Int { lastRow } + mutating func append(row: Int, kind: UInt8, payload: UInt64) { guard !kinds.isEmpty else { firstRow = row lastRow = row kinds.append(kind) payloads.append(payload) + storedCount = 1 return } if row == lastRow, isSorted { @@ -85,7 +98,8 @@ struct XLSXColumnBuilder { } let gap = row - lastRow - 1 if isDense, gap > 0 { - if gap <= max(64, kinds.count / 8) { + let spread = kinds.count + gap < Self.largestDenseSpread * (storedCount + 1) + if gap <= max(64, kinds.count / 8), spread { kinds.append(XLSXStoredCell.empty, times: gap) payloads.append(0, times: gap) } else { @@ -95,13 +109,13 @@ struct XLSXColumnBuilder { if !isDense { rows.append(UInt32(row)) } kinds.append(kind) payloads.append(payload) + storedCount += 1 lastRow = row } - mutating func clear(rows range: ClosedRange, keeping kept: Int?) { + mutating func clear(where isHidden: (Int) -> Bool) { normalize() - for slot in slots(in: range) { - if let kept, row(at: slot) == kept { continue } + for slot in 0..) -> Range { - guard !kinds.isEmpty else { return 0..<0 } - if isDense { - let lower = max(0, range.lowerBound - firstRow) - let upper = min(kinds.count, range.upperBound - firstRow + 1) - return lower < upper ? lower.., isFinal: Bool) -> Int { @@ -86,13 +86,10 @@ struct XLSXSheetReader { } } - mutating func finish(name: String, sharedStrings: XLSXSharedStrings) -> XLSXSheetSource { - for merge in merges { - for column in merge.columns where column < builders.count { - let kept = column == merge.start.column ? merge.start.row : nil - builders[column].clear(rows: merge.rows, keeping: kept) - } - } + mutating func finish(name: String, sharedStrings: XLSXSharedStrings, isCancelled: () -> Bool) throws -> XLSXSheetSource { + var seen = Set() + let distinctMerges = merges.filter { seen.insert($0).inserted } + try hideMergedCells(distinctMerges, isCancelled: isCancelled) var rowBounds: ClosedRange? var columnBounds: ClosedRange? for column in builders.indices { @@ -104,7 +101,7 @@ struct XLSXSheetReader { guard let rowBounds, let columnBounds else { return XLSXSheetSource( name: name, - layout: XLSXSheetSource.Layout(rowCount: 0, firstRowIndex: 0, firstColumnIndex: 0, mergedRanges: merges), + layout: XLSXSheetSource.Layout(rowCount: 0, firstRowIndex: 0, firstColumnIndex: 0, mergedRanges: distinctMerges), columns: [], arena: arenaHeap, sharedStrings: sharedStrings, @@ -123,7 +120,7 @@ struct XLSXSheetReader { rowCount: rowBounds.count, firstRowIndex: rowBounds.lowerBound, firstColumnIndex: columnBounds.lowerBound, - mergedRanges: merges + mergedRanges: distinctMerges ), columns: stores, arena: arenaHeap, @@ -132,6 +129,40 @@ struct XLSXSheetReader { ) } + // A merge hides every cell it covers except its top-left one. Sweeping columns with a running count of + // the ranges over each row costs each range two updates and each stored cell one lookup, however the + // ranges overlap. + private mutating func hideMergedCells(_ merges: [XLSXCellRange], isCancelled: () -> Bool) throws { + let lastRow = builders.reduce(-1) { max($0, $1.lastRowIndex) } + let ranges = merges.filter { $0.start.column < builders.count && $0.start.row <= lastRow } + guard !ranges.isEmpty else { return } + var topLeftCounts: [XLSXCellReference: Int] = [:] + for range in ranges { + topLeftCounts[range.start, default: 0] += 1 + } + let byStart = ranges.sorted { $0.start.column < $1.start.column } + let byEnd = ranges.sorted { $0.end.column < $1.end.column } + var coverage = XLSXRowCoverage(rowCount: lastRow + 1) + var nextStart = 0 + var nextEnd = 0 + for column in builders.indices { + if isCancelled() { throw TabularCancellation() } + while nextStart < byStart.count, byStart[nextStart].start.column <= column { + coverage.add(1, rows: byStart[nextStart].start.row...min(byStart[nextStart].end.row, lastRow)) + nextStart += 1 + } + while nextEnd < byEnd.count, byEnd[nextEnd].end.column < column { + coverage.add(-1, rows: byEnd[nextEnd].start.row...min(byEnd[nextEnd].end.row, lastRow)) + nextEnd += 1 + } + guard nextStart > nextEnd else { continue } + builders[column].clear { row in + let covering = coverage.count(at: row) + return covering > 0 && covering > topLeftCounts[XLSXCellReference(row: row, column: column), default: 0] + } + } + } + private mutating func beginRow(_ scanner: XMLByteScanner, attributes: Range) { let declared = scanner.attribute("r", in: attributes).flatMap(scanner.integer) if let declared, declared >= 1, declared <= XLSXCellReference.maximumRowCount { @@ -344,3 +375,34 @@ struct XLSXSheetReader { builders[column].append(row: position.row, kind: kind, payload: payload) } } + +private struct XLSXRowCoverage { + private var tree: [Int] + + init(rowCount: Int) { + tree = [Int](repeating: 0, count: rowCount + 1) + } + + mutating func add(_ delta: Int, rows: ClosedRange) { + update(at: rows.lowerBound, by: delta) + update(at: rows.upperBound + 1, by: -delta) + } + + func count(at row: Int) -> Int { + var position = row + 1 + var total = 0 + while position > 0 { + total += tree[position] + position &= position - 1 + } + return total + } + + private mutating func update(at row: Int, by delta: Int) { + var position = row + 1 + while position < tree.count { + tree[position] += delta + position += position & -position + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXSheetSource.swift b/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXSheetSource.swift index 0033ca1ffe..fc913a19a7 100644 --- a/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXSheetSource.swift +++ b/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXSheetSource.swift @@ -79,9 +79,29 @@ public struct XLSXSheetSource: TabularSource { return (0.. [Int] { + var occupied = [Bool](repeating: false, count: rowCount) + for store in columns { + store.forEachOccupiedRow { row in + if row >= 0, row < occupied.count { occupied[row] = true } + } + } + return occupied.indices.filter { occupied[$0] } + } + + public func scan( + columns requested: [Int], + rows: Rows, + _ body: (Int, TabularRowCells) -> Bool + ) where Rows.Element == Int { + scan(columns: requested, rows: rows, absentKind: .text, body) + } + + // `.missing` tells a slot with no stored cell apart from a stored cell whose text is empty. public func scan( columns requested: [Int], rows: Rows, + absentKind: TabularCellKind, _ body: (Int, TabularRowCells) -> Bool ) where Rows.Element == Int { guard !requested.isEmpty else { return } @@ -90,7 +110,7 @@ public struct XLSXSheetSource: TabularSource { var hints = [Int](repeating: -1, count: requested.count) for row in rows { guard row >= 0, row < rowCount else { continue } - buffer.reset(slots: requested.count, fill: .text) + buffer.reset(slots: requested.count, fill: absentKind) for (slot, column) in requested.enumerated() where column >= 0 && column < columnCount { let store = columns[column] guard let index = store.slot(forRow: row, hint: &hints[slot]) else { continue } diff --git a/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXStoredCell.swift b/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXStoredCell.swift index 0d99efb68a..b50f6ed1cd 100644 --- a/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXStoredCell.swift +++ b/apple/Packages/TableProCore/Sources/TableProTabularIO/XLSXStoredCell.swift @@ -48,9 +48,9 @@ enum XLSXDecimal { var mantissa: Int64 = 0 var digits = 0 while index < count, isDigit(bytes[index]) { + guard digits < XLSXStoredCell.largestDecimalScale else { return nil } mantissa = mantissa * 10 + Int64(bytes[index] - 0x30) digits += 1 - guard digits <= XLSXStoredCell.largestDecimalScale else { return nil } index += 1 } let integerDigits = index - integerStart @@ -59,10 +59,10 @@ enum XLSXDecimal { if index < count, bytes[index] == 0x2E { index += 1 while index < count, isDigit(bytes[index]) { + guard digits < XLSXStoredCell.largestDecimalScale else { return nil } mantissa = mantissa * 10 + Int64(bytes[index] - 0x30) digits += 1 scale += 1 - guard digits <= XLSXStoredCell.largestDecimalScale else { return nil } index += 1 } guard scale > 0 else { return nil } diff --git a/apple/Packages/TableProCore/Sources/TableProTabularIO/ZipArchive.swift b/apple/Packages/TableProCore/Sources/TableProTabularIO/ZipArchive.swift index e1beff1339..242e2e1e77 100644 --- a/apple/Packages/TableProCore/Sources/TableProTabularIO/ZipArchive.swift +++ b/apple/Packages/TableProCore/Sources/TableProTabularIO/ZipArchive.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWireIO public struct ZipArchive: Sendable { public struct Entry: Sendable, Equatable { @@ -46,6 +47,13 @@ public struct ZipArchive: Sendable { public static let storedMethod: UInt16 = 0 public static let deflateMethod: UInt16 = 8 public static let bufferedEntryLimit = 256 << 20 + public static let bufferedBudget = DecompressionBudget( + limit: bufferedEntryLimit, + ratioLimit: DecompressionBudget.standard.ratioLimit, + grace: DecompressionBudget.standard.grace + ) + // The declared size is unchecked until the entry inflates; past this, Data's own doubling costs little. + private static let largestReservation = 16 << 20 public let bytes: Data public let entries: [String: Entry] @@ -82,15 +90,11 @@ public struct ZipArchive: Sendable { return try data(for: entry) } - public func data( - for entry: Entry, - limit: Int = bufferedEntryLimit, - policy: ZipExpansionPolicy = .standard - ) throws -> Data { - guard entry.uncompressedSize <= limit else { throw Failure.entryTooLarge(entry.path) } - return try withReader(for: entry, policy: policy) { reader in + public func data(for entry: Entry, budget: DecompressionBudget = bufferedBudget) throws -> Data { + guard entry.uncompressedSize <= budget.remaining else { throw Failure.entryTooLarge(entry.path) } + return try withReader(for: entry, budget: budget) { reader in var output = Data() - output.reserveCapacity(max(entry.uncompressedSize, 0)) + output.reserveCapacity(min(max(entry.uncompressedSize, 0), Self.largestReservation)) let chunk = UnsafeMutableBufferPointer.allocate(capacity: 256 * 1_024) defer { chunk.deallocate() } while true { @@ -103,13 +107,13 @@ public struct ZipArchive: Sendable { public func withReader( for entry: Entry, - policy: ZipExpansionPolicy = .standard, + budget: DecompressionBudget = .standard, _ body: (inout ZipEntryReader) throws -> Result ) throws -> Result { try bytes.withUnsafeBytes { raw in let archive = raw.bindMemory(to: UInt8.self) let payload = try ZipCentralDirectory.payload(of: entry, in: archive) - var reader = try ZipEntryReader(entry: entry, payload: payload, policy: policy) + var reader = try ZipEntryReader(entry: entry, payload: payload, budget: budget) return try body(&reader) } } diff --git a/apple/Packages/TableProCore/Sources/TableProTabularIO/ZipEntryReader.swift b/apple/Packages/TableProCore/Sources/TableProTabularIO/ZipEntryReader.swift index e01634c919..babf78b19f 100644 --- a/apple/Packages/TableProCore/Sources/TableProTabularIO/ZipEntryReader.swift +++ b/apple/Packages/TableProCore/Sources/TableProTabularIO/ZipEntryReader.swift @@ -1,31 +1,21 @@ import Compression import Foundation - -public struct ZipExpansionPolicy: Sendable, Equatable { - public let ratioLimit: Int - public let grace: Int - - public init(ratioLimit: Int, grace: Int) { - self.ratioLimit = ratioLimit - self.grace = grace - } - - public static let standard = ZipExpansionPolicy(ratioLimit: 250, grace: 64 << 20) -} +import TableProWireIO public struct ZipEntryReader: ~Copyable { public let entry: ZipArchive.Entry - private let policy: ZipExpansionPolicy + private var budget: DecompressionBudget private let payload: UnsafeBufferPointer private let inflater: UnsafeMutablePointer? private var storedOffset = 0 + private var recordedConsumed = 0 private var isFinished: Bool public private(set) var bytesProduced = 0 - init(entry: ZipArchive.Entry, payload: UnsafeBufferPointer, policy: ZipExpansionPolicy = .standard) throws { + init(entry: ZipArchive.Entry, payload: UnsafeBufferPointer, budget: DecompressionBudget = .standard) throws { self.entry = entry self.payload = payload - self.policy = policy + self.budget = budget switch entry.compressionMethod { case ZipArchive.storedMethod: inflater = nil @@ -69,7 +59,10 @@ public struct ZipEntryReader: ~Copyable { public mutating func read(into destination: UnsafeMutableBufferPointer) throws -> Int { guard !isFinished, let destinationBase = destination.baseAddress, !destination.isEmpty else { return 0 } guard let inflater else { - return readStored(into: destinationBase, capacity: destination.count) + let copied = readStored(into: destinationBase, capacity: destination.count) + bytesProduced += copied + try record(copied) + return copied } inflater.pointee.dst_ptr = destinationBase inflater.pointee.dst_size = destination.count @@ -95,12 +88,18 @@ public struct ZipEntryReader: ~Copyable { isFinished = true throw ZipArchive.Failure.corruptEntry(entry.path) } - guard bytesProduced > policy.grace else { return } - let consumed = max(compressedBytesConsumed, 1) - guard bytesProduced / consumed <= policy.ratioLimit else { + try record(produced) + } + + private mutating func record(_ produced: Int) throws { + let consumed = compressedBytesConsumed + do { + try budget.record(produced: produced, consumed: consumed - recordedConsumed) + } catch { isFinished = true throw ZipArchive.Failure.entryTooLarge(entry.path) } + recordedConsumed = consumed } private mutating func readStored(into destination: UnsafeMutablePointer, capacity: Int) -> Int { diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/ByteIO.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/ByteIO.swift index 8d7b3e65a1..a3a7e4a7e5 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/ByteIO.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/ByteIO.swift @@ -45,45 +45,3 @@ struct ByteWriter { bytes.append(contentsOf: encoded) } } - -struct ByteReader { - private let bytes: [UInt8] - private(set) var offset: Int - - init(_ bytes: [UInt8], offset: Int = 0) { - self.bytes = bytes - self.offset = offset - } - - var remaining: Int { bytes.count - offset } - - mutating func u8() throws -> UInt8 { - guard remaining >= 1 else { throw TeradataWireError.truncated("u8") } - defer { offset += 1 } - return bytes[offset] - } - - mutating func u16() throws -> UInt16 { - guard remaining >= 2 else { throw TeradataWireError.truncated("u16") } - defer { offset += 2 } - return UInt16(bytes[offset]) << 8 | UInt16(bytes[offset + 1]) - } - - mutating func u32() throws -> UInt32 { - guard remaining >= 4 else { throw TeradataWireError.truncated("u32") } - defer { offset += 4 } - return UInt32(bytes[offset]) << 24 | UInt32(bytes[offset + 1]) << 16 - | UInt32(bytes[offset + 2]) << 8 | UInt32(bytes[offset + 3]) - } - - mutating func take(_ count: Int) throws -> [UInt8] { - guard remaining >= count else { throw TeradataWireError.truncated("take(\(count))") } - defer { offset += count } - return Array(bytes[offset..= count else { throw TeradataWireError.truncated("skip(\(count))") } - offset += count - } -} diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Der.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Der.swift index f321f96cf8..f13b73bac6 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Der.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Der.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWireIO enum Der { static let objectIdentifierTag: UInt8 = 0x06 @@ -51,10 +52,13 @@ enum Der { static func decodeLength(_ reader: inout ByteReader) throws -> Int { let first = try reader.u8() if first < 0x80 { return Int(first) } - let count = Int(first & 0x7F) - var length = 0 - for _ in 0.. [UInt8] { diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift index e6cef5e0ed..e00a7551a5 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWireIO enum ParcelFlavor: UInt16 { case request = 1 diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift index 893c4ad86b..157d694a8c 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWireIO enum RecordDecoder { static func nullBitmapLength(columnCount: Int) -> Int { diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Td2Token.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Td2Token.swift index 08c68bdca1..54cf350e95 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Td2Token.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Td2Token.swift @@ -2,6 +2,9 @@ import Foundation enum Td2Token { static let keyDataOffset = 80 + // 8,192 bits, the largest standard MODP group; the client token asks for 2,048. The modular + // exponentiation costs the square of this, and the connect deadline cannot interrupt it. + static let maxKeyByteCount = 1_024 static let clientInitToken: [UInt8] = HexBytes.decode(""" 01 01 01 00 00 00 00 9D 00 00 00 15 02 00 00 00 @@ -46,6 +49,9 @@ enum Td2Token { let nY = Int(be32(token, 28)) let verifyDHKey = be32(token, 32) let nQ = (peerCapabilities & 4) != 0 ? Int(be32(token, 36)) : 0 + for (length, what) in [(nN, "prime"), (nG, "generator"), (nY, "server public key")] where length > maxKeyByteCount { + throw TeradataWireError.malformed("TD2 \(what) of \(length) bytes") + } var cursor = keyDataOffset func slice(_ length: Int, _ what: String) throws -> [UInt8] { @@ -54,6 +60,10 @@ enum Td2Token { return Array(token[cursor.. 0 ? try slice(nQ, "qopDer") : [] diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Td2Wrap.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Td2Wrap.swift index 85108c9f95..abf3ee649c 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Td2Wrap.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Td2Wrap.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWireIO enum Td2Wrap { static let macLen = 32 diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift index 65f139e2db..b8c409d28f 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWireIO enum TeradataResultParser { static func parse(_ parcels: [Parcel]) throws -> TeradataResultSet { diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift index 664b4a53f3..a82563f33b 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift @@ -32,6 +32,8 @@ protocol TeradataTransport: AnyObject { } final class TeradataSocket: TeradataTransport { + private static let receiveChunkLength = 64 * 1_024 + private let lock = NSLock() private var descriptor: Int32 = -1 private var closed = false @@ -131,29 +133,31 @@ final class TeradataSocket: TeradataTransport { } } + // The count comes from a reply header, so the result grows with the bytes that actually arrive. func receive(_ count: Int) throws -> [UInt8] { guard count > 0 else { return [] } - var buffer = [UInt8](repeating: 0, count: count) - var offset = 0 - try buffer.withUnsafeMutableBytes { raw in - guard let base = raw.baseAddress else { - throw TeradataWireError.truncated("receive buffer unavailable") - } - while offset < count { - try applyCurrentTimeout() - let read = Darwin.recv(descriptor, base.advanced(by: offset), count - offset, 0) - if read == 0 { - if isClosed { throw TeradataWireError.cancelled } - throw TeradataWireError.truncated("peer closed after \(offset)/\(count)") - } - if read < 0 { - if isClosed { throw TeradataWireError.cancelled } - throw TeradataWireError.truncated("recv errno \(errno)") + var received: [UInt8] = [] + var chunk = [UInt8](repeating: 0, count: min(count, Self.receiveChunkLength)) + while received.count < count { + try applyCurrentTimeout() + let wanted = min(count - received.count, chunk.count) + let read = try chunk.withUnsafeMutableBytes { raw -> Int in + guard let base = raw.baseAddress else { + throw TeradataWireError.truncated("receive buffer unavailable") } - offset += read + return Darwin.recv(descriptor, base, wanted, 0) + } + if read == 0 { + if isClosed { throw TeradataWireError.cancelled } + throw TeradataWireError.truncated("peer closed after \(received.count)/\(count)") + } + if read < 0 { + if isClosed { throw TeradataWireError.cancelled } + throw TeradataWireError.truncated("recv errno \(errno)") } + received.append(contentsOf: chunk[.. timeval { diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTLSTransport.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTLSTransport.swift index aeaf0f7d03..6283e318a9 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTLSTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTLSTransport.swift @@ -2,6 +2,7 @@ import Foundation import Network import os import Security +import TableProWireIO internal enum TeradataTrustPolicy { case acceptAny @@ -14,12 +15,17 @@ final class TeradataTLSTransport: TeradataTransport { private let condition = NSCondition() private let deadlineLock = NSLock() private var connectDeadline: TeradataConnectDeadline? - private var rawBuffer: [UInt8] = [] - private var messageBuffer: [UInt8] = [] + private var rawBuffer = ByteQueue() + private var messageBuffer = ByteQueue() private var receiveError: Error? private var peerClosed = false private var cancelled = false private var handshakeComplete = false + private var receivePaused = false + + // Above this rawBuffer holds at least one whole frame, so the next drain always makes room. + private static let rawBufferLimit = WebSocketFrame.maxPayload + WebSocketFrame.maxHeaderLength + private static let maxUpgradeResponseLength = 64 * 1_024 init(host: String, options: TeradataTLSOptions, deadline: TeradataConnectDeadline) throws { connectDeadline = deadline @@ -78,16 +84,14 @@ final class TeradataTLSTransport: TeradataTransport { defer { condition.unlock() } let deadline = try waitDeadline() while messageBuffer.count < count { - try drainFramesLocked() + try drainFramesLocked(until: count) if messageBuffer.count >= count { break } if cancelled { throw TeradataWireError.cancelled } if let receiveError { throw TeradataWireError.truncated("TLS recv: \(receiveError)") } if peerClosed { throw TeradataWireError.truncated("WebSocket closed after \(messageBuffer.count)/\(count)") } if !condition.wait(until: deadline) { throw TeradataWireError.truncated("TLS recv timed out") } } - let result = Array(messageBuffer.prefix(count)) - messageBuffer.removeFirst(count) - return result + return messageBuffer.removeFirst(count) } func cancel() { stop() } @@ -134,12 +138,14 @@ final class TeradataTLSTransport: TeradataTransport { defer { condition.unlock() } let deadline = try waitDeadline() let terminator: [UInt8] = [0x0D, 0x0A, 0x0D, 0x0A] + var searchStart = 0 while true { - if let range = Self.range(of: terminator, in: rawBuffer) { - let end = range.upperBound - let header = Array(rawBuffer[.. Self.maxUpgradeResponseLength { + throw TeradataWireError.connectionFailed("WebSocket upgrade response has no end in its first \(Self.maxUpgradeResponseLength) bytes") } if cancelled { throw TeradataWireError.cancelled } if let receiveError { throw TeradataWireError.truncated("WebSocket handshake recv: \(receiveError)") } @@ -148,8 +154,10 @@ final class TeradataTLSTransport: TeradataTransport { } } - private func drainFramesLocked() throws { - while let frame = try WebSocketFrame.decode(&rawBuffer) { + // Frames wait in rawBuffer until a read needs them, and the receive loop pauses while rawBuffer is + // over its limit, so a peer streaming data nobody reads cannot grow either buffer without bound. + private func drainFramesLocked(until wanted: Int) throws { + while messageBuffer.count < wanted, let frame = try WebSocketFrame.decode(&rawBuffer) { switch frame.opcode { case .binary, .continuation: messageBuffer.append(contentsOf: frame.payload) @@ -161,6 +169,10 @@ final class TeradataTLSTransport: TeradataTransport { break } } + if receivePaused, rawBuffer.count <= Self.rawBufferLimit, !cancelled { + receivePaused = false + startReceiveLoop() + } } private func sendPong(_ payload: [UInt8]) { @@ -204,21 +216,17 @@ final class TeradataTLSTransport: TeradataTransport { if let data, !data.isEmpty { self.rawBuffer.append(contentsOf: data) } if let error { self.receiveError = error } if isComplete { self.peerClosed = true } - let shouldContinue = error == nil && !isComplete && !self.cancelled + var shouldContinue = error == nil && !isComplete && !self.cancelled + if shouldContinue, self.rawBuffer.count > Self.rawBufferLimit { + self.receivePaused = true + shouldContinue = false + } self.condition.signal() self.condition.unlock() if shouldContinue { self.startReceiveLoop() } } } - private static func range(of needle: [UInt8], in haystack: [UInt8]) -> Range? { - guard !needle.isEmpty, haystack.count >= needle.count else { return nil } - for start in 0...(haystack.count - needle.count) where Array(haystack[start.. TeradataTrustPolicy { guard options.verifiesCertificate else { return .acceptAny } let hostname = options.verifiesHostname ? host : nil diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/WebSocketFrame.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/WebSocketFrame.swift index 9c5a88780a..e4c4cd40cf 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/WebSocketFrame.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/WebSocketFrame.swift @@ -1,5 +1,6 @@ import CryptoKit import Foundation +import TableProWireIO enum WebSocketOpcode: UInt8 { case continuation = 0x0 @@ -13,7 +14,8 @@ struct WebSocketFrame { let opcode: WebSocketOpcode let payload: [UInt8] - private static let maxPayload = 64 * 1_024 * 1_024 + static let maxPayload = 64 * 1_024 * 1_024 + static let maxHeaderLength = 14 static func acceptKey(for clientKey: String) -> String { let magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" @@ -44,40 +46,41 @@ struct WebSocketFrame { return frame } - static func decode(_ buffer: inout [UInt8]) throws -> WebSocketFrame? { + static func decode(_ buffer: inout ByteQueue) throws -> WebSocketFrame? { guard buffer.count >= 2 else { return nil } let first = buffer[0] let second = buffer[1] let masked = second & 0x80 != 0 - var length = Int(second & 0x7F) + var declaredLength = UInt64(second & 0x7F) var offset = 2 - if length == 126 { + if declaredLength == 126 { guard buffer.count >= offset + 2 else { return nil } - length = Int(buffer[offset]) << 8 | Int(buffer[offset + 1]) + declaredLength = UInt64(buffer[offset]) << 8 | UInt64(buffer[offset + 1]) offset += 2 - } else if length == 127 { + } else if declaredLength == 127 { guard buffer.count >= offset + 8 else { return nil } - var value: UInt64 = 0 - for index in 0..<8 { value = value << 8 | UInt64(buffer[offset + index]) } - length = Int(value) + declaredLength = 0 + for index in offset..<(offset + 8) { declaredLength = declaredLength << 8 | UInt64(buffer[index]) } offset += 8 } - guard length <= maxPayload else { throw TeradataWireError.malformed("WebSocket frame length \(length)") } + guard let length = Int(exactly: declaredLength), length <= maxPayload else { + throw TeradataWireError.malformed("WebSocket frame length \(declaredLength)") + } var maskKey: [UInt8] = [] if masked { guard buffer.count >= offset + 4 else { return nil } - maskKey = Array(buffer[offset..= offset + length else { return nil } - var payload = Array(buffer[offset.. Int? { + guard let value = Double(text) else { return nil } + return Int(exactly: value.rounded(.towardZero)) + } + public static func showCreateTable(catalog: String?, schema: String?, table: String) -> String { "SHOW CREATE TABLE \(qualifiedName(catalog: catalog, schema: schema, table: table))" } diff --git a/apple/Packages/TableProCore/Sources/TableProWireIO/ByteQueue.swift b/apple/Packages/TableProCore/Sources/TableProWireIO/ByteQueue.swift new file mode 100644 index 0000000000..f45bd3b4fb --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProWireIO/ByteQueue.swift @@ -0,0 +1,60 @@ +public struct ByteQueue: Sendable { + private var storage: [UInt8] = [] + private var head = 0 + + private static let compactionThreshold = 64 * 1_024 + + public init() {} + + public var count: Int { storage.count - head } + public var isEmpty: Bool { head == storage.count } + + public subscript(position: Int) -> UInt8 { + storage[head + position] + } + + public mutating func append(contentsOf bytes: some Sequence) { + storage.append(contentsOf: bytes) + } + + public func prefix(_ length: Int) -> [UInt8] { + Array(storage[head..<(head + min(max(length, 0), count))]) + } + + @discardableResult + public mutating func removeFirst(_ length: Int) -> [UInt8] { + let taken = prefix(length) + head += taken.count + compactIfNeeded() + return taken + } + + public func firstIndex(of byte: UInt8, from start: Int = 0) -> Int? { + let lower = head + min(max(start, 0), count) + return storage[lower...].firstIndex(of: byte).map { $0 - head } + } + + public func firstRange(of pattern: [UInt8], from start: Int = 0) -> Range? { + guard let first = pattern.first, pattern.count <= count else { return nil } + var candidate = min(max(start, 0), count) + while let found = firstIndex(of: first, from: candidate), found + pattern.count <= count { + if storage[(head + found)..<(head + found + pattern.count)].elementsEqual(pattern) { + return found..<(found + pattern.count) + } + candidate = found + 1 + } + return nil + } + + // Moving the unread tail only once the consumed head is both large and at least half the + // storage keeps every byte's share of the copying constant. + private mutating func compactIfNeeded() { + if head == storage.count { + storage.removeAll(keepingCapacity: storage.capacity <= Self.compactionThreshold * 16) + head = 0 + } else if head >= Self.compactionThreshold, head * 2 >= storage.count { + storage.removeSubrange(0..= bytes.count } + + public mutating func u8() throws(WireIOError) -> UInt8 { + try require(1) + defer { offset += 1 } + return bytes[offset] + } + + public mutating func u16() throws(WireIOError) -> UInt16 { + try bigEndian(UInt16.self) + } + + public mutating func u32() throws(WireIOError) -> UInt32 { + try bigEndian(UInt32.self) + } + + public mutating func u64() throws(WireIOError) -> UInt64 { + try bigEndian(UInt64.self) + } + + public mutating func take(_ count: some BinaryInteger) throws(WireIOError) -> [UInt8] { + let length = try checkedCount(count) + defer { offset += length } + return Array(bytes[offset..<(offset + length)]) + } + + public mutating func skip(_ count: some BinaryInteger) throws(WireIOError) { + offset += try checkedCount(count) + } + + // A declared element count can only be honest if the bytes left could hold that many elements. + public func elementCount(_ declared: some BinaryInteger, minimumElementSize: Int) throws(WireIOError) -> Int { + let ceiling = remaining / max(minimumElementSize, 1) + return try WireLength.checked(declared, limit: ceiling) + } + + private func require(_ count: Int) throws(WireIOError) { + guard remaining >= count else { throw .truncated(needed: count, available: remaining) } + } + + private func checkedCount(_ count: some BinaryInteger) throws(WireIOError) -> Int { + guard let length = Int(exactly: count), length >= 0 else { + throw .lengthOutOfRange(String(count), limit: remaining) + } + try require(length) + return length + } + + private mutating func bigEndian(_: Value.Type) throws(WireIOError) -> Value { + let width = MemoryLayout.size + try require(width) + var value: Value = 0 + for index in offset..<(offset + width) { + value = value << 8 | Value(bytes[index]) + } + offset += width + return value + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProWireIO/DecompressionBudget.swift b/apple/Packages/TableProCore/Sources/TableProWireIO/DecompressionBudget.swift new file mode 100644 index 0000000000..8724f69d4c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProWireIO/DecompressionBudget.swift @@ -0,0 +1,36 @@ +public struct DecompressionBudget: Sendable, Equatable { + public let limit: Int + public let ratioLimit: Int + public let grace: Int + public private(set) var produced = 0 + public private(set) var consumed = 0 + + public init(limit: Int = .max, ratioLimit: Int, grace: Int) { + self.limit = limit + self.ratioLimit = ratioLimit + self.grace = grace + } + + // Past 64 MiB of output, refuse more than 250 bytes out per byte in: a deflate stream of one + // repeated byte packs about 1,000 to 1, while real exports stay well under the limit. + public static let standard = DecompressionBudget(ratioLimit: 250, grace: 64 << 20) + + public var remaining: Int { max(limit - produced, 0) } + + public mutating func record(produced newlyProduced: Int, consumed newlyConsumed: Int) throws(WireIOError) { + produced = Self.saturatingSum(produced, newlyProduced) + consumed = Self.saturatingSum(consumed, newlyConsumed) + guard produced <= limit else { + throw .expansionExceeded(produced: produced, consumed: consumed) + } + guard produced > grace else { return } + guard produced / max(consumed, 1) <= ratioLimit else { + throw .expansionExceeded(produced: produced, consumed: consumed) + } + } + + private static func saturatingSum(_ total: Int, _ addition: Int) -> Int { + let (sum, overflow) = total.addingReportingOverflow(max(addition, 0)) + return overflow ? .max : sum + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProWireIO/WireIOError.swift b/apple/Packages/TableProCore/Sources/TableProWireIO/WireIOError.swift new file mode 100644 index 0000000000..20155c7d76 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProWireIO/WireIOError.swift @@ -0,0 +1,20 @@ +import Foundation + +public enum WireIOError: Error, Equatable, Sendable, CustomStringConvertible, LocalizedError { + case truncated(needed: Int, available: Int) + case lengthOutOfRange(String, limit: Int) + case expansionExceeded(produced: Int, consumed: Int) + + public var description: String { + switch self { + case .truncated(let needed, let available): + return "truncated: needed \(needed) bytes, \(available) left" + case .lengthOutOfRange(let length, let limit): + return "length \(length) is outside 0...\(limit)" + case .expansionExceeded(let produced, let consumed): + return "decompressed data passed its limit: \(produced) bytes from \(consumed)" + } + } + + public var errorDescription: String? { description } +} diff --git a/apple/Packages/TableProCore/Sources/TableProWireIO/WireLength.swift b/apple/Packages/TableProCore/Sources/TableProWireIO/WireLength.swift new file mode 100644 index 0000000000..f7a3d3832e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProWireIO/WireLength.swift @@ -0,0 +1,8 @@ +public enum WireLength { + public static func checked(_ value: some BinaryInteger, limit: Int) throws(WireIOError) -> Int { + guard let length = Int(exactly: value), length >= 0, length <= limit else { + throw .lengthOutOfRange(String(value), limit: limit) + } + return length + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraVarintTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraVarintTests.swift new file mode 100644 index 0000000000..59071ddf13 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraVarintTests.swift @@ -0,0 +1,109 @@ +// +// CassandraVarintTests.swift +// TableProCassandraCoreTests +// + +import Foundation +import TableProCassandraCore +import Testing + +struct CassandraVarintTests { + @Test("Known varints read as their decimal text", arguments: [ + ([UInt8](), "0"), + ([0x00], "0"), + ([0x7F], "127"), + ([0x80], "-128"), + ([0xFF], "-1"), + ([0x01, 0x00], "256"), + ([0x3B, 0x9A, 0xCA, 0x00], "1000000000"), + ([0x0D, 0xE0, 0xB6, 0xB3, 0xA7, 0x64, 0x00, 0x00], "1000000000000000000"), + ([0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], "18446744073709551616"), + ([0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], "-18446744073709551616"), + ]) + func knownValues(bytes: [UInt8], expected: String) { + #expect(CassandraVarint.decimalString(fromTwosComplement: Data(bytes)) == expected) + } + + @Test("Values up to 512 bytes read the same as long division by ten, both signs") + func matchesLongDivision() throws { + var generator = SplitMix64(seed: 0x5EED) + for length in [1, 2, 3, 4, 5, 7, 8, 9, 16, 31, 64, 100, 257, 512] { + var magnitude = (0.. 9_800) + } + + @Test("A varint past the decimal limit is shown as hex instead of a conversion that grows with its square") + func longVarintIsHex() { + let bytes = Data([0x01] + [UInt8](repeating: 0x5A, count: 16 * 1_024 - 1)) + #expect(CassandraVarint.decimalString(fromTwosComplement: bytes) == CassandraCellText.hex(bytes)) + } + + @Test("A decimal whose unscaled integer is past the limit is shown as hex and keeps its scale") + func longDecimalIsHexWithScale() { + let unscaled = Data([0x01] + [UInt8](repeating: 0x5A, count: 16 * 1_024 - 1)) + #expect(CassandraVarint.decimalString(unscaled: unscaled, scale: 2) == CassandraCellText.hex(unscaled) + "E-2") + #expect(CassandraVarint.decimalString(unscaled: unscaled, scale: -3) == CassandraCellText.hex(unscaled) + "E3") + } + + @Test("Short decimals keep their plain form") + func shortDecimals() { + #expect(CassandraVarint.decimalString(unscaled: Data([0x30, 0x39]), scale: 2) == "123.45") + #expect(CassandraVarint.decimalString(unscaled: Data([0xCF, 0xC7]), scale: 2) == "-123.45") + #expect(CassandraVarint.decimalString(unscaled: Data([0x05]), scale: -2) == "500") + } + + @Test("Hex spells every byte in lower case") + func hexText() { + #expect(CassandraCellText.hex(Data()) == "0x") + #expect(CassandraCellText.hex(Data([0x00, 0x0F, 0xA0, 0xFF])) == "0x000fa0ff") + } + + private static func longDivisionDigits(_ magnitude: [UInt8]) -> String { + var remaining = Array(magnitude.drop { $0 == 0 }) + guard !remaining.isEmpty else { return "0" } + var digits: [Character] = [] + while !remaining.isEmpty { + var remainder = 0 + var quotient: [UInt8] = [] + for byte in remaining { + let value = remainder << 8 | Int(byte) + if !(quotient.isEmpty && value / 10 == 0) { + quotient.append(UInt8(value / 10)) + } + remainder = value % 10 + } + digits.append(Character(String(remainder))) + remaining = quotient + } + return String(digits.reversed()) + } +} + +private struct SplitMix64 { + var state: UInt64 + + init(seed: UInt64) { + state = seed + } + + mutating func next() -> UInt64 { + state &+= 0x9E37_79B9_7F4A_7C15 + var value = state + value = (value ^ (value >> 30)) &* 0xBF58_476D_1CE4_E5B9 + value = (value ^ (value >> 27)) &* 0x94D0_49BB_1331_11EB + return value ^ (value >> 31) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCoreTypesTests/UniqueNameAllocatorTests.swift b/apple/Packages/TableProCore/Tests/TableProCoreTypesTests/UniqueNameAllocatorTests.swift new file mode 100644 index 0000000000..c96b301fbf --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCoreTypesTests/UniqueNameAllocatorTests.swift @@ -0,0 +1,57 @@ +import TableProCoreTypes +import Testing + +private func suffixed(_ base: String, _ ordinal: Int) -> String { + ordinal == 1 ? base : "\(base) (\(ordinal))" +} + +@Test func givesEachRepeatTheLowestFreeOrdinal() { + var names = UniqueNameAllocator(candidate: suffixed) + let allocated = ["id", "id", "name", "id"].map { names.allocate($0) } + #expect(allocated == ["id", "id (2)", "name", "id (3)"]) +} + +@Test func skipsNamesThatAreAlreadyTaken() { + var names = UniqueNameAllocator(taken: ["id", "id (2)", "id (4)"], candidate: suffixed) + let first = names.allocate("id") + let second = names.allocate("id") + #expect(first == "id (3)") + #expect(second == "id (5)") + #expect(names.contains("id (5)")) +} + +@Test func aNameReservedLaterIsNeverHandedOut() { + var names = UniqueNameAllocator(candidate: suffixed) + let first = names.allocate("a") + let reserved = names.reserve("a (2)") + let reservedAgain = names.reserve("a (2)") + let next = names.allocate("a") + #expect(first == "a") + #expect(reserved) + #expect(!reservedAgain) + #expect(next == "a (3)") +} + +@Test func comparesNamesThroughTheKey() { + var names = UniqueNameAllocator(taken: ["Orders"], key: { $0.lowercased() }, candidate: suffixed) + let allocated = names.allocate("orders") + #expect(allocated == "orders (2)") +} + +@Test func aHeaderOfOneRepeatedNameStaysLinear() { + var keyCalls = 0 + var names = UniqueNameAllocator( + key: { name in + keyCalls += 1 + return name + }, + candidate: suffixed + ) + let count = 40_000 + var last = "" + for _ in 0.. Data { + var stream = z_stream() + let status = deflateInit2_( + &stream, Z_BEST_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 9, Z_DEFAULT_STRATEGY, + ZLIB_VERSION, Int32(MemoryLayout.size) + ) + precondition(status == Z_OK, "deflateInit2_ failed") + defer { deflateEnd(&stream) } + var input = [UInt8](repeating: byte, count: min(count, 1 << 20)) + var output = [UInt8](repeating: 0, count: 1 << 20) + var result = Data() + var left = count + repeat { + let take = min(left, input.count) + left -= take + let flush = left == 0 ? Z_FINISH : Z_NO_FLUSH + input.withUnsafeMutableBufferPointer { source in + stream.next_in = source.baseAddress + stream.avail_in = uInt(take) + repeat { + output.withUnsafeMutableBufferPointer { target in + stream.next_out = target.baseAddress + stream.avail_out = uInt(target.count) + deflate(&stream, flush) + guard let base = target.baseAddress else { return } + result.append(base, count: target.count - Int(stream.avail_out)) + } + } while stream.avail_out == 0 + } + } while left > 0 + return result + } + + static func gzip(_ text: String) -> Data { + var stream = z_stream() + _ = deflateInit2_( + &stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY, + ZLIB_VERSION, Int32(MemoryLayout.size) + ) + defer { deflateEnd(&stream) } + var input = Array(text.utf8) + var output = [UInt8](repeating: 0, count: input.count + 1_024) + input.withUnsafeMutableBufferPointer { source in + output.withUnsafeMutableBufferPointer { target in + stream.next_in = source.baseAddress + stream.avail_in = uInt(source.count) + stream.next_out = target.baseAddress + stream.avail_out = uInt(target.count) + deflate(&stream, Z_FINISH) + } + } + return Data(output.prefix(Int(stream.total_out))) + } +} + +@Test func inflatesOneMember() throws { + let text = String(repeating: "id,name\n1,alice\n", count: 500) + let output = try GzipInflater.decompress(GzipFixture.gzip(text)) + #expect(String(bytes: output, encoding: .utf8) == text) +} + +@Test func inflatesEveryConcatenatedMember() throws { + let output = try GzipInflater.decompress(GzipFixture.gzip("first\n") + GzipFixture.gzip("second\n")) + #expect(String(bytes: output, encoding: .utf8) == "first\nsecond\n") +} + +@Test func ignoresPaddingAfterTheLastMember() throws { + let output = try GzipInflater.decompress(GzipFixture.gzip("rows\n") + Data(repeating: 0, count: 512)) + #expect(String(bytes: output, encoding: .utf8) == "rows\n") +} + +@Test func refusesTruncatedInput() throws { + let whole = GzipFixture.gzip(String(repeating: "abcdefgh", count: 4_000)) + #expect(throws: GzipError.truncated) { + try GzipInflater.decompress(whole.prefix(whole.count / 2)) + } +} + +@Test func refusesCorruptInput() throws { + var bytes = [UInt8](GzipFixture.gzip(String(repeating: "abcdefgh", count: 4_000))) + for index in 20..<40 { bytes[index] ^= 0xFF } + #expect(throws: GzipError.corrupt) { + try GzipInflater.decompress(Data(bytes)) + } +} + +@Test func stopsAtTheBudgetLimit() throws { + let bomb = GzipFixture.gzip(repeating: 0, count: 8 << 20) + let budget = DecompressionBudget(limit: 1 << 20, ratioLimit: .max, grace: 0) + var produced = 0 + #expect(throws: GzipError.expandsTooFar) { + try GzipInflater.decompress(bomb, budget: budget) { produced += $0.count } + } + #expect(produced <= 1 << 20) +} + +// 80 MiB of empty lines packs into about 80 KB, past the ratio a reply decoded in memory may reach. +@Test func inMemoryDecodingRefusesPastTheExpansionRatio() { + let packed = GzipFixture.gzip(repeating: 0x0A, count: 80 << 20) + var produced = 0 + #expect(throws: GzipError.expandsTooFar) { + try GzipInflater.decompress(packed) { produced += $0.count } + } + #expect(produced < 80 << 20) +} + +// A dump of zero-filled BLOBs or a one-hot CSV packs as far, and a file on disk keeps to no ratio. +@Test func decompressesAFilePastTheExpansionRatio() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let source = directory.appendingPathComponent("export.csv.gz") + try GzipFixture.gzip(repeating: 0x0A, count: 80 << 20).write(to: source) + let destination = directory.appendingPathComponent("export.csv") + + try await GzipFile.decompress(source: source, destination: destination, outputLimit: 1 << 30) + let attributes = try FileManager.default.attributesOfItem(atPath: destination.path(percentEncoded: false)) + #expect(attributes[.size] as? Int == 80 << 20) +} + +@Test func refusesAFilePastItsOutputLimitAndLeavesNothing() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let source = directory.appendingPathComponent("dump.sql.gz") + try GzipFixture.gzip(repeating: 0, count: 8 << 20).write(to: source) + let destination = directory.appendingPathComponent("dump.sql") + + await #expect(throws: GzipError.needsMoreDiskSpace) { + try await GzipFile.decompress(source: source, destination: destination, outputLimit: 1 << 20) + } + #expect(!FileManager.default.fileExists(atPath: destination.path(percentEncoded: false))) +} + +@Test func fileBudgetLeavesTheReserveFreeWithNoRatio() throws { + let twentyGigabytes = 20 << 30 + var budget = GzipFile.budget(availableSpace: twentyGigabytes) + #expect(budget.limit == twentyGigabytes - GzipFile.reservedSpace) + #expect(GzipFile.budget(availableSpace: GzipFile.reservedSpace / 2).limit == GzipFile.reservedSpace / 4) + #expect(GzipFile.budget(availableSpace: 0).limit == 0) + #expect(GzipFile.budget(availableSpace: nil).limit == .max) + // A dump that expands to 12 GB on a volume with 20 GB free is read whole, at any ratio. + try budget.record(produced: 12 << 30, consumed: 1) +} + +@Test func decompressesAFileUnderTheRatio() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let text = (0..<20_000).map { "\($0),row \($0)\n" }.joined() + let source = directory.appendingPathComponent("dump.sql.gz") + try GzipFixture.gzip(text).write(to: source) + let destination = directory.appendingPathComponent("dump.sql") + + try await GzipFile.decompress(source: source, destination: destination, outputLimit: 1 << 30) + #expect(try String(contentsOf: destination, encoding: .utf8) == text) +} + +@Test func removesThePartialFileWhenTheSourceIsCut() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let whole = GzipFixture.gzip((0..<20_000).map { "\($0)\n" }.joined()) + let source = directory.appendingPathComponent("cut.sql.gz") + try whole.prefix(whole.count - 100).write(to: source) + let destination = directory.appendingPathComponent("cut.sql") + + await #expect(throws: GzipError.truncated) { + try await GzipFile.decompress(source: source, destination: destination, outputLimit: 1 << 30) + } + #expect(!FileManager.default.fileExists(atPath: destination.path(percentEncoded: false))) +} diff --git a/apple/Packages/TableProCore/Tests/TableProImportTests/ImportPlannerConnectionTests.swift b/apple/Packages/TableProCore/Tests/TableProImportTests/ImportPlannerConnectionTests.swift index ec321769e1..1f791729dc 100644 --- a/apple/Packages/TableProCore/Tests/TableProImportTests/ImportPlannerConnectionTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProImportTests/ImportPlannerConnectionTests.swift @@ -71,6 +71,21 @@ struct ImportPlannerConnectionTests { #expect(result.connections.map(\.settings.name) == ["Orders (Imported)", "Orders (Imported 2)"]) } + @Test("Two thousand copies of one name are numbered Imported, Imported 2, up to Imported 2000") + func copiesOfOneNameAreNumberedInOrder() throws { + let count = 2_000 + let library = ImportLibrarySnapshot(connections: [Fixtures.existing(Fixtures.settings(), name: "Orders")]) + let connections = (0.. Data { } struct InfluxResponseDecoderTests { - @Test func aNullInAOneColumnV3ResultIsAQuotedEmptyCell() { - let table = InfluxResponseDecoder.decodeV3CSV(bytes("note\n\"\"\n\"hot, \"\"quoted\"\"\"\na;b\n")) + @Test func aNullInAOneColumnV3ResultIsAQuotedEmptyCell() throws { + let table = try InfluxResponseDecoder.decodeV3CSV(bytes("note\n\"\"\n\"hot, \"\"quoted\"\"\"\na;b\n")) #expect(table == InfluxResultTable( columns: ["note"], typeNames: [InfluxTypeName.string], @@ -16,32 +16,32 @@ struct InfluxResponseDecoderTests { )) } - @Test func aNullV3AggregateIsOneRowOfNil() { - let table = InfluxResponseDecoder.decodeV3CSV(bytes("max(cpu.usage)\n\"\"\n")) + @Test func aNullV3AggregateIsOneRowOfNil() throws { + let table = try InfluxResponseDecoder.decodeV3CSV(bytes("max(cpu.usage)\n\"\"\n")) #expect(table.columns == ["max(cpu.usage)"]) #expect(table.rows == [[nil]]) } - @Test func anEmptyV3CellBesideOthersIsNil() { - let table = InfluxResponseDecoder.decodeV3CSV(bytes("host,note\na,\nb,\"hot, \"\"quoted\"\"\"\nc,a;b\n")) + @Test func anEmptyV3CellBesideOthersIsNil() throws { + let table = try InfluxResponseDecoder.decodeV3CSV(bytes("host,note\na,\nb,\"hot, \"\"quoted\"\"\"\nc,a;b\n")) #expect(table.columns == ["host", "note"]) #expect(table.rows == [["a", nil], ["b", "hot, \"quoted\""], ["c", "a;b"]]) } - @Test func aZeroByteV3BodyIsEmpty() { - #expect(InfluxResponseDecoder.decodeV3CSV(Data()) == .empty) + @Test func aZeroByteV3BodyIsEmpty() throws { + #expect(try InfluxResponseDecoder.decodeV3CSV(Data()) == .empty) } - @Test func aV3TimestampWithoutAZoneGetsZ() { + @Test func aV3TimestampWithoutAZoneGetsZ() throws { let csv = "time,usage\n2024-10-27T03:33:20,12.5\n2024-10-27T03:34:20.500,50.1\n2024-10-27T03:35:20Z,7.0\n" - let table = InfluxResponseDecoder.decodeV3CSV(bytes(csv)) + let table = try InfluxResponseDecoder.decodeV3CSV(bytes(csv)) #expect(table.typeNames == [InfluxTypeName.timestamp, InfluxTypeName.float]) #expect(table.column(named: "time") == ["2024-10-27T03:33:20Z", "2024-10-27T03:34:20.500Z", "2024-10-27T03:35:20Z"]) } - @Test func v3ColumnTypesAreInferredFromEveryValue() { + @Test func v3ColumnTypesAreInferredFromEveryValue() throws { let csv = "i,f,b,s,mixed,u\n-3,1.5,true,s,1,18446744073709551615\n4,2,false,t,2.5,\n" - let table = InfluxResponseDecoder.decodeV3CSV(bytes(csv)) + let table = try InfluxResponseDecoder.decodeV3CSV(bytes(csv)) #expect(table.typeNames == [ InfluxTypeName.integer, InfluxTypeName.float, @@ -53,16 +53,16 @@ struct InfluxResponseDecoderTests { #expect(table.rows == [["-3", "1.5", "true", "s", "1", "18446744073709551615"], ["4", "2", "false", "t", "2.5", nil]]) } - @Test func invalidUTF8InOneV3CellKeepsTheOtherRows() { + @Test func invalidUTF8InOneV3CellKeepsTheOtherRows() throws { var body = bytes("note\nok\n") body.append(contentsOf: [0x61, 0xFF, 0x62, 0x0A]) body.append(bytes("fine\n")) - let table = InfluxResponseDecoder.decodeV3CSV(body) + let table = try InfluxResponseDecoder.decodeV3CSV(body) #expect(table.rows == [["ok"], ["a\u{FFFD}b"], ["fine"]]) } - @Test func v3CSVReadsCRLFLineEnds() { - let table = InfluxResponseDecoder.decodeV3CSV(bytes("host,usage\r\na,1\r\nb,2\r\n")) + @Test func v3CSVReadsCRLFLineEnds() throws { + let table = try InfluxResponseDecoder.decodeV3CSV(bytes("host,usage\r\na,1\r\nb,2\r\n")) #expect(table.columns == ["host", "usage"]) #expect(table.rows == [["a", "1"], ["b", "2"]]) } @@ -208,6 +208,83 @@ struct InfluxResponseDecoderTests { #expect(try InfluxResponseDecoder.decodeFluxCSV(bytes("\r\n")) == .empty) } + private static let tooWide = InfluxError.malformedResponse( + String(localized: "the result has far more columns than its rows fill") + ) + + private static let wideHeader = (0 ..< 40_000).map { "c\($0)" }.joined(separator: ",") + + @Test("A v3 header of 40,000 columns over 1,000 empty records is refused, not padded to 40 million cells") + func shortV3RecordsUnderAWideHeaderAreRefused() throws { + let body = bytes(Self.wideHeader + "\n" + String(repeating: "\n", count: 1_000)) + #expect(throws: Self.tooWide) { try InfluxResponseDecoder.decodeV3CSV(body) } + #expect(try InfluxResponseDecoder.decodeV3CSV(bytes("host,note\na\n")).rows == [["a", nil]]) + } + + @Test("A dense v3 result past a million cells is read whole") + func denseV3ResultPastTheFloor() throws { + let header = (0 ..< 1_000).map { "c\($0)" }.joined(separator: ",") + let record = Array(repeating: "1", count: 1_000).joined(separator: ",") + let body = bytes(header + "\n" + String(repeating: record + "\n", count: 1_100)) + let table = try InfluxResponseDecoder.decodeV3CSV(body) + #expect(table.columns.count == 1_000) + #expect(table.rows.count == 1_100) + #expect(table.rows.allSatisfy { $0.count == 1_000 }) + #expect(table.typeNames.allSatisfy { $0 == InfluxTypeName.integer }) + } + + @Test("A v1 series of 40,000 columns over 1,000 empty value rows is refused") + func shortV1RowsUnderWideColumnsAreRefused() { + let columns = (0 ..< 40_000).map { "\"c\($0)\"" }.joined(separator: ",") + let values = Array(repeating: "[]", count: 1_000).joined(separator: ",") + let body = bytes(#"{"results":[{"statement_id":0,"series":[{"name":"m","columns":["# + columns + #"],"values":["# + values + "]}]}]}") + #expect(throws: Self.tooWide) { try InfluxResponseDecoder.decodeV1JSON(body) } + } + + @Test("Flux rows under a wider earlier or later table are refused, not padded to 40 million cells", arguments: [true, false]) + func shortFluxRowsUnderAWideTableAreRefused(wideFirst: Bool) { + let wide = ",result,table," + Self.wideHeader + "\r\n\r\n" + let narrow = ",result,table\r\n" + String(repeating: ",_result,0\r\n", count: 1_000) + "\r\n" + let body = bytes(wideFirst ? wide + narrow : narrow + wide) + #expect(throws: Self.tooWide) { try InfluxResponseDecoder.decodeFluxCSV(body) } + } + + // `SELECT * FROM /.*/ LIMIT 10` over 300 measurements, each with its own 10 fields. + @Test("A sparse result of 3,000 rows by 3,002 columns from a few hundred KB is read in v1 JSON and Flux") + func sparseResultAcrossManyMeasurements() throws { + let fields = (0 ..< 300).map { measurement in (0 ..< 10).map { "m\(measurement)_f\($0)" } } + let values = Array(repeating: "12.5", count: 10).joined(separator: ",") + + do { + let series = fields.enumerated().map { measurement, names in + let columns = (["time"] + names).map { "\"\($0)\"" }.joined(separator: ",") + let rows = Array(repeating: "[\"2024-10-27T03:33:20Z\"," + values + "]", count: 10).joined(separator: ",") + return #"{"name":"sensor\#(measurement)","columns":[\#(columns)],"values":[\#(rows)]}"# + } + let table = try InfluxResponseDecoder.decodeV1JSON( + bytes(#"{"results":[{"statement_id":0,"series":["# + series.joined(separator: ",") + "]}]}") + ) + #expect(table.columns.count == 3_002) + #expect(table.rows.count == 3_000) + #expect(table.rows[2_999][0] == "sensor299") + #expect(table.rows[2_999][3_001] == "12.5") + #expect(table.rows[0][3_001] == nil) + } + do { + let tables = fields.enumerated().map { measurement, names in + ",result,table,_time," + names.joined(separator: ",") + "\r\n" + + String(repeating: ",_result,\(measurement),2024-10-27T03:33:20Z," + values + "\r\n", count: 10) + } + let table = try InfluxResponseDecoder.decodeFluxCSV(bytes(tables.joined(separator: "\r\n"))) + #expect(table.columns.count == 3_002) + #expect(table.rows.count == 3_000) + #expect(table.rows.allSatisfy { $0.count == 3_002 }) + #expect(table.rows[2_999][0] == "299") + #expect(table.rows[2_999][3_001] == "12.5") + #expect(table.rows[0][3_001] == nil) + } + } + @Test func schemaTypesReplaceInferredTypesByColumnName() { let table = InfluxResultTable( columns: ["time", "host", "usage", "extra"], @@ -256,7 +333,7 @@ struct InfluxResponseDecoderTests { #expect(InfluxSchema.v3Columns(from: table) == [InfluxColumn(name: "c", typeName: typeName, role: role)]) } - @Test func v3ColumnsPutTimeFirstAndTheRestByName() { + @Test func v3ColumnsPutTimeFirstAndTheRestByName() throws { let csv = """ column_name,data_type b,Boolean @@ -270,7 +347,7 @@ struct InfluxResponseDecoderTests { zone,"Dictionary(Int32, Utf8)" """ - let columns = InfluxSchema.v3Columns(from: InfluxResponseDecoder.decodeV3CSV(bytes(csv))) + let columns = try InfluxSchema.v3Columns(from: InfluxResponseDecoder.decodeV3CSV(bytes(csv))) #expect(columns == [ InfluxColumn(name: "time", typeName: InfluxTypeName.timestamp, role: .time), InfluxColumn(name: "b", typeName: InfluxTypeName.boolean, role: .field), diff --git a/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaBatchFixture.swift b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaBatchFixture.swift new file mode 100644 index 0000000000..6f31218495 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaBatchFixture.swift @@ -0,0 +1,209 @@ +import Foundation +import TableProKafkaCore +import zlib + +enum KafkaBatchFixture { + static let transactional: Int16 = 0x10 + static let control: Int16 = 0x20 + + // The decoder never reads the CRC, so it stays zero. + static func batch( + baseOffset: Int64 = 0, + baseTimestamp: Int64 = 1_700_000_000_000, + codec: KafkaCompressionCodec = .none, + flags: Int16 = 0, + producerId: Int64 = -1, + recordCount: Int32 = 1, + payload: Data + ) -> Data { + var body = KafkaProtocolWriter() + body.int32(-1) // partitionLeaderEpoch + body.int8(2) // magic + body.uint32(0) // crc + body.int16(codec.rawValue | flags) // attributes + body.int32(recordCount - 1) // lastOffsetDelta + body.int64(baseTimestamp) + body.int64(baseTimestamp) // maxTimestamp + body.int64(producerId) + body.int16(-1) // producerEpoch + body.int32(-1) // baseSequence + body.int32(recordCount) + body.raw(payload) + + var batch = KafkaProtocolWriter() + batch.int64(baseOffset) + batch.int32(Int32(body.count)) + batch.raw(body.bytes) + return batch.data + } + + static func record(offsetDelta: Int32 = 0, timestampDelta: Int64 = 0, value: Data?) -> Data { + var fields = KafkaProtocolWriter() + fields.int8(0) + fields.varlong(timestampDelta) + fields.varint(offsetDelta) + fields.varint(-1) // null key + if let value { + fields.varint(Int32(value.count)) + fields.raw(value) + } else { + fields.varint(-1) + } + fields.varint(0) // header count + var framed = KafkaProtocolWriter() + framed.varint(Int32(fields.count)) + framed.raw(fields.bytes) + return framed.data + } + + // The bytes around a value of `valueLength` zeros, so a large value never has to exist uncompressed. + static func zeroValueRecord(offsetDelta: Int32 = 0, valueLength: Int) -> (head: Data, tail: Data) { + var fields = KafkaProtocolWriter() + fields.int8(0) + fields.varlong(0) + fields.varint(offsetDelta) + fields.varint(-1) + fields.varint(Int32(valueLength)) + var tail = KafkaProtocolWriter() + tail.varint(0) + var head = KafkaProtocolWriter() + head.varint(Int32(fields.count + valueLength + tail.count)) + head.raw(fields.bytes) + return (head.data, tail.data) + } + + static func gzip(_ head: Data, zeros: Int = 0, _ tail: Data = Data()) -> Data { + var stream = z_stream() + let status = deflateInit2_( + &stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY, + ZLIB_VERSION, Int32(MemoryLayout.size) + ) + precondition(status == Z_OK, "deflateInit2_ failed") + defer { deflateEnd(&stream) } + var output = Data() + var chunk = [UInt8](repeating: 0, count: 1 << 16) + + func feed(_ bytes: [UInt8], _ flush: Int32) { + var input = bytes + input.withUnsafeMutableBufferPointer { source in + stream.next_in = source.baseAddress + stream.avail_in = uInt(source.count) + repeat { + chunk.withUnsafeMutableBufferPointer { target in + stream.next_out = target.baseAddress + stream.avail_out = uInt(target.count) + deflate(&stream, flush) + output.append(contentsOf: target.prefix(target.count - Int(stream.avail_out))) + } + } while stream.avail_out == 0 + } + } + + feed([UInt8](head), Z_NO_FLUSH) + let zeroBlock = [UInt8](repeating: 0, count: 1 << 16) + var left = zeros + while left > 0 { + let take = min(left, zeroBlock.count) + feed(Array(zeroBlock.prefix(take)), Z_NO_FLUSH) + left -= take + } + feed([UInt8](tail), Z_FINISH) + return output + } + + static let xerialHeader: [UInt8] = [0x82, 0x53, 0x4E, 0x41, 0x50, 0x50, 0x59, 0x00, 0, 0, 0, 1, 0, 0, 0, 1] + + static func xerial(_ blocks: [[UInt8]]) -> Data { + var framed = xerialHeader + for block in blocks { + let length = UInt32(block.count) + framed += [24, 16, 8, 0].map { UInt8(truncatingIfNeeded: length >> $0) } + framed += block + } + return Data(framed) + } + + static func varint(_ value: Int) -> [UInt8] { + var encoded: [UInt8] = [] + var remaining = value + while remaining >= 0x80 { + encoded.append(UInt8(remaining & 0x7F) | 0x80) + remaining >>= 7 + } + encoded.append(UInt8(remaining)) + return encoded + } + + // One snappy block of a single literal, up to 64 KiB. + static func snappyLiteral(_ bytes: [UInt8]) -> [UInt8] { + let length = bytes.count - 1 + let tag: [UInt8] + if length < 60 { + tag = [UInt8(length) << 2] + } else if length < 256 { + tag = [60 << 2, UInt8(length)] + } else { + tag = [61 << 2, UInt8(length & 0xFF), UInt8(length >> 8)] + } + return varint(bytes.count) + tag + bytes + } + + // Frame flags 0x60 and 64 KiB blocks, with the header checksum the reference lz4 tool expects. + static func lz4Frame(_ blocks: [(bytes: [UInt8], compressed: Bool)]) -> Data { + var frame: [UInt8] = [0x04, 0x22, 0x4D, 0x18, 0x60, 0x40, 0x82] + for block in blocks { + let size = UInt32(block.bytes.count) | (block.compressed ? 0 : 0x8000_0000) + frame += [0, 8, 16, 24].map { UInt8(truncatingIfNeeded: size >> $0) } + frame += block.bytes + } + frame += [0, 0, 0, 0] + return Data(frame) + } + + // Frame flags 0x60 and 4 MiB blocks, each block a single literal byte. + static func lz4FrameOfOneByteBlocks(_ byte: UInt8, count: Int) -> Data { + var blocks: [UInt8] = [0x02, 0x00, 0x00, 0x00, 0x10, byte] + while blocks.count < count * 6 { blocks += blocks } + return Data([0x04, 0x22, 0x4D, 0x18, 0x60, 0x70, 0x73] + blocks.prefix(count * 6) + [0, 0, 0, 0]) + } + + // A block holding one literal-only sequence, which is a complete LZ4 block. + static func lz4Literal(_ bytes: [UInt8]) -> [UInt8] { + guard bytes.count >= 15 else { return [UInt8(bytes.count) << 4] + bytes } + var block: [UInt8] = [0xF0] + var remaining = bytes.count - 15 + while remaining >= 255 { + block.append(255) + remaining -= 255 + } + block.append(UInt8(remaining)) + return block + bytes + } + + static let zstdMagic: [UInt8] = [0x28, 0xB5, 0x2F, 0xFD] + + static func zstdBlockHeader(size: Int, rle: Bool, last: Bool) -> [UInt8] { + let value = size << 3 | (rle ? 1 : 0) << 1 | (last ? 1 : 0) + return [UInt8(value & 0xFF), UInt8(value >> 8 & 0xFF), UInt8(value >> 16 & 0xFF)] + } + + // A frame that declares no content size, with a 1 MiB window, holding `bytes` as one raw block. + static func zstdRawFrame(_ bytes: [UInt8]) -> Data { + Data(zstdMagic + [0x00, 0x50] + zstdBlockHeader(size: bytes.count, rle: false, last: true) + bytes) + } + + // A frame that declares no content size and repeats one byte for `blocks` blocks of 128 KiB. + static func zstdRunFrame(of byte: UInt8, blocks: Int) -> Data { + var frame = zstdMagic + [0x00, 0x50] + for index in 0 ..< blocks { + frame += zstdBlockHeader(size: 128 * 1_024, rle: true, last: index == blocks - 1) + frame.append(byte) + } + return Data(frame) + } +} + +func refusedCodec(_ error: KafkaError?) -> String? { + guard case .decompressionExceeded(let codec)? = error else { return nil } + return codec +} diff --git a/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaDecompressionBudgetTests.swift b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaDecompressionBudgetTests.swift new file mode 100644 index 0000000000..75fb3b19c1 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaDecompressionBudgetTests.swift @@ -0,0 +1,207 @@ +import Foundation +import TableProKafkaCore +import TableProWireIO +import Testing + +private func compressed(_ records: Data, as codec: KafkaCompressionCodec) -> Data { + let bytes = [UInt8](records) + switch codec { + case .none: return records + case .gzip: return KafkaBatchFixture.gzip(records) + case .snappy: return KafkaBatchFixture.xerial([KafkaBatchFixture.snappyLiteral(bytes)]) + case .lz4: return KafkaBatchFixture.lz4Frame([(bytes: KafkaBatchFixture.lz4Literal(bytes), compressed: true)]) + case .zstd: return KafkaBatchFixture.zstdRawFrame(bytes) + } +} + +private func smallBudget(limit: Int) -> DecompressionBudget { + DecompressionBudget(limit: limit, ratioLimit: 250, grace: .max) +} + +@Test(arguments: [KafkaCompressionCodec.none, .gzip, .snappy, .lz4, .zstd]) +func everyCodecStillDecodesAValidBatch(codec: KafkaCompressionCodec) throws { + let records = KafkaBatchFixture.record(offsetDelta: 0, value: Data("first".utf8)) + + KafkaBatchFixture.record(offsetDelta: 1, value: Data(String(repeating: "second ", count: 40).utf8)) + let blob = KafkaBatchFixture.batch(baseOffset: 100, codec: codec, recordCount: 2, payload: compressed(records, as: codec)) + var budget = DecompressionBudget.standard + let decoded = try KafkaRecordBatchDecoder.decode(blob: blob, partition: 3, skipAborted: true, budget: &budget) + #expect(decoded.records.map { $0.offset } == [100, 101]) + #expect(decoded.records.map { $0.value } == [Data("first".utf8), Data(String(repeating: "second ", count: 40).utf8)]) + #expect(decoded.records.allSatisfy { $0.partition == 3 }) + #expect(!decoded.truncated) +} + +@Test func codecsDecodeReferenceFrames() throws { + var budget = DecompressionBudget.standard + let declaredZstd: [UInt8] = KafkaBatchFixture.zstdMagic + [0x20, 0x08, 0x28, 0x00, 0x00] + Array("hello".utf8) + [0x1B, 0x00, 0x00, 0x21] + let fromDeclaredZstd = try KafkaCompression.decompress(Data(declaredZstd), codec: .zstd, budget: &budget) + #expect(fromDeclaredZstd == Data("hello!!!".utf8)) + + let fromStreamedZstd = try KafkaCompression.decompress(KafkaBatchFixture.zstdRunFrame(of: 0x41, blocks: 3), codec: .zstd, budget: &budget) + #expect(fromStreamedZstd == Data(repeating: 0x41, count: 3 * 128 * 1_024)) + + let lz4Match: [UInt8] = [0x3E, 0x61, 0x62, 0x63, 0x03, 0x00, 0x50, 0x31, 0x32, 0x33, 0x34, 0x35] + let lz4 = KafkaBatchFixture.lz4Frame([(bytes: lz4Match, compressed: true), (bytes: Array("xyz".utf8), compressed: false)]) + let fromLz4 = try KafkaCompression.decompress(lz4, codec: .lz4, budget: &budget) + #expect(fromLz4 == Data("abcabcabcabcabcabcabc12345xyz".utf8)) + + let snappyLiteral: [UInt8] = [0x05, 0x10] + Array("hello".utf8) + let snappyCopy: [UInt8] = [0x08, 0x04, 0x61, 0x62, 0x09, 0x02] + let fromSnappy = try KafkaCompression.decompress(KafkaBatchFixture.xerial([snappyLiteral, snappyCopy]), codec: .snappy, budget: &budget) + #expect(fromSnappy == Data("helloabababab".utf8)) +} + +@Test func gzipBatchOfZerosIsRefusedPastTheLimit() { + let record = KafkaBatchFixture.zeroValueRecord(valueLength: 16 << 20) + let payload = KafkaBatchFixture.gzip(record.head, zeros: 16 << 20, record.tail) + let blob = KafkaBatchFixture.batch(codec: .gzip, payload: payload) + var budget = smallBudget(limit: 1 << 20) + let error = #expect(throws: KafkaError.self) { + try KafkaRecordBatchDecoder.decode(blob: blob, partition: 0, skipAborted: true, budget: &budget) + } + #expect(refusedCodec(error) == "gzip") +} + +// A reply holding one 1 GiB record in about 1 MiB of gzip, under the budget a real fetch asks for. +@Test func gzipBombIsRefusedOnceItOutrunsTheRatio() throws { + let member = KafkaBatchFixture.gzip(Data(), zeros: 1 << 20) + let payload = Data((0 ..< 1_024).flatMap { _ in member }) + let blob = KafkaBatchFixture.batch(codec: .gzip, payload: payload) + var budget = try KafkaCompression.fetchBudget(requesting: 1 << 20) + let error = #expect(throws: KafkaError.self) { + try KafkaRecordBatchDecoder.decode(blob: blob, partition: 0, skipAborted: true, budget: &budget) + } + #expect(refusedCodec(error) == "gzip") +} + +@Test func runningTotalRefusesBatchesThatFitOneAtATime() throws { + let record = KafkaBatchFixture.zeroValueRecord(valueLength: 1 << 20) + let payload = KafkaBatchFixture.gzip(record.head, zeros: 1 << 20, record.tail) + let one = KafkaBatchFixture.batch(codec: .gzip, payload: payload) + + var roomForTwo = smallBudget(limit: (5 << 20) / 2) + let two = try KafkaRecordBatchDecoder.decode(blob: one + one, partition: 0, skipAborted: true, budget: &roomForTwo) + #expect(two.records.map { $0.value?.count } == [1 << 20, 1 << 20]) + + var budget = smallBudget(limit: (5 << 20) / 2) + let error = #expect(throws: KafkaError.self) { + try KafkaRecordBatchDecoder.decode(blob: one + one + one, partition: 0, skipAborted: true, budget: &budget) + } + #expect(refusedCodec(error) == "gzip") +} + +@Test func zstdFrameWithoutDeclaredSizeIsBounded() { + var budget = smallBudget(limit: 1 << 20) + let error = #expect(throws: KafkaError.self) { + try KafkaCompression.decompress(KafkaBatchFixture.zstdRunFrame(of: 0, blocks: 64), codec: .zstd, budget: &budget) + } + #expect(refusedCodec(error) == "zstd") +} + +@Test func zstdFrameThatStopsEarlyIsAnError() { + var budget = DecompressionBudget.standard + let frame = KafkaBatchFixture.zstdRunFrame(of: 0x41, blocks: 3).dropLast() + let error = #expect(throws: KafkaError.self) { + try KafkaCompression.decompress(Data(frame), codec: .zstd, budget: &budget) + } + guard case .decompressionFailed(let codec, _)? = error else { + Issue.record("expected a decompression failure, got \(String(describing: error))") + return + } + #expect(codec == "zstd") +} + +// Each block repeats one byte to 64 KiB from a few hundred bytes; only the total across blocks is bounded. +@Test func snappyAndLz4BlocksAreBoundedAcrossTheFrame() { + let snappyBlock: [UInt8] = KafkaBatchFixture.varint(1 + 64 * 1_024) + [0x00, 0x61] + + Array([[UInt8]](repeating: [0xFE, 0x01, 0x00], count: 1_024).joined()) + var snappyBudget = smallBudget(limit: 200_000) + let snappyError = #expect(throws: KafkaError.self) { + try KafkaCompression.decompress( + KafkaBatchFixture.xerial(Array(repeating: snappyBlock, count: 8)), + codec: .snappy, + budget: &snappyBudget + ) + } + #expect(refusedCodec(snappyError) == "snappy") + + let lz4Block: [UInt8] = [0x1F, 0x61, 0x01, 0x00] + [UInt8](repeating: 0xFF, count: 256) + [231, 0x50] + + Array("aaaaa".utf8) + var lz4Budget = smallBudget(limit: 200_000) + let lz4Error = #expect(throws: KafkaError.self) { + try KafkaCompression.decompress( + KafkaBatchFixture.lz4Frame(Array(repeating: (bytes: lz4Block, compressed: true), count: 8)), + codec: .lz4, + budget: &lz4Budget + ) + } + #expect(refusedCodec(lz4Error) == "lz4") +} + +// Written by `lz4 -9 -BX --content-size`: a content size, block checksums and a content checksum. +@Test func lz4DecodesAFrameFromTheLz4Tool() throws { + let frame: [UInt8] = [ + 0x04, 0x22, 0x4D, 0x18, 0x7C, 0x40, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xCF, 0x26, 0x00, 0x00, 0x00, 0xF3, 0x02, 0x69, 0x64, 0x2C, + 0x6E, 0x61, 0x6D, 0x65, 0x0A, 0x31, 0x2C, 0x6B, 0x61, 0x66, 0x6B, 0x61, + 0x0A, 0x32, 0x08, 0x00, 0x13, 0x33, 0x08, 0x00, 0x13, 0x34, 0x08, 0x00, + 0x80, 0x35, 0x2C, 0x6B, 0x61, 0x66, 0x6B, 0x61, 0x0A, 0xB6, 0x36, 0x44, + 0xC3, 0x00, 0x00, 0x00, 0x00, 0x41, 0x82, 0x17, 0x17 + ] + var budget = DecompressionBudget.standard + let decoded = try KafkaCompression.decompress(Data(frame), codec: .lz4, budget: &budget) + #expect(decoded == Data("id,name\n1,kafka\n2,kafka\n3,kafka\n4,kafka\n5,kafka\n".utf8)) +} + +// The second block decodes into the buffer the first one filled, so only its own bytes may follow. +@Test func lz4BlocksOfOneFrameKeepToTheirOwnBytes() throws { + let fill: [UInt8] = [0x1F, 0x61, 0x01, 0x00] + [UInt8](repeating: 0xFF, count: 256) + [231, 0x50] + + Array("aaaaa".utf8) + let frame = KafkaBatchFixture.lz4Frame([ + (bytes: fill, compressed: true), + (bytes: KafkaBatchFixture.lz4Literal(Array("xyz".utf8)), compressed: true) + ]) + var budget = DecompressionBudget.standard + let decoded = try KafkaCompression.decompress(frame, codec: .lz4, budget: &budget) + #expect(decoded == Data(repeating: 0x61, count: 64 * 1_024) + Data("xyz".utf8)) +} + +// 4 Mi one-byte blocks in a frame that declares 4 MiB blocks: a 4 MiB zero fill per block runs +// far past the time limit. +@Test(.timeLimit(.minutes(1))) +func lz4FrameOfTinyBlocksCostsWhatItHolds() throws { + let count = 1 << 22 + var budget = DecompressionBudget.standard + let decoded = try KafkaCompression.decompress( + KafkaBatchFixture.lz4FrameOfOneByteBlocks(0x41, count: count), + codec: .lz4, + budget: &budget + ) + #expect(decoded == Data(repeating: 0x41, count: count)) +} + +@Test func fetchBudgetCarriesItsTotalsAcrossPages() throws { + let first = try KafkaCompression.fetchBudget(requesting: 1 << 20) + #expect(first.limit == 250 << 20) + #expect(first.ratioLimit == DecompressionBudget.standard.ratioLimit) + #expect(first.grace == DecompressionBudget.standard.grace) + + var spent = first + try spent.record(produced: 10 << 20, consumed: 1 << 20) + let next = try KafkaCompression.fetchBudget(requesting: 4 << 20, after: spent) + #expect(next.produced == 10 << 20) + #expect(next.consumed == 1 << 20) + #expect(next.limit == (10 << 20) + (1_000 << 20)) +} + +// Escalated to a 64 MiB request, the standard ratio alone would let one read hold 16 GiB. +@Test func fetchBudgetGivesEachPageAtMostOneGibibyte() throws { + #expect(try KafkaCompression.fetchBudget(requesting: 64 << 20).limit == 1 << 30) + #expect(try KafkaCompression.fetchBudget(requesting: .max).limit == 1 << 30) + + // A long read of large records keeps paging past the ceiling, one page's allowance at a time. + var spent = DecompressionBudget(ratioLimit: DecompressionBudget.standard.ratioLimit, grace: DecompressionBudget.standard.grace) + try spent.record(produced: 1_100 << 20, consumed: 40 << 20) + #expect(try KafkaCompression.fetchBudget(requesting: 1 << 20, after: spent).limit == (1_100 + 250) << 20) + #expect(try KafkaCompression.fetchBudget(requesting: 64 << 20, after: spent).limit == (1_100 << 20) + (1 << 30)) +} diff --git a/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift new file mode 100644 index 0000000000..bbfe76b995 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift @@ -0,0 +1,88 @@ +import Foundation +import TableProKafkaCore +import TableProWireIO +import Testing + +private func decodeOne(baseOffset: Int64, baseTimestamp: Int64, offsetDelta: Int32, timestampDelta: Int64 = 0) throws -> [KafkaRecord] { + let blob = KafkaBatchFixture.batch( + baseOffset: baseOffset, + baseTimestamp: baseTimestamp, + payload: KafkaBatchFixture.record(offsetDelta: offsetDelta, timestampDelta: timestampDelta, value: Data("hi".utf8)) + ) + var budget = DecompressionBudget.standard + return try KafkaRecordBatchDecoder.decode(blob: blob, partition: 0, skipAborted: true, budget: &budget).records +} + +@Test func decodesAnOrdinaryBatchAtItsOffset() throws { + let records = try decodeOne(baseOffset: 41, baseTimestamp: 1_700_000_000_000, offsetDelta: 1) + #expect(records.map { $0.offset } == [42]) + #expect(records.map { $0.timestamp } == [1_700_000_000_000]) + #expect(records.first?.value == Data("hi".utf8)) +} + +@Test func offsetPastInt64IsAMalformedReplyNotACrash() { + let error = #expect(throws: KafkaError.self) { + try decodeOne(baseOffset: .max, baseTimestamp: 1_700_000_000_000, offsetDelta: 1) + } + guard case .malformedResponse? = error else { + Issue.record("expected a malformed reply, got \(String(describing: error))") + return + } +} + +@Test func recordAtTheLastOffsetDecodesAndEndsPaging() throws { + let records = try decodeOne(baseOffset: .max, baseTimestamp: 1_700_000_000_000, offsetDelta: 0) + #expect(records.map { $0.offset } == [.max]) + #expect(KafkaOffsetMath.offset(after: .max) == nil) + #expect(KafkaOffsetMath.offset(after: 41) == 42) +} + +@Test func timestampPastInt64WrapsLikeKafkasClient() throws { + let records = try decodeOne(baseOffset: 0, baseTimestamp: .max, offsetDelta: 0, timestampDelta: 1) + #expect(records.map { $0.timestamp } == [.min]) +} + +@Test func messageCountSaturatesOnBrokerOffsets() { + #expect(KafkaOffsetMath.messageCount(earliest: 3, latest: 10) == 7) + #expect(KafkaOffsetMath.messageCount(earliest: 10, latest: 3) == 0) + #expect(KafkaOffsetMath.messageCount(earliest: .min, latest: .max) == .max) + #expect(KafkaOffsetMath.messageCount(earliest: .max, latest: .min) == 0) +} + +@Test func topicTotalSaturates() { + #expect(KafkaOffsetMath.total([4, 5]) == 9) + #expect(KafkaOffsetMath.total([.max, 1, 1]) == .max) +} + +@Test func tailWindowStepsBackWithoutLeavingInt64() { + #expect(KafkaOffsetMath.stepBack(from: 1_000, by: 100, floor: 0) == 900) + #expect(KafkaOffsetMath.stepBack(from: 50, by: 100, floor: 0) == 0) + #expect(KafkaOffsetMath.stepBack(from: .min + 1, by: 100, floor: -5) == -5) + #expect(KafkaOffsetMath.stepBack(from: .min + 1, by: 100, floor: .min) == .min) +} + +@Test func abortedRunSkipsOnlyItsOwnProducerUntilItsMarker() throws { + let aborted = KafkaAbortedTransaction(producerId: 7, firstOffset: 5) + func batch(_ producerId: Int64, at offset: Int64, flags: Int16 = KafkaBatchFixture.transactional) -> Data { + KafkaBatchFixture.batch( + baseOffset: offset, + flags: flags, + producerId: producerId, + payload: KafkaBatchFixture.record(value: Data("\(producerId)@\(offset)".utf8)) + ) + } + let blob = batch(7, at: 4) + + batch(7, at: 10) + + batch(8, at: 11) + + batch(7, at: 12, flags: KafkaBatchFixture.transactional | KafkaBatchFixture.control) + + batch(7, at: 13, flags: 0) + var budget = DecompressionBudget.standard + let decoded = try KafkaRecordBatchDecoder.decode( + blob: blob, + partition: 0, + abortedTransactions: [KafkaAbortedTransaction(producerId: 7, firstOffset: 40), aborted], + skipAborted: true, + budget: &budget + ) + #expect(decoded.records.map { $0.offset } == [4, 11, 13]) +} diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift index ff68599232..62c2c55cf8 100644 --- a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift @@ -36,6 +36,42 @@ struct MemcachedKeyListingTests { "STAT items:1:number 3", "STAT items:1:age 10", "STAT items:12:number 1", "STAT items:5:number 0", ] #expect(MemcachedKeyListing.slabClasses(fromItemStats: lines) == [1, 12]) + #expect(MemcachedKeyListing.slabClasses(fromItemStats: ["STAT items:3:number 1", "STAT items:3:number 2"]) == [3]) + } + + @Test("A stats items reply naming 300,000 slab classes is read in one pass", .timeLimit(.minutes(1))) + func manySlabClasses() { + let lines = (1 ... 300_000).reversed().map { "STAT items:\($0):number 1" } + let classes = MemcachedKeyListing.slabClasses(fromItemStats: lines) + #expect(classes.count == 300_000) + #expect(classes.first == 1) + #expect(classes.last == 300_000) + } + + @Test("An expiry too far from the server clock to subtract keeps the item listed, with no time left shown") + func expiryPastTheClockRange() throws { + let far = try #require(MemcachedKeyListing.metadumpEntry("key=a exp=9223372036854775807 la=1 cas=1 fetch=no cls=1 size=1")) + #expect(far.secondsLeft(at: -1) == nil) + #expect(!far.isExpired(at: -1)) + + let entry = MemcachedKeyEntry(key: "a", expiresAt: 1_793_500_167, flags: nil) + #expect(entry.secondsLeft(at: .min) == nil) + #expect(!entry.isExpired(at: .min)) + #expect(entry.secondsLeft(at: 1_793_500_067) == 100) + #expect(entry.isExpired(at: 1_793_500_167)) + #expect(entry.isExpired(at: .max)) + } + + @Test("A STAT time at either end of Int64 falls back to the local clock instead of crashing") + func serverClockAtTheEnds() { + #expect(MemcachedServerClock.offset(serverTime: 1_793_500_167, localTime: 1_793_499_867) == 300) + #expect(MemcachedServerClock.serverTime(localTime: 1_793_499_900, offset: 300) == 1_793_500_200) + #expect(MemcachedServerClock.serverTime(localTime: 1_793_499_900, offset: -300) == 1_793_499_600) + + #expect(MemcachedServerClock.offset(serverTime: .min, localTime: 1_793_499_867) == 0) + let ahead = MemcachedServerClock.offset(serverTime: .max, localTime: 1_793_499_867) + #expect(MemcachedServerClock.serverTime(localTime: 1_793_499_867, offset: ahead) == .max) + #expect(MemcachedServerClock.serverTime(localTime: 1_793_499_868, offset: ahead) == 1_793_499_868) } @Test("Stat pairs keep a value with spaces whole") diff --git a/apple/Packages/TableProCore/Tests/TableProNeo4jCoreTests/Neo4jValueTextTests.swift b/apple/Packages/TableProCore/Tests/TableProNeo4jCoreTests/Neo4jValueTextTests.swift index 1d23038ddc..e13a1da6a6 100644 --- a/apple/Packages/TableProCore/Tests/TableProNeo4jCoreTests/Neo4jValueTextTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProNeo4jCoreTests/Neo4jValueTextTests.swift @@ -42,6 +42,52 @@ struct Neo4jValueTextTests { == "PT-2562047788015215H-30M-8S") } + @Test("A temporal outside what Neo4j can store reads like any undecodable structure instead of crashing") + func temporalsOutOfRange() { + let hostile: [(UInt8, [PackStreamValue])] = [ + (0x44, [.int(.max)]), + (0x44, [.int(.min)]), + (0x74, [.int(.max)]), + (0x54, [.int(0), .int(.min)]), + (0x64, [.int(.min), .int(0)]), + (0x64, [.int(0), .int(.max)]), + (0x49, [.int(.max), .int(0), .int(1)]), + (0x46, [.int(0), .int(0), .int(.max)]), + (0x69, [.int(.max), .int(0), .string("Asia/Ho_Chi_Minh")]) + ] + for (tag, fields) in hostile { + #expect(Neo4jValueText.text(structure(tag, fields)) == "") + } + #expect(Neo4jValueText.text(structure(0x45, [.int(0), .int(0), .int(0), .int(.min)])) != nil) + #expect(Neo4jValueText.text(structure(0x45, [.int(0), .int(0), .int(0), .int(-500_000_000)])) == "PT-0.5S") + } + + @Test("A DateTime at either end of the range reads at any offset, since its seconds are UTC") + func dateTimesAtTheRangeEnds() { + let maxLocal: Int64 = 365_241_780_471 * 86_400 + 86_399 + let minLocal: Int64 = -365_243_219_162 * 86_400 + #expect(Neo4jValueText.text(structure(0x49, [.int(maxLocal + 64_800), .int(999_999_999), .int(-64_800)])) + == "+999999999-12-31T23:59:59.999999999-18:00") + #expect(Neo4jValueText.text(structure(0x49, [.int(minLocal - 64_800), .int(0), .int(64_800)])) + == "-999999999-01-01T00:00:00+18:00") + #expect(Neo4jValueText.text(structure(0x49, [.int(maxLocal + 64_801), .int(0), .int(-64_800)])) == "") + #expect(Neo4jValueText.text(structure(0x49, [.int(minLocal - 64_801), .int(0), .int(64_800)])) == "") + #expect(Neo4jValueText.text(structure(0x69, [.int(maxLocal + 43_200), .int(999_999_999), .string("Etc/GMT+12")])) + == "+999999999-12-31T23:59:59.999999999-12:00[Etc/GMT+12]") + #expect(Neo4jValueText.text(structure(0x69, [.int(minLocal - 50_400), .int(0), .string("Etc/GMT-14")])) + == "-999999999-01-01T00:00:00+14:00[Etc/GMT-14]") + #expect(Neo4jValueText.text(structure(0x46, [.int(maxLocal + 1), .int(0), .int(-64_800)])) == "") + #expect(Neo4jValueText.text(structure(0x66, [.int(minLocal - 1), .int(0), .string("Etc/GMT-14")])) == "") + } + + @Test("A path step of Int64.min is refused instead of crashing") + func pathStepAtInt64Min() { + let node = structure(0x4E, [.int(1), .list([.string("A")]), .map([:]), .string("n1")]) + let knows = structure(0x72, [.int(9), .string("KNOWS"), .map([:]), .string("r9")]) + let path = structure(0x50, [.list([node]), .list([knows]), .list([.int(.min), .int(0)])]) + #expect(Neo4jValueText.text(path) != nil) + } + @Test("A path lists its steps in order, with each relationship pointing the way it was walked") func pathSteps() throws { let first = structure(0x4E, [.int(1), .list([.string("A")]), .map([:]), .string("n1")]) @@ -86,6 +132,11 @@ struct Neo4jValueTextTests { #expect(created.counters.map(\.counter) == [.nodesCreated, .relationshipsCreated, .labelsAdded]) #expect(Neo4jUpdateCounts(summary: [:]).counters.isEmpty) + + let huge = Neo4jUpdateCounts(summary: ["stats": .map([ + "nodes-created": .int(.max), "relationships-created": .int(1) + ])]) + #expect(huge.entitiesChanged == .max) } @Test("Points name their SRID and coordinates") diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseTabSeparatedRowDecoderChunkingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseTabSeparatedRowDecoderChunkingTests.swift new file mode 100644 index 0000000000..bc2c64f260 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseTabSeparatedRowDecoderChunkingTests.swift @@ -0,0 +1,92 @@ +import Foundation +import TableProPluginKit +import Testing + +struct ClickHouseTabSeparatedRowDecoderChunkingTests { + private static func decode(_ chunks: [Data]) -> [[PluginCellValue]] { + var decoder = ClickHouseTabSeparatedRowDecoder() + var rows: [[PluginCellValue]] = [] + for chunk in chunks { + rows.append(contentsOf: decoder.consume(chunk)) + } + rows.append(contentsOf: decoder.finish()) + return rows + } + + private static func chunks(of body: Data, size: Int) -> [Data] { + stride(from: 0, to: body.count, by: size).map { body.subdata(in: $0..\(number("A\(row)", "\(row)"))\(number("B\(row)", "\(row * 10)"))\(number("C\(row)", "\(row * 100)"))" + }.joined() + // B4:C6 comes twice and A3:C5 covers its top-left cell; A2:A3 covers the top-left cell of A3:C5. + let merges = ["B4:C6", "A2:A3", "B4:C6", "A3:C5", "B1:C1"].map { "" }.joined() + let source = try source(rows: rows, trailer: "\(merges)") + #expect(grid(source) == [ + ["1", "10", ""], + ["2", "20", "200"], + ["", "", ""], + ["", "", ""], + ["", "", ""], + ["6", "", ""] + ]) + #expect(source.mergedRanges.map { "\($0.start.text):\($0.end.text)" } == ["B4:C6", "A2:A3", "A3:C5", "B1:C1"]) + } + + @Test(.timeLimit(.minutes(1))) + func testManyNestedFullWidthMergesReadInLinearTime() throws { + let count = 20_000 + let rows = (1...count).map { "\(number("A\($0)", "\($0)"))" }.joined() + let merges = (1...count).map { "" }.joined() + let read = try source(rows: rows, trailer: "\(merges)") + #expect(grid(read) == [["1"]]) + #expect(read.mergedRanges.count == count) + } + + @Test func testHidingMergedCellsStopsWhenCancelled() throws { + let body = """ + \(number("A1", "1"))\(number("B1", "2"))\ + + """ + func reader() -> XLSXSheetReader { + var reader = XLSXSheetReader(styles: .empty, sharedStringCount: 0, dateSystem: .base1900) + let bytes = Array(body.utf8) + _ = bytes.withUnsafeBufferPointer { reader.consume($0, isFinal: true) } + return reader + } + var cancelled = reader() + #expect(throws: TabularCancellation.self) { + try cancelled.finish(name: "Data", sharedStrings: .empty, isCancelled: { true }) + } + var running = reader() + let finished = try running.finish(name: "Data", sharedStrings: .empty, isCancelled: { false }) + #expect(grid(finished) == [["1"]]) + } + + @Test func testOccupiedRowsSkipRowsWithNoValue() throws { + let rows = """ + \(number("B2", "1"))\ + \(number("D5", "2"))\(number("B5", "3"))\(number("C9", "4")) + """ + let source = try source(rows: rows) + #expect(source.rowCount == 8) + #expect(source.occupiedRows() == [0, 3, 7]) + } + + @Test func testAScanCanTellAnAbsentCellFromAnEmptyOne() throws { + let rows = """ + \(number("A1", "1"))""\ + 0\ + \(number("E2", "2")) + """ + let source = try source(rows: rows, sharedStrings: [""]) + var kinds: [[TabularCellKind]] = [] + source.scan(columns: Array(0..110\ diff --git a/apple/Packages/TableProCore/Tests/TableProTabularIOTests/ZipArchiveTests.swift b/apple/Packages/TableProCore/Tests/TableProTabularIOTests/ZipArchiveTests.swift index 226d0b4bc2..57cef6fc85 100644 --- a/apple/Packages/TableProCore/Tests/TableProTabularIOTests/ZipArchiveTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTabularIOTests/ZipArchiveTests.swift @@ -15,11 +15,47 @@ struct ZipArchiveTests { @Test func testAnEntryLargerThanTheBufferLimitIsRefusedBeforeInflating() throws { let archive = try archive([("xl/styles.xml", String(repeating: "a", count: 4_096), .deflate)]) let entry = try #require(archive.entry(named: "xl/styles.xml")) - if let error = #expect(throws: (any Error).self, performing: { try archive.data(for: entry, limit: 1_024) }) { + if let error = #expect(throws: (any Error).self, performing: { + try archive.data(for: entry, budget: .init(limit: 1_024, ratioLimit: .max, grace: .max)) + }) { #expect(error as? ZipArchive.Failure == .entryTooLarge("xl/styles.xml")) } } + @Test func testAnEntryThatInflatesPastTheBudgetLimitIsRefused() throws { + var builder = TestZipBuilder() + builder.usesZip64Extra = true + builder.zip64UncompressedSizeOverride = 1 << 46 + builder.add("xl/styles.xml", String(repeating: "a", count: 4_096), method: .deflate) + let archive = try ZipArchive(bytes: builder.build()) + let entry = try #require(archive.entry(named: "xl/styles.xml")) + if let error = #expect(throws: (any Error).self, performing: { + try archive.withReader(for: entry, budget: .init(limit: 1_024, ratioLimit: .max, grace: .max)) { reader in + let chunk = UnsafeMutableBufferPointer.allocate(capacity: 512) + defer { chunk.deallocate() } + while try reader.read(into: chunk) > 0 {} + } + }) { + #expect(error as? ZipArchive.Failure == .entryTooLarge("xl/styles.xml")) + } + } + + @Test func testADeclaredSizeIsNotTrustedForTheBuffer() throws { + let body = String(repeating: "", count: 1_000) + var builder = TestZipBuilder() + builder.usesZip64Extra = true + builder.zip64UncompressedSizeOverride = 1 << 46 + builder.add("xl/worksheets/sheet1.xml", body, method: .deflate) + let archive = try ZipArchive(bytes: builder.build()) + let entry = try #require(archive.entry(named: "xl/worksheets/sheet1.xml")) + #expect(entry.uncompressedSize == 1 << 46) + if let error = #expect(throws: (any Error).self, performing: { try archive.data(for: entry) }) { + #expect(error as? ZipArchive.Failure == .entryTooLarge("xl/worksheets/sheet1.xml")) + } + let unlimited = try archive.data(for: entry, budget: .init(ratioLimit: ZipArchive.bufferedBudget.ratioLimit, grace: 1 << 20)) + #expect(unlimited == Data(body.utf8)) + } + @Test func testAnEntryThatInflatesPastItsDeclaredSizeIsRefused() throws { var builder = TestZipBuilder() builder.add("xl/workbook.xml", String(repeating: "", count: 10_000), method: .deflate) @@ -34,13 +70,15 @@ struct ZipArchiveTests { } @Test func testAnEntryThatExpandsFarBeyondItsCompressedSizeIsRefused() throws { - let policy = ZipExpansionPolicy(ratioLimit: ZipExpansionPolicy.standard.ratioLimit, grace: 1 << 20) + let ratioLimit = ZipArchive.bufferedBudget.ratioLimit let body = String(repeating: "A", count: 4 << 20) let archive = try archive([("xl/worksheets/sheet1.xml", body, .deflate)]) let entry = try #require(archive.entry(named: "xl/worksheets/sheet1.xml")) - #expect(entry.uncompressedSize / max(entry.compressedSize, 1) > policy.ratioLimit) + #expect(entry.uncompressedSize / max(entry.compressedSize, 1) > ratioLimit) #expect(try archive.data(for: entry).count == body.utf8.count) - if let error = #expect(throws: (any Error).self, performing: { try archive.data(for: entry, policy: policy) }) { + if let error = #expect(throws: (any Error).self, performing: { + try archive.data(for: entry, budget: .init(ratioLimit: ratioLimit, grace: 1 << 20)) + }) { #expect(error as? ZipArchive.Failure == .entryTooLarge("xl/worksheets/sheet1.xml")) } } diff --git a/apple/Packages/TableProCore/Tests/TableProTabularTests/TabularEditingTests.swift b/apple/Packages/TableProCore/Tests/TableProTabularTests/TabularEditingTests.swift index 4bc4bd1d8b..5b155e27c1 100644 --- a/apple/Packages/TableProCore/Tests/TableProTabularTests/TabularEditingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTabularTests/TabularEditingTests.swift @@ -176,6 +176,57 @@ struct TabularEditingTests { #expect(TabularTypeInference.infer([(.text, "05/01/2024")]) == .text) } + @Test func testInferenceOfAWideSparseSheetReadsABoundedNumberOfCells() throws { + let budget = 100_000 + let source = GeneratedSource(rowCount: 1_048_576, columnCount: 16_384) { row, column in + if row == 0, column == 0 { return "1" } + return row == 1_048_575 && column == 16_383 ? "2" : nil + } + let table = TabularTable(source: source, usesFirstRowAsHeader: false) + let kinds = try TabularTypeInference.inferKinds(of: table, cellBudget: budget) + #expect(source.scanned.cells <= budget) + #expect(kinds.count == 16_384) + #expect(kinds[table.columns[0].id] == .integer) + #expect(kinds[table.columns[16_383].id] == .text) + } + + @Test func testInferenceOfALongTableWithNoValuesStopsAtItsBudget() throws { + let budget = 8_000 + let source = GeneratedSource(rowCount: 250_000, columnCount: 8) { _, _ in nil } + let kinds = try TabularTypeInference.inferKinds( + of: TabularTable(source: source, usesFirstRowAsHeader: false), + cellBudget: budget + ) + #expect(source.scanned.cells <= budget) + #expect(Set(kinds.values) == [.text]) + } + + // Reading all eight columns, the budget runs out at row 12,500, before the sparse column's value. + @Test func testASparseColumnKeepsTheBudgetTheFilledColumnsNoLongerNeed() throws { + let source = GeneratedSource(rowCount: 20_000, columnCount: 8) { row, column in + if column < 7 { return "\(row)" } + return row == 15_000 ? "42" : nil + } + let table = TabularTable(source: source, usesFirstRowAsHeader: false) + let kinds = try TabularTypeInference.inferKinds(of: table, cellBudget: 100_000) + #expect(kinds[table.columns[0].id] == .integer) + #expect(kinds[table.columns[7].id] == .integer) + #expect(source.scanned.cells <= 8 * 200 + 19_800) + } + + @Test func testInferenceStopsWhenItsTaskIsCancelled() async { + let budget = 1_000_000 + let source = GeneratedSource(rowCount: 250_000, columnCount: 8) { _, _ in nil } + let table = TabularTable(source: source, usesFirstRowAsHeader: false) + let task = Task { + withUnsafeCurrentTask { $0?.cancel() } + return try TabularTypeInference.inferKinds(of: table, cellBudget: budget) + } + let result = await task.result + #expect(throws: CancellationError.self) { try result.get() } + #expect(source.scanned.cells < budget) + } + @Test func testARunawayRegularExpressionStopsWhenItsTaskIsCancelled() async throws { let expression = try NSRegularExpression(pattern: "(a+)+b") let text = String(repeating: "a", count: 64) @@ -208,3 +259,39 @@ struct TabularEditingTests { #expect(replaced.replacements == 3) } } + +private final class ScannedCells: @unchecked Sendable { + var cells = 0 +} + +private struct GeneratedSource: TabularSource { + let rowCount: Int + let columnCount: Int + let value: @Sendable (_ row: Int, _ column: Int) -> String? + let scanned = ScannedCells() + + var intrinsicColumnNames: [String]? { nil } + var absentCell: TabularCell { .text("") } + + func cell(row: Int, column: Int) -> TabularCell { + value(row, column).map { TabularCell(kind: .number, text: $0) } ?? absentCell + } + + func cells(row: Int) -> [TabularCell] { + (0..(columns: [Int], rows: Rows, _ body: (Int, TabularRowCells) -> Bool) where Rows.Element == Int { + var buffer = TabularCellBuffer() + for row in rows { + buffer.reset(slots: columns.count, fill: .text) + for (slot, column) in columns.enumerated() { + if let text = value(row, column) { + buffer.setUTF8(slot, text, kind: .number) + } + } + scanned.cells += columns.count + guard buffer.withResolved({ body(row, $0) }) else { return } + } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTabularTests/TabularTableTests.swift b/apple/Packages/TableProCore/Tests/TableProTabularTests/TabularTableTests.swift index a591506456..9fdb838f03 100644 --- a/apple/Packages/TableProCore/Tests/TableProTabularTests/TabularTableTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTabularTests/TabularTableTests.swift @@ -57,6 +57,29 @@ struct TabularTableTests { #expect(seen == [["1", "x"], ["new", ""], ["2", ""]]) } + @Test func testColumnLookupsFollowInsertsMovesAndDeletes() async throws { + let source = try await Self.delimitedSource("a,b,c\n1,2,3\n") + var table = TabularTable(source: source, usesFirstRowAsHeader: true) + let inserted = table.insertColumn(named: "d", at: 1, fill: .text("4")) + table.moveColumn(table.columns[3].id, to: 0) + let removed = table.columns[1].id + table.deleteColumns([removed]) + #expect(table.columns.map(\.name) == ["c", "d", "b"]) + for (index, column) in table.columns.enumerated() { + #expect(table.columnIndex(of: column.id) == index) + #expect(table.column(column.id)?.name == column.name) + } + #expect(table.columnIndex(of: removed) == nil) + #expect(table.column(removed) == nil) + #expect(table.columnIndex(of: inserted) == 1) + var seen: [[String]] = [] + table.scan(columns: table.columnIDs, rows: 0.. [UInt8] { + var token = [UInt8](repeating: 0, count: Td2Token.keyDataOffset) + token[0] = 1 + token[1] = 1 + token[16] = 6 + for (offset, length) in [(20, prime.count), (24, generator.count), (28, publicKey.count)] { + for index in 0..<4 { + token[offset + index] = UInt8(truncatingIfNeeded: length >> (8 * (3 - index))) + } + } + return token + prime + generator + publicKey + } + + @Test("An empty, zero or even prime is refused before it reaches the modular arithmetic") + func unusablePrimeIsRefused() { + #expect(Self.serverToken(prime: []).count == 82) + let primes: [[UInt8]] = [ + [], + [0x00, 0x00, 0x00, 0x00], + [UInt8](repeating: 0xFF, count: 255) + [0xFE], + ] + for prime in primes { + let token = Self.serverToken(prime: prime) + #expect(throws: TeradataWireError.self) { try Td2Token.parseServerParams(token) } + } + } + + @Test("A prime, generator or server key longer than 1,024 bytes is refused") + func oversizedKeyMaterialIsRefused() { + let tooLong = [UInt8](repeating: 0xFF, count: Td2Token.maxKeyByteCount + 1) + let prime = [UInt8](repeating: 0xFF, count: 256) + let tokens = [ + Self.serverToken(prime: tooLong), + Self.serverToken(prime: prime, generator: tooLong), + Self.serverToken(prime: prime, publicKey: tooLong), + ] + for token in tokens { + #expect(throws: TeradataWireError.self) { try Td2Token.parseServerParams(token) } + } + } + + @Test("A 2,048-bit odd prime with its generator and key parses as sent") + func usableParamsParse() throws { + let prime = [UInt8](repeating: 0xFF, count: 256) + let params = try Td2Token.parseServerParams(Self.serverToken(prime: prime, generator: [0x02], publicKey: [0x05])) + #expect(params.prime == prime) + #expect(params.generator == [0x02]) + #expect(params.serverPublicKey == [0x05]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift new file mode 100644 index 0000000000..76bf57dbd6 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift @@ -0,0 +1,51 @@ +@testable import TableProTeradataCore +import TableProWireIO +import Testing + +struct TeradataReplyDecodingTests { + private static let key = [UInt8](repeating: 0x42, count: 32) + + private static func reply(charLength: [UInt8], record: [UInt8]) -> [UInt8] { + var bytes: [UInt8] = [ + 0x00, 0x7D, 0x00, 0x23, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x01, 0xC4, + ] + bytes += charLength + bytes += [0x00, 0x00, 0x00, 0x01, 0x63, 0x00, 0x00, 0x00, 0x00] + bytes += [0x00, 0x0A, 0x00, UInt8(4 + record.count)] + bytes += record + bytes += [0x00, 0x0C, 0x00, 0x04] + return bytes + } + + @Test("A result description whose CHAR column has length -1 fails the reply instead of crashing") + func negativeColumnLengthFailsTheReply() throws { + let reply = Self.reply(charLength: [UInt8](repeating: 0xFF, count: 8), record: [0x00, 0x78]) + #expect(reply.count == 45) + let parcels = try Parcel.decodeAll(reply) + let flavors = parcels.map(\.flavor) + #expect(flavors == [ParcelFlavor.prepInfoX.rawValue, ParcelFlavor.record.rawValue, ParcelFlavor.endRequest.rawValue]) + #expect(throws: WireIOError.lengthOutOfRange("-1", limit: 1)) { try TeradataResultParser.parse(parcels) } + } + + @Test("A CHAR column of the declared length still decodes") + func charColumnDecodes() throws { + let reply = Self.reply(charLength: [0, 0, 0, 0, 0, 0, 0, 3], record: [0x00, 0x61, 0x62, 0x63]) + let result = try TeradataResultParser.parse(try Parcel.decodeAll(reply)) + #expect(result.rows == [[TeradataValue.text("abc")]]) + } + + @Test("A DER length of eight 0xFF bytes in an encrypted reply fails instead of crashing") + func oversizedDerLengthFailsTheReply() { + let der: [UInt8] = [0xE0, 0x10, 0xC0, 0x88] + [UInt8](repeating: 0xFF, count: 8) + [UInt8](repeating: 0x00, count: 6) + #expect(throws: WireIOError.self) { try Td2Wrap.unwrap(der: der, key: Self.key) } + } + + @Test("An encrypted message with long-form DER lengths unwraps to its plaintext") + func encryptedMessageRoundTrips() throws { + let plaintext = (0..<300).map { UInt8(truncatingIfNeeded: $0) } + let der = try Td2Wrap.wrap(plaintext: plaintext, key: Self.key, sequenceNumber: 7) + #expect(try Td2Wrap.unwrap(der: der, key: Self.key) == plaintext) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataSocketReceiveTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataSocketReceiveTests.swift new file mode 100644 index 0000000000..9b40acec11 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataSocketReceiveTests.swift @@ -0,0 +1,93 @@ +import Darwin +import Dispatch +@testable import TableProTeradataCore +import Testing + +private struct LoopbackFailure: Error { + let step: String + let code: Int32 +} + +private struct LoopbackListener { + let descriptor: Int32 + let port: UInt16 + + init() throws { + let handle = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP) + guard handle >= 0 else { throw LoopbackFailure(step: "socket", code: errno) } + var address = sockaddr_in() + address.sin_len = UInt8(MemoryLayout.size) + address.sin_family = sa_family_t(AF_INET) + address.sin_addr.s_addr = inet_addr("127.0.0.1") + var length = socklen_t(MemoryLayout.size) + let listening = withUnsafeMutablePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { generic in + bind(handle, generic, length) == 0 && listen(handle, 1) == 0 && getsockname(handle, generic, &length) == 0 + } + } + guard listening else { + let code = errno + Darwin.close(handle) + throw LoopbackFailure(step: "listen", code: code) + } + descriptor = handle + port = UInt16(bigEndian: address.sin_port) + } + + func acceptPeer() throws -> Int32 { + let peer = accept(descriptor, nil, nil) + guard peer >= 0 else { throw LoopbackFailure(step: "accept", code: errno) } + var one: Int32 = 1 + setsockopt(peer, SOL_SOCKET, SO_NOSIGPIPE, &one, socklen_t(MemoryLayout.size)) + return peer + } +} + +private func writeAll(_ bytes: [UInt8], to descriptor: Int32) { + bytes.withUnsafeBytes { raw in + guard let base = raw.baseAddress else { return } + var offset = 0 + while offset < bytes.count { + let written = Darwin.write(descriptor, base + offset, bytes.count - offset) + if written <= 0 { return } + offset += written + } + } +} + +struct TeradataSocketReceiveTests { + private static func connect(to listener: LoopbackListener) throws -> TeradataSocket { + try TeradataSocket(host: "127.0.0.1", port: listener.port, deadline: TeradataConnectDeadline(milliseconds: 5_000)) + } + + @Test("A reply longer than one read chunk arrives whole and in order") + func longReplyArrivesWhole() throws { + let listener = try LoopbackListener() + defer { Darwin.close(listener.descriptor) } + let socket = try Self.connect(to: listener) + defer { socket.close() } + let peer = try listener.acceptPeer() + let reply = (0..<200_000).map { UInt8(truncatingIfNeeded: $0 &* 31) } + DispatchQueue.global().async { + writeAll(reply, to: peer) + Darwin.close(peer) + } + + let header = try socket.receive(52) + let body = try socket.receive(reply.count - 52) + #expect(header + body == reply) + } + + @Test("A length far past what the peer sent fails when it closes, without allocating the claimed size") + func claimedLengthIsNotAllocatedUpFront() throws { + let listener = try LoopbackListener() + defer { Darwin.close(listener.descriptor) } + let socket = try Self.connect(to: listener) + defer { socket.close() } + let peer = try listener.acceptPeer() + writeAll([0x01, 0x02, 0x03], to: peer) + Darwin.close(peer) + + #expect(throws: TeradataWireError.self) { try socket.receive(.max) } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/WebSocketFrameTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/WebSocketFrameTests.swift new file mode 100644 index 0000000000..089ad23cd1 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/WebSocketFrameTests.swift @@ -0,0 +1,60 @@ +@testable import TableProTeradataCore +import TableProWireIO +import Testing + +struct WebSocketFrameTests { + private static func queue(_ bytes: [UInt8]) -> ByteQueue { + var queue = ByteQueue() + queue.append(contentsOf: bytes) + return queue + } + + @Test("A 64-bit frame length with the top bit set is refused instead of crashing") + func topBitLengthIsRefused() { + var topBit = Self.queue([0x82, 0x7F, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) + #expect(throws: TeradataWireError.self) { try WebSocketFrame.decode(&topBit) } + var intMax = Self.queue([0x82, 0x7F, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]) + #expect(throws: TeradataWireError.self) { try WebSocketFrame.decode(&intMax) } + } + + @Test("Frames of each length form and a masked frame decode to their payloads, and a partial frame waits") + func validFramesDecode() throws { + let payloads = [5, 300, 70_000].map { [UInt8](repeating: UInt8(truncatingIfNeeded: $0), count: $0) } + var buffer = ByteQueue() + for payload in payloads { + buffer.append(contentsOf: WebSocketFrame.encodeBinary(payload)) + } + // A ping masked with 11 22 33 44 whose payload is "abc". + buffer.append(contentsOf: [0x89, 0x83, 0x11, 0x22, 0x33, 0x44, 0x70, 0x40, 0x50]) + buffer.append(contentsOf: [0x82, 0x7E, 0x01]) + + for payload in payloads { + let frame = try WebSocketFrame.decode(&buffer) + #expect(frame?.opcode == WebSocketOpcode.binary) + #expect(frame?.payload == payload) + } + let ping = try WebSocketFrame.decode(&buffer) + #expect(ping?.opcode == WebSocketOpcode.ping) + #expect(ping?.payload == Array("abc".utf8)) + let partial = try WebSocketFrame.decode(&buffer) + #expect(partial == nil) + #expect(buffer.count == 3) + } + + // Removing each frame from the front of an array moves the rest of the buffer: minutes for this input. + @Test("Eight megabytes of two-byte frames drain in time linear in their size", .timeLimit(.minutes(1))) + func manySmallFramesDrainLinearly() throws { + let frameCount = 1 << 22 + var bytes = [UInt8](repeating: 0x00, count: frameCount * 2) + for index in stride(from: 0, to: bytes.count, by: 2) { + bytes[index] = 0x8A + } + var buffer = Self.queue(bytes) + var drained = 0 + while try WebSocketFrame.decode(&buffer) != nil { + drained += 1 + } + #expect(drained == frameCount) + #expect(buffer.isEmpty) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTrinoCoreTests/TrinoIntrospectionSQLTests.swift b/apple/Packages/TableProCore/Tests/TableProTrinoCoreTests/TrinoIntrospectionSQLTests.swift index 7eec294eba..49834a1f2d 100644 --- a/apple/Packages/TableProCore/Tests/TableProTrinoCoreTests/TrinoIntrospectionSQLTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTrinoCoreTests/TrinoIntrospectionSQLTests.swift @@ -34,4 +34,18 @@ struct TrinoIntrospectionSQLTests { #expect(TrinoIntrospectionSQL.quoteLiteral("x'\u{301} OR true--") == "'x''\u{301} OR true--'") #expect(TrinoIntrospectionSQL.quoteIdentifier("a\"\u{301}") == "\"a\"\"\u{301}\"") } + + @Test("A SHOW STATS row count reads as whole rows, a fraction cut off", arguments: [ + ("1234.0", 1_234), ("1234", 1_234), ("12.7", 12), ("1.5e+18", 1_500_000_000_000_000_000), ("0.0", 0), + ] as [(String, Int)]) + func statsRowCount(text: String, count: Int) { + #expect(TrinoIntrospectionSQL.rowCount(fromStatsValue: text) == count) + } + + @Test("A row count that is not a finite number inside Int is no count instead of a crash", arguments: [ + "NaN", "Infinity", "-Infinity", "1e+30", "-1e+30", "1e19", "9223372036854775807", "rows", "", + ]) + func unusableStatsRowCount(text: String) { + #expect(TrinoIntrospectionSQL.rowCount(fromStatsValue: text) == nil) + } } diff --git a/apple/Packages/TableProCore/Tests/TableProWireIOTests/WireIOTests.swift b/apple/Packages/TableProCore/Tests/TableProWireIOTests/WireIOTests.swift new file mode 100644 index 0000000000..ac973b1978 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProWireIOTests/WireIOTests.swift @@ -0,0 +1,91 @@ +import TableProWireIO +import Testing + +@Test func readsBigEndianIntegers() throws { + var reader = ByteReader([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F]) + #expect(try reader.u8() == 0x01) + #expect(try reader.u16() == 0x0203) + #expect(try reader.u32() == 0x0405_0607) + #expect(try reader.u64() == 0x0809_0A0B_0C0D_0E0F) + #expect(reader.isAtEnd) +} + +@Test func takeRejectsANegativeCount() { + var reader = ByteReader([0x00, 0x78]) + _ = try? reader.u8() + #expect(throws: WireIOError.lengthOutOfRange("-1", limit: 1)) { try reader.take(-1) } + #expect(throws: WireIOError.lengthOutOfRange("-1", limit: 1)) { try reader.skip(-1) } + #expect(reader.offset == 1) +} + +@Test func takeRejectsACountPastTheBytes() { + var reader = ByteReader([0x01, 0x02]) + #expect(throws: WireIOError.truncated(needed: 3, available: 2)) { try reader.take(3) } + #expect(throws: WireIOError.lengthOutOfRange("18446744073709551615", limit: 2)) { try reader.take(UInt64.max) } +} + +@Test func elementCountIsBoundedByTheBytesLeft() throws { + let reader = ByteReader([UInt8](repeating: 0, count: 16)) + #expect(try reader.elementCount(Int32(4), minimumElementSize: 4) == 4) + #expect(throws: WireIOError.self) { try reader.elementCount(Int32(5), minimumElementSize: 4) } + #expect(throws: WireIOError.self) { try reader.elementCount(Int32(-1), minimumElementSize: 1) } +} + +@Test func checkedLengthNarrowsWithoutTrapping() throws { + #expect(try WireLength.checked(UInt64(64), limit: 64) == 64) + #expect(throws: WireIOError.self) { try WireLength.checked(UInt64.max, limit: .max) } + #expect(throws: WireIOError.self) { try WireLength.checked(Int64(-1), limit: 10) } + #expect(throws: WireIOError.self) { try WireLength.checked(65, limit: 64) } +} + +@Test func queueHandsBackBytesInOrder() { + var queue = ByteQueue() + queue.append(contentsOf: [1, 2, 3, 4, 5]) + #expect(queue.removeFirst(2) == [1, 2]) + queue.append(contentsOf: [6]) + #expect(queue.count == 4) + #expect(queue[0] == 3) + #expect(queue.prefix(10) == [3, 4, 5, 6]) + #expect(queue.removeFirst(10) == [3, 4, 5, 6]) + #expect(queue.isEmpty) +} + +@Test func queueFindsAPatternFromAnOffset() { + var queue = ByteQueue() + queue.append(contentsOf: Array("xx\r\n\r\nyy\r\n\r\n".utf8)) + #expect(queue.firstRange(of: [0x0D, 0x0A, 0x0D, 0x0A]) == 2..<6) + #expect(queue.firstRange(of: [0x0D, 0x0A, 0x0D, 0x0A], from: 3) == 8..<12) + #expect(queue.firstIndex(of: 0x79) == 6) + queue.removeFirst(6) + #expect(queue.firstRange(of: [0x0D, 0x0A, 0x0D, 0x0A]) == 2..<6) +} + +@Test func drainingManySmallMessagesStaysLinear() { + var queue = ByteQueue() + let message: [UInt8] = [0x8A, 0x00] + for _ in 0..<(1 << 19) { queue.append(contentsOf: message) } + var drained = 0 + while !queue.isEmpty { + drained += queue.removeFirst(2).count + } + #expect(drained == 1 << 20) +} + +@Test func budgetAllowsOutputInsideTheGrace() throws { + var budget = DecompressionBudget(ratioLimit: 250, grace: 1_000) + try budget.record(produced: 1_000, consumed: 1) + #expect(budget.produced == 1_000) +} + +@Test func budgetRefusesARatioPastTheGrace() { + var budget = DecompressionBudget(ratioLimit: 250, grace: 1_000) + #expect(throws: WireIOError.expansionExceeded(produced: 1_001, consumed: 1)) { + try budget.record(produced: 1_001, consumed: 1) + } +} + +@Test func budgetRefusesOutputPastTheLimitAcrossCalls() throws { + var budget = DecompressionBudget(limit: 100, ratioLimit: .max, grace: .max) + try budget.record(produced: 60, consumed: 10) + #expect(throws: WireIOError.self) { try budget.record(produced: 60, consumed: 10) } +} diff --git a/apple/Packages/TableProEditor/Package.resolved b/apple/Packages/TableProEditor/Package.resolved index 7b2ddfe572..48db09f897 100644 --- a/apple/Packages/TableProEditor/Package.resolved +++ b/apple/Packages/TableProEditor/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "1b5dffbc76dc5f663746a71dadea6dce0ab402033adf018419de50c3a2d0b442", + "originHash" : "927b5c5dbac0fdea39cb9a45d4ce0ca1623ec4b9f96b87636dbed6ca9bea9b8d", "pins" : [ { "identity" : "rearrange", @@ -54,6 +54,15 @@ "revision" : "da6fe9beb4f7f67beb75914ca8e0d48ae48d6406", "version" : "0.25.10" } + }, + { + "identity" : "zstd", + "kind" : "remoteSourceControl", + "location" : "https://github.com/facebook/zstd", + "state" : { + "revision" : "f8745da6ff1ad1e7bab384bd1f9d742439278e99", + "version" : "1.5.7" + } } ], "version" : 3 diff --git a/apple/Packages/TableProMacCore/Package.resolved b/apple/Packages/TableProMacCore/Package.resolved new file mode 100644 index 0000000000..c787a21fdc --- /dev/null +++ b/apple/Packages/TableProMacCore/Package.resolved @@ -0,0 +1,15 @@ +{ + "originHash" : "8416d6d6bf8d3efe83e2ee5a577c19c0a4be27502e2168e80ca90b420e98f493", + "pins" : [ + { + "identity" : "zstd", + "kind" : "remoteSourceControl", + "location" : "https://github.com/facebook/zstd", + "state" : { + "revision" : "f8745da6ff1ad1e7bab384bd1f9d742439278e99", + "version" : "1.5.7" + } + } + ], + "version" : 3 +} diff --git a/apple/Packages/TableProMacCore/Package.swift b/apple/Packages/TableProMacCore/Package.swift index f2e3cf9f4f..e0afd53917 100644 --- a/apple/Packages/TableProMacCore/Package.swift +++ b/apple/Packages/TableProMacCore/Package.swift @@ -49,7 +49,9 @@ let package = Package( name: "TableProMCP", dependencies: [ "TableProUtilities", - .product(name: "TableProLogRedaction", package: "TableProCore") + .product(name: "TableProCoreTypes", package: "TableProCore"), + .product(name: "TableProLogRedaction", package: "TableProCore"), + .product(name: "TableProWireIO", package: "TableProCore") ], path: "Sources/TableProMCP", swiftSettings: appModuleSettings diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/CommandLine/CLIRowWriter.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/CommandLine/CLIRowWriter.swift index 275ed881fc..5814fea5f6 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/CommandLine/CLIRowWriter.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/CommandLine/CLIRowWriter.swift @@ -1,4 +1,5 @@ import Foundation +import TableProCoreTypes public struct CLIResultSet: Sendable, Equatable { public let columns: [String] @@ -129,17 +130,10 @@ public enum CLIRowWriter { // `SELECT 1 AS a, 2 AS a` would otherwise give an object whose second key replaces the first. static func uniqueKeys(_ columns: [String]) -> [String] { - var taken = Set() - return columns.map { column in - var candidate = column - var suffix = 2 - while taken.contains(candidate) { - candidate = "\(column)_\(suffix)" - suffix += 1 - } - taken.insert(candidate) - return candidate - } + var keys = UniqueNameAllocator(candidate: { column, ordinal in + ordinal == 1 ? column : "\(column)_\(ordinal)" + }) + return columns.map { keys.allocate($0) } } private static func jsonLiteral(_ value: JsonValue) -> String { diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Wire/SseDecoder.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Wire/SseDecoder.swift index d8a9d61f82..ba5d969f34 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Wire/SseDecoder.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Wire/SseDecoder.swift @@ -1,24 +1,23 @@ import Foundation +import TableProWireIO public actor SseDecoder { - private var buffer: Data + private static let lineFeed: UInt8 = 0x0A + private static let carriageReturn: UInt8 = 0x0D + + private var buffer = ByteQueue() + // The front of the buffer already searched for a line break, so a line split over many chunks is scanned once. + private var searchedCount = 0 private var pendingEvent: String? private var pendingId: String? private var pendingRetry: Int? - private var pendingDataLines: [String] - private var hasPendingFields: Bool + private var pendingDataLines: [String] = [] + private var hasPendingFields = false - public init() { - buffer = Data() - pendingEvent = nil - pendingId = nil - pendingRetry = nil - pendingDataLines = [] - hasPendingFields = false - } + public init() {} public func feed(_ chunk: Data) -> [SseFrame] { - buffer.append(chunk) + buffer.append(contentsOf: chunk) var frames: [SseFrame] = [] @@ -36,33 +35,32 @@ public actor SseDecoder { } private func takeLine() -> String? { - var index = buffer.startIndex - while index < buffer.endIndex { - let byte = buffer[index] - if byte == 0x0A { - let lineData = buffer[buffer.startIndex.. String { - String(data: data, encoding: .utf8) ?? "" + private func removeLine(length: Int, terminatorLength: Int) -> String { + let line = buffer.removeFirst(length) + buffer.removeFirst(terminatorLength) + searchedCount = 0 + return String(bytes: line, encoding: .utf8) ?? "" } private func processLine(_ line: String) { diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/GzipProcess.swift b/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/GzipProcess.swift deleted file mode 100644 index 70f1cbee8b..0000000000 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/GzipProcess.swift +++ /dev/null @@ -1,130 +0,0 @@ -import Foundation -import os - -private let logger = Logger(subsystem: "com.TablePro", category: "GzipProcess") - -public enum GzipProcess { - public enum GzipError: LocalizedError { - case executableNotFound - case compressionFailed(Int32, String) - case decompressionFailed(Int32, String) - case destinationCreateFailed - - public var errorDescription: String? { - switch self { - case .executableNotFound: - return String(localized: "gzip executable not found") - case .compressionFailed(let status, let message): - return message.isEmpty - ? String(format: String(localized: "Compression failed with exit status %d"), status) - : message - case .decompressionFailed(let status, let message): - return message.isEmpty - ? String(format: String(localized: "Decompression failed with exit status %d"), status) - : message - case .destinationCreateFailed: - return String(localized: "Could not create destination file") - } - } - } - - public static func compress(source: URL, destination: URL) async throws { - let gzipPath = "/usr/bin/gzip" - guard FileManager.default.isExecutableFile(atPath: gzipPath) else { - throw GzipError.executableNotFound - } - - let sourcePath = source.standardizedFileURL.path(percentEncoded: false) - - guard FileManager.default.createFile( - atPath: destination.path(percentEncoded: false), - contents: nil - ) else { - throw GzipError.destinationCreateFailed - } - - let outputHandle = try FileHandle(forWritingTo: destination) - let errorPipe = Pipe() - - let process = Process() - process.executableURL = URL(fileURLWithPath: gzipPath) - process.arguments = ["-c", sourcePath] - process.standardOutput = outputHandle - process.standardError = errorPipe - - try await withTaskCancellationHandler { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - process.terminationHandler = { proc in - try? outputHandle.close() - let status = proc.terminationStatus - if status == 0 { - continuation.resume() - } else { - let errData = errorPipe.fileHandleForReading.readDataToEndOfFile() - let errMsg = String(data: errData, encoding: .utf8)? - .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" - continuation.resume(throwing: GzipError.compressionFailed(status, errMsg)) - } - } - do { - try process.run() - } catch { - try? outputHandle.close() - continuation.resume(throwing: error) - } - } - } onCancel: { - process.terminate() - } - } - - public static func decompress(source: URL, destination: URL) async throws { - let gzipPath = "/usr/bin/gzip" - guard FileManager.default.isExecutableFile(atPath: gzipPath) else { - throw GzipError.executableNotFound - } - - let sourcePath = source.standardizedFileURL.path(percentEncoded: false) - - guard FileManager.default.createFile( - atPath: destination.path(percentEncoded: false), - contents: nil - ) else { - throw GzipError.destinationCreateFailed - } - - let outputHandle = try FileHandle(forWritingTo: destination) - let errorPipe = Pipe() - - let process = Process() - process.executableURL = URL(fileURLWithPath: gzipPath) - process.arguments = ["-d", "-c", sourcePath] - process.standardOutput = outputHandle - process.standardError = errorPipe - - try await withTaskCancellationHandler { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - process.terminationHandler = { proc in - try? outputHandle.close() - let status = proc.terminationStatus - if status == 0 { - continuation.resume() - } else { - let errData = errorPipe.fileHandleForReading.readDataToEndOfFile() - let errMsg = String(data: errData, encoding: .utf8)? - .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" - continuation.resume(throwing: GzipError.decompressionFailed(status, errMsg)) - } - } - do { - try process.run() - } catch { - try? outputHandle.close() - continuation.resume(throwing: error) - } - } - } onCancel: { - process.terminate() - } - } -} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/CommandLine/CLIRowWriterTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/CommandLine/CLIRowWriterTests.swift index 2f3998c412..3ef2a74787 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/CommandLine/CLIRowWriterTests.swift +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/CommandLine/CLIRowWriterTests.swift @@ -71,6 +71,13 @@ struct CLIRowWriterTests { #expect(CLIRowWriter.uniqueKeys(["a", "a", "a_2", "b"]) == ["a", "a_2", "a_2_2", "b"]) } + @Test("A result whose columns all share one name gets its keys in linear time", .timeLimit(.minutes(1))) + func manyDuplicateKeys() { + let keys = CLIRowWriter.uniqueKeys(Array(repeating: "a", count: 40_000)) + #expect(keys.last == "a_40000") + #expect(Set(keys).count == 40_000) + } + @Test("Control characters are escaped in JSON strings") func jsonControlCharacters() { #expect(CLIRowWriter.jsonString("a\u{1}b\nc") == "\"a\\u0001b\\nc\"") diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Wire/SseDecoderChunkingTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Wire/SseDecoderChunkingTests.swift new file mode 100644 index 0000000000..c0ed152855 --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Wire/SseDecoderChunkingTests.swift @@ -0,0 +1,80 @@ +import Foundation +import TableProMCP +import Testing + +struct SseDecoderChunkingTests { + private static func decode(_ chunks: [Data]) async -> [SseFrame] { + let decoder = SseDecoder() + var frames: [SseFrame] = [] + for chunk in chunks { + frames.append(contentsOf: await decoder.feed(chunk)) + } + return frames + } + + private static func chunks(of stream: Data, size: Int) -> [Data] { + stride(from: 0, to: stream.count, by: size).map { stream.subdata(in: $0..(_ total: Value, _ addition: Value) -> Value { + let (sum, overflow) = total.addingReportingOverflow(addition) + return overflow ? (addition < 0 ? .min : .max) : sum + } + // MARK: - DDL and metadata /// Statements that recreate the table: its `CreateTable` request, then Time to Live and @@ -262,7 +267,7 @@ extension DynamoDBPluginDriver { return PluginTableMetadata( tableName: description.name, dataSize: description.sizeBytes, - indexSize: description.indexes.compactMap(\.sizeBytes).reduce(0, +), + indexSize: description.indexes.compactMap(\.sizeBytes).reduce(Int64(0), Self.saturatingSum), totalSize: description.sizeBytes, rowCount: description.itemCount, comment: DynamoDBTableDefinition.summary(description), diff --git a/apple/Plugins/FormatPluginTests/XLSXImportFixture.swift b/apple/Plugins/FormatPluginTests/XLSXImportFixture.swift new file mode 100644 index 0000000000..eefb5b975e --- /dev/null +++ b/apple/Plugins/FormatPluginTests/XLSXImportFixture.swift @@ -0,0 +1,107 @@ +// +// XLSXImportFixture.swift +// FormatPluginTests +// + +import Foundation +import zlib + +struct XLSXImportFixture { + struct Sheet { + let name: String + var isHidden = false + let rows: String + } + + private static let mainNamespace = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + private static let relationshipNamespace = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + + var sheets: [Sheet] + var sharedStrings: [String]? + // The numFmtId of each cell style, in the order a cell's `s` attribute indexes them. + var cellFormatIDs: [Int] + + init(rows: String, sharedStrings: [String]? = nil, cellFormatIDs: [Int] = []) { + self.init(sheets: [Sheet(name: "Data", rows: rows)], sharedStrings: sharedStrings, cellFormatIDs: cellFormatIDs) + } + + init(sheets: [Sheet], sharedStrings: [String]? = nil, cellFormatIDs: [Int] = []) { + self.sheets = sheets + self.sharedStrings = sharedStrings + self.cellFormatIDs = cellFormatIDs + } + + func write() throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("XLSXImport-\(UUID().uuidString).xlsx") + try Self.zip(entries()).write(to: url) + return url + } + + private func entries() -> [(path: String, body: String)] { + let relationshipType = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/" + let sheetEntries = sheets.enumerated().map { index, sheet in + "" + } + var relationships = sheets.indices.map { index in + "" + } + var entries: [(path: String, body: String)] = [ + ("_rels/.rels", ""), + ("xl/workbook.xml", "\(sheetEntries.joined())"), + ] + if let sharedStrings { + relationships.append("") + entries.append(("xl/sharedStrings.xml", "\(sharedStrings.joined())")) + } + if !cellFormatIDs.isEmpty { + relationships.append("") + let formats = cellFormatIDs.map { "" }.joined() + entries.append(("xl/styles.xml", "\(formats)")) + } + entries.append(("xl/_rels/workbook.xml.rels", "\(relationships.joined())")) + for (index, sheet) in sheets.enumerated() { + entries.append(("xl/worksheets/sheet\(index + 1).xml", "\(sheet.rows)")) + } + return entries + } + + private static func zip(_ entries: [(path: String, body: String)]) -> Data { + var output = Data() + var directory = Data() + for entry in entries { + let name = Data(entry.path.utf8) + let body = Data(entry.body.utf8) + let checksum = body.withUnsafeBytes { raw in + UInt32(zlib.crc32(0, raw.bindMemory(to: Bytef.self).baseAddress, uInt(raw.count))) + } + let offset = UInt32(output.count) + output.append(contentsOf: [0x50, 0x4B, 0x03, 0x04]) + output.append(littleEndian: [UInt16(20), 0, 0, 0, 0]) + output.append(littleEndian: [checksum, UInt32(body.count), UInt32(body.count)]) + output.append(littleEndian: [UInt16(name.count), 0]) + output.append(name) + output.append(body) + directory.append(contentsOf: [0x50, 0x4B, 0x01, 0x02]) + directory.append(littleEndian: [UInt16(20), 20, 0, 0, 0, 0]) + directory.append(littleEndian: [checksum, UInt32(body.count), UInt32(body.count)]) + directory.append(littleEndian: [UInt16(name.count), 0, 0, 0, 0]) + directory.append(littleEndian: [UInt32(0), offset]) + directory.append(name) + } + let directoryOffset = UInt32(output.count) + output.append(directory) + output.append(contentsOf: [0x50, 0x4B, 0x05, 0x06]) + output.append(littleEndian: [UInt16(0), 0, UInt16(entries.count), UInt16(entries.count)]) + output.append(littleEndian: [UInt32(directory.count), directoryOffset]) + output.append(littleEndian: [UInt16(0)]) + return output + } +} + +private extension Data { + mutating func append(littleEndian values: [Value]) { + for value in values { + Swift.withUnsafeBytes(of: value.littleEndian) { append(contentsOf: $0) } + } + } +} diff --git a/apple/Plugins/FormatPluginTests/XLSXImportTests.swift b/apple/Plugins/FormatPluginTests/XLSXImportTests.swift index 6deb1b93c0..fe5b08b0d1 100644 --- a/apple/Plugins/FormatPluginTests/XLSXImportTests.swift +++ b/apple/Plugins/FormatPluginTests/XLSXImportTests.swift @@ -7,129 +7,254 @@ import Foundation import TableProPluginKit import Testing -struct XLSXSheetParserTests { - /// `A` is 0 and `AA` is 26, so the letters are base-26 with no zero digit. Getting this wrong - /// puts every column past Z in the wrong place. - @Test("A cell reference resolves to its column index") - func columnIndexFromReference() { - #expect(XLSXSheetParser.columnIndex(fromReference: "A1") == 0) - #expect(XLSXSheetParser.columnIndex(fromReference: "B2") == 1) - #expect(XLSXSheetParser.columnIndex(fromReference: "Z9") == 25) - #expect(XLSXSheetParser.columnIndex(fromReference: "AA1") == 26) - #expect(XLSXSheetParser.columnIndex(fromReference: "AB1") == 27) - #expect(XLSXSheetParser.columnIndex(fromReference: "BA10") == 52) +private final class RecordingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "mock" + let targetTable: String? = "people" + + private(set) var rows: [[String: PluginCellValue]] = [] + + func execute(statement: String) async throws {} + func insertRow(_ values: [String: PluginCellValue]) async throws { rows.append(values) } + func deleteAllRowsFromTargetTable() async throws {} + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} +} + +// Serialized because `XLSXImportPlugin.settings` persists through plugin storage, so two instances in flight read +// each other's options. +@Suite(.serialized) +struct XLSXImportSheetTests { + private func read(_ fixture: XLSXImportFixture, hasHeaderRow: Bool = true) throws -> XLSXImportSheet { + let url = try fixture.write() + defer { try? FileManager.default.removeItem(at: url) } + return try XLSXImportSheet.read(at: url, hasHeaderRow: hasHeaderRow, isCancelled: { false }) + } + + private func values(_ sheet: XLSXImportSheet) -> [[String?]] { + sheet.rows(0 ..< sheet.rowCount).map(\.values) + } + + private func number(_ reference: String, _ value: String) -> String { + "\(value)" + } + + private func inline(_ reference: String, _ text: String) -> String { + "\(text)" + } + + // Row 3 holds only cells that store empty text, row 4 only a styled cell with nothing stored. + private var emptyCells: XLSXImportFixture { + let rows = """ + \(inline("A1", "id"))\(inline("B1", "note"))\(inline("C1", "code"))\ + \(number("A2", "1"))IF(A2="","",A2)0\ + ""\ + \ + \(number("A5", "2")) + """ + return XLSXImportFixture(rows: rows, sharedStrings: [""]) + } + + private func imported(_ fixture: XLSXImportFixture, emptyAsNull: Bool) async throws -> [[String: PluginCellValue]] { + let url = try fixture.write() + defer { try? FileManager.default.removeItem(at: url) } + let plugin = XLSXImportPlugin() + let saved = plugin.settings + defer { plugin.settings = saved } + var options = XLSXImportOptions() + options.emptyAsNull = emptyAsNull + plugin.settings = options + let sink = RecordingSink() + _ = try await plugin.performImport( + source: RoundTripImportFile(url), + sink: sink, + progress: PluginImportProgress(progress: Progress(totalUnitCount: 1)) + ) + return sink.rows + } + + @Test("A cell lands in the column its reference names") + func cellsArePlacedByReference() throws { + let cells = [("A1", "a"), ("B1", "b"), ("Z1", "z"), ("AA1", "aa"), ("AB1", "ab"), ("BA1", "ba")] + let sheet = try read(XLSXImportFixture(rows: "\(cells.map { inline($0.0, $0.1) }.joined())"), hasHeaderRow: false) + let row = try #require(values(sheet).first) + #expect(row.count == 53) + #expect([0, 1, 25, 26, 27, 52].map { row[$0] } == ["a", "b", "z", "aa", "ab", "ba"]) + #expect(row.compactMap { $0 }.count == 6) } @Test("A repeated or blank header gets a name of its own") func headerNamesAreUnique() { - let names = XLSXSheetParser.headerNames(from: ["Notes", "Notes", nil, " Id "]) - - #expect(names == ["Notes", "Notes 2", "column3", "Id"]) + #expect(XLSXImportSheet.headerNames(from: ["Notes", "Notes", nil, " Id "]) == ["Notes", "Notes 2", "column3", "Id"]) } @Test("A header spelled out keeps its name over a blank cell's placeholder that comes first") func literalHeaderKeepsItsName() { - let names = XLSXSheetParser.headerNames(from: [nil, "column1", "Code", "Code", "Code 2"]) - + let names = XLSXImportSheet.headerNames(from: [nil, "column1", "Code", "Code", "Code 2"]) #expect(names == ["column1 2", "column1", "Code", "Code 3", "Code 2"]) } - @Test("A reference with no letters is refused") - func invalidReference() { - #expect(XLSXSheetParser.columnIndex(fromReference: "1") == nil) - #expect(XLSXSheetParser.columnIndex(fromReference: "") == nil) + @Test("A reference past the last column is dropped instead of widening the row") + func referencePastTheLastColumnIsDropped() throws { + let rows = """ + \(inline("A1", "id"))\(inline("B1", "name"))\ + \(number("A2", "1"))\(number("AAAAAAAAAAAAAAA2", "2"))\(number("ZZZZZ2", "3"))\(number("XFE2", "4")) + """ + let sheet = try read(XLSXImportFixture(rows: rows)) + #expect(sheet.header == ["id", "name"]) + #expect(values(sheet) == [["1", nil]]) } - /// Part names vary between writers, so the first worksheet is found rather than assumed to be - /// `sheet1.xml`. - @Test("The first worksheet is found by path, not assumed") - func firstWorksheetIsFound() { - let paths = ["xl/workbook.xml", "xl/worksheets/sheet2.xml", "xl/worksheets/sheet1.xml", "[Content_Types].xml"] - #expect(XLSXSheetParser.firstWorksheetPath(in: paths) == "xl/worksheets/sheet1.xml") - #expect(XLSXSheetParser.firstWorksheetPath(in: ["xl/workbook.xml"]) == nil) + @Test("A cell at the last column and many empty rows import as one row") + func emptyRowsBelowAFarCellAreNotMaterialised() throws { + let empty = (2...100_001).map { "" }.joined() + let rows = "\(inline("A1", "id"))\(inline("XFD1", "far"))\(empty)\(number("A100002", "7"))" + let sheet = try read(XLSXImportFixture(rows: rows)) + let header = try #require(sheet.header) + #expect(header.count == 16_384) + #expect(header.first == "id") + #expect(header[1] == "column2") + #expect(header.last == "far") + #expect(sheet.rowCount == 1) + let row = try #require(sheet.rows(0 ..< 1).first) + #expect(row.line == 100_002) + #expect(row.values.first == "7") + #expect(row.values.dropFirst().allSatisfy { $0 == nil }) } - /// A string split across formatting runs is one value, not several. A styled word mid-cell - /// would otherwise truncate it. - @Test("Shared strings concatenate their runs") - func sharedStringRuns() { - let xml = """ - - AdaGrace Hopper - """ - let strings = XLSXSheetParser.sharedStrings(from: Data(xml.utf8)) - #expect(strings == ["Ada", "Grace Hopper"]) - } - - /// A cell typed `s` holds an index into the shared string table rather than the text. - @Test("A shared-string cell resolves through the table") - func sharedStringCellResolves() { - let sheet = """ - - - 01 - - """ - let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: ["id", "name"]) - #expect(rows == [["id", "name"]]) + @Test("The first visible sheet is read, in workbook order") + func firstVisibleSheetIsRead() throws { + let fixture = XLSXImportFixture(sheets: [ + .init(name: "Archive", isHidden: true, rows: "\(inline("A1", "old"))"), + .init(name: "Data", rows: "\(inline("A1", "current"))"), + ]) + #expect(try read(fixture).header == ["current"]) + } + + @Test("Shared strings concatenate their runs and resolve by index") + func sharedStringsResolve() throws { + let rows = "01" + let strings = ["Ada", "Grace Hopper"] + #expect(try read(XLSXImportFixture(rows: rows, sharedStrings: strings)).header == ["Ada", "Grace Hopper"]) + } + + @Test("A shared-string index the table lacks keeps the raw value") + func outOfRangeIndexKeepsRawValue() throws { + let rows = "99" + #expect(values(try read(XLSXImportFixture(rows: rows, sharedStrings: ["only"]), hasHeaderRow: false)) == [["99"]]) } - /// A row omits the cells it has no value for, so position comes from each cell's own reference. - /// Counting cells instead shifts every value after a gap into the wrong column. @Test("A gap in a row is filled from the cell references") - func gapsArePlacedByReference() { - let sheet = """ - - - 13 - - """ - let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []) - #expect(rows == [["1", nil, "3"]]) + func gapsArePlacedByReference() throws { + let rows = "\(number("A1", "1"))\(number("C1", "3"))" + #expect(values(try read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["1", nil, "3"]]) } - @Test("An inline string is read from the cell itself") - func inlineStringsAreRead() { - let sheet = """ - - - Ada - - """ - let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []) - #expect(rows == [["Ada"]]) + @Test("Rows are padded to the widest row") + func rowsArePadded() throws { + let rows = "\(number("A1", "1"))\(number("B1", "2"))\(number("A2", "3"))" + #expect(values(try read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["1", "2"], ["3", nil]]) } - /// A damaged workbook still imports something the user can see is wrong, rather than dropping - /// the value silently. - @Test("A shared-string index the table lacks keeps the raw value") - func outOfRangeIndexKeepsRawValue() { - let sheet = """ - - 99 + @Test("Numbers keep their text and booleans read as 1 and 0") + func valuesKeepTheirStoredText() throws { + let rows = "\(number("A1", "12.50"))\(number("B1", "007"))10" + #expect(values(try read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["12.50", "007", "1", "0"]]) + } + + @Test("A number of nineteen digits reads as its text") + func nineteenDigitNumberIsRead() throws { + let rows = "\(number("A1", "9999999999999999999"))\(number("B1", "-9223372036854775808"))" + #expect(values(try read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["9999999999999999999", "-9223372036854775808"]]) + } + + @Test("A row's line is its row number in the sheet") + func linesFollowTheSheet() throws { + let rows = "\(inline("A3", "id"))\(number("A5", "1"))" + let sheet = try read(XLSXImportFixture(rows: rows)) + #expect(sheet.rows(0 ..< sheet.rowCount).map(\.line) == [5]) + } + + @Test("An empty sheet reads as no rows rather than failing") + func emptySheet() throws { + let headed = try read(XLSXImportFixture(rows: "")) + #expect(headed.header == nil) + #expect(headed.rowCount == 0) + let plain = try read(XLSXImportFixture(rows: ""), hasHeaderRow: false) + #expect(plain.header?.isEmpty == true) + #expect(plain.rowCount == 0) + } + + @Test("A stored cell with empty text reads as empty text, a cell with nothing stored as nil") + func emptyTextIsNotAMissingCell() throws { + let sheet = try read(emptyCells) + #expect(sheet.header == ["id", "note", "code"]) + #expect(values(sheet) == [["1", "", ""], [nil, "", ""], ["2", nil, nil]]) + #expect(sheet.rows(0 ..< sheet.rowCount).map(\.line) == [2, 3, 5]) + } + + @Test("With Empty as NULL off, a cell that stores empty text imports as empty text") + func emptyTextImportsAsEmptyText() async throws { + let rows = try await imported(emptyCells, emptyAsNull: false) + #expect(rows == [ + ["id": .text("1"), "note": .text(""), "code": .text("")], + ["id": .null, "note": .text(""), "code": .text("")], + ["id": .text("2"), "note": .null, "code": .null], + ]) + } + + @Test("With Empty as NULL on, a cell that stores empty text imports as NULL") + func emptyTextImportsAsNull() async throws { + let rows = try await imported(emptyCells, emptyAsNull: true) + #expect(rows == [ + ["id": .text("1"), "note": .null, "code": .null], + ["id": .null, "note": .null, "code": .null], + ["id": .text("2"), "note": .null, "code": .null], + ]) + } + + @Test("Date, time and duration cells read as ISO dates and elapsed time, not their serial number") + func temporalCellsReadAsText() throws { + let rows = """ + 4529245292.5\ + 0.51.5\(number("E1", "45292")) """ - let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: ["only"]) - #expect(rows == [["99"]]) + let fixture = XLSXImportFixture(rows: rows, cellFormatIDs: [0, 14, 22, 20, 46]) + #expect(values(try read(fixture, hasHeaderRow: false)) == [["2024-01-01", "2024-01-01 12:00:00", "12:00:00", "36:00:00", "45292"]]) } - @Test("Rows are padded to the widest row") - func rowsArePadded() { - let sheet = """ - - - 12 - 3 - + @Test("Field detection types a date column as text") + func detectsADateColumnAsText() throws { + let rows = """ + \(inline("A1", "day"))\ + 4529245293 """ - let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []) - #expect(rows.count == 2) - #expect(rows.allSatisfy { $0.count == 2 }) - #expect(rows[1] == ["3", nil]) + let url = try XLSXImportFixture(rows: rows, cellFormatIDs: [0, 14]).write() + defer { try? FileManager.default.removeItem(at: url) } + let plugin = XLSXImportPlugin() + let saved = plugin.settings + defer { plugin.settings = saved } + plugin.settings = XLSXImportOptions() + let fields = try plugin.detectSourceFields(at: url, targetTable: nil) + #expect(fields.map(\.sampleValue) == ["2024-01-01"]) + #expect(fields.map(\.inferredType) == [.text]) } - @Test("An empty sheet reads as no rows rather than failing") - func emptySheet() { - let sheet = "" - #expect(XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []).isEmpty) + @Test("Field detection names the header and infers from the rows below it") + func detectsFieldsFromTheHeaderRow() throws { + let rows = """ + \(inline("A1", "id"))\(inline("B1", "name"))\ + \(number("A2", "1"))\(inline("B2", "Ada"))\(number("A3", "2")) + """ + let url = try XLSXImportFixture(rows: rows).write() + defer { try? FileManager.default.removeItem(at: url) } + let plugin = XLSXImportPlugin() + let saved = plugin.settings + defer { plugin.settings = saved } + plugin.settings = XLSXImportOptions() + let fields = try plugin.detectSourceFields(at: url, targetTable: nil) + #expect(fields.map(\.name) == ["id", "name"]) + #expect(fields.map(\.sampleValue) == ["1", "Ada"]) + #expect(fields.map(\.inferredType) == [.integer, .text]) } } diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift index f33e85c768..99a420548f 100644 --- a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift @@ -227,7 +227,7 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send guard response.isSuccess else { throw response.error } switch (server.generation, statement.language) { case (.v3, _): - return InfluxResponseDecoder.decodeV3CSV(response.data) + return try InfluxResponseDecoder.decodeV3CSV(response.data) case (_, .flux): return try InfluxResponseDecoder.decodeFluxCSV(response.data) default: diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaApiKey.swift b/apple/Plugins/KafkaDriverPlugin/KafkaApiKey.swift index 02fdd4dd52..2e24ed1732 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaApiKey.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaApiKey.swift @@ -1,24 +1,5 @@ import Foundation - -struct KafkaUUID: Hashable, Sendable { - let bytes: [UInt8] - - static let zero = KafkaUUID(bytes: [UInt8](repeating: 0, count: 16)) - - init(bytes: [UInt8]) { - self.bytes = bytes.count == 16 ? bytes : [UInt8](repeating: 0, count: 16) - } - - var isZero: Bool { bytes.allSatisfy { $0 == 0 } } - - var uuidString: String { - let hex = bytes.map { String(format: "%02x", $0) }.joined() - let groups = [0 ..< 8, 8 ..< 12, 12 ..< 16, 16 ..< 20, 20 ..< 32] - return groups - .map { String(hex[hex.index(hex.startIndex, offsetBy: $0.lowerBound) ..< hex.index(hex.startIndex, offsetBy: $0.upperBound)]) } - .joined(separator: "-") - } -} +import TableProKafkaCore enum KafkaApiKey: Int16, CaseIterable, Sendable { case produce = 0 diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaBrowseEngine.swift b/apple/Plugins/KafkaDriverPlugin/KafkaBrowseEngine.swift index 2d0ed6bfe2..2273558179 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaBrowseEngine.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaBrowseEngine.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore /// Where a scan reads from, and which end of the merged run is the page. struct KafkaScanWindow: Sendable { @@ -103,11 +104,7 @@ enum KafkaBrowseEngine { return Set(requested).sorted() } - /// Resolves a start mode into one concrete window. - /// - /// `.resolved` and `.tail` short-circuit, and that is the point: the browse path bakes the - /// resolved window into the query string it hands back, so re-running it for page two reads - /// the same window rather than re-deriving "newest" against a tail that has since moved. + // `.resolved` and `.tail` come from the window baked into page one's query, so page two reads that window, not a moved tail. static func resolveWindow( _ query: KafkaConsumeQuery, partitions: [Int32], @@ -140,7 +137,7 @@ enum KafkaBrowseEngine { // an unknown floor, and anchoring it at zero would send the page to the start of // a log whose first surviving message may be far past it. guard let floor = earliest[partition] else { continue } - starts[partition] = max(floor, end - step) + starts[partition] = KafkaOffsetMath.stepBack(from: end, by: step, floor: floor) } return KafkaScanWindow(start: starts, tail: kept, readsBackward: true) @@ -204,7 +201,7 @@ enum KafkaBrowseEngine { var starts: [Int32: Int64] = [:] var ends: [Int32: Int64] = [:] for bound in bounds { - starts[bound.partition] = max(bound.earliest, bound.latest - step) + starts[bound.partition] = KafkaOffsetMath.stepBack(from: bound.latest, by: step, floor: bound.earliest) ends[bound.partition] = bound.latest } return KafkaScanWindow(start: starts, tail: ends, readsBackward: true) diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaCluster+Routing.swift b/apple/Plugins/KafkaDriverPlugin/KafkaCluster+Routing.swift index 8e667b795c..f455f9a8eb 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaCluster+Routing.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaCluster+Routing.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore /// One partition's answer inside a reply that carries several. /// diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaCluster.swift b/apple/Plugins/KafkaDriverPlugin/KafkaCluster.swift index eea039eb0c..8a828070d0 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaCluster.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaCluster.swift @@ -2,6 +2,7 @@ import Foundation import NIOCore import NIOPosix import os +import TableProKafkaCore import TableProPluginKit /// Owns the connections to a cluster: the bootstrap dial, the per-broker pool, and the routing diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaClusterTypes.swift b/apple/Plugins/KafkaDriverPlugin/KafkaClusterTypes.swift index ffb1272a56..b7b769050c 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaClusterTypes.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaClusterTypes.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore struct KafkaEndpoint: Hashable, Sendable { let host: String diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaCompression.swift b/apple/Plugins/KafkaDriverPlugin/KafkaCompression.swift deleted file mode 100644 index 4b7b207d5d..0000000000 --- a/apple/Plugins/KafkaDriverPlugin/KafkaCompression.swift +++ /dev/null @@ -1,236 +0,0 @@ -import Compression -import Foundation -import libzstd -import zlib - -/// A Kafka broker never transcodes a batch for the consumer: it hands back exactly the bytes -/// the producer stored, in whatever codec the producer chose. Measured against a live broker -/// by producing one topic per codec and reading the stored attribute plus the payload's magic -/// bytes back: gzip `1f8b0800`, snappy `82 53 4e 41 50 50 59 00` (xerial framing, not raw -/// snappy), lz4 `04224d18` (the LZ4 frame format, not a raw block), zstd `28b52ffd`. -/// -/// So all four have to be decoded here. macOS supplies only some of that: zlib is in the SDK, -/// Compression.framework has a raw LZ4 block decoder, and there is no snappy and no zstd -/// anywhere on the system (verified by reading compression.h and by dlopen'ing libarchive, -/// which exports no ZSTD_ symbol). zstd therefore comes from the upstream SPM package and -/// snappy is implemented here. -enum KafkaCompression { - /// The ceiling on what any codec may claim it decompresses to. A declared size is written - /// by the producer and arrives over the network, so it is a claim rather than a fact. - static let maximumDecompressedSize = 512 * 1024 * 1024 - - static func decompress(_ payload: Data, codec: KafkaCompressionCodec) throws -> Data { - switch codec { - case .none: return payload - case .gzip: return try gunzip(payload) - case .snappy: return try snappyXerial(payload) - case .lz4: return try lz4Frame(payload) - case .zstd: return try zstd(payload) - } - } - - // MARK: - gzip - - /// zlib's own gzip mode, selected by adding 16 to the window bits. Compression.framework's - /// COMPRESSION_ZLIB is raw DEFLATE with no gzip wrapper, so it cannot read this directly. - private static func gunzip(_ payload: Data) throws -> Data { - guard !payload.isEmpty else { return Data() } - var stream = z_stream() - guard inflateInit2_(&stream, 16 + MAX_WBITS, ZLIB_VERSION, Int32(MemoryLayout.size)) == Z_OK else { - throw KafkaError.decompressionFailed(codec: "gzip", reason: "could not start zlib") - } - defer { inflateEnd(&stream) } - - var output = Data() - var input = [UInt8](payload) - let chunkSize = max(payload.count * 4, 64 * 1024) - var chunk = [UInt8](repeating: 0, count: chunkSize) - - let status: Int32 = input.withUnsafeMutableBufferPointer { inputBuffer -> Int32 in - stream.next_in = inputBuffer.baseAddress - stream.avail_in = uInt(inputBuffer.count) - while true { - let result: Int32 = chunk.withUnsafeMutableBufferPointer { outputBuffer -> Int32 in - stream.next_out = outputBuffer.baseAddress - stream.avail_out = uInt(outputBuffer.count) - let code = inflate(&stream, Z_NO_FLUSH) - let produced = outputBuffer.count - Int(stream.avail_out) - if produced > 0 { - output.append(contentsOf: outputBuffer.prefix(produced)) - } - return code - } - if result == Z_STREAM_END { return Z_STREAM_END } - if result != Z_OK { return result } - // Only Z_STREAM_END means the stream ended. Running out of input without it - // means the payload is truncated, and returning what decoded so far would - // hand back a partial message that looks like a whole one. - if stream.avail_in == 0, stream.avail_out != 0 { return Z_DATA_ERROR } - } - } - - guard status == Z_STREAM_END else { - throw KafkaError.decompressionFailed( - codec: "gzip", - reason: status == Z_DATA_ERROR - ? String(localized: "the compressed data is corrupt or was cut short") - : "zlib error \(status)" - ) - } - return output - } - - // MARK: - snappy - - private static let xerialMagic: [UInt8] = [0x82, 0x53, 0x4E, 0x41, 0x50, 0x50, 0x59, 0x00] - - /// The Java producer wraps snappy in "xerial" framing: an 8-byte magic, two version ints, - /// then a run of big-endian length-prefixed snappy blocks. A payload without the magic is - /// a single bare snappy block, which is what some non-Java clients write. - private static func snappyXerial(_ payload: Data) throws -> Data { - let bytes = [UInt8](payload) - guard bytes.count >= xerialMagic.count, Array(bytes.prefix(8)) == xerialMagic else { - return Data(try SnappyDecoder.decode(bytes)) - } - var cursor = 16 // magic + version + compatibleVersion - var output = Data() - while cursor + 4 <= bytes.count { - let length = Int(bytes[cursor]) << 24 | Int(bytes[cursor + 1]) << 16 - | Int(bytes[cursor + 2]) << 8 | Int(bytes[cursor + 3]) - cursor += 4 - guard length >= 0, cursor + length <= bytes.count else { - throw KafkaError.decompressionFailed(codec: "snappy", reason: "block runs past the payload") - } - output.append(contentsOf: try SnappyDecoder.decode(Array(bytes[cursor ..< cursor + length]))) - cursor += length - } - return output - } - - // MARK: - lz4 - - /// Kafka uses the LZ4 FRAME format. Compression.framework's COMPRESSION_LZ4 is Apple's own - /// container with a `bv41` magic and is not interchangeable with it; reaching for that - /// instead of parsing the frame yields corrupt output rather than an error. The blocks - /// inside the frame are raw LZ4, which COMPRESSION_LZ4_RAW does decode. - private static func lz4Frame(_ payload: Data) throws -> Data { - let bytes = [UInt8](payload) - guard bytes.count > 7, - bytes[0] == 0x04, bytes[1] == 0x22, bytes[2] == 0x4D, bytes[3] == 0x18 else { - throw KafkaError.decompressionFailed(codec: "lz4", reason: "not an LZ4 frame") - } - let flags = bytes[4] - let blockDescriptor = bytes[5] - let hasContentSize = flags & 0x08 != 0 - let hasDictionaryId = flags & 0x01 != 0 - let hasBlockChecksum = flags & 0x10 != 0 - let maxBlockSize = lz4MaxBlockSize(blockDescriptor) - - var cursor = 7 // magic + FLG + BD + header checksum - if hasContentSize { cursor += 8 } - if hasDictionaryId { cursor += 4 } - - var output = Data() - while cursor + 4 <= bytes.count { - let rawSize = UInt32(bytes[cursor]) | UInt32(bytes[cursor + 1]) << 8 - | UInt32(bytes[cursor + 2]) << 16 | UInt32(bytes[cursor + 3]) << 24 - cursor += 4 - if rawSize == 0 { break } // EndMark - let isUncompressed = rawSize & 0x8000_0000 != 0 - let blockSize = Int(rawSize & 0x7FFF_FFFF) - guard blockSize >= 0, cursor + blockSize <= bytes.count else { - throw KafkaError.decompressionFailed(codec: "lz4", reason: "block runs past the frame") - } - let block = Array(bytes[cursor ..< cursor + blockSize]) - cursor += blockSize - if hasBlockChecksum { cursor += 4 } - - if isUncompressed { - output.append(contentsOf: block) - } else { - output.append(contentsOf: try lz4RawBlock(block, capacity: maxBlockSize)) - } - } - return output - } - - private static func lz4MaxBlockSize(_ descriptor: UInt8) -> Int { - switch (descriptor >> 4) & 0x07 { - case 4: return 64 * 1024 - case 5: return 256 * 1024 - case 6: return 1024 * 1024 - case 7: return 4 * 1024 * 1024 - default: return 4 * 1024 * 1024 - } - } - - /// `capacity` is the frame's declared maximum block size, which is the real bound. Sizing - /// to LZ4's 255x worst case instead means zero-filling 16 MB for every 64 KB block, which - /// is what the Java producer writes by default. - private static func lz4RawBlock(_ block: [UInt8], capacity: Int) throws -> [UInt8] { - var destination = [UInt8](repeating: 0, count: max(capacity, block.count + 64)) - let produced = block.withUnsafeBufferPointer { source -> Int in - destination.withUnsafeMutableBufferPointer { target -> Int in - guard let sourceBase = source.baseAddress, let targetBase = target.baseAddress else { return 0 } - return compression_decode_buffer( - targetBase, target.count, - sourceBase, source.count, - nil, COMPRESSION_LZ4_RAW - ) - } - } - guard produced > 0 else { - throw KafkaError.decompressionFailed(codec: "lz4", reason: "the block did not decode") - } - return Array(destination.prefix(produced)) - } - - // MARK: - zstd - - private static func zstd(_ payload: Data) throws -> Data { - guard !payload.isEmpty else { return Data() } - let bytes = [UInt8](payload) - - // The frame usually declares its decompressed size. When it does not (a streamed - // frame writes ZSTD_CONTENTSIZE_UNKNOWN), fall back to a growing buffer. - let declared = bytes.withUnsafeBufferPointer { buffer -> UInt64 in - guard let base = buffer.baseAddress else { return 0 } - return ZSTD_getFrameContentSize(base, buffer.count) - } - let unknown = UInt64(bitPattern: -1) - let error = UInt64(bitPattern: -2) - guard declared != error else { - throw KafkaError.decompressionFailed(codec: "zstd", reason: "not a zstd frame") - } - - // The declared size comes from whoever produced the frame. Snappy caps its own declared - // length for the same reason: converting an arbitrary UInt64 to Int traps, and honouring - // one names an allocation nothing here can satisfy. - guard declared == unknown || declared <= UInt64(maximumDecompressedSize) else { - throw KafkaError.decompressionFailed(codec: "zstd", reason: "the frame declares an implausible size") - } - var capacity = declared == unknown ? max(payload.count * 8, 128 * 1024) : Int(declared) - for _ in 0 ..< 8 { - var destination = [UInt8](repeating: 0, count: max(capacity, 1)) - let produced = bytes.withUnsafeBufferPointer { source -> Int in - destination.withUnsafeMutableBufferPointer { target -> Int in - guard let sourceBase = source.baseAddress, let targetBase = target.baseAddress else { return 0 } - return ZSTD_decompress(targetBase, target.count, sourceBase, source.count) - } - } - if ZSTD_isError(produced) == 0 { - return Data(destination.prefix(produced)) - } - guard declared == unknown else { - throw KafkaError.decompressionFailed(codec: "zstd", reason: zstdMessage(produced)) - } - capacity *= 4 - } - throw KafkaError.decompressionFailed(codec: "zstd", reason: "the frame did not fit a reasonable buffer") - } - - private static func zstdMessage(_ code: Int) -> String { - guard let raw = ZSTD_getErrorName(code) else { return "unknown zstd error" } - return String(cString: raw) - } -} diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaConnection.swift b/apple/Plugins/KafkaDriverPlugin/KafkaConnection.swift index ce3102c704..cd38c12878 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaConnection.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaConnection.swift @@ -3,6 +3,7 @@ import NIOConcurrencyHelpers import NIOCore import NIOPosix import NIOSSL +import TableProKafkaCore import TableProPluginKit /// Splits the TCP stream into whole Kafka responses. Every response is a 4-byte big-endian diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaDeleteTopicsRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaDeleteTopicsRequest.swift index f9dc96c708..c79b176c81 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaDeleteTopicsRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaDeleteTopicsRequest.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore enum KafkaDeleteTopicsRequest { /// How long the broker may spend on the deletion before answering REQUEST_TIMED_OUT. diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift index 9af9972d87..9365bfa506 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift @@ -1,4 +1,6 @@ import Foundation +import TableProKafkaCore +import TableProWireIO struct KafkaFetchResult: Sendable { let records: [KafkaRecord] @@ -18,13 +20,7 @@ enum KafkaFetchRequest { private static let defaultPartitionBudget: Int32 = 1 * 1_024 * 1_024 private static let maximumPartitionBudget: Int32 = 64 * 1_024 * 1_024 - /// Fetches forward from `startOffset` in one partition. - /// - /// The byte budget is escalated rather than fixed. Kafka's per-partition limit guarantees - /// progress only for a message that is FIRST in the requested range (KIP-74): a 10 MB - /// message sitting behind a 1 MB budget returns zero records forever, and a fixed budget - /// turns that into a page that stops early with no explanation. Doubling on an empty - /// reply that the high watermark says should not be empty is what breaks the stall. + // Kafka guarantees progress only for the first message in range (KIP-74), so the byte budget grows on an empty reply. static func fetch( topic: String, partition: Int32, @@ -37,16 +33,20 @@ enum KafkaFetchRequest { var highWatermark: Int64 = startOffset var offset = startOffset var sawTruncation = false + var decompression: DecompressionBudget? while collected.count < maximumRecords { try Task.checkCancellation() - let page = try await fetchOnce( + let allowance = try KafkaCompression.fetchBudget(requesting: budget, after: decompression) + let (page, spent) = try await fetchOnce( topic: topic, partition: partition, startOffset: offset, budget: budget, + decompression: allowance, cluster: cluster ) + decompression = spent highWatermark = page.highWatermark if page.records.isEmpty { @@ -62,8 +62,8 @@ enum KafkaFetchRequest { } collected.append(contentsOf: page.records) - guard let last = page.records.last else { break } - offset = last.offset + 1 + guard let last = page.records.last, let next = KafkaOffsetMath.offset(after: last.offset) else { break } + offset = next budget = defaultPartitionBudget if offset >= page.highWatermark { break } } @@ -75,13 +75,15 @@ enum KafkaFetchRequest { return KafkaFetchResult(records: collected, highWatermark: highWatermark, truncated: sawTruncation) } + // Returns the decompression budget as the reply left it, so the next page continues from it. private static func fetchOnce( topic: String, partition: Int32, startOffset: Int64, budget: Int32, + decompression: DecompressionBudget, cluster: KafkaCluster - ) async throws -> KafkaFetchResult { + ) async throws -> (KafkaFetchResult, DecompressionBudget) { try await cluster.withLeader(of: partition, topic: topic, api: "Fetch") { connection in let version = try await connection.negotiatedVersion(for: .fetch) let flexible = KafkaApiKey.fetch.isFlexible(version: version) @@ -133,6 +135,7 @@ enum KafkaFetchRequest { } var body = try await connection.send(request) + var spent = decompression if version >= 1 { _ = try body.int32() } // throttleTimeMs if version >= 7 { let errorCode = try body.int16() @@ -173,7 +176,8 @@ enum KafkaFetchRequest { blob: blob, partition: index, abortedTransactions: abortedTransactions, - skipAborted: true + skipAborted: true, + budget: &spent ) // A batch can start before the offset asked for, because the broker returns // whole batches. Dropping the head here is what keeps paging exact. @@ -188,8 +192,9 @@ enum KafkaFetchRequest { return partitions } // One topic and one partition were requested, so there is one answer. - return topics.flatMap { $0 }.first + let result = topics.flatMap { $0 }.first ?? KafkaFetchResult(records: [], highWatermark: startOffset, truncated: false) + return (result, spent) } } } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaFindCoordinatorRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaFindCoordinatorRequest.swift index 12ef4350c6..563b74bc66 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaFindCoordinatorRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaFindCoordinatorRequest.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore /// Asks any broker which broker coordinates a consumer group. /// diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaGroupsRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaGroupsRequest.swift index 21f804dd14..7a5dd680a4 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaGroupsRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaGroupsRequest.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore struct KafkaGroupSummary: Sendable { let groupId: String diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift b/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift index b3e27c33ff..1dfb15d2d6 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore import TableProPluginKit /// Turns Kafka messages into grid rows. diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaMetadataRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaMetadataRequest.swift index 90e02616b5..760084afd7 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaMetadataRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaMetadataRequest.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore enum KafkaMetadataRequest { /// Metadata, which names the brokers, the topics and each partition's leader. diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaOffsetsRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaOffsetsRequest.swift index edc8989e66..f3420ceff0 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaOffsetsRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaOffsetsRequest.swift @@ -1,11 +1,12 @@ import Foundation +import TableProKafkaCore struct KafkaPartitionOffsets: Sendable { let partition: Int32 let earliest: Int64 let latest: Int64 - var messageCount: Int64 { max(0, latest - earliest) } + var messageCount: Int64 { KafkaOffsetMath.messageCount(earliest: earliest, latest: latest) } } enum KafkaOffsetsRequest { diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift b/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift index c71a3f8b62..9f811a0625 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift @@ -1,5 +1,6 @@ import Foundation import os +import TableProKafkaCore import TableProPluginKit // Follows topics from each partition's end offset with plain Fetch requests and no consumer group, so it never @@ -72,7 +73,12 @@ final class KafkaLiveSubscription: PluginLiveSubscription, @unchecked Sendable { init(cluster: KafkaCluster, cursors: [KafkaFollowCursor]) { self.cluster = cluster - let (stream, continuation) = AsyncThrowingStream.makeStream() + // The window's live buffer holds 10,000, so dropping older messages loses nothing it could show. + let (stream, continuation) = AsyncThrowingStream.makeStream( + of: PluginLiveMessage.self, + throwing: Error.self, + bufferingPolicy: .bufferingNewest(10_000) + ) messages = stream reader = Task { do { @@ -94,7 +100,8 @@ final class KafkaLiveSubscription: PluginLiveSubscription, @unchecked Sendable { continuation.yield(KafkaFollowCursor.message(from: record, topic: cursor.topic)) } if let last = page.records.last { - offset = last.offset + 1 + guard let next = KafkaOffsetMath.offset(after: last.offset) else { return } + offset = next } else if ContinuousClock.now - started < Self.idleWait / 2 { // A trailing transaction marker answers at once and reads as no record, // where an idle partition makes the broker wait out the long poll. diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver.swift b/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver.swift index d532868e48..11c5fd60e4 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore import TableProPluginKit /// The Kafka driver. @@ -158,7 +159,7 @@ final class KafkaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { partitions: topic.partitions.map(\.index), cluster: cluster ) - let total = bounds.reduce(Int64(0)) { $0 + $1.messageCount } + let total = KafkaOffsetMath.total(bounds.map(\.messageCount)) return PluginTableMetadata(tableName: table, rowCount: total) } @@ -494,8 +495,6 @@ final class KafkaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) } - /// A group's lag, per partition. This is the number a Kafka debugging session is usually - /// after: how far behind the consumers are, and on which partition. private func runDescribeGroup(_ group: String) async throws -> PluginQueryResult { // Asked first, and only to tell a typo apart from a group with nothing committed. Kafka // answers OffsetFetch for a group it has never heard of with an empty topic list and no @@ -520,7 +519,7 @@ final class KafkaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // A group that never committed a partition reports -1, and calling that a lag // of `end + 1` would be a fabricated number. let lag: PluginCellValue = offset.committedOffset >= 0 - ? .text(String(max(0, end - offset.committedOffset))) + ? .text(String(KafkaOffsetMath.messageCount(earliest: offset.committedOffset, latest: end))) : .null rows.append([ .text(topic), diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaProduceRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaProduceRequest.swift index 1560b45204..afeee1f846 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaProduceRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaProduceRequest.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore struct KafkaProduceResult: Sendable { let partition: Int32 diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaQL.swift b/apple/Plugins/KafkaDriverPlugin/KafkaQL.swift index e688d8e8db..34b2f3b6f5 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaQL.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaQL.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore /// Where a browse starts reading. Kafka has no ORDER BY and no skip-N, so a scan is defined by /// an anchor plus a direction, which is the vocabulary every Kafka UI converges on. diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaRecordOrdering.swift b/apple/Plugins/KafkaDriverPlugin/KafkaRecordOrdering.swift index 7e1950ed1b..f63df99b16 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaRecordOrdering.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaRecordOrdering.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore /// How a multi-partition read is presented as one list. /// diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaRequest.swift index 3f08dddabc..d46c6af0eb 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaRequest.swift @@ -1,4 +1,5 @@ import Foundation +import TableProKafkaCore /// One Kafka request: an API key, the negotiated version, and a body encoder that is handed a /// writer already told whether this version is flexible. diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaSASL.swift b/apple/Plugins/KafkaDriverPlugin/KafkaSASL.swift index 96dda3c242..9017cab269 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaSASL.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaSASL.swift @@ -1,6 +1,7 @@ import CommonCrypto import CryptoKit import Foundation +import TableProKafkaCore /// SASL over Kafka's own framing: SaslHandshake names the mechanism, then each SASL token /// travels inside a SaslAuthenticate request rather than as a bare byte block on the socket. diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaScram.swift b/apple/Plugins/KafkaDriverPlugin/KafkaScram.swift index 823fb331fe..1b07764b33 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaScram.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaScram.swift @@ -1,6 +1,7 @@ import CommonCrypto import CryptoKit import Foundation +import TableProKafkaCore /// SCRAM's cryptography and the rules that decide whether a broker's half of the exchange /// is acceptable. Split from the exchange itself because none of it needs a connection, so diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift index fb5fa15397..c1b97999fd 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -208,7 +208,9 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } lock.withLock { - serverClockOffset = serverTime - Int64(Date().timeIntervalSince1970) + serverClockOffset = MemcachedServerClock.offset( + serverTime: serverTime, localTime: Int64(Date().timeIntervalSince1970) + ) if shownExpiries.count + page.count > Self.shownExpiryLimit { shownExpiries.removeAll() } for entry in page { shownExpiries[entry.key] = MemcachedKnownExpiry(expiresAt: entry.expiresAt) } } @@ -414,7 +416,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let expiries = lock.withLock { shownExpiries } let generator = MemcachedStatementGenerator( columns: columns, - now: { Int64(Date().timeIntervalSince1970) + offset }, + now: { MemcachedServerClock.serverTime(localTime: Int64(Date().timeIntervalSince1970), offset: offset) }, knownExpiry: { expiries[$0] } ) return try generator.generateRowWrites( diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+DatabaseMetadata.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+DatabaseMetadata.swift index 252b48ee32..9e2c25512d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+DatabaseMetadata.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+DatabaseMetadata.swift @@ -106,9 +106,7 @@ internal extension MySQLPluginDriver { """) } - /// Before 5.0.2 there is no catalog to total, so each database answers its own `SHOW TABLE STATUS`. - /// One the account may list but not open is refused, and keeps its name alone rather than failing - /// the whole list. + // Before 5.0.2 there is no catalog to total, so each database answers its own `SHOW TABLE STATUS`. private func tableStatusMetadata(of database: String, isSystem: Bool) async throws -> PluginDatabaseMetadata { let rows: [[PluginCellValue]] do { @@ -118,16 +116,24 @@ internal extension MySQLPluginDriver { Self.logger.info("SHOW TABLE STATUS refused for one database code=\(error.code, privacy: .public)") return PluginDatabaseMetadata(name: database, isSystemDatabase: isSystem) } - let sizeBytes = rows.reduce(Int64(0)) { total, row in - let data = row[safe: 6]?.asText.flatMap { Int64($0) } ?? 0 - let index = row[safe: 8]?.asText.flatMap { Int64($0) } ?? 0 - return total + data + index - } return PluginDatabaseMetadata( name: database, tableCount: rows.count, - sizeBytes: sizeBytes, + sizeBytes: Self.tableStatusSizeBytes(rows), isSystemDatabase: isSystem ) } + + // The lengths are the server's own numbers, so a total past Int64 pins to its end. + static func tableStatusSizeBytes(_ rows: [[PluginCellValue]]) -> Int64 { + var total: Int64 = 0 + for row in rows { + for column in [6, 8] { + let length = row[safe: column]?.asText.flatMap { Int64($0) } ?? 0 + let (sum, overflow) = total.addingReportingOverflow(length) + total = overflow ? (length < 0 ? .min : .max) : sum + } + } + return total + } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLTableListing.swift b/apple/Plugins/MySQLDriverPlugin/MySQLTableListing.swift index 3d607345f2..3a0e2e8b20 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLTableListing.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLTableListing.swift @@ -117,8 +117,7 @@ enum MySQLTableStatusRow { ?? rows.first { $0[safe: 0]?.asText?.caseInsensitiveCompare(table) == .orderedSame } } - /// The positions `SHOW TABLE STATUS` documents, read in one place so the per-table and - /// whole-schema reads cannot index the same row differently. + // One place reads these positions, so the per-table and whole-schema reads cannot index a row differently. static func metadata( from row: [PluginCellValue], tableName: String, @@ -131,7 +130,11 @@ enum MySQLTableStatusRow { let totalSize: Int64? = { guard let data = dataSize else { return nil } guard reportsIndexLength else { return data } - return indexSize.map { data + $0 } + // The lengths are the server's own numbers, so a total past Int64 pins to its end. + return indexSize.map { index in + let (sum, overflow) = data.addingReportingOverflow(index) + return overflow ? .max : sum + } }() return PluginTableMetadata( diff --git a/apple/Plugins/MySQLDriverTests/MySQLDatabaseMetadataTests.swift b/apple/Plugins/MySQLDriverTests/MySQLDatabaseMetadataTests.swift new file mode 100644 index 0000000000..0ce0e79f80 --- /dev/null +++ b/apple/Plugins/MySQLDriverTests/MySQLDatabaseMetadataTests.swift @@ -0,0 +1,49 @@ +// +// MySQLDatabaseMetadataTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct MySQLDatabaseMetadataTests { + // SHOW TABLE STATUS puts Data_length at column 6 and Index_length at column 8. + private func statusRow(data: String?, index: String?) -> [PluginCellValue] { + var cells: [String?] = Array(repeating: nil, count: 18) + cells[0] = "orders" + cells[6] = data + cells[8] = index + return cells.map(PluginCellValue.fromOptional) + } + + @Test("A database's size adds every table's data and index lengths") + func sumsLengths() { + let rows = [statusRow(data: "16384", index: "32768"), statusRow(data: "100", index: nil)] + #expect(MySQLPluginDriver.tableStatusSizeBytes(rows) == 49_252) + #expect(MySQLPluginDriver.tableStatusSizeBytes([]) == 0) + } + + @Test("Lengths whose total passes Int64 stop at its end instead of crashing") + func hugeLengthsSaturate() { + let rows = [statusRow(data: "9223372036854775807", index: "1"), statusRow(data: "5", index: "5")] + #expect(MySQLPluginDriver.tableStatusSizeBytes(rows) == Int64.max) + } + + @Test("A table's total size is its data length plus its index length") + func tableTotalAddsLengths() { + let metadata = MySQLTableStatusRow.metadata(from: statusRow(data: "16384", index: "32768"), tableName: "orders") + #expect(metadata.dataSize == 16_384) + #expect(metadata.indexSize == 32_768) + #expect(metadata.totalSize == 49_152) + } + + @Test("A table whose two lengths pass Int64 together reads its total as the end of the range instead of crashing") + func tableTotalSaturates() { + let row = statusRow(data: "9223372036854775807", index: "9223372036854775807") + let metadata = MySQLTableStatusRow.metadata(from: row, tableName: "orders") + #expect(metadata.dataSize == Int64.max) + #expect(metadata.indexSize == Int64.max) + #expect(metadata.totalSize == Int64.max) + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+LiveMessages.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+LiveMessages.swift index 36d7f6ed9b..e349ae04fc 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+LiveMessages.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+LiveMessages.swift @@ -55,7 +55,12 @@ final class PostgreSQLLiveSubscription: PluginLiveSubscription, @unchecked Senda init(listener: PostgreSQLPluginDriver) { self.listener = listener - let (stream, continuation) = AsyncThrowingStream.makeStream() + // The newest 10,000, the most the live window keeps: a channel faster than the window drains cannot queue without bound. + let (stream, continuation) = AsyncThrowingStream.makeStream( + of: PluginLiveMessage.self, + throwing: Error.self, + bufferingPolicy: .bufferingNewest(10_000) + ) messages = stream reader = Task { do { diff --git a/apple/Plugins/RedisDriverPlugin/RedisClusterAggregator.swift b/apple/Plugins/RedisDriverPlugin/RedisClusterAggregator.swift index d2ef58a445..038ba5b147 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisClusterAggregator.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisClusterAggregator.swift @@ -11,10 +11,7 @@ import Foundation enum RedisClusterAggregator { - /// A shard that refused its part, or queued it into an open `MULTI` block, wins over every - /// policy but one_succeeded, whose whole meaning is that some shards may fail. Folding it in - /// instead counted it as zero: a split `DEL` one shard refused reported the other shard's - /// deletions as the total, and `DBSIZE` reported part of the keyspace as all of it. + // A refused or queued shard reply wins over every policy but one_succeeded; folded in, it counted as zero. static func combine(_ replies: [RedisReply], policy: RedisResponsePolicy?) -> RedisReply { guard let first = replies.first else { return .array([]) } guard replies.count > 1 else { return first } @@ -22,7 +19,7 @@ enum RedisClusterAggregator { switch policy { case .aggSum: - return aggregated(replies) { $0.reduce(0, +) } + return aggregated(replies) { $0.reduce(0, saturatingSum) } case .aggMin: return aggregated(replies) { $0.min() ?? 0 } case .aggMax: @@ -40,17 +37,23 @@ enum RedisClusterAggregator { } } - /// Each primary's `INFO keyspace` added up per database. Nil when any primary declined, the - /// same rule `DBSIZE` follows, because a count missing one primary is short, not zero. + // Nil when any primary declined, as for `DBSIZE`: a count missing one primary is short, not zero. static func keyspace(_ perShard: [[Int: Int]?]) -> [Int: Int]? { var merged: [Int: Int] = [:] for shard in perShard { guard let shard else { return nil } - merged.merge(shard, uniquingKeysWith: +) + merged.merge(shard, uniquingKeysWith: saturatingSum) } return merged } + // Each shard's count is the server's own number, so a total past the integer range pins to its end. + private static func saturatingSum(_ lhs: Value, _ rhs: Value) -> Value { + let (sum, overflow) = lhs.addingReportingOverflow(rhs) + guard overflow else { return sum } + return rhs < 0 ? .min : .max + } + /// The first shard error in the order the shards were asked, else the first `+QUEUED`. static func firstNonAnswer(in replies: [RedisReply]) -> RedisReply? { replies.first(where: \.isError) ?? replies.first(where: \.isQueued) diff --git a/apple/Plugins/RedisDriverPlugin/RedisClusterTopology.swift b/apple/Plugins/RedisDriverPlugin/RedisClusterTopology.swift index fcdbc881e5..9914a980f5 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisClusterTopology.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisClusterTopology.swift @@ -49,10 +49,10 @@ struct RedisClusterTopology: Equatable, Sendable { allNodes.first { $0.address == address } } - /// Applies a MOVED without a full reload. Redis clients patch the single slot and reload - /// periodically rather than on every redirect. + // Redis clients patch the one slot on MOVED and reload the full map only periodically. func movingSlot(_ slot: Int, to address: RedisNodeAddress) -> RedisClusterTopology { - guard let targetShardIndex = shards.firstIndex(where: { $0.master.address == address }) else { + guard (0 ..< RedisKeySlot.slotCount).contains(slot), + let targetShardIndex = shards.firstIndex(where: { $0.master.address == address }) else { return self } var updated: [RedisClusterShard] = [] @@ -84,7 +84,6 @@ struct RedisClusterTopology: Equatable, Sendable { } enum RedisClusterTopologyParser { - /// CLUSTER SHARDS: [ [ "slots", [start, end, ...], "nodes", [ {id, port, ip, endpoint, role, health}, ... ] ], ... ] static func parseShards(_ reply: RedisReply, fallbackHost: String) -> RedisClusterTopology? { guard case .array(let shardReplies) = reply, !shardReplies.isEmpty else { return nil } var shards: [RedisClusterShard] = [] @@ -94,8 +93,8 @@ enum RedisClusterTopologyParser { let bounds = slotNumbers.compactMap(\.intValue) guard bounds.count % 2 == 0 else { return nil } var ranges: [ClosedRange] = [] - for index in stride(from: 0, to: bounds.count, by: 2) where bounds[index] <= bounds[index + 1] { - ranges.append(bounds[index] ... bounds[index + 1]) + for index in stride(from: 0, to: bounds.count, by: 2) { + if let range = slotRange(bounds[index], bounds[index + 1]) { ranges.append(range) } } guard case .array(let nodeReplies)? = map["nodes"] else { return nil } @@ -111,13 +110,13 @@ enum RedisClusterTopologyParser { return shards.isEmpty ? nil : RedisClusterTopology(shards: shards) } - /// CLUSTER SLOTS: [ [ start, end, [ip, port, id, meta?], [replica...] ], ... ] static func parseSlots(_ reply: RedisReply, fallbackHost: String) -> RedisClusterTopology? { guard case .array(let rangeReplies) = reply, !rangeReplies.isEmpty else { return nil } var shards: [RedisClusterShard] = [] for rangeReply in rangeReplies { guard case .array(let parts) = rangeReply, parts.count >= 3, - let start = parts[0].intValue, let end = parts[1].intValue, start <= end else { continue } + let start = parts[0].intValue, let end = parts[1].intValue, + let range = slotRange(start, end) else { continue } var master: RedisClusterNode? var replicas: [RedisClusterNode] = [] for (offset, nodeReply) in parts.dropFirst(2).enumerated() { @@ -127,11 +126,16 @@ enum RedisClusterTopologyParser { if node.isReplica { replicas.append(node) } else { master = node } } guard let master else { continue } - shards.append(RedisClusterShard(slots: [start ... end], master: master, replicas: replicas)) + shards.append(RedisClusterShard(slots: [range], master: master, replicas: replicas)) } return shards.isEmpty ? nil : RedisClusterTopology(shards: shards) } + private static func slotRange(_ start: Int, _ end: Int) -> ClosedRange? { + guard start >= 0, start <= end, end < RedisKeySlot.slotCount else { return nil } + return start ... end + } + private static func parseShardNode(_ reply: RedisReply, fallbackHost: String) -> RedisClusterNode? { guard let map = flatMap(reply) else { return nil } let endpoint = map["endpoint"]?.stringValue ?? "" diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift index c63ca1f85b..92f619e150 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift @@ -479,8 +479,8 @@ extension RedisPluginConnection { throw transportFailure(ctx, delivered: true) } guard let reply = rawReply else { return replies } - replies.append(parseReply(reply.assumingMemoryBound(to: redisReply.self))) - freeReplyObject(reply) + replies.append(HiredisReply.value(of: reply.assumingMemoryBound(to: redisReply.self))) + HiredisReply.release(reply) } } #endif @@ -524,6 +524,7 @@ private extension RedisPluginConnection { logger.error("Failed to create Redis context") throw RedisPluginError.connectionFailed } + HiredisReply.installIterativeFree(on: ctx) if ctx.pointee.err != 0 { let errMsg = Self.contextErrorMessage(ctx) @@ -796,9 +797,7 @@ private extension RedisPluginConnection { !failure.wasDelivered || routing.isReadOnly(args) } - /// The append/flush/read split that `redisCommandArgv` performs internally, spelled out so a - /// failure can say whether the command reached the server. hiredis does exactly these three - /// steps, so the behaviour is unchanged. + // redisCommandArgv's append, flush and read, split so a failure can say whether the command reached the server. func executeCommandSync(_ args: [Data]) throws -> RedisReply { stateLock.lock() guard let ctx = context else { @@ -830,9 +829,8 @@ private extension RedisPluginConnection { throw transportFailure(ctx, delivered: true) } - let replyPtr = reply.assumingMemoryBound(to: redisReply.self) - let parsed = parseReply(replyPtr) - freeReplyObject(reply) + let parsed = HiredisReply.value(of: reply.assumingMemoryBound(to: redisReply.self)) + HiredisReply.release(reply) return parsed } @@ -864,7 +862,7 @@ private extension RedisPluginConnection { for _ in 0 ..< appendedCount { var discard: UnsafeMutableRawPointer? if redisGetReply(ctx, &discard) != REDIS_OK { break } - if let d = discard { freeReplyObject(d) } + if let d = discard { HiredisReply.release(d) } } let failure = transportFailure(ctx, delivered: false) markDisconnected() @@ -884,15 +882,14 @@ private extension RedisPluginConnection { for _ in (i + 1) ..< commands.count { var discard: UnsafeMutableRawPointer? if redisGetReply(ctx, &discard) == REDIS_OK, let d = discard { - freeReplyObject(d) + HiredisReply.release(d) } } markDisconnected() throw failure } - let replyPtr = reply.assumingMemoryBound(to: redisReply.self) - let parsed = parseReply(replyPtr) - freeReplyObject(reply) + let parsed = HiredisReply.value(of: reply.assumingMemoryBound(to: redisReply.self)) + HiredisReply.release(reply) replies.append(parsed) } return replies @@ -915,37 +912,50 @@ private extension RedisPluginConnection { #endif } - func parseReply(_ reply: UnsafeMutablePointer) -> RedisReply { - let type = reply.pointee.type - - switch type { - case REDIS_REPLY_STRING: - if let str = reply.pointee.str { - let len = reply.pointee.len - let data = Data(bytes: str, count: len) - if let string = String(data: data, encoding: .utf8) { - return .string(string) - } - return .data(data) - } - return .null + func fetchServerVersionSync() throws -> String? { + stateLock.lock() + guard context != nil else { + stateLock.unlock() + return nil + } + stateLock.unlock() + let reply = try executeCommandSync(["INFO", "server"]) + if case .string(let info) = reply { + return RedisServerInfo.version(from: info) + } + return nil + } +} +#endif - case REDIS_REPLY_INTEGER: +#if canImport(CRedis) +// A server decides how deep a reply nests, a few bytes a level. +enum HiredisReply { + // Deeper than any command or module replies. Past it a reply is not built, so nothing that walks one later + // runs out of stack. + static let maximumDepth = 128 + + static func value(of reply: UnsafeMutablePointer, depth: Int = 0) -> RedisReply { + switch reply.pointee.type { + case REDIS_REPLY_STRING, REDIS_REPLY_BIGNUM, REDIS_REPLY_VERB: + guard let data = bytes(of: reply) else { return .null } + return String(data: data, encoding: .utf8).map(RedisReply.string) ?? .data(data) + + case REDIS_REPLY_INTEGER, REDIS_REPLY_BOOL: return .integer(reply.pointee.integer) - case REDIS_REPLY_ARRAY: + case REDIS_REPLY_ARRAY, REDIS_REPLY_MAP, REDIS_REPLY_SET, REDIS_REPLY_PUSH: + guard depth < maximumDepth else { + return .error(String(localized: "Reply nested too deeply to show")) + } let count = reply.pointee.elements guard count > 0, let elements = reply.pointee.element else { return .array([]) } var items: [RedisReply] = [] items.reserveCapacity(count) - for i in 0 ..< count { - if let element = elements[i] { - items.append(parseReply(element)) - } else { - items.append(.null) - } + for index in 0 ..< count { + items.append(elements[index].map { value(of: $0, depth: depth + 1) } ?? .null) } return .array(items) @@ -953,95 +963,63 @@ private extension RedisPluginConnection { return .null case REDIS_REPLY_STATUS: - if let str = reply.pointee.str { - let len = reply.pointee.len - let data = Data(bytes: str, count: len) - return .status(String(data: data, encoding: .utf8) ?? "") - } - return .status("") + return .status(bytes(of: reply).flatMap { String(data: $0, encoding: .utf8) } ?? "") case REDIS_REPLY_ERROR: - if let str = reply.pointee.str { - let len = reply.pointee.len - let data = Data(bytes: str, count: len) - return .error(String(data: data, encoding: .utf8) ?? "Unknown error") - } - return .error("Unknown error") + return .error(bytes(of: reply).flatMap { String(data: $0, encoding: .utf8) } ?? "Unknown error") case REDIS_REPLY_DOUBLE: - if let str = reply.pointee.str { - let len = reply.pointee.len - let data = Data(bytes: str, count: len) - if let string = String(data: data, encoding: .utf8) { - return .string(string) - } - } - return .string(String(reply.pointee.dval)) - - case REDIS_REPLY_BOOL: - return .integer(reply.pointee.integer) - - case REDIS_REPLY_MAP: - let count = reply.pointee.elements - guard count > 0, let elements = reply.pointee.element else { - return .array([]) - } - var items: [RedisReply] = [] - items.reserveCapacity(count) - for i in 0 ..< count { - if let element = elements[i] { - items.append(parseReply(element)) - } else { - items.append(.null) - } - } - return .array(items) + return .string(bytes(of: reply).flatMap { String(data: $0, encoding: .utf8) } ?? String(reply.pointee.dval)) - case REDIS_REPLY_SET, REDIS_REPLY_PUSH: - let count = reply.pointee.elements - guard count > 0, let elements = reply.pointee.element else { - return .array([]) - } - var items: [RedisReply] = [] - items.reserveCapacity(count) - for i in 0 ..< count { - if let element = elements[i] { - items.append(parseReply(element)) - } else { - items.append(.null) - } - } - return .array(items) + default: + logger.warning("Unknown Redis reply type: \(reply.pointee.type)") + return .null + } + } - case REDIS_REPLY_BIGNUM, REDIS_REPLY_VERB: - if let str = reply.pointee.str { - let len = reply.pointee.len - let data = Data(bytes: str, count: len) - if let string = String(data: data, encoding: .utf8) { - return .string(string) + // freeReplyObject recurses once per level, which a deeply nested reply overflows. Detaching each node's + // children first leaves every call one node to free. + static func release(_ reply: UnsafeMutableRawPointer) { + var pending = [reply.assumingMemoryBound(to: redisReply.self)] + while let node = pending.popLast() { + if let elements = node.pointee.element { + for index in 0 ..< node.pointee.elements { + guard let child = elements[index] else { continue } + pending.append(child) + elements[index] = nil } - return .data(data) } - return .null - - default: - logger.warning("Unknown Redis reply type: \(type)") - return .null + freeReplyObject(node) } } - func fetchServerVersionSync() throws -> String? { - stateLock.lock() - guard context != nil else { - stateLock.unlock() - return nil + // hiredis frees a reply it has not handed over (cut short by a protocol error or a freed context) and every + // RESP3 push it drops with freeReplyObject. Both go through release instead. + static func installIterativeFree(on context: UnsafeMutablePointer) { + if let functions, let reader = context.pointee.reader { + reader.pointee.fn = functions } - stateLock.unlock() - let reply = try executeCommandSync(["INFO", "server"]) - if case .string(let info) = reply { - return RedisServerInfo.version(from: info) + redisSetPushCallback(context) { _, reply in + if let reply { HiredisReply.release(reply) } } - return nil + } + + // A reader keeps this pointer for its whole life, so the table is built once and never written again. + nonisolated(unsafe) private static let functions: UnsafeMutablePointer? = { + guard let reader = redisReaderCreate() else { return nil } + defer { redisReaderFree(reader) } + guard let defaults = reader.pointee.fn else { return nil } + let table = UnsafeMutablePointer.allocate(capacity: 1) + table.initialize(to: defaults.pointee) + table.pointee.freeObject = { reply in + if let reply { HiredisReply.release(reply) } + } + return table + }() + + private static func bytes(of reply: UnsafeMutablePointer) -> Data? { + guard let str = reply.pointee.str else { return nil } + return Data(bytes: str, count: reply.pointee.len) } } #endif diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+LiveMessages.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+LiveMessages.swift index 43a610d083..b0abe22935 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+LiveMessages.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+LiveMessages.swift @@ -75,7 +75,12 @@ final class RedisLiveSubscription: PluginLiveSubscription, @unchecked Sendable { init(subscriber: RedisPluginConnection, sessions: any PluginLiveSessionSource) { self.sessions = sessions - let (stream, continuation) = AsyncThrowingStream.makeStream() + // The newest 10,000, the most the live window keeps: a channel faster than the window drains cannot queue without bound. + let (stream, continuation) = AsyncThrowingStream.makeStream( + of: PluginLiveMessage.self, + throwing: Error.self, + bufferingPolicy: .bufferingNewest(10_000) + ) messages = stream reader = Task { do { diff --git a/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift b/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift new file mode 100644 index 0000000000..a1e5a4aa40 --- /dev/null +++ b/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift @@ -0,0 +1,144 @@ +// +// HiredisReplyTests.swift +// RedisDriverTests +// + +#if canImport(CRedis) +import CRedis +import Foundation +import Testing + +struct HiredisReplyTests { + private static func convert(_ wire: String) throws -> RedisReply { + let reader = try #require(redisReaderCreate()) + defer { redisReaderFree(reader) } + #expect(redisReaderFeed(reader, wire, wire.utf8.count) == REDIS_OK) + var raw: UnsafeMutableRawPointer? + #expect(redisReaderGetReply(reader, &raw) == REDIS_OK) + let reply = try #require(raw) + defer { HiredisReply.release(reply) } + return HiredisReply.value(of: reply.assumingMemoryBound(to: redisReply.self)) + } + + private static func firstElementChain(_ reply: RedisReply) -> (depth: Int, innermost: RedisReply) { + var depth = 0 + var current = reply + while case .array(let items) = current, let first = items.first { + depth += 1 + current = first + } + return (depth, current) + } + + @Test("A reply nested deeper than the cap ends in a placeholder and is freed without recursing per level") + func deeplyNestedReply() throws { + let reply = try Self.convert(String(repeating: "*1\r\n", count: 20_000) + ":1\r\n") + let chain = Self.firstElementChain(reply) + #expect(chain.depth == HiredisReply.maximumDepth) + guard case .error = chain.innermost else { + Issue.record("expected the too-deep placeholder, got \(chain.innermost.displayText)") + return + } + } + + @Test("A reply nested just inside the cap reads in full") + func replyInsideTheCap() throws { + let depth = HiredisReply.maximumDepth - 1 + let reply = try Self.convert(String(repeating: "*1\r\n", count: depth) + ":7\r\n") + let chain = Self.firstElementChain(reply) + #expect(chain.depth == depth) + #expect(chain.innermost.displayText == "7") + } + + @Test("Flat and RESP3 replies read as before", arguments: [ + ("*3\r\n$3\r\nfoo\r\n:42\r\n_\r\n", "[foo, 42, (nil)]"), + ("%1\r\n+key\r\n,1.5\r\n", "[key, 1.5]"), + ("~2\r\n#t\r\n#f\r\n", "[1, 0]"), + ("-ERR boom\r\n", "(error) ERR boom"), + ("*0\r\n", "[]"), + ]) + func flatReplies(wire: String, expected: String) throws { + #expect(try Self.convert(wire).displayText == expected) + } + + private struct Outcome: Sendable { + let status: Int32 + let error: Int32 + let reply: String? + } + + private static let deepNesting = String(repeating: "*1\r\n", count: 20_000) + + // 512 KiB is a Dispatch worker's stack, which a free that recurses once per level overflows at this depth. + private static func onSmallStack( + fed wire: String, + _ read: @escaping @Sendable (UnsafeMutablePointer) -> Outcome + ) async -> Outcome? { + await withCheckedContinuation { continuation in + let thread = Thread { + continuation.resume(returning: Self.withConnection(fed: wire, read)) + } + thread.stackSize = 512 * 1_024 + thread.start() + } + } + + private static func withConnection( + fed wire: String, + _ read: (UnsafeMutablePointer) -> Outcome + ) -> Outcome? { + var sockets: [Int32] = [-1, -1] + guard socketpair(AF_UNIX, SOCK_STREAM, 0, &sockets) == 0 else { return nil } + // A closed peer turns any socket read into an error rather than a wait. + close(sockets[1]) + guard let context = redisConnectFd(sockets[0]) else { + close(sockets[0]) + return nil + } + defer { redisFree(context) } + HiredisReply.installIterativeFree(on: context) + guard redisReaderFeed(context.pointee.reader, wire, wire.utf8.count) == REDIS_OK else { return nil } + return read(context) + } + + @Test("A protocol error under a deeply nested partial reply fails without recursing per level") + func protocolErrorUnderDeepReply() async throws { + let outcome = await Self.onSmallStack(fed: Self.deepNesting + "@\r\n") { context in + var raw: UnsafeMutableRawPointer? + let status = redisReaderGetReply(context.pointee.reader, &raw) + return Outcome(status: status, error: context.pointee.reader.pointee.err, reply: nil) + } + let read = try #require(outcome) + #expect(read.status == REDIS_ERR) + #expect(read.error == REDIS_ERR_PROTOCOL) + } + + @Test("Freeing a connection that holds a deeply nested partial reply does not recurse per level") + func freeWithDeepPartialReply() async throws { + let outcome = await Self.onSmallStack(fed: Self.deepNesting) { context in + var raw: UnsafeMutableRawPointer? + let status = redisReaderGetReply(context.pointee.reader, &raw) + return Outcome(status: status, error: context.pointee.reader.pointee.err, reply: raw.map { _ in "reply" }) + } + let read = try #require(outcome) + #expect(read.status == REDIS_OK) + #expect(read.reply == nil) + } + + @Test("A deeply nested RESP3 push hiredis drops is freed without recursing per level") + func droppedDeepPush() async throws { + let outcome = await Self.onSmallStack(fed: ">1\r\n" + Self.deepNesting + ":1\r\n+OK\r\n") { context in + var raw: UnsafeMutableRawPointer? + let status = redisGetReply(context, &raw) + let text = raw.map { reply in + defer { HiredisReply.release(reply) } + return HiredisReply.value(of: reply.assumingMemoryBound(to: redisReply.self)).displayText + } + return Outcome(status: status, error: context.pointee.err, reply: text) + } + let read = try #require(outcome) + #expect(read.status == REDIS_OK) + #expect(read.reply == "OK") + } +} +#endif diff --git a/apple/Plugins/RedisDriverTests/RedisClusterAggregatorTests.swift b/apple/Plugins/RedisDriverTests/RedisClusterAggregatorTests.swift index 88321f24fc..b3b92736da 100644 --- a/apple/Plugins/RedisDriverTests/RedisClusterAggregatorTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisClusterAggregatorTests.swift @@ -28,6 +28,12 @@ struct RedisClusterAggregatorKeyspaceTests { func declinedPrimaryIsUnknown() { #expect(RedisClusterAggregator.keyspace([[0: 2], nil]) == nil) } + + @Test("Key counts whose total passes the integer range stop at its end instead of crashing") + func hugeCountsSaturate() { + #expect(RedisClusterAggregator.keyspace([[0: Int.max, 1: 3], [0: 1, 1: 4]]) == [0: Int.max, 1: 7]) + #expect(RedisClusterAggregator.keyspace([[0: Int.min], [0: -1]]) == [0: Int.min]) + } } struct RedisClusterAggregatorNumericTests { @@ -43,6 +49,21 @@ struct RedisClusterAggregatorNumericTests { #expect(intValue(combined) == 3) } + @Test("agg_sum of shard replies whose total passes the integer range stops at its end instead of crashing") + func hugeSumsSaturate() { + let overflowing = RedisClusterAggregator.combine([.integer(.max), .string("1"), .integer(5)], policy: .aggSum) + #expect(intValue(overflowing) == Int64.max) + + let underflowing = RedisClusterAggregator.combine([.integer(.min), .integer(-1)], policy: .aggSum) + #expect(intValue(underflowing) == Int64.min) + + let perPosition = RedisClusterAggregator.combine( + [.array([.integer(.max), .integer(1)]), .array([.integer(.max), .integer(2)])], + policy: .aggSum + ) + #expect(perPosition.arrayValue?.map(intValue) == [Int64.max, 3]) + } + @Test("agg_min takes the smallest") func takesMin() { #expect(intValue(RedisClusterAggregator.combine([.integer(3), .integer(1)], policy: .aggMin)) == 1) diff --git a/apple/Plugins/RedisDriverTests/RedisClusterTopologyTests.swift b/apple/Plugins/RedisDriverTests/RedisClusterTopologyTests.swift index e834548f14..f203b41eb2 100644 --- a/apple/Plugins/RedisDriverTests/RedisClusterTopologyTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisClusterTopologyTests.swift @@ -189,4 +189,62 @@ struct RedisClusterTopologyMigrationTests { let moved = topology.movingSlot(100, to: RedisNodeAddress(host: "10.9.9.9", port: 1)) #expect(moved == topology) } + + @Test("A MOVED naming a slot outside 0...16383 changes nothing", arguments: [Int.max, Int.min, -1, 16_384]) + func ignoresSlotOutsideTheRange(slot: Int) throws { + let topology = try #require(RedisClusterTopologyParser.parseShards(shardsReply(), fallbackHost: "seed")) + let target = RedisNodeAddress(host: "127.0.0.1", port: 7_102) + let other = RedisNodeAddress(host: "127.0.0.1", port: 7_101) + let moved = topology.movingSlot(slot, to: target).movingSlot(slot, to: other) + #expect(moved == topology) + #expect(moved.coversAllSlots) + } +} + +struct RedisClusterSlotBoundsTests { + private static func slotsNode(port: Int64) -> RedisReply { + .array([.string("127.0.0.1"), .integer(port), .string("n\(port)")]) + } + + private static func shard(slots: [Int64], port: Int64) -> RedisReply { + .array([ + .string("slots"), .array(slots.map(RedisReply.integer)), + .string("nodes"), .array([ + .array([ + .string("id"), .string("n\(port)"), .string("port"), .integer(port), + .string("endpoint"), .string("127.0.0.1"), .string("role"), .string("master"), + ]), + ]), + ]) + } + + @Test("CLUSTER SLOTS drops a range outside 0...16383 instead of crashing on its size") + func slotsDropOutOfRange() throws { + let reply = RedisReply.array([ + .array([.integer(.min), .integer(.max), Self.slotsNode(port: 7_101)]), + .array([.integer(16_000), .integer(20_000), Self.slotsNode(port: 7_102)]), + .array([.integer(-5), .integer(10), Self.slotsNode(port: 7_103)]), + .array([.integer(0), .integer(16_383), Self.slotsNode(port: 7_104)]), + ]) + let topology = try #require(RedisClusterTopologyParser.parseSlots(reply, fallbackHost: "seed")) + #expect(topology.masters.map(\.address.port) == [7_104]) + #expect(topology.coversAllSlots) + } + + @Test("CLUSTER SLOTS with only a range outside 0...16383 is no topology") + func slotsWithNoValidRange() { + let reply = RedisReply.array([.array([.integer(.min), .integer(.max), Self.slotsNode(port: 7_101)])]) + #expect(RedisClusterTopologyParser.parseSlots(reply, fallbackHost: "seed") == nil) + } + + @Test("CLUSTER SHARDS keeps a shard's valid ranges and drops the rest") + func shardsDropOutOfRange() throws { + let reply = RedisReply.array([ + Self.shard(slots: [0, 16_383, 16_384, 20_000, .min, .max, 9, 3], port: 7_101), + Self.shard(slots: [.min, .max], port: 7_102), + ]) + let topology = try #require(RedisClusterTopologyParser.parseShards(reply, fallbackHost: "seed")) + #expect(topology.shards.map(\.slots) == [[0 ... 16_383]]) + #expect(topology.coversAllSlots) + } } diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift index 765b66b67c..74d0746f0c 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift @@ -1,7 +1,7 @@ import AppKit -import Compression import Foundation import os +import TableProGzip import TableProNumberFormatting import TableProPluginKit @@ -966,53 +966,13 @@ final class SnowflakeConnection: @unchecked Sendable { // MARK: - Gzip - private static func gunzipIfNeeded(_ data: Data) -> Data { - guard data.count > 18, data[data.startIndex] == 0x1F, data[data.startIndex + 1] == 0x8B else { - return data - } - let bytes = [UInt8](data) - let flags = bytes[3] - var offset = 10 - if flags & 0x04 != 0, offset + 2 <= bytes.count { - let extraLen = Int(bytes[offset]) | (Int(bytes[offset + 1]) << 8) - offset += 2 + extraLen - } - if flags & 0x08 != 0 { - while offset < bytes.count, bytes[offset] != 0 { offset += 1 } - offset += 1 - } - if flags & 0x10 != 0 { - while offset < bytes.count, bytes[offset] != 0 { offset += 1 } - offset += 1 - } - if flags & 0x02 != 0 { offset += 2 } - guard offset < bytes.count - 8 else { return data } - - let deflateBytes = Array(bytes[offset..<(bytes.count - 8)]) - return inflate(deflateBytes) ?? data - } - - private static func inflate(_ deflate: [UInt8]) -> Data? { - guard !deflate.isEmpty else { return nil } - var capacity = max(deflate.count * 8, 65_536) - for _ in 0..<6 { - var output = Data(count: capacity) - let written = output.withUnsafeMutableBytes { (outPtr: UnsafeMutableRawBufferPointer) -> Int in - deflate.withUnsafeBufferPointer { (inPtr: UnsafeBufferPointer) -> Int in - guard let outBase = outPtr.bindMemory(to: UInt8.self).baseAddress, - let inBase = inPtr.baseAddress else { return 0 } - return compression_decode_buffer( - outBase, capacity, inBase, deflate.count, nil, COMPRESSION_ZLIB - ) - } - } - if written > 0, written < capacity { - output.removeSubrange(written.. Data { + guard data.starts(with: [0x1F, 0x8B]) else { return data } + do { + return try GzipInflater.decompress(data) + } catch { + throw SnowflakeError.invalidResponse(error.localizedDescription) } - return nil } } diff --git a/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedRowDecoder.swift b/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedRowDecoder.swift index f4ff899bf2..f7bd68413c 100644 --- a/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedRowDecoder.swift +++ b/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedRowDecoder.swift @@ -22,6 +22,8 @@ public struct ClickHouseTabSeparatedRowDecoder: Sendable { public private(set) var header: Header? private var pendingBytes: [UInt8] = [] + // The pending bytes already searched for a separator, so a line split over many chunks is scanned once. + private var searchedCount = 0 private var columnNames: [String]? public init() {} @@ -29,31 +31,43 @@ public struct ClickHouseTabSeparatedRowDecoder: Sendable { public mutating func consume(_ data: Data) -> [[PluginCellValue]] { guard !data.isEmpty else { return [] } - var buffer = pendingBytes - pendingBytes = [] - buffer.append(contentsOf: data) + pendingBytes.append(contentsOf: data) var rows: [[PluginCellValue]] = [] var lineStart = 0 - for index in buffer.indices where buffer[index] == ClickHouseTabSeparatedBytes.lineSeparator { - if let row = ingest(buffer[lineStart.. 0 { + pendingBytes.removeFirst(lineStart) } - buffer.removeFirst(lineStart) - pendingBytes = buffer + searchedCount = pendingBytes.count return rows } - /// A body that ended without its closing newline still holds one whole row. + // A body that ended without its closing newline still holds one whole row. public mutating func finish() -> [[PluginCellValue]] { let buffer = pendingBytes pendingBytes = [] + searchedCount = 0 guard !buffer.isEmpty, let row = ingest(buffer[...]) else { return [] } return [row] } + private static func lineSeparatorIndex(in bytes: [UInt8], from start: Int) -> Int? { + bytes.withUnsafeBufferPointer { buffer -> Int? in + guard start < buffer.count, let base = buffer.baseAddress, + let found = memchr(base + start, Int32(ClickHouseTabSeparatedBytes.lineSeparator), buffer.count - start) + else { return nil } + return UnsafeRawPointer(found) - UnsafeRawPointer(base) + } + } + private mutating func ingest(_ line: ArraySlice) -> [PluginCellValue]? { guard let columnNames else { self.columnNames = ClickHouseTabSeparatedBytes.fields(line).map(ClickHouseTabSeparatedBytes.headerText) diff --git a/apple/Plugins/TableProPluginKit/ImportFieldNaming.swift b/apple/Plugins/TableProPluginKit/ImportFieldNaming.swift index 472ffac869..4779b99a6d 100644 --- a/apple/Plugins/TableProPluginKit/ImportFieldNaming.swift +++ b/apple/Plugins/TableProPluginKit/ImportFieldNaming.swift @@ -5,8 +5,7 @@ import Foundation -/// Headers the file spells out keep their names first; blanks and repeats are named afterwards, so a -/// placeholder can never take a name a later header spells literally. +// Literal headers claim their names first, so a placeholder never takes a name a later header spells out. public enum ImportFieldNaming { public static func uniqueNames(for header: [String?], placeholder: (Int) -> String) -> [String] { let trimmed = header.map { ($0 ?? "").trimmingCharacters(in: .whitespacesAndNewlines) } @@ -17,14 +16,17 @@ public enum ImportFieldNaming { names[index] = name } + // Resuming each base where it last stopped keeps a header of one repeated name linear. + var nextOrdinal: [String: Int] = [:] for index in names.indices where names[index] == nil { let base = trimmed[index].isEmpty ? placeholder(index) : trimmed[index] - var unique = base - var suffix = 2 + var ordinal = nextOrdinal[base, default: 1] + var unique = ordinal == 1 ? base : "\(base) \(ordinal)" while !used.insert(unique).inserted { - unique = "\(base) \(suffix)" - suffix += 1 + ordinal += 1 + unique = "\(base) \(ordinal)" } + nextOrdinal[base] = ordinal + 1 names[index] = unique } diff --git a/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver+Schema.swift b/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver+Schema.swift index 6fc6b3733e..02626a53e7 100644 --- a/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver+Schema.swift +++ b/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver+Schema.swift @@ -184,10 +184,11 @@ extension TrinoPluginDriver { ) guard let result = try? await execute(query: sql) else { return nil } for row in result.rows { - guard case .null? = row.first, row.count > 4, let value = text(row[4]), let count = Double(value) else { + guard case .null? = row.first, row.count > 4, let value = text(row[4]), + let count = TrinoIntrospectionSQL.rowCount(fromStatsValue: value) else { continue } - return Int(count) + return count } return nil } diff --git a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift index b9cf0455fe..6f7592ab3a 100644 --- a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift +++ b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift @@ -26,8 +26,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug private static let batchSize = 500 - /// How many rows are read to guess a column's type in the mapping sheet. The whole sheet is - /// already in memory by then, so this only bounds the inference work. + // The whole sheet is already in memory, so this bounds only the type guess for the mapping sheet. private static let detectionSampleRows = 200 @Published var settings = XLSXImportOptions() { @@ -60,9 +59,9 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug } func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { - let sheet = try readSheet(at: url) + let sheet = try XLSXImportSheet.read(at: url, hasHeaderRow: settings.hasHeaderRow, isCancelled: { Task.isCancelled }) guard let header = sheet.header else { return [] } - let sample = Array(sheet.rows.prefix(Self.detectionSampleRows)) + let sample = sheet.rows(0 ..< Self.detectionSampleRows).map(\.values) return header.enumerated().map { index, name in let values = sample.compactMap { $0[safe: index] ?? nil } return PluginImportField( @@ -79,27 +78,37 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug progress: PluginImportProgress ) async throws -> PluginImportResult { let started = Date() - let sheet = try readSheet(at: source.fileURL()) + let sheet: XLSXImportSheet + do { + sheet = try XLSXImportSheet.read( + at: source.fileURL(), + hasHeaderRow: settings.hasHeaderRow, + isCancelled: { progress.isCancelled } + ) + } catch is TabularCancellation { + throw PluginImportCancellationError() + } guard let header = sheet.header else { throw PluginImportError.importFailed( String(localized: "The first sheet has no header row to map columns from.")) } - progress.setEstimatedTotal(max(1, sheet.rows.count)) + progress.setEstimatedTotal(max(1, sheet.rowCount)) var cursor = 0 let outcome = try await RowImportRunner.run( configuration: runConfiguration, sink: sink, progress: progress ) { - guard cursor < sheet.rows.count else { return nil } - let end = min(cursor + Self.batchSize, sheet.rows.count) - let batch = (cursor ..< end).map { index -> RowImportRunner.Entry in - /// The line number counts the header, so it matches what the user sees in Excel. - (line: index + 2, row: self.row(sheet.rows[index], header: header)) - } + guard cursor < sheet.rowCount else { return nil } + let end = min(cursor + Self.batchSize, sheet.rowCount) + let rows = sheet.rows(cursor ..< end) cursor = end - return batch + let filled = rows.filter { $0.values.contains { $0 != nil } } + if filled.count < rows.count { + progress.incrementStatement(by: rows.count - filled.count) + } + return filled.map { (line: $0.line, row: self.row($0.values, header: header)) } } return PluginImportResult( @@ -112,11 +121,6 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug // MARK: - Private - private struct Sheet { - let header: [String]? - let rows: [[String?]] - } - private func row(_ values: [String?], header: [String]) -> [String: PluginCellValue] { var mapped: [String: PluginCellValue] = [:] for (index, column) in header.enumerated() { @@ -137,41 +141,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug return mapped } - /// The workbook is read whole. A sheet's rows are interleaved with the shared string table it - /// refers to, so a streaming read would have to hold that table anyway, and the format has no - /// way to say how large it is before parsing. - private func readSheet(at url: URL) throws -> Sheet { - let archive = try ZipArchive(contentsOf: url) - - guard let worksheetPath = XLSXSheetParser.firstWorksheetPath(in: archive.paths), - let worksheet = archive.entry(named: worksheetPath) else { - throw XLSXSheetParser.ParseError.noWorksheet - } - - var strings: [String] = [] - if let sharedEntry = archive.entry(named: "xl/sharedStrings.xml") { - strings = XLSXSheetParser.sharedStrings(from: try archive.data(for: sharedEntry, limit: .max)) - } - - var rows = XLSXSheetParser.rows( - from: try archive.data(for: worksheet, limit: .max), - sharedStrings: strings - ) - - guard settings.hasHeaderRow else { - let width = rows.first?.count ?? 0 - return Sheet( - header: (0 ..< width).map(XLSXSheetParser.placeholderName), - rows: rows - ) - } - guard !rows.isEmpty else { return Sheet(header: nil, rows: []) } - let headerRow = rows.removeFirst() - return Sheet(header: XLSXSheetParser.headerNames(from: headerRow), rows: rows) - } - - /// Every value arrives as text, so the type is inferred from what the values look like. The - /// sink casts on the way in, so this only drives what the mapping sheet shows. + // The sink casts on the way in, so the guess only drives what the mapping sheet shows. static func inferType(from values: [String]) -> PluginImportFieldType { let candidates = values.filter { !$0.isEmpty } guard !candidates.isEmpty else { return .text } diff --git a/apple/Plugins/XLSXImportPlugin/XLSXImportSheet.swift b/apple/Plugins/XLSXImportPlugin/XLSXImportSheet.swift new file mode 100644 index 0000000000..6cbd595b09 --- /dev/null +++ b/apple/Plugins/XLSXImportPlugin/XLSXImportSheet.swift @@ -0,0 +1,97 @@ +// +// XLSXImportSheet.swift +// XLSXImportPlugin +// + +import Foundation +import TableProPluginKit +import TableProTabularIO + +struct XLSXImportSheet { + enum ReadError: LocalizedError { + case noWorksheet + + var errorDescription: String? { + switch self { + case .noWorksheet: + return String(localized: "The workbook has no worksheet to read.") + } + } + } + + let header: [String]? + private let source: XLSXSheetSource + private let columns: [Int] + private let dataRows: [Int] + + init(source: XLSXSheetSource, hasHeaderRow: Bool) { + let columns = Array(0.. Bool) throws -> XLSXImportSheet { + let workbook = try XLSXWorkbook(contentsOf: url, isCancelled: isCancelled) + guard let sheet = workbook.sheets.first(where: { !$0.isHidden }) ?? workbook.sheets.first else { + throw ReadError.noWorksheet + } + let source = try workbook.source(for: sheet, isCancelled: isCancelled) + return XLSXImportSheet(source: source, hasHeaderRow: hasHeaderRow) + } + + static func headerNames(from row: [String?]) -> [String] { + ImportFieldNaming.uniqueNames(for: row, placeholder: placeholderName) + } + + static func placeholderName(_ index: Int) -> String { + "column\(index + 1)" + } + + var rowCount: Int { dataRows.count } + + func rows(_ range: Range) -> [(line: Int, values: [String?])] { + Self.rows(dataRows[range.clamped(to: dataRows.indices)], of: source, columns: columns) + } + + private static func rows( + _ picked: some Collection, + of source: XLSXSheetSource, + columns: [Int] + ) -> [(line: Int, values: [String?])] { + var result: [(line: Int, values: [String?])] = [] + result.reserveCapacity(picked.count) + source.scan(columns: columns, rows: picked, absentKind: .missing) { row, cells in + result.append((source.sheetReference(row: row, column: 0).row + 1, values(of: cells))) + return true + } + return result + } + + private static func values(of cells: TabularRowCells) -> [String?] { + (0.. [String] { - ImportFieldNaming.uniqueNames(for: row, placeholder: placeholderName) - } - - static func placeholderName(_ index: Int) -> String { - "column\(index + 1)" - } - - static func columnIndex(fromReference reference: String) -> Int? { - var index = 0 - var sawLetter = false - for character in reference.uppercased() { - guard let ascii = character.asciiValue else { return nil } - if ascii >= 65, ascii <= 90 { - index = index * 26 + Int(ascii - 64) - sawLetter = true - } else if ascii >= 48, ascii <= 57 { - break - } else { - return nil - } - } - return sawLetter ? index - 1 : nil - } - - /// The worksheet part of the first sheet. Named parts vary, so the first `sheet*.xml` under - /// `xl/worksheets/` is taken rather than assuming `sheet1.xml`. - static func firstWorksheetPath(in paths: [String]) -> String? { - paths - .filter { $0.hasPrefix("xl/worksheets/") && $0.hasSuffix(".xml") } - .sorted() - .first - } - - static func sharedStrings(from data: Data) -> [String] { - let delegate = SharedStringsDelegate() - let parser = XMLParser(data: data) - parser.delegate = delegate - parser.parse() - return delegate.strings - } - - /// Every row of the sheet, each padded to the widest row so a caller can zip them with a header - /// without checking lengths. - static func rows(from data: Data, sharedStrings: [String]) -> [[String?]] { - let delegate = SheetDelegate(sharedStrings: sharedStrings) - let parser = XMLParser(data: data) - parser.delegate = delegate - parser.parse() - - let width = delegate.rows.map(\.count).max() ?? 0 - return delegate.rows.map { row in - row.count == width ? row : row + Array(repeating: nil, count: width - row.count) - } - } -} - -/// `sharedStrings.xml` holds one `` per string, and a string can be split across several `` -/// runs when parts of it are formatted differently. The runs are concatenated, or a styled word in -/// the middle of a cell would truncate the value. -private final class SharedStringsDelegate: NSObject, XMLParserDelegate { - private(set) var strings: [String] = [] - private var current: String? - private var isInsideText = false - - func parser( - _ parser: XMLParser, - didStartElement elementName: String, - namespaceURI: String?, - qualifiedName: String?, - attributes: [String: String] - ) { - switch elementName { - case "si": current = "" - case "t": isInsideText = true - default: break - } - } - - func parser(_ parser: XMLParser, foundCharacters string: String) { - guard isInsideText, current != nil else { return } - current? += string - } - - func parser( - _ parser: XMLParser, - didEndElement elementName: String, - namespaceURI: String?, - qualifiedName: String? - ) { - switch elementName { - case "t": isInsideText = false - case "si": - strings.append(current ?? "") - current = nil - default: break - } - } -} - -private final class SheetDelegate: NSObject, XMLParserDelegate { - private(set) var rows: [[String?]] = [] - - private let sharedStrings: [String] - private var currentRow: [String?] = [] - private var currentColumn = 0 - private var cellType = "" - private var value: String? - private var isInsideValue = false - private var isInsideInlineText = false - - init(sharedStrings: [String]) { - self.sharedStrings = sharedStrings - } - - func parser( - _ parser: XMLParser, - didStartElement elementName: String, - namespaceURI: String?, - qualifiedName: String?, - attributes: [String: String] - ) { - switch elementName { - case "row": - currentRow = [] - case "c": - cellType = attributes["t"] ?? "" - value = nil - currentColumn = attributes["r"] - .flatMap { XLSXSheetParser.columnIndex(fromReference: $0) } - ?? currentRow.count - case "v": - isInsideValue = true - value = "" - case "t": - /// An inline string lives in the cell rather than in `sharedStrings.xml`. Numbers and - /// LibreOffice both write them. - isInsideInlineText = true - value = value ?? "" - default: - break - } - } - - func parser(_ parser: XMLParser, foundCharacters string: String) { - guard isInsideValue || isInsideInlineText else { return } - value = (value ?? "") + string - } - - func parser( - _ parser: XMLParser, - didEndElement elementName: String, - namespaceURI: String?, - qualifiedName: String? - ) { - switch elementName { - case "v": - isInsideValue = false - case "t": - isInsideInlineText = false - case "c": - place(resolvedValue(), at: currentColumn) - case "row": - rows.append(currentRow) - currentRow = [] - default: - break - } - } - - /// A cell whose type is `s` holds an index into the shared strings rather than the text itself. - /// An index the table does not have is left as the raw value rather than dropped, so a damaged - /// workbook still imports something the user can see is wrong. - private func resolvedValue() -> String? { - guard let value else { return nil } - guard cellType == "s" else { return value } - guard let index = Int(value), index >= 0, index < sharedStrings.count else { return value } - return sharedStrings[index] - } - - /// A row omits the cells it has no value for, so a cell lands at the index its own reference - /// names and the gap before it is filled with nulls. - private func place(_ value: String?, at column: Int) { - guard column >= 0 else { return } - while currentRow.count < column { - currentRow.append(nil) - } - if currentRow.count == column { - currentRow.append(value) - } else { - currentRow[column] = value - } - } -} diff --git a/apple/TablePro/Core/DataFiles/DataFileColumnNames.swift b/apple/TablePro/Core/DataFiles/DataFileColumnNames.swift index fd3f6834bf..0d4b3209a2 100644 --- a/apple/TablePro/Core/DataFiles/DataFileColumnNames.swift +++ b/apple/TablePro/Core/DataFiles/DataFileColumnNames.swift @@ -4,33 +4,33 @@ // import Foundation +import TableProCoreTypes import TableProTabular struct DataFileColumnNames: Equatable { let ids: [TabularColumnID] let displayNames: [String] private let idByName: [String: TabularColumnID] + private let indexByID: [TabularColumnID: Int] init(columns: [TabularColumn]) { - var used = Set() + var allocator = UniqueNameAllocator(candidate: { base, ordinal in + ordinal == 1 ? base : "\(base) (\(ordinal))" + }) var names: [String] = [] var byName: [String: TabularColumnID] = [:] + var positions: [TabularColumnID: Int] = [:] names.reserveCapacity(columns.count) for (index, column) in columns.enumerated() { - let base = Self.baseName(column.name, position: index) - var candidate = base - var suffix = 2 - while used.contains(candidate) { - candidate = "\(base) (\(suffix))" - suffix += 1 - } - used.insert(candidate) - names.append(candidate) - byName[candidate] = column.id + let name = allocator.allocate(Self.baseName(column.name, position: index)) + names.append(name) + byName[name] = column.id + if positions[column.id] == nil { positions[column.id] = index } } ids = columns.map(\.id) displayNames = names idByName = byName + indexByID = positions } var count: Int { ids.count } @@ -40,12 +40,11 @@ struct DataFileColumnNames: Equatable { } func name(for id: TabularColumnID) -> String? { - guard let index = ids.firstIndex(of: id) else { return nil } - return displayNames[index] + indexByID[id].map { displayNames[$0] } } func index(of id: TabularColumnID) -> Int? { - ids.firstIndex(of: id) + indexByID[id] } private static func baseName(_ raw: String, position: Int) -> String { diff --git a/apple/TablePro/Core/DataFiles/DataFileLoader.swift b/apple/TablePro/Core/DataFiles/DataFileLoader.swift index a6ad7b0f0a..19b9b14125 100644 --- a/apple/TablePro/Core/DataFiles/DataFileLoader.swift +++ b/apple/TablePro/Core/DataFiles/DataFileLoader.swift @@ -127,7 +127,7 @@ enum DataFileLoader { throw DataFileLoadError.invalidJSON(error) } let table = TabularTable(source: source, usesFirstRowAsHeader: false) - let kinds = TabularTypeInference.inferKinds(of: table) + let kinds = try TabularTypeInference.inferKinds(of: table) progress(1) let sheet = DataFileSheet( name: request.url.lastPathComponent, @@ -157,7 +157,7 @@ enum DataFileLoader { ) async throws -> DataFileLoadedSheet { let source = try workbook.source(for: sheet, progress: progress, isCancelled: { Task.isCancelled }) let table = TabularTable(source: source, usesFirstRowAsHeader: source.firstRowLooksLikeHeader) - return DataFileLoadedSheet(table: table, kinds: TabularTypeInference.inferKinds(of: table)) + return try DataFileLoadedSheet(table: table, kinds: TabularTypeInference.inferKinds(of: table)) } private static func loadWorkbook( @@ -176,7 +176,7 @@ enum DataFileLoader { let first = workbook.sheets[initial] let source = try workbook.source(for: first, progress: { progress(0.1 + $0 * 0.9) }, isCancelled: { Task.isCancelled }) let table = TabularTable(source: source, usesFirstRowAsHeader: source.firstRowLooksLikeHeader) - let kinds = TabularTypeInference.inferKinds(of: table) + let kinds = try TabularTypeInference.inferKinds(of: table) let sheets = workbook.sheets.map { sheet in DataFileSheet( name: sheet.name, @@ -234,7 +234,7 @@ enum DataFileLoader { } let source = contents.source let table = TabularTable(source: source, usesFirstRowAsHeader: source.dialect.hasHeaderRow) - let kinds = TabularTypeInference.inferKinds(of: table) + let kinds = try TabularTypeInference.inferKinds(of: table) progress(1) let sheet = DataFileSheet( name: request.url.lastPathComponent, diff --git a/apple/TablePro/Core/DataFiles/DataFileWorkingCopy.swift b/apple/TablePro/Core/DataFiles/DataFileWorkingCopy.swift index cf16af6e22..2a71f4a7de 100644 --- a/apple/TablePro/Core/DataFiles/DataFileWorkingCopy.swift +++ b/apple/TablePro/Core/DataFiles/DataFileWorkingCopy.swift @@ -5,7 +5,7 @@ import Foundation import os -import TableProUtilities +import TableProGzip final class DataFileWorkingCopy: @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "DataFiles") @@ -36,7 +36,7 @@ final class DataFileWorkingCopy: @unchecked Sendable { func snapshot(of url: URL, kind: DataFileKind) async throws -> URL { if kind.isCompressed { let destination = file(named: "content-\(UUID().uuidString).\(kind.contentExtension)") - try await GzipProcess.decompress(source: url, destination: destination) + try await GzipFile.decompress(source: url, destination: destination) return destination } let destination = file(named: "content-\(UUID().uuidString).\(kind.contentExtension)") diff --git a/apple/TablePro/Core/LiveMessages/LiveMessageBuffer.swift b/apple/TablePro/Core/LiveMessages/LiveMessageBuffer.swift index 9092b1678a..a043229131 100644 --- a/apple/TablePro/Core/LiveMessages/LiveMessageBuffer.swift +++ b/apple/TablePro/Core/LiveMessages/LiveMessageBuffer.swift @@ -35,10 +35,14 @@ internal struct LiveMessageBuffer: Sendable { static let defaultCapacity = 10_000 let capacity: Int - private(set) var entries: [LiveMessageEntry] = [] + // A ring once full: each new entry takes the slot of the oldest, which is released then. + private var storage: [LiveMessageEntry] = [] + private var oldest = 0 private(set) var droppedCount = 0 private var nextId = 0 + var entries: [LiveMessageEntry] { Array(storage[oldest...]) + storage[.. 0 else { return } - entries.removeFirst(excess) - droppedCount += excess + guard storage.count == capacity else { + storage.append(entry) + return + } + storage[oldest] = entry + oldest = oldest + 1 == capacity ? 0 : oldest + 1 + droppedCount += 1 } } diff --git a/apple/TablePro/Core/Plugins/SqlFileImportSource.swift b/apple/TablePro/Core/Plugins/SqlFileImportSource.swift index 26695e3b3f..5c9365defa 100644 --- a/apple/TablePro/Core/Plugins/SqlFileImportSource.swift +++ b/apple/TablePro/Core/Plugins/SqlFileImportSource.swift @@ -126,7 +126,7 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { return existing } - let result = try await FileDecompressor.decompressIfNeeded(url) { $0.path() } + let result = try await FileDecompressor.decompressIfNeeded(url) if result != url { _decompressedURL.withLock { $0 = result } diff --git a/apple/TablePro/Core/ServerDashboard/Providers/ClickHouseDashboardProvider.swift b/apple/TablePro/Core/ServerDashboard/Providers/ClickHouseDashboardProvider.swift index ffe243c656..7b225927d8 100644 --- a/apple/TablePro/Core/ServerDashboard/Providers/ClickHouseDashboardProvider.swift +++ b/apple/TablePro/Core/ServerDashboard/Providers/ClickHouseDashboardProvider.swift @@ -24,7 +24,7 @@ struct ClickHouseDashboardProvider: ServerDashboardQueryProvider { let readRows = value(row, at: col["read_rows"]) let memUsage = value(row, at: col["memory_usage"]) let stateDescription = "rows: \(readRows), mem: \(formatBytes(memUsage))" - let secs = Int(elapsed) + let secs = SaturatingArithmetic.int(elapsed) return DashboardSession( id: value(row, at: col["query_id"]), user: value(row, at: col["user"]), @@ -88,7 +88,8 @@ struct ClickHouseDashboardProvider: ServerDashboardQueryProvider { let result = try await execute(sql) let col = columnIndex(from: result.columns) return result.rows.map { row in - let secs = Int(value(row, at: col["duration_secs"])) ?? 0 + // ClickHouse's `/` returns a Float64, such as "1.5". + let secs = SaturatingArithmetic.int(Double(value(row, at: col["duration_secs"])) ?? 0) return DashboardSlowQuery( duration: formatDuration(seconds: secs), query: value(row, at: col["query"]), diff --git a/apple/TablePro/Core/ServerDashboard/Providers/MemcachedDashboardProvider.swift b/apple/TablePro/Core/ServerDashboard/Providers/MemcachedDashboardProvider.swift index 80cd0c9a99..9e4112ec51 100644 --- a/apple/TablePro/Core/ServerDashboard/Providers/MemcachedDashboardProvider.swift +++ b/apple/TablePro/Core/ServerDashboard/Providers/MemcachedDashboardProvider.swift @@ -31,7 +31,7 @@ struct MemcachedDashboardProvider: ServerDashboardQueryProvider { )) } if let hits = number("get_hits"), let misses = number("get_misses") { - let lookups = Double(hits + misses) + let lookups = Double(hits) + Double(misses) metrics.append(DashboardMetric( id: "hit_ratio", label: String(localized: "Hit Ratio"), diff --git a/apple/TablePro/Core/ServerDashboard/Providers/MongoDBDashboardProvider.swift b/apple/TablePro/Core/ServerDashboard/Providers/MongoDBDashboardProvider.swift index 145c43de25..3132d5bd59 100644 --- a/apple/TablePro/Core/ServerDashboard/Providers/MongoDBDashboardProvider.swift +++ b/apple/TablePro/Core/ServerDashboard/Providers/MongoDBDashboardProvider.swift @@ -162,7 +162,7 @@ struct MongoDBDashboardProvider: ServerDashboardQueryProvider { metrics.append(DashboardMetric( id: "resident_memory", label: String(localized: "Resident Memory"), - value: ByteSizeFormatting.string(bytes: mebibytes * 1_048_576), + value: ByteSizeFormatting.string(mebibytes: mebibytes), unit: "", icon: "memorychip" )) @@ -192,7 +192,7 @@ struct MongoDBDashboardProvider: ServerDashboardQueryProvider { return result.rows.map { row in let millis = Int(value(row, at: col["millis"])) ?? 0 return DashboardSlowQuery( - duration: DurationFormatting.string(microseconds: millis * 1_000), + duration: DurationFormatting.string(milliseconds: millis), query: String(value(row, at: col["command"]).prefix(Self.queryTextLimit)), user: value(row, at: col["user"]), database: Self.database(ofNamespace: value(row, at: col["ns"])) diff --git a/apple/TablePro/Core/Services/Export/ExportService.swift b/apple/TablePro/Core/Services/Export/ExportService.swift index fbcf13983b..836808875b 100644 --- a/apple/TablePro/Core/Services/Export/ExportService.swift +++ b/apple/TablePro/Core/Services/Export/ExportService.swift @@ -488,10 +488,7 @@ final class ExportService: ObservableObject { ) } - /// The non-SQL count goes through the data source, which knows the container each object was - /// listed under. Asking the driver directly answers about whichever one it is leased to, so an - /// export spanning two databases counted one of them twice and reported a total no progress bar - /// could reach. + // The non-SQL count asks the data source: the driver answers for whichever database it is leased to. private func fetchTotalRowCount( for tables: [ExportObjectItem], driver: DatabaseDriver, @@ -510,7 +507,7 @@ final class ExportService: ObservableObject { table: table.name, databaseName: table.databaseName ) if let count { - total += count + total = Self.saturatingSum(total, count) } } catch { failedCount += 1 @@ -540,7 +537,7 @@ final class ExportService: ObservableObject { let result = try await driver.execute(query: batchQuery) for row in result.rows { if let cell = row.first, let count = Int(cell.asText ?? "0") { - total += count + total = Self.saturatingSum(total, count) } } } catch { @@ -549,7 +546,7 @@ final class ExportService: ObservableObject { let tableRef = qualifiedTableRef(for: table, driver: driver, databaseType: databaseType) let result = try await driver.execute(query: "SELECT COUNT(*) FROM \(tableRef)") if let cell = result.rows.first?.first, let count = Int(cell.asText ?? "0") { - total += count + total = Self.saturatingSum(total, count) } } catch { failedCount += 1 @@ -566,6 +563,11 @@ final class ExportService: ObservableObject { return total } + private static func saturatingSum(_ total: Int, _ count: Int) -> Int { + let (sum, overflow) = total.addingReportingOverflow(count) + return overflow ? (count < 0 ? .min : .max) : sum + } + /// Counts pick between an explicit singular and plural key. Automatic grammar agreement is a /// SwiftUI `Text` facility: `String(localized:)` returns the markup verbatim. private static func estimatedProgressMessage(uncountedTables: Int) -> String { diff --git a/apple/TablePro/Core/Services/Export/ForeignApp/JetBrains/KdbxDatabase.swift b/apple/TablePro/Core/Services/Export/ForeignApp/JetBrains/KdbxDatabase.swift index 5b14a57ed8..70d1f2729c 100644 --- a/apple/TablePro/Core/Services/Export/ForeignApp/JetBrains/KdbxDatabase.swift +++ b/apple/TablePro/Core/Services/Export/ForeignApp/JetBrains/KdbxDatabase.swift @@ -4,9 +4,9 @@ // import CommonCrypto -import Compression import Foundation import os +import TableProGzip enum KdbxError: Error { case malformedHeader @@ -259,71 +259,8 @@ enum KdbxDatabase { // MARK: - GZIP private static func gunzip(_ data: [UInt8]) -> [UInt8]? { - guard data.count > 18, data[0] == 0x1f, data[1] == 0x8b, data[2] == 0x08 else { return nil } - let flags = data[3] - var offset = 10 - if flags & 0x04 != 0 { - guard offset + 2 <= data.count else { return nil } - let extraLength = Int(readUInt16LE(data, offset)) - offset += 2 + extraLength - } - if flags & 0x08 != 0 { - while offset < data.count, data[offset] != 0 { offset += 1 } - offset += 1 - } - if flags & 0x10 != 0 { - while offset < data.count, data[offset] != 0 { offset += 1 } - offset += 1 - } - if flags & 0x02 != 0 { - offset += 2 - } - guard offset < data.count - 8 else { return nil } - return inflateRawDeflate(Array(data[offset..<(data.count - 8)])) - } - - private static func inflateRawDeflate(_ input: [UInt8]) -> [UInt8]? { - let bufferSize = 65_536 - let destination = UnsafeMutablePointer.allocate(capacity: bufferSize) - defer { destination.deallocate() } - - var stream = compression_stream( - dst_ptr: destination, - dst_size: bufferSize, - src_ptr: UnsafePointer(destination), - src_size: 0, - state: nil - ) - guard compression_stream_init(&stream, COMPRESSION_STREAM_DECODE, COMPRESSION_ZLIB) == COMPRESSION_STATUS_OK else { - return nil - } - defer { compression_stream_destroy(&stream) } - - return input.withUnsafeBufferPointer { source -> [UInt8]? in - guard let base = source.baseAddress else { return nil } - stream.src_ptr = base - stream.src_size = source.count - stream.dst_ptr = destination - stream.dst_size = bufferSize - - var output: [UInt8] = [] - while true { - let status = compression_stream_process(&stream, Int32(COMPRESSION_STREAM_FINALIZE.rawValue)) - switch status { - case COMPRESSION_STATUS_OK: - if stream.dst_size == 0 { - output.append(contentsOf: UnsafeBufferPointer(start: destination, count: bufferSize)) - stream.dst_ptr = destination - stream.dst_size = bufferSize - } - case COMPRESSION_STATUS_END: - output.append(contentsOf: UnsafeBufferPointer(start: destination, count: bufferSize - stream.dst_size)) - return output - default: - return nil - } - } - } + guard let inflated = try? GzipInflater.decompress(Data(data)) else { return nil } + return [UInt8](inflated) } // MARK: - Little-Endian Readers diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift index 4d0159e00a..b449bc09e2 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift @@ -66,7 +66,7 @@ internal enum PhpSerializeWriter { output += "r:\(id);" case .unsupported(let token): output += token - case .depthExceeded, .tooLarge: + case .tooLarge: output += "…" } } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift index 8ed18de9a8..5cba571bdb 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift @@ -9,12 +9,34 @@ import Foundation internal enum ProtoFileParser { static let maxTokens = 2_000_000 static let maxDepth = 100 + static let maxPackageComponents = 100 static func parse(_ text: String) throws -> [ProtobufFileDefinition] { var parser = Parser(tokens: try Tokenizer.tokens(text)) return [try parser.file()] } + // Each field's type is looked up through every part of the package, so a long one costs every field. + static func checkPackage(_ package: String) throws { + guard package.utf8.count(where: { $0 == UInt8(ascii: ".") }) < maxPackageComponents else { + throw error(String(localized: "The package name has too many parts.")) + } + } + + static let maxNameLength = 1_024 + + // Every message and enum keeps its full name, so a long scope over many small definitions would copy far + // more than the file holds. Both are bounded before the name is built. + static func qualifiedName(_ scope: String, _ name: String, defined: Int) throws -> String { + guard defined < ProtobufSchema.maxMessages else { + throw error(String(localized: "The schema defines more messages than TablePro reads.")) + } + guard scope.utf8.count + name.utf8.count < maxNameLength else { + throw error(String(localized: "A type name is too long.")) + } + return scope.isEmpty ? name : "\(scope).\(name)" + } + fileprivate enum Token: Equatable { case identifier(String) case number(String) @@ -104,6 +126,7 @@ internal enum ProtoFileParser { mutating func file() throws -> ProtobufFileDefinition { var file = ProtobufFileDefinition() let package = packageName() + try ProtoFileParser.checkPackage(package) while let token = peek() { switch token { case .identifier("syntax"): @@ -154,8 +177,8 @@ internal enum ProtoFileParser { } private mutating func message(scope: String, depth: Int) throws -> String { - let name = try identifier() - let fullName = scope.isEmpty ? name : "\(scope).\(name)" + let name = try declaredName() + let fullName = try ProtoFileParser.qualifiedName(scope, name, defined: messages.count) try expect("{") try messageBody(fullName: fullName, depth: depth) return fullName @@ -165,11 +188,17 @@ internal enum ProtoFileParser { let slot = messages.count messages.append(ProtobufMessageDefinition(fullName: fullName)) var definition = messages[slot] - try body(of: &definition, depth: depth, oneof: nil) + var oneofCount = 0 + try body(of: &definition, depth: depth, oneof: nil, oneofCount: &oneofCount) messages[slot] = definition } - private mutating func body(of definition: inout ProtobufMessageDefinition, depth: Int, oneof: Int?) throws { + private mutating func body( + of definition: inout ProtobufMessageDefinition, + depth: Int, + oneof: Int?, + oneofCount: inout Int + ) throws { guard depth < ProtoFileParser.maxDepth else { throw ProtoFileParser.error(String(localized: "Messages are nested too deeply.")) } while true { guard let token = peek() else { throw ProtoFileParser.error(String(localized: "A message is never closed.")) } @@ -189,8 +218,9 @@ internal enum ProtoFileParser { advance() _ = try identifier() try expect("{") - let index = (definition.fields.compactMap(\.oneofIndex).max() ?? -1) + 1 - try body(of: &definition, depth: depth + 1, oneof: index) + let index = oneofCount + oneofCount += 1 + try body(of: &definition, depth: depth + 1, oneof: index, oneofCount: &oneofCount) // A Schema Registry stores a map field as the entry message protoc synthesizes for it. case .identifier("option") where peek(1) == .identifier("map_entry") && peek(3) == .identifier("true"): definition.isMapEntry = true @@ -218,14 +248,14 @@ internal enum ProtoFileParser { advance() } let typeName = try identifier() - let name = try identifier() + let name = try declaredName() try expect("=") let number = try fieldNumber() let features = try fieldOptions() let type: ProtobufFieldDefinition.TypeReference if typeName == "group" { try expect("{") - let fullName = "\(definition.fullName).\(name)" + let fullName = try ProtoFileParser.qualifiedName(definition.fullName, name, defined: messages.count) try messageBody(fullName: fullName, depth: depth + 1) definition.nestedMessages.append(fullName) type = .group(fullName) @@ -252,12 +282,14 @@ internal enum ProtoFileParser { try expect(",") let valueType = try identifier() try expect(">") - let name = try identifier() + let name = try declaredName() try expect("=") let number = try fieldNumber() _ = try fieldOptions() try expect(";") - let entryName = "\(definition.fullName).\(Self.camelCased(name))Entry" + let entryName = try ProtoFileParser.qualifiedName( + definition.fullName, "\(Self.camelCased(name))Entry", defined: messages.count + ) func reference(_ type: String) -> ProtobufFieldDefinition.TypeReference { ProtobufScalarType(rawValue: type).map { .scalar($0) } ?? .named(type) } @@ -274,8 +306,8 @@ internal enum ProtoFileParser { } private mutating func enumeration(scope: String) throws -> String { - let name = try identifier() - let fullName = scope.isEmpty ? name : "\(scope).\(name)" + let name = try declaredName() + let fullName = try ProtoFileParser.qualifiedName(scope, name, defined: enums.count) try expect("{") var enumeration = ProtobufEnumDefinition(fullName: fullName) while true { @@ -438,6 +470,14 @@ internal enum ProtoFileParser { return name } + // protoc takes a declared name as one identifier, and a dotted one would lengthen the scope each field's + // type is looked up through. + private mutating func declaredName() throws -> String { + guard case .identifier(let name)? = peek(), !name.contains(".") else { throw unexpected() } + advance() + return name + } + private mutating func expect(_ symbol: Character) throws { guard peek() == .symbol(symbol) else { throw unexpected() } advance() diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift index 7b493dd14d..3d51a7968e 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift @@ -110,7 +110,23 @@ internal struct ProtobufSchema: Equatable, Sendable { } static func parse(protoFiles texts: [String]) throws -> ProtobufSchema { - try build(texts.flatMap { try ProtoFileParser.parse($0) }) + var files: [ProtobufFileDefinition] = [] + var defined = 0 + for text in texts { + for file in try ProtoFileParser.parse(text) { + try countDefinitions(of: file, into: &defined) + files.append(file) + } + } + return try build(files) + } + + // Each file bounds its own definitions, so many files are bounded together as they are read. + static func countDefinitions(of file: ProtobufFileDefinition, into defined: inout Int) throws { + defined += file.allMessages.count + file.allEnums.count + guard defined <= 2 * maxMessages else { + throw ProtobufSchemaError(reason: String(localized: "The schema defines more messages than TablePro reads.")) + } } func messageIndexed(by indexes: [Int]) -> ProtobufMessageDescriptor? { @@ -157,10 +173,11 @@ internal struct ProtobufSchema: Equatable, Sendable { var resolved: [String: ProtobufMessageDescriptor] = [:] for (name, message) in definitions { let features = scopes.messages[name] ?? ProtobufFeatures(syntax: .proto2) + let scope = resolver.scope(of: name) resolved[name] = ProtobufMessageDescriptor( fullName: name, fields: message.fields.map { - resolver.resolve($0, scope: name, features: features, inMapEntry: message.isMapEntry) + resolver.resolve($0, scope: scope, features: features, inMapEntry: message.isMapEntry) }, nestedMessages: message.nestedMessages, isMapEntry: message.isMapEntry @@ -296,15 +313,34 @@ private struct ProtobufFeatureScopes { } } -// C++ scoping: a name is looked up from the innermost scope outward, by its first component. +// C++ scoping, as protoc does it: the first component is looked up from the innermost scope outward, +// and the rest is resolved inside the first scope that has it. private struct ProtobufTypeResolver { let messageNames: Set let enumNames: Set let mapEntryNames: Set + private let paths: ProtobufNamePaths + + init(messageNames: Set, enumNames: Set, mapEntryNames: Set) { + self.messageNames = messageNames + self.enumNames = enumNames + self.mapEntryNames = mapEntryNames + paths = ProtobufNamePaths(messageNames.union(enumNames)) + } + + // Found once per message: splitting the name again for each field cost fields times its length. + func scope(of messageName: String) -> [Int] { + var nodes = [ProtobufNamePaths.root] + for component in messageName.split(separator: ".") { + guard let last = nodes.last, let next = paths.child(of: last, component) else { break } + nodes.append(next) + } + return nodes + } func resolve( _ field: ProtobufFieldDefinition, - scope: String, + scope: [Int], features: ProtobufFeatures, inMapEntry: Bool ) -> ProtobufFieldDescriptor { @@ -342,17 +378,26 @@ private struct ProtobufTypeResolver { return !mapEntryNames.contains(name) } - private func resolve(_ reference: String, scope: String) -> ProtobufFieldType { + private func resolve(_ reference: String, scope: [Int]) -> ProtobufFieldType { if reference.hasPrefix(".") { return lookup(String(reference.dropFirst())) ?? .unresolved(reference) } - var components = scope.split(separator: ".").map(String.init) - while true { - let candidate = (components + [reference]).joined(separator: ".") - if let found = lookup(candidate) { return found } - guard !components.isEmpty else { return .unresolved(reference) } - components.removeLast() + let parts = reference.split(separator: ".", omittingEmptySubsequences: false) + guard let first = parts.first else { return .unresolved(reference) } + for scopeNode in scope.reversed() { + guard var node = paths.child(of: scopeNode, first) else { continue } + if parts.count == 1 { + guard let name = paths.name(at: node) else { continue } + return lookup(name) ?? .unresolved(reference) + } + for part in parts.dropFirst() { + guard let next = paths.child(of: node, part) else { return .unresolved(reference) } + node = next + } + guard let name = paths.name(at: node) else { return .unresolved(reference) } + return lookup(name) ?? .unresolved(reference) } + return .unresolved(reference) } private func lookup(_ name: String) -> ProtobufFieldType? { @@ -362,14 +407,54 @@ private struct ProtobufTypeResolver { } } +// Every known type name as a path of components. A node that ends a name remembers it. +private struct ProtobufNamePaths { + private struct Step: Hashable { + let parent: Int + let component: Substring + } + + static let root = 0 + private var steps: [Step: Int] = [:] + private var names: [Int: String] = [:] + + init(_ fullNames: Set) { + for fullName in fullNames { + var node = Self.root + for component in fullName.split(separator: ".", omittingEmptySubsequences: false) { + let step = Step(parent: node, component: component) + if let next = steps[step] { + node = next + } else { + let next = steps.count + 1 + steps[step] = next + node = next + } + } + names[node] = fullName + } + } + + func child(of node: Int, _ component: Substring) -> Int? { + steps[Step(parent: node, component: component)] + } + + func name(at node: Int) -> String? { + names[node] + } +} + // descriptor.proto read with the wire reader. Field numbers are descriptor.proto's. internal enum ProtobufDescriptorSetParser { static func parse(_ data: Data) throws -> [ProtobufFileDefinition] { var files: [ProtobufFileDefinition] = [] + var defined = 0 do { for (number, value) in try fields(data) { guard number == 1, case .bytes(let bytes) = value else { throw ProtobufSchemaError.notASchema } - files.append(try file(bytes)) + let parsed = try file(bytes) + try ProtobufSchema.countDefinitions(of: parsed, into: &defined) + files.append(parsed) } } catch let error as ProtobufSchemaError { throw error @@ -466,6 +551,7 @@ internal enum ProtobufDescriptorSetParser { default: break } } + try ProtoFileParser.checkPackage(package) let syntax = bytes(entries, 12).flatMap { String(data: $0, encoding: .utf8) } switch syntax { case "proto3": file.syntax = .proto3 @@ -490,8 +576,8 @@ internal enum ProtobufDescriptorSetParser { throw ProtobufSchemaError(reason: String(localized: "Messages are nested too deeply.")) } let entries = try ProtobufDescriptorSetParser.fields(data) - let name = entries.first { $0.0 == 1 }.flatMap { string($0.1) } ?? "" - let fullName = scope.isEmpty ? name : "\(scope).\(name)" + let name = try Self.declaredName(entries.first { $0.0 == 1 }.flatMap { string($0.1) } ?? "") + let fullName = try ProtoFileParser.qualifiedName(scope, name, defined: messages.count) let slot = messages.count messages.append(ProtobufMessageDefinition(fullName: fullName)) var definition = messages[slot] @@ -515,7 +601,7 @@ internal enum ProtobufDescriptorSetParser { var name = "" var values: [Int32: String] = [:] for (number, value) in entries { - if number == 1 { name = string(value) ?? "" } + if number == 1 { name = try Self.declaredName(string(value) ?? "") } guard number == 2, case .bytes(let bytes) = value else { continue } var valueName = "" var valueNumber: Int32 = 0 @@ -526,13 +612,19 @@ internal enum ProtobufDescriptorSetParser { // protoc prints an aliased number by its first name. if values[valueNumber] == nil { values[valueNumber] = valueName } } - let fullName = scope.isEmpty ? name : "\(scope).\(name)" + let fullName = try ProtoFileParser.qualifiedName(scope, name, defined: enums.count) enums.append(ProtobufEnumDefinition( fullName: fullName, values: values, features: try features(entries, options: 3, features: 7) )) return fullName } + // protoc refuses a dotted name, and one would lengthen the scope each field's type is looked up through. + private static func declaredName(_ name: String) throws -> String { + guard !name.contains(".") else { throw ProtobufSchemaError.notASchema } + return name + } + private func field(_ data: Data) throws -> ProtobufFieldDefinition { let entries = try ProtobufDescriptorSetParser.fields(data) let typeCode = varint(entries, 5) ?? 0 diff --git a/apple/TablePro/Core/Services/Formatting/ByteSizeFormatting.swift b/apple/TablePro/Core/Services/Formatting/ByteSizeFormatting.swift index ed6a99141d..bc98a8f565 100644 --- a/apple/TablePro/Core/Services/Formatting/ByteSizeFormatting.swift +++ b/apple/TablePro/Core/Services/Formatting/ByteSizeFormatting.swift @@ -38,6 +38,10 @@ internal enum ByteSizeFormatting { guard value < Double(Int64.max) else { return byteString } return string(bytes: Int64(value)) } + + internal static func string(mebibytes: Int64) -> String { + string(bytes: SaturatingArithmetic.product(mebibytes, 1_048_576)) + } } internal enum DurationFormatting { @@ -53,13 +57,17 @@ internal enum DurationFormatting { } internal static func string(seconds: Double) -> String { - string(seconds: Int(seconds.rounded())) + string(seconds: SaturatingArithmetic.int(seconds.rounded())) + } + + internal static func string(milliseconds: Int) -> String { + string(microseconds: SaturatingArithmetic.product(milliseconds, 1_000)) } // Slow log entries mostly finish within a second, which the whole-second form shows as 0s. internal static func string(microseconds: Int) -> String { let microseconds = max(0, microseconds) - let milliseconds = (microseconds + 500) / 1_000 + let milliseconds = microseconds / 1_000 + (microseconds % 1_000 >= 500 ? 1 : 0) if milliseconds >= 1_000 { return string(seconds: Double(microseconds) / 1_000_000) } @@ -69,3 +77,20 @@ internal enum DurationFormatting { return Duration.microseconds(microseconds).formatted(.units(allowed: [.microseconds], width: .narrow)) } } + +// A number a server reports for display pins to the end of the range instead of trapping. +internal enum SaturatingArithmetic { + // Truncates toward zero like `Int(_:)`; NaN reads as zero. + internal static func int(_ value: Double) -> Int { + if value.isNaN { return 0 } + if value >= Double(Int.max) { return .max } + if value <= Double(Int.min) { return .min } + return Int(value) + } + + internal static func product(_ lhs: Value, _ rhs: Value) -> Value { + let (result, overflow) = lhs.multipliedReportingOverflow(by: rhs) + guard overflow else { return result } + return (lhs < 0) == (rhs < 0) ? .max : .min + } +} diff --git a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift index dd19369787..96a55ad3a5 100644 --- a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift +++ b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift @@ -24,7 +24,6 @@ internal indirect enum PhpValue: Equatable { // `r:`, the object numbered `id` again. case objectReference(id: Int) case unsupported(token: String) - case depthExceeded case tooLarge static func == (lhs: PhpValue, rhs: PhpValue) -> Bool { @@ -49,7 +48,6 @@ internal indirect enum PhpValue: Equatable { case (.reference(let a), .reference(let b)): return a == b case (.objectReference(let a), .objectReference(let b)): return a == b case (.unsupported(let a), .unsupported(let b)): return a == b - case (.depthExceeded, .depthExceeded): return true case (.tooLarge, .tooLarge): return true default: return false } @@ -105,14 +103,19 @@ internal enum PhpSerializeParser { } private struct Cursor { + private static let cancellationInterval = 4_096 + private static let reservationLimit = 1_024 + let bytes: [UInt8] var index: Int = 0 + private var valuesRead = 0 var isAtEnd: Bool { index >= bytes.count } mutating func parseValue(depth: Int) -> PhpValue? { - guard depth <= PhpSerializeParser.depthCap else { return .depthExceeded } - guard index < bytes.count else { return nil } + guard depth <= PhpSerializeParser.depthCap, index < bytes.count else { return nil } + valuesRead += 1 + if valuesRead.isMultiple(of: Self.cancellationInterval), Task.isCancelled { return nil } let token = bytes[index] switch token { case UInt8(ascii: "N"): return parseNull() @@ -197,13 +200,15 @@ private struct Cursor { guard let count = Int(countRaw), count >= 0 else { return nil } guard expect(UInt8(ascii: ":")), expect(UInt8(ascii: "{")) else { return nil } - // A forged count would otherwise reserve it. guard count <= bytes.count - index else { return nil } var entries: [PhpKeyValue] = [] - entries.reserveCapacity(count) + entries.reserveCapacity(min(count, Self.reservationLimit)) for _ in 0.. start else { return nil } entries.append(PhpKeyValue(key: key, value: value)) } guard expect(UInt8(ascii: "}")) else { return nil } @@ -226,10 +231,11 @@ private struct Cursor { guard count <= bytes.count - index else { return nil } var properties: [PhpProperty] = [] - properties.reserveCapacity(count) + properties.reserveCapacity(min(count, Self.reservationLimit)) for _ in 0.. start else { return nil } switch key { case .string(let rawKey)?: let decoded = decodePhpPropertyKey(rawKey) diff --git a/apple/TablePro/Core/Services/ProjectImport/DockerComposeExtractor.swift b/apple/TablePro/Core/Services/ProjectImport/DockerComposeExtractor.swift index 67626be4f7..ce35a84ed3 100644 --- a/apple/TablePro/Core/Services/ProjectImport/DockerComposeExtractor.swift +++ b/apple/TablePro/Core/Services/ProjectImport/DockerComposeExtractor.swift @@ -26,10 +26,11 @@ enum DockerComposeExtractor { static func extract( contents: String, relativePath: String, - environment: DotenvDocument? + environment: DotenvDocument?, + budget: ProjectScanExpansionBudget = ProjectScanExpansionBudget() ) -> [ScannedConnectionCandidate] { - guard let parsed = YamlMappingSupport.loadMapping(contents), - let root = ComposeInterpolator.interpolate(parsed, environment: environment) as? [String: Any], + guard let parsed = YamlMappingSupport.loadMapping(contents, budget: budget), + let root = ComposeInterpolator.interpolate(parsed, environment: environment, budget: budget) as? [String: Any], let services = YamlMappingSupport.mapping(root["services"]) else { return [] } @@ -296,20 +297,30 @@ enum DockerComposeExtractor { } enum ComposeInterpolator { - static func interpolate(_ value: Any, environment: DotenvDocument?) -> Any { + private enum Substitution { + case value(String) + case missing + case overBudget + } + + static func interpolate(_ value: Any, environment: DotenvDocument?, budget: ProjectScanExpansionBudget) -> Any { if let text = value as? String { - return interpolate(text: text, environment: environment) + return interpolate(text: text, environment: environment, budget: budget) } if let mapping = value as? [String: Any] { - return mapping.mapValues { interpolate($0, environment: environment) } + return mapping.mapValues { interpolate($0, environment: environment, budget: budget) } } if let list = value as? [Any] { - return list.map { interpolate($0, environment: environment) } + return list.map { interpolate($0, environment: environment, budget: budget) } } return value } - static func interpolate(text contents: String, environment: DotenvDocument?) -> String { + static func interpolate( + text contents: String, + environment: DotenvDocument?, + budget: ProjectScanExpansionBudget + ) -> String { guard contents.contains("$") else { return contents } @@ -323,14 +334,18 @@ enum ComposeInterpolator { return result } let reference = String(afterOpen[afterOpen.startIndex.. String { + private static func resolve( + _ reference: String, + environment: DotenvDocument?, + budget: ProjectScanExpansionBudget + ) -> String { let separators = [":-", ":?", "-", "?"] for separator in separators { guard let range = reference.range(of: separator) else { @@ -341,12 +356,34 @@ enum ComposeInterpolator { guard !name.isEmpty else { continue } - if let value = environment?[name] { + switch substitution(for: name, environment: environment, budget: budget) { + case .value(let value): return value + case .overBudget: + return unresolvedMarker(reference) + case .missing: + return separator.hasSuffix("?") ? unresolvedMarker(reference) : fallback } - return separator.hasSuffix("?") ? unresolvedMarker(reference) : fallback } - return environment?[reference] ?? unresolvedMarker(reference) + guard case .value(let value) = substitution(for: reference, environment: environment, budget: budget) else { + return unresolvedMarker(reference) + } + return value + } + + // Charged at the stored length before the subscript trims it, so each trim of a long value is paid for too. + private static func substitution( + for name: String, + environment: DotenvDocument?, + budget: ProjectScanExpansionBudget + ) -> Substitution { + guard let environment, let entry = environment.entry(for: name), !entry.hasUnresolvedReference else { + return .missing + } + guard budget.spend(entry.value.utf8.count) else { + return .overBudget + } + return environment[name].map(Substitution.value) ?? .missing } static func isUnresolved(_ value: String) -> Bool { diff --git a/apple/TablePro/Core/Services/ProjectImport/DotenvParser+Interpolation.swift b/apple/TablePro/Core/Services/ProjectImport/DotenvParser+Interpolation.swift index 2439ac3b6c..a27d073299 100644 --- a/apple/TablePro/Core/Services/ProjectImport/DotenvParser+Interpolation.swift +++ b/apple/TablePro/Core/Services/ProjectImport/DotenvParser+Interpolation.swift @@ -14,7 +14,8 @@ extension DotenvParser { static func resolve( _ assignment: DotenvAssignment, document: DotenvDocument, - processEnvironment: [String: String] + processEnvironment: [String: String], + budget: ProjectScanExpansionBudget ) -> DotenvEntry { guard assignment.quoting != .single else { return DotenvEntry( @@ -39,9 +40,11 @@ extension DotenvParser { index += length unresolved = true case .valid(let name, let defaultValue, let length): - if let resolved = lookUp(name, document: document, processEnvironment: processEnvironment) { + let resolved = lookUp(name, document: document, processEnvironment: processEnvironment) + // A reference over the budget stays literal and unresolved, so the lines built on it stop growing too. + if let resolved, budget.spend(resolved.utf8.count) { value.append(resolved) - } else if let defaultValue { + } else if resolved == nil, let defaultValue { value.append(defaultValue) } else { value.append(literal(characters, from: index, length: length)) diff --git a/apple/TablePro/Core/Services/ProjectImport/DotenvParser.swift b/apple/TablePro/Core/Services/ProjectImport/DotenvParser.swift index c2195004ef..c17e896b39 100644 --- a/apple/TablePro/Core/Services/ProjectImport/DotenvParser.swift +++ b/apple/TablePro/Core/Services/ProjectImport/DotenvParser.swift @@ -62,7 +62,8 @@ enum DotenvParser { static func parse( _ contents: String, - processEnvironment: [String: String] + processEnvironment: [String: String], + budget: ProjectScanExpansionBudget = ProjectScanExpansionBudget() ) -> DotenvDocument { var document = DotenvDocument() let characters = Array(normalize(contents)) @@ -75,7 +76,12 @@ enum DotenvParser { guard let assignment = readAssignment(characters, &index) else { continue } - document.upsert(resolve(assignment, document: document, processEnvironment: processEnvironment)) + document.upsert(resolve( + assignment, + document: document, + processEnvironment: processEnvironment, + budget: budget + )) } return document } diff --git a/apple/TablePro/Core/Services/ProjectImport/ProjectFolderFileWalker.swift b/apple/TablePro/Core/Services/ProjectImport/ProjectFolderFileWalker.swift index bdd76fb9ff..8073c2fcf7 100644 --- a/apple/TablePro/Core/Services/ProjectImport/ProjectFolderFileWalker.swift +++ b/apple/TablePro/Core/Services/ProjectImport/ProjectFolderFileWalker.swift @@ -58,7 +58,7 @@ enum ProjectFolderFileWalker { var visited = 0 for case let url as URL in enumerator { visited += 1 - guard visited <= maxVisitedEntries else { + guard visited <= maxVisitedEntries, !Task.isCancelled else { break } guard let values = try? url.resourceValues(forKeys: Set(keys)) else { diff --git a/apple/TablePro/Core/Services/ProjectImport/ProjectFolderScanner.swift b/apple/TablePro/Core/Services/ProjectImport/ProjectFolderScanner.swift index 854e21ec4d..a201c0e7bf 100644 --- a/apple/TablePro/Core/Services/ProjectImport/ProjectFolderScanner.swift +++ b/apple/TablePro/Core/Services/ProjectImport/ProjectFolderScanner.swift @@ -34,16 +34,25 @@ enum ProjectFolderScanner { return .failure(.rootUnreadable) } let files = ProjectFolderFileWalker.walk(root: rootURL, fileManager: fileManager) - let index = ProjectScanEnvironmentIndex(files: files, rootURL: rootURL) + let budget = ProjectScanExpansionBudget() + let index = ProjectScanEnvironmentIndex(files: files, rootURL: rootURL, budget: budget) var candidates: [ScannedConnectionCandidate] = [] for file in files { + guard !Task.isCancelled else { + break + } candidates.append(contentsOf: extract( file: file, rootURL: rootURL, index: index, + budget: budget, fileManager: fileManager )) } + // Nothing waits for a cancelled scan, so it offers no partial result. + guard !Task.isCancelled else { + return .success(ProjectFolderScanResult(candidates: [], scannedFileCount: files.count)) + } candidates.append(contentsOf: parentWordPressCandidates(rootURL: rootURL, fileManager: fileManager)) let ranked = ScannedConnectionRanking.rankAndDeduplicate(candidates) logger.info( @@ -56,6 +65,7 @@ enum ProjectFolderScanner { file: ProjectScanFile, rootURL: URL, index: ProjectScanEnvironmentIndex, + budget: ProjectScanExpansionBudget, fileManager: FileManager ) -> [ScannedConnectionCandidate] { let directoryURL = file.url.deletingLastPathComponent() @@ -103,7 +113,8 @@ enum ProjectFolderScanner { return SpringYamlExtractor.extract( contents: contents, relativePath: file.relativePath, - processEnvironment: [:] + processEnvironment: [:], + budget: budget ) case .appSettingsJson: guard let data = ProjectFileReader.data(at: file.url) else { @@ -118,7 +129,8 @@ enum ProjectFolderScanner { contents: contents, relativePath: file.relativePath, projectRootURL: rootURL, - environment: index.environment(near: file.url) + environment: index.environment(near: file.url), + budget: budget ) case .dockerCompose: guard let contents = ProjectFileReader.text(at: file.url) else { @@ -127,7 +139,8 @@ enum ProjectFolderScanner { return DockerComposeExtractor.extract( contents: contents, relativePath: file.relativePath, - environment: index.environment(near: file.url) + environment: index.environment(near: file.url), + budget: budget ) } } diff --git a/apple/TablePro/Core/Services/ProjectImport/ProjectScanEnvironmentIndex.swift b/apple/TablePro/Core/Services/ProjectImport/ProjectScanEnvironmentIndex.swift index ee42e7bd7f..aee6a76ad3 100644 --- a/apple/TablePro/Core/Services/ProjectImport/ProjectScanEnvironmentIndex.swift +++ b/apple/TablePro/Core/Services/ProjectImport/ProjectScanEnvironmentIndex.swift @@ -23,14 +23,17 @@ struct ProjectScanEnvironmentIndex { private var documentsByDirectory: [String: DotenvDocument] = [:] private var rootDocument: DotenvDocument? - init(files: [ProjectScanFile], rootURL: URL) { + init(files: [ProjectScanFile], rootURL: URL, budget: ProjectScanExpansionBudget) { let rootPath = rootURL.standardizedFileURL.path for file in files where file.match.kind == .dotenv { + guard !Task.isCancelled else { + return + } guard let contents = ProjectFileReader.text(at: file.url) else { continue } // The folder may be someone else's, so a reference it does not define stays unresolved. - let document = DotenvParser.parse(contents, processEnvironment: [:]) + let document = DotenvParser.parse(contents, processEnvironment: [:], budget: budget) let filePath = file.url.standardizedFileURL.path let directory = file.url.deletingLastPathComponent().standardizedFileURL.path documentsByFile[filePath] = document diff --git a/apple/TablePro/Core/Services/ProjectImport/ProjectScanExpansionBudget.swift b/apple/TablePro/Core/Services/ProjectImport/ProjectScanExpansionBudget.swift new file mode 100644 index 0000000000..a2540f7d49 --- /dev/null +++ b/apple/TablePro/Core/Services/ProjectImport/ProjectScanExpansionBudget.swift @@ -0,0 +1,24 @@ +// +// ProjectScanExpansionBudget.swift +// TablePro +// + +import Foundation + +// What dotenv references, compose substitutions and YAML aliases may add to a folder scan beyond the bytes of its +// files. Each of them can turn a few hundred bytes into gigabytes, so one scan shares one allowance. +final class ProjectScanExpansionBudget { + static let standardLimit = 1 << 20 + + private(set) var remaining: Int + + init(limit: Int = standardLimit) { + remaining = max(limit, 0) + } + + func spend(_ amount: Int) -> Bool { + guard amount <= remaining else { return false } + remaining -= max(amount, 0) + return true + } +} diff --git a/apple/TablePro/Core/Services/ProjectImport/RailsDatabaseYamlExtractor.swift b/apple/TablePro/Core/Services/ProjectImport/RailsDatabaseYamlExtractor.swift index d5750e50f7..2229c3da80 100644 --- a/apple/TablePro/Core/Services/ProjectImport/RailsDatabaseYamlExtractor.swift +++ b/apple/TablePro/Core/Services/ProjectImport/RailsDatabaseYamlExtractor.swift @@ -12,9 +12,10 @@ enum RailsDatabaseYamlExtractor { contents: String, relativePath: String, projectRootURL: URL, - environment: DotenvDocument? + environment: DotenvDocument?, + budget: ProjectScanExpansionBudget = ProjectScanExpansionBudget() ) -> [ScannedConnectionCandidate] { - guard let root = YamlMappingSupport.loadMapping(contents) else { + guard let root = YamlMappingSupport.loadMapping(contents, budget: budget) else { return [] } guard let selected = selectEnvironment(root) else { diff --git a/apple/TablePro/Core/Services/ProjectImport/SpringYamlExtractor.swift b/apple/TablePro/Core/Services/ProjectImport/SpringYamlExtractor.swift index 13d385f1a1..6fa22a3980 100644 --- a/apple/TablePro/Core/Services/ProjectImport/SpringYamlExtractor.swift +++ b/apple/TablePro/Core/Services/ProjectImport/SpringYamlExtractor.swift @@ -9,9 +9,10 @@ enum SpringYamlExtractor { static func extract( contents: String, relativePath: String, - processEnvironment: [String: String] + processEnvironment: [String: String], + budget: ProjectScanExpansionBudget = ProjectScanExpansionBudget() ) -> [ScannedConnectionCandidate] { - for document in YamlMappingSupport.loadDocuments(contents) { + for document in YamlMappingSupport.loadDocuments(contents, budget: budget) { guard let rawURL = value(in: document, path: ["spring", "datasource", "url"]) else { continue } diff --git a/apple/TablePro/Core/Services/ProjectImport/YamlMappingSupport.swift b/apple/TablePro/Core/Services/ProjectImport/YamlMappingSupport.swift index 9544474536..94d72c7abc 100644 --- a/apple/TablePro/Core/Services/ProjectImport/YamlMappingSupport.swift +++ b/apple/TablePro/Core/Services/ProjectImport/YamlMappingSupport.swift @@ -7,21 +7,105 @@ import Foundation import Yams enum YamlMappingSupport { - static func loadMapping(_ contents: String) -> [String: Any]? { - guard let object = try? Yams.load(yaml: contents) else { - return nil + // The readers below walk loaded values one call per level on the scan's 512 KiB worker stack. No config file + // nests near this. + static let maximumNesting = 128 + + static func loadMapping( + _ contents: String, + budget: ProjectScanExpansionBudget = ProjectScanExpansionBudget() + ) -> [String: Any]? { + nonisolated(unsafe) let budget = budget + return onComposerThread { + guard let parser = try? Yams.Parser(yaml: contents), + let node = try? parser.singleRoot(), + isWithinLimits(node, of: parser, budget: budget), + let mapping = node.any as? [String: Any] else { + return nil + } + return expandingMergeKeys(mapping) } - guard let mapping = object as? [String: Any] else { - return nil + } + + static func loadDocuments( + _ contents: String, + budget: ProjectScanExpansionBudget = ProjectScanExpansionBudget() + ) -> [[String: Any]] { + nonisolated(unsafe) let budget = budget + return onComposerThread { + guard let parser = try? Yams.Parser(yaml: contents) else { + return [] + } + var documents: [[String: Any]] = [] + while let node = try? parser.nextRoot() { + guard isWithinLimits(node, of: parser, budget: budget), let mapping = node.any as? [String: Any] else { + continue + } + documents.append(expandingMergeKeys(mapping)) + } + return documents + } ?? [] + } + + // Yams composes one call per level. libyaml allows 1,000 levels, and a 512 KiB worker stack runs out a few + // hundred levels down. + private static func onComposerThread(_ body: @escaping @Sendable () -> Value?) -> Value? { + nonisolated(unsafe) var value: Value? + let finished = DispatchSemaphore(value: 0) + let thread = Thread { + value = body() + finished.signal() } - return expandingMergeKeys(mapping) + thread.stackSize = 8 << 20 + thread.start() + finished.wait() + return value } - static func loadDocuments(_ contents: String) -> [[String: Any]] { - guard let objects = try? Yams.load_all(yaml: contents) else { - return [] + // Yams shares an anchored node with every alias to it, so nested aliases cost nothing to compose and grow + // exponentially once built into values. Every node reached again through an alias is charged before building. + // `Node.anchor` is weak and only the parser holds the anchors, so the parser stays alive for the walk. + static func isWithinLimits(_ root: Node, of parser: Yams.Parser, budget: ProjectScanExpansionBudget) -> Bool { + withExtendedLifetime(parser) { + var anchors: Set = [] + var pending: [(node: Node, nesting: Int, isAliased: Bool)] = [(root, 0, false)] + while let entry = pending.popLast() { + let nesting = entry.nesting + var isAliased = entry.isAliased + if !isAliased, let anchor = entry.node.anchor { + isAliased = !anchors.insert(ObjectIdentifier(anchor)).inserted + } + switch entry.node { + case .scalar(let scalar): + guard !isAliased || budget.spend(max(scalar.string.utf8.count, 1)) else { return false } + guard !overflowsSexagesimal(scalar) else { return false } + case .sequence(let sequence): + guard nesting < maximumNesting, !isAliased || budget.spend(1) else { return false } + for child in sequence.reversed() { + pending.append((child, nesting + 1, isAliased)) + } + case .mapping(let mapping): + guard nesting < maximumNesting, !isAliased || budget.spend(1) else { return false } + for (key, value) in mapping.reversed() { + // Yams' constructor force-unwraps a key that is not a scalar. + guard case .scalar = key else { return false } + pending.append((value, nesting + 1, isAliased)) + pending.append((key, nesting + 1, isAliased)) + } + case .alias: + // Yams resolves aliases while composing; the constructor traps on one left in the tree. + return false + } + } + return true } - return objects.compactMap { $0 as? [String: Any] }.map(expandingMergeKeys) + } + + // YAML 1.1 reads `1:30` as the base-60 integer 90, and Yams' constructor traps when one overflows Int. At most + // 18 characters it cannot. + private static func overflowsSexagesimal(_ scalar: Node.Scalar) -> Bool { + guard scalar.string.utf8.count > 18, scalar.string.contains(":") else { return false } + return Node.scalar(scalar).tag == Tag(.int) } static func expandingMergeKeys(_ mapping: [String: Any]) -> [String: Any] { diff --git a/apple/TablePro/Core/Utilities/File/FileDecompressor.swift b/apple/TablePro/Core/Utilities/File/FileDecompressor.swift index c67bcf1b7d..1575d7d57e 100644 --- a/apple/TablePro/Core/Utilities/File/FileDecompressor.swift +++ b/apple/TablePro/Core/Utilities/File/FileDecompressor.swift @@ -2,11 +2,9 @@ // FileDecompressor.swift // TablePro // -// Utility for decompressing .gz files using system gunzip command. -// import Foundation -import TableProUtilities +import TableProGzip enum DecompressionError: LocalizedError { case decompressFailed @@ -22,42 +20,23 @@ enum DecompressionError: LocalizedError { } } -/// Utility for decompressing gzip-compressed files enum FileDecompressor { - /// Derive the inner extension from a .gz filename (e.g., "dump.sql.gz" -> "sql") private static func innerExtension(for url: URL) -> String { let name = url.deletingPathExtension().pathExtension.lowercased() return name.isEmpty ? "sql" : name } - /// Case-folded, because the file panel accepts `DUMP.SQL.GZ` as readily as `dump.sql.gz` and a - /// compressed file that slips past this reaches the statement parser still gzipped. + // The file panel accepts DUMP.SQL.GZ too, and a missed one reaches the statement parser still gzipped. static func isCompressed(_ url: URL) -> Bool { url.pathExtension.lowercased() == ImportFileFormatResolver.compressedFileExtension } - /// Decompress a .gz file to a temporary location - /// - Parameters: - /// - url: URL to the .gz file - /// - fileSystemPath: Helper function to get filesystem path for URL - /// - Returns: URL to the decompressed temporary file, or original URL if not compressed - /// - Throws: DecompressionError or GzipProcess.GzipError if decompression fails - static func decompressIfNeeded( - _ url: URL, - fileSystemPath: (URL) -> String - ) async throws -> URL { + static func decompressIfNeeded(_ url: URL) async throws -> URL { guard isCompressed(url) else { return url } - let ext = innerExtension(for: url) let tempURL = FileManager.default.temporaryDirectory - .appendingPathComponent(UUID().uuidString + "." + ext) - - do { - try await GzipProcess.decompress(source: url, destination: tempURL) - } catch { - throw DecompressionError.fileReadFailed(error.localizedDescription) - } - + .appendingPathComponent(UUID().uuidString + "." + innerExtension(for: url)) + try await GzipFile.decompress(source: url, destination: tempURL) return tempURL } } diff --git a/apple/TablePro/Models/UI/PhpTreeNode.swift b/apple/TablePro/Models/UI/PhpTreeNode.swift index 14e168c3c5..399082e9e3 100644 --- a/apple/TablePro/Models/UI/PhpTreeNode.swift +++ b/apple/TablePro/Models/UI/PhpTreeNode.swift @@ -298,12 +298,6 @@ internal enum PhpTreeBuilder { visibilityBadge: badge ) - case .depthExceeded: - return PhpTreeNode( - key: key, keyPath: keyPath, path: path, nodeType: .truncated, - displayValue: String(localized: "Maximum depth reached"), - visibilityBadge: badge - ) case .tooLarge: return PhpTreeNode( diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index f189097bee..f09788de2e 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -14953,6 +14953,10 @@ } } }, + "A %@ batch decompresses to far more data than the broker sent, so it was not read." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A %@ index has no equivalent on this engine." : { "localizations" : { "fr" : { @@ -17315,6 +17319,9 @@ } } } + }, + "A type name is too long." : { + }, "A user or role with this name already exists." : { "localizations" : { @@ -148224,6 +148231,10 @@ } } }, + "Reply nested too deeply to show" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Report" : { "extractionState" : "manual", "generatesSymbol" : false @@ -185202,6 +185213,10 @@ } } }, + "The compressed data expands to far more than it holds, so it was not read." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "the compressed data is corrupt or was cut short" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -186312,6 +186327,10 @@ } } }, + "The decompressed file needs more free disk space than this Mac has." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The default %@ is not written in a form this engine shares." : { "localizations" : { "fr" : { @@ -187426,6 +187445,14 @@ }, "The group end at byte %lld matches no group start." : { + }, + "The gzip data ends before its last block." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The gzip data is corrupt." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The header names version %lld, which this format does not have." : { @@ -188756,6 +188783,9 @@ } } } + }, + "The package name has too many parts." : { + }, "The pad directive carries no date, so its accounts were not projected." : { "extractionState" : "manual", @@ -190102,6 +190132,10 @@ } } }, + "the result has far more columns than its rows fill" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The right result stopped at the row cap and is compared on the %@ rows it holds. A row past the cap can show as only on the left." : { }, diff --git a/apple/TablePro/Views/Connection/ProjectFolder/ProjectFolderScanSheet.swift b/apple/TablePro/Views/Connection/ProjectFolder/ProjectFolderScanSheet.swift index 193859cfda..fc0329f0e8 100644 --- a/apple/TablePro/Views/Connection/ProjectFolder/ProjectFolderScanSheet.swift +++ b/apple/TablePro/Views/Connection/ProjectFolder/ProjectFolderScanSheet.swift @@ -151,9 +151,16 @@ struct ProjectFolderScanSheet: View { step = .scanning selection = nil let url = rootURL - let result = await Task.detached(priority: .userInitiated) { + let scan = Task.detached(priority: .userInitiated) { ProjectFolderScanner.scan(rootURL: url) - }.value + } + // Closing the sheet or picking another folder stops the scan instead of leaving it running. + let result = await withTaskCancellationHandler { + await scan.value + } onCancel: { + scan.cancel() + } + guard !Task.isCancelled else { return } switch result { case .success(let scan): step = scan.candidates.isEmpty ? .empty : .results(scan.candidates) diff --git a/apple/TablePro/Views/Import/ImportDialog.swift b/apple/TablePro/Views/Import/ImportDialog.swift index 7763ff9013..60b2d4c8fb 100644 --- a/apple/TablePro/Views/Import/ImportDialog.swift +++ b/apple/TablePro/Views/Import/ImportDialog.swift @@ -411,10 +411,12 @@ struct ImportDialog: View { let urlToRead: URL do { - urlToRead = try await decompressIfNeeded(url) + urlToRead = try await FileDecompressor.decompressIfNeeded(url) if urlToRead != url { tempPreviewURL = urlToRead } + } catch is CancellationError { + return } catch { filePreview = String(format: String(localized: "Failed to decompress file: %@"), error.localizedDescription) hasPreviewError = true @@ -570,12 +572,4 @@ struct ImportDialog: View { tempPreviewURL = nil } } - - private func fileSystemPath(for url: URL) -> String { - url.path() - } - - private func decompressIfNeeded(_ url: URL) async throws -> URL { - try await FileDecompressor.decompressIfNeeded(url, fileSystemPath: fileSystemPath) - } } diff --git a/apple/TablePro/Views/Results/PhpViewerView.swift b/apple/TablePro/Views/Results/PhpViewerView.swift index df7030b9fb..16919815ec 100644 --- a/apple/TablePro/Views/Results/PhpViewerView.swift +++ b/apple/TablePro/Views/Results/PhpViewerView.swift @@ -157,14 +157,21 @@ internal struct PhpViewerView: View { private func loadParse() async { parseResult = .parsing let raw = rawValue - let parsed = await Task.detached(priority: .userInitiated) { () -> PhpParseResult in + let work = Task.detached(priority: .userInitiated) { () -> PhpParseResult in if (raw as NSString).length > PhpSerializeParser.sizeCapBytes { return .tooLarge } - guard let value = PhpSerializeParser.parse(raw) else { return .failed } + guard let value = PhpSerializeParser.parse(raw), !Task.isCancelled else { return .failed } let tree = PhpTreeBuilder.build(from: value) return .parsed(tree) - }.value + } + let parsed = await withTaskCancellationHandler { + await work.value + } onCancel: { + work.cancel() + } + // A parse cancelled with its view leaves no result, so the view parses again when it returns. + guard !Task.isCancelled else { return } parseResult = parsed } } diff --git a/apple/TableProMobile/TableProMobile.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/apple/TableProMobile/TableProMobile.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 638336afe0..fce127506d 100644 --- a/apple/TableProMobile/TableProMobile.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/apple/TableProMobile/TableProMobile.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "fa5148407a13f55422d2128e0ec0fd6c4d156ebce806581c5ca4377fbaa42305", + "originHash" : "fcb1fbfb1cf41d99fb533f3d60a346045e6e48c2906e97b28b4f53ce4285e6f8", "pins" : [ { "identity" : "bigint", @@ -143,6 +143,15 @@ "revision" : "704705c5c51156ede21172a38654d522ce487074", "version" : "1.8.0" } + }, + { + "identity" : "zstd", + "kind" : "remoteSourceControl", + "location" : "https://github.com/facebook/zstd", + "state" : { + "revision" : "f8745da6ff1ad1e7bab384bd1f9d742439278e99", + "version" : "1.5.7" + } } ], "version" : 3 diff --git a/apple/TableProMobile/TableProMobile/Drivers/RedisDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/RedisDriver.swift index 5325809ee0..ba5713ea0d 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/RedisDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/RedisDriver.swift @@ -206,7 +206,6 @@ private actor RedisActor { database: Int, ssl: DriverSSLConfiguration ) throws { - // Close existing connection if reconnecting close() guard let portI32 = Int32(exactly: port), (1...65_535).contains(port) else { @@ -218,6 +217,7 @@ private actor RedisActor { guard let context = redisConnectWithTimeout(host, portI32, tv) else { throw RedisError.connectionFailed("Failed to create Redis context") } + HiredisReply.installIterativeFree(on: context) if context.pointee.err != 0 { let msg = withUnsafePointer(to: &context.pointee.errstr.0) { String(cString: $0) } @@ -317,55 +317,46 @@ private actor RedisActor { throw RedisError.connectionFailed("No reply from server") } - let reply = rawReply.assumingMemoryBound(to: redisReply.self) - defer { freeReplyObject(rawReply) } + defer { HiredisReply.release(rawReply) } - return parseReply(reply) + return HiredisReply.value(of: rawReply.assumingMemoryBound(to: redisReply.self)) } +} - // Read by length: a bulk string can hold a NUL. - private nonisolated static func text(of reply: UnsafeMutablePointer) -> String? { - guard let str = reply.pointee.str else { return nil } - return String(decoding: UnsafeRawBufferPointer(start: str, count: reply.pointee.len), as: UTF8.self) - } +// A server decides how deep a reply nests, a few bytes a level. +nonisolated enum HiredisReply { + // Deeper than any command or module replies. Past it a reply is not built, so nothing that walks one later + // runs out of stack. + static let maximumDepth = 128 - private func parseReply(_ reply: UnsafeMutablePointer) -> RedisReplyValue { + static func value(of reply: UnsafeMutablePointer, depth: Int = 0) -> RedisReplyValue { switch reply.pointee.type { case REDIS_REPLY_STRING: - if let text = Self.text(of: reply) { - return .string(text) - } - return .null + return text(of: reply).map(RedisReplyValue.string) ?? .null case REDIS_REPLY_INTEGER: return .integer(reply.pointee.integer) case REDIS_REPLY_ARRAY: + guard depth < maximumDepth else { + return .error(String(localized: "Reply nested too deeply to show")) + } let count = reply.pointee.elements guard count > 0, let elements = reply.pointee.element else { return .array([]) } var items: [RedisReplyValue] = [] - for i in 0..) { + if let functions, let reader = context.pointee.reader { + reader.pointee.fn = functions + } + redisSetPushCallback(context) { _, reply in + if let reply { HiredisReply.release(reply) } + } + } + + // A reader keeps this pointer for its whole life, so the table is built once and never written again. + nonisolated(unsafe) private static let functions: UnsafeMutablePointer? = { + guard let reader = redisReaderCreate() else { return nil } + defer { redisReaderFree(reader) } + guard let defaults = reader.pointee.fn else { return nil } + let table = UnsafeMutablePointer.allocate(capacity: 1) + table.initialize(to: defaults.pointee) + table.pointee.freeObject = { reply in + if let reply { HiredisReply.release(reply) } + } + return table + }() + + // Read by length: a bulk string can hold a NUL. + private static func text(of reply: UnsafeMutablePointer) -> String? { + guard let str = reply.pointee.str else { return nil } + return String(decoding: UnsafeRawBufferPointer(start: str, count: reply.pointee.len), as: UTF8.self) + } } // MARK: - Errors diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index 7d6a1d957a..5d0684efba 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -18889,6 +18889,9 @@ }, "Replica Set" : { + }, + "Reply nested too deeply to show" : { + }, "Report" : { "extractionState" : "manual", diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/HiredisReplyTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/HiredisReplyTests.swift new file mode 100644 index 0000000000..853bf56b89 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/HiredisReplyTests.swift @@ -0,0 +1,132 @@ +import CRedis +import Foundation +@testable import TableProMobile +import Testing + +struct HiredisReplyTests { + private static func convert(_ wire: String) throws -> RedisReplyValue { + let reader = try #require(redisReaderCreate()) + defer { redisReaderFree(reader) } + #expect(redisReaderFeed(reader, wire, wire.utf8.count) == REDIS_OK) + var raw: UnsafeMutableRawPointer? + #expect(redisReaderGetReply(reader, &raw) == REDIS_OK) + let reply = try #require(raw) + defer { HiredisReply.release(reply) } + return HiredisReply.value(of: reply.assumingMemoryBound(to: redisReply.self)) + } + + private static func firstElementChain(_ reply: RedisReplyValue) -> (depth: Int, innermost: RedisReplyValue) { + var depth = 0 + var current = reply + while case .array(let items) = current, let first = items.first { + depth += 1 + current = first + } + return (depth, current) + } + + @Test("A reply nested deeper than the cap ends in a placeholder and is freed without recursing per level") + func deeplyNestedReply() throws { + let reply = try Self.convert(String(repeating: "*1\r\n", count: 20_000) + ":1\r\n") + let chain = Self.firstElementChain(reply) + #expect(chain.depth == HiredisReply.maximumDepth) + #expect(chain.innermost.errorMessage != nil) + } + + @Test("A reply nested just inside the cap reads in full") + func replyInsideTheCap() throws { + let depth = HiredisReply.maximumDepth - 1 + let reply = try Self.convert(String(repeating: "*1\r\n", count: depth) + ":7\r\n") + let chain = Self.firstElementChain(reply) + #expect(chain.depth == depth) + #expect(chain.innermost == .integer(7)) + } + + @Test("Flat replies read as before") + func flatReplies() throws { + #expect(try Self.convert("*3\r\n$3\r\nfoo\r\n:42\r\n$-1\r\n") == .array([.string("foo"), .integer(42), .null])) + #expect(try Self.convert("+OK\r\n") == .status("OK")) + #expect(try Self.convert("-ERR boom\r\n") == .error("ERR boom")) + #expect(try Self.convert("*0\r\n") == .array([])) + } + + private struct Outcome: Sendable { + let status: Int32 + let error: Int32 + let reply: RedisReplyValue? + } + + private static let deepNesting = String(repeating: "*1\r\n", count: 20_000) + + // 512 KiB is a Dispatch worker's stack, which a free that recurses once per level overflows at this depth. + private static func onSmallStack( + fed wire: String, + _ read: @escaping @Sendable (UnsafeMutablePointer) -> Outcome + ) async -> Outcome? { + await withCheckedContinuation { continuation in + let thread = Thread { + continuation.resume(returning: Self.withConnection(fed: wire, read)) + } + thread.stackSize = 512 * 1_024 + thread.start() + } + } + + private static func withConnection( + fed wire: String, + _ read: (UnsafeMutablePointer) -> Outcome + ) -> Outcome? { + var sockets: [Int32] = [-1, -1] + guard socketpair(AF_UNIX, SOCK_STREAM, 0, &sockets) == 0 else { return nil } + // A closed peer turns any socket read into an error rather than a wait. + close(sockets[1]) + guard let context = redisConnectFd(sockets[0]) else { + close(sockets[0]) + return nil + } + defer { redisFree(context) } + HiredisReply.installIterativeFree(on: context) + guard redisReaderFeed(context.pointee.reader, wire, wire.utf8.count) == REDIS_OK else { return nil } + return read(context) + } + + @Test("A protocol error under a deeply nested partial reply fails without recursing per level") + func protocolErrorUnderDeepReply() async throws { + let outcome = await Self.onSmallStack(fed: Self.deepNesting + "@\r\n") { context in + var raw: UnsafeMutableRawPointer? + let status = redisReaderGetReply(context.pointee.reader, &raw) + return Outcome(status: status, error: context.pointee.reader.pointee.err, reply: nil) + } + let read = try #require(outcome) + #expect(read.status == REDIS_ERR) + #expect(read.error == REDIS_ERR_PROTOCOL) + } + + @Test("Freeing a connection that holds a deeply nested partial reply does not recurse per level") + func freeWithDeepPartialReply() async throws { + let outcome = await Self.onSmallStack(fed: Self.deepNesting) { context in + var raw: UnsafeMutableRawPointer? + let status = redisReaderGetReply(context.pointee.reader, &raw) + return Outcome(status: status, error: context.pointee.reader.pointee.err, reply: raw.map { _ in RedisReplyValue.null }) + } + let read = try #require(outcome) + #expect(read.status == REDIS_OK) + #expect(read.reply == nil) + } + + @Test("A deeply nested RESP3 push hiredis drops is freed without recursing per level") + func droppedDeepPush() async throws { + let outcome = await Self.onSmallStack(fed: ">1\r\n" + Self.deepNesting + ":1\r\n+OK\r\n") { context in + var raw: UnsafeMutableRawPointer? + let status = redisGetReply(context, &raw) + let value = raw.map { reply in + defer { HiredisReply.release(reply) } + return HiredisReply.value(of: reply.assumingMemoryBound(to: redisReply.self)) + } + return Outcome(status: status, error: context.pointee.err, reply: value) + } + let read = try #require(outcome) + #expect(read.status == REDIS_OK) + #expect(read.reply == .status("OK")) + } +} diff --git a/apple/TableProTests/Core/DataFiles/DataFileColumnNamesTests.swift b/apple/TableProTests/Core/DataFiles/DataFileColumnNamesTests.swift new file mode 100644 index 0000000000..0a49c203e8 --- /dev/null +++ b/apple/TableProTests/Core/DataFiles/DataFileColumnNamesTests.swift @@ -0,0 +1,31 @@ +import Foundation +@testable import TablePro +import TableProTabular +import Testing + +private func columns(named names: [String]) -> [TabularColumn] { + names.enumerated().map { index, name in + TabularColumn(id: TabularColumnID(rawValue: index), name: name, values: ColumnValues(base: .source(index))) + } +} + +struct DataFileColumnNamesTests { + @Test func repeatedHeadersGetNumberedNames() { + let names = DataFileColumnNames(columns: columns(named: ["id", "id", "", "id (2)", "id"])) + #expect(names.displayNames == ["id", "id (2)", "Column 3", "id (2) (2)", "id (3)"]) + #expect(names.id(forName: "id (3)") == TabularColumnID(rawValue: 4)) + #expect(names.name(for: TabularColumnID(rawValue: 3)) == "id (2) (2)") + #expect(names.index(of: TabularColumnID(rawValue: 2)) == 2) + } + + // A header repeating `id` 40,000 times froze the main actor for minutes. A quadratic loop takes over 20 minutes on 200,000. + @Test(.timeLimit(.minutes(1))) + func aHeaderOfOneRepeatedNameNamesEveryColumnQuickly() { + let count = 200_000 + let names = DataFileColumnNames(columns: columns(named: Array(repeating: "id", count: count))) + #expect(names.count == count) + #expect(names.displayNames.last == "id (\(count))") + #expect(Set(names.displayNames).count == count) + #expect(names.index(of: TabularColumnID(rawValue: count - 1)) == count - 1) + } +} diff --git a/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift b/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift index e1eb618f07..8113e6f18b 100644 --- a/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift +++ b/apple/TableProTests/Core/Export/ExportServiceRowCountTests.swift @@ -57,6 +57,59 @@ private final class RowsOnlyFormat: ExportFormatPlugin, @unchecked Sendable { } } +private final class HugeCountDriver: DatabaseDriver, @unchecked Sendable { + let connection = TestFixtures.makeConnection(type: .postgresql) + var status: ConnectionStatus = .connected + var serverVersion: String? { nil } + + func execute(query: String) async throws -> QueryResult { + let count = PluginCellValue.text(String(Int.max)) + return QueryResult( + columns: ["c"], columnTypes: [.integer(rawType: "BIGINT")], + rows: query.contains("COUNT(") ? [[count], [count]] : [], + rowsAffected: 0, executionTime: 0, error: nil + ) + } + + func executeUserQuery(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryResult { + try await execute(query: query) + } + func executeParameterized(query: String, parameters: [Any?]) async throws -> QueryResult { + try await execute(query: query) + } + func connect() async throws {} + func disconnect() {} + func testConnection() async throws -> Bool { true } + func ping() async throws {} + func cancelQuery() throws {} + func applyQueryTimeout(_ seconds: Int) async throws {} + func fetchTables() async throws -> [TableInfo] { [] } + func fetchTables(schema: String?) async throws -> [TableInfo] { [] } + func fetchColumns(table: String) async throws -> [ColumnInfo] { [] } + func fetchAllColumns() async throws -> [String: [ColumnInfo]] { [:] } + func fetchIndexes(table: String) async throws -> [IndexInfo] { [] } + func fetchForeignKeys(table: String) async throws -> [ForeignKeyInfo] { [] } + func fetchApproximateRowCount(table: String) async throws -> Int? { nil } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> DatabaseMetadata { + DatabaseMetadata( + id: database, name: database, tableCount: nil, sizeBytes: nil, + lastAccessed: nil, isSystemDatabase: false, icon: "cylinder" + ) + } + func fetchTableDDL(table: String) async throws -> String { "" } + func fetchTableMetadata(tableName: String) async throws -> TableMetadata { + TableMetadata( + tableName: tableName, dataSize: nil, indexSize: nil, totalSize: nil, avgRowLength: nil, + rowCount: nil, comment: nil, engine: nil, collation: nil, createTime: nil, updateTime: nil + ) + } + func fetchViewDefinition(view: String) async throws -> String { "" } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} +} + @MainActor struct ExportServiceRowCountTests { private func run( @@ -109,4 +162,21 @@ struct ExportServiceRowCountTests { #expect(counts.first?.contains("customers") == false) #expect(service.state.readsRows) } + + @Test("Two tables whose counts add past Int.max total Int.max instead of crashing") + func countsSaturate() async throws { + let driver = HugeCountDriver() + let format = StructureAndDataFormat() + let service = ExportService(driver: driver, databaseType: .postgresql) { _ in format } + var config = ExportConfiguration() + config.formatId = StructureAndDataFormat.formatId + let url = FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).txt") + defer { try? FileManager.default.removeItem(at: url) } + try await service.export(objects: [ + ExportObjectItem(name: "orders", kind: .table, optionValues: [true, true]), + ExportObjectItem(name: "customers", kind: .table, optionValues: [true, true]) + ], config: config, to: url) + + #expect(service.state.totalRows == Int.max) + } } diff --git a/apple/TableProTests/Core/LiveMessages/LiveMessageBufferTests.swift b/apple/TableProTests/Core/LiveMessages/LiveMessageBufferTests.swift index c86488a1da..9a41ff3a82 100644 --- a/apple/TableProTests/Core/LiveMessages/LiveMessageBufferTests.swift +++ b/apple/TableProTests/Core/LiveMessages/LiveMessageBufferTests.swift @@ -17,6 +17,35 @@ struct LiveMessageBufferTests { #expect(buffer.droppedCount == 2) } + @Test("A full buffer fed one message at a time keeps the newest in order across compactions") + func oneAtATimePastTheCap() { + var buffer = LiveMessageBuffer(capacity: 4) + for index in 1...23 { + buffer.append([Self.message("m\(index)")]) + } + + let ids = buffer.entries.map { $0.id } + #expect(buffer.entries.compactMap { $0.message?.payload } == ["m20", "m21", "m22", "m23"]) + #expect(ids == [19, 20, 21, 22]) + #expect(buffer.droppedCount == 19) + } + + @Test("Batches of every size wrap the buffer many times and it still holds exactly the newest, oldest first") + func wrapsInOrder() { + var buffer = LiveMessageBuffer(capacity: 5) + var sent = 0 + for batch in [1, 2, 7, 5, 3, 11, 1, 4] + Array(repeating: 6, count: 160) { + buffer.append((sent + 1...sent + batch).map { Self.message("m\($0)") }) + sent += batch + } + + #expect(sent == 994) + #expect(buffer.entries.count == buffer.capacity) + #expect(buffer.entries.compactMap { $0.message?.payload } == (990...994).map { "m\($0)" }) + #expect(buffer.entries.map { $0.id } == Array(989...993)) + #expect(buffer.droppedCount == 989) + } + @Test("Entry ids keep growing after a trim and a clear, so a table never reuses one") func idsNeverRepeat() { var buffer = LiveMessageBuffer(capacity: 2) diff --git a/apple/TableProTests/Core/ServerDashboard/ClickHouseDashboardProviderTests.swift b/apple/TableProTests/Core/ServerDashboard/ClickHouseDashboardProviderTests.swift new file mode 100644 index 0000000000..9ca5bfcd18 --- /dev/null +++ b/apple/TableProTests/Core/ServerDashboard/ClickHouseDashboardProviderTests.swift @@ -0,0 +1,67 @@ +// +// ClickHouseDashboardProviderTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +struct ClickHouseDashboardProviderTests { + private let provider = ClickHouseDashboardProvider(serverVersion: "24.8.1") + + // The TabSeparatedWithNamesAndTypes body system.processes answers with, as the driver hands it on. + private static func processes(elapsed: String) -> QueryResult { + QueryResult( + columns: ["query_id", "user", "current_database", "elapsed", "read_rows", "memory_usage", "query"], + columnTypes: [], + rows: [[ + "0c1f4a52-7d3e-4b8a-9f10-2a6b5c7d8e9f", "default", "analytics", .text(elapsed), "81920", "4194304", + "SELECT count() FROM events", + ]], + rowsAffected: 0, + executionTime: 0, + error: nil + ) + } + + // `query_duration_ms / 1000` from system.query_log, which ClickHouse answers as a Float64. + private static func queryLog(durationSeconds: String) -> QueryResult { + QueryResult( + columns: ["user", "duration_secs", "query"], + columnTypes: [], + rows: [["default", .text(durationSeconds), "SELECT count() FROM events"]], + rowsAffected: 0, + executionTime: 0, + error: nil + ) + } + + @Test("A session's elapsed time is cut to whole seconds") + func elapsedIsCutToWholeSeconds() async throws { + let sessions = try await provider.fetchSessions { _ in Self.processes(elapsed: "83.254913") } + #expect(sessions.map(\.durationSeconds) == [83]) + #expect(sessions.first?.duration == DurationFormatting.string(seconds: 83)) + } + + @Test("An elapsed time that is not a number an Int holds is listed instead of crashing the refresh", arguments: [ + ("inf", Int.max), ("Infinity", .max), ("1e400", .max), ("1e30", .max), ("9.3e18", .max), + ("-inf", .min), ("-1e30", .min), ("nan", 0), ("", 0), ("soon", 0), + ] as [(String, Int)]) + func elapsedThatIsNotARepresentableNumber(elapsed: String, seconds: Int) async throws { + let sessions = try await provider.fetchSessions { _ in Self.processes(elapsed: elapsed) } + #expect(sessions.map(\.durationSeconds) == [seconds]) + #expect(sessions.first?.duration == DurationFormatting.string(seconds: seconds)) + } + + @Test("A slow query's fractional duration is cut to whole seconds, not read as zero", arguments: [ + ("1.5", 1), ("83.254913", 83), ("3600", 3_600), ("inf", Int.max), ("1e30", .max), ("nan", 0), ("", 0), + ] as [(String, Int)]) + func slowQueryDurationIsAFloat(duration: String, seconds: Int) async throws { + let slow = try await provider.fetchSlowQueries { _ in Self.queryLog(durationSeconds: duration) } + #expect(slow.map(\.duration) == [DurationFormatting.string(seconds: seconds)]) + #expect(slow.first?.user == "default") + #expect(slow.first?.query == "SELECT count() FROM events") + } +} diff --git a/apple/TableProTests/Core/ServerDashboard/MemcachedDashboardProviderTests.swift b/apple/TableProTests/Core/ServerDashboard/MemcachedDashboardProviderTests.swift index ce576feba2..113ab36f02 100644 --- a/apple/TableProTests/Core/ServerDashboard/MemcachedDashboardProviderTests.swift +++ b/apple/TableProTests/Core/ServerDashboard/MemcachedDashboardProviderTests.swift @@ -48,4 +48,12 @@ struct MemcachedDashboardProviderTests { #expect(metrics.map(\.id) == ["hit_ratio"]) #expect(metrics.first?.value == "0") } + + @Test("Hit and miss counters whose sum passes Int64 still give a ratio instead of crashing") + func hugeCountersGiveARatio() async throws { + let metrics = try await MemcachedDashboardProvider().fetchMetrics { _ in + Self.stats([("get_hits", "9223372036854775807"), ("get_misses", "1")]) + } + #expect(metrics.first { $0.id == "hit_ratio" }?.value == "100.0") + } } diff --git a/apple/TableProTests/Core/ServerDashboard/MongoDBDashboardProviderTests.swift b/apple/TableProTests/Core/ServerDashboard/MongoDBDashboardProviderTests.swift index 3ae04ec534..eca27d9c54 100644 --- a/apple/TableProTests/Core/ServerDashboard/MongoDBDashboardProviderTests.swift +++ b/apple/TableProTests/Core/ServerDashboard/MongoDBDashboardProviderTests.swift @@ -123,6 +123,22 @@ struct MongoDBDashboardProviderTests { #expect(entry.query.contains("orders")) } + @Test("Resident memory past Int64 bytes shows the largest size instead of crashing") + func residentMemoryPastInt64() async throws { + let metrics = try await MongoDBDashboardProvider().fetchMetrics { _ in + Self.result(["resident"], [["8796093022208"]]) + } + #expect(metrics.first { $0.id == "resident_memory" }?.value == ByteSizeFormatting.string(bytes: Int64.max)) + } + + @Test("A profiled operation's milliseconds past the integer range show the longest duration instead of crashing") + func slowQueryMillisecondsPastInt() async throws { + let slow = try await MongoDBDashboardProvider().fetchSlowQueries { _ in + Self.result(["command", "millis", "ns", "user"], [["{}", "9223372036854776", "shop.orders", "app"]]) + } + #expect(slow.first?.duration == DurationFormatting.string(microseconds: .max)) + } + @Test("The database is the namespace before its first dot", arguments: [ ("shop.orders", "shop"), ("admin.$cmd", "admin"), diff --git a/apple/TableProTests/Core/ServerDashboard/RedisDashboardProviderTests.swift b/apple/TableProTests/Core/ServerDashboard/RedisDashboardProviderTests.swift index a9cfb4f1de..4c6e0b0b60 100644 --- a/apple/TableProTests/Core/ServerDashboard/RedisDashboardProviderTests.swift +++ b/apple/TableProTests/Core/ServerDashboard/RedisDashboardProviderTests.swift @@ -145,6 +145,14 @@ struct RedisDashboardProviderTests { #expect(slow.last?.duration == DurationFormatting.string(microseconds: 626)) } + @Test("A slow log duration at the top of the integer range is listed instead of crashing the refresh") + func slowLogDurationAtIntMax() async throws { + let rows: [[String?]] = [["1", "1792537100", String(Int64.max), "GET a", "127.0.0.1:59713", ""]] + let slow = try await provider.fetchSlowQueries { _ in Self.result(RedisSlowLogGrid.columns, rows) } + #expect(slow.count == 1) + #expect(slow.first?.duration == DurationFormatting.string(seconds: Double(Int.max) / 1_000_000)) + } + @Test("Terminate is CLIENT KILL ID and Cancel is CLIENT UNBLOCK, for a numeric id only", arguments: [ ("21", true), ("21 SKIPME no", false), diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaResolutionTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaResolutionTests.swift new file mode 100644 index 0000000000..16997ad1c0 --- /dev/null +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaResolutionTests.swift @@ -0,0 +1,175 @@ +import Foundation +@testable import TablePro +import Testing + +private func fieldTypes(_ schema: ProtobufSchema, _ message: String) -> [String: ProtobufFieldType] { + Dictionary(uniqueKeysWithValues: (schema.messages[message]?.fields ?? []).map { ($0.name, $0.type) }) +} + +private func lengthDelimited(_ number: Int, _ payload: [UInt8]) -> [UInt8] { + var bytes = [UInt8(number << 3 | 2)] + var length = payload.count + while length >= 0x80 { + bytes.append(UInt8(length & 0x7F | 0x80)) + length >>= 7 + } + bytes.append(UInt8(length)) + return bytes + payload +} + +struct ProtobufSchemaResolutionTests { + @Test func referencesResolveFromTheInnermostScopeOutward() throws { + let proto = """ + syntax = "proto3"; + package a.b; + enum Color { RED = 0; } + message Outer { + message Inner { int32 v = 1; } + Inner inner = 1; + b.Outer qualified = 2; + Outer.Inner nested = 3; + Missing missing = 4; + .a.b.Color absolute = 5; + Color color = 6; + } + """ + let types = fieldTypes(try ProtobufSchema.parse(Data(proto.utf8)), "a.b.Outer") + #expect(types["inner"] == .message("a.b.Outer.Inner")) + #expect(types["qualified"] == .message("a.b.Outer")) + #expect(types["nested"] == .message("a.b.Outer.Inner")) + #expect(types["missing"] == .unresolved("Missing")) + #expect(types["absolute"] == .enumeration("a.b.Color")) + #expect(types["color"] == .enumeration("a.b.Color")) + } + + @Test func oneofsAreNumberedInDeclarationOrder() throws { + let proto = """ + syntax = "proto3"; + message M { + oneof first { int32 a = 1; string b = 2; } + int32 plain = 3; + oneof second { int32 c = 4; } + } + """ + let fields = try #require(try ProtobufSchema.parse(Data(proto.utf8)).messages["M"]?.fields) + let indexes = Dictionary(uniqueKeysWithValues: fields.map { ($0.name, $0.oneofIndex) }) + #expect(indexes == ["a": 0, "b": 0, "plain": nil, "c": 1]) + } + + @Test func aCompoundNameWhoseFirstPartIsAnEnumIsNotLookedUpFurtherOut() throws { + let inner = """ + syntax = "proto3"; + package a.b; + message M { + enum E { X = 0; } + E.Inner f = 1; + } + """ + let outer = """ + syntax = "proto3"; + package a; + message E { message Inner { int32 v = 1; } } + """ + let types = fieldTypes(try ProtobufSchema.parse(protoFiles: [inner, outer]), "a.b.M") + #expect(types["f"] == .unresolved("E.Inner")) + } + + // Each field looked its type up through every part of the package: 10,000 parts and 2,000 fields took 15 s. + @Test(.timeLimit(.minutes(1))) + func aPackageOfTooManyPartsIsRefused() throws { + let fields = (1...2_000).map { " Missing\($0) f\($0) = \($0);" }.joined(separator: "\n") + func proto(parts: Int) -> String { + let package = Array(repeating: "p", count: parts).joined(separator: ".") + return "syntax = \"proto3\";\npackage \(package);\nmessage M {\n\(fields)\n}\n" + } + #expect(throws: ProtobufSchemaError(reason: String(localized: "The package name has too many parts."))) { + try ProtobufSchema.parse(Data(proto(parts: 10_000).utf8)) + } + let package = Array(repeating: "p", count: ProtoFileParser.maxPackageComponents).joined(separator: ".") + let types = fieldTypes(try ProtobufSchema.parse(Data(proto(parts: ProtoFileParser.maxPackageComponents).utf8)), "\(package).M") + #expect(types["f2000"] == .unresolved("Missing2000")) + } + + @Test(.timeLimit(.minutes(1))) + func aDottedMessageNameIsRefused() { + let name = Array(repeating: "p", count: 10_000).joined(separator: ".") + let fields = (1...2_000).map { " Missing\($0) f\($0) = \($0);" }.joined(separator: "\n") + let proto = "syntax = \"proto3\";\nmessage \(name) {\n\(fields)\n}\n" + #expect(throws: ProtobufSchemaError.self) { + try ProtobufSchema.parse(Data(proto.utf8)) + } + } + + // Splitting the package again for each field cost fields times its length in characters. + @Test(.timeLimit(.minutes(1))) + func aLongPackageNameResolvesInLinearTime() throws { + let package = String(repeating: "p", count: ProtoFileParser.maxNameLength - 10) + let fields = (1...30_000).map { " Missing\($0) f\($0) = \($0);" }.joined(separator: "\n") + let proto = "syntax = \"proto3\";\npackage \(package);\nmessage M {\n\(fields)\n}\n" + let types = fieldTypes(try ProtobufSchema.parse(Data(proto.utf8)), "\(package).M") + #expect(types.count == 30_000) + #expect(types["f30000"] == .unresolved("Missing30000")) + } + + // Each message keeps its full name, so a 2 MB package over 180,000 empty messages asked for 360 GB. + @Test(.timeLimit(.minutes(1))) + func aTypeNamePastTheLimitIsRefused() { + let package = String(repeating: "p", count: 2_000_000) + let messages = (1...180_000).map { "message A\($0) {}" }.joined(separator: "\n") + let proto = "syntax = \"proto3\";\npackage \(package);\n\(messages)\n" + #expect(throws: ProtobufSchemaError(reason: String(localized: "A type name is too long."))) { + try ProtobufSchema.parse(Data(proto.utf8)) + } + } + + @Test func moreEnumsThanTheLimitAreRefused() { + let enums = (0...ProtobufSchema.maxMessages).map { "enum E\($0) { V\($0) = 0; }" }.joined(separator: "\n") + let proto = "syntax = \"proto3\";\nmessage M {}\n\(enums)\n" + #expect(throws: ProtobufSchemaError(reason: String(localized: "The schema defines more messages than TablePro reads."))) { + try ProtobufSchema.parse(Data(proto.utf8)) + } + } + + @Test func definitionsAcrossFilesAreCountedTogether() { + let files = (0..<3).map { file in + let enums = (0..<15_000).map { "enum E\(file)x\($0) { V\(file)x\($0) = 0; }" }.joined(separator: "\n") + return "syntax = \"proto3\";\npackage f\(file);\nmessage M {}\n\(enums)\n" + } + #expect(throws: ProtobufSchemaError(reason: String(localized: "The schema defines more messages than TablePro reads."))) { + try ProtobufSchema.parse(protoFiles: files) + } + } + + @Test func moreMessagesThanTheLimitAreRefused() { + let messages = (0...ProtobufSchema.maxMessages).map { "message A\($0) {}" }.joined(separator: "\n") + let proto = "syntax = \"proto3\";\n\(messages)\n" + #expect(throws: ProtobufSchemaError(reason: String(localized: "The schema defines more messages than TablePro reads."))) { + try ProtobufSchema.parse(Data(proto.utf8)) + } + } + + @Test func aDescriptorSetPackageOfTooManyPartsIsRefused() throws { + func descriptorSet(packageParts: Int) -> Data { + let package = Array(repeating: "p", count: packageParts).joined(separator: ".") + let message = lengthDelimited(1, Array("M".utf8)) + let file = lengthDelimited(1, Array("m.proto".utf8)) + lengthDelimited(2, Array(package.utf8)) + + lengthDelimited(4, message) + return Data(lengthDelimited(1, file)) + } + #expect(throws: ProtobufSchemaError(reason: String(localized: "The package name has too many parts."))) { + try ProtobufSchema.parse(descriptorSet(packageParts: ProtoFileParser.maxPackageComponents + 1)) + } + let schema = try ProtobufSchema.parse(descriptorSet(packageParts: ProtoFileParser.maxPackageComponents)) + #expect(schema.messageNames.count == 1) + } + + @Test(.timeLimit(.minutes(1))) + func manyOneofsAreNumberedInLinearTime() throws { + let count = 40_000 + let oneofs = (1...count).map { " oneof o\($0) { int32 f\($0) = \($0); }" }.joined(separator: "\n") + let proto = "syntax = \"proto3\";\nmessage M {\n\(oneofs)\n}\n" + let fields = try #require(try ProtobufSchema.parse(Data(proto.utf8)).messages["M"]?.fields) + #expect(fields.count == count) + #expect(fields.last?.oneofIndex == count - 1) + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/ByteSizeFormattingTests.swift b/apple/TableProTests/Core/Services/Formatting/ByteSizeFormattingTests.swift index 932133db86..ef2d5383de 100644 --- a/apple/TableProTests/Core/Services/Formatting/ByteSizeFormattingTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/ByteSizeFormattingTests.swift @@ -71,6 +71,69 @@ struct ByteSizeFormattingTests { func negativeDurationIsClamped() { #expect(!DurationFormatting.string(seconds: -10).contains("-")) } + + @Test("Microseconds round half up to whole milliseconds", arguments: [ + (499, Duration.microseconds(499)), (500, .milliseconds(1)), (1_499, .milliseconds(1)), + (1_500, .milliseconds(2)), (12_345, .milliseconds(12)), (999_499, .milliseconds(999)), + ] as [(Int, Duration)]) + func microsecondsRoundToMilliseconds(microseconds: Int, expected: Duration) { + let unit: Set = expected < .milliseconds(1) ? [.microseconds] : [.milliseconds] + #expect(DurationFormatting.string(microseconds: microseconds) == expected.formatted(.units(allowed: unit, width: .narrow))) + } + + @Test("A slow log duration at the top of the integer range formats instead of crashing", arguments: [ + Int.max, Int.max - 499, Int.max - 500, + ]) + func microsecondsNearIntMax(microseconds: Int) { + #expect(DurationFormatting.string(microseconds: microseconds) == DurationFormatting.string(seconds: Double(microseconds) / 1_000_000)) + } + + @Test("Fractional seconds round to the nearest second") + func fractionalSecondsRound() { + #expect(DurationFormatting.string(seconds: 89.4) == DurationFormatting.string(seconds: 89)) + #expect(DurationFormatting.string(seconds: 89.6) == DurationFormatting.string(seconds: 90)) + #expect(DurationFormatting.string(seconds: 7_199.7) == DurationFormatting.string(seconds: 7_200)) + } + + @Test("Seconds that are not a finite number inside Int format instead of crashing", arguments: [ + (Double.nan, 0), (.infinity, .max), (-.infinity, 0), (1e300, .max), (-1e300, 0), (Double(Int.max), .max), + ] as [(Double, Int)]) + func unrepresentableSeconds(seconds: Double, expected: Int) { + #expect(DurationFormatting.string(seconds: seconds) == DurationFormatting.string(seconds: expected)) + } + + @Test("Milliseconds format as the same number of microseconds, saturating past the integer range") + func millisecondsFormat() { + #expect(DurationFormatting.string(milliseconds: 320) == DurationFormatting.string(microseconds: 320_000)) + #expect(DurationFormatting.string(milliseconds: 0) == DurationFormatting.string(microseconds: 0)) + #expect(DurationFormatting.string(milliseconds: 9_223_372_036_854_776) == DurationFormatting.string(microseconds: .max)) + #expect(DurationFormatting.string(milliseconds: .max) == DurationFormatting.string(microseconds: .max)) + #expect(!DurationFormatting.string(milliseconds: .min).contains("-")) + } + + @Test("Mebibytes format as bytes, saturating past Int64") + func mebibytesFormat() { + #expect(ByteSizeFormatting.string(mebibytes: 136) == ByteSizeFormatting.string(bytes: Int64(136 * 1_048_576))) + #expect(ByteSizeFormatting.string(mebibytes: 8_796_093_022_208) == ByteSizeFormatting.string(bytes: Int64.max)) + #expect(ByteSizeFormatting.string(mebibytes: .max) == ByteSizeFormatting.string(bytes: Int64.max)) + } + + @Test("A Double converts to Int by truncation, pinned to Int's range, with NaN as zero", arguments: [ + (83.9, 83), (-83.9, -83), (0, 0), (.nan, 0), (.infinity, .max), (-.infinity, .min), + (1e30, .max), (-1e30, .min), (9.3e18, .max), (Double(Int.max), .max), (Double(Int.min), .min), + ] as [(Double, Int)]) + func saturatingInt(value: Double, expected: Int) { + #expect(SaturatingArithmetic.int(value) == expected) + } + + @Test("A product past the integer range pins to the end its sign points at") + func saturatingProduct() { + #expect(SaturatingArithmetic.product(Int64(136), 1_048_576) == 142_606_336) + #expect(SaturatingArithmetic.product(Int64.max, 2) == .max) + #expect(SaturatingArithmetic.product(Int64.min, 2) == .min) + #expect(SaturatingArithmetic.product(Int64.min, -1) == .max) + #expect(SaturatingArithmetic.product(Int.max / 2, -3) == .min) + } } @MainActor diff --git a/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift b/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift index 22e2ab9b59..ef88e9b8d5 100644 --- a/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift @@ -304,4 +304,43 @@ struct PhpSerializeParserDepthTests { #expect(!PhpSerializeParser.looksLikePhpSerialized("a quick brown fox")) #expect(!PhpSerializeParser.looksLikePhpSerialized("some text")) } + + private static func nested(_ inner: String, wrappers: Int, padding: Int = 0) -> String { + String(repeating: "a:1:{i:0;", count: wrappers) + inner + String(repeating: "}", count: wrappers) + + String(repeating: " ", count: padding) + } + + @Test("A value nested to the depth cap parses, one level deeper is refused") + func nestingAtTheCap() { + let cap = PhpSerializeParser.depthCap + #expect(PhpSerializeParser.parse(Self.nested("i:1;", wrappers: cap)) != nil) + #expect(PhpSerializeParser.parse(Self.nested("i:1;", wrappers: cap + 1)) == nil) + } + + @Test("An array at the depth cap that declares entries it does not carry is refused") + func forgedCountAtTheCap() { + let value = Self.nested("a:60000:{}", wrappers: PhpSerializeParser.depthCap, padding: 60_000) + #expect(PhpSerializeParser.parse(value) == nil) + } + + @Test("Sibling arrays at the depth cap cannot each claim the trailing padding") + func forgedSiblingCountsAtTheCap() { + let siblings = (0..<50).map { "i:\($0);a:5000:{}" }.joined() + let value = Self.nested("a:50:{\(siblings)}", wrappers: PhpSerializeParser.depthCap - 1, padding: 5_000) + #expect(PhpSerializeParser.parse(value) == nil) + } + + @Test("A parse in a cancelled task stops and returns nil") + func cancelledParse() async { + let value = "a:5000:{" + (0..<5_000).map { "i:\($0);i:\($0);" }.joined() + "}" + #expect(PhpSerializeParser.parse(value) != nil) + let stopped = await Task.detached { () -> Bool in + withUnsafeCurrentTask { task in + guard let task else { return } + task.cancel() + } + return PhpSerializeParser.parse(value) == nil + }.value + #expect(stopped) + } } diff --git a/apple/TableProTests/Core/Services/ProjectImport/DotenvParserTests.swift b/apple/TableProTests/Core/Services/ProjectImport/DotenvParserTests.swift index 0660b62498..b9273c0b1f 100644 --- a/apple/TableProTests/Core/Services/ProjectImport/DotenvParserTests.swift +++ b/apple/TableProTests/Core/Services/ProjectImport/DotenvParserTests.swift @@ -154,6 +154,45 @@ struct DotenvParserTests { #expect(document["K"] == nil) } + // MARK: - Expansion budget + + @Test("Lines that each repeat the line before ten times stop growing at the expansion budget") + func testInterpolationExpansionIsBounded() { + var lines = ["A=" + String(repeating: "x", count: 100)] + for (previous, name) in zip(["A", "B", "C", "D", "E"], ["B", "C", "D", "E", "F"]) { + lines.append("\(name)=" + String(repeating: "$\(previous)", count: 10)) + } + let source = lines.joined(separator: "\n") + let document = DotenvParser.parse(source, processEnvironment: [:]) + let expanded = document.entries.reduce(0) { $0 + $1.value.utf8.count } + #expect(expanded <= source.utf8.count + ProjectScanExpansionBudget.standardLimit) + #expect(document.entry(for: "F")?.hasUnresolvedReference == true) + #expect(document["F"] == nil) + } + + @Test("A reference over the budget stays literal and does not fall back to its default") + func testOverBudgetReferenceIsLiteral() { + let source = "A=12345\nB=${A:-fallback}\nC=$A" + let document = DotenvParser.parse(source, processEnvironment: [:], budget: ProjectScanExpansionBudget(limit: 4)) + #expect(document.entry(for: "B")?.value == "${A:-fallback}") + #expect(document.entry(for: "B")?.hasUnresolvedReference == true) + #expect(document.entry(for: "C")?.value == "$A") + #expect(document.entry(for: "C")?.hasUnresolvedReference == true) + + let fitting = DotenvParser.parse(source, processEnvironment: [:], budget: ProjectScanExpansionBudget(limit: 5)) + #expect(fitting.entry(for: "B")?.value == "12345") + #expect(fitting.entry(for: "C")?.hasUnresolvedReference == true) + } + + @Test("Files parsed with one budget share it, so a later file cannot expand once an earlier one spent it") + func testBudgetIsShared() { + let budget = ProjectScanExpansionBudget(limit: 10) + let first = DotenvParser.parse("A=0123456789\nB=$A", processEnvironment: [:], budget: budget) + let second = DotenvParser.parse("A=1\nB=$A", processEnvironment: [:], budget: budget) + #expect(first["B"] == "0123456789") + #expect(second.entry(for: "B")?.hasUnresolvedReference == true) + } + // MARK: - Security @Test("Command substitution is never executed") diff --git a/apple/TableProTests/Core/Services/ProjectImport/ProjectConfigExtractorTests.swift b/apple/TableProTests/Core/Services/ProjectImport/ProjectConfigExtractorTests.swift index c2ae52732d..baea77e891 100644 --- a/apple/TableProTests/Core/Services/ProjectImport/ProjectConfigExtractorTests.swift +++ b/apple/TableProTests/Core/Services/ProjectImport/ProjectConfigExtractorTests.swift @@ -111,6 +111,110 @@ struct YamlMappingSupportTests { func testNonMappingDocument() { #expect(YamlMappingSupport.loadMapping("- just\n- a\n- list") == nil) } + + private static func aliasChain(levels: Int, width: Int) -> String { + var lines = ["l0: &l0 [" + Array(repeating: "x", count: width).joined(separator: ", ") + "]"] + for level in 1.. String { + let levels = 990 + switch shape { + case "flow": + return "k: " + String(repeating: "[", count: levels) + "x" + String(repeating: "]", count: levels) + case "sequence": + return "k:\n" + String(repeating: "- ", count: levels) + "x" + default: + return (0.. + """ + let root = try #require(YamlMappingSupport.loadMapping(contents)) + let services = try #require(YamlMappingSupport.mapping(root["services"])) + let db = try #require(YamlMappingSupport.mapping(services["db"])) + #expect(YamlMappingSupport.string(db["driver"]) == "json-file") + #expect(YamlMappingSupport.string(db["image"]) == "postgres:${PG_VERSION:-16}") + #expect((db["ports"] as? [Any])?.count == 2) + #expect((db["command"] as? [Any])?.count == 3) + } } struct WordPressConfigExtractorTests { diff --git a/apple/TableProTests/Core/Services/ProjectImport/ProjectFolderScannerTests.swift b/apple/TableProTests/Core/Services/ProjectImport/ProjectFolderScannerTests.swift index 9df04f968e..c90a1dca7f 100644 --- a/apple/TableProTests/Core/Services/ProjectImport/ProjectFolderScannerTests.swift +++ b/apple/TableProTests/Core/Services/ProjectImport/ProjectFolderScannerTests.swift @@ -186,6 +186,21 @@ struct ProjectFolderScannerTests { return (name, secret) } + @Test("A scan whose task is cancelled stops and offers nothing") + func testCancelledScanOffersNothing() async throws { + try write("DATABASE_URL=postgresql://u:p@localhost:5432/appdb", to: ".env") + let rootURL = root + let cancelled = await Task.detached { () -> Result in + withUnsafeCurrentTask { task in + guard let task else { return } + task.cancel() + } + return ProjectFolderScanner.scan(rootURL: rootURL) + }.value + #expect(try cancelled.get().candidates.isEmpty) + #expect(try !scan().candidates.isEmpty) + } + @Test("Results are ordered with the most trustworthy sources first") func testOrdering() throws { try write("DATABASE_URL=postgresql://u:p@localhost:5432/fromlocal", to: ".env.local") diff --git a/apple/TableProTests/Core/Services/ProjectImport/ProjectYamlExtractorTests.swift b/apple/TableProTests/Core/Services/ProjectImport/ProjectYamlExtractorTests.swift index f966d1d4ed..4e9ebc944c 100644 --- a/apple/TableProTests/Core/Services/ProjectImport/ProjectYamlExtractorTests.swift +++ b/apple/TableProTests/Core/Services/ProjectImport/ProjectYamlExtractorTests.swift @@ -507,6 +507,51 @@ struct DockerComposeExtractorTests { #expect(candidate?.parsedURL.type == .mariadb) #expect(candidate?.parsedURL.port == 3_399) } + + @Test("A .env value referenced thousands of times from one compose value stops at the expansion budget") + func testSubstitutionIsBounded() throws { + let environment = DotenvParser.parse("V=" + String(repeating: "x", count: 1_000), processEnvironment: [:]) + let references = String(repeating: "${V}", count: 5_000) + let contents = """ + services: + db: + image: postgres:16 + environment: + POSTGRES_PASSWORD: "\(references)" + ports: + - "5432:5432" + """ + let candidate = try #require(extract(contents, environment: environment).first) + #expect(candidate.parsedURL.password.utf8.count <= references.utf8.count + ProjectScanExpansionBudget.standardLimit) + #expect(candidate.warnings.contains(String(localized: "Some values are set outside this file"))) + } + + @Test("A compose file and the .env beside it draw on one budget") + func testSubstitutionSharesTheBudget() throws { + let budget = ProjectScanExpansionBudget(limit: 8) + let environment = DotenvParser.parse("A=pass\nB=$A$A", processEnvironment: [:], budget: budget) + let contents = """ + services: + db: + image: postgres:16 + environment: + POSTGRES_USER: ${A} + POSTGRES_PASSWORD: ${A} + ports: + - "5432:5432" + """ + let candidate = try #require( + DockerComposeExtractor.extract( + contents: contents, + relativePath: "docker-compose.yml", + environment: environment, + budget: budget + ).first + ) + #expect(environment["B"] == "passpass") + #expect(candidate.parsedURL.username == "${A}") + #expect(candidate.parsedURL.password == "${A}") + } } struct SpringYamlExtractorTests { diff --git a/apple/TableProTests/Models/UI/PhpTreeBuilderTests.swift b/apple/TableProTests/Models/UI/PhpTreeBuilderTests.swift index 8528ffcc17..afef53a715 100644 --- a/apple/TableProTests/Models/UI/PhpTreeBuilderTests.swift +++ b/apple/TableProTests/Models/UI/PhpTreeBuilderTests.swift @@ -238,7 +238,7 @@ struct PhpTreeBuilderTests { PhpKeyValue(key: .int(7), value: .serializable(className: "T", rawPayload: "https://example.com/")), PhpKeyValue(key: .int(8), value: .reference(id: 2)), PhpKeyValue(key: .int(9), value: .unsupported(token: "o")), - PhpKeyValue(key: .int(10), value: .depthExceeded) + PhpKeyValue(key: .int(10), value: .tooLarge) ]) ) let contents = root.children.map(\.rowContent) diff --git a/apple/changelog.d/3991.changed.md b/apple/changelog.d/3991.changed.md new file mode 100644 index 0000000000..bd8c30db83 --- /dev/null +++ b/apple/changelog.d/3991.changed.md @@ -0,0 +1 @@ +Importing an Excel file reads date and time cells as dates and times, as the Data Files window shows them. diff --git a/apple/changelog.d/3991.fixed.1.md b/apple/changelog.d/3991.fixed.1.md new file mode 100644 index 0000000000..4be0876647 --- /dev/null +++ b/apple/changelog.d/3991.fixed.1.md @@ -0,0 +1 @@ +A crafted Excel, CSV or compressed file could hang TablePro or fill memory and disk. diff --git a/apple/changelog.d/3991.fixed.md b/apple/changelog.d/3991.fixed.md new file mode 100644 index 0000000000..0d6b269bf6 --- /dev/null +++ b/apple/changelog.d/3991.fixed.md @@ -0,0 +1 @@ +Malformed replies from Kafka, Teradata, Redis, Neo4j, Trino and other servers could crash TablePro. diff --git a/apple/project.yml b/apple/project.yml index 73a919362c..2262c6b848 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -58,11 +58,6 @@ packages: Yams: url: https://github.com/jpsim/Yams majorVersion: 5.1.0 - # Kafka brokers hand a consumer whatever codec the producer stored, and never transcode. - # macOS supplies no zstd at any layer, so a zstd topic is unreadable without this. - zstd: - url: https://github.com/facebook/zstd - majorVersion: 1.5.0 # Already in the graph through Packages/TableProOracle. Declared here so the Kafka plugin # can link them directly: Kafka is a length-prefixed binary protocol over TCP, and NIOSSL # reads the PEM paths SSLConfiguration already carries. @@ -408,7 +403,7 @@ targets: - package: Sparkle - package: Yams - package: TableProCore - products: [TableProAnalytics, TableProAutocomplete, TableProConnectionLibrary, TableProGeometry, TableProGoogleCloud, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme] + products: [TableProAnalytics, TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme] - package: TableProMacCore products: [TableProMCP, TableProSSHConfig, TableProUtilities] # Writes nothing for a version CHANGELOG.md has no section for, so a 0.0.0-dev build ships no notes. @@ -952,6 +947,9 @@ targets: - -lc++ - -lz SWIFT_INCLUDE_PATHS: $(SRCROOT)/Plugins/CassandraDriverPlugin/CCassandra + dependencies: + - package: TableProCore + product: TableProCassandraCore CloudflareD1DriverPlugin: templates: [DriverPlugin] @@ -1062,8 +1060,10 @@ targets: sources: - Plugins/SharedNIO dependencies: - - package: zstd - product: libzstd + - package: TableProCore + product: TableProKafkaCore + - package: TableProCore + product: TableProWireIO - package: swift-nio product: NIOCore - package: swift-nio @@ -1218,6 +1218,8 @@ targets: folder: SnowflakeDriverPlugin principalClass: SnowflakePlugin dependencies: + - package: TableProCore + product: TableProGzip - package: TableProCore product: TableProNumberFormatting