fix(parsing): stop malformed replies and files from crashing, hanging or filling memory

.oss-scanner/swift/Package.swift

@@ -15,8 +15,12 @@ let package = Package(
         .target(name: "TableProDocumentPath"),
         .target(name: "TableProJavaScriptText"),
         .target(name: "TableProDamengCore", dependencies: ["TableProSQLGrammar"]),
+        .target(name: "TableProWireIO"),
+        .target(name: "TableProCassandraCore"),
         .testTarget(name: "TableProSQLGrammarTests", dependencies: ["TableProSQLGrammar"]),
         .testTarget(name: "TableProLogRedactionTests", dependencies: ["TableProLogRedaction"]),
-        .testTarget(name: "TableProDamengCoreTests", dependencies: ["TableProDamengCore", "TableProSQLGrammar"])
+        .testTarget(name: "TableProDamengCoreTests", dependencies: ["TableProDamengCore", "TableProSQLGrammar"]),
+        .testTarget(name: "TableProWireIOTests", dependencies: ["TableProWireIO"]),
+        .testTarget(name: "TableProCassandraCoreTests", dependencies: ["TableProCassandraCore"])
     ]
 )

apple/Packages/TableProCore/Package.swift

@@ -60,7 +60,15 @@ let package = Package(
         .library(name: "TableProPostgreSQLCore", targets: ["TableProPostgreSQLCore"]),
         .library(name: "TableProRedisCore", targets: ["TableProRedisCore"]),
         .library(name: "TableProMemcachedCore", targets: ["TableProMemcachedCore"]),
-        .library(name: "TableProTestSupport", targets: ["TableProTestSupport"])
+        .library(name: "TableProTestSupport", targets: ["TableProTestSupport"]),
+        .library(name: "TableProWireIO", targets: ["TableProWireIO"]),
+        .library(name: "TableProGzip", targets: ["TableProGzip"]),
+        .library(name: "TableProKafkaCore", targets: ["TableProKafkaCore"]),
+        .library(name: "TableProCassandraCore", targets: ["TableProCassandraCore"])
+    ],
+    // Kafka brokers hand back whatever codec the producer stored, and macOS has no zstd.
+    dependencies: [
+        .package(url: "https://github.com/facebook/zstd", from: "1.5.0")
     ],
     targets: [
         .target(
@@ -110,7 +118,7 @@ let package = Package(
         ),
         .target(
             name: "TableProImport",
-            dependencies: ["TableProConnectionLibrary", "TableProSSHTransport"],
+            dependencies: ["TableProConnectionLibrary", "TableProCoreTypes", "TableProSSHTransport"],
             path: "Sources/TableProImport"
         ),
         .target(
@@ -152,7 +160,7 @@ let package = Package(
         ),
         .target(
             name: "TableProTeradataCore",
-            dependencies: [],
+            dependencies: ["TableProWireIO"],
             path: "Sources/TableProTeradataCore"
         ),
         .target(
@@ -263,7 +271,7 @@ let package = Package(
         ),
         .target(
             name: "TableProTabularIO",
-            dependencies: [],
+            dependencies: ["TableProWireIO"],
             path: "Sources/TableProTabularIO"
         ),
         .target(
@@ -301,6 +309,33 @@ let package = Package(
             path: "Sources/TableProRedisCore",
             swiftSettings: pluginCoreSettings
         ),
+        .target(
+            name: "TableProWireIO",
+            dependencies: [],
+            path: "Sources/TableProWireIO"
+        ),
+        .target(
+            name: "TableProGzip",
+            dependencies: ["TableProWireIO"],
+            path: "Sources/TableProGzip"
+        ),
+        // Linked by a plugin bundle, so never TableProPluginKit.
+        .target(
+            name: "TableProKafkaCore",
+            dependencies: [
+                "TableProWireIO",
+                "TableProGzip",
+                .product(name: "libzstd", package: "zstd")
+            ],
+            path: "Sources/TableProKafkaCore",
+            swiftSettings: pluginCoreSettings
+        ),
+        .target(
+            name: "TableProCassandraCore",
+            dependencies: [],
+            path: "Sources/TableProCassandraCore",
+            swiftSettings: pluginCoreSettings
+        ),
         .testTarget(
             name: "TableProBigtableCoreTests",
             dependencies: ["TableProBigtableCore", "TableProGoogleCloud"],
@@ -323,7 +358,7 @@ let package = Package(
         ),
         .testTarget(
             name: "TableProTeradataCoreTests",
-            dependencies: ["TableProTeradataCore"],
+            dependencies: ["TableProTeradataCore", "TableProWireIO"],
             path: "Tests/TableProTeradataCoreTests"
         ),
         .testTarget(
@@ -493,6 +528,28 @@ let package = Package(
             dependencies: ["TableProRedisCore"],
             path: "Tests/TableProRedisCoreTests",
             swiftSettings: pluginCoreSettings
+        ),
+        .testTarget(
+            name: "TableProWireIOTests",
+            dependencies: ["TableProWireIO"],
+            path: "Tests/TableProWireIOTests"
+        ),
+        .testTarget(
+            name: "TableProGzipTests",
+            dependencies: ["TableProGzip", "TableProWireIO"],
+            path: "Tests/TableProGzipTests"
+        ),
+        .testTarget(
+            name: "TableProKafkaCoreTests",
+            dependencies: ["TableProKafkaCore", "TableProWireIO"],
+            path: "Tests/TableProKafkaCoreTests",
+            swiftSettings: pluginCoreSettings
+        ),
+        .testTarget(
+            name: "TableProCassandraCoreTests",
+            dependencies: ["TableProCassandraCore"],
+            path: "Tests/TableProCassandraCoreTests",
+            swiftSettings: pluginCoreSettings
         )
     ]
 )

apple/Plugins/CassandraDriverPlugin/CassandraCellText.swift → apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellText.swift

@@ -1,17 +1,18 @@
 //
 //  CassandraCellText.swift
-//  CassandraDriverPlugin
+//  TableProCassandraCore
 //
 
 import Foundation
 
-/// The text a cell shows for the CQL types the C driver hands over as raw parts rather than as a string.
-enum CassandraCellText {
+// The text a cell shows for the CQL types the C driver hands over as raw parts rather than as a string.
+public enum CassandraCellText {
     private static let nanosecondsPerSecond: Int64 = 1_000_000_000
+    private static let hexDigits = Array("0123456789abcdef".unicodeScalars)
 
-    /// Cassandra's own spelling of a duration, the form cqlsh prints and CQL accepts back: `1y2mo3d4h5m6s7ms`.
-    /// A duration's parts all carry the same sign, so a negative one is the positive spelling with a leading `-`.
-    static func durationText(months: Int32, days: Int32, nanoseconds: Int64) -> String {
+    // Cassandra's own spelling of a duration, the form cqlsh prints and CQL accepts back: `1y2mo3d4h5m6s7ms`.
+    // A duration's parts all carry the same sign, so a negative one is the positive spelling with a leading `-`.
+    public static func durationText(months: Int32, days: Int32, nanoseconds: Int64) -> String {
         let isNegative = months < 0 || days < 0 || nanoseconds < 0
         let months = Int64(months).magnitude
         let days = Int64(days).magnitude
@@ -27,9 +28,9 @@ enum CassandraCellText {
         return isNegative ? "-" + text : text
     }
 
-    /// A `vector<float, n>` arrives as a custom type whose class name carries the element type, with the values
-    /// packed as big-endian 32-bit floats. Any other custom value is shown as its bytes.
-    static func customText(className: String?, bytes: Data) -> String {
+    // A `vector<float, n>` arrives as a custom type whose class name carries the element type, with the values
+    // packed as big-endian 32-bit floats. Any other custom value is shown as its bytes.
+    public static func customText(className: String?, bytes: Data) -> String {
         guard let className, isFloatVector(className), bytes.count.isMultiple(of: 4) else {
             return hex(bytes)
         }
@@ -41,8 +42,14 @@ enum CassandraCellText {
         return "[" + values.joined(separator: ", ") + "]"
     }
 
-    static func hex(_ bytes: Data) -> String {
-        "0x" + bytes.map { String(format: "%02x", $0) }.joined()
+    public static func hex(_ bytes: Data) -> String {
+        var text = "0x"
+        text.reserveCapacity(2 + bytes.count * 2)
+        for byte in bytes {
+            text.unicodeScalars.append(hexDigits[Int(byte >> 4)])
+            text.unicodeScalars.append(hexDigits[Int(byte & 0x0F)])
+        }
+        return text
     }
 
     private static func isFloatVector(_ className: String) -> Bool {

apple/Plugins/CassandraDriverPlugin/CassandraVarint.swift → apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraVarint.swift

@@ -1,16 +1,21 @@
 //
 //  CassandraVarint.swift
-//  CassandraDriverPlugin
+//  TableProCassandraCore
 //
 
 import Foundation
 
-/// CQL's `varint` is a big-endian two's complement integer of any length, and `decimal` is that integer unscaled
-/// with a 32-bit scale beside it. The C driver hands both over as raw bytes, so reading and writing them as numbers
-/// is done here.
-enum CassandraVarint {
-    static func decimalString(fromTwosComplement bytes: Data) -> String {
+// CQL's `varint` is a big-endian two's complement integer of any length, and `decimal` is that integer unscaled
+// with a 32-bit scale beside it. The C driver hands both over as raw bytes, so reading and writing them as numbers
+// is done here.
+public enum CassandraVarint {
+    // Each pass over the number yields nine digits, so the cost grows with the square of its length and a longer
+    // value is shown as hex. 4 KiB is about 9,900 digits, near the 10,000 characters the grid shows of a cell.
+    public static let maximumDecimalBytes = 4_096
+
+    public static func decimalString(fromTwosComplement bytes: Data) -> String {
         guard let first = bytes.first else { return "0" }
+        guard bytes.count <= maximumDecimalBytes else { return CassandraCellText.hex(bytes) }
         let isNegative = first & 0x80 != 0
         var magnitude = [UInt8](bytes)
         if isNegative {
@@ -20,11 +25,14 @@ enum CassandraVarint {
         return isNegative ? "-" + digits : digits
     }
 
-    /// A scale the plain form would pad with more zeros than this is written with an exponent instead. The wire
-    /// scale is any 32-bit value, and expanding one near its limits would allocate billions of characters.
+    // A scale the plain form would pad with more zeros than this is written with an exponent instead. The wire
+    // scale is any 32-bit value, and expanding one near its limits would allocate billions of characters.
     static let maximumExpandedZeros = 64
 
-    static func decimalString(unscaled: Data, scale: Int32) -> String {
+    public static func decimalString(unscaled: Data, scale: Int32) -> String {
+        guard unscaled.count <= maximumDecimalBytes else {
+            return CassandraCellText.hex(unscaled) + "E" + String(-Int(scale))
+        }
         let integer = decimalString(fromTwosComplement: unscaled)
         let isNegative = integer.hasPrefix("-")
         var digits = isNegative ? String(integer.dropFirst()) : integer
@@ -46,7 +54,7 @@ enum CassandraVarint {
         return isNegative ? "-" + rendered : String(rendered)
     }
 
-    static func bytes(fromDecimalInteger text: String) -> Data? {
+    public static func bytes(fromDecimalInteger text: String) -> Data? {
         var body = Substring(text)
         var isNegative = false
         if body.first == "-" || body.first == "+" {
@@ -72,7 +80,7 @@ enum CassandraVarint {
         return Data(twosComplement(magnitude: trimmedMagnitude(magnitude), isNegative: isNegative))
     }
 
-    static func decimal(fromText text: String) -> (unscaled: Data, scale: Int32)? {
+    public static func decimal(fromText text: String) -> (unscaled: Data, scale: Int32)? {
         var mantissa = Substring(text)
         var exponent = 0
         if let marker = mantissa.firstIndex(where: { $0 == "e" || $0 == "E" }) {
@@ -96,25 +104,48 @@ enum CassandraVarint {
         return (unscaled, scale32)
     }
 
+    private static let chunkDivisor: UInt64 = 1_000_000_000
+    private static let chunkDigits = 9
+
+    // Long division of 32-bit limbs by 10^9, nine digits a pass.
     private static func decimalDigits(ofMagnitude magnitude: [UInt8]) -> String {
-        var remaining = trimmedMagnitude(magnitude)
-        guard remaining != [0] else { return "0" }
-        var digits: [Character] = []
-        while !(remaining.count == 1 && remaining[0] == 0) {
-            var remainder = 0
-            var quotient: [UInt8] = []
-            for byte in remaining {
-                let value = remainder << 8 | Int(byte)
-                let digit = value / 10
-                remainder = value % 10
-                if !(quotient.isEmpty && digit == 0) {
-                    quotient.append(UInt8(digit))
-                }
+        var limbs = Self.limbs(ofMagnitude: magnitude)
+        guard !limbs.isEmpty else { return "0" }
+        var chunks: [UInt64] = []
+        while !limbs.isEmpty {
+            var remainder: UInt64 = 0
+            for index in limbs.indices {
+                let value = remainder << 32 | UInt64(limbs[index])
+                limbs[index] = UInt32(value / chunkDivisor)
+                remainder = value % chunkDivisor
+            }
+            chunks.append(remainder)
+            limbs.removeFirst(limbs.prefix { $0 == 0 }.count)
+        }
+        var text = String(chunks.removeLast())
+        for chunk in chunks.reversed() {
+            let digits = String(chunk)
+            text += String(repeating: "0", count: chunkDigits - digits.count) + digits
+        }
+        return text
+    }
+
+    private static func limbs(ofMagnitude magnitude: [UInt8]) -> [UInt32] {
+        let significant = magnitude.drop { $0 == 0 }
+        var limbs: [UInt32] = []
+        limbs.reserveCapacity((significant.count + 3) / 4)
+        var limb: UInt32 = 0
+        var filled = (4 - significant.count % 4) % 4
+        for byte in significant {
+            limb = limb << 8 | UInt32(byte)
+            filled += 1
+            if filled == 4 {
+                limbs.append(limb)
+                limb = 0
+                filled = 0
             }
-            digits.append(Character(String(remainder)))
-            remaining = quotient.isEmpty ? [0] : quotient
         }
-        return String(digits.reversed())
+        return limbs
     }
 
     private static func trimmedMagnitude(_ magnitude: [UInt8]) -> [UInt8] {

apple/Packages/TableProCore/Sources/TableProCoreTypes/UniqueNameAllocator.swift

@@ -0,0 +1,40 @@
+// Gives each base name the lowest free ordinal, starting at 1. Remembering where each base
+// stopped keeps a run of repeated names linear: the ordinals below the mark are taken for good,
+// because a name, once taken, stays taken.
+public struct UniqueNameAllocator {
+    private var taken: Set<String>
+    private var nextOrdinal: [String: Int] = [:]
+    private let key: (String) -> String
+    private let candidate: (String, Int) -> String
+
+    public init(
+        taken: [String] = [],
+        key: @escaping (String) -> String = { $0 },
+        candidate: @escaping (_ base: String, _ ordinal: Int) -> String
+    ) {
+        self.key = key
+        self.candidate = candidate
+        self.taken = Set(taken.map(key))
+    }
+
+    public func contains(_ name: String) -> Bool {
+        taken.contains(key(name))
+    }
+
+    @discardableResult
+    public mutating func reserve(_ name: String) -> Bool {
+        taken.insert(key(name)).inserted
+    }
+
+    public mutating func allocate(_ base: String) -> String {
+        var ordinal = nextOrdinal[base, default: 1]
+        while true {
+            let name = candidate(base, ordinal)
+            ordinal += 1
+            if taken.insert(key(name)).inserted {
+                nextOrdinal[base] = ordinal
+                return name
+            }
+        }
+    }
+}