diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift new file mode 100644 index 0000000000..fee14447d7 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift @@ -0,0 +1,449 @@ +// +// AvroDecoder.swift +// TablePro +// + +import Foundation + +internal enum AvroFraming: Equatable, Sendable { + // Confluent Schema Registry: magic byte 0, then the schema ID, big-endian. + case confluent(schemaID: UInt32) + // Avro single-object encoding: C3 01, then the CRC-64-AVRO fingerprint of the writer schema, little-endian. + case singleObject(fingerprint: UInt64) +} + +internal struct AvroMessage: Equatable, Sendable { + let value: MessagePackValue + let framing: AvroFraming? +} + +// Values land in MessagePack's model, which already has the tree, the JSON writer and the timestamp badge. +// A union shows the branch it holds, as Avro's own readers return it. +internal enum AvroDecoder { + static let maxDepth = 256 + // Items that take no bytes (null, an empty record) are bounded by this, not by the bytes left. + static let maxZeroByteItems: Int64 = 1_000_000 + // Records of records that take no bytes describe exponentially many values in a short schema, so every value + // read counts, built or not. A grid cell reads synchronously and stops far sooner, showing hex past that. + static let maxValuesRead = 10_000_000 + static let maxValuesReadInCell = 1_000_000 + + // A single-object header that names this schema's fingerprint is certain, so it goes first. A Confluent + // prefix proves nothing (a plain datum can start with zero), so it is only tried once the plain read fails. + static func decode( + _ data: Data, + schema: AvroSchema, + maxValues: Int = .max, + maxValuesRead: Int = AvroDecoder.maxValuesRead + ) throws -> AvroMessage { + let limits = (build: maxValues, read: maxValuesRead) + let singleObject = singleObjectFingerprint(data) + if let singleObject, singleObject == schema.fingerprint { + let value = try datum(data.dropFirst(10), schema: schema, limits: limits) + return AvroMessage(value: value, framing: .singleObject(fingerprint: singleObject)) + } + do { + return AvroMessage(value: try datum(data, schema: schema, limits: limits), framing: nil) + } catch let plainError as BinaryValueDecodeError { + // Avro's own reader refuses a value written with a schema it does not have. + if let singleObject { throw BinaryValueDecodeError(kind: .otherSchema(fingerprint: singleObject), offset: 2) } + guard data.count >= 5, data.first == 0x00 else { throw plainError } + var header = BinaryValueReader(data) + _ = try header.readByte() + let framing = AvroFraming.confluent(schemaID: try header.readUInt32()) + let body = data.dropFirst(5) + // Confluent's serializer writes a bytes value as it is, with no length in front, a decimal on bytes included. + if let raw = rawBytes(Data(body), schema: schema) { + return AvroMessage(value: raw, framing: framing) + } + guard let value = try? datum(body, schema: schema, limits: limits) else { throw plainError } + return AvroMessage(value: value, framing: framing) + } + } + + private static func rawBytes(_ body: Data, schema: AvroSchema) -> MessagePackValue? { + switch schema.resolve(schema.root) { + case .bytes: + return .binary(body) + case .logical(let logical, let base): + guard case .bytes = schema.resolve(base) else { return nil } + return AvroLogicalRendering.render(.binary(body), as: logical) + default: + return nil + } + } + + private static func singleObjectFingerprint(_ data: Data) -> UInt64? { + guard data.count >= 10, data.starts(with: [0xC3, 0x01]) else { return nil } + var header = BinaryValueReader(data) + try? header.skip(2) + return try? header.readUInt64LittleEndian() + } + + private static func datum(_ data: Data, schema: AvroSchema, limits: (build: Int, read: Int)) throws -> MessagePackValue { + var parser = Parser(reader: BinaryValueReader(Data(data)), schema: schema, budget: limits.build, readLimit: limits.read) + let value = try parser.value(schema.root, depth: 0) + if !parser.reader.isAtEnd { + throw parser.reader.error(.trailingBytes(parser.reader.remaining)) + } + return value + } + + // Past the budget the parser keeps reading and stops building, so the cell judges the frame and the + // trailing bytes on the whole value, the same as the viewer does. + private struct Parser { + var reader: BinaryValueReader + let schema: AvroSchema + private var budget: Int + private var isTruncated = false + private var zeroByteItems: Int64 = 0 + private var valuesRead = 0 + private let readLimit: Int + + init(reader: BinaryValueReader, schema: AvroSchema, budget: Int, readLimit: Int) { + self.reader = reader + self.schema = schema + self.budget = budget + self.readLimit = readLimit + } + + // Only a dispatch: every kind of type is its own function, so the frames a deep value stacks up stay small. + mutating func value(_ type: AvroType, depth: Int) throws -> MessagePackValue { + guard depth <= AvroDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } + valuesRead += 1 + guard valuesRead <= readLimit else { throw reader.error(.tooLarge) } + if !isTruncated { + budget -= 1 + isTruncated = budget < 0 + } + let builds = !isTruncated + let resolved = schema.resolve(type) + switch resolved { + case .record(_, let fields): + return try record(fields, depth: depth, builds: builds) + case .array(let items): + return try array(items, depth: depth, builds: builds) + case .map(let valueType): + return try map(valueType, depth: depth, builds: builds) + case .union(let branches): + // The branch is the value itself, so it takes no share of the budget of its own. + budget += builds ? 1 : 0 + return try value(branches[try index(below: branches.count)], depth: depth) + case .logical(let logical, let base): + budget += builds ? 1 : 0 + return try annotated(logical, base, depth: depth, builds: builds) + case .named: + throw reader.error(.invalidField) + default: + return try primitive(resolved, builds: builds) + } + } + + private mutating func primitive(_ type: AvroType, builds: Bool) throws -> MessagePackValue { + switch type { + case .boolean: + let start = reader.offset + let byte = try reader.readByte() + guard byte <= 1 else { throw BinaryValueDecodeError(kind: .invalidByte(byte), offset: start) } + return builds ? .bool(byte == 1) : .truncated + case .int: + let number = try int() + return builds ? .int(number) : .truncated + case .long: + let number = try long() + return builds ? .int(number) : .truncated + case .float: + let bits = try reader.readUInt32LittleEndian() + return builds ? .float(Float(bitPattern: bits)) : .truncated + case .double: + let bits = try reader.readUInt64LittleEndian() + return builds ? .double(Double(bitPattern: bits)) : .truncated + case .bytes, .string, .fixed: + let count: Int + if case .fixed(_, let size) = type { count = size } else { count = try byteCount() } + guard builds else { + try reader.skip(count) + return .truncated + } + let data = try reader.readBytes(count) + guard case .string = type else { return .binary(data) } + // Writers that ignore the encoding put any bytes in a string, so those show as bytes. + return String(data: data, encoding: .utf8).map(MessagePackValue.string) ?? .binary(data) + case .enumeration(_, let symbols): + let symbol = symbols[try index(below: symbols.count)] + return builds ? .string(symbol) : .truncated + default: + return builds ? .null : .truncated + } + } + + private mutating func record(_ fields: [AvroField], depth: Int, builds: Bool) throws -> MessagePackValue { + var pairs: [MessagePackPair] = [] + pairs.reserveCapacity(builds ? fields.count : 0) + for field in fields { + let wasTruncated = isTruncated + let item = try value(field.type, depth: depth + 1) + if !wasTruncated { pairs.append(MessagePackPair(key: .string(field.name), value: item)) } + } + return builds ? .map(pairs) : .truncated + } + + private mutating func array(_ items: AvroType, depth: Int, builds: Bool) throws -> MessagePackValue { + var values: [MessagePackValue] = [] + try blocks(takesNoBytes: schema.takesNoBytes(items)) { parser in + let wasTruncated = parser.isTruncated + let item = try parser.value(items, depth: depth + 1) + if !wasTruncated { values.append(item) } + } + return builds ? .array(values) : .truncated + } + + private mutating func map(_ valueType: AvroType, depth: Int, builds: Bool) throws -> MessagePackValue { + var pairs: [MessagePackPair] = [] + try blocks(takesNoBytes: false) { parser in + let wasTruncated = parser.isTruncated + let key = try parser.value(.string, depth: depth + 1) + let item = try parser.value(valueType, depth: depth + 1) + if !wasTruncated { pairs.append(MessagePackPair(key: key, value: item)) } + } + return builds ? .map(pairs) : .truncated + } + + private mutating func annotated(_ logical: AvroLogicalType, _ base: AvroType, depth: Int, builds: Bool) throws -> MessagePackValue { + let raw = try value(base, depth: depth) + guard builds, raw != .truncated else { return raw } + if case .decimal = logical, case .binary(let bytes) = raw { + // Long division costs the square of the width, so a wide decimal draws on the budget by its width. + budget -= AvroDecimal.significantBytes([UInt8](bytes)).count + } + return AvroLogicalRendering.render(raw, as: logical) + } + + // Each block is a count, then that many items; a negative count is followed by the block's byte size. + private mutating func blocks(takesNoBytes: Bool, _ item: (inout Parser) throws -> Void) throws { + while true { + let start = reader.offset + var count = try long() + if count == 0 { return } + if count < 0 { + guard count != .min else { throw BinaryValueDecodeError(kind: .integerOutOfRange, offset: start) } + count = -count + _ = try long() + } + // Every item takes a byte at least, unless its type takes none; then only the cap bounds the loop. + if takesNoBytes { + guard count <= AvroDecoder.maxZeroByteItems - zeroByteItems else { + throw BinaryValueDecodeError(kind: .tooLarge, offset: start) + } + zeroByteItems += count + } else if count > Int64(reader.remaining) { + throw BinaryValueDecodeError(kind: .unexpectedEnd, offset: reader.end) + } + for _ in 0.. Int { + let start = reader.offset + let index = try long() + guard index >= 0, index < count else { + throw BinaryValueDecodeError(kind: .indexOutOfRange(Int(clamping: index)), offset: start) + } + return Int(index) + } + + private mutating func byteCount() throws -> Int { + let start = reader.offset + let count = try long() + guard count >= 0 else { throw BinaryValueDecodeError(kind: .invalidLength, offset: start) } + guard count <= Int64(reader.remaining) else { throw reader.error(.unexpectedEnd) } + return Int(count) + } + + private mutating func int() throws -> Int64 { + let start = reader.offset + let value = try long() + guard let narrowed = Int32(exactly: value) else { + throw BinaryValueDecodeError(kind: .integerOutOfRange, offset: start) + } + return Int64(narrowed) + } + + private mutating func long() throws -> Int64 { + let raw = try reader.readVarint() + return Int64(bitPattern: raw >> 1) ^ -Int64(bitPattern: raw & 1) + } + } +} + +internal enum AvroLogicalRendering { + static func render(_ value: MessagePackValue, as logical: AvroLogicalType) -> MessagePackValue { + switch (logical, value) { + case (.decimal(let precision, let scale), .binary(let data)): + let unscaled = AvroDecimal.significantBytes([UInt8](data)) + guard unscaled.count <= AvroDecimal.maxBytes(precision: precision) else { return value } + return .string(AvroDecimal.text(unscaled: unscaled, scale: scale)) + case (.uuid, .string): + return value + case (.uuid, .binary(let data)) where data.count == 16: + return .string(UUID(uuid: uuidTuple([UInt8](data))).uuidString.lowercased()) + case (.date, .int(let days)): + return .string(dateText(days: days)) + case (.timeMillis, .int(let millis)): + return timeText(units: millis, perSecond: 1_000).map(MessagePackValue.string) ?? value + case (.timeMicros, .int(let micros)): + return timeText(units: micros, perSecond: 1_000_000).map(MessagePackValue.string) ?? value + case (.timestampMillis, .int(let millis)): + return timestamp(units: millis, perSecond: 1_000) + case (.timestampMicros, .int(let micros)): + return timestamp(units: micros, perSecond: 1_000_000) + case (.timestampNanos, .int(let nanos)): + return timestamp(units: nanos, perSecond: 1_000_000_000) + case (.localTimestampMillis, .int(let millis)): + return localText(timestamp(units: millis, perSecond: 1_000)).map(MessagePackValue.string) ?? value + case (.localTimestampMicros, .int(let micros)): + return localText(timestamp(units: micros, perSecond: 1_000_000)).map(MessagePackValue.string) ?? value + case (.localTimestampNanos, .int(let nanos)): + return localText(timestamp(units: nanos, perSecond: 1_000_000_000)).map(MessagePackValue.string) ?? value + case (.duration, .binary(let data)) where data.count == 12: + return .string(durationText([UInt8](data))) + default: + return value + } + } + + private static func timestamp(units: Int64, perSecond: Int64) -> MessagePackValue { + let (seconds, remainder) = floorDivide(units, perSecond) + return .timestamp(seconds: seconds, nanoseconds: UInt32(remainder * (1_000_000_000 / perSecond))) + } + + // A local timestamp names no zone, so it reads as the wall-clock text without the trailing Z. + private static func localText(_ timestamp: MessagePackValue) -> String? { + guard case .timestamp(let seconds, let nanoseconds) = timestamp else { return nil } + let text = RFC3339Timestamp.format(seconds: seconds, nanoseconds: nanoseconds) + return text.hasSuffix("Z") ? String(text.dropLast()) : nil + } + + private static func dateText(days: Int64) -> String { + let (seconds, overflow) = days.multipliedReportingOverflow(by: 86_400) + guard !overflow else { return String(days) } + let text = RFC3339Timestamp.format(seconds: seconds, nanoseconds: 0) + return text.firstIndex(of: "T").map { String(text[..<$0]) } ?? text + } + + private static func timeText(units: Int64, perSecond: Int64) -> String? { + guard units >= 0, units < 86_400 * perSecond else { return nil } + let seconds = units / perSecond + var text = String(format: "%02lld:%02lld:%02lld", seconds / 3_600, seconds % 3_600 / 60, seconds % 60) + let fraction = units % perSecond + let digits = perSecond == 1_000 ? 3 : 6 + if fraction > 0 { + text += "." + String(format: "%0\(digits)lld", fraction) + } + return text + } + + // Three little-endian unsigned ints: months, days, milliseconds, written as an ISO 8601 duration. + private static func durationText(_ bytes: [UInt8]) -> String { + func part(_ offset: Int) -> UInt64 { + (0..<4).reduce(0) { $0 | UInt64(bytes[offset + $1]) << (8 * UInt64($1)) } + } + let months = part(0), days = part(4), millis = part(8) + var text = "P" + if months > 0 { text += "\(months)M" } + if days > 0 { text += "\(days)D" } + if millis > 0 { + text += "T\(millis / 1_000)" + if millis % 1_000 > 0 { text += String(format: ".%03llu", millis % 1_000) } + text += "S" + } + return text == "P" ? "PT0S" : text + } + + private static func floorDivide(_ value: Int64, _ divisor: Int64) -> (Int64, Int64) { + let quotient = value / divisor + let remainder = value % divisor + return remainder < 0 ? (quotient - 1, remainder + divisor) : (quotient, remainder) + } + + private static func uuidTuple(_ bytes: [UInt8]) -> uuid_t { + (bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]) + } +} + +// Big-endian two's complement of any width, as Avro's decimal stores the unscaled value. +internal enum AvroDecimal { + // The most digits a two's complement of this many bytes holds: floor(log10(2^(8n-1) - 1)). + static func maxPrecision(bytes: Int) -> Int { + guard bytes > 0 else { return 0 } + return Int((Double(8 * min(bytes, 100_000) - 1) * log10(2.0)).rounded(.down)) + } + + // The fewest bytes that hold any value of this many digits, sign bit included. + static func maxBytes(precision: Int) -> Int { + Int(((Double(precision) * log2(10.0) + 1) / 8).rounded(.up)) + } + + // Leading bytes that only repeat the sign; a fixed pads with them. + static func significantBytes(_ bytes: [UInt8]) -> ArraySlice { + var start = bytes.startIndex + while bytes.endIndex - start > 1 { + let lead = bytes[start], next = bytes[start + 1] + guard (lead == 0x00 && next < 0x80) || (lead == 0xFF && next >= 0x80) else { break } + start += 1 + } + return bytes[start...] + } + + static func text(unscaled bytes: ArraySlice, scale: Int) -> String { + guard let first = bytes.first else { return "0" } + let isNegative = first & 0x80 != 0 + var magnitude = Array(bytes) + if isNegative { + var carry = true + for index in magnitude.indices.reversed() { + magnitude[index] = ~magnitude[index] + if carry { + let (sum, overflow) = magnitude[index].addingReportingOverflow(1) + magnitude[index] = sum + carry = overflow + } + } + } + var words = Self.words(magnitude) + // Base 10^9 groups, least significant first: one pass of division yields nine digits. + var groups: [UInt32] = [] + while let lead = words.firstIndex(where: { $0 != 0 }) { + words.removeFirst(lead) + var remainder: UInt64 = 0 + for index in words.indices { + let current = remainder << 32 | UInt64(words[index]) + words[index] = UInt32(current / 1_000_000_000) + remainder = current % 1_000_000_000 + } + groups.append(UInt32(remainder)) + } + var text = groups.last.map { String($0) } ?? "0" + for group in groups.dropLast().reversed() { + let digits = String(group) + text += String(repeating: "0", count: 9 - digits.count) + digits + } + if scale > 0 { + if text.count <= scale { + text = String(repeating: "0", count: scale - text.count + 1) + text + } + text.insert(".", at: text.index(text.endIndex, offsetBy: -scale)) + } + return isNegative ? "-" + text : text + } + + private static func words(_ bytes: [UInt8]) -> [UInt32] { + let padded = [UInt8](repeating: 0, count: (4 - bytes.count % 4) % 4) + bytes + return stride(from: 0, to: padded.count, by: 4).map { start in + padded[start..<(start + 4)].reduce(0) { $0 << 8 | UInt32($1) } + } + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift new file mode 100644 index 0000000000..583dc7fc02 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift @@ -0,0 +1,339 @@ +// +// AvroSchema.swift +// TablePro +// + +import Foundation + +internal enum AvroLogicalType: Equatable, Sendable { + case decimal(precision: Int, scale: Int) + case uuid + case date + case timeMillis + case timeMicros + case timestampMillis + case timestampMicros + case timestampNanos + case localTimestampMillis + case localTimestampMicros + case localTimestampNanos + case duration +} + +internal struct AvroField: Equatable, Sendable { + let name: String + let type: AvroType +} + +internal indirect enum AvroType: Equatable, Sendable { + case null + case boolean + case int + case long + case float + case double + case bytes + case string + case record(name: String, fields: [AvroField]) + case enumeration(name: String, symbols: [String]) + case array(AvroType) + case map(AvroType) + case union([AvroType]) + case fixed(name: String, size: Int) + case named(String) + case logical(AvroLogicalType, AvroType) +} + +internal struct AvroSchemaError: Error, Equatable, Sendable { + let reason: String +} + +// https://avro.apache.org/docs/1.12.0/specification/ +// A named type is stored once and referenced by full name, so a recursive record parses. +internal struct AvroSchema: Equatable, Sendable { + let root: AvroType + let namedTypes: [String: AvroType] + // CRC-64-AVRO of the Parsing Canonical Form, what a single-object encoded value names its writer schema by. + let fingerprint: UInt64 + // Named types whose values take no bytes, worked out once: walking a shared type on every check is exponential. + let zeroByteNames: Set + + static func parse(_ text: String) throws -> AvroSchema { + let json: Any + do { + json = try JSONSerialization.jsonObject(with: Data(text.utf8), options: [.fragmentsAllowed]) + } catch { + throw AvroSchemaError(reason: String(localized: "The schema is not valid JSON.")) + } + var parser = Parser() + let root = try parser.type(json, namespace: nil) + return AvroSchema( + root: root, + namedTypes: parser.namedTypes, + fingerprint: fingerprint(of: canonicalForm(root)), + zeroByteNames: zeroByteNames(parser.completed) + ) + } + + // In the order definitions finish, so every name a type refers to is settled before it, except its own and its + // enclosing types', which a value cannot hold without bytes anyway. No recursion: a schema can chain thousands. + private static func zeroByteNames(_ completed: [(name: String, type: AvroType)]) -> Set { + var names: Set = [] + func takesNoBytes(_ type: AvroType) -> Bool { + switch type { + case .null: return true + case .fixed(_, let size): return size == 0 + case .named(let name), .record(let name, _), .enumeration(let name, _): return names.contains(name) + case .logical(_, let base): return takesNoBytes(base) + default: return false + } + } + for (name, type) in completed { + let base: AvroType + if case .logical(_, let inner) = type { base = inner } else { base = type } + switch base { + case .record(_, let fields) where fields.allSatisfy({ takesNoBytes($0.type) }): names.insert(name) + case .fixed(_, 0): names.insert(name) + default: break + } + } + return names + } + + // https://avro.apache.org/docs/1.12.0/specification/#parsing-canonical-form-for-schemas + // Full names, only the attributes that shape the bytes, in the specified order, no whitespace. A named + // type is written out where the schema defines it and by name everywhere else, as the parsed tree holds it. + static func canonicalForm(_ type: AvroType) -> String { + switch type { + case .null: return #""null""# + case .boolean: return #""boolean""# + case .int: return #""int""# + case .long: return #""long""# + case .float: return #""float""# + case .double: return #""double""# + case .bytes: return #""bytes""# + case .string: return #""string""# + case .named(let name): return quoted(name) + case .logical(_, let base): return canonicalForm(base) + case .union(let branches): return "[" + branches.map(canonicalForm).joined(separator: ",") + "]" + case .array(let items): return #"{"type":"array","items":"# + canonicalForm(items) + "}" + case .map(let values): return #"{"type":"map","values":"# + canonicalForm(values) + "}" + case .fixed(let name, let size): return #"{"name":"# + quoted(name) + #","type":"fixed","size":"# + "\(size)}" + case .enumeration(let name, let symbols): + return #"{"name":"# + quoted(name) + #","type":"enum","symbols":["# + symbols.map(quoted).joined(separator: ",") + "]}" + case .record(let name, let fields): + let members = fields.map { #"{"name":"# + quoted($0.name) + #","type":"# + canonicalForm($0.type) + "}" } + return #"{"name":"# + quoted(name) + #","type":"record","fields":["# + members.joined(separator: ",") + "]}" + } + } + + static func fingerprint(of canonicalForm: String) -> UInt64 { + canonicalForm.utf8.reduce(emptyFingerprint) { ($0 >> 8) ^ fingerprintTable[Int(($0 ^ UInt64($1)) & 0xFF)] } + } + + private static let emptyFingerprint: UInt64 = 0xC15D_213A_A4D7_A795 + + private static let fingerprintTable: [UInt64] = (0..<256).map { index in + var value = UInt64(index) + for _ in 0..<8 { + value = (value >> 1) ^ (emptyFingerprint & (0 &- (value & 1))) + } + return value + } + + private static func quoted(_ text: String) -> String { + "\"" + text.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "\"", with: "\\\"") + "\"" + } + + // Far past any engine's decimal; a larger one would only let a schema file pad a cell with zeros. + static let maxDecimalPrecision = 1_000 + + // The root type's own name, without its namespace, for menus. + var displayName: String? { + Self.name(of: root) + } + + private static func name(of type: AvroType) -> String? { + switch type { + case .record(let name, _), .enumeration(let name, _), .fixed(let name, _), .named(let name): + return name.split(separator: ".").last.map(String.init) + case .logical(_, let base): + return Self.name(of: base) + case .union(let branches): + return branches.lazy.compactMap { Self.name(of: $0) }.first + case .array, .map, .null, .boolean, .int, .long, .float, .double, .bytes, .string: + return nil + } + } + + func resolve(_ type: AvroType) -> AvroType { + if case .named(let name) = type, let defined = namedTypes[name] { return defined } + return type + } + + // Items of these types take no bytes, so an array of them can hold more items than bytes remain. + func takesNoBytes(_ type: AvroType) -> Bool { + switch type { + case .null: return true + case .fixed(_, let size): return size == 0 + case .logical(_, let base): return takesNoBytes(base) + case .named(let name), .record(let name, _): return zeroByteNames.contains(name) + default: return false + } + } + + private struct Parser { + var namedTypes: [String: AvroType] = [:] + private(set) var completed: [(name: String, type: AvroType)] = [] + + mutating func type(_ json: Any, namespace: String?) throws -> AvroType { + if let name = json as? String { + return try reference(name, namespace: namespace) + } + if let branches = json as? [Any] { + return .union(try branches.map { try type($0, namespace: namespace) }) + } + guard let object = json as? [String: Any] else { + throw AvroSchemaError(reason: String(localized: "The schema has a type that is not a name, an object or a union.")) + } + let base = try complex(object, namespace: namespace) + guard let logicalName = object["logicalType"] as? String, + let logical = Self.logical(logicalName, object: object, base: base) else { return base } + let annotated = AvroType.logical(logical, base) + // The logical type belongs to the named type, so a later reference by name keeps it. + if case .fixed(let name, _) = base { namedTypes[name] = annotated } + return annotated + } + + private mutating func complex(_ object: [String: Any], namespace: String?) throws -> AvroType { + guard let typeName = object["type"] else { + throw AvroSchemaError(reason: String(localized: "A schema object has no \"type\".")) + } + guard let kind = typeName as? String else { return try type(typeName, namespace: namespace) } + switch kind { + case "record", "error": + let name = try newName(object, namespace: namespace) + namedTypes[name] = .record(name: name, fields: []) + let inner = Self.namespace(of: name) + guard let rawFields = object["fields"] as? [[String: Any]] else { + throw AvroSchemaError(reason: String(format: String(localized: "Record %@ has no fields list."), name)) + } + let fields = try rawFields.map { field -> AvroField in + guard let fieldName = field["name"] as? String, let fieldType = field["type"] else { + throw AvroSchemaError(reason: String(format: String(localized: "A field of record %@ has no name or type."), name)) + } + return AvroField(name: fieldName, type: try type(fieldType, namespace: inner)) + } + let record = AvroType.record(name: name, fields: fields) + namedTypes[name] = record + completed.append((name, record)) + return record + case "enum": + let name = try newName(object, namespace: namespace) + guard let symbols = object["symbols"] as? [String] else { + throw AvroSchemaError(reason: String(format: String(localized: "Enum %@ has no symbols."), name)) + } + let enumeration = AvroType.enumeration(name: name, symbols: symbols) + namedTypes[name] = enumeration + completed.append((name, enumeration)) + return enumeration + case "fixed": + let name = try newName(object, namespace: namespace) + guard let size = Self.integer(object["size"]), size >= 0 else { + throw AvroSchemaError(reason: String(format: String(localized: "Fixed type %@ has no size."), name)) + } + let fixed = AvroType.fixed(name: name, size: size) + namedTypes[name] = fixed + completed.append((name, fixed)) + return fixed + case "array": + guard let items = object["items"] else { + throw AvroSchemaError(reason: String(localized: "An array has no \"items\".")) + } + return .array(try type(items, namespace: namespace)) + case "map": + guard let values = object["values"] else { + throw AvroSchemaError(reason: String(localized: "A map has no \"values\".")) + } + return .map(try type(values, namespace: namespace)) + default: + return try reference(kind, namespace: namespace) + } + } + + private func reference(_ name: String, namespace: String?) throws -> AvroType { + switch name { + case "null": return .null + case "boolean": return .boolean + case "int": return .int + case "long": return .long + case "float": return .float + case "double": return .double + case "bytes": return .bytes + case "string": return .string + default: break + } + if !name.contains("."), let namespace, namedTypes["\(namespace).\(name)"] != nil { + return .named("\(namespace).\(name)") + } + guard namedTypes[name] != nil else { + throw AvroSchemaError(reason: String(format: String(localized: "The schema names type %@ before defining it."), name)) + } + return .named(name) + } + + private func newName(_ object: [String: Any], namespace: String?) throws -> String { + guard let name = object["name"] as? String, !name.isEmpty else { + throw AvroSchemaError(reason: String(localized: "A record, enum or fixed type has no name.")) + } + var fullName = name + if !name.contains("."), let space = object["namespace"] as? String ?? namespace, !space.isEmpty { + fullName = "\(space).\(name)" + } + guard namedTypes[fullName] == nil else { + throw AvroSchemaError(reason: String(format: String(localized: "The schema defines type %@ twice."), fullName)) + } + return fullName + } + + private static func namespace(of fullName: String) -> String? { + guard let dot = fullName.lastIndex(of: ".") else { return nil } + return String(fullName[.. AvroLogicalType? { + switch (name, base) { + case ("decimal", .bytes): + return decimal(object, maxPrecision: AvroSchema.maxDecimalPrecision) + case ("decimal", .fixed(_, let size)): + return decimal(object, maxPrecision: min(AvroSchema.maxDecimalPrecision, AvroDecimal.maxPrecision(bytes: size))) + case ("uuid", .string), ("uuid", .fixed(_, 16)): return .uuid + case ("date", .int): return .date + case ("time-millis", .int): return .timeMillis + case ("time-micros", .long): return .timeMicros + case ("timestamp-millis", .long): return .timestampMillis + case ("timestamp-micros", .long): return .timestampMicros + case ("timestamp-nanos", .long): return .timestampNanos + case ("local-timestamp-millis", .long): return .localTimestampMillis + case ("local-timestamp-micros", .long): return .localTimestampMicros + case ("local-timestamp-nanos", .long): return .localTimestampNanos + case ("duration", .fixed(_, 12)): return .duration + default: return nil + } + } + + private static func decimal(_ object: [String: Any], maxPrecision: Int) -> AvroLogicalType? { + guard let precision = integer(object["precision"]), precision >= 1, precision <= maxPrecision else { return nil } + let scale = object["scale"] == nil ? 0 : integer(object["scale"]) ?? -1 + return (0...precision).contains(scale) ? .decimal(precision: precision, scale: scale) : nil + } + + // `as? Int` also takes `true` and `4.0`, which Avro's own parser refuses as a size, precision or scale. + private static func integer(_ value: Any?) -> Int? { + guard let number = value as? NSNumber, CFGetTypeID(number) != CFBooleanGetTypeID(), + !CFNumberIsFloatType(number) else { return nil } + return Int(exactly: number.int64Value) + } + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift index 2daceec8b2..4bd1fcbc58 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift @@ -5,14 +5,18 @@ import Foundation -internal enum BinaryValueFormat: String, Equatable, Sendable { +internal enum BinaryValueFormat: Hashable, Sendable { case messagePack case protobuf + case igbinary + case avro(schemaID: String) var displayName: String { switch self { case .messagePack: return "MessagePack" case .protobuf: return "Protobuf" + case .igbinary: return "igbinary" + case .avro: return "Avro" } } } @@ -20,11 +24,14 @@ internal enum BinaryValueFormat: String, Equatable, Sendable { internal enum BinaryValueTree: Sendable { case json(JSONTreeNode) case protobuf(ProtobufTreeNode) + case php(PhpTreeNode) } internal enum BinaryValueDocument: Sendable { case decoded(tree: BinaryValueTree, text: String, note: String?) case failed(BinaryValueDecodeError) + // Nothing to judge the bytes by: the format names a schema this Mac no longer has. + case missingSchema } // Pure: the grid calls it per cell on the main actor, the viewer off it. @@ -48,6 +55,14 @@ internal enum BinaryValueDecoding { case .protobuf: let fields = try ProtobufWireDecoder.decode(data, maxValues: budget).fields text = ProtobufTextWriter.text(fields, multiLine: false, limit: limit) + case .igbinary: + text = PhpSerializeWriter.text(try IgbinaryDecoder.decode(data, maxValues: budget), limit: limit) + case .avro(let schemaID): + guard let schema = AvroSchemaStore.shared.schema(id: schemaID) else { return nil } + let value = try AvroDecoder.decode( + data, schema: schema, maxValues: budget, maxValuesRead: AvroDecoder.maxValuesReadInCell + ).value + text = MessagePackTextWriter.json(value, pretty: false, limit: limit) } } catch { return nil @@ -92,6 +107,27 @@ internal enum BinaryValueDecoding { ), note: message.framing.map(framingNote) ) + case .igbinary: + let value = try IgbinaryDecoder.decode(data) + return .decoded( + tree: .php(PhpTreeBuilder.build(from: value)), + text: capped( + PhpSerializeWriter.text(value, limit: writerLimit(forCap: maxViewerTextLength)), + at: maxViewerTextLength + ), + note: nil + ) + case .avro(let schemaID): + guard let schema = AvroSchemaStore.shared.schema(id: schemaID) else { return .missingSchema } + let message = try AvroDecoder.decode(data, schema: schema) + return .decoded( + tree: .json(MessagePackTreeBuilder.build(from: message.value)), + text: capped( + MessagePackTextWriter.json(message.value, pretty: true, limit: writerLimit(forCap: maxViewerTextLength)), + at: maxViewerTextLength + ), + note: message.framing.map(framingNote) + ) } } catch let error as BinaryValueDecodeError { return .failed(error) @@ -100,6 +136,18 @@ internal enum BinaryValueDecoding { } } + private static func framingNote(_ framing: AvroFraming) -> String { + switch framing { + case .confluent(let schemaID): + return String(format: String(localized: "Schema Registry framing: schema ID %lld"), Int(schemaID)) + case .singleObject(let fingerprint): + return String( + format: String(localized: "Avro single-object encoding: schema fingerprint %@"), + String(format: "%016llX", fingerprint) + ) + } + } + private static func framingNote(_ framing: ProtobufFraming) -> String { let indexes = framing.messageIndexes.map(String.init).joined(separator: ", ") return String( diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift index a33a2bf9f1..edcb95f1c1 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift @@ -15,6 +15,11 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { case integerOutOfRange case invalidField case unmatchedGroupEnd + case unknownVersion(UInt32) + case unknownReference(Int) + case indexOutOfRange(Int) + case invalidLength + case otherSchema(fingerprint: UInt64) } let kind: Kind @@ -42,6 +47,23 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { return String(format: String(localized: "Byte %lld does not start a valid field."), offset) case .unmatchedGroupEnd: return String(format: String(localized: "The group end at byte %lld matches no group start."), offset) + case .unknownVersion(let version): + return String(format: String(localized: "The header names version %lld, which this format does not have."), Int(version)) + case .unknownReference(let index): + return String( + format: String(localized: "Byte %lld refers back to item %lld, which comes later or not at all."), + offset, + index + ) + case .indexOutOfRange(let index): + return String(format: String(localized: "Byte %lld picks entry %lld, which the schema does not have."), offset, index) + case .invalidLength: + return String(format: String(localized: "The length at byte %lld is negative."), offset) + case .otherSchema(let fingerprint): + return String( + format: String(localized: "The value was written with the schema whose fingerprint is %@, not this column's."), + String(format: "%016llX", fingerprint) + ) } } } diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift new file mode 100644 index 0000000000..d69f1b506e --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/IgbinaryDecoder.swift @@ -0,0 +1,402 @@ +// +// IgbinaryDecoder.swift +// TablePro +// + +import Foundation + +// https://github.com/igbinary/igbinary/blob/master/src/php7/igbinary.c +// Values are numbered the way PHP's serialize() numbers them (php_add_var_hash in ext/standard/var.c), so the +// text reads the same `r:` and `R:` serialize() writes. +internal enum IgbinaryDecoder { + static let maxDepth = PhpSerializeParser.depthCap + // A repeated array or string ID costs a few bytes and expands in the tree, its text and Find. + static let maxExpandedValues = 10_000_000 + static let maxExpandedStringBytes = 20_000_000 + + static func decode(_ data: Data, maxValues: Int = .max) throws -> PhpValue { + var reader = BinaryValueReader(data) + let version = try reader.readUInt32() + guard version == 1 || version == 2 else { + throw BinaryValueDecodeError(kind: .unknownVersion(version), offset: 0) + } + var parser = Parser(reader: reader, budget: maxValues) + let value = try parser.value(depth: 0).value + if !parser.isTruncated, !parser.reader.isAtEnd { + throw parser.reader.error(.trailingBytes(parser.reader.remaining)) + } + return value + } + + private enum Tag: UInt8 { + case null = 0x00 + case ref8, ref16, ref32 + case boolFalse, boolTrue + case long8p, long8n, long16p, long16n, long32p, long32n + case double + case stringEmpty + case stringID8, stringID16, stringID32 + case string8, string16, string32 + case array8, array16, array32 + case object8, object16, object32 + case objectID8, objectID16, objectID32 + case objectSer8, objectSer16, objectSer32 + case long64p, long64n + case objref8, objref16, objref32 + case reference + case string64 + case enumCase + } + + // What serialize() writes when this value turns up again inside an array igbinary repeats by value: + // a PHP reference as `R:`, an object as `r:`, anything else written out again. + private struct Copy { + let value: PhpValue + let slots: Int + let stringBytes: Int + // Levels of nesting below the copy, so one placed deeper than the original still fits the depth cap. + let height: Int + + static func of(_ value: PhpValue, slots: Int, stringBytes: Int = 0) -> Copy { + Copy(value: value, slots: slots, stringBytes: stringBytes, height: 0) + } + } + + private struct Decoded { + let value: PhpValue + let copy: Copy + } + + private struct Entry { + let key: PhpValue + let keyBytes: Int + let item: Decoded + } + + private struct Reference { + let slot: Int + // Nil while an array is still being read, so a repeat of it from inside is refused. + var copy: Copy? + } + + private struct Parser { + var reader: BinaryValueReader + var budget: Int + var isTruncated = false + private var strings: [PhpValue] = [] + // igbinary's table: every array, every object, and every PHP reference to anything else. + private var references: [Reference] = [] + // The number serialize() gives the value being read. + private var slot = 0 + private var stringBytes = 0 + + init(reader: BinaryValueReader, budget: Int) { + self.reader = reader + self.budget = budget + } + + mutating func value(depth: Int) throws -> Decoded { + guard depth <= IgbinaryDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } + guard budget > 0 else { + isTruncated = true + return Decoded(value: .tooLarge, copy: .of(.tooLarge, slots: 1)) + } + budget -= 1 + try expand(slots: 1, stringBytes: 0) + return try value(tag: try readTag(), depth: depth, isReference: false) + } + + // Only a dispatch: every case is its own function, so the frames a deep value stacks up stay small. + private mutating func value(tag: Tag, depth: Int, isReference: Bool) throws -> Decoded { + switch tag { + case .reference: + guard !isReference else { throw invalid(tag) } + return try phpReference(depth: depth) + case .ref8, .ref16, .ref32, .objref8, .objref16, .objref32: + return try repeated(tag, depth: depth, isReference: isReference) + case .array8, .array16, .array32: + return try array(tag, depth: depth) + case .object8, .object16, .object32, .objectID8, .objectID16, .objectID32: + return try object(tag, depth: depth) + case .objectSer8, .objectSer16, .objectSer32, .enumCase: + throw invalid(tag) + default: + return try scalar(tag) + } + } + + private mutating func scalar(_ tag: Tag) throws -> Decoded { + let value: PhpValue + var bytes = 0 + switch tag { + case .null: value = .null + case .boolFalse: value = .bool(false) + case .boolTrue: value = .bool(true) + case .double: value = .float(Double(bitPattern: try reader.readUInt64())) + case .stringEmpty, .stringID8, .stringID16, .stringID32, .string8, .string16, .string32, .string64: + let text = try string(tag) + value = text.value + bytes = text.bytes + default: value = .int(try integer(tag)) + } + return Decoded(value: value, copy: .of(value, slots: 1, stringBytes: bytes)) + } + + private mutating func phpReference(depth: Int) throws -> Decoded { + let own = slot + let target = try value(tag: try readTag(), depth: depth, isReference: true) + if case .reference = target.value { return target } + let copy = Copy.of(.reference(id: own), slots: 0) + switch target.value { + case .array, .object, .serializable, .binarySerializable, .enumCase: break + default: references.append(Reference(slot: own, copy: copy)) + } + return Decoded(value: target.value, copy: copy) + } + + private mutating func repeated(_ tag: Tag, depth: Int, isReference: Bool) throws -> Decoded { + let start = reader.offset + let target = try reference(width: tag) + if isReference { + // A PHP reference to a value already numbered takes no number of its own. + slot -= 1 + let value = PhpValue.reference(id: target.slot) + return Decoded(value: value, copy: .of(value, slots: 0)) + } + if tag == .objref8 || tag == .objref16 || tag == .objref32 { + let value = PhpValue.objectReference(id: target.slot) + return Decoded(value: value, copy: .of(value, slots: 1)) + } + // The same array again by value, which serialize() writes out in full. + guard let copy = target.copy else { + throw BinaryValueDecodeError(kind: .unknownReference(target.index), offset: start) + } + guard depth + copy.height <= IgbinaryDecoder.maxDepth else { throw reader.error(.nestingTooDeep) } + try expand(slots: copy.slots - 1, stringBytes: copy.stringBytes) + return Decoded(value: copy.value, copy: copy) + } + + private mutating func array(_ tag: Tag, depth: Int) throws -> Decoded { + let own = slot + let count = try length(tag) + let index = references.count + references.append(Reference(slot: own, copy: nil)) + let entries = try entries(count: count, depth: depth) + let copy = Self.copy(of: entries) + references[index].copy = copy + return Decoded(value: .array(entries.map { PhpKeyValue(key: $0.key, value: $0.item.value) }), copy: copy) + } + + private static func copy(of entries: [Entry]) -> Copy { + Copy( + value: .array(entries.map { PhpKeyValue(key: $0.key, value: $0.item.copy.value) }), + slots: entries.reduce(1) { $0 + $1.item.copy.slots }, + stringBytes: entries.reduce(0) { $0 + $1.keyBytes + $1.item.copy.stringBytes }, + height: entries.map { $0.item.copy.height + 1 }.max() ?? 0 + ) + } + + // An object takes its number before its properties are read, so one that holds itself refers back to it. + private mutating func object(_ nameTag: Tag, depth: Int) throws -> Decoded { + let own = slot + let className = Self.text(try string(nameTag).value) + let copy = Copy.of(.objectReference(id: own), slots: 1) + references.append(Reference(slot: own, copy: copy)) + let tag = try readTag() + let value: PhpValue + switch tag { + case .array8, .array16, .array32: + let properties = try entries(count: try length(tag), depth: depth).map { entry -> PhpProperty in + if case .int(let index) = entry.key { + return PhpProperty(name: String(index), visibility: .publicVisibility, value: entry.item.value, index: index) + } + let property = decodePhpPropertyKey(Self.text(entry.key)) + var binaryKey: Data? + if case .binaryString(let bytes) = entry.key { binaryKey = bytes } + return PhpProperty( + name: property.name, visibility: property.visibility, value: entry.item.value, binaryKey: binaryKey + ) + } + value = .object(className: className, properties: properties) + case .objectSer8, .objectSer16, .objectSer32: + let payload = try reader.readBytes(try length(tag)) + value = String(data: payload, encoding: .utf8).map { .serializable(className: className, rawPayload: $0) } + ?? .binarySerializable(className: className, payload: payload) + case .enumCase: + let caseTag = try readTag() + switch caseTag { + case .string8, .string16, .string32, .string64, .stringID8, .stringID16, .stringID32: + value = .enumCase(className: className, caseName: Self.text(try string(caseTag).value)) + default: + throw invalid(caseTag) + } + default: + throw invalid(tag) + } + return Decoded(value: value, copy: copy) + } + + // A count past the bytes left is forged; checking first keeps it from reserving gigabytes. + private mutating func entries(count: Int, depth: Int) throws -> [Entry] { + guard count <= reader.remaining else { throw reader.error(.unexpectedEnd) } + var entries: [Entry] = [] + entries.reserveCapacity(min(count, 1_024)) + for _ in 0.. (value: PhpValue, bytes: Int)? { + let tag = try readTag() + switch tag { + case .null: + return nil + case .long8p, .long8n, .long16p, .long16n, .long32p, .long32n, .long64p, .long64n: + return (.int(try integer(tag)), 0) + case .stringEmpty, .stringID8, .stringID16, .stringID32, .string8, .string16, .string32, .string64: + return try string(tag) + default: + throw invalid(tag) + } + } + + // Every string igbinary writes in full joins the table its string IDs index: keys, values, class names. + private mutating func string(_ tag: Tag) throws -> (value: PhpValue, bytes: Int) { + switch tag { + case .stringEmpty: + return (.string(""), 0) + case .stringID8, .stringID16, .stringID32, .objectID8, .objectID16, .objectID32: + let start = reader.offset + let id = try index(width: tag) + guard id < strings.count else { + throw BinaryValueDecodeError(kind: .unknownReference(id), offset: start) + } + let value = strings[id] + let bytes = Self.byteCount(value) + try expand(slots: 0, stringBytes: bytes) + return (value, bytes) + default: + let data = try reader.readBytes(try length(tag)) + let value = Self.phpString(data) + strings.append(value) + try expand(slots: 0, stringBytes: data.count) + return (value, data.count) + } + } + + private mutating func expand(slots: Int, stringBytes added: Int) throws { + slot += slots + stringBytes += added + guard slot <= IgbinaryDecoder.maxExpandedValues, stringBytes <= IgbinaryDecoder.maxExpandedStringBytes else { + throw reader.error(.tooLarge) + } + } + + private static func phpString(_ bytes: Data) -> PhpValue { + String(data: bytes, encoding: .utf8).map(PhpValue.string) ?? .binaryString(bytes) + } + + private static func byteCount(_ value: PhpValue) -> Int { + switch value { + case .string(let text): return text.utf8.count + case .binaryString(let bytes): return bytes.count + default: return 0 + } + } + + // Names and keys PHP keeps as text; bytes that are not UTF-8 show one character each. + private static func text(_ value: PhpValue) -> String { + switch value { + case .int(let number): return String(number) + case .string(let text): return text + case .binaryString(let bytes): return String(data: bytes, encoding: .isoLatin1) ?? "" + default: return "" + } + } + + private mutating func integer(_ tag: Tag) throws -> Int { + let start = reader.offset + let magnitude: UInt64 + let isNegative: Bool + switch tag { + case .long8p, .long8n: + magnitude = UInt64(try reader.readByte()) + isNegative = tag == .long8n + case .long16p, .long16n: + magnitude = UInt64(try reader.readUInt16()) + isNegative = tag == .long16n + case .long32p, .long32n: + magnitude = UInt64(try reader.readUInt32()) + isNegative = tag == .long32n + default: + magnitude = try reader.readUInt64() + isNegative = tag == .long64n + } + if isNegative { + guard magnitude <= UInt64(Int64.max) + 1 else { + throw BinaryValueDecodeError(kind: .integerOutOfRange, offset: start) + } + return Int(Int64(bitPattern: 0 &- magnitude)) + } + guard magnitude <= UInt64(Int64.max) else { + throw BinaryValueDecodeError(kind: .integerOutOfRange, offset: start) + } + return Int(magnitude) + } + + private mutating func length(_ tag: Tag) throws -> Int { + switch tag { + case .string8, .array8, .object8, .objectSer8: + return Int(try reader.readByte()) + case .string16, .array16, .object16, .objectSer16: + return Int(try reader.readUInt16()) + case .string64: + let start = reader.offset + let count = try reader.readUInt64() + guard count <= UInt64(reader.remaining) else { + throw BinaryValueDecodeError(kind: .unexpectedEnd, offset: start) + } + return Int(count) + default: + return Int(try reader.readUInt32()) + } + } + + private mutating func index(width tag: Tag) throws -> Int { + switch tag { + case .ref8, .objref8, .stringID8, .objectID8: + return Int(try reader.readByte()) + case .ref16, .objref16, .stringID16, .objectID16: + return Int(try reader.readUInt16()) + default: + return Int(try reader.readUInt32()) + } + } + + private mutating func reference(width tag: Tag) throws -> (index: Int, slot: Int, copy: Copy?) { + let start = reader.offset + let index = try index(width: tag) + guard index < references.count else { + throw BinaryValueDecodeError(kind: .unknownReference(index), offset: start) + } + return (index, references[index].slot, references[index].copy) + } + + private mutating func readTag() throws -> Tag { + let start = reader.offset + let byte = try reader.readByte() + guard let tag = Tag(rawValue: byte) else { + throw BinaryValueDecodeError(kind: .invalidByte(byte), offset: start) + } + return tag + } + + private func invalid(_ tag: Tag) -> BinaryValueDecodeError { + BinaryValueDecodeError(kind: .invalidByte(tag.rawValue), offset: reader.offset - 1) + } + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift new file mode 100644 index 0000000000..4d0159e00a --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/PhpSerializeWriter.swift @@ -0,0 +1,134 @@ +// +// PhpSerializeWriter.swift +// TablePro +// + +import Foundation + +// Writes what PHP's serialize() writes, so the decoded text reads back in the PHP Serialized viewer. +internal enum PhpSerializeWriter { + static func text(_ value: PhpValue, limit: Int = .max) -> String { + var output = "" + write(value, into: &output, limit: limit) + return output + } + + // Text past `limit` is cut before it is copied, so a large string costs what the preview shows of it. The lengths + // stay the full ones, and the caller marks the cut. + private static func write(_ value: PhpValue, into output: inout String, limit: Int) { + let room = limit - output.utf8.count + guard room > 0 else { return } + switch value { + case .null: + output += "N;" + case .bool(let flag): + output += flag ? "b:1;" : "b:0;" + case .int(let number): + output += "i:\(number);" + case .float(let number): + output += "d:\(floatText(number));" + case .string(let text): + output += quoted("s", text, room: room) + ";" + case .binaryString(let bytes): + // The length is the byte count serialize() writes; the bytes show one character each. + output += "s:\(bytes.count):\"\(latin1(bytes.prefix(room)))\";" + case .array(let entries): + output += "a:\(entries.count):{" + for entry in entries { + guard output.utf8.count < limit else { return } + write(entry.key, into: &output, limit: limit) + write(entry.value, into: &output, limit: limit) + } + output += "}" + case .object(let className, let properties): + output += quoted("O", className, room: room) + ":\(properties.count):{" + for property in properties { + guard output.utf8.count < limit else { return } + if let index = property.index { + output += "i:\(index);" + } else if let binaryKey = property.binaryKey { + output += "s:\(binaryKey.count):\"\(latin1(binaryKey.prefix(limit - output.utf8.count)))\";" + } else { + output += quoted("s", mangled(property), room: limit - output.utf8.count) + ";" + } + write(property.value, into: &output, limit: limit) + } + output += "}" + case .serializable(let className, let rawPayload): + output += quoted("C", className, room: room) + ":\(rawPayload.utf8.count):{\(rawPayload.scalarPrefix(room))}" + case .binarySerializable(let className, let payload): + output += quoted("C", className, room: room) + ":\(payload.count):{\(latin1(payload.prefix(room)))}" + case .enumCase(let className, let caseName): + output += quoted("E", "\(className):\(caseName)", room: room) + ";" + case .reference(let id): + output += "R:\(id);" + case .objectReference(let id): + output += "r:\(id);" + case .unsupported(let token): + output += token + case .depthExceeded, .tooLarge: + output += "…" + } + } + + private static func quoted(_ token: String, _ text: String, room: Int) -> String { + "\(token):\(text.utf8.count):\"\(text.scalarPrefix(room))\"" + } + + private static func latin1(_ bytes: Data) -> String { + String(data: bytes, encoding: .isoLatin1) ?? "" + } + + private static func mangled(_ property: PhpProperty) -> String { + switch property.visibility { + case .publicVisibility: return property.name + case .protectedVisibility: return "\u{0}*\u{0}\(property.name)" + case .privateVisibility(let className): return "\u{0}\(className)\u{0}\(property.name)" + } + } + + // php_gcvt with serialize_precision -1: the shortest digits that read back, in E notation only when the + // decimal point falls more than 17 digits right or 4 digits left of the first digit. + static func floatText(_ value: Double) -> String { + if value.isNaN { return "NAN" } + if value.isInfinite { return value < 0 ? "-INF" : "INF" } + let sign = value.sign == .minus ? "-" : "" + if value == 0 { return sign + "0" } + + let (digits, point) = shortestDigits(abs(value)) + if point < -3 || point > 17 { + let mantissa = digits.count == 1 ? digits + ".0" : "\(digits.prefix(1)).\(digits.dropFirst())" + let exponent = point - 1 + return sign + mantissa + "E" + (exponent < 0 ? "-" : "+") + String(abs(exponent)) + } + if point <= 0 { + return sign + "0." + String(repeating: "0", count: -point) + digits + } + if digits.count <= point { + return sign + digits + String(repeating: "0", count: point - digits.count) + } + return sign + digits.prefix(point) + "." + digits.dropFirst(point) + } + + // Swift's description is the shortest round-trip text too; this reads the digits and the decimal point's + // position off it, as dtoa mode 0 returns them. + private static func shortestDigits(_ value: Double) -> (digits: String, point: Int) { + var text = "\(value)" + var exponent = 0 + if let marker = text.firstIndex(of: "e") { + exponent = Int(text[text.index(after: marker)...]) ?? 0 + text = String(text[.. 1 ? parts[1] : "")) + var point = parts[0].count + exponent + while digits.first == "0" { + digits = digits.dropFirst() + point -= 1 + } + while digits.last == "0" { + digits = digits.dropLast() + } + return (String(digits), point) + } +} diff --git a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift index f9c9990b61..dd19369787 100644 --- a/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift +++ b/apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift @@ -11,11 +11,18 @@ internal indirect enum PhpValue: Equatable { case int(Int) case float(Double) case string(String) + // A PHP string is bytes; igbinary hands over ones that are not UTF-8 as they are. + case binaryString(Data) case array([PhpKeyValue]) case object(className: String, properties: [PhpProperty]) case serializable(className: String, rawPayload: String) + // A Serializable payload that is not UTF-8, which igbinary hands over as it is. + case binarySerializable(className: String, payload: Data) case enumCase(className: String, caseName: String) + // `R:`, a PHP reference to the value numbered `id`. case reference(id: Int) + // `r:`, the object numbered `id` again. + case objectReference(id: Int) case unsupported(token: String) case depthExceeded case tooLarge @@ -29,14 +36,18 @@ internal indirect enum PhpValue: Equatable { if a.isNaN && b.isNaN { return true } return a == b case (.string(let a), .string(let b)): return a == b + case (.binaryString(let a), .binaryString(let b)): return a == b case (.array(let a), .array(let b)): return a == b case (.object(let aName, let aProps), .object(let bName, let bProps)): return aName == bName && aProps == bProps case (.serializable(let aName, let aPayload), .serializable(let bName, let bPayload)): return aName == bName && aPayload == bPayload + case (.binarySerializable(let aName, let aPayload), .binarySerializable(let bName, let bPayload)): + return aName == bName && aPayload == bPayload case (.enumCase(let aName, let aCase), .enumCase(let bName, let bCase)): return aName == bName && aCase == bCase case (.reference(let a), .reference(let b)): return a == b + case (.objectReference(let a), .objectReference(let b)): return a == b case (.unsupported(let a), .unsupported(let b)): return a == b case (.depthExceeded, .depthExceeded): return true case (.tooLarge, .tooLarge): return true @@ -54,6 +65,10 @@ internal struct PhpProperty: Equatable { let name: String let visibility: PhpVisibility let value: PhpValue + // An `i:` member, which __serialize data such as ArrayObject's holds; `name` spells the same number. + var index: Int? + // The key as igbinary stored it when it is not UTF-8, visibility prefix included; `name` shows it a byte a character. + var binaryKey: Data? } internal enum PhpVisibility: Equatable { @@ -111,7 +126,7 @@ private struct Cursor { case UInt8(ascii: "E"): return parseEnumCase() case UInt8(ascii: "o"): return .unsupported(token: "o") - case UInt8(ascii: "r"), UInt8(ascii: "R"): return parseReference() + case UInt8(ascii: "r"), UInt8(ascii: "R"): return parseReference(token: token) default: return nil } } @@ -213,10 +228,17 @@ private struct Cursor { var properties: [PhpProperty] = [] properties.reserveCapacity(count) for _ in 0.. PhpValue? { + private mutating func parseReference(token: UInt8) -> PhpValue? { index += 1 guard expect(UInt8(ascii: ":")) else { return nil } guard let raw = readUntil(UInt8(ascii: ";")) else { return nil } guard let value = Int(raw) else { return nil } guard expect(UInt8(ascii: ";")) else { return nil } - return .reference(id: value) + return token == UInt8(ascii: "R") ? .reference(id: value) : .objectReference(id: value) } private mutating func readUntil(_ terminator: UInt8) -> String? { @@ -276,7 +298,7 @@ private struct Cursor { } } -private func decodePropertyKey(_ raw: String) -> (name: String, visibility: PhpVisibility) { +internal func decodePhpPropertyKey(_ raw: String) -> (name: String, visibility: PhpVisibility) { let scalars = Array(raw.unicodeScalars) guard scalars.first?.value == 0 else { return (raw, .publicVisibility) diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift index 6676c6f15d..9ded656ca1 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift @@ -2,14 +2,10 @@ // ValueDisplayFormat.swift // TablePro // -// Semantic display formats for raw database values. -// Enables auto-detection and per-column overrides for values like -// UUIDs stored in BINARY(16) or Unix timestamps in INT columns. -// import Foundation -enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { +enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case raw case text case uuid @@ -20,6 +16,69 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { case phpSerialized case messagePack case protobuf + case igbinary + // The schema is kept by AvroSchemaStore under this ID, so every per-column path can decode with it. + case avro(schemaID: String) + + // Every format but Avro, which needs a schema chosen first. + static let presetFormats: [ValueDisplayFormat] = [ + .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized, + .messagePack, .protobuf, .igbinary, + ] + + private static let avroPrefix = "avro:" + + init?(rawValue: String) { + switch rawValue { + case "raw": self = .raw + case "text": self = .text + case "uuid": self = .uuid + case "uuidSwappedTime": self = .uuidSwappedTime + case "unixTimestamp": self = .unixTimestamp + case "unixTimestampMillis": self = .unixTimestampMillis + case "json": self = .json + case "phpSerialized": self = .phpSerialized + case "messagePack": self = .messagePack + case "protobuf": self = .protobuf + case "igbinary": self = .igbinary + default: + guard rawValue.hasPrefix(Self.avroPrefix) else { return nil } + let schemaID = String(rawValue.dropFirst(Self.avroPrefix.count)) + guard AvroSchemaStore.isSchemaID(schemaID) else { return nil } + self = .avro(schemaID: schemaID) + } + } + + var rawValue: String { + switch self { + case .raw: return "raw" + case .text: return "text" + case .uuid: return "uuid" + case .uuidSwappedTime: return "uuidSwappedTime" + case .unixTimestamp: return "unixTimestamp" + case .unixTimestampMillis: return "unixTimestampMillis" + case .json: return "json" + case .phpSerialized: return "phpSerialized" + case .messagePack: return "messagePack" + case .protobuf: return "protobuf" + case .igbinary: return "igbinary" + case .avro(let schemaID): return Self.avroPrefix + schemaID + } + } + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let rawValue = try container.decode(String.self) + guard let format = Self(rawValue: rawValue) else { + throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unknown display format \(rawValue)") + } + self = format + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(rawValue) + } var id: String { rawValue } @@ -33,7 +92,10 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { case .unixTimestampMillis: return String(localized: "Unix Timestamp (milliseconds)") case .json: return String(localized: "JSON") case .phpSerialized: return String(localized: "PHP Serialized") - case .messagePack, .protobuf: return binaryValueFormat?.displayName ?? rawValue + case .messagePack, .protobuf, .igbinary: return binaryValueFormat?.displayName ?? rawValue + case .avro(let schemaID): + guard let name = AvroSchemaStore.shared.schema(id: schemaID)?.displayName else { return "Avro" } + return String(format: String(localized: "Avro (%@)"), name) } } @@ -41,17 +103,18 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { switch self { case .messagePack: return .messagePack case .protobuf: return .protobuf + case .igbinary: return .igbinary + case .avro(let schemaID): return .avro(schemaID: schemaID) case .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: return nil } } - /// Column types this format can apply to. var applicableColumnTypes: Set { switch self { case .raw: return [] - case .text, .uuidSwappedTime, .messagePack, .protobuf: + case .text, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro: return ["blob"] case .uuid: return ["blob", "text"] @@ -80,24 +143,25 @@ enum ValueDisplayFormat: String, Codable, CaseIterable, Identifiable { return applicableColumnTypes.contains(typeKey) } - /// Whether this format turns stored bytes into readable characters. A binary column is worth - /// searching exactly when one of these is on it; hex is not what anyone types into Find. + // A binary column is worth searching exactly when one of these is on it; hex is not what anyone types into Find. var rendersBinaryAsText: Bool { switch self { - case .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf: + case .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro: return true case .raw, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: return false } } - /// Returns applicable formats for a given column type. - /// Always includes `.raw` as the first option. static func applicableFormats( for columnType: ColumnType?, databaseType: DatabaseType? = nil ) -> [ValueDisplayFormat] { - allCases.filter { $0.isApplicable(to: columnType, databaseType: databaseType) } + presetFormats.filter { $0.isApplicable(to: columnType, databaseType: databaseType) } + } + + static func offersAvro(for columnType: ColumnType?, databaseType: DatabaseType? = nil) -> Bool { + avro(schemaID: "").isApplicable(to: columnType, databaseType: databaseType) } } diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatService.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatService.swift index 6417114bf4..4248baf06d 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatService.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatService.swift @@ -8,11 +8,11 @@ import Foundation import os -/// Owns which format a column is shown in, and where a user's choice is persisted. -/// -/// Rendering a value under a format is a separate job that must not reach the preference layer; -/// it lives in `ValueDisplayFormatter`. Keeping the two apart is what lets the data grid's per -/// cell path stay free of storage, and it is why this type no longer exposes `applyFormat`. +// Owns which format a column is shown in, and where a user's choice is persisted. +// +// Rendering a value under a format is a separate job that must not reach the preference layer; +// it lives in `ValueDisplayFormatter`. Keeping the two apart is what lets the data grid's per +// cell path stay free of storage, and it is why this type no longer exposes `applyFormat`. @MainActor final class ValueDisplayFormatService { static let shared = ValueDisplayFormatService() @@ -83,11 +83,7 @@ final class ValueDisplayFormatService { overrides.removeValue(forKey: columnKey) } - if overrides.isEmpty { - storage.clear(for: scope) - } else { - storage.save(overrides, for: scope) - } + storage.save(overrides, for: scope) overridesVersion &+= 1 } diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift index 8e5cd8ed53..83e8941087 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift @@ -6,16 +6,16 @@ import Foundation import TableProUtilities -/// Renders one raw cell value under one display format. -/// -/// Nothing here reads a setting or a store, which is the point: this is what the data grid calls -/// per cell, and it has to stay reachable without dragging the preference layer behind it. -/// Resolving *which* format a column carries is a separate job with separate dependencies, and it -/// lives in `ValueDisplayFormatService`. +// Renders one raw cell value under one display format. +// +// Nothing here reads a setting or a store, which is the point: this is what the data grid calls +// per cell, and it has to stay reachable without dragging the preference layer behind it. +// Resolving *which* format a column carries is a separate job with separate dependencies, and it +// lives in `ValueDisplayFormatService`. @MainActor enum ValueDisplayFormatter { - /// nil where the format cannot render this value, so the caller falls back to how the column - /// would have read without it rather than showing a value the format did not actually produce. + // nil where the format cannot render this value, so the caller falls back to how the column + // would have read without it rather than showing a value the format did not actually produce. static func apply( _ rawValue: String, format: ValueDisplayFormat, @@ -36,7 +36,7 @@ enum ValueDisplayFormatter { return formatAsTimestamp(rawValue, divideBy: 1_000) case .json, .phpSerialized: return rawValue - case .messagePack, .protobuf: + case .messagePack, .protobuf, .igbinary, .avro: guard let data = rawValue.data(using: .isoLatin1), let decoder = format.binaryValueFormat else { return nil } return BinaryValueDecoding.text(data, format: decoder) } @@ -54,7 +54,7 @@ enum ValueDisplayFormatter { guard rawValue.count == 16 else { return nil } let order: UUIDByteOrder = format == .uuidSwappedTime ? .swappedTime : .stored return insertUuidHyphens(order.uuidBytes(fromStored: [UInt8](rawValue)).hexEncoded) - case .messagePack, .protobuf: + case .messagePack, .protobuf, .igbinary, .avro: guard let decoder = format.binaryValueFormat else { return nil } return BinaryValueDecoding.text(rawValue, format: decoder) case .raw, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: diff --git a/apple/TablePro/Core/Services/Infrastructure/AvroSchemaFilePanel.swift b/apple/TablePro/Core/Services/Infrastructure/AvroSchemaFilePanel.swift new file mode 100644 index 0000000000..5ef39b8a7a --- /dev/null +++ b/apple/TablePro/Core/Services/Infrastructure/AvroSchemaFilePanel.swift @@ -0,0 +1,71 @@ +// +// AvroSchemaFilePanel.swift +// TablePro +// + +import AppKit + +// Every file is enabled, since schemas come as .avsc, .json or no extension at all. The panel's validate hook +// is the gate: a file that is not a schema shows why and leaves the panel open on it. +@MainActor +internal enum AvroSchemaFilePanel { + static func chooseSchemaText(in window: NSWindow) async -> String? { + let panel = NSOpenPanel() + panel.allowedContentTypes = [] + panel.canChooseFiles = true + panel.canChooseDirectories = false + panel.allowsMultipleSelection = false + panel.prompt = String(localized: "Choose Schema") + panel.message = String(localized: "Choose the Avro schema the values in this column were written with.") + let gate = AvroSchemaFileGate() + panel.delegate = gate + let response = await panel.presentAsSheet(for: window) + return withExtendedLifetime(gate) { response == .OK ? gate.text : nil } + } +} + +@MainActor +private final class AvroSchemaFileGate: NSObject, NSOpenSavePanelDelegate { + private(set) var text: String? + + func panel(_ sender: Any, validate url: URL) throws { + let name = url.lastPathComponent + let size = (try? url.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0 + guard size <= AvroSchemaStore.maxFileBytes else { throw AvroSchemaFileError(fileName: name, reason: .tooLarge) } + guard let text = try? String(contentsOf: url, encoding: .utf8) else { + throw AvroSchemaFileError(fileName: name, reason: .notText) + } + do { + _ = try AvroSchema.parse(text) + } catch let error as AvroSchemaError { + throw AvroSchemaFileError(fileName: name, reason: .invalid(error.reason)) + } + self.text = text + } +} + +private struct AvroSchemaFileError: LocalizedError { + enum Reason { + case tooLarge + case notText + case invalid(String) + } + + let fileName: String + let reason: Reason + + var errorDescription: String? { + String(format: String(localized: "“%@” is not a usable Avro schema."), fileName) + } + + var recoverySuggestion: String? { + switch reason { + case .tooLarge: + return String(localized: "The file is larger than any Avro schema.") + case .notText: + return String(localized: "The file is not UTF-8 text.") + case .invalid(let reason): + return reason + } + } +} diff --git a/apple/TablePro/Core/Storage/AvroSchemaStore.swift b/apple/TablePro/Core/Storage/AvroSchemaStore.swift new file mode 100644 index 0000000000..3b3bdb9a1a --- /dev/null +++ b/apple/TablePro/Core/Storage/AvroSchemaStore.swift @@ -0,0 +1,110 @@ +// +// AvroSchemaStore.swift +// TablePro +// + +import CryptoKit +import Foundation +import os + +// The schema files chosen for Avro columns, one file each, named by a hash of the text, so a column still +// decodes after the original file moves and the same file chosen twice is kept once. Read from the grid on +// the main actor and from the viewer off it. +internal final class AvroSchemaStore: Sendable { + static let shared = AvroSchemaStore( + directory: AppStorageEnvironment.shared.supportDirectory.appendingPathComponent("AvroSchemas", isDirectory: true) + ) + + // Far past any real schema; a file this large is the wrong file. + static let maxFileBytes = 4_000_000 + + // Posted when a schema this Mac did not have is kept, so a viewer that found it missing decodes again. + static let didRestoreSchema = Notification.Name("com.TablePro.avroSchemaStore.didRestoreSchema") + static let schemaIDKey = "schemaID" + + private static let fileExtension = "avsc" + + private struct State { + var parsed: [String: AvroSchema] = [:] + // Kept from a sweep: a schema can be added before the column's format is saved. + var addedThisLaunch: Set = [] + // Asked for and not found, so adding one later is a restore that grids and viewers have to hear about. + var missing: Set = [] + } + + private let directory: URL + private let state = OSAllocatedUnfairLock(initialState: State()) + + init(directory: URL) { + self.directory = directory + } + + static func isSchemaID(_ text: String) -> Bool { + text.utf8.count == 32 && text.utf8.allSatisfy { (0x30...0x39).contains($0) || (0x61...0x66).contains($0) } + } + + static func schemaID(of text: String) -> String { + SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() + } + + // Parses first, so nothing is kept for a file that is not a schema. + func add(_ text: String) throws -> String { + let schema = try AvroSchema.parse(text) + let id = Self.schemaID(of: text) + let url = fileURL(id) + // A copy that went bad on disk is written over, or the next launch would find it missing again. + let contents = Data(text.utf8) + if (try? Data(contentsOf: url)) != contents { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try contents.write(to: url, options: .atomic) + } + let wasMissing = state.withLock { state -> Bool in + state.parsed[id] = schema + state.addedThisLaunch.insert(id) + return state.missing.remove(id) != nil + } + if wasMissing { + NotificationCenter.default.post(name: Self.didRestoreSchema, object: self, userInfo: [Self.schemaIDKey: id]) + } + return id + } + + func schema(id: String) -> AvroSchema? { + if let cached = state.withLock({ $0.parsed[id] }) { return cached } + guard Self.isSchemaID(id), + let data = try? Data(contentsOf: fileURL(id)), + let text = String(data: data, encoding: .utf8), + let schema = try? AvroSchema.parse(text) else { + // An add() may have landed since the read failed; recorded as missing only if it did not. + return state.withLock { state in + if let added = state.parsed[id] { return added } + state.missing.insert(id) + return nil + } + } + state.withLock { $0.parsed[id] = schema } + return schema + } + + var storedIDs: Set { + let names = (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] + return Set(names.compactMap { name in + let url = URL(fileURLWithPath: name) + guard url.pathExtension == Self.fileExtension else { return nil } + let id = url.deletingPathExtension().lastPathComponent + return Self.isSchemaID(id) ? id : nil + }) + } + + func removeAll(except kept: Set) { + let addedThisLaunch = state.withLock { $0.addedThisLaunch } + for id in storedIDs where !kept.contains(id) && !addedThisLaunch.contains(id) { + try? FileManager.default.removeItem(at: fileURL(id)) + state.withLock { _ = $0.parsed.removeValue(forKey: id) } + } + } + + private func fileURL(_ id: String) -> URL { + directory.appendingPathComponent(id, isDirectory: false).appendingPathExtension(Self.fileExtension) + } +} diff --git a/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift b/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift index 287d1b59b2..fbc2b90b7b 100644 --- a/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift +++ b/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift @@ -5,45 +5,67 @@ import Foundation +// 0.83 decodes a table's formats as one dictionary and drops all of them on a name it does not know, so a +// format added since lives in a second key under the same prefix. Each key is read entry by entry, the first +// wins for a column in both (0.83 writes only the first), and a name this build does not know is kept. @MainActor internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { static let shared = ValueDisplayFormatStorage() + private static let namesReadBy083: Set = [ + "raw", "text", "uuid", "uuidSwappedTime", "unixTimestamp", "unixTimestampMillis", "json", "phpSerialized", + "messagePack", "protobuf", + ] + private let store: KeyValueStore + private let schemas: AvroSchemaStore - init(defaults: KeyValueStore = AppStorageEnvironment.shared.defaults) { + init(defaults: KeyValueStore = AppStorageEnvironment.shared.defaults, schemas: AvroSchemaStore = .shared) { store = defaults + self.schemas = schemas } func save(_ formats: [String: ValueDisplayFormat], for scope: TableScope) { - guard !formats.isEmpty else { - clear(for: scope) - return + let key = Self.key(for: scope) + let addedKey = Self.addedKey(for: scope) + var first: [String: String] = [:] + var added = (Self.names(store.dataValue(forKey: addedKey)) ?? [:]) + .filter { ValueDisplayFormat(rawValue: $0.value) == nil && formats[$0.key] == nil } + for (column, format) in formats { + if Self.namesReadBy083.contains(format.rawValue) { + first[column] = format.rawValue + } else { + added[column] = format.rawValue + } } - guard let data = try? JSONEncoder().encode(formats) else { return } - store.setDataValue(data, forKey: PreferenceKeys.columnDisplayFormats(scope).name) + write(first, forKey: key) + write(added, forKey: addedKey) removeLegacy(for: scope) + sweepSchemas() } func load(for scope: TableScope) -> [String: ValueDisplayFormat]? { - if let data = store.dataValue(forKey: PreferenceKeys.columnDisplayFormats(scope).name), - let formats = try? JSONDecoder().decode([String: ValueDisplayFormat].self, from: data) { - return formats - } - return migrateLegacy(for: scope) + let first = Self.names(store.dataValue(forKey: Self.key(for: scope))) + let added = Self.names(store.dataValue(forKey: Self.addedKey(for: scope))) + guard first != nil || added != nil else { return migrateLegacy(for: scope) } + let names = (added ?? [:]).merging(first ?? [:]) { _, firstName in firstName } + return names.compactMapValues(ValueDisplayFormat.init(rawValue:)) } func clear(for scope: TableScope) { - store.setDataValue(nil, forKey: PreferenceKeys.columnDisplayFormats(scope).name) + store.setDataValue(nil, forKey: Self.key(for: scope)) + store.setDataValue(nil, forKey: Self.addedKey(for: scope)) removeLegacy(for: scope) + sweepSchemas() } func renameTable(from oldScope: TableScope, to newScope: TableScope) { - let oldKey = PreferenceKeys.columnDisplayFormats(oldScope).name + let oldKey = Self.key(for: oldScope) if store.dataValue(forKey: oldKey) == nil { migrateLegacy(for: oldScope) } - store.moveValue(fromKey: oldKey, toKey: PreferenceKeys.columnDisplayFormats(newScope).name) + store.moveValue(fromKey: oldKey, toKey: Self.key(for: newScope)) + store.moveValue(fromKey: Self.addedKey(for: oldScope), toKey: Self.addedKey(for: newScope)) } func renameContainer( @@ -65,15 +87,15 @@ internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { clear(for: scope) } - /// The legacy key is deliberately left alone. It is `prefix + connectionId + table` and names no - /// database or schema, so nothing here can tell which container it belonged to, and sweeping the - /// connection's would take the formats of tables in databases that still exist. `dropTable` - /// clears it, because there the table is named. + // The legacy key names no database or schema, so nothing here can tell which container it belonged to, + // and sweeping the connection's would take the formats of tables in databases that still exist. + // `dropTable` clears it, because there the table is named. func dropContainer(connectionId: UUID, database: String, schema: String?) { store.removeValues( withPrefix: Self.keyPrefix + TableScope.storagePrefix(connectionId: connectionId, database: database, schema: schema) ) + sweepSchemas() } func purgeConnections(_ connectionIds: Set, leavesTombstones: Bool) { @@ -81,29 +103,59 @@ internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { store.removeValues(withPrefix: Self.keyPrefix + TableScope.storagePrefix(connectionId: connectionId)) store.removeValues(withPrefix: Self.legacyKeyPrefix(for: connectionId)) } + sweepSchemas() + } + + // A schema file lives as long as a saved format names it. + private func sweepSchemas() { + guard !schemas.storedIDs.isEmpty else { return } + var named: Set = [] + for key in store.keys(withPrefix: Self.keyPrefix) where key.hasSuffix(Self.addedSuffix) { + for name in (Self.names(store.dataValue(forKey: key)) ?? [:]).values { + if case .avro(let schemaID)? = ValueDisplayFormat(rawValue: name) { named.insert(schemaID) } + } + } + schemas.removeAll(except: named) } @discardableResult private func migrateLegacy(for scope: TableScope) -> [String: ValueDisplayFormat]? { let legacyKey = Self.legacyKey(for: scope) - guard let data = store.dataValue(forKey: legacyKey), - let formats = try? JSONDecoder().decode([String: ValueDisplayFormat].self, from: data), - !formats.isEmpty else { - return nil - } - if let encoded = try? JSONEncoder().encode(formats) { - store.setDataValue(encoded, forKey: PreferenceKeys.columnDisplayFormats(scope).name) - } + guard let names = Self.names(store.dataValue(forKey: legacyKey)), !names.isEmpty else { return nil } + write(names, forKey: Self.key(for: scope)) store.setDataValue(nil, forKey: legacyKey) - return formats + return names.compactMapValues(ValueDisplayFormat.init(rawValue:)) + } + + private func write(_ names: [String: String], forKey key: String) { + guard !names.isEmpty, let data = try? JSONEncoder().encode(names) else { + store.setDataValue(nil, forKey: key) + return + } + store.setDataValue(data, forKey: key) } private func removeLegacy(for scope: TableScope) { store.setDataValue(nil, forKey: Self.legacyKey(for: scope)) } + private static func names(_ data: Data?) -> [String: String]? { + data.flatMap { try? JSONDecoder().decode([String: String].self, from: $0) } + } + private static let keyPrefix = PreferenceKeys.columnDisplayFormatsPrefix + // A table key is four dot-joined, percent-encoded parts, so a fifth part never names another table. + private static let addedSuffix = ".added" + + private static func key(for scope: TableScope) -> String { + PreferenceKeys.columnDisplayFormats(scope).name + } + + private static func addedKey(for scope: TableScope) -> String { + key(for: scope) + addedSuffix + } + private static func legacyKey(for scope: TableScope) -> String { legacyKeyPrefix(for: scope.connectionId) + scope.table } diff --git a/apple/TablePro/Models/UI/PhpTreeNode.swift b/apple/TablePro/Models/UI/PhpTreeNode.swift index 87b0e71692..14e168c3c5 100644 --- a/apple/TablePro/Models/UI/PhpTreeNode.swift +++ b/apple/TablePro/Models/UI/PhpTreeNode.swift @@ -94,12 +94,12 @@ internal struct PhpTreeNode: Identifiable { } extension PhpTreeNode: FilterableTreeNode { - /// An integer key is stored in the value, unlike a JSON array index, so it stays searchable. + // An integer key is stored in the value, unlike a JSON array index, so it stays searchable. internal var searchableKey: String? { key } - /// A class name is data. A count and the cut marker are text the tree made up. + // A class name is data. A count and the cut marker are text the tree made up. internal var searchableText: String { switch nodeType { case .array, .truncated: return "" @@ -121,7 +121,7 @@ extension PhpTreeNode: FilterableTreeNode { rawValue ?? displayValue } - /// An array or an object copies its summary: there is no serializer to write its members back out. + // An array or an object copies its summary: there is no serializer to write its members back out. internal var copyableValueIncludesDescendants: Bool { false } @@ -201,6 +201,15 @@ internal enum PhpTreeBuilder { rawValue: stringValue, visibilityBadge: badge ) + case .binaryString(let bytes): + let hex = "0x" + bytes.uppercaseHex() + let display = TreeDisplayText.cut(hex, limit: maxDisplayLength) + return PhpTreeNode( + key: key, keyPath: keyPath, path: path, nodeType: .string, + displayValue: display.text, isDisplayCut: display.isCut, + rawValue: hex, visibilityBadge: badge + ) + case .array(let entries): var children: [PhpTreeNode] = [] children.reserveCapacity(entries.count) @@ -238,9 +247,10 @@ internal enum PhpTreeBuilder { } children.append( makeNode( - key: property.name, - keyPath: "\(keyPath).\(TreeKeyPathSyntax.member(property.name))", - path: path.appending(siblings.key(property.name)), + key: property.index.map { "[\($0)]" } ?? property.name, + keyPath: property.index.map { "\(keyPath)[\($0)]" } + ?? "\(keyPath).\(TreeKeyPathSyntax.member(property.name))", + path: path.appending(property.index.map { siblings.integerKey($0) } ?? siblings.key(property.name)), value: property.value, formats: formats, nodeCount: &nodeCount, @@ -254,6 +264,14 @@ internal enum PhpTreeBuilder { className: className, visibilityBadge: badge, children: children ) + case .binarySerializable(let className, let payload): + let hex = "0x" + payload.uppercaseHex() + return PhpTreeNode( + key: key, keyPath: keyPath, path: path, nodeType: .serializable, + displayValue: serializableDisplay(className: className, payload: hex), + rawValue: hex, className: className, visibilityBadge: badge + ) + case .serializable(let className, let rawPayload): return PhpTreeNode( key: key, keyPath: keyPath, path: path, nodeType: .serializable, @@ -267,7 +285,7 @@ internal enum PhpTreeBuilder { displayValue: "\(className)::\(caseName)", className: className, visibilityBadge: badge ) - case .reference(let identifier): + case .reference(let identifier), .objectReference(let identifier): return PhpTreeNode( key: key, keyPath: keyPath, path: path, nodeType: .reference, displayValue: "→ #\(identifier)", visibilityBadge: badge @@ -307,6 +325,7 @@ internal enum PhpTreeBuilder { switch key { case .int(let intValue): return "[\(intValue)]" case .string(let stringValue): return stringValue + case .binaryString(let bytes): return "0x" + bytes.uppercaseHex() default: return "?" } } @@ -315,6 +334,7 @@ internal enum PhpTreeBuilder { switch key { case .int(let intValue): return "\(parent)[\(intValue)]" case .string(let stringValue): return "\(parent).\(TreeKeyPathSyntax.member(stringValue))" + case .binaryString: return "\(parent)[\(arrayKeyDisplay(key))]" default: return parent } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 4b999ea5d9..117f59b770 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -939,6 +939,9 @@ } } } + }, + "“%@” is not a usable Avro schema." : { + }, "\"%@\" is not a valid %@. Example: %@" : { "extractionState" : "manual", @@ -15526,6 +15529,9 @@ }, "A field needs a type." : { + }, + "A field of record %@ has no name or type." : { + }, "A file" : { "localizations" : { @@ -15829,6 +15835,9 @@ } } } + }, + "A map has no \"values\"." : { + }, "A map needs a geometry or geography column. This result has none." : { "localizations" : { @@ -16286,6 +16295,9 @@ } } } + }, + "A record, enum or fixed type has no name." : { + }, "A record's id cannot be edited." : { "extractionState" : "manual", @@ -16478,6 +16490,9 @@ }, "A saved query with this name exists." : { + }, + "A schema object has no \"type\"." : { + }, "A schema with this name already exists." : { "localizations" : { @@ -24019,6 +24034,9 @@ } } } + }, + "An array has no \"items\"." : { + }, "An attribute value must name exactly one type" : { "extractionState" : "manual", @@ -29696,6 +29714,15 @@ } } } + }, + "Avro (%@)" : { + + }, + "Avro Schema…" : { + + }, + "Avro single-object encoding: schema fingerprint %@" : { + }, "AWS answered without the Redshift user and password." : { @@ -32409,6 +32436,26 @@ }, "Byte %lld does not start a valid field." : { + }, + "Byte %lld picks entry %lld, which the schema does not have." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Byte %1$lld picks entry %2$lld, which the schema does not have." + } + } + } + }, + "Byte %lld refers back to item %lld, which comes later or not at all." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Byte %1$lld refers back to item %2$lld, which comes later or not at all." + } + } + } }, "Byte 0x%@ at offset %lld is not valid here." : { "localizations" : { @@ -38359,6 +38406,12 @@ } } } + }, + "Choose Schema" : { + + }, + "Choose the Avro schema the values in this column were written with." : { + }, "Choose the base type." : { @@ -72526,6 +72579,9 @@ } } } + }, + "Enum %@ has no symbols." : { + }, "Enum Type" : { "localizations" : { @@ -83405,6 +83461,9 @@ } } } + }, + "Fixed type %@ has no size." : { + }, "Flag" : { "extractionState" : "manual", @@ -143422,6 +143481,9 @@ } } } + }, + "Record %@ has no fields list." : { + }, "Record Access" : { "extractionState" : "manual", @@ -157129,6 +157191,9 @@ } } } + }, + "Schema Registry framing: schema ID %lld" : { + }, "Schema Registry framing: schema ID %lld, message index %@" : { "localizations" : { @@ -186387,9 +186452,15 @@ }, "The file is larger than %@, too large for a rules file." : { + }, + "The file is larger than any Avro schema." : { + }, "The file is not a JSON object." : { + }, + "The file is not UTF-8 text." : { + }, "The file is not valid JSON." : { "extractionState" : "manual", @@ -186711,6 +186782,9 @@ }, "The group end at byte %lld matches no group start." : { + }, + "The header names version %lld, which this format does not have." : { + }, "The host cannot be passed to FreeTDS. Use a host name or an IP address, without spaces, brackets, “=”, “;” or “#”." : { "extractionState" : "manual", @@ -187008,6 +187082,9 @@ }, "The left result stopped at the row cap and is compared on the %@ rows it holds. A row past the cap can show as only on the right." : { + }, + "The length at byte %lld is negative." : { + }, "The license has been suspended." : { "localizations" : { @@ -189909,6 +189986,18 @@ }, "The schema changes staged in this diagram will be lost." : { + }, + "The schema defines type %@ twice." : { + + }, + "The schema has a type that is not a name, an object or a union." : { + + }, + "The schema is not valid JSON." : { + + }, + "The schema names type %@ before defining it." : { + }, "The script already ran. Compare again to see where the target stands." : { "localizations" : { @@ -193140,6 +193229,9 @@ } } } + }, + "The value was written with the schema whose fingerprint is %@, not this column's." : { + }, "The values this save replaced are no longer being kept." : { "localizations" : { @@ -197754,6 +197846,9 @@ } } } + }, + "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…" : { + }, "This Mac no longer holds a seat." : { "localizations" : { diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift index 1b2d3456ff..04815f481b 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift @@ -100,6 +100,18 @@ extension DataGridViewController { item.state = format == current ? .on : .off submenu.addItem(item) } + if ValueDisplayFormat.offersAvro(for: column.type, databaseType: model.configuration.traits.databaseType) { + if case .avro = current { + let item = menuItem( + current.displayName, + #selector(setDisplayFormatFromMenu(_:)), + DataGridDisplayFormatChoice(columnID: column.id, format: current) + ) + item.state = .on + submenu.addItem(item) + } + submenu.addItem(menuItem(String(localized: "Avro Schema…"), #selector(chooseAvroSchemaFromMenu(_:)), column.id)) + } let item = NSMenuItem(title: String(localized: "Display As"), action: nil, keyEquivalent: "") item.submenu = submenu return item @@ -142,6 +154,7 @@ extension DataGridViewController { #selector(DataGridViewController.clearValueFilterFromMenu(_:)), #selector(DataGridViewController.clearAllValueFiltersFromMenu(_:)), #selector(DataGridViewController.setDisplayFormatFromMenu(_:)), + #selector(DataGridViewController.chooseAvroSchemaFromMenu(_:)), #selector(DataGridViewController.sizeColumnToFitFromMenu(_:)), #selector(DataGridViewController.sizeAllColumnsToFitFromMenu(_:)), #selector(DataGridViewController.hideColumnFromMenu(_:)), @@ -224,11 +237,15 @@ extension DataGridViewController { // Only a value filter can renumber rows on a format change, so only then does the owner confirm it. @objc func setDisplayFormatFromMenu(_ sender: Any?) { - guard let choice = (sender as? NSMenuItem)?.representedObject as? DataGridDisplayFormatChoice, - model.columns.column(id: choice.columnID) != nil else { return } + guard let choice = (sender as? NSMenuItem)?.representedObject as? DataGridDisplayFormatChoice else { return } + setDisplayFormat(choice.format, columnID: choice.columnID) + } + + private func setDisplayFormat(_ format: ValueDisplayFormat, columnID: DataGridColumnID) { + guard model.columns.column(id: columnID) != nil else { return } let apply: @MainActor () -> Void = { [weak self] in - guard let self, let column = self.model.columns.column(id: choice.columnID) else { return } - self.model.setDisplayFormat(choice.format, for: column) + guard let self, let column = self.model.columns.column(id: columnID) else { return } + self.model.setDisplayFormat(format, for: column) } guard model.store.valueFilter.isActive, let actions = model.actions else { apply() @@ -237,6 +254,19 @@ extension DataGridViewController { actions.gridConfirmDisplayOrderChange(then: apply) } + // The schema is copied in, so the column still decodes after the file moves. + @objc func chooseAvroSchemaFromMenu(_ sender: Any?) { + guard let columnID = menuColumn(sender)?.id, let window = view.window else { return } + Task { @MainActor [weak self] in + guard let text = await AvroSchemaFilePanel.chooseSchemaText(in: window) else { return } + do { + self?.setDisplayFormat(.avro(schemaID: try AvroSchemaStore.shared.add(text)), columnID: columnID) + } catch { + NSAlert(error: error).beginSheetModal(for: window, completionHandler: nil) + } + } + } + @objc func sizeColumnToFitFromMenu(_ sender: Any?) { guard let column = menuColumn(sender) else { return } sizeColumnToFit(column) diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index 43d6f59ed2..e0e6df95ae 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -3,6 +3,7 @@ // TablePro // +import Combine import SwiftUI import TableProTheme @@ -23,6 +24,7 @@ internal struct BinaryValueViewerView: View { @State private var hexDump = "" @State private var decodedKey: TaskKey? @State private var searchText = "" + @State private var schemaRevision = 0 init(data: Data, format: BinaryValueFormat, onPopOut: (() -> Void)? = nil) { self.data = data @@ -37,14 +39,21 @@ internal struct BinaryValueViewerView: View { content .frame(maxWidth: .infinity, maxHeight: .infinity) } - .task(id: TaskKey(data: data, format: format)) { + .task(id: TaskKey(data: data, format: format, schemaRevision: schemaRevision)) { await decode() } + // Matched on the format, not the document: a decode still running may be about to report it missing. + .onReceive(NotificationCenter.default.publisher(for: AvroSchemaStore.didRestoreSchema).receive(on: RunLoop.main)) { notification in + guard case .avro(let schemaID) = format, + notification.userInfo?[AvroSchemaStore.schemaIDKey] as? String == schemaID else { return } + schemaRevision += 1 + } } private struct TaskKey: Equatable { let data: Data let format: BinaryValueFormat + let schemaRevision: Int } // MARK: - Toolbar @@ -100,6 +109,15 @@ internal struct BinaryValueViewerView: View { Divider() hexBody } + case .missingSchema?: + VStack(spacing: 0) { + banner( + String(localized: "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…"), + systemImage: "questionmark.folder" + ) + Divider() + hexBody + } case .decoded(let tree, let text, let note)?: VStack(spacing: 0) { if let note { @@ -129,6 +147,12 @@ internal struct BinaryValueViewerView: View { searchText: $searchText, fullValueModeName: String(localized: "Text") ) + case .php(let root): + FilterableTreeView( + rootNode: root, + searchText: $searchText, + fullValueModeName: String(localized: "Text") + ) } } @@ -174,7 +198,7 @@ internal struct BinaryValueViewerView: View { // A pane added back restarts the task with the same key. private func decode() async { - let key = TaskKey(data: data, format: format) + let key = TaskKey(data: data, format: format, schemaRevision: schemaRevision) guard decodedKey != key else { return } let data = data let format = format diff --git a/apple/TablePro/Views/Results/CellInteractionResolver.swift b/apple/TablePro/Views/Results/CellInteractionResolver.swift index aa64b5e819..5c9be4a713 100644 --- a/apple/TablePro/Views/Results/CellInteractionResolver.swift +++ b/apple/TablePro/Views/Results/CellInteractionResolver.swift @@ -99,7 +99,7 @@ internal struct CellInteractionResolver { case .phpSerialized: return .viewPhpSerialized case .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .messagePack, .protobuf, - .none: + .igbinary, .avro, .none: return plainText(for: context, isReadOnly: isReadOnly) } } diff --git a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift index 2976e16b9f..13bed1dadc 100644 --- a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift +++ b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift @@ -9,8 +9,8 @@ import TableProTextEngine @MainActor internal enum FieldEditorResolver { - /// Answers from the field's own resolution when it has one. Only a field built before - /// `MultiRowEditState` filled it in pays the detectors here. + // Answers from the field's own resolution when it has one. Only a field built before + // `MultiRowEditState` filled it in pays the detectors here. static func resolve(field: FieldEditState) -> FieldEditorKind { if let resolved = field.resolvedEditor { return resolved } return resolve( @@ -51,7 +51,7 @@ internal enum FieldEditorResolver { return .phpSerialized case .json: return .json - case .messagePack, .protobuf: + case .messagePack, .protobuf, .igbinary, .avro: // A NULL keeps the hex editor, the way bytes get into it. if type.isBlobType || isBinaryValue, originalValue != nil, let format = override.binaryValueFormat { return .binaryValue(format) @@ -105,19 +105,19 @@ internal enum FieldEditorResolver { return .singleLine } - /// The element editor an array column's value opens on, or nil where the list cannot represent - /// it and the plain text editor stays. - /// - /// The value is parsed, not the type alone: `jsonb[]` and `jsonb[][]` are one type in - /// PostgreSQL's catalog, and a dimension-prefixed literal such as `[0:2]={a,b,c}` is legal in - /// any array column, so the declared type cannot rule either out on a given row. This is the - /// same gate the grid applies before opening the popover. - /// - /// It is also what scopes the editor to the engines whose arrays are written this way. The - /// classifier's `[]` rule takes no engine, and a document store's `object[]` classifies the - /// same, so a field with no literal to read keeps the plain editor rather than being offered a - /// list that commits PostgreSQL `{…}` syntax. That leaves a stored NULL and a multi-row - /// selection on the plain editor, which is where they already were. + // The element editor an array column's value opens on, or nil where the list cannot represent + // it and the plain text editor stays. + // + // The value is parsed, not the type alone: `jsonb[]` and `jsonb[][]` are one type in + // PostgreSQL's catalog, and a dimension-prefixed literal such as `[0:2]={a,b,c}` is legal in + // any array column, so the declared type cannot rule either out on a given row. This is the + // same gate the grid applies before opening the popover. + // + // It is also what scopes the editor to the engines whose arrays are written this way. The + // classifier's `[]` rule takes no engine, and a document store's `object[]` classifies the + // same, so a field with no literal to read keeps the plain editor rather than being offered a + // list that commits PostgreSQL `{…}` syntax. That leaves a stored NULL and a multi-row + // selection on the plain editor, which is where they already were. private static func arrayElementEditor(for type: ColumnType, originalValue: String?) -> ArrayElementEditor? { guard let elementEditor = type.arrayElementEditor, let originalValue, @@ -129,11 +129,11 @@ internal enum FieldEditorResolver { return elementEditor } - /// Runs ahead of the JSON rule, which would claim GeoJSON in a spatial column. The readers are - /// the ones `GeometryFieldPreview` draws with, so the kind and the Map segment cannot disagree. - /// - /// This runs on every selection change, so a value too long for the preview to read on the main - /// actor is classified by how it opens instead. + // Runs ahead of the JSON rule, which would claim GeoJSON in a spatial column. The readers are + // the ones `GeometryFieldPreview` draws with, so the kind and the Map segment cannot disagree. + // + // This runs on every selection change, so a value too long for the preview to read on the main + // actor is classified by how it opens instead. private static func geometryEditor( for type: ColumnType, originalValue: String?, @@ -150,8 +150,8 @@ internal enum FieldEditorResolver { } } - /// GeoJSON only: the sniffing reader takes any pair of numbers, and any document with `lat` and - /// `lon`, for a point. The parse goes first because on a large document it is cheaper. + // GeoJSON only: the sniffing reader takes any pair of numbers, and any document with `lat` and + // `lon`, for a point. The parse goes first because on a large document it is cheaper. private static func isGeoJSON(_ value: String) -> Bool { guard GeometryFieldPreview.readsSynchronously(value) else { return GeometryValueSniffer.opensGeoJSON(value) @@ -160,9 +160,9 @@ internal enum FieldEditorResolver { return GeoJSONGeometryReader.looksLikeGeoJSON(value) } - /// Bytes never fall through to a text editor: they arrive as one character per byte, and one - /// keystroke in a text field would stage that string over the blob. A type the map names but - /// cannot draw is still a geometry field, as it is for text. + // Bytes never fall through to a text editor: they arrive as one character per byte, and one + // keystroke in a text field would stage that string over the blob. A type the map names but + // cannot draw is still a geometry field, as it is for text. private static func binaryGeometryEditor(_ originalValue: String?) -> FieldEditorKind { guard let originalValue else { return .blobHex } let geometry = FieldEditorKind.geometry(GeometryFieldDescriptor(textEditor: .hex, source: .binary)) @@ -177,8 +177,8 @@ internal enum FieldEditorResolver { } } - /// A value with nothing to read yet is still a geometry field, and so is a type the map names - /// but cannot draw. Text no reader takes, as SQL Server, Oracle and Teradata send, is not. + // A value with nothing to read yet is still a geometry field, and so is a type the map names + // but cannot draw. Text no reader takes, as SQL Server, Oracle and Teradata send, is not. private static func spatialTextEditor(_ originalValue: String?) -> FieldEditorKind? { guard let originalValue, !originalValue.isEmpty else { return .geometry(GeometryFieldDescriptor(textEditor: .multiLine, source: .spatialColumn)) @@ -199,11 +199,11 @@ internal enum FieldEditorResolver { } } - /// `isLongText` only matches six exact type names, so a large value in `VARCHAR(MAX)`, - /// `NCLOB` or ClickHouse's `Nullable(String)` never reached the multi-line editor. Whether a - /// value belongs on one line is a property of the value, so ask the value as well. - /// A one-line field cannot mark an invisible character, so a value holding one takes the text - /// view that can, while the marks are on. + // `isLongText` only matches six exact type names, so a large value in `VARCHAR(MAX)`, + // `NCLOB` or ClickHouse's `Nullable(String)` never reached the multi-line editor. Whether a + // value belongs on one line is a property of the value, so ask the value as well. + // A one-line field cannot mark an invisible character, so a value holding one takes the text + // view that can, while the marks are on. static func needsMultiLineEditor(_ value: String?, revealsInvisibleCharacters: Bool = false) -> Bool { guard let value, !value.isEmpty else { return false } let text = value as NSString @@ -216,8 +216,8 @@ internal enum FieldEditorResolver { (0 ..< text.length).contains { SpecialCharacter.classify(in: text, at: $0) != nil } } - /// Two lines' worth at the inspector's minimum width. The value font is user-configurable, so how - /// many characters actually fit a line moves with it; the threshold stays a fixed count on purpose, - /// because which editor a value gets must not change under the user when they resize the font. + // Two lines' worth at the inspector's minimum width. The value font is user-configurable, so how + // many characters actually fit a line moves with it; the threshold stays a fixed count on purpose, + // because which editor a value gets must not change under the user when they resize the font. static let multiLineValueThreshold = 80 } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift new file mode 100644 index 0000000000..e84d7d6e3d --- /dev/null +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/AvroDecoderTests.swift @@ -0,0 +1,352 @@ +// +// AvroDecoderTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +// Fixtures written by fastavro, `schemaless_writer(buffer, parse_schema(schema), record)`: 1.13.1, and 1.12.2 for +// the named decimal, the padded decimal and the nanosecond timestamps. +struct AvroDecoderTests { + private static let userSchema = #"{"type":"record","name":"User","namespace":"com.example","fields":["# + + #"{"name":"id","type":"long"},{"name":"name","type":"string"},{"name":"email","type":["null","string"]},"# + + #"{"name":"active","type":"boolean"},{"name":"score","type":"double"},{"name":"ratio","type":"float"},"# + + #"{"name":"tags","type":{"type":"array","items":"string"}},{"name":"attrs","type":{"type":"map","values":"int"}},"# + + #"{"name":"kind","type":{"type":"enum","name":"Kind","symbols":["A","B","C"]}},"# + + #"{"name":"hash","type":{"type":"fixed","name":"Hash","size":4}},{"name":"raw","type":"bytes"},"# + + #"{"name":"nothing","type":"null"}]}"# + + private static let logicalSchema = #"{"type":"record","name":"Logical","fields":["# + + #"{"name":"id","type":{"type":"string","logicalType":"uuid"}},"# + + #"{"name":"day","type":{"type":"int","logicalType":"date"}},"# + + #"{"name":"tms","type":{"type":"int","logicalType":"time-millis"}},"# + + #"{"name":"tus","type":{"type":"long","logicalType":"time-micros"}},"# + + #"{"name":"tsms","type":{"type":"long","logicalType":"timestamp-millis"}},"# + + #"{"name":"tsus","type":{"type":"long","logicalType":"timestamp-micros"}},"# + + #"{"name":"local","type":{"type":"long","logicalType":"local-timestamp-millis"}},"# + + #"{"name":"price","type":{"type":"bytes","logicalType":"decimal","precision":10,"scale":2}},"# + + #"{"name":"weight","type":{"type":"fixed","name":"Weight","size":8,"logicalType":"decimal","precision":12,"scale":3}}]}"# + + private static func json(_ hex: String, schema: String) throws -> String { + let message = try AvroDecoder.decode(BinaryFixture.data(hex), schema: try AvroSchema.parse(schema)) + return MessagePackTextWriter.json(message.value, pretty: false) + } + + @Test("A record with every primitive and complex type") + func record() throws { + let hex = "9593d89fee470e416e6e204c6565021e616e6e406578616d706c652e636f6d01000000000000f83f000080be0402610662c3a9" + + "000402780202790300040102feff040010" + let expected = #"{"id":-1234567890123,"name":"Ann Lee","email":"ann@example.com","active":true,"score":1.5,"# + + #""ratio":-0.25,"tags":["a","bé"],"attrs":{"x":1,"y":-2},"kind":"C","hash":"0x0102FEFF","raw":"0x0010","# + + #""nothing":null}"# + #expect(try Self.json(hex, schema: Self.userSchema) == expected) + } + + @Test("A datum that starts with zero bytes reads as it is, not as a Kafka frame") + func leadingZeroDatum() throws { + let message = try AvroDecoder.decode( + BinaryFixture.data(String(repeating: "00", count: 24)), + schema: try AvroSchema.parse(Self.userSchema) + ) + #expect(message.framing == nil) + let expected = #"{"id":0,"name":"","email":null,"active":false,"score":0.0,"ratio":0.0,"tags":[],"attrs":{},"# + + #""kind":"A","hash":"0x00000000","raw":"0x","nothing":null}"# + #expect(MessagePackTextWriter.json(message.value, pretty: false) == expected) + } + + @Test("A Kafka value keeps its schema ID and decodes past the prefix") + func confluentFrame() throws { + let hex = "000000002a0e024b000100000000000000400000803f0202740002026b06000261626364027f" + let message = try AvroDecoder.decode(BinaryFixture.data(hex), schema: try AvroSchema.parse(Self.userSchema)) + #expect(message.framing == .confluent(schemaID: 42)) + let expected = #"{"id":7,"name":"K","email":null,"active":true,"score":2.0,"ratio":1.0,"tags":["t"],"attrs":{"k":3},"# + + #""kind":"B","hash":"0x61626364","raw":"0x7F","nothing":null}"# + #expect(MessagePackTextWriter.json(message.value, pretty: false) == expected) + } + + @Test("Logical types render as the values they stand for") + func logicalTypes() throws { + let hex = "4836636364373830632d626162612d313032362d393536342d35623863363536303234646296b502aab2992bcaa1c4de1b" + + "aa82e1e9c163018ef6e0e9c16306fe1dc00000000000003039" + let expected = #"{"id":"6ccd780c-baba-1026-9564-5b8c656024db","day":"2024-03-05","tms":"12:34:56.789","# + + #""tus":"01:02:03.004005","tsms":"2024-03-05T12:34:56.789Z","tsus":"1969-12-31T23:59:59.999999Z","# + + #""local":"2024-03-05T12:34:56.007","price":"-1234.56","weight":"12.345"}"# + #expect(try Self.json(hex, schema: Self.logicalSchema) == expected) + } + + @Test("A record that refers to itself by name") + func recursiveRecord() throws { + let schema = #"{"type":"record","name":"Node","namespace":"com.example.list","fields":["# + + #"{"name":"value","type":"int"},{"name":"next","type":["null","Node"]}]}"# + #expect(try Self.json("020204020600", schema: schema) + == #"{"value":1,"next":{"value":2,"next":{"value":3,"next":null}}}"#) + } + + @Test("A bare long at its minimum, and a union of several branches") + func primitivesAndUnions() throws { + #expect(try Self.json("ffffffffffffffffff01", schema: #""long""#) == "-9223372036854775808") + let union = #"{"type":"array","items":["int","string",{"type":"array","items":"long"}]}"# + #expect(try Self.json("06000a020273040402010000", schema: union) == #"[5,"s",[1,-1]]"#) + } + + @Test("An array block with a negative count carries its byte size") + func negativeBlockCount() throws { + #expect(try Self.json("0304020400", schema: #"{"type":"array","items":"int"}"#) == "[1,2]") + } + + @Test("An array of nulls holds more items than bytes, up to a limit") + func zeroByteItems() throws { + let nulls = #"{"type":"array","items":"null"}"# + #expect(try Self.json("0400", schema: nulls) == "[null,null]") + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("feffffffffffffffff0100"), schema: try AvroSchema.parse(nulls)) + } + #expect(error?.kind == .tooLarge) + } + + @Test("Items that take no bytes count toward the limit across every block, nested ones included") + func zeroByteItemsAcrossBlocks() throws { + let nested = try AvroSchema.parse(#"{"type":"array","items":{"type":"array","items":"null"}}"#) + let fourThousandNulls = "c03e00" + #expect(throws: Never.self) { + try AvroDecoder.decode(BinaryFixture.data("04" + String(repeating: fourThousandNulls, count: 2) + "00"), schema: nested) + } + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("d804" + String(repeating: fourThousandNulls, count: 300) + "00"), schema: nested) + } + #expect(error?.kind == .tooLarge) + } + + @Test("A cell's budget stops building, not reading, so it finds the Kafka frame the viewer finds") + func budgetKeepsTheFramingDecision() throws { + let schema = try AvroSchema.parse( + #"{"type":"record","name":"Six","fields":[{"name":"a","type":"long"},{"name":"b","type":"long"},"# + + #"{"name":"c","type":"long"},{"name":"d","type":"long"},{"name":"e","type":"long"},{"name":"f","type":"long"}]}"# + ) + let framed = BinaryFixture.data("0000000005" + "020406080a0c") + let full = try AvroDecoder.decode(framed, schema: schema) + #expect(full.framing == .confluent(schemaID: 5)) + #expect(MessagePackTextWriter.json(full.value, pretty: false) == #"{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6}"#) + let budgeted = try AvroDecoder.decode(framed, schema: schema, maxValues: 3) + #expect(budgeted.framing == .confluent(schemaID: 5)) + #expect(MessagePackTextWriter.json(budgeted.value, pretty: false) == #"{"a":1,"b":2,"c":…}"#) + } + + @Test("Fingerprints match Avro Java's CRC-64-AVRO of the Parsing Canonical Form") + func fingerprints() throws { + #expect(try AvroSchema.parse(#""int""#).fingerprint == 0x7275_D51A_3F39_5C8F) + #expect(try AvroSchema.parse(#"{"type":"string"}"#).fingerprint == 0x8F01_4872_6345_03C7) + #expect(try AvroSchema.parse(#""bytes""#).fingerprint == 0x4FC0_16DA_C320_1965) + let record = #"{"type":"record","name":"R","doc":"ignored","fields":[{"name":"a","type":"long"},"# + + #"{"name":"b","type":{"type":"string","logicalType":"uuid"}}]}"# + #expect(try AvroSchema.parse(record).fingerprint == 0xD99C_4645_DD9B_2051) + let payment = #"{"type":"record","name":"Payment","namespace":"com.example","fields":[{"name":"id","type":"string"},"# + + #"{"name":"amount","type":{"type":"fixed","name":"Money","size":8,"logicalType":"decimal","precision":12,"scale":2}},"# + + #"{"name":"fee","type":["null","Money"]},{"name":"kind","type":{"type":"enum","name":"Kind","symbols":["A","B"]}},"# + + #"{"name":"tags","type":{"type":"map","values":{"type":"array","items":"int"}}}]}"# + let parsed = try AvroSchema.parse(payment) + #expect(AvroSchema.canonicalForm(parsed.root) == #"{"name":"com.example.Payment","type":"record","fields":["# + + #"{"name":"id","type":"string"},{"name":"amount","type":{"name":"com.example.Money","type":"fixed","size":8}},"# + + #"{"name":"fee","type":["null","com.example.Money"]},"# + + #"{"name":"kind","type":{"name":"com.example.Kind","type":"enum","symbols":["A","B"]}},"# + + #"{"name":"tags","type":{"type":"map","values":{"type":"array","items":"int"}}}]}"#) + #expect(parsed.fingerprint == 0x51D2_8FF2_5F71_BF64) + } + + @Test("A single-object encoded value is read past its header when the fingerprint is the schema's, and refused when not") + func singleObjectEncoding() throws { + let schema = try AvroSchema.parse( + #"{"type":"record","name":"R","fields":[{"name":"a","type":"long"},{"name":"b","type":"string"}]}"# + ) + let data = BinaryFixture.data("c301" + "51209bdd45469cd9" + "020278") + let message = try AvroDecoder.decode(data, schema: schema) + #expect(message.framing == .singleObject(fingerprint: 0xD99C_4645_DD9B_2051)) + #expect(MessagePackTextWriter.json(message.value, pretty: false) == #"{"a":1,"b":"x"}"#) + + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(data, schema: try AvroSchema.parse(#""long""#)) + } + #expect(error?.kind == .otherSchema(fingerprint: 0xD99C_4645_DD9B_2051)) + } + + @Test("A Kafka value of a bytes schema is the bytes themselves, with no length in front") + func confluentBytes() throws { + let message = try AvroDecoder.decode(BinaryFixture.data("0000000007dead"), schema: try AvroSchema.parse(#""bytes""#)) + #expect(message.framing == .confluent(schemaID: 7)) + #expect(message.value == .binary(Data([0xDE, 0xAD]))) + let decimal = try AvroSchema.parse(#"{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}"#) + #expect(try AvroDecoder.decode(BinaryFixture.data("000000000704d2"), schema: decimal).value == .string("12.34")) + } + + @Test("Records that take no bytes, nested two to a level, are refused rather than walked exponentially") + func zeroByteRecordsDoNotExpand() throws { + var definitions = [#"{"type":"record","name":"R0","fields":[]}"#] + for level in 1...30 { + definitions.append(#"{"type":"record","name":"R"# + "\(level)" + #"","fields":[{"name":"a","type":"R"# + + "\(level - 1)" + #""},{"name":"b","type":"R"# + "\(level - 1)" + #""}]}"#) + } + let fields = definitions.enumerated().map { #"{"name":"f"# + "\($0.offset)" + #"","type":"# + $0.element + "}" } + let holder = #"{"type":"record","name":"Holder","fields":["# + fields.dropLast().joined(separator: ",") + "]}" + let schema = try AvroSchema.parse(#"{"type":"array","items":{"type":"record","name":"Wrap","fields":[{"name":"h","type":"# + + holder + #"},{"name":"r","type":"# + definitions[30] + "}]}}") + #expect(try AvroDecoder.decode(Data([0x00]), schema: schema).value == .array([])) + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(Data([0x02, 0x00]), schema: schema) + } + #expect(error?.kind == .tooLarge) + } + + @Test("A grid cell reads a value only so far, then leaves it to the viewer") + func cellReadLimit() throws { + let booleans = try AvroSchema.parse(#"{"type":"array","items":"boolean"}"#) + let data = Data([0x90, 0x03]) + Data(repeating: 0x01, count: 200) + Data([0x00]) + #expect(throws: Never.self) { try AvroDecoder.decode(data, schema: booleans, maxValues: 10) } + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(data, schema: booleans, maxValues: 10, maxValuesRead: 100) + } + #expect(error?.kind == .tooLarge) + } + + @Test("A schema that chains thousands of named records parses without walking the chain") + func longNamedChain() throws { + var records = [#"{"type":"record","name":"N0","fields":[]}"#] + for index in 1...20_000 { + records.append(#"{"type":"record","name":"N"# + "\(index)" + #"","fields":[{"name":"p","type":"N"# + "\(index - 1)" + #""}]}"#) + } + let schema = try AvroSchema.parse("[" + records.joined(separator: ",") + "]") + #expect(schema.takesNoBytes(.named("N20000"))) + #expect(!schema.takesNoBytes(.union([.null]))) + } + + @Test("A huge zero-byte block after a small one is refused, not added up until it overflows") + func zeroByteCountOverflow() throws { + let nulls = try AvroSchema.parse(#"{"type":"array","items":"null"}"#) + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("c801feffffffffffffffff0100"), schema: nulls) + } + #expect(error?.kind == .tooLarge) + } + + @Test("A size, precision or scale that is not a whole number is refused, as Avro's parser refuses it") + func strictIntegerAttributes() throws { + #expect(throws: AvroSchemaError.self) { try AvroSchema.parse(#"{"type":"fixed","name":"F","size":true}"#) } + #expect(throws: AvroSchemaError.self) { try AvroSchema.parse(#"{"type":"fixed","name":"F","size":4.5}"#) } + #expect(try Self.json("0230", schema: #"{"type":"bytes","logicalType":"decimal","precision":4.0}"#) == #""0x30""#) + #expect(try Self.json("0230", schema: #"{"type":"bytes","logicalType":"decimal","precision":4,"scale":true}"#) == #""0x30""#) + } + + @Test("A decimal without a usable precision and scale reads as its bytes") + func invalidDecimalFallsBack() throws { + for annotation in [#""scale":2"#, #""precision":2,"scale":3"#, #""precision":100000000,"scale":100000000"#] { + let schema = #"{"type":"bytes","logicalType":"decimal","# + annotation + "}" + #expect(try Self.json("0230", schema: schema) == #""0x30""#, "\(annotation)") + } + } + + @Test("Decimals of any width keep their sign and scale") + func decimalText() { + #expect(AvroDecimal.text(unscaled: [0x00], scale: 2) == "0.00") + #expect(AvroDecimal.text(unscaled: [0xFF], scale: 3) == "-0.001") + #expect(AvroDecimal.text(unscaled: [0x30, 0x39], scale: 0) == "12345") + #expect(AvroDecimal.text(unscaled: [0x80] + Array(repeating: 0, count: 15), scale: 0) + == "-170141183460469231731687303715884105728") + #expect(AvroDecimal.text(unscaled: [0x0D, 0xE0, 0xB6, 0xB3, 0xA7, 0x64, 0x00, 0x00], scale: 18) == "1.000000000000000000") + #expect(AvroDecimal.text(unscaled: [0x3B, 0x9A, 0xCA, 0x00], scale: 0) == "1000000000") + } + + @Test("A decimal reads as its bytes once its significant width passes what its precision holds") + func decimalWidthFollowsPrecision() throws { + let schema = #"{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}"# + #expect(try Self.json("0404d2", schema: schema) == #""12.34""#) + #expect(try Self.json("08000004d2", schema: schema) == #""12.34""#) + #expect(try Self.json("067fffff", schema: schema) == #""0x7FFFFF""#) + let padded = #"{"type":"fixed","name":"Wide","size":16,"logicalType":"decimal","precision":4,"scale":2}"# + #expect(try Self.json("000000000000000000000000000004d2", schema: padded) == #""12.34""#) + #expect(AvroDecimal.maxBytes(precision: 38) == 16) + #expect(AvroDecimal.maxPrecision(bytes: 16) == 38) + } + + @Test("A fixed's decimal precision past what its size holds is ignored, as Avro's reader does") + func fixedDecimalPrecisionFitsSize() throws { + let schema = #"{"type":"fixed","name":"Small","size":2,"logicalType":"decimal","precision":5,"scale":0}"# + #expect(try Self.json("04d2", schema: schema) == #""0x04D2""#) + } + + @Test("A named fixed keeps its logical type where it is referred to by name") + func namedLogicalType() throws { + let schema = #"{"type":"record","name":"Prices","fields":["# + + #"{"name":"a","type":{"type":"fixed","name":"Money","size":8,"logicalType":"decimal","precision":12,"scale":2}},"# + + #"{"name":"b","type":"Money"}]}"# + #expect(try Self.json("00000000000004d2fffffffffffffffb", schema: schema) == #"{"a":"12.34","b":"-0.05"}"#) + } + + @Test("Nanosecond timestamps and durations render as values") + func nanosAndDuration() throws { + let nanos = #"{"type":"record","name":"N","fields":["# + + #"{"name":"ts","type":{"type":"long","logicalType":"timestamp-nanos"}},"# + + #"{"name":"lts","type":{"type":"long","logicalType":"local-timestamp-nanos"}}]}"# + #expect(try Self.json("aab4aed8c7bfce972faab4aed8c7bfce972f", schema: nanos) + == #"{"ts":"2023-11-14T22:13:20.123456789Z","lts":"2023-11-14T22:13:20.123456789"}"#) + let duration = #"{"type":"fixed","name":"D","size":12,"logicalType":"duration"}"# + #expect(try Self.json("0100000002000000bc0b0000", schema: duration) == #""P1M2DT3.004S""#) + #expect(try Self.json("000000000000000000000000", schema: duration) == #""PT0S""#) + } + + @Test("A value that does not fit the schema names what is wrong and where") + func brokenValues() throws { + let schema = try AvroSchema.parse(Self.userSchema) + let cases: [(hex: String, kind: BinaryValueDecodeError.Kind)] = [ + ("02", .unexpectedEnd), + ("020e416e6e", .unexpectedEnd), + ("02000600", .indexOutOfRange(3)), + ("0201", .invalidLength), + ("02000002", .invalidByte(0x02)), + ] + for testCase in cases { + let error = #expect(throws: BinaryValueDecodeError.self, "\(testCase.hex)") { + try AvroDecoder.decode(BinaryFixture.data(testCase.hex), schema: schema) + } + #expect(error?.kind == testCase.kind, "\(testCase.hex)") + } + let int = try AvroSchema.parse(#""int""#) + let error = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("8080808020"), schema: int) + } + #expect(error?.kind == .integerOutOfRange) + let trailing = #expect(throws: BinaryValueDecodeError.self) { + try AvroDecoder.decode(BinaryFixture.data("0202"), schema: int) + } + #expect(trailing?.kind == .trailingBytes(1)) + } + + @Test("A schema that is not one says why") + func brokenSchemas() { + for text in [ + "not json", + #"{"type":"record","name":"R"}"#, + #"{"type":"record","name":"R","fields":[{"name":"x","type":"Missing"}]}"#, + #"{"type":"enum","name":"E"}"#, + #"{"type":"fixed","name":"F"}"#, + #"{"type":"array"}"#, + "42", + #"{"type":"record","name":"R","fields":[{"name":"a","type":{"type":"enum","name":"R","symbols":["X"]}}]}"#, + ] { + #expect(throws: AvroSchemaError.self, "\(text)") { try AvroSchema.parse(text) } + } + } + + @Test("A named type resolves inside its namespace and by full name") + func namespaces() throws { + let schema = try AvroSchema.parse( + #"{"type":"record","name":"Outer","namespace":"a.b","fields":["# + + #"{"name":"e","type":{"type":"enum","name":"Color","symbols":["RED"]}},"# + + #"{"name":"f","type":"Color"},{"name":"g","type":"a.b.Color"}]}"# + ) + #expect(schema.namedTypes["a.b.Color"] == .enumeration(name: "a.b.Color", symbols: ["RED"])) + #expect(schema.displayName == "Outer") + let value = try AvroDecoder.decode(BinaryFixture.data("000000"), schema: schema).value + #expect(MessagePackTextWriter.json(value, pretty: false) == #"{"e":"RED","f":"RED","g":"RED"}"#) + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift index 298f4fa66a..44057ed83b 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift @@ -10,7 +10,7 @@ import Testing @MainActor struct BinaryDisplayFormatTests { - /// `HEX(UUID_TO_BIN('6ccd780c-baba-1026-9564-5b8c656024db', 1))` on MySQL 8.4. + // `HEX(UUID_TO_BIN('6ccd780c-baba-1026-9564-5b8c656024db', 1))` on MySQL 8.4. private static let swappedStored = BinaryFixture.data("1026BABA6CCD780C95645B8C656024DB") private static let blob = ColumnType.blob(rawType: "BINARY(16)") @@ -131,4 +131,48 @@ struct BinaryDisplayFormatTests { #expect(text == "1: 150\n") #expect(note?.contains("42") == true) } + + @Test("igbinary and Avro render in the cell, and a broken value falls back to hex") + func igbinaryAndAvroRender() throws { + let blob = ColumnType.blob(rawType: "BLOB") + let igbinary = BinaryFixture.data("000000021705506f696e7414011101780601") + #expect(CellDisplayFormatter.format(.bytes(igbinary), columnType: blob, displayFormat: .igbinary) + == #"O:5:"Point":1:{s:1:"x";i:1;}"#) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("0000000211")), columnType: blob, displayFormat: .igbinary) + == "0x0000000211") + + let schemaID = try AvroSchemaStore.shared.add(#"{"type":"record","name":"Reading","fields":[{"name":"id","type":"long"}]}"#) + let avro = ValueDisplayFormat.avro(schemaID: schemaID) + #expect(avro.displayName.contains("Reading")) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("54")), columnType: blob, displayFormat: avro) == #"{"id":42}"#) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("80")), columnType: blob, displayFormat: avro) == "0x80") + } + + @Test("The viewer shows igbinary in the PHP tree and Avro with its frame's schema ID") + func igbinaryAndAvroDocuments() throws { + let igbinary = BinaryValueDecoding.document(BinaryFixture.data("000000020607"), format: .igbinary) + guard case .decoded(.php, let text, nil) = igbinary else { + Issue.record("expected a decoded PHP tree") + return + } + #expect(text == "i:7;") + + let schemaID = try AvroSchemaStore.shared.add(#"{"type":"array","items":"int"}"#) + let framed = BinaryValueDecoding.document(BinaryFixture.data("000000002a020200"), format: .avro(schemaID: schemaID)) + guard case .decoded(.json, _, let note) = framed else { + Issue.record("expected a decoded JSON tree") + return + } + #expect(note?.contains("42") == true) + } + + @Test("An Avro column whose schema is gone says so, rather than calling the value invalid, and the cell stays hex") + func avroSchemaMissing() { + let format = BinaryValueFormat.avro(schemaID: String(repeating: "0", count: 32)) + guard case .missingSchema = BinaryValueDecoding.document(Data([0x02]), format: format) else { + Issue.record("expected a missing schema") + return + } + #expect(BinaryValueDecoding.text(Data([0x02]), format: format) == nil) + } } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift new file mode 100644 index 0000000000..2b39e3908a --- /dev/null +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/IgbinaryDecoderTests.swift @@ -0,0 +1,293 @@ +// +// IgbinaryDecoderTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +// Fixtures from igbinary 3.2.17RC1 on PHP 8.3: igbinary_serialize($value) beside serialize($value). +struct IgbinaryDecoderTests { + private static let fixtures: [(name: String, hex: String, serialized: String)] = [ + ("null", "0000000200", #"N;"#), + ("true", "0000000205", #"b:1;"#), + ("false", "0000000204", #"b:0;"#), + ("small int", "000000020607", #"i:7;"#), + ("negative int", "0000000209012c", #"i:-300;"#), + ("32-bit int", "000000020a00011170", #"i:70000;"#), + ("64-bit int", "0000000220000000012a05f200", #"i:5000000000;"#), + ("PHP_INT_MAX", "00000002207fffffffffffffff", #"i:9223372036854775807;"#), + ("PHP_INT_MIN", "00000002218000000000000000", #"i:-9223372036854775808;"#), + ("empty string", "000000020d", #"s:0:"";"#), + ("UTF-8 string", "00000002110d68c3a96c6c6f2077c3b6726c64", #"s:13:"héllo wörld";"#), + ( + "list of scalars", + "000000021406060006010601060206021105746872656506030c4012000000000000060400060504", + #"a:6:{i:0;i:1;i:1;i:2;i:2;s:5:"three";i:3;d:4.5;i:4;N;i:5;b:0;}"# + ), + ( + "nested map with int keys", + "00000002140511046e616d651103416e6e11047461677314020600110161060111016211046d65746114020e00110569" + + "6e6e657211016e0701060a110374656e070511036e6567", + #"a:5:{s:4:"name";s:3:"Ann";s:4:"tags";a:2:{i:0;s:1:"a";i:1;s:1:"b";}s:4:"meta";a:2:{s:4:"na"# + + #"me";s:5:"inner";s:1:"n";i:-1;}i:10;s:3:"ten";i:-5;s:3:"neg";}"# + ), + ( + "repeated strings by ID", + "00000002140506001105616c706861060111046265746106020e0006030e0106040e00", + #"a:5:{i:0;s:5:"alpha";i:1;s:4:"beta";i:2;s:5:"alpha";i:3;s:4:"beta";i:4;s:5:"alpha";}"# + ), + ( + "keys and values share string IDs", + "00000002140311016b1101760e010e0011017a14010e000e01", + #"a:3:{s:1:"k";s:1:"v";s:1:"v";s:1:"k";s:1:"z";a:1:{s:1:"k";s:1:"v";}}"# + ), + ( + "object with every visibility", + "000000021705506f696e74140311017806011104002a00790602110800506f696e74007a0603", + #"O:5:"Point":3:{s:1:"x";i:1;s:4:"\#u{0}*\#u{0}y";i:2;s:8:"\#u{0}Point\#u{0}z";i:3;}"# + ), + ("stdClass", "000000021708737464436c617373140211016106011101621401060005", #"O:8:"stdClass":2:{s:1:"a";i:1;s:1:"b";a:1:{i:0;b:1;}}"#), + ("empty stdClass", "000000021708737464436c6173731400", #"O:8:"stdClass":0:{}"#), + ("enum case", "00000002170453756974271106486561727473", #"E:11:"Suit:Hearts";"#), + ( + "repeated enum case", + "00000002140406001704537569742711064865617274730601220106021a0027110653706164657306030601", + #"a:4:{i:0;E:11:"Suit:Hearts";i:1;r:2;i:2;E:11:"Suit:Spades";i:3;i:1;}"# + ), + ( + "PHP reference to an enum case", + "000000021403060025170453756974271106486561727473060125220106020602", + #"a:3:{i:0;E:11:"Suit:Hearts";i:1;R:2;i:2;i:2;}"# + ), + ( + "repeated object", + "00000002140306001705506f696e74140311017806011104002a00790602110800506f696e74007a0603060122010602" + + "2201", + #"a:3:{i:0;O:5:"Point":3:{s:1:"x";i:1;s:4:"\#u{0}*\#u{0}y";i:2;s:8:"\#u{0}Point\#u{0}z";i:3;}i:1;r:2;i:2;r:2;}"# + ), + ( + "__serialize object repeated", + "00000002140306001703426f78140111017606050601220106020603", + #"a:3:{i:0;O:3:"Box":1:{s:1:"v";i:5;}i:1;r:2;i:2;i:3;}"# + ), + ("Serializable object repeated", "000000021403060017034f6c641d036162630601220106020604", #"a:3:{i:0;C:3:"Old":3:{abc}i:1;r:2;i:2;i:4;}"#), + ("object holding itself", "000000021708737464436c6173731401110473656c662200", #"O:8:"stdClass":1:{s:4:"self";r:1;}"#), + ("PHP reference inside an array repeated by value", "00000002140206001401060025060106010101", #"a:2:{i:0;a:1:{i:0;i:1;}i:1;a:1:{i:0;R:3;}}"#), + ( + "PHP reference to an object, then objects", + "0000000214040600251708737464436c6173731400060125220106021a00140006032202", + #"a:4:{i:0;O:8:"stdClass":0:{}i:1;R:2;i:2;O:8:"stdClass":0:{}i:3;r:3;}"# + ), + ( + "PHP reference to an array", + "00000002140306002514010600060106012501010602140106000601", + #"a:3:{i:0;a:1:{i:0;i:1;}i:1;R:2;i:2;a:1:{i:0;i:1;}}"# + ), + ( + "PHP reference inside an object", + "00000002140206001708737464436c617373140211016125110373747211016225010206012201", + #"a:2:{i:0;O:8:"stdClass":2:{s:1:"a";s:3:"str";s:1:"b";R:3;}i:1;r:2;}"# + ), + ( + "object and its reference in an array repeated by value", + "0000000214020600140206001708737464436c6173731400060125220206010101", + #"a:2:{i:0;a:2:{i:0;O:8:"stdClass":0:{}i:1;R:3;}i:1;a:2:{i:0;r:3;i:1;R:3;}}"# + ), + ("scalar reference, then its value", "0000000214030600251102616106010e000602250101", #"a:3:{i:0;s:2:"aa";i:1;s:2:"aa";i:2;R:2;}"#), + ("PHP reference to null", "000000021402060025000601250101", #"a:2:{i:0;N;i:1;R:2;}"#), + ( + "array repeated by value", + "0000000214030600140111016b1101760601010106020101", + #"a:3:{i:0;a:1:{s:1:"k";s:1:"v";}i:1;a:1:{s:1:"k";s:1:"v";}i:2;a:1:{s:1:"k";s:1:"v";}}"# + ), + ( + "arrays repeated by value inside one another", + "000000021402060014020600140111017806010601010206010101", + #"a:2:{i:0;a:2:{i:0;a:1:{s:1:"x";i:1;}i:1;a:1:{s:1:"x";i:1;}}i:1;a:2:{i:0;a:1:{s:1:"x";i:1;}"# + + #"i:1;a:1:{s:1:"x";i:1;}}}"# + ), + ( + "object inside an array repeated by value", + "0000000214030600140206001705506f696e74140311017806011104002a00790602110800506f696e74007a06030601" + + "06020601010106022202", + #"a:3:{i:0;a:2:{i:0;O:5:"Point":3:{s:1:"x";i:1;s:4:"\#u{0}*\#u{0}y";i:2;s:8:"\#u{0}Point\#u{0}z";i:3;}i:1;i:2;}i"# + + #":1;a:2:{i:0;r:3;i:1;i:2;}i:2;r:3;}"# + ), + ( + "floats", + "00000002140e06000c800000000000000006010c430c6bf52634000006020c4341c37937e0800006030c4376345785d8" + + "a00006040c3fb999999999999a06050c3ee4f8b588e368f106060c3f1a36e2eb1c432d06070c3e8421f5f40d83760608" + + "0c40fe240c9fbe76c906090c0000000000000001060a0c7fefffffffffffff060b0c4345ee2a2eb5a5c4060c0c7ff000" + + "0000000000060d0cfff0000000000000", + #"a:14:{i:0;d:-0;i:1;d:1000000000000000;i:2;d:10000000000000000;i:3;d:1.0E+17;i:4;d:0.1;i:5;"# + + #"d:1.0E-5;i:6;d:0.0001;i:7;d:1.5E-7;i:8;d:123456.789;i:9;d:5.0E-324;i:10;d:1.79769313486231"# + + #"57E+308;i:11;d:12345678901234568;i:12;d:INF;i:13;d:-INF;}"# + ), + ( + "ArrayObject, whose members have integer keys", + "00000002170b41727261794f626a65637414040600060006011402060006010601060206021400060300", + #"O:11:"ArrayObject":4:{i:0;i:0;i:1;a:2:{i:0;i:1;i:1;i:2;}i:2;a:0:{}i:3;N;}"# + ), + ("__serialize list", "000000021401060017074c697374426f7814020600060a0601110178", #"a:1:{i:0;O:7:"ListBox":2:{i:0;i:10;i:1;s:1:"x";}}"#), + ] + + @Test("Each fixture decodes to what serialize() wrote, numbering, r and R included") + func fixturesMatchSerialize() throws { + for fixture in Self.fixtures { + let decoded = try IgbinaryDecoder.decode(BinaryFixture.data(fixture.hex)) + #expect(PhpSerializeWriter.text(decoded) == fixture.serialized, "\(fixture.name)") + #expect(decoded == PhpSerializeParser.parse(fixture.serialized), "\(fixture.name)") + } + } + + @Test("A string that is not UTF-8 keeps its bytes, and its length is the byte count serialize() writes") + func binaryStrings() throws { + let value = try IgbinaryDecoder.decode(BinaryFixture.data("00000002110600ff80616263")) + #expect(value == .binaryString(Data([0x00, 0xFF, 0x80, 0x61, 0x62, 0x63]))) + #expect(PhpSerializeWriter.text(value) == Self.latin1("733a363a2200ff80616263223b")) + + let keyed = try IgbinaryDecoder.decode(BinaryFixture.data("0000000214021102fffe06011103fffe781101ff")) + #expect(keyed == .array([ + PhpKeyValue(key: .binaryString(Data([0xFF, 0xFE])), value: .int(1)), + PhpKeyValue(key: .binaryString(Data([0xFF, 0xFE, 0x78])), value: .binaryString(Data([0xFF]))), + ])) + #expect( + PhpSerializeWriter.text(keyed) + == Self.latin1("613a323a7b733a323a22fffe223b693a313b733a333a22fffe78223b733a313a22ff223b7d") + ) + } + + @Test("A Serializable payload that is not UTF-8 keeps its bytes and its byte count") + func binarySerializablePayload() throws { + let value = try IgbinaryDecoder.decode(BinaryFixture.data("0000000217034f6c641d01ff")) + #expect(value == .binarySerializable(className: "Old", payload: Data([0xFF]))) + #expect(PhpSerializeWriter.text(value) == #"C:3:"Old":1:{"# + "\u{FF}}") + } + + @Test("Text past the limit is cut before it is copied, and its length stays the full one") + func writerCutsLargePayloads() { + let text = PhpSerializeWriter.text(.string(String(repeating: "a", count: 1_000_000)), limit: 100) + #expect(text.hasPrefix(#"s:1000000:"aaa"#)) + #expect(text.utf8.count < 200) + let bytes = PhpSerializeWriter.text(.binarySerializable(className: "Old", payload: Data(count: 1_000_000)), limit: 100) + #expect(bytes.hasPrefix(#"C:3:"Old":1000000:{"#)) + #expect(bytes.utf8.count < 200) + } + + @Test("An object property name that is not UTF-8 is written with its own bytes") + func binaryPropertyName() throws { + let value = try IgbinaryDecoder.decode(BinaryFixture.data("000000021708737464436c617373140111" + "01ff" + "0601")) + #expect(PhpSerializeWriter.text(value) == #"O:8:"stdClass":1:{s:1:""# + "\u{FF}" + #"";i:1;}"#) + } + + @Test("A string of 300 bytes uses the 16-bit length") + func longString() throws { + let hex = "0000000212012c" + String(repeating: "61", count: 300) + #expect(try IgbinaryDecoder.decode(BinaryFixture.data(hex)) == .string(String(repeating: "a", count: 300))) + } + + @Test("Arrays nested by value forty deep are refused rather than expanded") + func exponentialCopiesAreRefused() { + func level(_ depth: Int, of total: Int) -> String { + guard depth > 0 else { return "14020600060106010601" } + return "14020600" + level(depth - 1, of: total) + "060101" + String(format: "%02x", total - depth + 1) + } + let error = #expect(throws: BinaryValueDecodeError.self) { + try IgbinaryDecoder.decode(BinaryFixture.data("00000002" + level(40, of: 40))) + } + #expect(error?.kind == .tooLarge) + #expect(throws: Never.self) { try IgbinaryDecoder.decode(BinaryFixture.data("00000002" + level(3, of: 3))) } + } + + @Test("A long string repeated by ID past the limit is refused") + func repeatedLongStringIsRefused() { + var hex = "0000000215" + String(format: "%04x", 6_000) + "080000" + "1300001000" + String(repeating: "61", count: 4_096) + for index in 1..<6_000 { + hex += "08" + String(format: "%04x", index) + "0e00" + } + let error = #expect(throws: BinaryValueDecodeError.self) { try IgbinaryDecoder.decode(BinaryFixture.data(hex)) } + #expect(error?.kind == .tooLarge) + } + + @Test("Strings inside an array repeated by value count toward the string limit") + func repeatedArrayStringsAreRefused() { + let text = Data(repeating: 0x61, count: 1_000_000) + func value(copies: Int) -> Data { + var data = Data([0, 0, 0, 2, 0x15]) + Self.bigEndian(UInt16(copies + 1)) + data += Data([0x06, 0x00, 0x14, 0x01, 0x06, 0x00, 0x13]) + Self.bigEndian(UInt32(text.count)) + text + for index in 1...copies { + data += Data([0x08]) + Self.bigEndian(UInt16(index)) + Data([0x01, 0x01]) + } + return data + } + #expect(throws: Never.self) { try IgbinaryDecoder.decode(value(copies: 10)) } + let error = #expect(throws: BinaryValueDecodeError.self) { try IgbinaryDecoder.decode(value(copies: 30)) } + #expect(error?.kind == .tooLarge) + } + + @Test("A copy placed deeper than the array it repeats still fits the depth cap") + func copiesKeepTheDepthCap() { + func value(levels: Int) -> Data { + let chain = 80 + var data = Data([0, 0, 0, 2, 0x15]) + Self.bigEndian(UInt16(levels + 1)) + Data([0x06, 0x00]) + data += Data(Array(repeating: [0x14, 0x01, 0x06, 0x00] as [UInt8], count: chain).joined()) + Data([0x00]) + for level in 1...levels { + let target = level == 1 ? 1 : chain + level - 1 + data += Data([0x08]) + Self.bigEndian(UInt16(level)) + data += Data([0x14, 0x01, 0x06, 0x00, 0x02]) + Self.bigEndian(UInt16(target)) + } + return data + } + #expect(throws: Never.self) { try IgbinaryDecoder.decode(value(levels: 100)) } + let error = #expect(throws: BinaryValueDecodeError.self) { try IgbinaryDecoder.decode(value(levels: 200)) } + #expect(error?.kind == .nestingTooDeep) + } + + @Test("The version 1 header reads too") + func versionOne() throws { + #expect(try IgbinaryDecoder.decode(BinaryFixture.data("000000010607")) == .int(7)) + } + + @Test("A value that is not igbinary names what is wrong and where") + func brokenValues() { + let cases: [(hex: String, kind: BinaryValueDecodeError.Kind)] = [ + ("613a313a7b", .unknownVersion(0x613A_313A)), + ("00000002", .unexpectedEnd), + ("00000002110568656c", .unexpectedEnd), + ("00000002ff", .invalidByte(0xFF)), + ("000000020607aa", .trailingBytes(1)), + ("00000002140106000e05", .unknownReference(5)), + ("0000000214010600220a", .unknownReference(10)), + ("000000021401060001" + "00", .unknownReference(0)), + ("0000000214011400", .invalidByte(0x14)), + ("0000000220ffffffffffffffff", .integerOutOfRange), + ("0000000214ff", .unexpectedEnd), + ("000000022525" + "00", .invalidByte(0x25)), + ] + for testCase in cases { + let error = #expect(throws: BinaryValueDecodeError.self, "\(testCase.hex)") { + try IgbinaryDecoder.decode(BinaryFixture.data(testCase.hex)) + } + #expect(error?.kind == testCase.kind, "\(testCase.hex)") + } + } + + @Test("Floats PHP writes with serialize_precision -1, past what the fixtures hold") + func floatText() { + #expect(PhpSerializeWriter.floatText(.nan) == "NAN") + #expect(PhpSerializeWriter.floatText(0.5) == "0.5") + #expect(PhpSerializeWriter.floatText(-1.5e-7) == "-1.5E-7") + #expect(PhpSerializeWriter.floatText(1e25) == "1.0E+25") + #expect(PhpSerializeWriter.floatText(99_999_999_999_999_984) == "99999999999999980") + } + + private static func latin1(_ hex: String) -> String { + String(data: BinaryFixture.data(hex), encoding: .isoLatin1) ?? "" + } + + private static func bigEndian(_ value: T) -> Data { + withUnsafeBytes(of: value.bigEndian) { Data($0) } + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift b/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift index ab31cb0d41..22e2ab9b59 100644 --- a/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/PhpSerializeParserTests.swift @@ -211,7 +211,7 @@ struct PhpSerializeParserSpecialTests { #expect(payload == "xyzab") } - /// PHP 8.1 writes an enum case as `E:`. Before, any value holding one failed to parse at all. + // PHP 8.1 writes an enum case as `E:`. Before, any value holding one failed to parse at all. @Test("E token returns .enumCase, inside an array too") func enumCaseToken() { #expect(PhpSerializeParser.parse(#"E:11:"Suit:Hearts";"#) == .enumCase(className: "Suit", caseName: "Hearts")) @@ -228,9 +228,20 @@ struct PhpSerializeParserSpecialTests { #expect(tree.badgeLabel == "enum") } - @Test("r token returns .reference") + @Test("An object member with an integer key, as ArrayObject and __serialize data have, parses and writes back") + func integerObjectKeys() { + let text = #"O:11:"ArrayObject":4:{i:0;i:0;i:1;a:2:{i:0;i:1;i:1;i:2;}i:2;a:0:{}i:3;N;}"# + guard case .object("ArrayObject", let properties)? = PhpSerializeParser.parse(text) else { + Issue.record("expected an object") + return + } + #expect(properties.map(\.index) == [0, 1, 2, 3]) + #expect(PhpSerializeWriter.text(.object(className: "ArrayObject", properties: properties)) == text) + } + + @Test("r token returns .objectReference, so it is written back as r and not as R") func referenceLowerR() { - #expect(PhpSerializeParser.parse("r:7;") == .reference(id: 7)) + #expect(PhpSerializeParser.parse("r:7;") == .objectReference(id: 7)) } @Test("R token returns .reference") diff --git a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift index a49ee6578c..72d719c5e2 100644 --- a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift @@ -21,6 +21,23 @@ struct ValueDisplayFormatTests { #expect(ValueDisplayFormat.uuidSwappedTime.rawValue == "uuidSwappedTime") #expect(ValueDisplayFormat.messagePack.rawValue == "messagePack") #expect(ValueDisplayFormat.protobuf.rawValue == "protobuf") + #expect(ValueDisplayFormat.igbinary.rawValue == "igbinary") + #expect(ValueDisplayFormat.avro(schemaID: "ab12").rawValue == "avro:ab12") + } + + @Test("Every format reads back from its stored form, an Avro one with its schema ID") + func codableRoundTrip() throws { + let formats: [String: ValueDisplayFormat] = [ + "a": .messagePack, "b": .igbinary, "c": .avro(schemaID: String(repeating: "ab12", count: 8)), "d": .raw, + ] + let data = try JSONEncoder().encode(formats) + #expect(try JSONDecoder().decode([String: ValueDisplayFormat].self, from: data) == formats) + #expect(try JSONDecoder().decode([String: ValueDisplayFormat].self, from: Data(#"{"x":"protobuf"}"#.utf8)) + == ["x": .protobuf]) + #expect(ValueDisplayFormat(rawValue: "avro:") == nil) + #expect(ValueDisplayFormat(rawValue: "avro:ab12") == nil) + #expect(ValueDisplayFormat(rawValue: "avro:" + String(repeating: "AB12", count: 8)) == nil) + #expect(ValueDisplayFormat(rawValue: "nope") == nil) } @Test("only the formats that produce characters count as rendering binary as text") @@ -30,6 +47,8 @@ struct ValueDisplayFormatTests { #expect(ValueDisplayFormat.uuidSwappedTime.rendersBinaryAsText) #expect(ValueDisplayFormat.messagePack.rendersBinaryAsText) #expect(ValueDisplayFormat.protobuf.rendersBinaryAsText) + #expect(ValueDisplayFormat.igbinary.rendersBinaryAsText) + #expect(ValueDisplayFormat.avro(schemaID: "ab12").rendersBinaryAsText) #expect(!ValueDisplayFormat.raw.rendersBinaryAsText) #expect(!ValueDisplayFormat.unixTimestamp.rendersBinaryAsText) #expect(!ValueDisplayFormat.json.rendersBinaryAsText) @@ -60,7 +79,7 @@ struct ValueDisplayFormatTests { let formats = ValueDisplayFormat.applicableFormats(for: .blob(rawType: "BLOB")) #expect(!formats.contains(.json)) #expect(!formats.contains(.phpSerialized)) - #expect(formats == [.raw, .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf]) + #expect(formats == [.raw, .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf, .igbinary]) } @Test("The binary decoders and the swapped UUID stay off text and integer columns") @@ -70,6 +89,8 @@ struct ValueDisplayFormatTests { #expect(!formats.contains(.uuidSwappedTime)) #expect(!formats.contains(.messagePack)) #expect(!formats.contains(.protobuf)) + #expect(!formats.contains(.igbinary)) + #expect(!ValueDisplayFormat.avro(schemaID: "ab12").isApplicable(to: columnType)) } } @@ -81,7 +102,7 @@ struct ValueDisplayFormatTests { ) #expect(!formats.contains(.uuid)) - #expect(formats == [.raw, .text, .messagePack, .protobuf]) + #expect(formats == [.raw, .text, .messagePack, .protobuf, .igbinary]) } @Test("nil column type returns only raw") diff --git a/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift new file mode 100644 index 0000000000..65749dc6e9 --- /dev/null +++ b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift @@ -0,0 +1,93 @@ +// +// AvroSchemaStoreTests.swift +// TableProTests +// + +import Foundation +import os +@testable import TablePro +import Testing + +struct AvroSchemaStoreTests { + private static let schema = #"{"type":"record","name":"Reading","namespace":"com.example","fields":[{"name":"id","type":"long"}]}"# + + private func withDirectory(_ body: (URL) throws -> Void) rethrows { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + try body(directory) + } + + @Test("The same text kept twice is one file, read back by a later launch") + func sameTextIsOneFile() throws { + try withDirectory { directory in + let store = AvroSchemaStore(directory: directory) + let id = try store.add(Self.schema) + #expect(try store.add(Self.schema) == id) + #expect(AvroSchemaStore.isSchemaID(id)) + #expect(store.storedIDs == [id]) + #expect(AvroSchemaStore(directory: directory).schema(id: id)?.displayName == "Reading") + } + } + + @Test("Text that is not a schema is refused and leaves nothing behind") + func invalidTextIsRefused() throws { + try withDirectory { directory in + let store = AvroSchemaStore(directory: directory) + #expect(throws: AvroSchemaError.self) { try store.add(#"{"type":"record","name":"R"}"#) } + #expect(store.storedIDs.isEmpty) + } + } + + @Test("An ID that is not one of the store's never reaches the file system") + func foreignIDsAreRefused() throws { + try withDirectory { directory in + let store = AvroSchemaStore(directory: directory) + #expect(store.schema(id: "../../etc/hosts") == nil) + #expect(ValueDisplayFormat(rawValue: "avro:../../etc/hosts") == nil) + #expect(ValueDisplayFormat(rawValue: "avro:") == nil) + let id = try store.add(Self.schema) + #expect(ValueDisplayFormat(rawValue: "avro:" + id) == .avro(schemaID: id)) + } + } + + @Test("Adding a schema that was looked up and missing announces it, and only then") + func restoreIsAnnounced() throws { + try withDirectory { directory in + let store = AvroSchemaStore(directory: directory) + let announced = OSAllocatedUnfairLock<[String]>(initialState: []) + let observer = NotificationCenter.default.addObserver( + forName: AvroSchemaStore.didRestoreSchema, object: store, queue: nil + ) { notification in + let schemaID = notification.userInfo?[AvroSchemaStore.schemaIDKey] as? String ?? "" + announced.withLock { $0.append(schemaID) } + } + defer { NotificationCenter.default.removeObserver(observer) } + + _ = try store.add(#""int""#) + #expect(announced.withLock { $0 }.isEmpty) + + let lost = AvroSchemaStore.schemaID(of: Self.schema) + #expect(store.schema(id: lost) == nil) + #expect(try store.add(Self.schema) == lost) + #expect(announced.withLock { $0 } == [lost]) + _ = try store.add(Self.schema) + #expect(announced.withLock { $0 } == [lost]) + } + } + + @Test("A sweep keeps what is named and what this launch added") + func sweepKeepsNamedAndNew() throws { + try withDirectory { directory in + let earlier = AvroSchemaStore(directory: directory) + let named = try earlier.add(Self.schema) + let orphan = try earlier.add(#""long""#) + let store = AvroSchemaStore(directory: directory) + let fresh = try store.add(#""string""#) + + store.removeAll(except: [named]) + + #expect(store.storedIDs == [named, fresh]) + #expect(store.schema(id: orphan) == nil) + } + } +} diff --git a/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift b/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift index 8b3d551c94..d55432ed9e 100644 --- a/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift +++ b/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift @@ -235,4 +235,83 @@ struct ValueDisplayFormatStorageTests { #expect(defaults.data(forKey: legacyKey) == nil) #expect(storage.load(for: scope("public", connectionId: other)) == ["id": .json]) } + + @Test("A format 0.83 does not have goes in a second key, so the key 0.83 decodes whole still decodes") + func newFormatsStayOutOfThe083Key() throws { + let (storage, defaults) = try makeStorage() + let target = scope("public", connectionId: UUID()) + let avro = ValueDisplayFormat.avro(schemaID: String(repeating: "a", count: 32)) + storage.save(["a": .json, "b": .igbinary, "c": avro], for: target) + + let data = try #require(defaults.data(forKey: PreferenceKeys.columnDisplayFormats(target).name)) + #expect(try JSONDecoder().decode([String: FormatsIn083].self, from: data) == ["a": .json]) + #expect(storage.load(for: target) == ["a": .json, "b": .igbinary, "c": avro]) + } + + @Test("A format 0.83 sets later wins over the one it could not read") + func formatSetBy083Wins() throws { + let (storage, defaults) = try makeStorage() + let target = scope("public", connectionId: UUID()) + storage.save(["b": .igbinary], for: target) + + defaults.set(try JSONEncoder().encode(["b": FormatsIn083.json]), forKey: PreferenceKeys.columnDisplayFormats(target).name) + + #expect(storage.load(for: target) == ["b": .json]) + } + + @Test("A format name this build does not know is kept through other columns' changes") + func unknownFormatsAreKept() throws { + let (storage, defaults) = try makeStorage() + let service = ValueDisplayFormatService(storage: storage) + let target = scope("public", connectionId: UUID()) + let addedKey = PreferenceKeys.columnDisplayFormats(target).name + ".added" + defaults.set(try JSONEncoder().encode(["x": "formatFromALaterBuild"]), forKey: addedKey) + + service.setOverride(.igbinary, columnKey: "a", scope: target) + service.setOverride(nil, columnKey: "a", scope: target) + + #expect(storage.load(for: target) == [:]) + let kept = try #require(defaults.data(forKey: addedKey)) + #expect(try JSONDecoder().decode([String: String].self, from: kept) == ["x": "formatFromALaterBuild"]) + } + + @Test("A table rename moves the formats in both keys") + func renameTableMovesBothKeys() throws { + let (storage, _) = try makeStorage() + let conn = UUID() + storage.save(["id": .uuid, "payload": .igbinary], for: scope("public", connectionId: conn)) + + storage.renameTable( + from: scope("public", connectionId: conn), + to: scope("public", connectionId: conn, table: "purchases") + ) + + #expect(storage.load(for: scope("public", connectionId: conn)) == nil) + #expect(storage.load(for: scope("public", connectionId: conn, table: "purchases")) == ["id": .uuid, "payload": .igbinary]) + } + + @Test("A schema file goes once no saved format names it") + func schemaFilesFollowTheFormatsThatNameThem() throws { + let defaults = try pool.make().defaults + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + let earlier = AvroSchemaStore(directory: directory) + let kept = try earlier.add(#"{"type":"array","items":"int"}"#) + let orphan = try earlier.add(#"{"type":"array","items":"long"}"#) + let schemas = AvroSchemaStore(directory: directory) + let storage = ValueDisplayFormatStorage(defaults: defaults, schemas: schemas) + let target = scope("public", connectionId: UUID()) + + storage.save(["payload": .avro(schemaID: kept)], for: target) + #expect(schemas.storedIDs == [kept]) + + storage.dropTable(target) + #expect(schemas.storedIDs.isEmpty) + #expect(orphan != kept) + } +} + +// What 0.83.1 decodes a table's formats as: a String enum of the names it shipped. +private enum FormatsIn083: String, Codable { + case raw, text, uuid, uuidSwappedTime, unixTimestamp, unixTimestampMillis, json, phpSerialized, messagePack, protobuf } diff --git a/apple/TableProTests/Views/Results/CellInteractionResolverTests.swift b/apple/TableProTests/Views/Results/CellInteractionResolverTests.swift index 2629294ba0..2475fe52f5 100644 --- a/apple/TableProTests/Views/Results/CellInteractionResolverTests.swift +++ b/apple/TableProTests/Views/Results/CellInteractionResolverTests.swift @@ -238,6 +238,8 @@ struct CellInteractionResolverBinaryTests { arguments: [ (ValueDisplayFormat.messagePack, BinaryValueFormat.messagePack), (.protobuf, .protobuf), + (.igbinary, .igbinary), + (.avro(schemaID: String(repeating: "c", count: 32)), .avro(schemaID: String(repeating: "c", count: 32))), ] ) func decoderFormatOpensDecodedViewer(format: ValueDisplayFormat, expected: BinaryValueFormat) { diff --git a/apple/TableProTests/Views/Shared/FieldEditorResolverTests.swift b/apple/TableProTests/Views/Shared/FieldEditorResolverTests.swift index 2bce527453..e42d85d1df 100644 --- a/apple/TableProTests/Views/Shared/FieldEditorResolverTests.swift +++ b/apple/TableProTests/Views/Shared/FieldEditorResolverTests.swift @@ -8,7 +8,7 @@ import Foundation @testable import TablePro import Testing -/// What a binary cell is handed to the inspector as: one character per stored byte. +// What a binary cell is handed to the inspector as: one character per stored byte. private func binaryText(hex: String) -> String { var bytes: [UInt8] = [] var rest = Substring(hex) @@ -23,9 +23,9 @@ private enum GeometrySample { static let wkbPoint = "0101000000000000000000F03F0000000000000040" static let ewkbPoint = "0101000020E6100000000000000000F03F0000000000000040" static let geoJsonPoint = #"{"type":"Point","coordinates":[-122.4194,37.7749]}"# - /// A GeoPackage blob: the `GP` header and SRS id ahead of the same WKB point. + // A GeoPackage blob: the `GP` header and SRS id ahead of the same WKB point. static let geoPackagePoint = "47500001E6100000" + wkbPoint - /// A SpatiaLite blob: start byte, byte order, SRID, bounding box, then the point and an end byte. + // A SpatiaLite blob: start byte, byte order, SRID, bounding box, then the point and an end byte. static let spatiaLitePoint = "0001E6100000" + "000000000000F03F0000000000000040000000000000F03F0000000000000040" + "7C01000000000000000000F03F0000000000000040FE" @@ -65,14 +65,21 @@ struct FieldEditorResolverTests { @Test("A binary decoder format on a binary column opens the decoded viewer") func binaryDecoderFormat() { - let kind = FieldEditorResolver.resolve( - for: .blob(rawType: "BLOB"), - isLongText: false, - originalValue: "\u{82}", - displayFormatOverride: .protobuf, - isBinaryValue: true - ) - #expect(kind == .binaryValue(.protobuf)) + let schemaID = String(repeating: "b", count: 32) + let formats: [(ValueDisplayFormat, BinaryValueFormat)] = [ + (.protobuf, .protobuf), (.messagePack, .messagePack), (.igbinary, .igbinary), + (.avro(schemaID: schemaID), .avro(schemaID: schemaID)), + ] + for (format, expected) in formats { + let kind = FieldEditorResolver.resolve( + for: .blob(rawType: "BLOB"), + isLongText: false, + originalValue: "\u{82}", + displayFormatOverride: format, + isBinaryValue: true + ) + #expect(kind == .binaryValue(expected), "\(format)") + } let null = FieldEditorResolver.resolve( for: .blob(rawType: "BLOB"), isLongText: false, @@ -295,15 +302,15 @@ struct FieldEditorResolverTests { } } - /// PostgreSQL hands the raw EWKB hex over as text when its `ST_AsEWKT` rewrite fails. + // PostgreSQL hands the raw EWKB hex over as text when its `ST_AsEWKT` rewrite fails. @Test("EWKB hex text in a spatial column resolves to the geometry field") func spatialEwkbHexTextResolvesToGeometry() { #expect(resolveSpatial(GeometrySample.ewkbPoint) == geometry(.multiLine, .spatialColumn)) #expect(resolveSpatial(GeometrySample.wkbPoint) == geometry(.multiLine, .spatialColumn)) } - /// The JSON rule used to answer first, so a spatial column holding GeoJSON was a JSON field - /// and nothing downstream knew it was a geometry. + // The JSON rule used to answer first, so a spatial column holding GeoJSON was a JSON field + // and nothing downstream knew it was a geometry. @Test("JSON-shaped geometry in a spatial column is a geometry field over the JSON editor") func spatialJsonShapedValueKeepsTheJsonEditor() { let values = [ @@ -329,7 +336,7 @@ struct FieldEditorResolverTests { } } - /// SQL Server, Oracle and Teradata send text no reader takes, and MySQL falls back to `0x` hex. + // SQL Server, Oracle and Teradata send text no reader takes, and MySQL falls back to `0x` hex. @Test("Spatial text no reader takes keeps the editor it had") func unreadableSpatialTextKeepsItsEditor() { #expect(resolveSpatial("0xE61000000101000000000000000000F03F0000000000000040") == .singleLine) @@ -350,8 +357,8 @@ struct FieldEditorResolverTests { } } - /// WKB type 8 is CIRCULARSTRING and 10 is CURVEPOLYGON. The reader names the type from the - /// header, so the Map segment can say which one it cannot draw, as it does for text. + // WKB type 8 is CIRCULARSTRING and 10 is CURVEPOLYGON. The reader names the type from the + // header, so the Map segment can say which one it cannot draw, as it does for text. @Test("WKB of a type the map cannot draw is still a geometry field over the hex editor") func unsupportedBinaryTypeIsStillGeometry() { let zero = "0000000000000000" @@ -370,8 +377,8 @@ struct FieldEditorResolverTests { } } - /// The field was a text field over the Latin-1 spelling of the bytes, where one keystroke - /// staged that string over the blob. + // The field was a text field over the Latin-1 spelling of the bytes, where one keystroke + // staged that string over the blob. @Test("Bytes in a spatial column that are not WKB get the hex editor, never a text field") func unreadableBinarySpatialIsHex() { let blobs = [ @@ -388,8 +395,8 @@ struct FieldEditorResolverTests { #expect(resolveSpatial(nil, isBinaryValue: true) == .blobHex) } - /// The text readers never see a blob: its bytes can spell WKT, and the hex of a short one is a - /// valid geohash. + // The text readers never see a blob: its bytes can spell WKT, and the hex of a short one is a + // valid geohash. @Test("A binary cell is read as bytes, not as the text its bytes spell") func binarySpatialIsNotReadAsText() { #expect(resolveSpatial("POINT(1 2)", isBinaryValue: true) == .blobHex) @@ -421,8 +428,8 @@ struct FieldEditorResolverTests { } } - /// The sniffing reader takes each of these for a point. A JSON column is held to GeoJSON, or - /// every pair of numbers and every document with `lat` and `lon` would open on a map. + // The sniffing reader takes each of these for a point. A JSON column is held to GeoJSON, or + // every pair of numbers and every document with `lat` and `lon` would open on a map. @Test("JSON that is not GeoJSON stays a JSON field") func jsonThatIsNotGeoJsonStaysJson() { let values = [ @@ -487,8 +494,8 @@ struct FieldEditorResolverTests { // MARK: - Large geometry values - /// No reader accepts this after any opening, so a large value that still resolves to a - /// geometry field was classified by how it opens, not read in full. + // No reader accepts this after any opening, so a large value that still resolves to a + // geometry field was classified by how it opens, not read in full. private static let unreadableTail = String(repeating: "x", count: 2_000_000) @Test("A large spatial value is classified by how it opens") @@ -598,8 +605,8 @@ struct FieldEditorResolverImageTests { #expect(kind == .image(.svg)) } - /// The image branch runs before the blob branch, or a PNG in a BLOB column would only ever be - /// a hex dump in the inspector while the grid popover drew it. + // The image branch runs before the blob branch, or a PNG in a BLOB column would only ever be + // a hex dump in the inspector while the grid popover drew it. @Test("PNG bytes in a blob column resolve to the image editor") func pngBlobResolvesToImage() throws { let value = try #require(String(data: encodedPng(), encoding: .isoLatin1)) @@ -670,11 +677,11 @@ struct FieldEditorResolverImageTests { #expect(kind == .arrayElements(element: .scalar, values: [])) } - /// `jsonb[]` and `jsonb[][]` are one type in PostgreSQL's catalog and any array column may - /// carry a dimension prefix, so the declared type cannot rule either out on a given row. The - /// text editor over the raw literal is the lossless fallback, as it is in the grid, and which - /// text editor is the value's own length talking: both literals here sit under - /// `multiLineValueThreshold`. + // `jsonb[]` and `jsonb[][]` are one type in PostgreSQL's catalog and any array column may + // carry a dimension prefix, so the declared type cannot rule either out on a given row. The + // text editor over the raw literal is the lossless fallback, as it is in the grid, and which + // text editor is the value's own length talking: both literals here sit under + // `multiLineValueThreshold`. @Test("A literal the element list cannot represent falls back to the text editor") func unrepresentableArrayFallsBackToText() { #expect( @@ -693,8 +700,8 @@ struct FieldEditorResolverImageTests { ) } - /// An engine whose list literal is not PostgreSQL's reaches the same gate and fails it, so the - /// classifier's engine-blind `[]` rule cannot hand another driver's value to this editor. + // An engine whose list literal is not PostgreSQL's reaches the same gate and fails it, so the + // classifier's engine-blind `[]` rule cannot hand another driver's value to this editor. @Test("A list literal from another engine does not reach the element editor") func foreignListLiteralFallsBackToText() { let kind = FieldEditorResolver.resolve( @@ -705,9 +712,9 @@ struct FieldEditorResolverImageTests { #expect(kind == .singleLine) } - /// The parse is also what scopes the editor to the engines whose arrays are written this way: - /// the classifier's `[]` rule takes no engine, and a document store's `object[]` classifies the - /// same, so a field with no literal to read must not be offered a list that commits `{…}`. + // The parse is also what scopes the editor to the engines whose arrays are written this way: + // the classifier's `[]` rule takes no engine, and a document store's `object[]` classifies the + // same, so a field with no literal to read must not be offered a list that commits `{…}`. @Test("An array column with no value keeps the plain editor") func absentArrayValueKeepsPlainEditor() { #expect( @@ -720,8 +727,8 @@ struct FieldEditorResolverImageTests { ) } - /// The labels arrive separately in `TableRows.columnEnumValues` and are injected into the type - /// before the inspector resolves it, so they have to reach the element inside the array. + // The labels arrive separately in `TableRows.columnEnumValues` and are injected into the type + // before the inspector resolves it, so they have to reach the element inside the array. @Test("An enum array carries its declared labels into the element editor") func enumArrayCarriesItsLabels() { let labels = ["sad", "ok", "happy"] @@ -734,8 +741,8 @@ struct FieldEditorResolverImageTests { ) } - /// An empty array literal is itself valid JSON, so resolving the array before the JSON branch - /// is what keeps `{}` out of the JSON editor. + // An empty array literal is itself valid JSON, so resolving the array before the JSON branch + // is what keeps `{}` out of the JSON editor. @Test("An empty array opens the element editor rather than the JSON editor") func emptyArrayResolvesToElements() { let kind = FieldEditorResolver.resolve( diff --git a/apple/changelog.d/3962.added.md b/apple/changelog.d/3962.added.md new file mode 100644 index 0000000000..95ab228dca --- /dev/null +++ b/apple/changelog.d/3962.added.md @@ -0,0 +1 @@ +Display As decodes igbinary and Avro values in binary columns, Avro with a schema file chosen for the column.