diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift b/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift index cadacdd281..5fb0d05fee 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaConnectionField.swift @@ -88,6 +88,10 @@ enum KafkaConnectionField { static let saslMechanism = "kafkaSaslMechanism" static let brokerRouting = "kafkaBrokerRouting" static let connectTimeout = "kafkaConnectTimeout" + // The app reads these to fetch schemas; the driver never does. + static let schemaRegistryUrl = "schemaRegistryUrl" + static let schemaRegistryUsername = "schemaRegistryUsername" + static let schemaRegistryPassword = "schemaRegistryPassword" enum SecurityProtocol: String, CaseIterable { case plaintext = "PLAINTEXT" @@ -202,6 +206,26 @@ enum KafkaConnectionField { ) ]), section: .advanced + ), + ConnectionField( + id: schemaRegistryUrl, + label: String(localized: "Schema Registry URL"), + placeholder: "http://localhost:8081", + required: false, + section: .authentication + ), + ConnectionField( + id: schemaRegistryUsername, + label: String(localized: "Schema Registry User"), + required: false, + section: .authentication + ), + ConnectionField( + id: schemaRegistryPassword, + label: String(localized: "Schema Registry Password"), + required: false, + fieldType: .secure, + section: .authentication ) ] } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift b/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift index 370504fe12..b3e27c33ff 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaMessageFlattener.swift @@ -95,11 +95,9 @@ enum KafkaMessageFlattener { return .text("{\(pairs.joined(separator: ","))}") } - /// A column is text only if every non-null payload on the page decodes as UTF-8. One - /// binary value is enough to make the whole column binary, because rendering a binary - /// payload through a text column is the lossier of the two mistakes. + // Bytes shown as text is the lossier mistake, and a Schema Registry value's leading zero can sit in valid UTF-8. private static func allDecodeAsText(_ payloads: S) -> Bool where S.Element == Data? { - for case let payload? in payloads where String(data: payload, encoding: .utf8) == nil { + for case let payload? in payloads where payload.first == 0 || String(data: payload, encoding: .utf8) == nil { return false } return true diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index 5e665060df..608496a96f 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -57,16 +57,25 @@ final class DataGridModel: ObservableObject { private(set) weak var mountedView: (any DataGridMountedView)? private var orderRevision = 0 + private let schemaRegistry: SchemaRegistryCache + private var schemaObservers: Set = [] + private var staleSchemaColumns: Set = [] + // Per connection, per data index: what each row a filter kept showed before its registry switched. + private var registrySwitchSnapshots: [UUID: [Int: [RowID: String]]] = [:] + init( store: any DataGridStateStore = InMemoryDataGridStateStore(), configuration: DataGridConfiguration = DataGridConfiguration(), themeScope: ThemeScope = .live, - selectionSummary: SelectionSummaryState? = nil + selectionSummary: SelectionSummaryState? = nil, + schemaRegistry: SchemaRegistryCache = .shared ) { self.store = store self.configuration = configuration self.themeScope = themeScope self.selectionSummary = selectionSummary + self.schemaRegistry = schemaRegistry + observeSchemas() } // MARK: - Rows @@ -89,6 +98,7 @@ final class DataGridModel: ObservableObject { func reloadRows() { let current = rows + prefetchSchemas(in: current, storageRows: nil, retryingFailures: true) store.valueFilter = store.valueFilter.remapped(toColumns: current.columns) let rebuilt = DataGridColumnSet(tableRows: current, layout: store.columnLayout) if rebuilt != columns { @@ -109,6 +119,18 @@ final class DataGridModel: ObservableObject { return } let current = rows + switch delta { + case .rowsInserted(let inserted): + prefetchSchemas(in: current, storageRows: inserted, retryingFailures: false) + case .columnsReplaced: + prefetchSchemas(in: current, storageRows: nil, retryingFailures: false) + case .cellChanged(let row, _): + prefetchSchemas(in: current, storageRows: IndexSet(integer: row), retryingFailures: false) + case .cellsChanged(let positions): + prefetchSchemas(in: current, storageRows: IndexSet(positions.map(\.row)), retryingFailures: false) + case .rowsRemoved, .fullReplace: + break + } if delta.changesRowSet || delta == .columnsReplaced { if delta == .columnsReplaced { columns = DataGridColumnSet(tableRows: current, layout: store.columnLayout) @@ -262,33 +284,221 @@ final class DataGridModel: ObservableObject { let current = rows let previous = displayFormat(for: column) guard previous != format else { return } - let formats = DisplayFormatArray.setting( + store.displayFormats = DisplayFormatArray.setting( format, at: column.dataIndex, in: store.displayFormats, columnCount: current.columns.count ) + // Before the remap reads it: another connection may have loaded the schemas already. + prefetchSchemas(in: current, storageRows: nil, retryingFailures: true) // A filter stores the shown text, so the values the user picked follow the new format. - if let remapped = remappedValueFilter(column: column, from: previous, to: format, in: current) { + if let remapped = remappedValueFilter( + dataIndex: column.dataIndex, name: column.name, type: column.type, from: previous, to: format, in: current + ) { store.valueFilter = remapped } - store.displayFormats = formats - display.cache.removeAll() - if store.valueFilter.isActive { + redrawFormatted([column.dataIndex], in: current) + } + + // Hidden columns too: their cached text and filters outlive the hiding. + private func redrawFormatted(_ dataIndexes: Set, in current: TableRows) { + display.cache.clearValues(forDataIndexes: dataIndexes) + if dataIndexes.contains(where: { store.valueFilter.filter(forColumn: $0) != nil }) { order = resolveOrder(in: current) restoreSelection(in: current) displayRevision &+= 1 mountedView?.apply(.reload) } else { - mountedView?.apply(.rowsUpdated(IndexSet(0.. Bool) { + let wasEmpty = staleSchemaColumns.isEmpty + for (dataIndex, format) in store.displayFormats.enumerated() { + if let format, matches(format) { staleSchemaColumns.insert(dataIndex) } + } + guard wasEmpty, !staleSchemaColumns.isEmpty else { return } + DispatchQueue.main.async { [weak self] in + MainActor.assumeIsolated { self?.redrawStaleSchemaColumns() } + } + } + + private func redrawStaleSchemaColumns() { + let current = rows + followRegistrySwitches(in: current) + let stale = staleSchemaColumns.filter { $0 < current.columns.count } + staleSchemaColumns = [] + guard !stale.isEmpty else { return } + let formats = store.displayFormats + // The format is the same, but the values move from the stored bytes to decoded text, and a filter has to follow. + for dataIndex in stale.sorted() { + let type = dataIndex < current.columnTypes.count ? current.columnTypes[dataIndex] : nil + let format = dataIndex < formats.count ? formats[dataIndex] : nil + if let remapped = remappedValueFilter( + dataIndex: dataIndex, name: current.columns[dataIndex], type: type, from: nil, to: format, in: current + ) { + store.valueFilter = remapped + } + } + redrawFormatted(stale, in: current) + } + + private func prefetchSchemas(in current: TableRows, storageRows: IndexSet?, retryingFailures: Bool) { + let databaseType = configuration.traits.databaseType + var wanted: [UUID: Set] = [:] + for (dataIndex, format) in store.displayFormats.enumerated() { + guard case .schemaRegistry(let connectionID)? = format else { continue } + let type = dataIndex < current.columnTypes.count ? current.columnTypes[dataIndex] : nil + guard format?.isApplicable(to: type, databaseType: databaseType) == true else { continue } + let holdsBytes = BinaryValueBytes.holdsBytes(columnType: type) + var ids = wanted[connectionID, default: []] + func collect(_ row: Row) { + guard dataIndex < row.values.count, + let id = Self.schemaID(of: row.values[dataIndex], holdsBytes: holdsBytes) else { return } + ids.insert(id) + } + if let storageRows { + for index in storageRows where index < current.rows.count { collect(current.rows[index]) } + } else { + for row in current.rows { collect(row) } + } + wanted[connectionID] = ids + } + for (connectionID, ids) in wanted { + if schemaRegistry.prefetch(connectionID: connectionID, schemaIDs: ids, retryingFailures: retryingFailures) { + followRegistrySwitches(in: current) + } + } + } + + private func registryWillSwitch(_ connectionID: UUID) { + let current = rows + let databaseType = configuration.traits.databaseType + for (dataIndex, format) in store.displayFormats.enumerated() where format == .schemaRegistry(connectionID: connectionID) { + guard let filter = store.valueFilter.filter(forColumn: dataIndex) else { continue } + let type = dataIndex < current.columnTypes.count ? current.columnTypes[dataIndex] : nil + var shown: [RowID: String] = [:] + for row in current.rows where dataIndex < row.values.count && row.values[dataIndex] != .null { + let text = GridDisplayOrderResolver.displayValue( + row.values[dataIndex], columnType: type, displayFormat: format, databaseType: databaseType + ) + if filter.selectedValues.contains(text) { shown[row.id] = text } + } + if !shown.isEmpty { registrySwitchSnapshots[connectionID, default: [:]][dataIndex] = shown } + } + } + + // A switched registry reads the same bytes differently, so the rows a filter kept stay kept under their new text. + private func followRegistrySwitches(in current: TableRows) { + guard !registrySwitchSnapshots.isEmpty else { return } + let snapshots = registrySwitchSnapshots + registrySwitchSnapshots = [:] + let databaseType = configuration.traits.databaseType + let rowsByID = Dictionary(current.rows.map { ($0.id, $0) }) { first, _ in first } + for (dataIndex, shown) in snapshots.values.flatMap({ $0 }) { + guard let existing = store.valueFilter.filter(forColumn: dataIndex), dataIndex < current.columns.count else { continue } + let type = dataIndex < current.columnTypes.count ? current.columnTypes[dataIndex] : nil + let format = displayFormatAt(dataIndex) + var removed = Set() + var added = Set() + for (rowID, old) in shown { + guard let row = rowsByID[rowID], dataIndex < row.values.count else { continue } + let new = GridDisplayOrderResolver.displayValue( + row.values[dataIndex], columnType: type, displayFormat: format, databaseType: databaseType + ) + if new != old { + removed.insert(old) + added.insert(new) + } + } + var selected = existing.selectedValues + selected.subtract(removed) + selected.formUnion(added) + guard selected != existing.selectedValues else { continue } + var filter = store.valueFilter + filter.set( + ColumnValueFilter(selectedValues: selected, includesNull: existing.includesNull), + columnName: current.columns[dataIndex], + forColumn: dataIndex + ) + store.valueFilter = filter + } + } + + private func displayFormatAt(_ dataIndex: Int) -> ValueDisplayFormat? { + let formats = store.displayFormats + return dataIndex < formats.count ? formats[dataIndex] : nil + } + + private static func schemaID(of value: PluginCellValue, holdsBytes: Bool) -> UInt32? { + switch value { + case .null: + return nil + case .bytes(let data): + return SchemaRegistryFrame.schemaID(prefix: data) + case .text(let text): + let head = String(String.UnicodeScalarView(text.unicodeScalars.prefix(SchemaRegistryFrame.headerLength))) + return BinaryValueBytes.data(of: head, holdsBytes: holdsBytes).flatMap { SchemaRegistryFrame.schemaID(prefix: $0) } + } + } + private func remappedValueFilter( - column: DataGridColumn, + dataIndex: Int, + name: String, + type: ColumnType?, from oldFormat: ValueDisplayFormat?, to newFormat: ValueDisplayFormat?, in rows: TableRows ) -> GridValueFilterState? { - let dataIndex = column.dataIndex guard let existing = store.valueFilter.filter(forColumn: dataIndex) else { return nil } let databaseType = configuration.traits.databaseType var removed = Set() @@ -297,11 +507,11 @@ final class DataGridModel: ObservableObject { let raw = row.values[dataIndex] guard raw != .null else { continue } let old = GridDisplayOrderResolver.displayValue( - raw, columnType: column.type, displayFormat: oldFormat, databaseType: databaseType + raw, columnType: type, displayFormat: oldFormat, databaseType: databaseType ) guard existing.selectedValues.contains(old) else { continue } let new = GridDisplayOrderResolver.displayValue( - raw, columnType: column.type, displayFormat: newFormat, databaseType: databaseType + raw, columnType: type, displayFormat: newFormat, databaseType: databaseType ) if old != new { removed.insert(old) @@ -315,7 +525,7 @@ final class DataGridModel: ObservableObject { var filter = store.valueFilter filter.set( ColumnValueFilter(selectedValues: selected, includesNull: existing.includesNull), - columnName: column.name, + columnName: name, forColumn: dataIndex ) return filter diff --git a/apple/TablePro/Core/DataGrid/RowDisplayBox.swift b/apple/TablePro/Core/DataGrid/RowDisplayBox.swift index 99f39e1bd2..209b20d4f5 100644 --- a/apple/TablePro/Core/DataGrid/RowDisplayBox.swift +++ b/apple/TablePro/Core/DataGrid/RowDisplayBox.swift @@ -94,6 +94,20 @@ final class RowDisplayCache { storage[id] = Entry(box: existing.box, cost: 0) } + // For a display format change: every other column's text is still right. + func clearValues(forDataIndexes dataIndexes: Set) { + guard !dataIndexes.isEmpty else { return } + highlights.removeAll(keepingCapacity: true) + storage = storage.mapValues { entry in + for index in dataIndexes where index < entry.box.values.count { + entry.box.values[index] = nil + } + let cost = Self.rowCost(entry.box.values) + totalCost += cost - entry.cost + return Entry(box: entry.box, cost: cost) + } + } + private func evictIfNeeded() { while storage.count > countLimit || totalCost > costLimit { guard insertionHead < insertionOrder.count else { break } diff --git a/apple/TablePro/Core/MCP/Outside/MCPRemoteServerTransport.swift b/apple/TablePro/Core/MCP/Outside/MCPRemoteServerTransport.swift index e180d8d471..ccfe17c629 100644 --- a/apple/TablePro/Core/MCP/Outside/MCPRemoteServerTransport.swift +++ b/apple/TablePro/Core/MCP/Outside/MCPRemoteServerTransport.swift @@ -30,7 +30,7 @@ internal actor MCPRemoteServerTransport { private let endpoint: URL private let bearerToken: String private let urlSession: URLSession - private let redirectGuard: MCPRedirectGuard + private let redirectGuard: SameOriginRedirectGuard private let timeout: Duration private var negotiatedSessionId: String? @@ -48,7 +48,7 @@ internal actor MCPRemoteServerTransport { self.endpoint = endpoint self.bearerToken = bearerToken self.timeout = timeout - self.redirectGuard = MCPRedirectGuard(origin: endpoint) + self.redirectGuard = SameOriginRedirectGuard(origin: endpoint) if let urlSession { self.urlSession = urlSession } else { @@ -276,40 +276,3 @@ internal actor MCPRemoteServerTransport { String(localized: "The connection to the server was closed.") } } - -/// Refuses to follow a redirect off the endpoint's own origin. -/// -/// `URLSession` follows redirects on its own and carries the request's headers to wherever it lands. -/// The header here is `Authorization: Bearer `, so a server that answers `302` pointing at -/// another host hands that host the credential, and the user never sees a request they did not make. -/// A same-origin redirect is ordinary routing and is followed; anything else is answered with the -/// redirect response itself, which surfaces as an HTTP failure naming the status. -private final class MCPRedirectGuard: NSObject, URLSessionTaskDelegate, @unchecked Sendable { - private let scheme: String? - private let host: String? - private let port: Int? - - init(origin: URL) { - scheme = origin.scheme?.lowercased() - host = origin.host?.lowercased() - port = origin.port - } - - func urlSession( - _ session: URLSession, - task: URLSessionTask, - willPerformHTTPRedirection response: HTTPURLResponse, - newRequest request: URLRequest, - completionHandler: @escaping (URLRequest?) -> Void - ) { - guard let target = request.url, - target.scheme?.lowercased() == scheme, - target.host?.lowercased() == host, - target.port == port - else { - completionHandler(nil) - return - } - completionHandler(request) - } -} diff --git a/apple/TablePro/Core/Network/SameOriginRedirectGuard.swift b/apple/TablePro/Core/Network/SameOriginRedirectGuard.swift new file mode 100644 index 0000000000..3eeadf2f31 --- /dev/null +++ b/apple/TablePro/Core/Network/SameOriginRedirectGuard.swift @@ -0,0 +1,56 @@ +// +// SameOriginRedirectGuard.swift +// TablePro +// + +import Foundation + +// URLSession carries a request's Authorization header to wherever a redirect points, so only same-origin redirects are followed. +internal final class SameOriginRedirectGuard: NSObject, URLSessionTaskDelegate, Sendable { + private let origin: Origin? + + init(origin: URL) { + self.origin = Origin(origin) + } + + func allows(_ target: URL) -> Bool { + guard let origin, let destination = Origin(target) else { return false } + return origin == destination + } + + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping @Sendable (URLRequest?) -> Void + ) { + guard let target = request.url, allows(target) else { + completionHandler(nil) + return + } + completionHandler(request) + } + + private struct Origin: Equatable, Sendable { + let scheme: String + let host: String + let port: Int + + init?(_ url: URL) { + guard let scheme = url.scheme?.lowercased(), let host = url.host?.lowercased(), !host.isEmpty else { return nil } + guard let port = url.port ?? Self.defaultPort(scheme) else { return nil } + self.scheme = scheme + self.host = host + self.port = port + } + + private static func defaultPort(_ scheme: String) -> Int? { + switch scheme { + case "http": return 80 + case "https": return 443 + default: return nil + } + } + } +} diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift index aa8ae62501..3ed9a9ab7c 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift @@ -141,6 +141,26 @@ private func kafkaConnectionFields() -> [ConnectionField] { .init(value: "bootstrapOnly", label: String(localized: "Only use the bootstrap address")) ]), section: .advanced + ), + ConnectionField( + id: SchemaRegistryConnectionField.url, + label: String(localized: "Schema Registry URL"), + placeholder: "http://localhost:8081", + required: false, + section: .authentication + ), + ConnectionField( + id: SchemaRegistryConnectionField.username, + label: String(localized: "Schema Registry User"), + required: false, + section: .authentication + ), + ConnectionField( + id: SchemaRegistryConnectionField.password, + label: String(localized: "Schema Registry Password"), + required: false, + fieldType: .secure, + section: .authentication ) ] } diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift index bb72c1db7e..99d29cb84a 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -689,7 +689,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { let parameterStyle = driverType.parameterStyle let schemes = driverType.urlSchemes let primaryScheme = schemes.first ?? driverType.databaseTypeId.lowercased() - let additionalConnectionFields = driverType.additionalConnectionFields.filter { + let pluginFields = driverType.additionalConnectionFields.filter { // Kafka releases before the shared timeout setting exposed a second control under // this id. An installed old bundle still registers its metadata over the curated // snapshot, so retire the field here too while its saved value migrates in the model. @@ -702,6 +702,13 @@ final class PluginMetadataRegistry: @unchecked Sendable { // the property crash with EXC_BAD_INSTRUCTION (missing witness table entry). let existingSnapshot = snapshot(forRegisteredTypeId: driverType.databaseTypeId) + // The app reads the Schema Registry fields itself, so an older bundle keeps the curated ones. + let declared = Set(pluginFields.map(\.id)) + let additionalConnectionFields = pluginFields + + (existingSnapshot?.connection.additionalConnectionFields ?? []).filter { + SchemaRegistryConnectionField.all.contains($0.id) && !declared.contains($0.id) + } + return PluginMetadataSnapshot( displayName: driverType.databaseDisplayName, iconName: driverType.iconName, diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift index fee14447d7..4be954eed8 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroDecoder.swift @@ -61,6 +61,17 @@ internal enum AvroDecoder { } } + // The payload behind a Schema Registry prefix the caller has read, so a datum that starts with zero stays a datum. + static func decodeRegistryPayload( + _ body: Data, + schema: AvroSchema, + maxValues: Int = .max, + maxValuesRead: Int = AvroDecoder.maxValuesRead + ) throws -> MessagePackValue { + if let raw = rawBytes(Data(body), schema: schema) { return raw } + return try datum(body, schema: schema, limits: (maxValues, maxValuesRead)) + } + private static func rawBytes(_ body: Data, schema: AvroSchema) -> MessagePackValue? { switch schema.resolve(schema.root) { case .bytes: diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift index 583dc7fc02..5c3b15ddb6 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/AvroSchema.swift @@ -58,15 +58,13 @@ internal struct AvroSchema: Equatable, Sendable { // Named types whose values take no bytes, worked out once: walking a shared type on every check is exponential. let zeroByteNames: Set - static func parse(_ text: String) throws -> AvroSchema { - let json: Any - do { - json = try JSONSerialization.jsonObject(with: Data(text.utf8), options: [.fragmentsAllowed]) - } catch { - throw AvroSchemaError(reason: String(localized: "The schema is not valid JSON.")) - } + // A Schema Registry schema can name types another subject defines; `references` are those, each before its users. + static func parse(_ text: String, references: [String] = []) throws -> AvroSchema { var parser = Parser() - let root = try parser.type(json, namespace: nil) + for reference in references { + _ = try parser.type(try json(reference), namespace: nil) + } + let root = try parser.type(try json(text), namespace: nil) return AvroSchema( root: root, namedTypes: parser.namedTypes, @@ -75,6 +73,14 @@ internal struct AvroSchema: Equatable, Sendable { ) } + private static func json(_ text: String) throws -> Any { + do { + return try JSONSerialization.jsonObject(with: Data(text.utf8), options: [.fragmentsAllowed]) + } catch { + throw AvroSchemaError(reason: String(localized: "The schema is not valid JSON.")) + } + } + // In the order definitions finish, so every name a type refers to is settled before it, except its own and its // enclosing types', which a value cannot hold without bytes anyway. No recursion: a schema can chain thousands. private static func zeroByteNames(_ completed: [(name: String, type: AvroType)]) -> Set { diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueBytes.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueBytes.swift new file mode 100644 index 0000000000..f5b11b3d03 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueBytes.swift @@ -0,0 +1,17 @@ +// +// BinaryValueBytes.swift +// TablePro +// + +import Foundation + +internal enum BinaryValueBytes { + // Bytes reach a String one Latin-1 character per byte; a text column's value is its own UTF-8. + static func data(of text: String, holdsBytes: Bool) -> Data? { + holdsBytes ? text.data(using: .isoLatin1) : Data(text.utf8) + } + + static func holdsBytes(columnType: ColumnType?, isBinaryValue: Bool = false) -> Bool { + isBinaryValue || columnType?.isBlobType != false + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift index 4bd1fcbc58..e675154fde 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueDecoding.swift @@ -10,13 +10,16 @@ internal enum BinaryValueFormat: Hashable, Sendable { case protobuf case igbinary case avro(schemaID: String) + case protobufSchema(schemaID: String, message: String) + case schemaRegistry(connectionID: UUID) var displayName: String { switch self { case .messagePack: return "MessagePack" - case .protobuf: return "Protobuf" + case .protobuf, .protobufSchema: return "Protobuf" case .igbinary: return "igbinary" case .avro: return "Avro" + case .schemaRegistry: return "Schema Registry" } } } @@ -32,6 +35,34 @@ internal enum BinaryValueDocument: Sendable { case failed(BinaryValueDecodeError) // Nothing to judge the bytes by: the format names a schema this Mac no longer has. case missingSchema + case registryUnavailable(schemaID: UInt32, reason: String) +} + +internal enum SchemaRegistryFrame { + static let headerLength = 5 + + static func schemaID(_ data: Data) throws -> UInt32 { + guard let id = schemaID(prefix: data) else { throw BinaryValueDecodeError(kind: .notFramed, offset: 0) } + return id + } + + static func schemaID>(prefix bytes: Bytes) -> UInt32? { + var iterator = bytes.makeIterator() + guard iterator.next() == 0 else { return nil } + var id: UInt32 = 0 + for _ in 0..<4 { + guard let byte = iterator.next() else { return nil } + id = id << 8 | UInt32(byte) + } + return id + } + + static func jsonText(_ data: Data) throws -> String { + guard let text = String(data: data.dropFirst(headerLength), encoding: .utf8) else { + throw BinaryValueDecodeError(kind: .invalidField, offset: headerLength) + } + return text + } } // Pure: the grid calls it per cell on the main actor, the viewer off it. @@ -63,6 +94,16 @@ internal enum BinaryValueDecoding { data, schema: schema, maxValues: budget, maxValuesRead: AvroDecoder.maxValuesReadInCell ).value text = MessagePackTextWriter.json(value, pretty: false, limit: limit) + case .protobufSchema(let schemaID, let message): + guard let schema = ProtobufSchemaStore.shared.schema(id: schemaID) else { return nil } + let decoded = try ProtobufSchemaDecoder.decode(data, schema: schema, message: message, maxValues: budget) + text = ProtobufSchemaTextWriter.text(decoded.fields, multiLine: false, limit: limit) + case .schemaRegistry(let connectionID): + let schemaID = try SchemaRegistryFrame.schemaID(data) + guard case .success(let schema)? = SchemaRegistryCache.shared.cachedSchema( + connectionID: connectionID, schemaID: schemaID + ) else { return nil } + text = try registryText(data, schema: schema, maxValues: budget, limit: limit) } } catch { return nil @@ -128,6 +169,21 @@ internal enum BinaryValueDecoding { ), note: message.framing.map(framingNote) ) + case .protobufSchema(let schemaID, let message): + guard let schema = ProtobufSchemaStore.shared.schema(id: schemaID) else { return .missingSchema } + return protobufDocument(try ProtobufSchemaDecoder.decode(data, schema: schema, message: message)) + case .schemaRegistry(let connectionID): + let schemaID = try SchemaRegistryFrame.schemaID(data) + switch SchemaRegistryCache.shared.cachedSchema(connectionID: connectionID, schemaID: schemaID) { + case .success(let schema)?: + return try registryDocument(data, schema: schema) + case .failure(let error)?: + return .registryUnavailable(schemaID: schemaID, reason: error.reason) + case nil: + return .registryUnavailable( + schemaID: schemaID, reason: String(localized: "The schema is still being fetched.") + ) + } } } catch let error as BinaryValueDecodeError { return .failed(error) @@ -136,6 +192,61 @@ internal enum BinaryValueDecoding { } } + static func registryText(_ data: Data, schema: RegistrySchema, maxValues: Int, limit: Int) throws -> String { + switch schema { + case .avro(let avro): + let value = try AvroDecoder.decodeRegistryPayload( + data.dropFirst(SchemaRegistryFrame.headerLength), schema: avro, + maxValues: maxValues, maxValuesRead: AvroDecoder.maxValuesReadInCell + ) + return MessagePackTextWriter.json(value, pretty: false, limit: limit) + case .protobuf(let protobuf): + let message = try ProtobufSchemaDecoder.decode(data, schema: protobuf, message: nil, maxValues: maxValues) + return ProtobufSchemaTextWriter.text(message.fields, multiLine: false, limit: limit) + case .json: + return try SchemaRegistryFrame.jsonText(data) + } + } + + static func registryDocument(_ data: Data, schema: RegistrySchema) throws -> BinaryValueDocument { + switch schema { + case .avro(let avro): + let schemaID = try SchemaRegistryFrame.schemaID(data) + let value = try AvroDecoder.decodeRegistryPayload(data.dropFirst(SchemaRegistryFrame.headerLength), schema: avro) + return .decoded( + tree: .json(MessagePackTreeBuilder.build(from: value)), + text: capped( + MessagePackTextWriter.json(value, pretty: true, limit: writerLimit(forCap: maxViewerTextLength)), + at: maxViewerTextLength + ), + note: framingNote(.confluent(schemaID: schemaID)) + ) + case .protobuf(let protobuf): + return protobufDocument(try ProtobufSchemaDecoder.decode(data, schema: protobuf, message: nil)) + case .json: + let text = try SchemaRegistryFrame.jsonText(data) + guard case .success(let tree) = JSONTreeParser.parse(text) else { + throw BinaryValueDecodeError(kind: .invalidField, offset: SchemaRegistryFrame.headerLength) + } + return .decoded( + tree: .json(tree), + text: capped(text, at: maxViewerTextLength), + note: framingNote(.confluent(schemaID: try SchemaRegistryFrame.schemaID(data))) + ) + } + } + + private static func protobufDocument(_ message: ProtobufTypedMessage) -> BinaryValueDocument { + .decoded( + tree: .protobuf(ProtobufSchemaTreeBuilder.build(from: message)), + text: capped( + ProtobufSchemaTextWriter.text(message.fields, multiLine: true, limit: writerLimit(forCap: maxViewerTextLength)), + at: maxViewerTextLength + ), + note: message.framing.map(framingNote) + ) + } + private static func framingNote(_ framing: AvroFraming) -> String { switch framing { case .confluent(let schemaID): diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift index edcb95f1c1..8337d63b40 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/BinaryValueReader.swift @@ -20,6 +20,8 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { case indexOutOfRange(Int) case invalidLength case otherSchema(fingerprint: UInt64) + case notFramed + case invalidUTF8 } let kind: Kind @@ -59,6 +61,10 @@ internal struct BinaryValueDecodeError: Error, Equatable, Sendable { return String(format: String(localized: "Byte %lld picks entry %lld, which the schema does not have."), offset, index) case .invalidLength: return String(format: String(localized: "The length at byte %lld is negative."), offset) + case .notFramed: + return String(localized: "The value does not start with the Schema Registry prefix: a zero byte and a schema ID.") + case .invalidUTF8: + return String(format: String(localized: "The text at byte %lld is not valid UTF-8."), offset) case .otherSchema(let fingerprint): return String( format: String(localized: "The value was written with the schema whose fingerprint is %@, not this column's."), diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift new file mode 100644 index 0000000000..8ed18de9a8 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtoFileParser.swift @@ -0,0 +1,467 @@ +// +// ProtoFileParser.swift +// TablePro +// + +import Foundation + +// https://protobuf.dev/reference/protobuf/proto3-spec/; only options that change how a value reads are kept. +internal enum ProtoFileParser { + static let maxTokens = 2_000_000 + static let maxDepth = 100 + + static func parse(_ text: String) throws -> [ProtobufFileDefinition] { + var parser = Parser(tokens: try Tokenizer.tokens(text)) + return [try parser.file()] + } + + fileprivate enum Token: Equatable { + case identifier(String) + case number(String) + case string(String) + case symbol(Character) + } + + private enum Tokenizer { + static func tokens(_ text: String) throws -> [Token] { + let scalars = Array(text.unicodeScalars) + var tokens: [Token] = [] + var index = scalars.first == "\u{FEFF}" ? 1 : 0 + while index < scalars.count { + guard tokens.count < ProtoFileParser.maxTokens else { throw error(String(localized: "The file is too large.")) } + let scalar = scalars[index] + if CharacterSet.whitespacesAndNewlines.contains(scalar) { + index += 1 + } else if scalar == "/", index + 1 < scalars.count, scalars[index + 1] == "/" { + while index < scalars.count, scalars[index] != "\n" { index += 1 } + } else if scalar == "/", index + 1 < scalars.count, scalars[index + 1] == "*" { + index += 2 + while index + 1 < scalars.count, !(scalars[index] == "*" && scalars[index + 1] == "/") { index += 1 } + guard index + 1 < scalars.count else { throw error(String(localized: "A comment is never closed.")) } + index += 2 + } else if isIdentifierStart(scalar) + || (scalar == "." && index + 1 < scalars.count && isIdentifierStart(scalars[index + 1])) { + let start = index + while index < scalars.count, isIdentifierPart(scalars[index]) || scalars[index] == "." { index += 1 } + tokens.append(.identifier(String(String.UnicodeScalarView(scalars[start..=;,:-+".unicodeScalars.contains(scalar) { + tokens.append(.symbol(Character(scalar))) + index += 1 + } else { + throw error(String(format: String(localized: "Unexpected character “%@”."), String(Character(scalar)))) + } + } + return tokens + } + + private static func isIdentifierStart(_ scalar: Unicode.Scalar) -> Bool { + scalar == "_" || (scalar.isASCII && CharacterSet.letters.contains(scalar)) + } + + private static func isIdentifierPart(_ scalar: Unicode.Scalar) -> Bool { + isIdentifierStart(scalar) || isDigit(scalar) + } + + private static func isDigit(_ scalar: Unicode.Scalar) -> Bool { + ("0"..."9").contains(scalar) + } + } + + fileprivate static func error(_ reason: String) -> ProtobufSchemaError { + ProtobufSchemaError(reason: reason) + } + + private struct Parser { + let tokens: [Token] + var position = 0 + var messages: [ProtobufMessageDefinition] = [] + var enums: [ProtobufEnumDefinition] = [] + + init(tokens: [Token]) { + self.tokens = tokens + } + + mutating func file() throws -> ProtobufFileDefinition { + var file = ProtobufFileDefinition() + let package = packageName() + while let token = peek() { + switch token { + case .identifier("syntax"): + if peek(1) == .symbol("="), peek(2) == .string("proto3") { file.syntax = .proto3 } + try skipStatement() + case .identifier("edition"): + file.syntax = .editions + try skipStatement() + case .identifier("package"): + try skipStatement() + case .identifier("import"): + try skipStatement() + case .identifier("option"): + feature(at: position + 1, into: &file.features) + try skipStatement() + case .identifier("message"): + advance() + file.messages.append(try message(scope: package, depth: 0)) + case .identifier("enum"): + advance() + file.enums.append(try enumeration(scope: package)) + case .identifier("service"), .identifier("extend"): + try skipStatement() + case .symbol(";"): + advance() + default: + throw unexpected() + } + } + file.allMessages = messages + file.allEnums = enums + return file + } + + // It qualifies every definition in the file, those written before it too. + private func packageName() -> String { + var depth = 0 + for index in tokens.indices { + switch tokens[index] { + case .symbol("{"): depth += 1 + case .symbol("}"): depth -= 1 + case .identifier("package") where depth == 0 && index + 1 < tokens.count: + if case .identifier(let name) = tokens[index + 1] { return name } + default: break + } + } + return "" + } + + private mutating func message(scope: String, depth: Int) throws -> String { + let name = try identifier() + let fullName = scope.isEmpty ? name : "\(scope).\(name)" + try expect("{") + try messageBody(fullName: fullName, depth: depth) + return fullName + } + + private mutating func messageBody(fullName: String, depth: Int) throws { + let slot = messages.count + messages.append(ProtobufMessageDefinition(fullName: fullName)) + var definition = messages[slot] + try body(of: &definition, depth: depth, oneof: nil) + messages[slot] = definition + } + + private mutating func body(of definition: inout ProtobufMessageDefinition, depth: Int, oneof: Int?) throws { + guard depth < ProtoFileParser.maxDepth else { throw ProtoFileParser.error(String(localized: "Messages are nested too deeply.")) } + while true { + guard let token = peek() else { throw ProtoFileParser.error(String(localized: "A message is never closed.")) } + switch token { + case .symbol("}"): + advance() + return + case .symbol(";"): + advance() + case .identifier("message"): + advance() + definition.nestedMessages.append(try message(scope: definition.fullName, depth: depth + 1)) + case .identifier("enum"): + advance() + definition.nestedEnums.append(try enumeration(scope: definition.fullName)) + case .identifier("oneof"): + advance() + _ = try identifier() + try expect("{") + let index = (definition.fields.compactMap(\.oneofIndex).max() ?? -1) + 1 + try body(of: &definition, depth: depth + 1, oneof: index) + // A Schema Registry stores a map field as the entry message protoc synthesizes for it. + case .identifier("option") where peek(1) == .identifier("map_entry") && peek(3) == .identifier("true"): + definition.isMapEntry = true + try skipStatement() + case .identifier("option"): + if oneof == nil { feature(at: position + 1, into: &definition.features) } + try skipStatement() + case .identifier("reserved"), .identifier("extensions"), .identifier("extend"): + try skipStatement() + case .identifier("map") where peek(1) == .symbol("<"): + advance() + try mapField(into: &definition) + default: + try field(into: &definition, depth: depth, oneof: oneof) + } + } + } + + private mutating func field(into definition: inout ProtobufMessageDefinition, depth: Int, oneof: Int?) throws { + var label: String? + // `optional foo = 1;` is a field of a message named `optional`: a label needs a type and a name after it. + if case .identifier(let word)? = peek(), ["optional", "required", "repeated"].contains(word), + peek(1).map(Self.isIdentifier) == true, peek(2).map(Self.isIdentifier) == true, peek(3) == .symbol("=") { + label = word + advance() + } + let typeName = try identifier() + let name = try identifier() + try expect("=") + let number = try fieldNumber() + let features = try fieldOptions() + let type: ProtobufFieldDefinition.TypeReference + if typeName == "group" { + try expect("{") + let fullName = "\(definition.fullName).\(name)" + try messageBody(fullName: fullName, depth: depth + 1) + definition.nestedMessages.append(fullName) + type = .group(fullName) + } else { + try expect(";") + type = ProtobufScalarType(rawValue: typeName).map { .scalar($0) } ?? .named(typeName) + } + var field = ProtobufFieldDefinition( + name: typeName == "group" ? name.lowercased() : name, + number: number, + type: type, + isRepeated: label == "repeated" + ) + field.isOptional = label == "optional" + field.oneofIndex = oneof + field.features = features + definition.fields.append(field) + } + + // protoc's synthesized entry: `Entry { key = 1; value = 2; }`, nested where the field is declared. + private mutating func mapField(into definition: inout ProtobufMessageDefinition) throws { + try expect("<") + let keyType = try identifier() + try expect(",") + let valueType = try identifier() + try expect(">") + let name = try identifier() + try expect("=") + let number = try fieldNumber() + _ = try fieldOptions() + try expect(";") + let entryName = "\(definition.fullName).\(Self.camelCased(name))Entry" + func reference(_ type: String) -> ProtobufFieldDefinition.TypeReference { + ProtobufScalarType(rawValue: type).map { .scalar($0) } ?? .named(type) + } + var entry = ProtobufMessageDefinition(fullName: entryName) + entry.fields = [ + ProtobufFieldDefinition(name: "key", number: 1, type: reference(keyType), isRepeated: false), + ProtobufFieldDefinition(name: "value", number: 2, type: reference(valueType), isRepeated: false), + ] + entry.isMapEntry = true + messages.append(entry) + definition.nestedMessages.append(entryName) + // The entry sits in the message's scope, so its value type resolves from there. + definition.fields.append(ProtobufFieldDefinition(name: name, number: number, type: .named(".\(entryName)"), isRepeated: true)) + } + + private mutating func enumeration(scope: String) throws -> String { + let name = try identifier() + let fullName = scope.isEmpty ? name : "\(scope).\(name)" + try expect("{") + var enumeration = ProtobufEnumDefinition(fullName: fullName) + while true { + guard let token = peek() else { throw ProtoFileParser.error(String(localized: "An enum is never closed.")) } + switch token { + case .symbol("}"): + advance() + enums.append(enumeration) + return fullName + case .symbol(";"): + advance() + case .identifier("option"): + feature(at: position + 1, into: &enumeration.features) + try skipStatement() + case .identifier("reserved"): + try skipStatement() + default: + let valueName = try identifier() + try expect("=") + var negative = false + if peek() == .symbol("-") { + negative = true + advance() + } + guard case .number(let digits)? = peek(), let magnitude = Self.integer(digits), + let number = Int32(exactly: negative ? -magnitude : magnitude) else { throw unexpected() } + advance() + try skipOptions() + try expect(";") + if enumeration.values[number] == nil { enumeration.values[number] = valueName } + } + } + } + + private static func isIdentifier(_ token: Token) -> Bool { + if case .identifier = token { return true } + return false + } + + // `features.field_presence = IMPLICIT` or `features = { field_presence: IMPLICIT }`, as a statement or a field option. + private func feature(at index: Int, into overrides: inout ProtobufFeatureOverrides) { + guard index + 2 < tokens.count, case .identifier(let name) = tokens[index], tokens[index + 1] == .symbol("=") else { return } + if name == "features", tokens[index + 2] == .symbol("{") { + var cursor = index + 3 + while cursor + 2 < tokens.count, tokens[cursor] != .symbol("}") { + if case .identifier(let key) = tokens[cursor], tokens[cursor + 1] == .symbol(":"), + case .identifier(let value) = tokens[cursor + 2] { + Self.apply("features.\(key)", value, into: &overrides) + cursor += 3 + } else { + cursor += 1 + } + } + return + } + guard case .identifier(let value) = tokens[index + 2] else { return } + Self.apply(name, value, into: &overrides) + } + + private static func apply(_ name: String, _ value: String, into overrides: inout ProtobufFeatureOverrides) { + switch (name, value) { + case ("features.field_presence", "IMPLICIT"): overrides.implicitPresence = true + case ("features.field_presence", "EXPLICIT"), ("features.field_presence", "LEGACY_REQUIRED"): + overrides.implicitPresence = false + case ("features.enum_type", "CLOSED"): overrides.closedEnum = true + case ("features.enum_type", "OPEN"): overrides.closedEnum = false + case ("features.utf8_validation", "VERIFY"): overrides.validatesUTF8 = true + case ("features.utf8_validation", "NONE"): overrides.validatesUTF8 = false + case ("features.message_encoding", "DELIMITED"): overrides.delimitedMessages = true + case ("features.message_encoding", "LENGTH_PREFIXED"): overrides.delimitedMessages = false + default: break + } + } + + private mutating func fieldOptions() throws -> ProtobufFeatureOverrides { + var overrides = ProtobufFeatureOverrides() + let start = position + try skipOptions() + var nesting = 0 + for index in start.. String { + var result = "" + var upper = true + for character in name { + if character == "_" { + upper = true + } else { + result += upper ? character.uppercased() : String(character) + upper = false + } + } + return result + } + + private static func integer(_ digits: String) -> Int64? { + let lower = digits.lowercased() + if lower.hasPrefix("0x") { return Int64(lower.dropFirst(2), radix: 16) } + if lower.count > 1, lower.hasPrefix("0") { return Int64(lower.dropFirst(), radix: 8) } + return Int64(lower) + } + + private mutating func fieldNumber() throws -> Int { + guard case .number(let digits)? = peek(), let value = Self.integer(digits), + value > 0, value <= Int64(ProtobufWireDecoder.maxFieldNumber) else { throw unexpected() } + advance() + return Int(value) + } + + private mutating func skipOptions() throws { + guard peek() == .symbol("[") else { return } + try skipBalanced(open: "[", close: "]") + } + + private mutating func skipStatement() throws { + while let token = peek() { + switch token { + case .symbol(";"): + advance() + return + case .symbol("{"): + try skipBalanced(open: "{", close: "}") + if peek() == .symbol(";") { advance() } + return + case .symbol("["): + try skipBalanced(open: "[", close: "]") + default: + advance() + } + } + throw ProtoFileParser.error(String(localized: "The file ends in the middle of a statement.")) + } + + private mutating func skipBalanced(open: Character, close: Character) throws { + var depth = 0 + while let token = peek() { + advance() + if token == .symbol(open) { depth += 1 } + if token == .symbol(close) { + depth -= 1 + if depth == 0 { return } + } + } + throw ProtoFileParser.error(String(localized: "The file ends in the middle of a statement.")) + } + + private mutating func identifier() throws -> String { + guard case .identifier(let name)? = peek() else { throw unexpected() } + advance() + return name + } + + private mutating func expect(_ symbol: Character) throws { + guard peek() == .symbol(symbol) else { throw unexpected() } + advance() + } + + private func peek(_ offset: Int = 0) -> Token? { + position + offset < tokens.count ? tokens[position + offset] : nil + } + + private mutating func advance() { + position += 1 + } + + private func unexpected() -> ProtobufSchemaError { + guard let token = peek() else { + return ProtoFileParser.error(String(localized: "The file ends in the middle of a statement.")) + } + let text: String + switch token { + case .identifier(let value), .number(let value): text = value + case .string(let value): text = "\"\(value)\"" + case .symbol(let value): text = String(value) + } + return ProtoFileParser.error(String(format: String(localized: "Unexpected “%@” in the .proto file."), text)) + } + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift index 622a3c6420..0cc87cddcf 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufRendering.swift @@ -24,7 +24,7 @@ internal enum ProtobufTextWriter { } } - private static func write( + static func write( _ fields: [ProtobufField], into output: inout String, multiLine: Bool, @@ -113,6 +113,7 @@ internal enum ProtobufTreeBuilder { )) break } + nodeCount += 1 let name = String(field.number) children.append(node( key: name, @@ -126,7 +127,8 @@ internal enum ProtobufTreeBuilder { return children } - private static func node( + // The caller counts the node itself. + static func node( key: String, keyPath: String, path: TreeNodePath, @@ -134,7 +136,6 @@ internal enum ProtobufTreeBuilder { formats: TreeSummaryFormats, nodeCount: inout Int ) -> ProtobufTreeNode { - nodeCount += 1 switch value { case .message(let members), .group(let members): let children = makeChildren(members, keyPath: keyPath, path: path, formats: formats, nodeCount: &nodeCount) diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift new file mode 100644 index 0000000000..7b493dd14d --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchema.swift @@ -0,0 +1,570 @@ +// +// ProtobufSchema.swift +// TablePro +// + +import Foundation + +internal enum ProtobufScalarType: String, Equatable, Sendable { + case double, float, int64, uint64, int32, fixed64, fixed32, bool, string, bytes, uint32 + case sfixed32, sfixed64, sint32, sint64 +} + +internal enum ProtobufFieldType: Equatable, Sendable { + case scalar(ProtobufScalarType) + case message(String) + case enumeration(String) + case group(String) + // Named by a file the schema does not include, so its values read as they would with no schema. + case unresolved(String) +} + +internal struct ProtobufFieldDescriptor: Equatable, Sendable { + let name: String + let number: Int + let type: ProtobufFieldType + let isRepeated: Bool + // A real oneof only: proto3 `optional` puts a field in a oneof of its own. + let oneofIndex: Int? + // Without presence, a field holding its default is not printed. + let hasPresence: Bool + // protoc refuses the whole value when such a string is not UTF-8. + let validatesUTF8: Bool + // An editions message field sent with group framing. + let isDelimited: Bool +} + +internal struct ProtobufMessageDescriptor: Equatable, Sendable { + let fullName: String + let fields: [ProtobufFieldDescriptor] + // Map entries included: a Schema Registry message index counts them. + let nestedMessages: [String] + let isMapEntry: Bool + private let fieldIndexes: [Int: Int] + + init(fullName: String, fields: [ProtobufFieldDescriptor], nestedMessages: [String], isMapEntry: Bool) { + self.fullName = fullName + self.fields = fields + self.nestedMessages = nestedMessages + self.isMapEntry = isMapEntry + fieldIndexes = Dictionary(fields.enumerated().map { ($0.element.number, $0.offset) }) { first, _ in first } + } + + var shortName: String { + fullName.split(separator: ".").last.map(String.init) ?? fullName + } + + func field(number: Int) -> ProtobufFieldDescriptor? { + fieldIndexes[number].map { fields[$0] } + } +} + +internal struct ProtobufEnumDescriptor: Equatable, Sendable { + let fullName: String + let values: [Int32: String] + // A closed enum keeps a number it does not declare out of the field, with the unknown fields. + let isClosed: Bool + + var shortName: String { + fullName.split(separator: ".").last.map(String.init) ?? fullName + } +} + +internal struct ProtobufSchemaError: Error, Equatable, Sendable { + let reason: String + + static var notASchema: ProtobufSchemaError { + ProtobufSchemaError(reason: String(localized: "The file is neither a .proto file nor a descriptor set.")) + } +} + +// The first file is the one chosen, the primary file of a descriptor set, or the subject a registry returned. +internal struct ProtobufSchema: Equatable, Sendable { + let messages: [String: ProtobufMessageDescriptor] + let enums: [String: ProtobufEnumDescriptor] + // A Schema Registry message index walks these. + let rootMessages: [String] + // The first file's messages first, map entries left out. + let messageNames: [String] + + static let maxMessages = 20_000 + + static func parse(_ data: Data) throws -> ProtobufSchema { + var textError: ProtobufSchemaError? + if let text = String(data: data, encoding: .utf8) { + let files: [ProtobufFileDefinition]? + do { + files = try ProtoFileParser.parse(text) + } catch { + files = nil + textError = error as? ProtobufSchemaError + } + if let files { return try build(files) } + } + do { + return try build(ProtobufDescriptorSetParser.parse(data)) + } catch { + if let textError, looksLikeText(data) { throw textError } + throw (error as? ProtobufSchemaError) ?? .notASchema + } + } + + static func parse(protoFiles texts: [String]) throws -> ProtobufSchema { + try build(texts.flatMap { try ProtoFileParser.parse($0) }) + } + + func messageIndexed(by indexes: [Int]) -> ProtobufMessageDescriptor? { + guard let first = indexes.first, rootMessages.indices.contains(first), + var message = messages[rootMessages[first]] else { return nil } + for index in indexes.dropFirst() { + guard message.nestedMessages.indices.contains(index), + let nested = messages[message.nestedMessages[index]] else { return nil } + message = nested + } + return message + } + + // A descriptor set always holds control bytes; a .proto file holds none but whitespace. + private static func looksLikeText(_ data: Data) -> Bool { + !data.contains { $0 < 0x20 && !(0x09...0x0D).contains($0) } + } + + private static func build(_ files: [ProtobufFileDefinition]) throws -> ProtobufSchema { + var definitions: [String: ProtobufMessageDefinition] = [:] + var enumDefinitions: [String: ProtobufEnumDefinition] = [:] + var order: [String] = [] + for file in files { + for message in file.allMessages { + if definitions[message.fullName] == nil { order.append(message.fullName) } + definitions[message.fullName] = message + } + for enumeration in file.allEnums { + enumDefinitions[enumeration.fullName] = enumeration + } + } + guard definitions.count <= maxMessages else { + throw ProtobufSchemaError(reason: String(localized: "The schema defines more messages than TablePro reads.")) + } + guard !definitions.isEmpty else { + throw ProtobufSchemaError(reason: String(localized: "The file defines no message.")) + } + let scopes = ProtobufFeatureScopes(files: files, messages: definitions, enums: enumDefinitions) + let resolver = ProtobufTypeResolver( + messageNames: Set(definitions.keys), + enumNames: Set(enumDefinitions.keys), + mapEntryNames: Set(definitions.values.filter(\.isMapEntry).map(\.fullName)) + ) + var resolved: [String: ProtobufMessageDescriptor] = [:] + for (name, message) in definitions { + let features = scopes.messages[name] ?? ProtobufFeatures(syntax: .proto2) + resolved[name] = ProtobufMessageDescriptor( + fullName: name, + fields: message.fields.map { + resolver.resolve($0, scope: name, features: features, inMapEntry: message.isMapEntry) + }, + nestedMessages: message.nestedMessages, + isMapEntry: message.isMapEntry + ) + } + var enums: [String: ProtobufEnumDescriptor] = [:] + for (name, enumeration) in enumDefinitions { + enums[name] = ProtobufEnumDescriptor( + fullName: name, values: enumeration.values, isClosed: scopes.closedEnums[name] ?? false + ) + } + return ProtobufSchema( + messages: resolved, + enums: enums, + rootMessages: files.first?.messages ?? [], + messageNames: order.filter { resolved[$0]?.isMapEntry == false } + ) + } +} + +internal enum ProtobufSyntax: Sendable { + case proto2, proto3, editions +} + +// The `features` a scope sets; only an editions file can set them. +internal struct ProtobufFeatureOverrides: Equatable, Sendable { + var implicitPresence: Bool? + var closedEnum: Bool? + var validatesUTF8: Bool? + var delimitedMessages: Bool? +} + +internal struct ProtobufFileDefinition: Sendable { + var name = "" + var dependencies: [String] = [] + var syntax = ProtobufSyntax.proto2 + var features = ProtobufFeatureOverrides() + var messages: [String] = [] + var enums: [String] = [] + var allMessages: [ProtobufMessageDefinition] = [] + var allEnums: [ProtobufEnumDefinition] = [] +} + +internal struct ProtobufMessageDefinition: Sendable { + let fullName: String + var fields: [ProtobufFieldDefinition] = [] + var nestedMessages: [String] = [] + var nestedEnums: [String] = [] + var isMapEntry = false + var features = ProtobufFeatureOverrides() +} + +internal struct ProtobufEnumDefinition: Sendable { + let fullName: String + var values: [Int32: String] = [:] + var features = ProtobufFeatureOverrides() +} + +internal struct ProtobufFieldDefinition: Sendable { + enum TypeReference: Sendable { + case scalar(ProtobufScalarType) + // As written: relative, dotted, or absolute with a leading dot. + case named(String) + case group(String) + } + + let name: String + let number: Int + let type: TypeReference + let isRepeated: Bool + var isOptional = false + var oneofIndex: Int? + var features = ProtobufFeatureOverrides() +} + +private struct ProtobufFeatures { + let syntax: ProtobufSyntax + var implicitPresence: Bool + var closedEnum: Bool + var validatesUTF8: Bool + var delimitedMessages = false + + init(syntax: ProtobufSyntax) { + self.syntax = syntax + implicitPresence = syntax == .proto3 + closedEnum = syntax == .proto2 + validatesUTF8 = syntax != .proto2 + } + + func applying(_ overrides: ProtobufFeatureOverrides?) -> ProtobufFeatures { + guard syntax == .editions, let overrides else { return self } + var result = self + if let implicitPresence = overrides.implicitPresence { result.implicitPresence = implicitPresence } + if let closedEnum = overrides.closedEnum { result.closedEnum = closedEnum } + if let validatesUTF8 = overrides.validatesUTF8 { result.validatesUTF8 = validatesUTF8 } + if let delimitedMessages = overrides.delimitedMessages { result.delimitedMessages = delimitedMessages } + return result + } +} + +// Features inherit from the file through each enclosing message, wherever in the scope the option is written. +private struct ProtobufFeatureScopes { + private(set) var messages: [String: ProtobufFeatures] = [:] + private(set) var closedEnums: [String: Bool] = [:] + + init( + files: [ProtobufFileDefinition], + messages definitions: [String: ProtobufMessageDefinition], + enums enumDefinitions: [String: ProtobufEnumDefinition] + ) { + for file in files { + let base = ProtobufFeatures(syntax: file.syntax).applying(file.features) + var pending = file.messages.map { ($0, base) } + for name in file.enums { + closedEnums[name] = base.applying(enumDefinitions[name]?.features).closedEnum + } + while let (name, inherited) = pending.popLast() { + guard messages[name] == nil, let definition = definitions[name] else { continue } + let features = inherited.applying(definition.features) + messages[name] = features + pending += definition.nestedMessages.map { ($0, features) } + for nested in definition.nestedEnums { + closedEnums[nested] = features.applying(enumDefinitions[nested]?.features).closedEnum + } + } + for message in file.allMessages where messages[message.fullName] == nil { + messages[message.fullName] = base.applying(message.features) + } + for enumeration in file.allEnums where closedEnums[enumeration.fullName] == nil { + closedEnums[enumeration.fullName] = base.applying(enumeration.features).closedEnum + } + } + } +} + +// C++ scoping: a name is looked up from the innermost scope outward, by its first component. +private struct ProtobufTypeResolver { + let messageNames: Set + let enumNames: Set + let mapEntryNames: Set + + func resolve( + _ field: ProtobufFieldDefinition, + scope: String, + features: ProtobufFeatures, + inMapEntry: Bool + ) -> ProtobufFieldDescriptor { + let type: ProtobufFieldType + switch field.type { + case .scalar(let scalar): + type = .scalar(scalar) + case .group(let fullName): + type = .group(fullName) + case .named(let reference): + type = resolve(reference, scope: scope) + } + let fieldFeatures = features.applying(field.features) + let isImplicit = fieldFeatures.implicitPresence && !field.isOptional + let isMessage: Bool + switch type { + case .message, .group: isMessage = true + case .scalar, .enumeration, .unresolved: isMessage = false + } + return ProtobufFieldDescriptor( + name: field.name, + number: field.number, + type: type, + isRepeated: field.isRepeated, + oneofIndex: field.oneofIndex, + hasPresence: !field.isRepeated && (field.oneofIndex != nil || isMessage || !isImplicit), + validatesUTF8: fieldFeatures.validatesUTF8, + isDelimited: !inMapEntry && isDelimited(type, features: fieldFeatures) + ) + } + + // A map's entries, and the messages they hold, stay length-prefixed whatever the file sets. + private func isDelimited(_ type: ProtobufFieldType, features: ProtobufFeatures) -> Bool { + guard features.delimitedMessages, case .message(let name) = type else { return false } + return !mapEntryNames.contains(name) + } + + private func resolve(_ reference: String, scope: String) -> ProtobufFieldType { + if reference.hasPrefix(".") { + return lookup(String(reference.dropFirst())) ?? .unresolved(reference) + } + var components = scope.split(separator: ".").map(String.init) + while true { + let candidate = (components + [reference]).joined(separator: ".") + if let found = lookup(candidate) { return found } + guard !components.isEmpty else { return .unresolved(reference) } + components.removeLast() + } + } + + private func lookup(_ name: String) -> ProtobufFieldType? { + if messageNames.contains(name) { return .message(name) } + if enumNames.contains(name) { return .enumeration(name) } + return nil + } +} + +// descriptor.proto read with the wire reader. Field numbers are descriptor.proto's. +internal enum ProtobufDescriptorSetParser { + static func parse(_ data: Data) throws -> [ProtobufFileDefinition] { + var files: [ProtobufFileDefinition] = [] + do { + for (number, value) in try fields(data) { + guard number == 1, case .bytes(let bytes) = value else { throw ProtobufSchemaError.notASchema } + files.append(try file(bytes)) + } + } catch let error as ProtobufSchemaError { + throw error + } catch { + throw ProtobufSchemaError.notASchema + } + guard files.contains(where: { !$0.allMessages.isEmpty }) else { throw ProtobufSchemaError.notASchema } + return primaryFirst(files) + } + + // `--include_imports` writes each dependency before the files that import it. + private static func primaryFirst(_ files: [ProtobufFileDefinition]) -> [ProtobufFileDefinition] { + let imported = Set(files.flatMap(\.dependencies)) + guard let primary = files.lastIndex(where: { !imported.contains($0.name) }) else { return files } + var ordered = files + let file = ordered.remove(at: primary) + return [file] + ordered + } + + private enum Value { + case varint(UInt64) + case bytes(Data) + } + + private static func fields(_ data: Data) throws -> [(Int, Value)] { + var reader = BinaryValueReader(data) + var result: [(Int, Value)] = [] + while !reader.isAtEnd { + let tag = try reader.readVarint() + let number = Int(truncatingIfNeeded: tag >> 3) + switch tag & 7 { + case 0: + result.append((number, .varint(try reader.readVarint()))) + case 1: + try reader.skip(8) + case 2: + let length = try reader.readVarint() + guard length <= UInt64(reader.remaining) else { throw reader.error(.unexpectedEnd) } + result.append((number, .bytes(try reader.readBytes(Int(length))))) + case 5: + try reader.skip(4) + default: + throw reader.error(.invalidField) + } + } + return result + } + + private static func string(_ value: Value) -> String? { + guard case .bytes(let data) = value else { return nil } + return String(data: data, encoding: .utf8) + } + + private static func varint(_ entries: [(Int, Value)], _ number: Int) -> UInt64? { + for (member, value) in entries.reversed() where member == number { + if case .varint(let raw) = value { return raw } + } + return nil + } + + private static func bytes(_ entries: [(Int, Value)], _ number: Int) -> Data? { + for (member, value) in entries.reversed() where member == number { + if case .bytes(let data) = value { return data } + } + return nil + } + + // FeatureSet: field_presence 1 (2 IMPLICIT), enum_type 2 (2 CLOSED), utf8_validation 4 (3 NONE), message_encoding 5. + private static func features(_ entries: [(Int, Value)], options: Int, features: Int) throws -> ProtobufFeatureOverrides { + guard let optionBytes = bytes(entries, options), + let featureBytes = bytes(try fields(optionBytes), features) else { return ProtobufFeatureOverrides() } + let set = try fields(featureBytes) + var overrides = ProtobufFeatureOverrides() + if let presence = varint(set, 1), (1...3).contains(presence) { overrides.implicitPresence = presence == 2 } + if let enumType = varint(set, 2), (1...2).contains(enumType) { overrides.closedEnum = enumType == 2 } + if let validation = varint(set, 4), (2...3).contains(validation) { overrides.validatesUTF8 = validation == 2 } + if let encoding = varint(set, 5), (1...2).contains(encoding) { overrides.delimitedMessages = encoding == 2 } + return overrides + } + + private static func file(_ data: Data) throws -> ProtobufFileDefinition { + let entries = try fields(data) + var file = ProtobufFileDefinition() + var package = "" + var messageData: [Data] = [] + var enumData: [Data] = [] + for (number, value) in entries { + switch (number, value) { + case (1, _): file.name = string(value) ?? "" + case (2, _): package = string(value) ?? "" + case (3, _): file.dependencies.append(string(value) ?? "") + case (4, .bytes(let bytes)): messageData.append(bytes) + case (5, .bytes(let bytes)): enumData.append(bytes) + default: break + } + } + let syntax = bytes(entries, 12).flatMap { String(data: $0, encoding: .utf8) } + switch syntax { + case "proto3": file.syntax = .proto3 + case "editions": file.syntax = .editions + default: file.syntax = (varint(entries, 14) ?? 0) >= 1_000 ? .editions : .proto2 + } + file.features = try features(entries, options: 8, features: 50) + var collector = Collector() + file.messages = try messageData.map { try collector.message($0, scope: package, depth: 0) } + file.enums = try enumData.map { try collector.enumeration($0, scope: package) } + file.allMessages = collector.messages + file.allEnums = collector.enums + return file + } + + private struct Collector { + var messages: [ProtobufMessageDefinition] = [] + var enums: [ProtobufEnumDefinition] = [] + + mutating func message(_ data: Data, scope: String, depth: Int) throws -> String { + guard depth < ProtoFileParser.maxDepth else { + throw ProtobufSchemaError(reason: String(localized: "Messages are nested too deeply.")) + } + let entries = try ProtobufDescriptorSetParser.fields(data) + let name = entries.first { $0.0 == 1 }.flatMap { string($0.1) } ?? "" + let fullName = scope.isEmpty ? name : "\(scope).\(name)" + let slot = messages.count + messages.append(ProtobufMessageDefinition(fullName: fullName)) + var definition = messages[slot] + definition.features = try features(entries, options: 7, features: 12) + for (number, value) in entries { + guard case .bytes(let bytes) = value else { continue } + switch number { + case 2: definition.fields.append(try field(bytes)) + case 3: definition.nestedMessages.append(try message(bytes, scope: fullName, depth: depth + 1)) + case 4: definition.nestedEnums.append(try enumeration(bytes, scope: fullName)) + case 7: definition.isMapEntry = varint(try fields(bytes), 7).map { $0 != 0 } ?? false + default: break + } + } + messages[slot] = definition + return fullName + } + + mutating func enumeration(_ data: Data, scope: String) throws -> String { + let entries = try ProtobufDescriptorSetParser.fields(data) + var name = "" + var values: [Int32: String] = [:] + for (number, value) in entries { + if number == 1 { name = string(value) ?? "" } + guard number == 2, case .bytes(let bytes) = value else { continue } + var valueName = "" + var valueNumber: Int32 = 0 + for (member, memberValue) in try ProtobufDescriptorSetParser.fields(bytes) { + if member == 1 { valueName = string(memberValue) ?? "" } + if member == 2, case .varint(let raw) = memberValue { valueNumber = Int32(truncatingIfNeeded: raw) } + } + // protoc prints an aliased number by its first name. + if values[valueNumber] == nil { values[valueNumber] = valueName } + } + let fullName = scope.isEmpty ? name : "\(scope).\(name)" + enums.append(ProtobufEnumDefinition( + fullName: fullName, values: values, features: try features(entries, options: 3, features: 7) + )) + return fullName + } + + private func field(_ data: Data) throws -> ProtobufFieldDefinition { + let entries = try ProtobufDescriptorSetParser.fields(data) + let typeCode = varint(entries, 5) ?? 0 + let typeName = bytes(entries, 6).flatMap { String(data: $0, encoding: .utf8) } ?? "" + let type: ProtobufFieldDefinition.TypeReference + switch typeCode { + case 10: type = .group(String(typeName.drop { $0 == "." })) + case 11, 14, 0: type = .named(typeName) + default: + guard let scalar = Self.scalarTypes[typeCode] else { + throw ProtobufSchemaError(reason: String( + format: String(localized: "The descriptor set names an unknown field type %llu."), typeCode + )) + } + type = .scalar(scalar) + } + let isProto3Optional = (varint(entries, 17) ?? 0) != 0 + var definition = ProtobufFieldDefinition( + name: bytes(entries, 1).flatMap { String(data: $0, encoding: .utf8) } ?? "", + number: Int(truncatingIfNeeded: varint(entries, 3) ?? 0), + type: type, + isRepeated: varint(entries, 4) == 3 + ) + definition.isOptional = isProto3Optional + if !isProto3Optional, let index = varint(entries, 9) { definition.oneofIndex = Int(truncatingIfNeeded: index) } + definition.features = try features(entries, options: 8, features: 21) + return definition + } + + private static let scalarTypes: [UInt64: ProtobufScalarType] = [ + 1: .double, 2: .float, 3: .int64, 4: .uint64, 5: .int32, 6: .fixed64, 7: .fixed32, 8: .bool, + 9: .string, 12: .bytes, 13: .uint32, 15: .sfixed32, 16: .sfixed64, 17: .sint32, 18: .sint64, + ] + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift new file mode 100644 index 0000000000..5c0fb341f0 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoder.swift @@ -0,0 +1,716 @@ +// +// ProtobufSchemaDecoder.swift +// TablePro +// + +import Foundation + +internal struct ProtobufTypedField: Equatable, Sendable { + // The field's name, or its number when the schema does not name it. + let name: String + // The declared type, which the viewer shows as the badge; nil for a field the schema does not name. + let typeName: String? + let value: ProtobufTypedValue +} + +internal indirect enum ProtobufTypedValue: Equatable, Sendable { + // Numbers, bools and enum values, already in `protoc --decode` spelling. + case scalar(String) + case string(String) + case bytes(Data) + case message([ProtobufTypedField]) + case unknown(ProtobufFieldValue) + case truncated +} + +internal struct ProtobufTypedMessage: Equatable, Sendable { + let typeName: String + let fields: [ProtobufTypedField] + let framing: ProtobufFraming? +} + +// Matches `protoc --decode=`: known fields by number, then the rest in wire order as `--decode_raw` prints them. +internal enum ProtobufSchemaDecoder { + // A nil message takes the one a Schema Registry frame's message indexes name. + static func decode( + _ data: Data, + schema: ProtobufSchema, + message messageName: String?, + maxValues: Int = .max + ) throws -> ProtobufTypedMessage { + var reader = BinaryValueReader(data) + var framing: ProtobufFraming? + if reader.peekByte() == 0 { + framing = try ProtobufWireDecoder.readFraming(&reader) + } + let message: ProtobufMessageDescriptor? + if let messageName { + message = schema.messages[messageName] + } else if let framing { + message = schema.messageIndexed(by: framing.messageIndexes) + if message == nil { + throw BinaryValueDecodeError(kind: .indexOutOfRange(framing.messageIndexes.last ?? 0), offset: 1) + } + } else { + message = schema.rootMessages.first.flatMap { schema.messages[$0] } + } + guard let message else { throw BinaryValueDecodeError(kind: .invalidField, offset: 0) } + var walker = Walker(schema: schema, budget: maxValues) + let payload = try reader.window(reader.remaining) + let fields = try walker.members(of: [payload], message: message, depth: 0, asMapEntry: false) + return ProtobufTypedMessage(typeName: message.fullName, fields: fields, framing: framing) + } + + private struct Occurrence { + enum Payload { + case varint(UInt64) + case fixed64(UInt64) + case fixed32(UInt32) + case lengthDelimited(BinaryValueReader) + case group(BinaryValueReader) + // A rejected map entry, kept as protoc writes it back: its last key and value. + case entry([Occurrence]) + } + + let number: Int + let payload: Payload + let position: Int + } + + private enum Match { + case accepted + case mismatched + case undeclaredEnumValue + } + + // protoc moves a map entry whose closed enum value is undeclared to the unknown fields whole. + // `kept` is what protoc writes back for the entry; nil keeps its bytes as they came. + private struct MapEntryRejected: Error { + let kept: [Occurrence]? + } + + private struct Level { + var fields: [ProtobufTypedField] = [] + var unknown: [Occurrence] = [] + var isCut = false + } + + private enum MapKey: Comparable { + case signed(Int64) + case unsigned(UInt64) + case bytes([UInt8]) + + static func < (lhs: MapKey, rhs: MapKey) -> Bool { + switch (lhs, rhs) { + case (.signed(let left), .signed(let right)): return left < right + case (.unsigned(let left), .unsigned(let right)): return left < right + case (.bytes(let left), .bytes(let right)): return left.lexicographicallyPrecedes(right) + default: return false + } + } + } + + // An occurrence and each packed value cost one, so a cell reads no more of a large value than it can show. + private struct Walker { + let schema: ProtobufSchema + var budget: Int + + mutating func members( + of windows: [BinaryValueReader], + message: ProtobufMessageDescriptor, + depth: Int, + asMapEntry: Bool + ) throws -> [ProtobufTypedField] { + guard depth < ProtobufWireDecoder.maxDepth else { + throw BinaryValueDecodeError(kind: .nestingTooDeep, offset: windows.first?.offset ?? 0) + } + var level = Level() + var occurrences: [Occurrence] = [] + scan: for window in windows { + var reader = window + while !reader.isAtEnd { + guard budget > 0 else { + level.isCut = true + break scan + } + let (occurrence, isCut) = try Self.occurrence(&reader, depth: depth, limit: budget) + occurrences.append(occurrence) + budget -= 1 + if isCut { + level.isCut = true + break scan + } + } + } + var accepted: [Int: [Occurrence]] = [:] + var lastInEntry: [Int: (occurrence: Occurrence, match: Match)] = [:] + for occurrence in occurrences { + let field = message.field(number: occurrence.number) + let result = field.map { match(occurrence.payload, field: $0) } ?? .mismatched + if asMapEntry, result != .mismatched { lastInEntry[occurrence.number] = (occurrence, result) } + switch result { + case .accepted: + if let field { try verifyUTF8(occurrence.payload, field: field) } + accepted[occurrence.number, default: []].append(occurrence) + case .undeclaredEnumValue where asMapEntry: + break + case .mismatched, .undeclaredEnumValue: + // A map entry drops what it does not declare. + if !asMapEntry { level.unknown.append(occurrence) } + } + } + // Last wins in an entry too: only an undeclared closed enum number left standing rejects it. + if lastInEntry.values.contains(where: { $0.match == .undeclaredEnumValue }) { + throw MapEntryRejected(kept: lastInEntry.keys.sorted().compactMap { lastInEntry[$0]?.occurrence }) + } + try parseDiscarded(keepLastOneofMember(&accepted, message: message), of: message, depth: depth) + var numbers = Set(accepted.keys) + // protoc prints an entry's key and value even when the bytes leave them out. + if asMapEntry, !level.isCut { numbers.formUnion(message.fields.map(\.number)) } + for number in numbers.sorted() { + guard let field = message.field(number: number) else { continue } + if let found = accepted[number] { + try append(field, found, to: &level, depth: depth, asMapEntry: asMapEntry) + } else if let value = defaultValue(field.type) { + level.fields.append(typed(field, value)) + } + } + // An entry that cannot hold a value goes to the unknown fields whole, as protoc's parse would fail. + if asMapEntry, !level.unknown.isEmpty { throw MapEntryRejected(kept: nil) } + var fields = level.fields + let unknown = level.unknown.enumerated().sorted { + ($0.element.position, $0.offset) < ($1.element.position, $1.offset) + } + for (_, occurrence) in unknown { + fields.append(try raw(occurrence, depth: depth)) + } + if level.isCut { fields.append(ProtobufTypedField(name: "", typeName: nil, value: .truncated)) } + return fields + } + + private static func occurrence( + _ reader: inout BinaryValueReader, + depth: Int, + limit: Int + ) throws -> (Occurrence, isCut: Bool) { + let start = reader.offset + let tag = try reader.readVarint() + let number = tag >> 3 + guard tag <= UInt64(UInt32.max), number > 0, number <= UInt64(ProtobufWireDecoder.maxFieldNumber) else { + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + var isCut = false + let payload: Occurrence.Payload + switch tag & 7 { + case 0: + payload = .varint(try reader.readVarint()) + case 1: + payload = .fixed64(try reader.readUInt64LittleEndian()) + case 2: + let length = try reader.readVarint() + guard length <= UInt64(reader.remaining) else { throw reader.error(.unexpectedEnd) } + payload = .lengthDelimited(try reader.window(Int(length))) + try reader.skip(Int(length)) + case 3: + let group = try ProtobufWireDecoder.groupContents(&reader, number: Int(number), depth: depth, limit: limit) + payload = .group(group.contents) + isCut = group.isCut + case 4: + throw BinaryValueDecodeError(kind: .unmatchedGroupEnd, offset: start) + case 5: + payload = .fixed32(try reader.readUInt32LittleEndian()) + default: + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + return (Occurrence(number: Int(number), payload: payload, position: start), isCut) + } + + private func match(_ payload: Occurrence.Payload, field: ProtobufFieldDescriptor) -> Match { + switch (field.type, payload) { + case (.scalar(.string), .lengthDelimited), (.scalar(.bytes), .lengthDelimited), (.group, .group): + return .accepted + case (.message, .lengthDelimited): + return field.isDelimited ? .mismatched : .accepted + case (.message, .group): + return field.isDelimited ? .accepted : .mismatched + case (.scalar(.string), _), (.scalar(.bytes), _): + return .mismatched + case (.scalar, .lengthDelimited), (.enumeration, .lengthDelimited): + return field.isRepeated ? .accepted : .mismatched + case (.scalar(let scalar), .varint): + return scalar.wireType == 0 ? .accepted : .mismatched + case (.scalar(let scalar), .fixed64): + return scalar.wireType == 1 ? .accepted : .mismatched + case (.scalar(let scalar), .fixed32): + return scalar.wireType == 5 ? .accepted : .mismatched + case (.enumeration(let name), .varint(let raw)): + return isDeclared(raw, enumeration: name) ? .accepted : .undeclaredEnumValue + default: + return .mismatched + } + } + + private func isDeclared(_ raw: UInt64, enumeration name: String) -> Bool { + guard let enumeration = schema.enums[name], enumeration.isClosed else { return true } + return enumeration.values[Int32(truncatingIfNeeded: raw)] != nil + } + + // The member set last wins, and only what follows the last switch to it counts. Returns what it drops. + private func keepLastOneofMember( + _ accepted: inout [Int: [Occurrence]], + message: ProtobufMessageDescriptor + ) -> [Occurrence] { + var oneofs: [Int: [Int]] = [:] + for field in message.fields where accepted[field.number] != nil { + if let index = field.oneofIndex { oneofs[index, default: []].append(field.number) } + } + var dropped: [Occurrence] = [] + for members in oneofs.values where members.count > 1 { + let lastPositions = members.map { (number: $0, position: accepted[$0]?.last?.position ?? -1) } + guard let winner = lastPositions.max(by: { $0.position < $1.position })?.number else { continue } + let cutoff = lastPositions.filter { $0.number != winner }.map { $0.position }.max() ?? -1 + for number in members where number != winner { + dropped += accepted[number] ?? [] + accepted[number] = nil + } + dropped += accepted[winner]?.filter { $0.position <= cutoff } ?? [] + accepted[winner] = accepted[winner]?.filter { $0.position > cutoff } + } + // Wire order, so the shared budget meets the same members first on every launch. + return dropped.sorted { $0.position < $1.position } + } + + // protoc parses a member a later one replaces, so a broken one fails the value; this spends the shared budget. + private mutating func parseDiscarded(_ dropped: [Occurrence], of message: ProtobufMessageDescriptor, depth: Int) throws { + for occurrence in dropped { + guard let field = message.field(number: occurrence.number) else { continue } + switch (field.type, occurrence.payload) { + case (.message(let name), .lengthDelimited(let window)), (.message(let name), .group(let window)), + (.group(let name), .group(let window)): + guard let nested = schema.messages[name], budget > 0 else { continue } + _ = try members(of: [window], message: nested, depth: depth + 1, asMapEntry: false) + default: + continue + } + } + } + + private mutating func append( + _ field: ProtobufFieldDescriptor, + _ found: [Occurrence], + to level: inout Level, + depth: Int, + asMapEntry: Bool + ) throws { + switch field.type { + case .message(let name), .group(let name): + guard let nested = schema.messages[name] else { + level.unknown += found + return + } + if !field.isRepeated { + try appendMessage(field, nested, found, to: &level, depth: depth) + } else if nested.isMapEntry { + try appendMapEntries(field, nested, found, to: &level, depth: depth) + } else { + for occurrence in found { try appendMessage(field, nested, [occurrence], to: &level, depth: depth) } + } + case .scalar, .enumeration: + if field.isRepeated { + for occurrence in found { try appendRepeated(field, occurrence, to: &level) } + } else if let last = found.last { + // Last wins first: `count: 5` then `count: 0` leaves an implicit field at its default. + if !asMapEntry, !field.hasPresence, holdsDefault(last.payload, type: field.type) { return } + level.fields.append(typed(field, value(last.payload, field: field))) + } + case .unresolved: + level.unknown += found + } + } + + // A singular message seen again merges, which reads the same as its occurrences back to back. + private mutating func appendMessage( + _ field: ProtobufFieldDescriptor, + _ nested: ProtobufMessageDescriptor, + _ found: [Occurrence], + to level: inout Level, + depth: Int + ) throws { + let windows = found.compactMap { occurrence -> BinaryValueReader? in + switch occurrence.payload { + case .lengthDelimited(let window), .group(let window): return window + case .varint, .fixed64, .fixed32, .entry: return nil + } + } + // A declared message that does not parse fails the value, as it fails protoc's parse. + let members = try self.members(of: windows, message: nested, depth: depth + 1, asMapEntry: false) + if case .group = field.type { + level.fields.append(ProtobufTypedField(name: nested.shortName, typeName: "group", value: .message(members))) + } else { + level.fields.append(typed(field, .message(members))) + } + } + + // protoc sorts a map by key and keeps every entry, a repeated key included. + private mutating func appendMapEntries( + _ field: ProtobufFieldDescriptor, + _ entry: ProtobufMessageDescriptor, + _ found: [Occurrence], + to level: inout Level, + depth: Int + ) throws { + var entries: [(key: MapKey, field: ProtobufTypedField)] = [] + for occurrence in found { + guard case .lengthDelimited(let window) = occurrence.payload else { continue } + let start = budget + do { + let members = try self.members(of: [window], message: entry, depth: depth + 1, asMapEntry: true) + entries.append((mapKey(members, entry: entry), typed(field, .message(members)))) + } catch let rejected as MapEntryRejected { + budget = start + level.unknown.append(rejected.kept.map { + Occurrence(number: occurrence.number, payload: .entry($0), position: occurrence.position) + } ?? occurrence) + } + } + let sorted = entries.enumerated().sorted { lhs, rhs in + lhs.element.key == rhs.element.key ? lhs.offset < rhs.offset : lhs.element.key < rhs.element.key + } + level.fields += sorted.map { $0.element.field } + } + + private mutating func appendRepeated(_ field: ProtobufFieldDescriptor, _ occurrence: Occurrence, to level: inout Level) throws { + guard case .lengthDelimited(let window) = occurrence.payload, + field.type != .scalar(.string), field.type != .scalar(.bytes) else { + level.fields.append(typed(field, value(occurrence.payload, field: field))) + return + } + var reader = window + var values: [ProtobufTypedField] = [] + var undeclared: [Occurrence] = [] + var remaining = budget + var isCut = false + while !reader.isAtEnd { + guard remaining > 0 else { + isCut = true + break + } + let item = try Self.packedItem(&reader, type: field.type) + remaining -= 1 + if case .varint(let raw) = item, case .enumeration(let name) = field.type, !isDeclared(raw, enumeration: name) { + undeclared.append(Occurrence(number: field.number, payload: item, position: occurrence.position)) + } else { + values.append(typed(field, .scalar(scalarText(item, type: field.type)))) + } + } + budget = remaining + level.fields += values + level.unknown += undeclared + if isCut { level.isCut = true } + } + + private static func packedItem(_ reader: inout BinaryValueReader, type: ProtobufFieldType) throws -> Occurrence.Payload { + guard case .scalar(let scalar) = type else { return .varint(try reader.readVarint()) } + switch scalar.wireType { + case 1: return .fixed64(try reader.readUInt64LittleEndian()) + case 5: return .fixed32(try reader.readUInt32LittleEndian()) + default: return .varint(try reader.readVarint()) + } + } + + private mutating func raw(_ occurrence: Occurrence, depth: Int) throws -> ProtobufTypedField { + ProtobufTypedField( + name: String(occurrence.number), typeName: nil, value: .unknown(try rawValue(occurrence.payload, depth: depth)) + ) + } + + private mutating func rawValue(_ payload: Occurrence.Payload, depth: Int) throws -> ProtobufFieldValue { + switch payload { + case .varint(let raw): return .varint(raw) + case .fixed64(let raw): return .fixed64(raw) + case .fixed32(let raw): return .fixed32(raw) + case .lengthDelimited(let window): + return ProtobufWireDecoder.lengthDelimited(window, depth: depth, budget: &budget) + case .group(let window): + return .group(try ProtobufWireDecoder.fields(in: window, depth: depth + 1, budget: &budget)) + case .entry(let kept): + return .message(try kept.map { ProtobufField(number: $0.number, value: try rawValue($0.payload, depth: depth + 1)) }) + } + } + + private func mapKey(_ members: [ProtobufTypedField], entry: ProtobufMessageDescriptor) -> MapKey { + guard let key = entry.field(number: 1) else { return .signed(0) } + let value = members.first { $0.name == key.name }?.value + switch key.type { + case .scalar(.string): + if case .string(let text)? = value { return .bytes(Array(text.utf8)) } + if case .bytes(let data)? = value { return .bytes(Array(data)) } + return .bytes([]) + case .scalar(.bool): + return .unsigned(value == .scalar("true") ? 1 : 0) + case .scalar(.uint32), .scalar(.uint64), .scalar(.fixed32), .scalar(.fixed64): + guard case .scalar(let text)? = value else { return .unsigned(0) } + return .unsigned(UInt64(text) ?? 0) + default: + guard case .scalar(let text)? = value else { return .signed(0) } + return .signed(Int64(text) ?? 0) + } + } + + // Floats compare by bit pattern, so -0 is not the default. + private func holdsDefault(_ payload: Occurrence.Payload, type: ProtobufFieldType) -> Bool { + switch payload { + case .varint(let raw): + switch type { + case .scalar(.int64), .scalar(.uint64), .scalar(.sint64), .scalar(.bool): return raw == 0 + default: return UInt32(truncatingIfNeeded: raw) == 0 + } + case .fixed64(let raw): return raw == 0 + case .fixed32(let raw): return raw == 0 + case .lengthDelimited(let window): return window.remaining == 0 + case .group, .entry: return false + } + } + + private func defaultValue(_ type: ProtobufFieldType) -> ProtobufTypedValue? { + switch type { + case .scalar(.string): return .string("") + case .scalar(.bytes): return .bytes(Data()) + case .scalar(.bool): return .scalar("false") + case .scalar: return .scalar("0") + case .enumeration(let name): return .scalar(schema.enums[name]?.values[0] ?? "0") + case .message, .group: return .message([]) + case .unresolved: return nil + } + } + + // Every occurrence, as protoc checks each one it parses, the ones a later value replaces included. + private func verifyUTF8(_ payload: Occurrence.Payload, field: ProtobufFieldDescriptor) throws { + guard field.validatesUTF8, field.type == .scalar(.string), case .lengthDelimited(let window) = payload else { return } + var reader = window + let data = (try? reader.readBytes(reader.remaining)) ?? Data() + guard String(data: data, encoding: .utf8) == nil else { return } + throw BinaryValueDecodeError(kind: .invalidUTF8, offset: window.offset) + } + + // An unverified string that is not UTF-8 prints escaped, as protoc prints a proto2 string. + private func value(_ payload: Occurrence.Payload, field: ProtobufFieldDescriptor) -> ProtobufTypedValue { + guard case .lengthDelimited(let window) = payload else { return .scalar(scalarText(payload, type: field.type)) } + var reader = window + let data = (try? reader.readBytes(reader.remaining)) ?? Data() + if field.type == .scalar(.string), let text = String(data: data, encoding: .utf8) { return .string(text) } + return .bytes(data) + } + + private func typed(_ descriptor: ProtobufFieldDescriptor, _ value: ProtobufTypedValue) -> ProtobufTypedField { + ProtobufTypedField(name: descriptor.name, typeName: typeName(descriptor.type), value: value) + } + + private func typeName(_ type: ProtobufFieldType) -> String { + switch type { + case .scalar(let scalar): return scalar.rawValue + case .message(let name): return schema.messages[name]?.shortName ?? name + case .enumeration(let name): return schema.enums[name]?.shortName ?? name + case .group: return "group" + case .unresolved(let name): return name + } + } + + private func scalarText(_ payload: Occurrence.Payload, type: ProtobufFieldType) -> String { + switch payload { + case .varint(let raw): return varintText(raw, type: type) ?? String(raw) + case .fixed64(let raw): return fixed64Text(raw, type: type) ?? String(raw) + case .fixed32(let raw): return fixed32Text(raw, type: type) ?? String(raw) + case .lengthDelimited, .group, .entry: return "" + } + } + + private func varintText(_ raw: UInt64, type: ProtobufFieldType) -> String? { + switch type { + case .scalar(.int32): return String(Int32(truncatingIfNeeded: raw)) + case .scalar(.int64): return String(Int64(bitPattern: raw)) + case .scalar(.uint32): return String(UInt32(truncatingIfNeeded: raw)) + case .scalar(.uint64): return String(raw) + case .scalar(.sint32): + // Truncated before the ZigZag step, as protoc reads a value a sint64 writer sent. + let bits = UInt32(truncatingIfNeeded: raw) + return String(Int32(bitPattern: (bits >> 1) ^ (0 &- (bits & 1)))) + case .scalar(.sint64): return String(ProtobufWireDecoder.zigzag(raw)) + case .scalar(.bool): return raw != 0 ? "true" : "false" + case .enumeration(let name): + let number = Int32(truncatingIfNeeded: raw) + return schema.enums[name]?.values[number] ?? String(number) + default: return nil + } + } + + private func fixed64Text(_ raw: UInt64, type: ProtobufFieldType) -> String? { + switch type { + case .scalar(.double): return ProtobufNumberText.double(Double(bitPattern: raw)) + case .scalar(.fixed64): return String(raw) + case .scalar(.sfixed64): return String(Int64(bitPattern: raw)) + default: return nil + } + } + + private func fixed32Text(_ raw: UInt32, type: ProtobufFieldType) -> String? { + switch type { + case .scalar(.float): return ProtobufNumberText.float(Float(bitPattern: raw)) + case .scalar(.fixed32): return String(raw) + case .scalar(.sfixed32): return String(Int32(bitPattern: raw)) + default: return nil + } + } + } +} + +private extension ProtobufScalarType { + var wireType: UInt64 { + switch self { + case .double, .fixed64, .sfixed64: return 1 + case .float, .fixed32, .sfixed32: return 5 + case .string, .bytes: return 2 + case .int32, .int64, .uint32, .uint64, .sint32, .sint64, .bool: return 0 + } + } +} + +// protobuf's SimpleDtoa and SimpleFtoa: the shortest of DBL_DIG or FLT_DIG digits that reads back, else the full count. +internal enum ProtobufNumberText { + static func double(_ value: Double) -> String { + if let special = special(value) { return special } + let short = String(format: "%.15g", value) + return Double(short) == value ? short : String(format: "%.17g", value) + } + + static func float(_ value: Float) -> String { + if let special = special(Double(value)) { return special } + let short = String(format: "%.6g", Double(value)) + return Float(short) == value ? short : String(format: "%.9g", Double(value)) + } + + private static func special(_ value: Double) -> String? { + if value.isNaN { return "nan" } + if value.isInfinite { return value < 0 ? "-inf" : "inf" } + return nil + } +} + +internal enum ProtobufSchemaTextWriter { + static func text(_ fields: [ProtobufTypedField], multiLine: Bool, limit: Int = .max) -> String { + var output = "" + write(fields, into: &output, multiLine: multiLine, depth: 0, limit: limit) + return multiLine ? output : output.trimmingCharacters(in: .whitespaces) + } + + static func scalarText(_ value: ProtobufTypedValue, limit: Int = .max) -> String? { + switch value { + case .scalar(let text): return text + case .string(let text): return ProtobufTextWriter.scalarText(.string(text), limit: limit) + case .bytes(let data): return ProtobufTextWriter.scalarText(.bytes(data), limit: limit) + case .unknown(let raw): return ProtobufTextWriter.scalarText(raw, limit: limit) + case .message, .truncated: return nil + } + } + + private static func write( + _ fields: [ProtobufTypedField], + into output: inout String, + multiLine: Bool, + depth: Int, + limit: Int + ) { + let indent = multiLine ? String(repeating: " ", count: depth) : "" + let separator = multiLine ? "\n" : " " + for field in fields { + let room = limit - output.utf8.count + guard room > 0 else { return } + switch field.value { + case .message(let members): + output += "\(indent)\(field.name) {\(separator)" + write(members, into: &output, multiLine: multiLine, depth: depth + 1, limit: limit) + output += "\(indent)}\(separator)" + case .unknown(let raw): + ProtobufTextWriter.write( + [ProtobufField(number: Int(field.name) ?? 0, value: raw)], + into: &output, multiLine: multiLine, depth: depth, limit: limit + ) + case .truncated: + output += "\(indent)…\(separator)" + case .scalar, .string, .bytes: + output += "\(indent)\(field.name): \(scalarText(field.value, limit: room) ?? "")\(separator)" + } + } + } +} + +internal enum ProtobufSchemaTreeBuilder { + private static let maxDisplayLength = 300 + + static func build(from message: ProtobufTypedMessage, formats: TreeSummaryFormats = .localized) -> ProtobufTreeNode { + var nodeCount = 0 + let fields = message.fields.filter { $0.value != .truncated } + return ProtobufTreeNode( + key: nil, keyPath: "", path: .root, kind: .message, + displayValue: "{\(formats.fields.text(fields.count))}", + typeBadge: message.typeName.split(separator: ".").last.map(String.init), + typedMembers: message.fields, + children: makeChildren(fields, keyPath: nil, path: .root, formats: formats, nodeCount: &nodeCount) + ) + } + + private static func makeChildren( + _ fields: [ProtobufTypedField], + keyPath: String?, + path: TreeNodePath, + formats: TreeSummaryFormats, + nodeCount: inout Int + ) -> [ProtobufTreeNode] { + var children: [ProtobufTreeNode] = [] + var siblings = TreeSiblingCounter() + for field in fields where field.value != .truncated { + guard nodeCount < TreeNodeLimits.maxNodes else { + children.append(ProtobufTreeNode( + key: nil, keyPath: "", path: path.appending(.truncationMarker), kind: .truncated, + displayValue: "… (\(fields.count - children.count) more)" + )) + break + } + nodeCount += 1 + let fieldKeyPath = keyPath.map { "\($0).\(field.name)" } ?? field.name + let fieldPath = path.appending(siblings.key(field.name)) + switch field.value { + case .message(let members): + let shown = members.filter { $0.value != .truncated } + children.append(ProtobufTreeNode( + key: field.name, keyPath: fieldKeyPath, path: fieldPath, kind: .message, + displayValue: "{\(formats.fields.text(shown.count))}", + typeBadge: field.typeName, + typedMembers: members, + children: makeChildren(shown, keyPath: fieldKeyPath, path: fieldPath, formats: formats, nodeCount: &nodeCount) + )) + case .unknown(let raw): + children.append(ProtobufTreeBuilder.node( + key: field.name, keyPath: fieldKeyPath, path: fieldPath, value: raw, formats: formats, nodeCount: &nodeCount + )) + case .string(let text): + let display = TreeDisplayText.quoted(text, limit: maxDisplayLength) + children.append(ProtobufTreeNode( + key: field.name, keyPath: fieldKeyPath, path: fieldPath, kind: .string, + displayValue: display.text, isDisplayCut: display.isCut, rawValue: text, typeBadge: field.typeName + )) + case .scalar, .bytes, .truncated: + let text = ProtobufSchemaTextWriter.scalarText(field.value) ?? "" + let display = TreeDisplayText.cut(text, limit: maxDisplayLength) + let kind: ProtobufTreeNode.Kind + if case .bytes = field.value { kind = .bytes } else { kind = .varint } + children.append(ProtobufTreeNode( + key: field.name, keyPath: fieldKeyPath, path: fieldPath, kind: kind, + displayValue: display.isCut ? display.text + "…" : display.text, + isDisplayCut: display.isCut, rawValue: text, typeBadge: field.typeName + )) + } + } + return children + } +} diff --git a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift index 980b986578..4e72c33fb8 100644 --- a/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift +++ b/apple/TablePro/Core/Services/Formatting/BinaryValues/ProtobufWireDecoder.swift @@ -49,7 +49,7 @@ internal enum ProtobufWireDecoder { return ProtobufMessage(fields: fields, framing: framing) } - private static func readFraming(_ reader: inout BinaryValueReader) throws -> ProtobufFraming { + static func readFraming(_ reader: inout BinaryValueReader) throws -> ProtobufFraming { _ = try reader.readByte() let schemaID = try reader.readUInt32() let start = reader.offset @@ -65,7 +65,71 @@ internal enum ProtobufWireDecoder { return ProtobufFraming(schemaID: schemaID, messageIndexes: indexes) } - private static func zigzag(_ value: UInt64) -> Int64 { + // Spends the caller's budget, so a cell stops inside a large field the schema does not name. + static func lengthDelimited(_ window: BinaryValueReader, depth: Int, budget: inout Int) -> ProtobufFieldValue { + var parser = Parser(budget: budget) + var window = window + let value = parser.lengthDelimited(&window, depth: depth) + budget = parser.budget + return value + } + + static func fields(in window: BinaryValueReader, depth: Int, budget: inout Int) throws -> [ProtobufField] { + var parser = Parser(budget: budget) + var window = window + let fields = try parser.fields(&window, depth: depth, group: nil) + budget = parser.budget + return fields + } + + // The bytes between a group's start tag, already read, and its end tag. Past `limit` tags it stops at the last whole + // field and reports the cut, so a cell never reads more of a group than its budget. + static func groupContents( + _ reader: inout BinaryValueReader, + number: Int, + depth: Int, + limit: Int + ) throws -> (contents: BinaryValueReader, isCut: Bool) { + let contents = reader + var openGroups = [number] + var boundary = reader.offset + var scanned = 0 + while true { + let start = reader.offset + if openGroups.count == 1 { boundary = start } + guard scanned < limit else { return (try contents.window(boundary - contents.offset), true) } + guard !reader.isAtEnd else { throw reader.error(.unexpectedEnd) } + let tag = try reader.readVarint() + scanned += 1 + let fieldNumber = tag >> 3 + guard tag <= UInt64(UInt32.max), fieldNumber > 0, fieldNumber <= UInt64(maxFieldNumber) else { + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + switch tag & 7 { + case 0: + _ = try reader.readVarint() + case 1: + try reader.skip(8) + case 2: + let length = try reader.readVarint() + guard length <= UInt64(reader.remaining) else { throw reader.error(.unexpectedEnd) } + try reader.skip(Int(length)) + case 3: + guard depth + openGroups.count < maxDepth else { throw BinaryValueDecodeError(kind: .nestingTooDeep, offset: start) } + openGroups.append(Int(fieldNumber)) + case 4: + guard openGroups.last == Int(fieldNumber) else { throw BinaryValueDecodeError(kind: .unmatchedGroupEnd, offset: start) } + openGroups.removeLast() + if openGroups.isEmpty { return (try contents.window(start - contents.offset), false) } + case 5: + try reader.skip(4) + default: + throw BinaryValueDecodeError(kind: .invalidField, offset: start) + } + } + } + + static func zigzag(_ value: UInt64) -> Int64 { Int64(bitPattern: value >> 1) ^ -Int64(bitPattern: value & 1) } @@ -128,7 +192,7 @@ internal enum ProtobufWireDecoder { return result } - private mutating func lengthDelimited(_ window: inout BinaryValueReader, depth: Int) -> ProtobufFieldValue { + mutating func lengthDelimited(_ window: inout BinaryValueReader, depth: Int) -> ProtobufFieldValue { if window.remaining > 0, depth < ProtobufWireDecoder.maxDepth, budget > 0 { var attempt = self var nested = window diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift index 9ded656ca1..f3833ba647 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift @@ -17,16 +17,19 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case messagePack case protobuf case igbinary - // The schema is kept by AvroSchemaStore under this ID, so every per-column path can decode with it. case avro(schemaID: String) + case protobufSchema(schemaID: String, message: String) + case schemaRegistry(connectionID: UUID) - // Every format but Avro, which needs a schema chosen first. + // The schema formats are left out: each needs a file or a registry chosen first. static let presetFormats: [ValueDisplayFormat] = [ .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized, .messagePack, .protobuf, .igbinary, ] private static let avroPrefix = "avro:" + private static let protobufSchemaPrefix = "protobuf:" + private static let schemaRegistryPrefix = "schemaRegistry:" init?(rawValue: String) { switch rawValue { @@ -42,10 +45,20 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case "protobuf": self = .protobuf case "igbinary": self = .igbinary default: - guard rawValue.hasPrefix(Self.avroPrefix) else { return nil } - let schemaID = String(rawValue.dropFirst(Self.avroPrefix.count)) - guard AvroSchemaStore.isSchemaID(schemaID) else { return nil } - self = .avro(schemaID: schemaID) + if rawValue.hasPrefix(Self.avroPrefix) { + let schemaID = String(rawValue.dropFirst(Self.avroPrefix.count)) + guard AvroSchemaStore.isSchemaID(schemaID) else { return nil } + self = .avro(schemaID: schemaID) + } else if rawValue.hasPrefix(Self.protobufSchemaPrefix) { + let parts = rawValue.dropFirst(Self.protobufSchemaPrefix.count).split(separator: ":", maxSplits: 1) + guard parts.count == 2, AvroSchemaStore.isSchemaID(String(parts[0])), !parts[1].isEmpty else { return nil } + self = .protobufSchema(schemaID: String(parts[0]), message: String(parts[1])) + } else if rawValue.hasPrefix(Self.schemaRegistryPrefix), + let connectionID = UUID(uuidString: String(rawValue.dropFirst(Self.schemaRegistryPrefix.count))) { + self = .schemaRegistry(connectionID: connectionID) + } else { + return nil + } } } @@ -63,6 +76,8 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case .protobuf: return "protobuf" case .igbinary: return "igbinary" case .avro(let schemaID): return Self.avroPrefix + schemaID + case .protobufSchema(let schemaID, let message): return Self.protobufSchemaPrefix + schemaID + ":" + message + case .schemaRegistry(let connectionID): return Self.schemaRegistryPrefix + connectionID.uuidString } } @@ -96,6 +111,10 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case .avro(let schemaID): guard let name = AvroSchemaStore.shared.schema(id: schemaID)?.displayName else { return "Avro" } return String(format: String(localized: "Avro (%@)"), name) + case .protobufSchema(_, let message): + return String(format: String(localized: "Protobuf (%@)"), message.split(separator: ".").last.map(String.init) ?? message) + case .schemaRegistry: + return String(localized: "Schema Registry") } } @@ -105,6 +124,8 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { case .protobuf: return .protobuf case .igbinary: return .igbinary case .avro(let schemaID): return .avro(schemaID: schemaID) + case .protobufSchema(let schemaID, let message): return .protobufSchema(schemaID: schemaID, message: message) + case .schemaRegistry(let connectionID): return .schemaRegistry(connectionID: connectionID) case .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: return nil } @@ -114,8 +135,11 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { switch self { case .raw: return [] - case .text, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro: + case .text, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro, .protobufSchema: return ["blob"] + // An older Kafka plugin puts a framed value whose bytes are valid UTF-8 in a text column. + case .schemaRegistry: + return ["blob", "text"] case .uuid: return ["blob", "text"] case .unixTimestamp, .unixTimestampMillis: @@ -146,7 +170,7 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { // A binary column is worth searching exactly when one of these is on it; hex is not what anyone types into Find. var rendersBinaryAsText: Bool { switch self { - case .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro: + case .text, .uuid, .uuidSwappedTime, .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: return true case .raw, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: return false @@ -163,6 +187,10 @@ enum ValueDisplayFormat: Hashable, Codable, Identifiable, RawRepresentable { static func offersAvro(for columnType: ColumnType?, databaseType: DatabaseType? = nil) -> Bool { avro(schemaID: "").isApplicable(to: columnType, databaseType: databaseType) } + + static func offersProtobufSchema(for columnType: ColumnType?, databaseType: DatabaseType? = nil) -> Bool { + protobufSchema(schemaID: "", message: "").isApplicable(to: columnType, databaseType: databaseType) + } } enum ValueDisplayFormatColumnKey { diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift index 83e8941087..3732e7d5b0 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift @@ -36,8 +36,9 @@ enum ValueDisplayFormatter { return formatAsTimestamp(rawValue, divideBy: 1_000) case .json, .phpSerialized: return rawValue - case .messagePack, .protobuf, .igbinary, .avro: - guard let data = rawValue.data(using: .isoLatin1), let decoder = format.binaryValueFormat else { return nil } + case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: + guard let data = BinaryValueBytes.data(of: rawValue, holdsBytes: BinaryValueBytes.holdsBytes(columnType: columnType)), + let decoder = format.binaryValueFormat else { return nil } return BinaryValueDecoding.text(data, format: decoder) } } @@ -54,7 +55,7 @@ enum ValueDisplayFormatter { guard rawValue.count == 16 else { return nil } let order: UUIDByteOrder = format == .uuidSwappedTime ? .swappedTime : .stored return insertUuidHyphens(order.uuidBytes(fromStored: [UInt8](rawValue)).hexEncoded) - case .messagePack, .protobuf, .igbinary, .avro: + case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: guard let decoder = format.binaryValueFormat else { return nil } return BinaryValueDecoding.text(rawValue, format: decoder) case .raw, .unixTimestamp, .unixTimestampMillis, .json, .phpSerialized: diff --git a/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift b/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift new file mode 100644 index 0000000000..768385daf1 --- /dev/null +++ b/apple/TablePro/Core/Services/Infrastructure/ProtobufSchemaFilePanel.swift @@ -0,0 +1,80 @@ +// +// ProtobufSchemaFilePanel.swift +// TablePro +// + +import AppKit + +internal struct ProtobufSchemaChoice { + let contents: Data + let message: String +} + +// Any extension: a descriptor set has none of its own, so the validate hook refuses a file that is not a schema. +@MainActor +internal enum ProtobufSchemaFilePanel { + static func chooseSchema(in window: NSWindow) async -> ProtobufSchemaChoice? { + let panel = NSOpenPanel() + panel.allowedContentTypes = [] + panel.canChooseFiles = true + panel.canChooseDirectories = false + panel.allowsMultipleSelection = false + panel.prompt = String(localized: "Choose Schema") + panel.message = String(localized: "Choose the .proto file or descriptor set the values in this column were written with.") + let gate = ProtobufSchemaFileGate() + panel.delegate = gate + let response = await panel.presentAsSheet(for: window) + guard response == .OK, let contents = gate.contents, let schema = gate.schema else { return nil } + guard let message = await chooseMessage(in: schema, window: window) else { return nil } + return ProtobufSchemaChoice(contents: contents, message: message) + } + + private static func chooseMessage(in schema: ProtobufSchema, window: NSWindow) async -> String? { + let names = schema.messageNames + guard names.count > 1 else { return names.first } + let alert = NSAlert() + alert.messageText = String(localized: "Which message are the values?") + alert.informativeText = String(localized: "The file defines more than one message. Choose the one this column holds.") + let popUp = NSPopUpButton(frame: NSRect(x: 0, y: 0, width: 320, height: 26), pullsDown: false) + popUp.addItems(withTitles: names) + if let first = schema.rootMessages.first { popUp.selectItem(withTitle: first) } + alert.accessoryView = popUp + alert.addButton(withTitle: String(localized: "Choose")) + alert.addButton(withTitle: String(localized: "Cancel")) + let response = await alert.beginSheetModal(for: window) + return response == .alertFirstButtonReturn ? popUp.titleOfSelectedItem : nil + } +} + +@MainActor +private final class ProtobufSchemaFileGate: NSObject, NSOpenSavePanelDelegate { + private(set) var contents: Data? + private(set) var schema: ProtobufSchema? + + func panel(_ sender: Any, validate url: URL) throws { + let name = url.lastPathComponent + let size = (try? url.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0 + guard size <= ProtobufSchemaStore.maxFileBytes, let data = try? Data(contentsOf: url) else { + throw ProtobufSchemaFileError(fileName: name, reason: String(localized: "The file is larger than any schema.")) + } + do { + schema = try ProtobufSchema.parse(data) + } catch { + throw ProtobufSchemaFileError(fileName: name, reason: ((error as? ProtobufSchemaError) ?? .notASchema).reason) + } + contents = data + } +} + +private struct ProtobufSchemaFileError: LocalizedError { + let fileName: String + let reason: String + + var errorDescription: String? { + String(format: String(localized: "“%@” is not a usable Protobuf schema."), fileName) + } + + var recoverySuggestion: String? { + reason + } +} diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift new file mode 100644 index 0000000000..9ff0ce2c8a --- /dev/null +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryCache.swift @@ -0,0 +1,257 @@ +// +// SchemaRegistryCache.swift +// TablePro +// + +import Foundation +import os + +// A registry never changes the schema behind an ID, so a loaded one is kept for the session, keyed by the endpoint it came from. +internal final class SchemaRegistryCache: Sendable { + typealias Lookup = Result + typealias EndpointResolver = @MainActor @Sendable (UUID) -> SchemaRegistryEndpoint? + + static let shared = SchemaRegistryCache() + + // Posted per loaded schema to every connection on that endpoint, and when a connection's endpoint changed. + static let didChange = Notification.Name("com.TablePro.schemaRegistryCache.didChange") + // Posted on the main actor while the old endpoint still answers, so every grid can note what its filters show. + static let willSwitch = Notification.Name("com.TablePro.schemaRegistryCache.willSwitch") + static let connectionIDKey = "connectionID" + + // A page of values can name thousands of IDs that are only bytes starting with zero, so fetches are bounded. + static let maxConcurrentFetches = 4 + static let maxPendingFetches = 256 + static let maxFailures = 1_024 + + private struct Key: Hashable { + let endpoint: SchemaRegistryEndpoint + let schemaID: UInt32 + } + + private enum Entry { + case loaded(RegistrySchema) + case failed(SchemaRegistryError, at: ContinuousClock.Instant) + } + + private enum Resolved { + case configured(SchemaRegistryEndpoint) + case unconfigured + } + + private struct State { + var endpoints: [UUID: Resolved] = [:] + var entries: [Key: Entry] = [:] + var inFlight: [Key: Task] = [:] + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + private let resolveEndpoint: EndpointResolver + private let transport: SchemaRegistryClient.Transport + private let failureLifetime: Duration + private let limiter = FetchLimiter(limit: SchemaRegistryCache.maxConcurrentFetches) + + init( + failureLifetime: Duration = .seconds(30), + resolveEndpoint: @escaping EndpointResolver = { SchemaRegistryEndpoint.resolve(for: $0) }, + transport: @escaping SchemaRegistryClient.Transport = SchemaRegistryClient.urlSessionTransport + ) { + self.failureLifetime = failureLifetime + self.resolveEndpoint = resolveEndpoint + self.transport = transport + } + + // Never fetches: nil until a prefetch or the viewer has asked for the schema. + func cachedSchema(connectionID: UUID, schemaID: UInt32) -> Lookup? { + state.withLock { state in + switch state.endpoints[connectionID] { + case nil: + return nil + case .unconfigured?: + return .failure(Self.noEndpoint) + case .configured(let endpoint)?: + switch state.entries[Key(endpoint: endpoint, schemaID: schemaID)] { + case .loaded(let schema)?: return .success(schema) + case .failed(let error, _)?: return .failure(error) + case nil: return nil + } + } + } + } + + // A reload retries failures at once; anything else waits out the failure lifetime. True when the endpoint switched. + @MainActor + @discardableResult + func prefetch(connectionID: UUID, schemaIDs: Set, retryingFailures: Bool) -> Bool { + guard !schemaIDs.isEmpty else { return false } + let allLoaded = state.withLock { state -> Bool in + guard case .configured(let endpoint)? = state.endpoints[connectionID] else { return false } + return schemaIDs.allSatisfy { + if case .loaded? = state.entries[Key(endpoint: endpoint, schemaID: $0)] { return true } + return false + } + } + // Resolving reads the Keychain, so a batch that needs nothing new skips it unless a reload asked to look again. + if allLoaded, !retryingFailures { return false } + let endpoint = resolveEndpoint(connectionID) + announceSwitch(to: endpoint, for: connectionID) + let changed = state.withLock { state -> Bool in + let changed = adopt(endpoint, for: connectionID, in: &state) + guard let endpoint else { return changed } + pruneFailures(in: &state) + for schemaID in schemaIDs.sorted() { + let key = Key(endpoint: endpoint, schemaID: schemaID) + guard state.inFlight[key] == nil, usable(state.entries[key], retryingFailures: retryingFailures) == nil, + state.inFlight.count < Self.maxPendingFetches else { continue } + _ = start(key, in: &state) + } + return changed + } + if changed { Self.post(connectionID) } + return changed + } + + @MainActor + func schema(connectionID: UUID, schemaID: UInt32) async -> Lookup { + let endpoint = resolveEndpoint(connectionID) + announceSwitch(to: endpoint, for: connectionID) + let (changed, pending, cached) = state.withLock { state -> (Bool, Task?, Lookup?) in + let changed = adopt(endpoint, for: connectionID, in: &state) + guard let endpoint else { return (changed, nil, .failure(Self.noEndpoint)) } + let key = Key(endpoint: endpoint, schemaID: schemaID) + if let running = state.inFlight[key] { return (changed, running, nil) } + if let cached = usable(state.entries[key], retryingFailures: false) { return (changed, nil, cached) } + return (changed, start(key, in: &state), nil) + } + if changed { Self.post(connectionID) } + guard let pending else { return cached ?? .failure(Self.noEndpoint) } + return await pending.value + } + + private static var noEndpoint: SchemaRegistryError { + SchemaRegistryError( + reason: String(localized: "The connection names no Schema Registry URL. Add one in the connection's settings.") + ) + } + + private func adopt(_ endpoint: SchemaRegistryEndpoint?, for connectionID: UUID, in state: inout State) -> Bool { + let previous = state.endpoints.updateValue(endpoint.map { .configured($0) } ?? .unconfigured, forKey: connectionID) + return Self.isSwitch(from: previous, to: endpoint) + } + + @MainActor + private func announceSwitch(to endpoint: SchemaRegistryEndpoint?, for connectionID: UUID) { + guard state.withLock({ Self.isSwitch(from: $0.endpoints[connectionID], to: endpoint) }) else { return } + NotificationCenter.default.post(name: Self.willSwitch, object: nil, userInfo: [Self.connectionIDKey: connectionID]) + } + + private static func isSwitch(from previous: Resolved?, to endpoint: SchemaRegistryEndpoint?) -> Bool { + switch (previous, endpoint) { + case (nil, _), (.unconfigured?, nil): + return false + case (.configured(let old)?, let new?): + return old != new + default: + return true + } + } + + private func usable(_ entry: Entry?, retryingFailures: Bool) -> Lookup? { + switch entry { + case .loaded(let schema)?: + return .success(schema) + case .failed(let error, let at)?: + return retryingFailures || ContinuousClock.now - at >= failureLifetime ? nil : .failure(error) + case nil: + return nil + } + } + + private func start(_ key: Key, in state: inout State) -> Task { + let client = SchemaRegistryClient(endpoint: key.endpoint, transport: transport) + let limiter = limiter + let task = Task { + await limiter.acquire() + let result: Lookup + do { + result = .success(try await client.schema(id: key.schemaID)) + } catch let error as SchemaRegistryError { + result = .failure(error) + } catch { + result = .failure(SchemaRegistryError(reason: error.localizedDescription)) + } + await limiter.release() + self.finish(key, with: result) + return result + } + state.inFlight[key] = task + return task + } + + // A load is announced as it lands, to every connection reading this endpoint; the grid coalesces the notices. + private func finish(_ key: Key, with result: Lookup) { + let readers = state.withLock { state -> [UUID] in + state.inFlight[key] = nil + switch result { + case .success(let schema): + state.entries[key] = .loaded(schema) + return state.endpoints.compactMap { id, resolved in + if case .configured(let endpoint) = resolved, endpoint == key.endpoint { return id } + return nil + } + case .failure(let error): + state.entries[key] = .failed(error, at: .now) + return [] + } + } + for connectionID in readers { Self.post(connectionID) } + } + + // Garbage prefixes would otherwise leave one failure each for the session. + private func pruneFailures(in state: inout State) { + let now = ContinuousClock.now + var failures: [(key: Key, at: ContinuousClock.Instant)] = [] + for (key, entry) in state.entries { + guard case .failed(_, let at) = entry else { continue } + if now - at >= failureLifetime { + state.entries[key] = nil + } else { + failures.append((key, at)) + } + } + guard failures.count > Self.maxFailures else { return } + for failure in failures.sorted(by: { $0.at < $1.at }).prefix(failures.count - Self.maxFailures) { + state.entries[failure.key] = nil + } + } + + private static func post(_ connectionID: UUID) { + NotificationCenter.default.post(name: didChange, object: nil, userInfo: [connectionIDKey: connectionID]) + } +} + +private actor FetchLimiter { + private let limit: Int + private var running = 0 + private var waiting: [CheckedContinuation] = [] + + init(limit: Int) { + self.limit = limit + } + + func acquire() async { + guard running >= limit else { + running += 1 + return + } + await withCheckedContinuation { waiting.append($0) } + } + + func release() { + guard !waiting.isEmpty else { + running -= 1 + return + } + waiting.removeFirst().resume() + } +} diff --git a/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift new file mode 100644 index 0000000000..1f1c9bcadf --- /dev/null +++ b/apple/TablePro/Core/Services/SchemaRegistry/SchemaRegistryClient.swift @@ -0,0 +1,204 @@ +// +// SchemaRegistryClient.swift +// TablePro +// + +import Foundation + +// Engine neutral: any driver that declares these connection fields gets the Schema Registry display format. +internal enum SchemaRegistryConnectionField { + static let url = "schemaRegistryUrl" + static let username = "schemaRegistryUsername" + static let password = "schemaRegistryPassword" + static let all: Set = [url, username, password] +} + +internal struct SchemaRegistryEndpoint: Hashable, Sendable { + let baseURL: URL + let username: String? + let password: String? + + init?(fields: [String: String]) { + let text = (fields[SchemaRegistryConnectionField.url] ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), ["http", "https"].contains(scheme), + url.host?.isEmpty == false else { return nil } + baseURL = url + let username = fields[SchemaRegistryConnectionField.username] ?? "" + self.username = username.isEmpty ? nil : username + password = fields[SchemaRegistryConnectionField.password] + } + + // The session's record, so the endpoint changes when the connection does: on the next connect, like every other field. + @MainActor + static func connection(for connectionID: UUID) -> DatabaseConnection? { + DatabaseManager.shared.session(for: connectionID)?.connection + ?? ConnectionStorage.shared.loadConnection(id: connectionID) + } + + @MainActor + static func isConfigured(for connectionID: UUID) -> Bool { + guard let connection = connection(for: connectionID) else { return false } + return SchemaRegistryEndpoint(fields: connection.additionalFields) != nil + } + + @MainActor + static func resolve(for connectionID: UUID) -> SchemaRegistryEndpoint? { + guard let connection = connection(for: connectionID) else { return nil } + let fields = DatabaseDriverFactory.resolvedAdditionalFields(for: connection) ?? connection.additionalFields + return SchemaRegistryEndpoint(fields: fields) + } +} + +internal enum RegistrySchema: Sendable { + case avro(AvroSchema) + case protobuf(ProtobufSchema) + case json +} + +internal struct SchemaRegistryError: Error, Equatable, Sendable { + let reason: String +} + +// https://docs.confluent.io/platform/current/schema-registry/develop/api.html +internal struct SchemaRegistryClient: Sendable { + typealias Transport = @Sendable (URLRequest) async throws -> (Data, URLResponse) + + static let maxResponseBytes = 4_000_000 + static let maxReferences = 200 + static let timeout: TimeInterval = 15 + + let endpoint: SchemaRegistryEndpoint + let transport: Transport + + init(endpoint: SchemaRegistryEndpoint, transport: @escaping Transport = Self.urlSessionTransport) { + self.endpoint = endpoint + self.transport = transport + } + + private static let session: URLSession = { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForResource = 60 + configuration.httpShouldSetCookies = false + return URLSession(configuration: configuration) + }() + + static let urlSessionTransport: Transport = { request in + guard let url = request.url else { throw URLError(.badURL) } + let (bytes, response) = try await session.bytes(for: request, delegate: SameOriginRedirectGuard(origin: url)) + var data = Data() + for try await byte in bytes { + data.append(byte) + // Stops a registry that never ends its answer from filling memory. + if data.count > maxResponseBytes { break } + } + return (data, response) + } + + func schema(id: UInt32) async throws -> RegistrySchema { + let root: SchemaResponse = try await get(["schemas", "ids", String(id)]) + switch root.schemaType?.uppercased() ?? "AVRO" { + case "AVRO": + let references = try await referencedSchemas(root.references ?? []) + do { + return .avro(try AvroSchema.parse(root.schema, references: references)) + } catch let error as AvroSchemaError { + throw SchemaRegistryError(reason: error.reason) + } + case "PROTOBUF": + let references = try await referencedSchemas(root.references ?? []) + do { + return .protobuf(try ProtobufSchema.parse(protoFiles: [root.schema] + references)) + } catch let error as ProtobufSchemaError { + throw SchemaRegistryError(reason: error.reason) + } + case "JSON": + return .json + case let other: + throw SchemaRegistryError(reason: String( + format: String(localized: "The Schema Registry names a schema type TablePro does not read: %@."), other + )) + } + } + + // Each subject version once, after the ones it references, so a type is defined before its users. + private func referencedSchemas(_ references: [SchemaReference]) async throws -> [String] { + var texts: [String] = [] + var seen: Set = [] + try await collect(references, into: &texts, seen: &seen) + return texts + } + + private func collect(_ references: [SchemaReference], into texts: inout [String], seen: inout Set) async throws { + for reference in references { + guard seen.insert("\(reference.subject)@\(reference.version)").inserted else { continue } + guard seen.count <= Self.maxReferences else { + throw SchemaRegistryError(reason: String(localized: "The schema imports more files than TablePro reads.")) + } + let version: SchemaResponse = try await get(["subjects", reference.subject, "versions", String(reference.version)]) + try await collect(version.references ?? [], into: &texts, seen: &seen) + texts.append(version.schema) + } + } + + private func get(_ components: [String]) async throws -> Response { + let data: Data + let response: URLResponse + do { + (data, response) = try await transport(request(components)) + } catch { + throw SchemaRegistryError(reason: String( + format: String(localized: "TablePro could not reach the Schema Registry: %@"), error.localizedDescription + )) + } + guard data.count <= Self.maxResponseBytes else { + throw SchemaRegistryError(reason: String(localized: "The Schema Registry's answer is larger than any schema.")) + } + let status = (response as? HTTPURLResponse)?.statusCode ?? 200 + guard (200..<300).contains(status) else { + let message = (try? JSONDecoder().decode(ErrorResponse.self, from: data))?.message + ?? HTTPURLResponse.localizedString(forStatusCode: status) + throw SchemaRegistryError(reason: String( + format: String(localized: "The Schema Registry answered %lld: %@"), status, message + )) + } + do { + return try JSONDecoder().decode(Response.self, from: data) + } catch { + throw SchemaRegistryError(reason: String(localized: "The Schema Registry's answer is not a schema.")) + } + } + + // Each component is encoded alone: a subject such as `demo/common.proto` holds a slash. + func request(_ components: [String]) -> URLRequest { + var allowed = CharacterSet.urlPathAllowed + allowed.remove("/") + let base = endpoint.baseURL.absoluteString.hasSuffix("/") + ? String(endpoint.baseURL.absoluteString.dropLast()) + : endpoint.baseURL.absoluteString + let path = components + .map { $0.addingPercentEncoding(withAllowedCharacters: allowed) ?? $0 } + .joined(separator: "/") + var request = URLRequest(url: URL(string: "\(base)/\(path)") ?? endpoint.baseURL, timeoutInterval: Self.timeout) + request.setValue("application/vnd.schemaregistry.v1+json, application/json", forHTTPHeaderField: "Accept") + if let username = endpoint.username { + let credentials = Data("\(username):\(endpoint.password ?? "")".utf8).base64EncodedString() + request.setValue("Basic \(credentials)", forHTTPHeaderField: "Authorization") + } + return request + } + + private struct SchemaResponse: Decodable { + let schema: String + let schemaType: String? + let references: [SchemaReference]? + } + + private struct SchemaReference: Decodable { + let subject: String + let version: Int + } + + private struct ErrorResponse: Decodable { + let message: String + } +} diff --git a/apple/TablePro/Core/Storage/AvroSchemaStore.swift b/apple/TablePro/Core/Storage/AvroSchemaStore.swift index 3b3bdb9a1a..2527c70379 100644 --- a/apple/TablePro/Core/Storage/AvroSchemaStore.swift +++ b/apple/TablePro/Core/Storage/AvroSchemaStore.swift @@ -3,13 +3,8 @@ // TablePro // -import CryptoKit import Foundation -import os -// The schema files chosen for Avro columns, one file each, named by a hash of the text, so a column still -// decodes after the original file moves and the same file chosen twice is kept once. Read from the grid on -// the main actor and from the viewer off it. internal final class AvroSchemaStore: Sendable { static let shared = AvroSchemaStore( directory: AppStorageEnvironment.shared.supportDirectory.appendingPathComponent("AvroSchemas", isDirectory: true) @@ -22,47 +17,27 @@ internal final class AvroSchemaStore: Sendable { static let didRestoreSchema = Notification.Name("com.TablePro.avroSchemaStore.didRestoreSchema") static let schemaIDKey = "schemaID" - private static let fileExtension = "avsc" - - private struct State { - var parsed: [String: AvroSchema] = [:] - // Kept from a sweep: a schema can be added before the column's format is saved. - var addedThisLaunch: Set = [] - // Asked for and not found, so adding one later is a restore that grids and viewers have to hear about. - var missing: Set = [] - } - - private let directory: URL - private let state = OSAllocatedUnfairLock(initialState: State()) + private let files: SchemaFileCache init(directory: URL) { - self.directory = directory + files = SchemaFileCache(directory: directory, fileExtension: "avsc") { data in + guard let text = String(data: data, encoding: .utf8) else { + throw AvroSchemaError(reason: String(localized: "The file is not UTF-8 text.")) + } + return try AvroSchema.parse(text) + } } static func isSchemaID(_ text: String) -> Bool { - text.utf8.count == 32 && text.utf8.allSatisfy { (0x30...0x39).contains($0) || (0x61...0x66).contains($0) } + SchemaFileCache.isSchemaID(text) } static func schemaID(of text: String) -> String { - SHA256.hash(data: Data(text.utf8)).prefix(16).map { String(format: "%02x", $0) }.joined() + SchemaFileCache.schemaID(of: Data(text.utf8)) } - // Parses first, so nothing is kept for a file that is not a schema. func add(_ text: String) throws -> String { - let schema = try AvroSchema.parse(text) - let id = Self.schemaID(of: text) - let url = fileURL(id) - // A copy that went bad on disk is written over, or the next launch would find it missing again. - let contents = Data(text.utf8) - if (try? Data(contentsOf: url)) != contents { - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - try contents.write(to: url, options: .atomic) - } - let wasMissing = state.withLock { state -> Bool in - state.parsed[id] = schema - state.addedThisLaunch.insert(id) - return state.missing.remove(id) != nil - } + let (id, wasMissing) = try files.add(Data(text.utf8)) if wasMissing { NotificationCenter.default.post(name: Self.didRestoreSchema, object: self, userInfo: [Self.schemaIDKey: id]) } @@ -70,41 +45,14 @@ internal final class AvroSchemaStore: Sendable { } func schema(id: String) -> AvroSchema? { - if let cached = state.withLock({ $0.parsed[id] }) { return cached } - guard Self.isSchemaID(id), - let data = try? Data(contentsOf: fileURL(id)), - let text = String(data: data, encoding: .utf8), - let schema = try? AvroSchema.parse(text) else { - // An add() may have landed since the read failed; recorded as missing only if it did not. - return state.withLock { state in - if let added = state.parsed[id] { return added } - state.missing.insert(id) - return nil - } - } - state.withLock { $0.parsed[id] = schema } - return schema + files.schema(id: id) } var storedIDs: Set { - let names = (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] - return Set(names.compactMap { name in - let url = URL(fileURLWithPath: name) - guard url.pathExtension == Self.fileExtension else { return nil } - let id = url.deletingPathExtension().lastPathComponent - return Self.isSchemaID(id) ? id : nil - }) + files.storedIDs } func removeAll(except kept: Set) { - let addedThisLaunch = state.withLock { $0.addedThisLaunch } - for id in storedIDs where !kept.contains(id) && !addedThisLaunch.contains(id) { - try? FileManager.default.removeItem(at: fileURL(id)) - state.withLock { _ = $0.parsed.removeValue(forKey: id) } - } - } - - private func fileURL(_ id: String) -> URL { - directory.appendingPathComponent(id, isDirectory: false).appendingPathExtension(Self.fileExtension) + files.removeAll(except: kept) } } diff --git a/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift b/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift new file mode 100644 index 0000000000..5646f2d8a8 --- /dev/null +++ b/apple/TablePro/Core/Storage/ProtobufSchemaStore.swift @@ -0,0 +1,45 @@ +// +// ProtobufSchemaStore.swift +// TablePro +// + +import Foundation + +internal final class ProtobufSchemaStore: Sendable { + static let shared = ProtobufSchemaStore( + directory: AppStorageEnvironment.shared.supportDirectory.appendingPathComponent("ProtobufSchemas", isDirectory: true) + ) + + static let maxFileBytes = 4_000_000 + + static let didRestoreSchema = Notification.Name("com.TablePro.protobufSchemaStore.didRestoreSchema") + static let schemaIDKey = "schemaID" + + private let files: SchemaFileCache + + init(directory: URL) { + files = SchemaFileCache(directory: directory, fileExtension: "pbschema") { try ProtobufSchema.parse($0) } + } + + func add(_ contents: Data) throws -> String { + let (id, wasMissing) = try files.add(contents) + if wasMissing { + NotificationCenter.default.post( + name: Self.didRestoreSchema, object: self, userInfo: [Self.schemaIDKey: id] + ) + } + return id + } + + func schema(id: String) -> ProtobufSchema? { + files.schema(id: id) + } + + var storedIDs: Set { + files.storedIDs + } + + func removeAll(except kept: Set) { + files.removeAll(except: kept) + } +} diff --git a/apple/TablePro/Core/Storage/SchemaFileCache.swift b/apple/TablePro/Core/Storage/SchemaFileCache.swift new file mode 100644 index 0000000000..2a54eded99 --- /dev/null +++ b/apple/TablePro/Core/Storage/SchemaFileCache.swift @@ -0,0 +1,97 @@ +// +// SchemaFileCache.swift +// TablePro +// + +import CryptoKit +import Foundation +import os + +// Named by a hash of the contents, so a column still decodes after the original file moves. +internal final class SchemaFileCache: Sendable { + private struct State { + var parsed: [String: Schema] = [:] + // Kept from a sweep: a schema can be added before the column's format is saved. + var addedThisLaunch: Set = [] + // Also answers later reads without the disk, since the grid asks once per cell. + var missing: Set = [] + } + + private let directory: URL + private let fileExtension: String + private let parse: @Sendable (Data) throws -> Schema + private let state = OSAllocatedUnfairLock(initialState: State()) + + init(directory: URL, fileExtension: String, parse: @escaping @Sendable (Data) throws -> Schema) { + self.directory = directory + self.fileExtension = fileExtension + self.parse = parse + } + + static func isSchemaID(_ text: String) -> Bool { + text.utf8.count == 32 && text.utf8.allSatisfy { (0x30...0x39).contains($0) || (0x61...0x66).contains($0) } + } + + static func schemaID(of contents: Data) -> String { + SHA256.hash(data: contents).prefix(16).map { String(format: "%02x", $0) }.joined() + } + + func add(_ contents: Data) throws -> (id: String, wasMissing: Bool) { + let schema = try parse(contents) + let id = Self.schemaID(of: contents) + let url = fileURL(id) + // A copy that went bad on disk is written over, or the next launch would find it missing again. + if (try? Data(contentsOf: url)) != contents { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try contents.write(to: url, options: .atomic) + } + let wasMissing = state.withLock { state -> Bool in + state.parsed[id] = schema + state.addedThisLaunch.insert(id) + return state.missing.remove(id) != nil + } + return (id, wasMissing) + } + + func schema(id: String) -> Schema? { + let known = state.withLock { state -> (schema: Schema?, isMissing: Bool) in + (state.parsed[id], state.missing.contains(id)) + } + if let cached = known.schema { return cached } + if known.isMissing { return nil } + guard Self.isSchemaID(id), + let data = try? Data(contentsOf: fileURL(id)), + let schema = try? parse(data) else { + // An add() may have landed since the read failed; recorded as missing only if it did not. + return state.withLock { state in + if let added = state.parsed[id] { return added } + state.missing.insert(id) + return nil + } + } + state.withLock { $0.parsed[id] = schema } + return schema + } + + var storedIDs: Set { + let names = (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] + return Set(names.compactMap { name in + let url = URL(fileURLWithPath: name) + guard url.pathExtension == fileExtension else { return nil } + let id = url.deletingPathExtension().lastPathComponent + return Self.isSchemaID(id) ? id : nil + }) + } + + func removeAll(except kept: Set) { + let addedThisLaunch = state.withLock { $0.addedThisLaunch } + for id in storedIDs where !kept.contains(id) && !addedThisLaunch.contains(id) { + try? FileManager.default.removeItem(at: fileURL(id)) + state.withLock { _ = $0.parsed.removeValue(forKey: id) } + } + } + + private func fileURL(_ id: String) -> URL { + directory.appendingPathComponent(id, isDirectory: false).appendingPathExtension(fileExtension) + } +} diff --git a/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift b/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift index fbc2b90b7b..9921cf8941 100644 --- a/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift +++ b/apple/TablePro/Core/Storage/ValueDisplayFormatStorage.swift @@ -19,10 +19,16 @@ internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { private let store: KeyValueStore private let schemas: AvroSchemaStore + private let protobufSchemas: ProtobufSchemaStore - init(defaults: KeyValueStore = AppStorageEnvironment.shared.defaults, schemas: AvroSchemaStore = .shared) { + init( + defaults: KeyValueStore = AppStorageEnvironment.shared.defaults, + schemas: AvroSchemaStore = .shared, + protobufSchemas: ProtobufSchemaStore = .shared + ) { store = defaults self.schemas = schemas + self.protobufSchemas = protobufSchemas } func save(_ formats: [String: ValueDisplayFormat], for scope: TableScope) { @@ -108,14 +114,20 @@ internal final class ValueDisplayFormatStorage: TableScopedSettingsStore { // A schema file lives as long as a saved format names it. private func sweepSchemas() { - guard !schemas.storedIDs.isEmpty else { return } - var named: Set = [] + guard !schemas.storedIDs.isEmpty || !protobufSchemas.storedIDs.isEmpty else { return } + var avro: Set = [] + var protobuf: Set = [] for key in store.keys(withPrefix: Self.keyPrefix) where key.hasSuffix(Self.addedSuffix) { for name in (Self.names(store.dataValue(forKey: key)) ?? [:]).values { - if case .avro(let schemaID)? = ValueDisplayFormat(rawValue: name) { named.insert(schemaID) } + switch ValueDisplayFormat(rawValue: name) { + case .avro(let schemaID)?: avro.insert(schemaID) + case .protobufSchema(let schemaID, _)?: protobuf.insert(schemaID) + default: break + } } } - schemas.removeAll(except: named) + schemas.removeAll(except: avro) + protobufSchemas.removeAll(except: protobuf) } @discardableResult diff --git a/apple/TablePro/Models/UI/ProtobufTreeNode.swift b/apple/TablePro/Models/UI/ProtobufTreeNode.swift index 737364b0d8..c3f67de671 100644 --- a/apple/TablePro/Models/UI/ProtobufTreeNode.swift +++ b/apple/TablePro/Models/UI/ProtobufTreeNode.swift @@ -5,7 +5,7 @@ import Foundation -// Without a schema the bytes give only a field number, the key, and a wire type, the badge. +// Without a schema the key is the field number and the badge the wire type; a schema gives name and declared type. internal struct ProtobufTreeNode: Identifiable { internal enum Kind: Equatable { case message @@ -61,6 +61,8 @@ internal struct ProtobufTreeNode: Identifiable { let isDisplayCut: Bool let rawValue: String? let members: [ProtobufField] + let typeBadge: String? + let typedMembers: [ProtobufTypedField] let children: [ProtobufTreeNode] init( @@ -73,6 +75,8 @@ internal struct ProtobufTreeNode: Identifiable { isDisplayCut: Bool = false, rawValue: String? = nil, members: [ProtobufField] = [], + typeBadge: String? = nil, + typedMembers: [ProtobufTypedField] = [], children: [ProtobufTreeNode] = [] ) { self.id = id @@ -84,6 +88,8 @@ internal struct ProtobufTreeNode: Identifiable { self.isDisplayCut = isDisplayCut self.rawValue = rawValue self.members = members + self.typeBadge = typeBadge + self.typedMembers = typedMembers self.children = children } } @@ -101,7 +107,7 @@ extension ProtobufTreeNode: FilterableTreeNode { } internal var badgeLabel: String { - kind.badgeLabel + typeBadge ?? kind.badgeLabel } internal var isTruncationMarker: Bool { @@ -111,6 +117,7 @@ extension ProtobufTreeNode: FilterableTreeNode { internal var copyableValue: String { switch kind { case .message, .group: + guard typedMembers.isEmpty else { return ProtobufSchemaTextWriter.text(typedMembers, multiLine: false) } return ProtobufTextWriter.text(members, multiLine: false) case .varint, .fixed64, .fixed32, .string, .bytes, .truncated: return rawValue ?? displayValue @@ -131,7 +138,7 @@ extension ProtobufTreeNode: FilterableTreeNode { string: kind == .string ? rawValue : nil, isCut: isDisplayCut, tone: kind.tone, - typeBadge: kind.badgeLabel, + typeBadge: badgeLabel, visibilityBadge: nil ) } @@ -140,7 +147,7 @@ extension ProtobufTreeNode: FilterableTreeNode { ProtobufTreeNode( id: id, key: key, keyPath: keyPath, path: path, kind: kind, displayValue: displayValue, isDisplayCut: isDisplayCut, rawValue: rawValue, members: members, - children: children + typeBadge: typeBadge, typedMembers: typedMembers, children: children ) } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 8aef509209..51a2c90290 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -954,6 +954,9 @@ }, "“%@” is not a usable Avro schema." : { + }, + "“%@” is not a usable Protobuf schema." : { + }, "\"%@\" is not a valid %@. Example: %@" : { "extractionState" : "manual", @@ -15206,6 +15209,9 @@ } } } + }, + "A comment is never closed." : { + }, "A comparison is already running." : { "localizations" : { @@ -15967,6 +15973,9 @@ } } } + }, + "A message is never closed." : { + }, "A MongoDB field has no type, default or nullability to change. Only its name can change." : { "extractionState" : "manual", @@ -16945,6 +16954,9 @@ } } } + }, + "A string is never closed." : { + }, "A string literal is missing its closing quote." : { "extractionState" : "manual", @@ -24166,6 +24178,9 @@ "An auto-increment column takes its values from a sequence, so it cannot also have a default." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "An enum is never closed." : { + }, "An enum's labels in declaration order" : { "localizations" : { @@ -38538,6 +38553,9 @@ }, "Choose Schema" : { + }, + "Choose the .proto file or descriptor set the values in this column were written with." : { + }, "Choose the Avro schema the values in this column were written with." : { @@ -109154,6 +109172,9 @@ } } } + }, + "Messages are nested too deeply." : { + }, "Metadata" : { "localizations" : { @@ -138498,6 +138519,12 @@ } } } + }, + "Protobuf (%@)" : { + + }, + "Protobuf Schema…" : { + }, "Protocol" : { "extractionState" : "manual", @@ -157051,6 +157078,16 @@ } } }, + "Schema %lld could not be read from the Schema Registry. %@ Showing the stored bytes." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Schema %1$lld could not be read from the Schema Registry. %2$@ Showing the stored bytes." + } + } + } + }, "Schema and table privileges in this database are only visible when the connection is using it." : { "localizations" : { "fr" : { @@ -157548,6 +157585,9 @@ } } } + }, + "Schema Registry" : { + }, "Schema Registry framing: schema ID %lld" : { @@ -157562,6 +157602,18 @@ } } }, + "Schema Registry Password" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Schema Registry URL" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Schema Registry User" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Schema response was empty." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -179644,6 +179696,9 @@ } } } + }, + "TablePro could not reach the Schema Registry: %@" : { + }, "TablePro could not read the database's own name from the connection." : { "localizations" : { @@ -185193,6 +185248,9 @@ } } } + }, + "The connection names no Schema Registry URL. Add one in the connection's settings." : { + }, "The connection stopped responding." : { "localizations" : { @@ -186145,6 +186203,9 @@ } } } + }, + "The descriptor set names an unknown field type %llu." : { + }, "The destination has no table named %@" : { "localizations" : { @@ -186767,10 +186828,19 @@ } } } + }, + "The file defines more than one message. Choose the one this column holds." : { + + }, + "The file defines no message." : { + }, "The file does not declare a theme format version." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The file ends in the middle of a statement." : { + }, "The file goes back to its last committed version. You can't undo this action." : { "localizations" : { @@ -186899,6 +186969,12 @@ }, "The file is larger than any Avro schema." : { + }, + "The file is larger than any schema." : { + + }, + "The file is neither a .proto file nor a descriptor set." : { + }, "The file is not a JSON object." : { @@ -186919,6 +186995,9 @@ } } } + }, + "The file is too large." : { + }, "The file is too large. A theme file can be at most %@." : { "extractionState" : "manual", @@ -190446,18 +190525,46 @@ }, "The schema changes staged in this diagram will be lost." : { + }, + "The schema defines more messages than TablePro reads." : { + }, "The schema defines type %@ twice." : { }, "The schema has a type that is not a name, an object or a union." : { + }, + "The schema imports more files than TablePro reads." : { + }, "The schema is not valid JSON." : { + }, + "The schema is still being fetched." : { + }, "The schema names type %@ before defining it." : { + }, + "The Schema Registry answered %lld: %@" : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "The Schema Registry answered %1$lld: %2$@" + } + } + } + }, + "The Schema Registry names a schema type TablePro does not read: %@." : { + + }, + "The Schema Registry's answer is larger than any schema." : { + + }, + "The Schema Registry's answer is not a schema." : { + }, "The script already ran. Compare again to see where the target stands." : { "localizations" : { @@ -192996,6 +193103,9 @@ } } } + }, + "The text at byte %lld is not valid UTF-8." : { + }, "The text could not be parsed as JSON." : { "localizations" : { @@ -193663,6 +193773,9 @@ } } } + }, + "The value does not start with the Schema Registry prefix: a zero byte and a schema ID." : { + }, "The value ends early, at byte %lld." : { @@ -198370,6 +198483,9 @@ }, "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…" : { + }, + "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Protobuf Schema…" : { + }, "This Mac no longer holds a seat." : { "localizations" : { @@ -205218,6 +205334,9 @@ } } } + }, + "Unexpected “%@” in the .proto file." : { + }, "Unexpected %@ in a CONSUME statement." : { "extractionState" : "manual", @@ -205242,6 +205361,9 @@ } } } + }, + "Unexpected character “%@”." : { + }, "Unexpected InfluxDB response: %@" : { "extractionState" : "manual", @@ -214266,6 +214388,9 @@ } } } + }, + "Which message are the values?" : { + }, "Whitespace in text" : { "localizations" : { diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift index 04815f481b..84d08cd831 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+HeaderMenu.swift @@ -87,8 +87,11 @@ extension DataGridViewController { } private func displayAsItem(for column: DataGridColumn) -> NSMenuItem? { - let formats = ValueDisplayFormat.applicableFormats(for: column.type, databaseType: model.configuration.traits.databaseType) - guard formats.count > 1 else { return nil } + let databaseType = model.configuration.traits.databaseType + let formats = ValueDisplayFormat.applicableFormats(for: column.type, databaseType: databaseType) + guard formats.count > 1 || schemaRegistryFormat?.isApplicable(to: column.type, databaseType: databaseType) == true else { + return nil + } let current = model.displayFormat(for: column) ?? .raw let submenu = NSMenu() for format in formats { @@ -100,7 +103,7 @@ extension DataGridViewController { item.state = format == current ? .on : .off submenu.addItem(item) } - if ValueDisplayFormat.offersAvro(for: column.type, databaseType: model.configuration.traits.databaseType) { + if ValueDisplayFormat.offersAvro(for: column.type, databaseType: databaseType) { if case .avro = current { let item = menuItem( current.displayName, @@ -112,11 +115,38 @@ extension DataGridViewController { } submenu.addItem(menuItem(String(localized: "Avro Schema…"), #selector(chooseAvroSchemaFromMenu(_:)), column.id)) } + if ValueDisplayFormat.offersProtobufSchema(for: column.type, databaseType: databaseType) { + if case .protobufSchema = current { + let item = menuItem( + current.displayName, + #selector(setDisplayFormatFromMenu(_:)), + DataGridDisplayFormatChoice(columnID: column.id, format: current) + ) + item.state = .on + submenu.addItem(item) + } + submenu.addItem(menuItem(String(localized: "Protobuf Schema…"), #selector(chooseProtobufSchemaFromMenu(_:)), column.id)) + } + if let registry = schemaRegistryFormat, registry.isApplicable(to: column.type, databaseType: databaseType) { + let item = menuItem( + registry.displayName, + #selector(setDisplayFormatFromMenu(_:)), + DataGridDisplayFormatChoice(columnID: column.id, format: registry) + ) + item.state = registry == current ? .on : .off + submenu.addItem(item) + } let item = NSMenuItem(title: String(localized: "Display As"), action: nil, keyEquivalent: "") item.submenu = submenu return item } + private var schemaRegistryFormat: ValueDisplayFormat? { + guard let connectionID = model.configuration.traits.connectionScope?.connectionId, + SchemaRegistryEndpoint.isConfigured(for: connectionID) else { return nil } + return .schemaRegistry(connectionID: connectionID) + } + // Refused, with the reason, when the frozen run would leave too little of the grid to scroll. private func freezeItems(position: Int, columnID: DataGridColumnID) -> [NSMenuItem] { var items: [NSMenuItem] = [] @@ -155,6 +185,7 @@ extension DataGridViewController { #selector(DataGridViewController.clearAllValueFiltersFromMenu(_:)), #selector(DataGridViewController.setDisplayFormatFromMenu(_:)), #selector(DataGridViewController.chooseAvroSchemaFromMenu(_:)), + #selector(DataGridViewController.chooseProtobufSchemaFromMenu(_:)), #selector(DataGridViewController.sizeColumnToFitFromMenu(_:)), #selector(DataGridViewController.sizeAllColumnsToFitFromMenu(_:)), #selector(DataGridViewController.hideColumnFromMenu(_:)), @@ -267,6 +298,19 @@ extension DataGridViewController { } } + @objc func chooseProtobufSchemaFromMenu(_ sender: Any?) { + guard let columnID = menuColumn(sender)?.id, let window = view.window else { return } + Task { @MainActor [weak self] in + guard let choice = await ProtobufSchemaFilePanel.chooseSchema(in: window) else { return } + do { + let schemaID = try ProtobufSchemaStore.shared.add(choice.contents) + self?.setDisplayFormat(.protobufSchema(schemaID: schemaID, message: choice.message), columnID: columnID) + } catch { + NSAlert(error: error).beginSheetModal(for: window, completionHandler: nil) + } + } + } + @objc func sizeColumnToFitFromMenu(_ sender: Any?) { guard let column = menuColumn(sender) else { return } sizeColumnToFit(column) diff --git a/apple/TablePro/Views/DataGrid/Editing/DataGridViewController+TypedEditors.swift b/apple/TablePro/Views/DataGrid/Editing/DataGridViewController+TypedEditors.swift index a0a4227f07..b7cb10390b 100644 --- a/apple/TablePro/Views/DataGrid/Editing/DataGridViewController+TypedEditors.swift +++ b/apple/TablePro/Views/DataGrid/Editing/DataGridViewController+TypedEditors.swift @@ -373,7 +373,9 @@ extension DataGridViewController { switch cell.value { case .null: return case .bytes(let bytes): data = bytes - case .text(let text): data = text.storedBytes + case .text(let text): + let holdsBytes = BinaryValueBytes.holdsBytes(columnType: cell.column.type) + data = BinaryValueBytes.data(of: text, holdsBytes: holdsBytes) ?? text.storedBytes } let name = cell.column.name presentTypedPopover(for: cell, edits: false, contentSize: NSSize(width: 560, height: 360)) { dismiss in diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index e0e6df95ae..1e0c45af07 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -48,6 +48,16 @@ internal struct BinaryValueViewerView: View { notification.userInfo?[AvroSchemaStore.schemaIDKey] as? String == schemaID else { return } schemaRevision += 1 } + .onReceive(NotificationCenter.default.publisher(for: ProtobufSchemaStore.didRestoreSchema).receive(on: RunLoop.main)) { notification in + guard case .protobufSchema(let schemaID, _) = format, + notification.userInfo?[ProtobufSchemaStore.schemaIDKey] as? String == schemaID else { return } + schemaRevision += 1 + } + .onReceive(NotificationCenter.default.publisher(for: SchemaRegistryCache.didChange).receive(on: RunLoop.main)) { notification in + guard case .schemaRegistry(let connectionID) = format, + notification.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID == connectionID else { return } + schemaRevision += 1 + } } private struct TaskKey: Equatable { @@ -110,10 +120,20 @@ internal struct BinaryValueViewerView: View { hexBody } case .missingSchema?: + VStack(spacing: 0) { + banner(missingSchemaMessage, systemImage: "questionmark.folder") + Divider() + hexBody + } + case .registryUnavailable(let schemaID, let reason)?: VStack(spacing: 0) { banner( - String(localized: "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…"), - systemImage: "questionmark.folder" + String( + format: String(localized: "Schema %lld could not be read from the Schema Registry. %@ Showing the stored bytes."), + Int(schemaID), + reason + ), + systemImage: "exclamationmark.triangle" ) Divider() hexBody @@ -173,6 +193,13 @@ internal struct BinaryValueViewerView: View { HexDumpDisplayView(text: hexDump, font: themeEngine.valueFont) } + private var missingSchemaMessage: String { + if case .protobufSchema = format { + return String(localized: "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Protobuf Schema…") + } + return String(localized: "This Mac no longer has the schema chosen for this column, so the stored bytes show. Choose it again with Display As > Avro Schema…") + } + private func failureBanner(_ error: BinaryValueDecodeError) -> some View { banner( String(format: String(localized: "Not valid %@. %@ Showing the stored bytes."), format.displayName, error.reason), @@ -202,6 +229,15 @@ internal struct BinaryValueViewerView: View { guard decodedKey != key else { return } let data = data let format = format + // A new value never shows the last one's document while its schema is fetched. + if decodedKey?.data != data || decodedKey?.format != format { + document = nil + hexDump = "" + } + if case .schemaRegistry(let connectionID) = format, let schemaID = try? SchemaRegistryFrame.schemaID(data) { + _ = await SchemaRegistryCache.shared.schema(connectionID: connectionID, schemaID: schemaID) + guard !Task.isCancelled else { return } + } let (decoded, dump) = await Task.detached(priority: .userInitiated) { ( BinaryValueDecoding.document(data, format: format), diff --git a/apple/TablePro/Views/Results/CellInteractionResolver.swift b/apple/TablePro/Views/Results/CellInteractionResolver.swift index 5c9be4a713..83255a105f 100644 --- a/apple/TablePro/Views/Results/CellInteractionResolver.swift +++ b/apple/TablePro/Views/Results/CellInteractionResolver.swift @@ -75,8 +75,9 @@ internal struct CellInteractionResolver { let holdsBytes = context.columnType?.isBlobType == true || context.isBinaryValue // Read-only: nothing writes a decoded tree back. A NULL keeps the hex editor, the way bytes get into it. - if holdsBytes, context.isBinaryValue || context.value != nil, - let format = context.displayFormatOverride?.binaryValueFormat { + if let override = context.displayFormatOverride, let format = override.binaryValueFormat, + context.isBinaryValue || context.value != nil, + holdsBytes || override.isApplicable(to: context.columnType) { return .viewBinaryValue(format) } @@ -99,7 +100,7 @@ internal struct CellInteractionResolver { case .phpSerialized: return .viewPhpSerialized case .raw, .text, .uuid, .uuidSwappedTime, .unixTimestamp, .unixTimestampMillis, .messagePack, .protobuf, - .igbinary, .avro, .none: + .igbinary, .avro, .protobufSchema, .schemaRegistry, .none: return plainText(for: context, isReadOnly: isReadOnly) } } diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/BinaryValueFieldView.swift b/apple/TablePro/Views/RowInspector/FieldEditors/BinaryValueFieldView.swift index da317d0891..46212c9190 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/BinaryValueFieldView.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/BinaryValueFieldView.swift @@ -23,7 +23,7 @@ internal struct BinaryValueFieldView: View { .padding(.vertical, 3) } else { BinaryValueViewerView( - data: context.value.wrappedValue.storedBytes, + data: bytes, format: format, onPopOut: onPopOut.map { popOut in { popOut(context.value.wrappedValue) } } ) @@ -33,4 +33,10 @@ internal struct BinaryValueFieldView: View { } .valueWell(clipsContent: true) } + + private var bytes: Data { + let text = context.value.wrappedValue + let holdsBytes = BinaryValueBytes.holdsBytes(columnType: context.columnType, isBinaryValue: context.isBinaryValue) + return BinaryValueBytes.data(of: text, holdsBytes: holdsBytes) ?? text.storedBytes + } } diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift b/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift index 46174a22b3..a8e66e7b6b 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift @@ -25,6 +25,7 @@ internal struct FieldEditorContext { // False where the store has no NULL or DEFAULT to write, as in a data file. internal let allowsNullAndDefault: Bool + internal let isBinaryValue: Bool internal init( columnName: String, @@ -36,7 +37,8 @@ internal struct FieldEditorContext { isReadOnly: Bool, commitBytes: ((Data) -> Void)? = nil, editor: FieldEditorKind? = nil, - allowsNullAndDefault: Bool = true + allowsNullAndDefault: Bool = true, + isBinaryValue: Bool = false ) { self.columnName = columnName self.columnType = columnType @@ -48,6 +50,7 @@ internal struct FieldEditorContext { self.commitBytes = commitBytes self.editor = editor self.allowsNullAndDefault = allowsNullAndDefault + self.isBinaryValue = isBinaryValue } internal var hasMultipleValues: Bool { valueState == .multipleValues } diff --git a/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift b/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift index c3ec4ce0c4..6f8a1f812c 100644 --- a/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift +++ b/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift @@ -179,7 +179,8 @@ internal struct InspectorFieldListView: View { ? { editState.setFieldToBytes(at: field.columnIndex, data: $0) } : nil, editor: kind, - allowsNullAndDefault: offersDatabaseValues + allowsNullAndDefault: offersDatabaseValues, + isBinaryValue: field.isBinaryValue ) } diff --git a/apple/TablePro/Views/RowInspector/RowInspectorView.swift b/apple/TablePro/Views/RowInspector/RowInspectorView.swift index 03a8555d7f..112c74b661 100644 --- a/apple/TablePro/Views/RowInspector/RowInspectorView.swift +++ b/apple/TablePro/Views/RowInspector/RowInspectorView.swift @@ -158,7 +158,12 @@ internal struct RowInspectorView: View { case .php: PhpViewerWindowController.open(text: text, columnName: field.columnName) case .binary(let format): - BinaryValueViewerWindowController.open(data: text.storedBytes, format: format, columnName: field.columnName) + let holdsBytes = BinaryValueBytes.holdsBytes(columnType: field.columnTypeEnum, isBinaryValue: field.isBinaryValue) + BinaryValueViewerWindowController.open( + data: BinaryValueBytes.data(of: text, holdsBytes: holdsBytes) ?? text.storedBytes, + format: format, + columnName: field.columnName + ) case .text: TextViewerWindowController.open( text: text, diff --git a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift index 13bed1dadc..c98ff27917 100644 --- a/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift +++ b/apple/TablePro/Views/Shared/FieldEditors/FieldEditorResolver.swift @@ -51,9 +51,10 @@ internal enum FieldEditorResolver { return .phpSerialized case .json: return .json - case .messagePack, .protobuf, .igbinary, .avro: + case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: // A NULL keeps the hex editor, the way bytes get into it. - if type.isBlobType || isBinaryValue, originalValue != nil, let format = override.binaryValueFormat { + if type.isBlobType || isBinaryValue || override.isApplicable(to: type), originalValue != nil, + let format = override.binaryValueFormat { return .binaryValue(format) } structuredAllowed = true diff --git a/apple/TableProTests/Core/Network/SameOriginRedirectGuardTests.swift b/apple/TableProTests/Core/Network/SameOriginRedirectGuardTests.swift new file mode 100644 index 0000000000..14b1ad02ea --- /dev/null +++ b/apple/TableProTests/Core/Network/SameOriginRedirectGuardTests.swift @@ -0,0 +1,31 @@ +// +// SameOriginRedirectGuardTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct SameOriginRedirectGuardTests { + private static func allows(_ origin: String, _ target: String) throws -> Bool { + SameOriginRedirectGuard(origin: try #require(URL(string: origin))).allows(try #require(URL(string: target))) + } + + @Test("A redirect on the same scheme, host and port is followed, the default port written or not") + func sameOrigin() throws { + #expect(try Self.allows("https://registry.example.com/base", "https://REGISTRY.example.com/other")) + #expect(try Self.allows("http://registry.example.com", "http://registry.example.com:80/schemas")) + #expect(try Self.allows("https://registry.example.com:443/", "https://registry.example.com/schemas")) + #expect(try Self.allows("http://localhost:8081", "http://localhost:8081/subjects")) + } + + @Test("A redirect to another host, port or scheme is refused, a downgrade to http included") + func otherOrigin() throws { + #expect(try !Self.allows("https://registry.example.com", "https://evil.example.com")) + #expect(try !Self.allows("https://registry.example.com", "http://registry.example.com")) + #expect(try !Self.allows("http://localhost:8081", "http://localhost:8082")) + #expect(try !Self.allows("https://registry.example.com", "https://registry.example.com:8443")) + #expect(try !Self.allows("https://registry.example.com", "file:///etc/passwd")) + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift index 44057ed83b..dce228b129 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/BinaryDisplayFormatTests.swift @@ -175,4 +175,15 @@ struct BinaryDisplayFormatTests { } #expect(BinaryValueDecoding.text(Data([0x02]), format: format) == nil) } + + @Test("A Protobuf schema column renders field names in the cell and falls back to hex on a broken value") + func protobufSchemaRenders() throws { + let blob = ColumnType.blob(rawType: "BLOB") + let schemaID = try ProtobufSchemaStore.shared.add(Data("syntax = \"proto3\"; message Point { sint32 x = 1; string label = 2; }".utf8)) + let format = ValueDisplayFormat.protobufSchema(schemaID: schemaID, message: "Point") + #expect(format.displayName.contains("Point")) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("08051202686b")), columnType: blob, displayFormat: format) + == #"x: -3 label: "hk""#) + #expect(CellDisplayFormatter.format(.bytes(BinaryFixture.data("0f")), columnType: blob, displayFormat: format) == "0x0F") + } } diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift new file mode 100644 index 0000000000..4054bfb2f2 --- /dev/null +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufSchemaDecoderTests.swift @@ -0,0 +1,747 @@ +// +// ProtobufSchemaDecoderTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +// libprotoc 36.2. Each fixture says how it was written; every expected text is `protoc --decode` output for it. +struct ProtobufSchemaDecoderTests { + private static let shopProto = """ + syntax = "proto3"; + package shop; + + message Item { + int32 i32 = 1; + int64 i64 = 2; + uint32 u32 = 3; + uint64 u64 = 4; + sint32 s32 = 5; + sint64 s64 = 6; + fixed32 f32 = 7; + fixed64 f64 = 8; + sfixed32 sf32 = 9; + sfixed64 sf64 = 10; + float fl = 11; + double db = 12; + bool flag = 13; + string name = 14; + bytes raw = 15; + Color color = 16; + repeated int32 packed = 17; + repeated string labels = 18; + map inners = 19; + Inner inner = 20; + oneof choice { + string text = 21; + int32 number = 22; + } + message Inner { + string note = 1; + repeated Color colors = 2; + } + } + + enum Color { + RED = 0; + GREEN = 1; + BLUE = 2; + } + + message Other { + string x = 1; + } + """ + + // protoc --descriptor_set_out=shop.desc shop.proto + private static let shopDescriptorSet = "0ae7050a0a73686f702e70726f746f120473686f70228c050a044974656d12100a03693332180120012805520369333212100a036936341802200128" + + "03520369363412100a0375333218032001280d520375333212100a03753634180420012804520375363412100a037333321805200128115203733332" + + "12100a03733634180620012812520373363412100a03663332180720012807520366333212100a03663634180820012806520366363412120a047366" + + "333218092001280f52047366333212120a0473663634180a20012810520473663634120e0a02666c180b200128025202666c120e0a026462180c2001" + + "28015202646212120a04666c6167180d200128085204666c616712120a046e616d65180e2001280952046e616d6512100a03726177180f2001280c52" + + "0372617712210a05636f6c6f7218102001280e320b2e73686f702e436f6c6f725205636f6c6f7212160a067061636b65641811200328055206706163" + + "6b656412160a066c6162656c7318122003280952066c6162656c73122e0a06696e6e65727318132003280b32162e73686f702e4974656d2e496e6e65" + + "7273456e7472795206696e6e65727312260a05696e6e657218142001280b32102e73686f702e4974656d2e496e6e65725205696e6e657212140a0474" + + "657874181520012809480052047465787412180a066e756d626572181620012805480052066e756d6265721a4b0a0b496e6e657273456e7472791210" + + "0a036b657918012001280952036b657912260a0576616c756518022001280b32102e73686f702e4974656d2e496e6e6572520576616c75653a023801" + + "1a400a05496e6e657212120a046e6f746518012001280952046e6f746512230a06636f6c6f727318022003280e320b2e73686f702e436f6c6f725206" + + "636f6c6f727342080a0663686f69636522150a054f74686572120c0a01781801200128095201782a250a05436f6c6f7212070a03524544100012090a" + + "05475245454e100112080a04424c55451002620670726f746f33" + + // protoc --encode=shop.Item shop.proto + private static let item = "08fbffffffffffffffff011080ccbbbcdeffffffff011880d0acf30e208080a0a89c94b6e6f901280d30ffef99c589043d7b00000041c801000000" + + "0000004df8ffffff51f7ffffffffffffff5d0000c03f619a9999999999b93f680172075461622250726f7a030102ff8001028a01040102ac0292010161" + + "920101629a010d0a026b3112070a016e12020102a201060a0464656570b0012a" + + private static let itemText = """ + i32: -5 + i64: -9000000000 + u32: 4000000000 + u64: 18000000000000000000 + s32: -7 + s64: -70000000000 + f32: 123 + f64: 456 + sf32: -8 + sf64: -9 + fl: 1.5 + db: 0.1 + flag: true + name: "Tab\\"Pro" + raw: "\\001\\002\\377" + color: BLUE + packed: 1 + packed: 2 + packed: 300 + labels: "a" + labels: "b" + inners { + key: "k1" + value { + note: "n" + colors: GREEN + colors: BLUE + } + } + inner { + note: "deep" + } + number: 42 + + """ + + private static func text(_ hex: String, schema: ProtobufSchema, message: String?, multiLine: Bool = true) throws -> String { + let decoded = try ProtobufSchemaDecoder.decode(BinaryFixture.data(hex), schema: schema, message: message) + return ProtobufSchemaTextWriter.text(decoded.fields, multiLine: multiLine) + } + + @Test("A .proto file reads every type the way protoc --decode prints it") + func protoFile() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + #expect(schema.messageNames == ["shop.Item", "shop.Item.Inner", "shop.Other"]) + #expect(try Self.text(Self.item, schema: schema, message: "shop.Item") == Self.itemText) + } + + @Test("A descriptor set from protoc reads the same as the .proto it was built from") + func descriptorSet() throws { + let schema = try ProtobufSchema.parse(BinaryFixture.data(Self.shopDescriptorSet)) + #expect(schema == (try ProtobufSchema.parse(Data(Self.shopProto.utf8)))) + #expect(try Self.text(Self.item, schema: schema, message: "shop.Item") == Self.itemText) + } + + @Test("Fields the message does not declare, or declares with another wire type, show by number") + func unknownFields() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + let text = try Self.text(Self.item, schema: schema, message: "shop.Other") + let lines = text.split(separator: "\n").map(String.init) + #expect(lines.prefix(3) == ["1: 18446744073709551611", "2: 18446744064709551616", "3: 4000000000"]) + #expect(lines.contains("7: 0x0000007b")) + #expect(lines.contains(#"17: "\001\002\254\002""#)) + #expect(lines.last == "22: 42") + } + + @Test("A proto2 group and packed and unpacked repeats") + func proto2() throws { + let proto = """ + syntax = "proto2"; + package legacy; + + message Msg { + optional int32 id = 1; + repeated group Entry = 2 { + optional string k = 3; + } + repeated int32 nums = 4 [packed = true]; + repeated int32 loose = 5; + optional float ratio = 6; + } + """ + let schema = try ProtobufSchema.parse(Data(proto.utf8)) + let expected = "id: 1\nEntry {\n k: \"a\"\n}\nEntry {\n k: \"b\"\n}\nnums: 5\nnums: 6\nloose: 7\nloose: 8\nratio: 0.1\n" + #expect(try Self.text("0801131a016114131a016214220205062807280835cdcccc3d", schema: schema, message: "legacy.Msg") == expected) + } + + @Test("The tree names each field and badges it with its declared type") + func tree() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(Self.item), schema: schema, message: "shop.Item") + let root = ProtobufSchemaTreeBuilder.build(from: message, formats: BinaryFixture.english) + #expect(root.badgeLabel == "Item") + let color = try #require(root.children.first { $0.key == "color" }) + #expect(color.badgeLabel == "Color") + #expect(color.displayValue == "BLUE") + let inner = try #require(root.children.first { $0.key == "inner" }) + #expect(inner.badgeLabel == "Inner") + #expect(inner.copyableValue == #"note: "deep""#) + #expect(root.children.first { $0.key == "s64" }?.badgeLabel == "sint64") + } + + @Test("The one-line text a cell shows") + func singleLine() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + #expect(try Self.text("a201060a0464656570", schema: schema, message: "shop.Item", multiLine: false) == #"inner { note: "deep" }"#) + } + + @Test("A singular message sent twice is merged, as a parser does") + func mergesSingularMessages() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + #expect(try Self.text("a201030a0161a20103120101", schema: schema, message: "shop.Item") + == "inner {\n note: \"a\"\n colors: GREEN\n}\n") + } + + @Test("A cell stops reading a packed field at its budget") + func packedFieldBudget() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + let packed = "8a01" + "c801" + String(repeating: "01", count: 200) + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(packed), schema: schema, message: "shop.Item", maxValues: 10) + #expect(message.fields.count < 20) + #expect(message.fields.last?.value == .truncated) + } + + @Test("A broken value is refused, so the cell falls back to hex") + func brokenValue() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + #expect(throws: BinaryValueDecodeError.self) { + try ProtobufSchemaDecoder.decode(BinaryFixture.data("0a05ab"), schema: schema, message: "shop.Item") + } + #expect(throws: BinaryValueDecodeError.self) { + try ProtobufSchemaDecoder.decode(BinaryFixture.data("0f"), schema: schema, message: "shop.Item") + } + } + + @Test("A file that is not a schema says why") + func invalidSchema() { + #expect(throws: ProtobufSchemaError.self) { try ProtobufSchema.parse(Data("message Item { int32 id = ; }".utf8)) } + #expect(throws: ProtobufSchemaError.self) { try ProtobufSchema.parse(Data("enum Only { A = 0; }".utf8)) } + #expect(throws: ProtobufSchemaError.self) { try ProtobufSchema.parse(Data([0xFF, 0x00, 0x13])) } + } + + private static let demoProto = """ + syntax = "proto3"; + package demo; + + message M { + map m = 1; + map im = 2; + oneof choice { + string text = 3; + int32 number = 4; + Inner pick = 5; + } + int32 count = 6; + optional int32 opt = 7; + string name = 8; + bytes data = 9; + E e = 10; + double ratio = 11; + Inner inner = 12; + } + + enum E { + E0 = 0; + E1 = 1; + } + + message Inner { + int32 x = 1; + int32 y = 2; + } + """ + + // protoc --descriptor_set_out=demo.desc demo.proto + private static let demoDescriptorSet = "0a90040a0a64656d6f2e70726f746f120464656d6f22b9030a014d121c0a016d18012003280b320e2e64656d6f2e4d2e4d456e74727952016d121f0a" + + "02696d18022003280b320f2e64656d6f2e4d2e496d456e7472795202696d12140a0474657874180320012809480052047465787412180a066e756d62" + + "6572180420012805480052066e756d62657212210a047069636b18052001280b320b2e64656d6f2e496e6e6572480052047069636b12140a05636f75" + + "6e741806200128055205636f756e7412150a036f7074180720012805480152036f707488010112120a046e616d6518082001280952046e616d651212" + + "0a046461746118092001280c52046461746112150a0165180a2001280e32072e64656d6f2e4552016512140a05726174696f180b2001280152057261" + + "74696f12210a05696e6e6572180c2001280b320b2e64656d6f2e496e6e65725205696e6e65721a340a064d456e74727912100a036b65791801200128" + + "0952036b657912140a0576616c7565180220012805520576616c75653a0238011a350a07496d456e74727912100a036b657918012001280552036b65" + + "7912140a0576616c7565180220012809520576616c75653a02380142080a0663686f69636542060a045f6f707422230a05496e6e6572120c0a017818" + + "0120012805520178120c0a01791802200128055201792a130a014512060a024530100012060a0245311001620670726f746f33" + + // protoc --encode=demo.M: m { key: "b" value: 1 } m { key: "a" value: 2 } m { key: "a" value: 3 } + // im { key: 5 value: "x" } im { key: -1 value: "y" } im { key: 2 value: "z" } + private static let maps = "0a050a016210010a050a016110020a050a0161100312050805120178120e08ffffffffffffffffff011201791205080212017a" + + private static let mapsText = """ + m { + key: "a" + value: 2 + } + m { + key: "a" + value: 3 + } + m { + key: "b" + value: 1 + } + im { + key: -1 + value: "y" + } + im { + key: 2 + value: "z" + } + im { + key: 5 + value: "x" + } + + """ + + // Hand-written: count, name, data, e and ratio at their defaults, then protoc --encode=demo.M of `opt: 0 inner {}`. + private static let defaults = "300042004a00500059000000000000000038006200" + + private static let editionsProto = """ + edition = "2023"; + package ed; + option features.field_presence = IMPLICIT; + + enum Open { + O0 = 0; + O1 = 1; + } + + enum Closed { + option features.enum_type = CLOSED; + C0 = 0; + C1 = 1; + } + + message M { + int32 a = 1; + int32 b = 2 [features.field_presence = EXPLICIT]; + int32 c = 3 [features.field_presence = LEGACY_REQUIRED]; + Sub sub = 4; + Open open = 5; + Closed closed = 6 [features.field_presence = EXPLICIT]; + string s = 7; + + message Sub { + int32 x = 1; + int32 y = 2 [features.field_presence = EXPLICIT]; + } + } + """ + + // protoc --descriptor_set_out=ed.desc ed.proto + private static let editionsDescriptorSet = "0ab6020a0865642e70726f746f1202656422d9010a014d120c0a016118012001280552016112130a01621802200128054205aa010208015201621213" + + "0a01631803200128054205aa01020803520163121b0a0373756218042001280b32092e65642e4d2e5375625203737562121c0a046f70656e18052001" + + "280e32082e65642e4f70656e52046f70656e12290a06636c6f73656418062001280e320a2e65642e436c6f7365644205aa010208015206636c6f7365" + + "64120c0a01731807200128095201731a280a03537562120c0a017818012001280552017812130a01791802200128054205aa010208015201792a160a" + + "044f70656e12060a024f30100012060a024f3110012a1e0a06436c6f73656412060a024330100012060a02433110011a043a02100242059203020802" + + "620865646974696f6e7370e807" + + // protoc --include_imports --descriptor_set_out=order.desc order.proto, where order.proto imports money.proto. + private static let orderDescriptorSet = "0a570a0b6d6f6e65792e70726f746f12056d6f6e657922390a054d6f6e657912140a05756e6974731801200128035205756e697473121a0a08637572" + + "72656e6379180220012809520863757272656e6379620670726f746f330acb010a0b6f726465722e70726f746f12056f726465721a0b6d6f6e65792e" + + "70726f746f2285010a054f72646572120e0a0269641801200128095202696412220a05746f74616c18022001280b320c2e6d6f6e65792e4d6f6e6579" + + "5205746f74616c122b0a0673746174757318032001280e32132e6f726465722e4f726465722e5374617475735206737461747573221b0a0653746174" + + "757312070a034e4557100012080a04444f4e45100122180a044c696e6512100a03736b751801200128095203736b75620670726f746f33" + + private static let budgetProto = """ + syntax = "proto2"; + package budget; + + message Box { + optional Part a = 1; + optional Part b = 2; + optional Part c = 3; + } + + message Part { + repeated int32 v = 1; + } + """ + + @Test("Map entries print sorted by key, and a repeated key keeps every entry in wire order") + func mapsSortByKey() throws { + let schema = try ProtobufSchema.parse(Data(Self.demoProto.utf8)) + #expect(try Self.text(Self.maps, schema: schema, message: "demo.M") == Self.mapsText) + // Hand-written: an empty entry, then one with a field 3 the entry does not declare. + #expect(try Self.text("0a000a050a017a1809", schema: schema, message: "demo.M") + == "m {\n key: \"\"\n value: 0\n}\nm {\n key: \"z\"\n value: 0\n}\n") + } + + @Test("The oneof member set last wins, and a message member keeps only what follows the last switch to it") + func oneofLastMemberWins() throws { + let schema = try ProtobufSchema.parse(Data(Self.demoProto.utf8)) + // protoc --encode=demo.M of `text: "hello"` and of `number: 7`, joined in both orders. + #expect(try Self.text("1a0568656c6c6f2007", schema: schema, message: "demo.M") == "number: 7\n") + #expect(try Self.text("20071a0568656c6c6f", schema: schema, message: "demo.M") == "text: \"hello\"\n") + // protoc --encode=demo.M of `pick { x: 1 }`, `number: 7` and `pick { y: 3 }`, joined. + #expect(try Self.text("2a02080120072a021003", schema: schema, message: "demo.M") == "pick {\n y: 3\n}\n") + } + + @Test("A proto3 field at its default is left out, after the last value wins; optional and message fields stay") + func implicitPresence() throws { + let schema = try ProtobufSchema.parse(Data(Self.demoProto.utf8)) + // protoc --encode=demo.M of `count: 5`, then a hand-written `count: 0`. + #expect(try Self.text("30053000", schema: schema, message: "demo.M").isEmpty) + #expect(try Self.text(Self.defaults, schema: schema, message: "demo.M") == "opt: 0\ninner {\n}\n") + // protoc --encode=demo.M of `ratio: -0` + #expect(try Self.text("590000000000000080", schema: schema, message: "demo.M") == "ratio: -0\n") + } + + @Test("A proto2 field prints whatever value the bytes carry") + func proto2KeepsDefaults() throws { + let proto = """ + syntax = "proto2"; + package legacy; + message Msg { + optional int32 n = 1; + optional string s = 2; + } + """ + let schema = try ProtobufSchema.parse(Data(proto.utf8)) + // protoc --encode=legacy.Msg of `n: 0 s: ""` + #expect(try Self.text("08001200", schema: schema, message: "legacy.Msg") == "n: 0\ns: \"\"\n") + } + + @Test("A number a closed enum does not declare goes to the unknown fields, a map entry with it whole") + func closedEnumUnknownNumbers() throws { + let proto = """ + syntax = "proto2"; + package legacy; + + enum Level { + LOW = 0; + HIGH = 1; + } + + message Msg { + optional int32 n = 1; + optional string s = 2; + optional Level level = 3; + repeated Level levels = 4 [packed = true]; + map named = 5; + } + """ + let schema = try ProtobufSchema.parse(Data(proto.utf8)) + // protoc --encode=legacy.Msg of `level: HIGH`, then hand-written: level 7, 99: 1, levels packed [1, 9, 0], + // named { 4: 8 } and named { 2: HIGH }. + let fixture = "1801180798060122030109002a04080410082a0408021001" + let expected = "level: HIGH\nlevels: HIGH\nlevels: LOW\nnamed {\n key: 2\n value: HIGH\n}\n3: 7\n99: 1\n4: 9\n" + + "5 {\n 1: 4\n 2: 8\n}\n" + #expect(try Self.text(fixture, schema: schema, message: "legacy.Msg") == expected) + } + + @Test("Editions presence follows features.field_presence on the file and the field") + func editionsPresence() throws { + let schema = try ProtobufSchema.parse(Data(Self.editionsProto.utf8)) + let descriptorSet = try ProtobufSchema.parse(BinaryFixture.data(Self.editionsDescriptorSet)) + #expect(descriptorSet == schema) + // Hand-written: every field of ed.M at zero, sub with x and y at zero. + let zeros = "080010001800220408001000280030003a00" + let expected = "b: 0\nc: 0\nsub {\n y: 0\n}\nclosed: C0\n" + #expect(try Self.text(zeros, schema: schema, message: "ed.M") == expected) + #expect(try Self.text(zeros, schema: descriptorSet, message: "ed.M") == expected) + // Hand-written: a 1, closed 7, open 7. + #expect(try Self.text("080130072807", schema: schema, message: "ed.M") == "a: 1\nopen: 7\n6: 7\n") + } + + @Test("A descriptor set reads the same as the .proto protoc built it from") + func descriptorSetMatchesProto() throws { + let schema = try ProtobufSchema.parse(Data(Self.demoProto.utf8)) + let descriptorSet = try ProtobufSchema.parse(BinaryFixture.data(Self.demoDescriptorSet)) + #expect(descriptorSet == schema) + #expect(try Self.text(Self.maps, schema: descriptorSet, message: "demo.M") == Self.mapsText) + #expect(try Self.text(Self.defaults, schema: descriptorSet, message: "demo.M") == "opt: 0\ninner {\n}\n") + } + + @Test("A descriptor set that does not read says it is not a schema") + func brokenDescriptorSet() { + // order.desc's money.proto with its last three bytes cut, framed as a set again. + let truncated = "0a540a0b6d6f6e65792e70726f746f12056d6f6e657922390a054d6f6e657912140a05756e6974731801200128035205756e697473121a0a0863" + + "757272656e6379180220012809520863757272656e6379620670726f" + #expect(throws: ProtobufSchemaError.notASchema) { try ProtobufSchema.parse(BinaryFixture.data(truncated)) } + // Hand-written: a file whose message names more bytes than it holds. + #expect(throws: ProtobufSchemaError.notASchema) { try ProtobufSchema.parse(BinaryFixture.data("0a0422020a05")) } + } + + @Test("A set built with --include_imports reads its last file as the one chosen") + func includeImports() throws { + let schema = try ProtobufSchema.parse(BinaryFixture.data(Self.orderDescriptorSet)) + #expect(schema.rootMessages == ["order.Order", "order.Line"]) + #expect(schema.messageNames == ["order.Order", "order.Line", "money.Money"]) + // protoc --encode=order.Order of `id: "A1" total { units: 12 currency: "EUR" } status: DONE` + #expect(try Self.text("0a0241311207080c12034555521801", schema: schema, message: nil) + == "id: \"A1\"\ntotal {\n units: 12\n currency: \"EUR\"\n}\nstatus: DONE\n") + } + + @Test("A nested enum in a descriptor set is read once") + func nestedEnumOnce() throws { + let files = try ProtobufDescriptorSetParser.parse(BinaryFixture.data(Self.orderDescriptorSet)) + let names = files.flatMap { $0.allEnums.map { $0.fullName } } + #expect(names == ["order.Order.Status"]) + } + + @Test("A field whose type is a message named optional is not read as a label") + func optionalAsTypeName() throws { + let proto = """ + syntax = "proto2"; + package opt; + + message optional { + optional int32 v = 1; + } + + message M { + optional foo = 1; + repeated optional bar = 2; + } + """ + let schema = try ProtobufSchema.parse(Data(proto.utf8)) + // protoc --encode=opt.M of `foo { v: 1 } bar { v: 2 } bar { v: 3 }`, with `foo` declared `optional optional foo = 1;` + #expect(try Self.text("0a0208011202080212020803", schema: schema, message: "opt.M") + == "foo {\n v: 1\n}\nbar {\n v: 2\n}\nbar {\n v: 3\n}\n") + } + + @Test("A verified string that is not UTF-8 refuses the whole value, as protoc does, and an unverified one prints escaped") + func stringValidation() throws { + let proto3 = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package u; + message Inner { string s = 1; } + message M { string s = 1; Inner inner = 2; repeated string r = 3; map m = 4; } + """.utf8)) + // protoc --decode=u.M fails "contains invalid UTF-8 data" on each of these. + for fixture in ["0a01ff", "12030a01ff", "1a01ff", "22050a01ff1001", "0a01ff0a0141"] { + #expect(throws: BinaryValueDecodeError.self, "\(fixture)") { + try Self.text(fixture, schema: proto3, message: "u.M") + } + } + let proto2 = try ProtobufSchema.parse(Data("syntax = \"proto2\"; message M { optional string s = 1; }".utf8)) + #expect(try Self.text("0a01ff", schema: proto2, message: "M") == "s: \"\\377\"\n") + let editions = try ProtobufSchema.parse(Data(""" + edition = "2023"; + package ue; + message M { string s = 1 [features.utf8_validation = NONE]; string v = 2; } + """.utf8)) + #expect(try Self.text("0a01ff", schema: editions, message: "ue.M") == "s: \"\\377\"\n") + #expect(throws: BinaryValueDecodeError.self) { try Self.text("1201ff", schema: editions, message: "ue.M") } + } + + @Test("A declared message or packed field whose bytes do not parse refuses the value, as protoc does") + func malformedDeclaredFields() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package n; + message Inner { int32 a = 1; } + message Inner2 { string s = 1; } + message M { Inner inner = 1; repeated int32 p = 2; Inner2 g = 3; } + """.utf8)) + // protoc --decode=n.M fails "Failed to parse input." on each of these. + for fixture in ["0a0108", "1203ffffff", "0a03ff0101"] { + #expect(throws: BinaryValueDecodeError.self, "\(fixture)") { + try Self.text(fixture, schema: schema, message: "n.M") + } + } + // protoc --decode=n.M of 1a020801: a wire type the nested field does not declare is only unknown. + #expect(try Self.text("1a020801", schema: schema, message: "n.M") == "g {\n 1: 1\n}\n") + } + + @Test("A oneof message member that a later member replaces must still parse, as protoc parses it first") + func discardedOneofMemberIsParsed() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package o2; + message Inner { int32 n = 1; } + message M { oneof c { Inner a = 1; int32 b = 2; } } + """.utf8)) + // protoc --decode=o2.M fails on both: `a` holds a truncated varint. + for fixture in ["0a01081007", "10070a0108"] { + #expect(throws: BinaryValueDecodeError.self, "\(fixture)") { + try Self.text(fixture, schema: schema, message: "o2.M") + } + } + #expect(try Self.text("0a0208011007", schema: schema, message: "o2.M") == "b: 7\n") + } + + @Test("An editions message field set DELIMITED reads its group framing, from a .proto and a descriptor set") + func delimitedMessageEncoding() throws { + let proto = """ + edition = "2023"; + package d; + message Inner { int32 n = 1; } + message M { Inner inner = 1 [features.message_encoding = DELIMITED]; Inner plain = 2; } + """ + // protoc --descriptor_set_out=d.desc of the file above. + let descriptorSet = "0a7c0a07642e70726f746f12016422150a05496e6e6572120c0a016e18012001280552016e224a0a014d12250a05696e6e6572" + + "18012001280b32082e642e496e6e65724205aa010228025205696e6e6572121e0a05706c61696e18022001280b32082e642e496e6e" + + "65725205706c61696e620865646974696f6e7370e807" + for schema in [try ProtobufSchema.parse(Data(proto.utf8)), try ProtobufSchema.parse(BinaryFixture.data(descriptorSet))] { + // protoc --decode=d.M of 0b08070c, then of 0a020807 sent length-prefixed. + #expect(try Self.text("0b08070c", schema: schema, message: "d.M") == "inner {\n n: 7\n}\n") + #expect(try Self.text("0a020807", schema: schema, message: "d.M") == "1 {\n 1: 7\n}\n") + } + let fileLevel = try ProtobufSchema.parse(Data(""" + edition = "2023"; + package d2; + option features.message_encoding = DELIMITED; + message Inner { int32 n = 1; } + message M { Inner inner = 1; repeated Inner many = 2; } + """.utf8)) + // protoc --decode=d2.M of 0b08070c1308011413080214 + #expect(try Self.text("0b08070c1308011413080214", schema: fileLevel, message: "d2.M") + == "inner {\n n: 7\n}\nmany {\n n: 1\n}\nmany {\n n: 2\n}\n") + } + + @Test("A cell reads no further into a group than its budget, so bytes past it, even broken ones, are never read") + func groupScanIsBudgeted() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto2"; + package gb; + message M { optional group Item = 1 { repeated int32 v = 2; } } + """.utf8)) + // Hand-written: a group of fifty `v: 1`, then a byte that starts no field and no end tag. + let fixture = "0b" + String(repeating: "1001", count: 50) + "ff" + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "gb.M", maxValues: 5) + let text = ProtobufSchemaTextWriter.text(message.fields, multiLine: false) + #expect(text.hasPrefix("Item { v: 1")) + #expect(text.hasSuffix("…")) + #expect(throws: BinaryValueDecodeError.self) { + try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "gb.M") + } + } + + @Test("A sint32 wider than 32 bits is cut to 32 before ZigZag, as protoc reads one a sint64 writer sent") + func wideSint32() throws { + let schema = try ProtobufSchema.parse(Data("syntax = \"proto3\"; message M { sint32 n = 1; }".utf8)) + // protoc --decode=M of 088280808010 + #expect(try Self.text("088280808010", schema: schema, message: "M") == "n: 1\n") + } + + @Test("Features written as one aggregate option apply like dotted ones") + func aggregateFeatures() throws { + let schema = try ProtobufSchema.parse(Data(""" + edition = "2023"; + package ag; + option features = { message_encoding: DELIMITED, field_presence: IMPLICIT }; + message Inner { int32 n = 1; } + message M { + Inner inner = 1; + int32 z = 2; + string t = 3 [features = { utf8_validation: NONE }]; + int32 e = 4 [features = { field_presence: EXPLICIT }]; + } + """.utf8)) + // protoc --decode=ag.M of 0b08070c10001a01ff2000 + #expect(try Self.text("0b08070c10001a01ff2000", schema: schema, message: "ag.M") + == "inner {\n n: 7\n}\nt: \"\\377\"\ne: 0\n") + } + + @Test("A map entry keeps its last value: a closed enum number it does not declare rejects it only when left standing") + func mapEntryLastValueWins() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto2"; + package me; + enum E { A = 0; B = 1; } + message M { map m = 1; } + """.utf8)) + // protoc --decode=me.M of 0a06080110091001, then of 0a06080110011009. + #expect(try Self.text("0a06080110091001", schema: schema, message: "me.M") == "m {\n key: 1\n value: B\n}\n") + #expect(try Self.text("0a06080110011009", schema: schema, message: "me.M") == "1 {\n 1: 1\n 2: 9\n}\n") + } + + @Test("A map's message values stay length-prefixed in a file that sets DELIMITED") + func delimitedFileKeepsMapValuesLengthPrefixed() throws { + let schema = try ProtobufSchema.parse(Data(""" + edition = "2023"; + package dm; + option features.message_encoding = DELIMITED; + message Child { int32 n = 1; } + message M { map m = 1; } + """.utf8)) + // protoc --decode=dm.M of 0a070a016112020805 + #expect(try Self.text("0a070a016112020805", schema: schema, message: "dm.M") + == "m {\n key: \"a\"\n value {\n n: 5\n }\n}\n") + } + + @Test("Parsing a replaced oneof member spends the cell's budget instead of a fresh one") + func discardedOneofSpendsBudget() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package ob; + message Inner { repeated int32 v = 1; } + message M { oneof c { Inner a = 1; int32 b = 2; } } + """.utf8)) + // Hand-written: `a` holding a hundred unpacked `v: 1`, then `b: 7`. + let fixture = "0ac801" + String(repeating: "0801", count: 100) + "1007" + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "ob.M", maxValues: 5) + #expect(ProtobufSchemaTextWriter.text(message.fields, multiLine: false) == "b: 7") + } + + @Test("Replaced oneof members parse in wire order, so a broken one fails the value whatever the hash order") + func discardedMembersParseInWireOrder() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto3"; + package wo; + message Inner { repeated int32 v = 1; } + message M { oneof x { Inner a = 1; int32 b = 2; } oneof y { Inner c = 3; int32 d = 4; } } + """.utf8)) + // Hand-written: `c` holding a truncated varint, `a` holding a hundred `v: 1`, then `b: 1` and `d: 1`. + let fixture = "1a0108" + "0ac801" + String(repeating: "0801", count: 100) + "1001" + "2001" + for _ in 0..<20 { + #expect(throws: BinaryValueDecodeError.self) { + try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "wo.M", maxValues: 6) + } + } + } + + @Test("The package qualifies the definitions written before it too") + func packageAfterMessage() throws { + let schema = try ProtobufSchema.parse(Data(""" + syntax = "proto2"; + message A { optional B b = 1; } + package x; + message B { optional int32 v = 1; } + """.utf8)) + #expect(schema.messageNames == ["x.A", "x.B"]) + // protoc --decode=x.A of 0a020807 + #expect(try Self.text("0a020807", schema: schema, message: "x.A") == "b {\n v: 7\n}\n") + } + + @Test("A .proto file saved with a UTF-8 byte order mark reads as protoc reads it") + func byteOrderMark() throws { + let schema = try ProtobufSchema.parse(Data([0xEF, 0xBB, 0xBF]) + Data("syntax = \"proto3\"; message Q { int32 a = 1; }".utf8)) + #expect(try Self.text("0805", schema: schema, message: "Q") == "a: 5\n") + } + + @Test("A cell spends its budget inside a field the schema does not name") + func unknownFieldBudget() throws { + let schema = try ProtobufSchema.parse(Data(Self.budgetProto.utf8)) + // Hand-written: field 9 holding fifty `1: 1`. + let fixture = "4a64" + String(repeating: "0801", count: 50) + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "budget.Box", maxValues: 5) + #expect(ProtobufSchemaTextWriter.text(message.fields, multiLine: false) == "9 { 1: 1 1: 1 1: 1 1: 1 … }") + } + + @Test("Messages read in field order whatever their wire order, and the budget runs out where it runs out") + func messagesInFieldOrder() throws { + let schema = try ProtobufSchema.parse(Data(Self.budgetProto.utf8)) + // protoc --encode=budget.Box of `c { v: 7 v: 8 v: 9 }`, `b { v: 4 v: 5 v: 6 }` and `a { v: 1 v: 2 v: 3 }`, joined. + let fixture = "1a0608070808080912060804080508060a06080108020803" + #expect(try Self.text(fixture, schema: schema, message: "budget.Box") + == "a {\n v: 1\n v: 2\n v: 3\n}\nb {\n v: 4\n v: 5\n v: 6\n}\nc {\n v: 7\n v: 8\n v: 9\n}\n") + for _ in 0..<5 { + let message = try ProtobufSchemaDecoder.decode(BinaryFixture.data(fixture), schema: schema, message: "budget.Box", maxValues: 7) + #expect(ProtobufSchemaTextWriter.text(message.fields, multiLine: false) == "a { v: 1 v: 2 v: 3 } b { v: 4 … } c { … }") + } + } + + @Test("A field the schema does not name counts once toward the tree's node limit") + func unknownFieldsCountOnce() throws { + let schema = try ProtobufSchema.parse(Data(Self.shopProto.utf8)) + // Hand-written: 3,000 copies of `2: 1`, which shop.Other does not declare. + let fixture = Data((0..<3_000).flatMap { _ in [UInt8(0x10), 0x01] }) + let message = try ProtobufSchemaDecoder.decode(fixture, schema: schema, message: "shop.Other") + let root = ProtobufSchemaTreeBuilder.build(from: message, formats: BinaryFixture.english) + #expect(root.children.count == 3_000) + #expect(root.children.last?.kind == .varint) + } +} diff --git a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufWireDecoderTests.swift b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufWireDecoderTests.swift index 47c95c8fb5..5e9f5094da 100644 --- a/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufWireDecoderTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/BinaryValues/ProtobufWireDecoderTests.swift @@ -8,9 +8,10 @@ import Testing @testable import TablePro -/// Fixtures written by `protoc --encode` (libprotoc 36.2). Each expectation is what -/// `protoc --decode_raw` printed for the same bytes, except that UTF-8 text keeps its characters. +// Each expectation is what `protoc --decode_raw` (libprotoc 36.2) printed for the same bytes, except that UTF-8 text +// keeps its characters. struct ProtobufWireDecoderTests { + // protoc --encode of a message setting every wire type, a nested and a repeated message, and field 1000. private static let order = "089601120774657374696e671a0908011205636166c3a925070000002909000000000000003206038e029ea7053a" + "030001ff400348ffffffffffffffffff01516e861bf0f92109405d0000c03f6202080262050803120178c03eb5b8f0fe2d" @@ -59,6 +60,7 @@ struct ProtobufWireDecoderTests { 5: "end" """ + // protoc --encode of a proto2 message with a group between two fields. #expect(try Self.text("0801131a08687474703a2f2f782002142a03656e64") == expected) let message = try ProtobufWireDecoder.decode(BinaryFixture.data("0801131a08687474703a2f2f782002142a03656e64")) #expect(message.fields[1].value == .group([ @@ -104,6 +106,23 @@ struct ProtobufWireDecoderTests { #expect(ProtobufTextWriter.text(message.fields, multiLine: false) == #"1: 150 2: "testing" …"#) } + @Test("A length-delimited value spends the caller's budget and leaves the rest of it") + func lengthDelimitedBudget() { + // Hand-written: `1: 1`, `1: 2`, `1: 3`. + let window = BinaryValueReader(BinaryFixture.data("080108020803")) + var budget = 2 + let value = ProtobufWireDecoder.lengthDelimited(window, depth: 0, budget: &budget) + #expect(value == .message([ + ProtobufField(number: 1, value: .varint(1)), + ProtobufField(number: 1, value: .varint(2)), + ProtobufField(number: 0, value: .truncated), + ])) + #expect(budget == 0) + var roomy = 10 + _ = ProtobufWireDecoder.lengthDelimited(window, depth: 0, budget: &roomy) + #expect(roomy == 7) + } + @Test("The tree keys fields by number and badges their wire type") func tree() throws { let message = try ProtobufWireDecoder.decode(BinaryFixture.data(Self.order)) diff --git a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift index 72d719c5e2..ff1d8b1d10 100644 --- a/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift +++ b/apple/TableProTests/Core/Services/Formatting/ValueDisplayFormatTests.swift @@ -40,6 +40,22 @@ struct ValueDisplayFormatTests { #expect(ValueDisplayFormat(rawValue: "nope") == nil) } + @Test("A Protobuf schema format keeps its schema and message, a Schema Registry one its connection") + func schemaFormatsRoundTrip() throws { + let schemaID = String(repeating: "ab12", count: 8) + let connection = UUID() + let protobuf = ValueDisplayFormat.protobufSchema(schemaID: schemaID, message: "shop.Item") + #expect(protobuf.rawValue == "protobuf:\(schemaID):shop.Item") + #expect(ValueDisplayFormat(rawValue: protobuf.rawValue) == protobuf) + #expect(ValueDisplayFormat(rawValue: "schemaRegistry:\(connection.uuidString)") == .schemaRegistry(connectionID: connection)) + #expect(ValueDisplayFormat(rawValue: "protobuf:\(schemaID):") == nil) + #expect(ValueDisplayFormat(rawValue: "protobuf:ab12:shop.Item") == nil) + #expect(ValueDisplayFormat(rawValue: "schemaRegistry:nope") == nil) + #expect(protobuf.isApplicable(to: .blob(rawType: "BLOB"))) + #expect(!protobuf.isApplicable(to: .text(rawType: "TEXT"))) + #expect(ValueDisplayFormat.schemaRegistry(connectionID: connection).isApplicable(to: .text(rawType: "TEXT"))) + } + @Test("only the formats that produce characters count as rendering binary as text") func rendersBinaryAsText() { #expect(ValueDisplayFormat.text.rendersBinaryAsText) diff --git a/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift new file mode 100644 index 0000000000..2e50b423fe --- /dev/null +++ b/apple/TableProTests/Core/Services/SchemaRegistry/SchemaRegistryTests.swift @@ -0,0 +1,425 @@ +// +// SchemaRegistryTests.swift +// TableProTests +// + +import Foundation +import os +import TableProTestSupport +@testable import TablePro +import TableProPluginKit +import Testing + +// Recorded from confluentinc/cp-schema-registry 7.7.1. The values were written by confluent-kafka 2.6.1's +// AvroSerializer and ProtobufSerializer against it, and the expected texts are what those libraries decode. +struct SchemaRegistryTests { + private static let responses: [String: (Int, String)] = [ + "/schemas/ids/1": (200, #"{"schema":"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"com.example\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"email\",\"type\":[\"null\",\"string\"]},{\"name\":\"kind\",\"type\":{\"type\":\"enum\",\"name\":\"Kind\",\"symbols\":[\"A\",\"B\"]}}]}"}"#), + "/schemas/ids/3": (200, #"{"schemaType":"PROTOBUF","schema":"syntax = \"proto3\";\npackage demo;\n\nimport \"demo/common.proto\";\n\nmessage Order {\n string id = 1;\n .demo.Money total = 2;\n repeated string tags = 3;\n .demo.Status status = 4;\n repeated .demo.Order.Line lines = 5;\n repeated .demo.Order.CountsEntry counts = 6;\n\n message Line {\n string sku = 1;\n sint32 qty = 2;\n }\n message CountsEntry {\n option map_entry = true;\n \n string key = 1;\n int32 value = 2;\n }\n}\nmessage Refund {\n string order_id = 1;\n double amount = 2;\n}\nenum Status {\n STATUS_UNKNOWN = 0;\n STATUS_PAID = 1;\n}\n","references":[{"name":"demo/common.proto","subject":"demo/common.proto","version":1}]}"#), + "/subjects/demo%2Fcommon.proto/versions/1": (200, #"{"subject":"demo/common.proto","version":1,"id":2,"schemaType":"PROTOBUF","schema":"syntax = \"proto3\";\npackage demo;\n\nmessage Money {\n string currency = 1;\n int64 units = 2;\n}\n"}"#), + "/schemas/ids/99999": (404, #"{"error_code":40403,"message":"Schema 99999 not found"}"#), + ] + + private static let avroUser = "00000000015406416e6e021e616e6e406578616d706c652e636f6d02" + private static let order = "0000000003000a036f2d3112080a0345555210e2091a01611a016220012a050a0178100532050a016b1007" + private static let refund = "000000000302020a036f2d31110000000000002940" + + private static let endpoint = SchemaRegistryEndpoint(fields: [ + SchemaRegistryConnectionField.url: "https://registry.example.com/base/", + SchemaRegistryConnectionField.username: "key", + SchemaRegistryConnectionField.password: "secret", + ]) + + private final class RecordedTransport: Sendable { + let requests = OSAllocatedUnfairLock<[URLRequest]>(initialState: []) + + var transport: SchemaRegistryClient.Transport { + { [self] request in + requests.withLock { $0.append(request) } + let path = String((request.url?.absoluteString ?? "").dropFirst("https://registry.example.com/base".count)) + let (status, body) = SchemaRegistryTests.responses[path] ?? (404, #"{"message":"no such path"}"#) + let response = HTTPURLResponse(url: request.url!, statusCode: status, httpVersion: nil, headerFields: nil)! + return (Data(body.utf8), response) + } + } + } + + private func fetch(_ id: UInt32, transport: RecordedTransport = RecordedTransport()) async throws -> RegistrySchema { + try await SchemaRegistryClient(endpoint: try #require(Self.endpoint), transport: transport.transport).schema(id: id) + } + + @Test("An Avro value decodes with the schema its prefix names") + func avro() async throws { + let schema = try await fetch(1) + let text = try BinaryValueDecoding.registryText(BinaryFixture.data(Self.avroUser), schema: schema, maxValues: .max, limit: .max) + #expect(text == #"{"id":42,"name":"Ann","email":"ann@example.com","kind":"B"}"#) + guard case .decoded(.json, _, let note) = try BinaryValueDecoding.registryDocument(BinaryFixture.data(Self.avroUser), schema: schema) else { + Issue.record("expected a decoded JSON tree") + return + } + #expect(note?.contains("1") == true) + } + + @Test("A Protobuf schema is fetched with the files it imports, and the message index picks the message") + func protobuf() async throws { + let transport = RecordedTransport() + let schema = try await fetch(3, transport: transport) + let paths = transport.requests.withLock { $0.compactMap { $0.url?.path(percentEncoded: true) } } + #expect(paths == ["/base/schemas/ids/3", "/base/subjects/demo%2Fcommon.proto/versions/1"]) + + let order = try BinaryValueDecoding.registryText(BinaryFixture.data(Self.order), schema: schema, maxValues: .max, limit: .max) + #expect(order == #"id: "o-1" total { currency: "EUR" units: 1250 } tags: "a" tags: "b" status: STATUS_PAID "# + + #"lines { sku: "x" qty: -3 } counts { key: "k" value: 7 }"#) + + guard case .decoded(.protobuf, let text, let note) = try BinaryValueDecoding.registryDocument( + BinaryFixture.data(Self.refund), schema: schema + ) else { + Issue.record("expected a decoded Protobuf tree") + return + } + #expect(text == "order_id: \"o-1\"\namount: 12.5\n") + #expect(note?.contains("3") == true) + } + + @Test("An Avro schema can name a type another subject defines") + func avroReferences() throws { + let money = #"{"type":"record","name":"Money","namespace":"demo","fields":[{"name":"units","type":"long"}]}"# + let order = #"{"type":"record","name":"Order","namespace":"demo","fields":[{"name":"total","type":"demo.Money"}]}"# + #expect(throws: AvroSchemaError.self) { try AvroSchema.parse(order) } + let schema = try AvroSchema.parse(order, references: [money]) + let text = try BinaryValueDecoding.registryText(BinaryFixture.data("000000000754"), schema: .avro(schema), maxValues: .max, limit: .max) + #expect(text == #"{"total":{"units":42}}"#) + } + + @Test("Every request carries the connection's credentials") + func basicAuth() async throws { + let transport = RecordedTransport() + _ = try await fetch(1, transport: transport) + let header = transport.requests.withLock { $0.first?.value(forHTTPHeaderField: "Authorization") } + #expect(header == "Basic " + Data("key:secret".utf8).base64EncodedString()) + } + + @Test("The registry's own error message is the reason a schema is missing") + func notFound() async { + await #expect(throws: SchemaRegistryError(reason: "The Schema Registry answered 404: Schema 99999 not found")) { + try await fetch(99_999) + } + } + + @Test("Only an http or https URL names a registry") + func endpointSchemes() { + #expect(SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: "file:///etc/passwd"]) == nil) + #expect(SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: ""]) == nil) + #expect(SchemaRegistryEndpoint(fields: [:]) == nil) + #expect(SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: " http://localhost:8081 "])?.username == nil) + } + + @Test("A value without the prefix, or one its schema cannot read, is refused, so the cell stays hex") + func brokenValues() async throws { + let avro = try await fetch(1) + #expect(throws: BinaryValueDecodeError(kind: .notFramed, offset: 0)) { + try SchemaRegistryFrame.schemaID(BinaryFixture.data("5406416e6e")) + } + #expect(throws: BinaryValueDecodeError.self) { + try BinaryValueDecoding.registryText(BinaryFixture.data("000000000154"), schema: avro, maxValues: .max, limit: .max) + } + let protobuf = try await fetch(3) + #expect(throws: BinaryValueDecodeError.self) { + try BinaryValueDecoding.registryText(BinaryFixture.data("000000000302080a"), schema: protobuf, maxValues: .max, limit: .max) + } + } + + private final class Resolver: Sendable { + let calls = OSAllocatedUnfairLock(initialState: 0) + let current: OSAllocatedUnfairLock + + init(_ endpoint: SchemaRegistryEndpoint?) { + current = OSAllocatedUnfairLock(initialState: endpoint) + } + + var resolve: SchemaRegistryCache.EndpointResolver { + { [self] _ in + calls.withLock { $0 += 1 } + return current.withLock { $0 } + } + } + } + + private static func changes(for connection: UUID) -> (AsyncStream, NSObjectProtocol) { + let (stream, continuation) = AsyncStream.makeStream() + let observer = NotificationCenter.default.addObserver(forName: SchemaRegistryCache.didChange, object: nil, queue: nil) { + if $0.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID == connection { continuation.yield() } + } + return (stream, observer) + } + + @Test("Reading the cache never fetches; a prefetch asks once per schema and announces the batch") + @MainActor + func prefetch() async throws { + let transport = RecordedTransport() + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: transport.transport) + let (changes, observer) = Self.changes(for: connection) + defer { NotificationCenter.default.removeObserver(observer) } + + #expect(cache.cachedSchema(connectionID: connection, schemaID: 1) == nil) + #expect(transport.requests.withLock { $0.isEmpty }) + + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: false) + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: false) + var iterator = changes.makeAsyncIterator() + _ = await iterator.next() + guard case .success(.avro)? = cache.cachedSchema(connectionID: connection, schemaID: 1) else { + Issue.record("expected the fetched Avro schema") + return + } + #expect(transport.requests.withLock { $0.count } == 1) + } + + @Test("A loaded batch skips the endpoint lookup, which reads the Keychain") + @MainActor + func loadedBatchSkipsResolve() async throws { + let resolver = Resolver(Self.endpoint) + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: resolver.resolve, transport: RecordedTransport().transport) + _ = await cache.schema(connectionID: connection, schemaID: 1) + let calls = resolver.calls.withLock { $0 } + + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: false) + #expect(resolver.calls.withLock { $0 } == calls) + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true) + #expect(resolver.calls.withLock { $0 } == calls + 1) + } + + @Test("A failure is kept for its lifetime, and a full reload asks again at once") + @MainActor + func failureRetry() async throws { + let transport = RecordedTransport() + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: transport.transport) + guard case .failure = await cache.schema(connectionID: connection, schemaID: 99_999) else { + Issue.record("expected a failure") + return + } + guard case .failure? = cache.cachedSchema(connectionID: connection, schemaID: 99_999) else { + Issue.record("expected the failure to be kept") + return + } + cache.prefetch(connectionID: connection, schemaIDs: [99_999], retryingFailures: false) + #expect(transport.requests.withLock { $0.count } == 1) + + cache.prefetch(connectionID: connection, schemaIDs: [99_999], retryingFailures: true) + try await waitUntil { transport.requests.withLock { $0.count } == 2 } + } + + @Test("A failure past its lifetime is fetched again by the next prefetch") + @MainActor + func failureExpires() async throws { + let transport = RecordedTransport() + let connection = UUID() + let cache = SchemaRegistryCache( + failureLifetime: .zero, resolveEndpoint: Resolver(Self.endpoint).resolve, transport: transport.transport + ) + _ = await cache.schema(connectionID: connection, schemaID: 99_999) + cache.prefetch(connectionID: connection, schemaIDs: [99_999], retryingFailures: false) + try await waitUntil { transport.requests.withLock { $0.count } == 2 } + } + + @Test("A connection whose registry changed reads schemas from the new one, and the change is announced") + @MainActor + func endpointChange() async throws { + let transport = RecordedTransport() + let resolver = Resolver(Self.endpoint) + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: resolver.resolve, transport: transport.transport) + guard case .success = await cache.schema(connectionID: connection, schemaID: 1) else { + Issue.record("expected the schema from the first registry") + return + } + let (changes, observer) = Self.changes(for: connection) + defer { NotificationCenter.default.removeObserver(observer) } + + let other = SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: "https://other.example.com"]) + resolver.current.withLock { $0 = other } + cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true) + var iterator = changes.makeAsyncIterator() + _ = await iterator.next() + + try await waitUntil { transport.requests.withLock { $0.count } == 2 } + let host = transport.requests.withLock { $0.last?.url?.host } + #expect(host == "other.example.com") + } + + private final class HeldTransport: Sendable { + let gate = AsyncGate() + let calls = OSAllocatedUnfairLock(initialState: 0) + let active = OSAllocatedUnfairLock(initialState: (now: 0, most: 0)) + + var transport: SchemaRegistryClient.Transport { + { [self] request in + calls.withLock { $0 += 1 } + active.withLock { $0 = (now: $0.now + 1, most: max($0.most, $0.now + 1)) } + await gate.wait() + active.withLock { $0.now -= 1 } + let response = HTTPURLResponse(url: request.url!, statusCode: 404, httpVersion: nil, headerFields: nil)! + return (Data(#"{"message":"not here"}"#.utf8), response) + } + } + } + + @Test("A page naming thousands of IDs queues a bounded number of fetches and runs a few at a time") + @MainActor + func prefetchIsBounded() async throws { + let held = HeldTransport() + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: held.transport) + cache.prefetch(connectionID: connection, schemaIDs: Set(UInt32(1)...5_000), retryingFailures: false) + + await held.gate.arrived(SchemaRegistryCache.maxConcurrentFetches) + #expect(held.calls.withLock { $0 } == SchemaRegistryCache.maxConcurrentFetches) + await held.gate.open() + try await waitUntil { held.calls.withLock { $0 } == SchemaRegistryCache.maxPendingFetches } + try await waitUntil { held.active.withLock { $0.now } == 0 } + #expect(held.active.withLock { $0.most } == SchemaRegistryCache.maxConcurrentFetches) + #expect(held.calls.withLock { $0 } == SchemaRegistryCache.maxPendingFetches) + } + + @Test("A connection on a registry another one already read gets its schemas as soon as it prefetches") + @MainActor + func sharedEndpointAdoptsLoadedSchemas() async throws { + let transport = RecordedTransport() + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: transport.transport) + guard case .success = await cache.schema(connectionID: UUID(), schemaID: 1) else { + Issue.record("expected the schema for the first connection") + return + } + let second = UUID() + #expect(cache.cachedSchema(connectionID: second, schemaID: 1) == nil) + cache.prefetch(connectionID: second, schemaIDs: [1], retryingFailures: false) + guard case .success? = cache.cachedSchema(connectionID: second, schemaID: 1) else { + Issue.record("expected the second connection to read the loaded schema at once") + return + } + #expect(transport.requests.withLock { $0.count } == 1) + } + + @Test("Every grid hears of a registry switch before it happens, while the old schemas still answer") + @MainActor + func willSwitch() async throws { + let resolver = Resolver(Self.endpoint) + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: resolver.resolve, transport: RecordedTransport().transport) + _ = await cache.schema(connectionID: connection, schemaID: 1) + let heard = OSAllocatedUnfairLock(initialState: (calls: 0, oldStillAnswered: false)) + let observer = NotificationCenter.default.addObserver(forName: SchemaRegistryCache.willSwitch, object: nil, queue: nil) { + guard $0.userInfo?[SchemaRegistryCache.connectionIDKey] as? UUID == connection else { return } + let answered: Bool + if case .success? = cache.cachedSchema(connectionID: connection, schemaID: 1) { answered = true } else { answered = false } + heard.withLock { $0 = (calls: $0.calls + 1, oldStillAnswered: answered) } + } + defer { NotificationCenter.default.removeObserver(observer) } + + #expect(!cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true)) + #expect(heard.withLock { $0.calls } == 0) + + resolver.current.withLock { $0 = SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: "https://other.example.com"]) } + #expect(cache.prefetch(connectionID: connection, schemaIDs: [1], retryingFailures: true)) + #expect(heard.withLock { $0.calls } == 1) + #expect(heard.withLock { $0.oldStillAnswered }) + if case .success? = cache.cachedSchema(connectionID: connection, schemaID: 1) { + Issue.record("expected the new registry's schema to be unread") + } + } + + @Test("A loaded schema is announced at once, not after a slower one in the same batch") + @MainActor + func loadsAreAnnouncedOneByOne() async throws { + let gate = AsyncGate() + let recorded = RecordedTransport() + let slow: SchemaRegistryClient.Transport = { request in + if request.url?.lastPathComponent == "99999" { await gate.wait() } + return try await recorded.transport(request) + } + let connection = UUID() + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: slow) + let (changes, observer) = Self.changes(for: connection) + defer { NotificationCenter.default.removeObserver(observer) } + + cache.prefetch(connectionID: connection, schemaIDs: [1, 99_999], retryingFailures: false) + var iterator = changes.makeAsyncIterator() + _ = await iterator.next() + guard case .success? = cache.cachedSchema(connectionID: connection, schemaID: 1) else { + Issue.record("expected schema 1 while 99999 is still being fetched") + return + } + #expect(cache.cachedSchema(connectionID: connection, schemaID: 99_999) == nil) + await gate.open() + } + + @Test("A schema one connection loads is announced to every connection on that registry") + @MainActor + func loadReachesEveryConnectionOnTheEndpoint() async throws { + let cache = SchemaRegistryCache(resolveEndpoint: Resolver(Self.endpoint).resolve, transport: RecordedTransport().transport) + let first = UUID() + let second = UUID() + _ = await cache.schema(connectionID: second, schemaID: 99_999) + let (changes, observer) = Self.changes(for: second) + defer { NotificationCenter.default.removeObserver(observer) } + + _ = await cache.schema(connectionID: first, schemaID: 1) + var iterator = changes.makeAsyncIterator() + _ = await iterator.next() + guard case .success? = cache.cachedSchema(connectionID: second, schemaID: 1) else { + Issue.record("expected the second connection to read the schema the first loaded") + return + } + } + + private final class OlderKafkaPlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Kafka" + static let pluginVersion = "1.0.0" + static let pluginDescription = "A Kafka bundle from before the Schema Registry fields" + static let capabilities: [PluginCapability] = [.databaseDriver] + static let databaseTypeId = "Kafka" + static let databaseDisplayName = "Kafka" + static let iconName = "kafka-icon" + static let defaultPort = 9_092 + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + fatalError("Not used in tests") + } + } + + @Test("A Kafka bundle from before the registry fields keeps them, the password among the secure fields") + @MainActor + func olderPluginKeepsRegistryFields() { + let built = PluginMetadataRegistry.shared.buildMetadataSnapshot(from: OlderKafkaPlugin.self) + let fields = built.connection.additionalConnectionFields + #expect(Set(fields.map(\.id)).isSuperset(of: SchemaRegistryConnectionField.all)) + #expect(fields.first { $0.id == SchemaRegistryConnectionField.password }?.isSecure == true) + } + + @Test("A connection with no registry URL says so instead of fetching") + @MainActor + func noEndpoint() async { + let cache = SchemaRegistryCache(resolveEndpoint: { _ in nil }, transport: { _ in throw URLError(.badURL) }) + let connection = UUID() + guard case .failure(let error) = await cache.schema(connectionID: connection, schemaID: 1) else { + Issue.record("expected a failure") + return + } + #expect(error.reason.contains("Schema Registry URL")) + guard case .failure? = cache.cachedSchema(connectionID: connection, schemaID: 1) else { + Issue.record("expected the grid to read the same failure") + return + } + } + + @Test("The frame's schema ID reads from any byte sequence, and anything shorter or unframed has none") + func framePrefix() { + #expect(SchemaRegistryFrame.schemaID(prefix: Data([0, 0, 0, 1, 2, 9])) == 258) + #expect(SchemaRegistryFrame.schemaID(prefix: Data([0, 0, 0, 1])) == nil) + #expect(SchemaRegistryFrame.schemaID(prefix: Data([1, 0, 0, 1, 2])) == nil) + #expect(SchemaRegistryFrame.schemaID(prefix: "\u{0}\u{0}\u{0}\u{0}\u{7}{}".utf8) == 7) + } +} diff --git a/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift index 65749dc6e9..0e59862593 100644 --- a/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift +++ b/apple/TableProTests/Core/Storage/AvroSchemaStoreTests.swift @@ -29,6 +29,23 @@ struct AvroSchemaStoreTests { } } + @Test("A schema file that cannot be read is not read again on every cell, until it is added back") + func unreadableFileIsRememberedUntilAdded() throws { + try withDirectory { directory in + let id = AvroSchemaStore.schemaID(of: Self.schema) + let file = directory.appendingPathComponent(id).appendingPathExtension("avsc") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try Data("not json".utf8).write(to: file) + let store = AvroSchemaStore(directory: directory) + #expect(store.schema(id: id) == nil) + + try Data(Self.schema.utf8).write(to: file) + #expect(store.schema(id: id) == nil) + #expect(try store.add(Self.schema) == id) + #expect(store.schema(id: id)?.displayName == "Reading") + } + } + @Test("Text that is not a schema is refused and leaves nothing behind") func invalidTextIsRefused() throws { try withDirectory { directory in diff --git a/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift b/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift index d55432ed9e..47ba0cff3e 100644 --- a/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift +++ b/apple/TableProTests/Core/Storage/ValueDisplayFormatStorageTests.swift @@ -309,6 +309,28 @@ struct ValueDisplayFormatStorageTests { #expect(schemas.storedIDs.isEmpty) #expect(orphan != kept) } + + @Test("A Protobuf schema file goes once no saved format names it, beside the Avro ones") + func protobufSchemaFilesFollowTheFormatsThatNameThem() throws { + let defaults = try pool.make().defaults + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + let protobuf = ProtobufSchemaStore(directory: directory.appendingPathComponent("proto", isDirectory: true)) + let kept = try protobuf.add(Data("message A { int32 id = 1; }".utf8)) + let storage = ValueDisplayFormatStorage( + defaults: defaults, + schemas: AvroSchemaStore(directory: directory.appendingPathComponent("avro", isDirectory: true)), + protobufSchemas: ProtobufSchemaStore(directory: directory.appendingPathComponent("proto", isDirectory: true)) + ) + let target = scope("public", connectionId: UUID()) + + storage.save(["payload": .protobufSchema(schemaID: kept, message: "A")], for: target) + #expect(storage.load(for: target) == ["payload": .protobufSchema(schemaID: kept, message: "A")]) + #expect(protobuf.storedIDs == [kept]) + + storage.dropTable(target) + #expect(protobuf.storedIDs.isEmpty) + } } // What 0.83.1 decodes a table's formats as: a String enum of the names it shipped. diff --git a/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift b/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift new file mode 100644 index 0000000000..133cfb9680 --- /dev/null +++ b/apple/TableProTests/Views/DataGrid/DataGridSchemaRegistryTests.swift @@ -0,0 +1,178 @@ +// +// DataGridSchemaRegistryTests.swift +// TableProTests +// + +import Foundation +import os +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct DataGridSchemaRegistryTests { + private final class Requests: Sendable { + let ids = OSAllocatedUnfairLock<[String]>(initialState: []) + + var transport: SchemaRegistryClient.Transport { + { [self] request in + ids.withLock { $0.append(request.url?.lastPathComponent ?? "") } + let response = HTTPURLResponse(url: request.url!, statusCode: 404, httpVersion: nil, headerFields: nil)! + return (Data(#"{"message":"not here"}"#.utf8), response) + } + } + } + + private struct Grid { + let owner: GridTestOwner + let model: DataGridModel + let presenter: DataGridCellPresenter + let requests: Requests + } + + private static let connection = UUID() + + private static func grid(_ rows: TableRows) -> Grid { + let requests = Requests() + let endpoint = SchemaRegistryEndpoint(fields: [SchemaRegistryConnectionField.url: "http://localhost:8081"]) + let cache = SchemaRegistryCache(resolveEndpoint: { _ in endpoint }, transport: requests.transport) + let owner = GridTestOwner(rows) + let model = DataGridModel(schemaRegistry: cache) + model.dataSource = owner + model.actions = owner + owner.model = model + model.reloadRows() + return Grid(owner: owner, model: model, presenter: DataGridCellPresenter(model: model), requests: requests) + } + + private static func framed(_ id: UInt8, _ payload: [UInt8] = [0x02]) -> PluginCellValue { + .bytes(Data([0, 0, 0, 0, id] + payload)) + } + + private static func cachedText(_ grid: Grid, row: Int, column: Int) -> String? { + guard let id = grid.model.row(at: row)?.id, let box = grid.model.display.cache.box(forID: id), + column < box.values.count else { return nil } + return box.values[column] + } + + private static func fill(_ grid: Grid, rows: Int, columns: Int) { + for row in 0..