From 2b82f416f4b5432941a0b91d219a7694b3bd3420 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 22 Oct 2026 17:39:45 +0700 Subject: [PATCH 1/7] fix(mongodb): search every field type and escape connection string options --- .../MongoDBConnection.swift | 108 +++------- .../MongoDBConnectionURI.swift | 123 +++++++++++ .../MongoDBPluginDriver.swift | 29 ++- .../MongoDBQueryBuilder.swift | 107 ++++++---- .../MongoDBSSLMapping.swift | 20 +- .../MongoDBShownText.swift | 52 +++++ .../MongoDBTimeoutPolicy.swift | 6 +- .../MongoScriptRuntime.swift | 8 + .../Core/Coordinators/FindCoordinator.swift | 3 +- .../Core/Coordinators/FindMatcher.swift | 6 +- .../Coordinators/PaginationCoordinator.swift | 2 +- .../QueryExecutionCoordinator+Helpers.swift | 2 +- .../DatabaseManager+ScopedDriver.swift | 18 ++ .../Database/DatabaseManager+Tunnel.swift | 4 + .../Database/MongoToolsConnectionString.swift | 38 +++- .../Core/Plugins/DatabaseType+Registry.swift | 8 + ...ginMetadataRegistry+RegistryDefaults.swift | 4 +- .../Core/Plugins/PluginMetadataRegistry.swift | 9 + .../Connection/ConnectionURLFormatter.swift | 85 +++++--- .../Connection/ConnectionURLParser.swift | 13 +- .../TransientConnectionFactory.swift | 8 +- .../Views/Main/MainContentCoordinator.swift | 9 +- .../Drivers/MongoDBBackedDriver.swift | 45 +--- .../Drivers/MongoDBDialPlan.swift | 4 + .../Drivers/MongoDBOnIOSTests.swift | 13 +- .../MongoDBPluginConnectionTests.swift | 150 +++++++++++++- .../Drivers/MongoDBQueryBuilderTests.swift | 102 +++++++++ .../Core/Coordinators/FindMatcherTests.swift | 15 ++ .../Database/ScopedDriverRoutingTests.swift | 16 ++ ...tadataRegistryCuratedCapabilityTests.swift | 16 ++ .../ConnectionURLFormatterTests.swift | 59 ++++++ .../Database/NativeDumpRegistryTests.swift | 66 +++++- ...luginBrowsedQueryTabEditabilityTests.swift | 47 ++++- .../probes/check-mongodb-shown-text.sh | 194 ++++++++++++++++++ 34 files changed, 1140 insertions(+), 249 deletions(-) create mode 100644 apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift create mode 100644 apple/Plugins/MongoDBDriverPlugin/MongoDBShownText.swift create mode 100644 apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift create mode 100755 apple/scripts/probes/check-mongodb-shown-text.sh diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift index 095d0af10d..85dbc0dd31 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift @@ -236,86 +236,24 @@ nonisolated final class MongoDBConnection: @unchecked Sendable { // MARK: - URI Construction private func buildUri() -> String { - let scheme = useSrv ? "mongodb+srv" : "mongodb" - var uri = "\(scheme)://" - - if !user.isEmpty { - let encodedUser = user.addingPercentEncoding(withAllowedCharacters: .urlUserAllowed) ?? user - if let password = password, !password.isEmpty { - let encodedPassword = password.addingPercentEncoding( - withAllowedCharacters: .urlPasswordAllowed - ) ?? password - uri += "\(encodedUser):\(encodedPassword)@" - } else { - uri += "\(encodedUser)@" - } - } - - if useSrv { - let srvHost = Self.stripPort(fromSrvHost: host) - let encodedHost = srvHost.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed) ?? srvHost - uri += encodedHost - } else if let hosts = MongoDBHostList.uriHosts(host, defaultPort: port) { - uri += hosts - } else if host.contains(",") { - let segments = host.split(separator: ",").compactMap { segment -> String? in - let parts = segment.split(separator: ":", maxSplits: 1) - guard let first = parts.first else { return nil } - let h = String(first).trimmingCharacters(in: .whitespaces) - guard !h.isEmpty else { return nil } - let encodedH = h.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed) ?? h - if parts.count > 1 { - return "\(encodedH):\(parts[1].trimmingCharacters(in: .whitespaces))" - } - return "\(encodedH):\(port)" - } - uri += segments.isEmpty ? "localhost:\(port)" : segments.joined(separator: ",") - } else { - let encodedHost = host.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed) ?? host - uri += "\(encodedHost):\(port)" - } - - let encodedDb = database.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? database - uri += database.isEmpty ? "/" : "/\(encodedDb)" - - let encodedAuthSource = resolvedAuthSource - .addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? resolvedAuthSource - var params = connectTimeout.uriParameters - params.append("authSource=\(encodedAuthSource)") - - params.append(contentsOf: MongoDBSSLMapping.uriParameters( - for: ssl, - certificateKeyFile: clientCertificateFile?.path - )) - - if let rp = readPreference, !rp.isEmpty { - params.append("readPreference=\(rp)") - } - if let wc = writeConcern, !wc.isEmpty { - params.append("w=\(wc)") - } - if let mechanism = authMechanism, !mechanism.isEmpty { - params.append("authMechanism=\(mechanism)") - } - if let rs = replicaSet, !rs.isEmpty { - params.append("replicaSet=\(rs)") - } - - var explicitKeys: Set = [ - "connectTimeoutMS", "serverSelectionTimeoutMS", - "authSource", "authMechanism", "replicaSet", - "tls", "tlsAllowInvalidCertificates", "tlsAllowInvalidHostnames", - "tlsCAFile", "tlsCertificateKeyFile" - ] - if let readPreference, !readPreference.isEmpty { explicitKeys.insert("readPreference") } - if let writeConcern, !writeConcern.isEmpty { explicitKeys.insert("w") } - for (key, value) in extraUriParams where !explicitKeys.contains(key) { - let encodedValue = value.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? value - params.append("\(key)=\(encodedValue)") - } - - uri += "?" + params.joined(separator: "&") - return uri + MongoDBConnectionURI( + useSrv: useSrv, + host: host, + port: port, + user: user, + password: password, + database: database, + options: MongoDBConnectionURI.options( + connectTimeout: connectTimeout, + authSource: resolvedAuthSource, + ssl: MongoDBSSLMapping.uriOptions(for: ssl, certificateKeyFile: clientCertificateFile?.path), + readPreference: readPreference, + writeConcern: writeConcern, + authMechanism: authMechanism, + replicaSet: replicaSet, + extraParams: extraUriParams + ) + ).text } /// Strips a trailing `:port` from a hostname intended for an SRV URI. @@ -388,10 +326,13 @@ nonisolated final class MongoDBConnection: @unchecked Sendable { self.client = newClient let configuredWriteConcern = MongoWriteConcern(client: newClient) + // Read here, on the connection's queue, so building a browse query never waits on the server. + let version = self.fetchServerVersionSync() self.stateLock.lock() self._isConnected = true self._writeConcern = configuredWriteConcern + self._cachedServerVersion = version self.stateLock.unlock() logger.info("Connected to MongoDB at \(self.host):\(self.port)") @@ -611,6 +552,13 @@ nonisolated final class MongoDBConnection: @unchecked Sendable { return nil #endif } + + var knownServerVersion: String? { + stateLock.lock() + defer { stateLock.unlock() } + return _cachedServerVersion + } + func currentDatabase() -> String { database } // MARK: - Command Execution diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift new file mode 100644 index 0000000000..ee24ece5ce --- /dev/null +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift @@ -0,0 +1,123 @@ +// +// MongoDBConnectionURI.swift +// MongoDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +nonisolated struct MongoDBURIOption: Equatable, Sendable { + let key: String + let value: String +} + +nonisolated struct MongoDBConnectionURI: Sendable { + let useSrv: Bool + let host: String + let port: Int + let user: String + let password: String? + let database: String + let options: [MongoDBURIOption] + + // libmongoc splits options on a raw `&` and decodes every value, and the Go tools also split on `;` and read `+` as a + // space, so only RFC 3986 unreserved characters stay raw. Not `.alphanumerics`, which holds non-ASCII letters. + static let unreserved = CharacterSet( + charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" + ) + + static func encoded(_ component: String) -> String { + component.addingPercentEncoding(withAllowedCharacters: unreserved) ?? component + } + + var text: String { + let scheme = useSrv ? "mongodb+srv" : "mongodb" + let encodedDb = database.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? database + let path = database.isEmpty ? "/" : "/\(encodedDb)" + let query = options.map { "\(Self.encoded($0.key))=\(Self.encoded($0.value))" }.joined(separator: "&") + return "\(scheme)://\(userInfo)\(hosts)\(path)?\(query)" + } + + private var userInfo: String { + guard !user.isEmpty else { return "" } + let encodedUser = user.addingPercentEncoding(withAllowedCharacters: .urlUserAllowed) ?? user + guard let password, !password.isEmpty else { return "\(encodedUser)@" } + let encodedPassword = password.addingPercentEncoding(withAllowedCharacters: .urlPasswordAllowed) ?? password + return "\(encodedUser):\(encodedPassword)@" + } + + private var hosts: String { + if useSrv { + let srvHost = MongoDBConnection.stripPort(fromSrvHost: host) + return srvHost.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed) ?? srvHost + } + if let hosts = MongoDBHostList.uriHosts(host, defaultPort: port) { + return hosts + } + guard host.contains(",") else { + let encodedHost = host.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed) ?? host + return "\(encodedHost):\(port)" + } + // A list MongoDBHostList cannot read is passed on for libmongoc to refuse, with its ports encoded so a typed + // `/?` cannot start the options. + let segments = host.split(separator: ",").compactMap { segment -> String? in + let parts = segment.split(separator: ":", maxSplits: 1) + guard let first = parts.first else { return nil } + let name = String(first).trimmingCharacters(in: .whitespaces) + guard !name.isEmpty else { return nil } + let encodedName = name.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed) ?? name + guard parts.count > 1 else { return "\(encodedName):\(port)" } + return "\(encodedName):\(Self.encoded(parts[1].trimmingCharacters(in: .whitespaces)))" + } + return segments.isEmpty ? "localhost:\(port)" : segments.joined(separator: ",") + } + + // The fields own their keys, and a typed parameter named like one is dropped rather than sent after it, where + // libmongoc would let it win. Keys compare without case, as libmongoc reads them. + static func options( + connectTimeout: MongoDBConnectTimeout, + authSource: String, + ssl: [MongoDBURIOption], + readPreference: String?, + writeConcern: String?, + authMechanism: String?, + replicaSet: String?, + extraParams: [String: String] + ) -> [MongoDBURIOption] { + var options = connectTimeout.uriOptions + options.append(MongoDBURIOption(key: "authSource", value: authSource)) + options += ssl + let typed: [(key: String, value: String?)] = [ + ("readPreference", readPreference), + ("w", writeConcern), + ("authMechanism", authMechanism), + ("replicaSet", replicaSet) + ] + for (key, value) in typed { + guard let value, !value.isEmpty else { continue } + options.append(MongoDBURIOption(key: key, value: value)) + } + + var taken = Set(fieldOwnedKeys.map { $0.lowercased() }) + taken.formUnion(options.map { $0.key.lowercased() }) + // With TLS on, the SSL mode decides certificate checking, and libmongoc refuses a revocation switch beside the + // `tlsAllowInvalidCertificates` that already turns it off. + if !ssl.isEmpty { + taken.insert("tlsinsecure") + } + if ssl.contains(where: { $0.key == "tlsAllowInvalidCertificates" }) { + taken.formUnion(["tlsdisableocspendpointcheck", "tlsdisablecertificaterevocationcheck"]) + } + for key in extraParams.keys.sorted() where taken.insert(key.lowercased()).inserted { + options.append(MongoDBURIOption(key: key, value: extraParams[key] ?? "")) + } + return options + } + + static let fieldOwnedKeys = [ + "connectTimeoutMS", "serverSelectionTimeoutMS", + "authSource", "authMechanism", "replicaSet", + "tls", "tlsAllowInvalidCertificates", "tlsAllowInvalidHostnames", + "tlsCAFile", "tlsCertificateKeyFile" + ] +} diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 6fb96f2ab9..5683f3ef03 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -243,7 +243,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se from: documents, startTime: startTime, isTruncated: isTruncated, collection: collection, declared: declared, - typesIdentity: self.readsCollectionIdentities(outcome) + recordsKinds: outcome.returnsStoredDocuments(in: self.currentDb) ) } ), @@ -260,8 +260,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se /// one never sees. Read again on a first page, so a refresh picks up a changed validator, and /// reused while paging. private func declaredColumns(for outcome: MongoScriptStatementResult) async throws -> MongoDBCollectionSchema { - guard outcome.producedDocuments, - let find = outcome.find, find.returnsWholeDocuments, find.database == currentDb, + guard outcome.producedDocuments, outcome.returnsStoredDocuments(in: currentDb), let find = outcome.find, let collection = outcome.collection, !collection.isEmpty, let conn = mongoConnection else { return .empty @@ -968,7 +967,8 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se documents: { documents, collection, isTruncated in self.buildPluginResult( from: MongoReadDocuments(dictionaries: documents.dictionaries, texts: []), startTime: Date(), - isTruncated: isTruncated, collection: collection + isTruncated: isTruncated, collection: collection, + recordsKinds: outcome.returnsStoredDocuments(in: self.currentDb) ) } ) @@ -991,7 +991,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se isTruncated: Bool = false, collection: String = "", declared: MongoDBCollectionSchema = .empty, - typesIdentity: Bool = false + recordsKinds: Bool = false ) -> PluginQueryResult { let documents = read.dictionaries if documents.isEmpty { @@ -1006,13 +1006,13 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se let sampledKinds = BsonDocumentFlattener.columnKinds( for: sampled, documents: documents, representation: uuidRepresentation ) - rememberColumnKinds(sampledKinds, for: sampled, collection: collection) - if typesIdentity { + if recordsKinds { + rememberColumnKinds(sampledKinds, for: sampled, collection: collection) rememberIdentityKinds(of: documents, collection: collection) + rememberFieldPathKinds(from: documents, collection: collection) + rememberBinarySubtypes(of: documents, collection: collection) + rememberFieldKinds(of: documents, collection: collection) } - rememberFieldPathKinds(from: documents, collection: collection) - rememberBinarySubtypes(of: documents, collection: collection) - rememberFieldKinds(of: documents, collection: collection) let unseen = MongoDBCollectionShape.declaredColumnsMissing(from: sampled, schema: declared) let columns = sampled + unseen let kinds = sampledKinds + unseen.map { declared.field(named: $0)?.valueKind ?? .null } @@ -1094,12 +1094,6 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se } } - // A find in this database returns the collection's own `_id`s. A pipeline's `_id` is often a group key, and typing - // a grid row's `_id` by it aims the row's write at another document or at none. - private func readsCollectionIdentities(_ outcome: MongoScriptStatementResult) -> Bool { - outcome.find?.database == currentDb - } - // Added to rather than replaced: a row keeps the `_id` kinds every find of its collection held, so a later read of // one document, a value lookup or another tab's, never types the `_id` of a row it did not return. private func rememberIdentityKinds(of documents: [[String: Any]], collection: String) { @@ -1141,7 +1135,8 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se let normalizer = rawFilterNormalizer return MongoDBQueryBuilder( columnKinds: filterKinds(for: collection, database: database), - rawFilterNormalizer: { normalizer.normalize($0) } + rawFilterNormalizer: { normalizer.normalize($0) }, + matchesShownText: MongoDBCapabilities.parse(mongoConnection?.knownServerVersion).supportsRegexMatch ) } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift index 8f0b087f08..ae3f7f42ea 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift @@ -14,23 +14,28 @@ nonisolated struct MongoDBQueryBuilder { /// typed by the user, instead of a field name and an operator. static let rawFilterColumn = "__RAW__" - /// Matches no document, on any collection, without a collection scan. Emitted when every - /// condition of a non-empty filter had to be dropped, because widening to the whole - /// collection would report success while showing rows the filter excludes. + /// Matches no document, on any collection, without a collection scan. Emitted when a condition + /// of a non-empty filter cannot be built, because running the rest would report success while + /// showing rows the filter excludes. static let impossibleFilter = "{\"_id\": {\"$in\": []}}" let columnKinds: [String: BsonValueKind] /// Rewrites a raw filter row into canonical Extended JSON, so `find` and `countDocuments` - /// receive the same document. Without one, raw filter rows are dropped. + /// receive the same document. Without one, a raw filter row refuses the filter. let rawFilterNormalizer: (@Sendable (String) -> String?)? + // From MongoDB 4.2 a contains, starts with or ends with filter also matches a value that is not a string by its text. + let matchesShownText: Bool + init( columnKinds: [String: BsonValueKind] = [:], - rawFilterNormalizer: (@Sendable (String) -> String?)? = nil + rawFilterNormalizer: (@Sendable (String) -> String?)? = nil, + matchesShownText: Bool = false ) { self.columnKinds = columnKinds self.rawFilterNormalizer = rawFilterNormalizer + self.matchesShownText = matchesShownText } // MARK: - Base Query @@ -58,12 +63,13 @@ nonisolated struct MongoDBQueryBuilder { columns: [String] = [], limit: Int = 200, offset: Int = 0 - ) -> String { - buildQuery( - collection: collection, - filterDocument: buildFilterDocument(from: queryFilters, logicMode: logicMode), - sortColumns: sortColumns, columns: columns, limit: limit, offset: offset - ) + ) -> String? { + exactFilterDocument(from: queryFilters, logicMode: logicMode).map { + buildQuery( + collection: collection, filterDocument: $0, + sortColumns: sortColumns, columns: columns, limit: limit, offset: offset + ) + } } func buildQuery( @@ -125,6 +131,15 @@ nonisolated struct MongoDBQueryBuilder { from filters: [PluginQueryFilter], logicMode: String = "and" ) -> String { + exactFilterDocument(from: filters, logicMode: logicMode) ?? Self.impossibleFilter + } + + /// Nil when a row cannot be built: a raw document that does not normalize, a name that reads as an operator, a + /// missing bound. Running the other rows without it would, under "match all", show rows the filter excludes. + func exactFilterDocument( + from filters: [PluginQueryFilter], + logicMode: String = "and" + ) -> String? { guard !filters.isEmpty else { return "{}" } var clauses: [MongoDBFilterClause] = [] @@ -133,15 +148,13 @@ nonisolated struct MongoDBQueryBuilder { for filter in filters { guard filter.column != Self.rawFilterColumn else { - if let raw = rawFilterClause(for: filter) { - clauses.append(raw) - } + guard let raw = rawFilterClause(for: filter) else { return nil } + clauses.append(raw) continue } guard let scope = filter.elementScope, !scope.isEmpty else { - if let clause = buildClause(for: filter, field: filter.column) { - clauses.append(clause) - } + guard let clause = buildClause(for: filter, field: filter.column) else { return nil } + clauses.append(clause) continue } if scoped[scope] == nil { @@ -153,12 +166,10 @@ nonisolated struct MongoDBQueryBuilder { for scope in scopeOrder { guard let clause = elementMatchClause( scope: scope, filters: scoped[scope] ?? [], logicMode: logicMode - ) else { continue } + ) else { return nil } clauses.append(clause) } - guard !clauses.isEmpty else { return Self.impossibleFilter } - if clauses.count == 1 { return "{\(clauses[0].json)}" } @@ -240,10 +251,11 @@ nonisolated struct MongoDBQueryBuilder { logicMode: String ) -> MongoDBFilterClause? { guard Self.isPlainFieldName(scope) else { return nil } + // `$expr` reads the top-level document only, so a condition on one element matches by `$regex` alone. let inner = filters.compactMap { filter -> MongoDBFilterClause? in - buildClause(for: filter, field: Self.relativePath(filter.column, under: scope)) + buildClause(for: filter, field: Self.relativePath(filter.column, under: scope), readsShownText: false) } - guard !inner.isEmpty else { return nil } + guard !inner.isEmpty, inner.count == filters.count else { return nil } let body: String if logicMode == "and" || inner.count == 1 { @@ -269,12 +281,22 @@ nonisolated struct MongoDBQueryBuilder { MongoCollectionAccessor.expression(for: name) } - private func buildClause(for filter: PluginQueryFilter, field rawField: String) -> MongoDBFilterClause? { + private func buildClause( + for filter: PluginQueryFilter, + field rawField: String, + readsShownText: Bool = true + ) -> MongoDBFilterClause? { guard Self.isPlainFieldName(rawField), Self.isPlainFieldName(filter.column) else { return nil } let kind = columnKinds[filter.column] let field = Self.escapeJsonString(rawField) let ignoresCase = !filter.isCaseSensitive && MongoDBFilterValue.supportsRegexMatching(kind) let value = filter.value + let pattern = { (pattern: String, negated: Bool) in + patternClause( + field: rawField, pattern: pattern, ignoresCase: !filter.isCaseSensitive, + negated: negated, readsShownText: readsShownText + ) + } switch filter.op { case "=": @@ -311,23 +333,13 @@ nonisolated struct MongoDBQueryBuilder { case "<=": return MongoDBFilterClause(key: field, body: "{\"$lte\": \(typed(value, kind))}") case "CONTAINS": - return MongoDBFilterClause( - key: field, body: Self.regexBody(pattern: escapeRegexChars(value), ignoresCase: ignoresCase) - ) + return pattern(escapeRegexChars(value), false) case "NOT CONTAINS": - return MongoDBFilterClause( - key: field, body: Self.negatedRegexBody(pattern: escapeRegexChars(value), ignoresCase: ignoresCase) - ) + return pattern(escapeRegexChars(value), true) case "STARTS WITH": - let pattern = "^\(escapeRegexChars(value))" - return MongoDBFilterClause( - key: field, body: Self.regexBody(pattern: pattern, ignoresCase: ignoresCase) - ) + return pattern("^\(escapeRegexChars(value))", false) case "ENDS WITH": - let pattern = "\(escapeRegexChars(value))$" - return MongoDBFilterClause( - key: field, body: Self.regexBody(pattern: pattern, ignoresCase: ignoresCase) - ) + return pattern("\(escapeRegexChars(value))$", false) case "IS NULL": return MongoDBFilterClause(key: field, body: "null") case "IS NOT NULL": @@ -361,6 +373,27 @@ nonisolated struct MongoDBQueryBuilder { } } + /// A `$regex` matches a string, or a string in an array, and nothing else. A pattern asks about text, so the case + /// it ignores is the filter's own, whatever the column's sampled kind. + private func patternClause( + field: String, + pattern: String, + ignoresCase: Bool, + negated: Bool, + readsShownText: Bool + ) -> MongoDBFilterClause { + let key = Self.escapeJsonString(field) + guard matchesShownText, readsShownText, MongoDBShownText.reads(field: field) else { + let body = negated + ? Self.negatedRegexBody(pattern: pattern, ignoresCase: ignoresCase) + : Self.regexBody(pattern: pattern, ignoresCase: ignoresCase) + return MongoDBFilterClause(key: key, body: body) + } + let regex = "{\"\(key)\": \(Self.regexBody(pattern: pattern, ignoresCase: ignoresCase))}" + let shown = MongoDBShownText.regexMatch(field: field, pattern: pattern, ignoresCase: ignoresCase) + return MongoDBFilterClause(key: negated ? "$nor" : "$or", body: "[\(regex), \(shown)]") + } + /// `secondValue` is authoritative when the caller supplies it. `value` still arrives joined as /// `lower + "," + upper`, because plugins built before `secondValue` existed parse that form, /// so the upper bound and its separator are stripped off the end rather than split on the diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBSSLMapping.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBSSLMapping.swift index d6b571d830..4ab18123b0 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBSSLMapping.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBSSLMapping.swift @@ -2,30 +2,24 @@ import Foundation import TableProPluginKit nonisolated enum MongoDBSSLMapping { - static func uriParameters(for ssl: SSLConfiguration, certificateKeyFile: String? = nil) -> [String] { + static func uriOptions(for ssl: SSLConfiguration, certificateKeyFile: String? = nil) -> [MongoDBURIOption] { guard ssl.isEnabled else { return [] } - var params: [String] = ["tls=true"] + var options = [MongoDBURIOption(key: "tls", value: "true")] switch ssl.mode { case .preferred, .required: - params.append("tlsAllowInvalidCertificates=true") + options.append(MongoDBURIOption(key: "tlsAllowInvalidCertificates", value: "true")) case .verifyCa: - params.append("tlsAllowInvalidHostnames=true") + options.append(MongoDBURIOption(key: "tlsAllowInvalidHostnames", value: "true")) case .disabled, .verifyIdentity: break } if ssl.verifiesCertificate, !ssl.caCertificatePath.isEmpty { - let encoded = ssl.caCertificatePath - .addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) - ?? ssl.caCertificatePath - params.append("tlsCAFile=\(encoded)") + options.append(MongoDBURIOption(key: "tlsCAFile", value: ssl.caCertificatePath)) } let certificateKeyFile = certificateKeyFile ?? ssl.clientCertificatePath if !certificateKeyFile.isEmpty { - let encoded = certificateKeyFile - .addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) - ?? certificateKeyFile - params.append("tlsCertificateKeyFile=\(encoded)") + options.append(MongoDBURIOption(key: "tlsCertificateKeyFile", value: certificateKeyFile)) } - return params + return options } } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBShownText.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBShownText.swift new file mode 100644 index 0000000000..a9b8bc4d63 --- /dev/null +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBShownText.swift @@ -0,0 +1,52 @@ +// +// MongoDBShownText.swift +// MongoDBDriverPlugin +// + +import Foundation + +// A `$regex` matches strings only, so a pattern also matches a number, a date or an ObjectId by the text the grid shows +// for it, the way a SQL filter casts the column. Measured on 8.0: `$convert` throws past its onError on binary data and +// on a date outside years 0 to 9999, alone or inside an array or a document, and one such value fails the whole find. So +// only scalars that always convert are converted, and every other value reads as empty, left to the `$regex` arm. +nonisolated enum MongoDBShownText { + static let convertedTypes = ["double", "int", "long", "decimal", "bool", "objectId"] + static let earliestDateMilliseconds: Int64 = -62_167_219_200_000 + static let latestDateMilliseconds: Int64 = 253_402_300_799_999 + + // An aggregation field path refuses an empty segment and one that starts with `$`, and the error fails the whole + // find, where a `$regex` on the same name only matches nothing. + static func reads(field: String) -> Bool { + field.split(separator: ".", omittingEmptySubsequences: false).allSatisfy { !$0.isEmpty && !$0.hasPrefix("$") } + } + + static func regexMatch(field: String, pattern: String, ignoresCase: Bool) -> String { + let regex = MongoDBQueryBuilder.escapeJsonString(pattern) + let options = ignoresCase ? ", \"options\": \"i\"" : "" + return "{\"$expr\": {\"$regexMatch\": {\"input\": \(expression(field: field)), \"regex\": \"\(regex)\"\(options)}}}" + } + + // The grid writes a date's milliseconds only when it has them. + static func expression(field: String) -> String { + let path = MongoDBQueryBuilder.escapeJsonString("$" + field) + let types = convertedTypes.map { "\"\($0)\"" }.joined(separator: ", ") + let branches = [ + "{\"case\": {\"$in\": [{\"$type\": \"$$v\"}, [\(types)]]}, \"then\": {\"$toString\": \"$$v\"}}", + "{\"case\": {\"$ne\": [{\"$type\": \"$$v\"}, \"date\"]}, \"then\": \"\"}", + "{\"case\": {\"$lt\": [\"$$v\", \(dateJson(earliestDateMilliseconds))]}, \"then\": \"\"}", + "{\"case\": {\"$gt\": [\"$$v\", \(dateJson(latestDateMilliseconds))]}, \"then\": \"\"}", + "{\"case\": {\"$eq\": [{\"$millisecond\": \"$$v\"}, 0]}, \"then\": \(dateText("%Y-%m-%dT%H:%M:%SZ"))}" + ] + let choice = "{\"$switch\": {\"branches\": [\(branches.joined(separator: ", "))], " + + "\"default\": \(dateText("%Y-%m-%dT%H:%M:%S.%LZ"))}}" + return "{\"$let\": {\"vars\": {\"v\": \"\(path)\"}, \"in\": \(choice)}}" + } + + private static func dateJson(_ milliseconds: Int64) -> String { + "{\"$date\": {\"$numberLong\": \"\(milliseconds)\"}}" + } + + private static func dateText(_ format: String) -> String { + "{\"$dateToString\": {\"date\": \"$$v\", \"format\": \"\(format)\"}}" + } +} diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift index 9769d7bbf2..086ec24abc 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift @@ -19,10 +19,10 @@ nonisolated struct MongoDBConnectTimeout: Equatable, Sendable { self.milliseconds = Int32(min(max(milliseconds, 1), Self.maximumMilliseconds)) } - var uriParameters: [String] { + var uriOptions: [MongoDBURIOption] { [ - "connectTimeoutMS=\(milliseconds)", - "serverSelectionTimeoutMS=\(milliseconds)" + MongoDBURIOption(key: "connectTimeoutMS", value: String(milliseconds)), + MongoDBURIOption(key: "serverSelectionTimeoutMS", value: String(milliseconds)) ] } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift b/apple/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift index 11346bdde7..2d68fe336e 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift @@ -22,6 +22,14 @@ nonisolated struct MongoScriptStatementResult: Sendable { var databaseSwitch: String? var producedDocuments = false var rowsAffected = 0 + + // Only a find in the session's database with no projection returns the collection's documents as stored. A + // pipeline's `_id` is often a group key and a projection can compute any field, so learning kinds from either would + // type the table tab's next write or filter by a value no document holds. + func returnsStoredDocuments(in currentDatabase: String) -> Bool { + guard let find else { return false } + return find.returnsWholeDocuments && find.database == currentDatabase + } } /// Runs mongosh-shaped JavaScript against the connection. diff --git a/apple/TablePro/Core/Coordinators/FindCoordinator.swift b/apple/TablePro/Core/Coordinators/FindCoordinator.swift index b100f0c842..c796b1f32e 100644 --- a/apple/TablePro/Core/Coordinators/FindCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/FindCoordinator.swift @@ -66,9 +66,10 @@ final class FindCoordinator: ObservableObject { guard !term.isEmpty else { return } let buffer = parent.tabSessionRegistry.tableRows(for: tab.id) + let readsAnyColumn = parent.connection.type.patternFiltersReadAnyColumn let searchable = buffer.columns.indices.filter { index in guard index < buffer.columnTypes.count else { return true } - return FindMatcher.isServerSearchable(buffer.columnTypes[index]) + return FindMatcher.isServerSearchable(buffer.columnTypes[index], patternReadsAnyColumn: readsAnyColumn) } guard !searchable.isEmpty else { return } diff --git a/apple/TablePro/Core/Coordinators/FindMatcher.swift b/apple/TablePro/Core/Coordinators/FindMatcher.swift index 68776ab149..766f9d7d36 100644 --- a/apple/TablePro/Core/Coordinators/FindMatcher.swift +++ b/apple/TablePro/Core/Coordinators/FindMatcher.swift @@ -20,8 +20,10 @@ enum FindMatcher { /// Columns a `LIKE` comparison can be built against without a cast. Postgres has no implicit /// integer to text conversion, so `id ILIKE '%abc%'` fails the whole query rather than matching /// nothing. Client-side matching is looser on purpose: it compares the text already on screen. - static func isServerSearchable(_ columnType: ColumnType?) -> Bool { - switch columnType { + /// An engine whose pattern filters read any column by its shown text matches what the page does. + static func isServerSearchable(_ columnType: ColumnType?, patternReadsAnyColumn: Bool = false) -> Bool { + if patternReadsAnyColumn { return isSearchable(columnType) } + return switch columnType { case .text, .enumType, .set, nil: true default: false } diff --git a/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift b/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift index ea15b59786..00e7e4e257 100644 --- a/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift @@ -251,7 +251,7 @@ final class PaginationCoordinator: ObservableObject { countSQL: String? ) async -> Result { do { - let count = try await DatabaseManager.shared.withMetadataDriver(scope: scope, workload: .bulk) { driver in + let count = try await DatabaseManager.shared.withRowCountDriver(scope: scope, filtered: !filters.isEmpty) { driver in if let browseFilters { return try await driver.fetchExactRowCount(table: tableName, browseFilters: browseFilters) } diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift index 5c4f368cdf..fa85298a11 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift @@ -780,7 +780,7 @@ extension QueryExecutionCoordinator { }) else { return nil } return .count(count, isApproximate: true) case let .filteredNonSQL(filters, logicMode): - guard let count = try? await DatabaseManager.shared.withMetadataDriver(scope: scope, workload: .bulk, { driver in + guard let count = try? await DatabaseManager.shared.withRowCountDriver(scope: scope, filtered: true, { driver in try await driver.fetchFilteredRowCount(table: tableName, filters: filters, logicMode: logicMode) }) else { return .clear } return .count(count, isApproximate: false) diff --git a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift index fb86a6fd57..5f3389174b 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift @@ -130,6 +130,24 @@ extension DatabaseManager { } } + /// On an engine that types a filter by the rows its driver read, a filtered count runs where the page did, or the + /// same filter would match other rows. + func rowCountRunsWithPage(_ scope: DatabaseScope, filtered: Bool) -> Bool { + filtered && activeSessions[scope.connectionId]?.connection.type.filtersTypedByReadRows == true + } + + /// A count that runs with the page is cancelled on the server when a page turn supersedes it. + func withRowCountDriver( + scope: DatabaseScope, + filtered: Bool, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T + ) async throws -> T { + guard rowCountRunsWithPage(scope, filtered: filtered) else { + return try await withMetadataDriver(scope: scope, workload: .bulk, body) + } + return try await withCancellableRead(scope: scope, route: executionRoute(for: scope), body) + } + /// A table tab's read is a SELECT the app built from the tab's own table, so it depends on /// nothing the session holds: no transaction, no temp table, no variable. That makes it the one /// kind of work that can follow a route change it waited through. A database switch on an engine diff --git a/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift b/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift index af4fef7ec8..9c63818a6b 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift @@ -72,6 +72,10 @@ extension DatabaseManager { } } if connection.type.pluginTypeId == "MongoDB", !connection.usesMongoSrv { + // The driver reads option keys without case, so an imported `directconnection=false` would still apply. + for key in effectiveFields.keys where key.lowercased() == "mongoparam_directconnection" { + effectiveFields[key] = nil + } effectiveFields["mongoParam_directConnection"] = "true" } if connection.type.pluginTypeId == "Redis" { diff --git a/apple/TablePro/Core/Database/MongoToolsConnectionString.swift b/apple/TablePro/Core/Database/MongoToolsConnectionString.swift index 0d8d7c1b03..b33521c85f 100644 --- a/apple/TablePro/Core/Database/MongoToolsConnectionString.swift +++ b/apple/TablePro/Core/Database/MongoToolsConnectionString.swift @@ -9,13 +9,25 @@ import TableProPluginKit internal enum MongoToolsConnectionString { private static let defaultAuthenticationDatabase = "admin" - private static let queryValueAllowed = CharacterSet.urlQueryAllowed.subtracting(CharacterSet(charactersIn: "&=+#")) + // The driver refuses these mechanisms with any other auth database, as the plugin's resolver knows. + private static let externalMechanisms: Set = ["MONGODB-X509", "MONGODB-AWS"] + + // The tools' Go driver splits options on `;` as well as `&` and reads `+` as a space, so only RFC 3986 unreserved + // characters stay raw. + private static let queryValueAllowed = CharacterSet( + charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" + ) private static let fieldOwnedParameterKeys: Set = [ "authSource", "authMechanism", "replicaSet", "tls", "tlsAllowInvalidCertificates", "tlsAllowInvalidHostnames", "tlsCAFile", "tlsCertificateKeyFile" ] + // With TLS on, the SSL mode decides certificate checking, and the driver refuses a revocation switch beside the + // `tlsInsecure` that already turns it off. + private static let ownedByTLS = ["tlsInsecure"] + private static let impliedByInsecure = ["tlsDisableOCSPEndpointCheck", "tlsDisableCertificateRevocationCheck"] + static func make(for connection: DatabaseConnection, host: String) -> String { let scheme = connection.usesMongoSrv ? "mongodb+srv" : "mongodb" let base = "\(scheme)://\(userInfo(connection))\(hosts(connection, host: host))/" @@ -27,6 +39,9 @@ internal enum MongoToolsConnectionString { if let explicit = connection.mongoAuthSource { return explicit } + if let mechanism = connection.mongoAuthMechanism, externalMechanisms.contains(mechanism.uppercased()) { + return "$external" + } guard !connection.usesMongoSrv, !connection.database.isEmpty else { return defaultAuthenticationDatabase } @@ -63,7 +78,7 @@ internal enum MongoToolsConnectionString { return encodedHost(srvName.host) } return entries - .map { "\(encodedHost($0.host)):\($0.port ?? String(connection.port))" } + .map { "\(encodedHost($0.host)):\($0.port.map(encodedValue) ?? String(connection.port))" } .joined(separator: ",") } @@ -81,6 +96,10 @@ internal enum MongoToolsConnectionString { let port = remainder.hasPrefix(":") ? String(remainder.dropFirst()) : "" return (String(entry[...closing]), port.isEmpty ? nil : port) } + // A bare IPv6 address has no port, and the driver reads it only in brackets. + if entry.filter({ $0 == ":" }).count > 1 { + return ("[\(entry)]", nil) + } guard let colon = entry.lastIndex(of: ":") else { return (entry, nil) } let port = String(entry[entry.index(after: colon)...]) return (String(entry[..) -> [String] { - connection.additionalFields + // The driver reads keys without case, so a typed parameter named like a field's key in any case is dropped. + private static func extraParameters(_ connection: DatabaseConnection, excluding keys: [String]) -> [String] { + var taken = Set(keys.map { $0.lowercased() }) + return connection.additionalFields .compactMap { key, value -> (String, String)? in guard key.hasPrefix("mongoParam_") else { return nil } let name = String(key.dropFirst("mongoParam_".count)) - guard !name.isEmpty, !keys.contains(name) else { return nil } + guard !name.isEmpty else { return nil } return (name, value) } .sorted { $0.0 < $1.0 } + .filter { taken.insert($0.0.lowercased()).inserted } .map { "\(encodedValue($0.0))=\(encodedValue($0.1))" } } diff --git a/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift b/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift index 78e5dd8cce..2834f53641 100644 --- a/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift +++ b/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift @@ -129,6 +129,14 @@ extension DatabaseType { PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.columnsAreSampled ?? false } + var patternFiltersReadAnyColumn: Bool { + PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.patternFiltersReadAnyColumn ?? false + } + + var filtersTypedByReadRows: Bool { + PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.filtersTypedByReadRows ?? false + } + var authenticationIsDatabaseScoped: Bool { PluginMetadataRegistry.shared.snapshot(for: self)? .capabilities.authenticationIsDatabaseScoped ?? false diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index f5e4420667..106c33ea2a 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -67,7 +67,9 @@ extension PluginMetadataRegistry { supportsDropIndex: false, supportsOpportunisticTLS: false, authenticationIsDatabaseScoped: true, - columnsAreSampled: true + columnsAreSampled: true, + patternFiltersReadAnyColumn: true, + filtersTypedByReadRows: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "public", diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift index 760039e9b1..9e15a69745 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -103,6 +103,13 @@ struct PluginMetadataSnapshot: Sendable { /// MongoDB collection lists the fields found in its first documents, so a field missing from /// one side's list says nothing about whether that side holds it. var columnsAreSampled: Bool = false + /// Whether a contains, starts with or ends with filter matches a column of any type by the text it shows, so a + /// search can send number and date columns too. MongoDB's builder converts the value before it matches. + var patternFiltersReadAnyColumn: Bool = false + /// Whether the driver types a filter's values by the kinds it learned from rows it read, so a count of a + /// filtered table runs on the driver that read the page. A pooled driver read nothing and would type the same + /// filter another way: a date compared as a string counts no rows. + var filtersTypedByReadRows: Bool = false var isEngineReadOnly: Bool = false /// Which connection field carries the path of the local database file this driver opens, @@ -774,6 +781,8 @@ final class PluginMetadataRegistry: @unchecked Sendable { supportsColumnSort: existingSnapshot?.capabilities.supportsColumnSort ?? true, supportsMatchAnyFilters: existingSnapshot?.capabilities.supportsMatchAnyFilters ?? true, columnsAreSampled: existingSnapshot?.capabilities.columnsAreSampled ?? false, + patternFiltersReadAnyColumn: existingSnapshot?.capabilities.patternFiltersReadAnyColumn ?? false, + filtersTypedByReadRows: existingSnapshot?.capabilities.filtersTypedByReadRows ?? false, isEngineReadOnly: existingSnapshot?.capabilities.isEngineReadOnly ?? false, localFilePathField: existingSnapshot?.capabilities.localFilePathField, newDatabaseFileExtensions: existingSnapshot?.capabilities.newDatabaseFileExtensions ?? [], diff --git a/apple/TablePro/Core/Utilities/Connection/ConnectionURLFormatter.swift b/apple/TablePro/Core/Utilities/Connection/ConnectionURLFormatter.swift index f15adf5280..3f456fdffd 100644 --- a/apple/TablePro/Core/Utilities/Connection/ConnectionURLFormatter.swift +++ b/apple/TablePro/Core/Utilities/Connection/ConnectionURLFormatter.swift @@ -34,6 +34,10 @@ struct ConnectionURLFormatter { ) } + if connection.type.pluginTypeId == DatabaseType.mongodb.rawValue, connection.mongoUseSrv { + return formatMongoSrv(connection, password: password) + } + return formatStandard(connection, scheme: scheme, defaultPort: schemeType.defaultPort, password: password) } @@ -83,6 +87,26 @@ struct ConnectionURLFormatter { return url } + // An SRV record names the servers and their ports, so the URL carries the lookup name alone. + private static func formatMongoSrv(_ connection: DatabaseConnection, password: String?) -> String { + var result = "mongodb+srv://" + if !connection.username.isEmpty { + result += percentEncodeUserinfo(connection.username) + if let password, !password.isEmpty { + result += ":\(percentEncodeUserinfo(password))" + } + result += "@" + } + let listed = connection.additionalFields["mongoHosts"]? + .split(separator: ",").first.map { $0.trimmingCharacters(in: .whitespaces) } + let host = listed.flatMap { $0.isEmpty ? nil : $0 } ?? connection.host + result += host.split(separator: ":").first.map(String.init) ?? host + let database = connection.database + result += "/\(database.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? database)" + let query = buildQueryString(connection) + return query.isEmpty ? result : "\(result)?\(query)" + } + private static func formatSSH( _ connection: DatabaseConnection, sshConfig ssh: SSHConfiguration, @@ -199,19 +223,19 @@ struct ConnectionURLFormatter { return result } - private static let socketPathAllowed = CharacterSet( + private static let unreservedAllowed = CharacterSet( charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" ) // MySQL's documented form, `socket=%2Ftmp%2Fmysql.sock`. Only unreserved characters stay // literal, so `+` and a space survive both percent and form decoding. private static func percentEncodeSocketPath(_ path: String) -> String { - path.addingPercentEncoding(withAllowedCharacters: socketPathAllowed) ?? path + path.addingPercentEncoding(withAllowedCharacters: unreservedAllowed) ?? path } // libpq's documented form, `?host=/var/run/postgresql`: a query keeps `/` as it is. private static func percentEncodeSocketDirectory(_ path: String) -> String { - path.addingPercentEncoding(withAllowedCharacters: socketPathAllowed.union(CharacterSet(charactersIn: "/"))) ?? path + path.addingPercentEncoding(withAllowedCharacters: unreservedAllowed.union(CharacterSet(charactersIn: "/"))) ?? path } // Trino and Presto read the path as /catalog/schema, as ConnectionURLParser does. @@ -280,26 +304,42 @@ struct ConnectionURLFormatter { params.append("env=\(encoded)") } - if let authSource = connection.mongoAuthSource, !authSource.isEmpty { - params.append("authSource=\(percentEncodeQueryValue(authSource))") - } - if let authMech = connection.mongoAuthMechanism, !authMech.isEmpty { - params.append("authMechanism=\(percentEncodeQueryValue(authMech))") - } - if let replicaSet = connection.mongoReplicaSet, !replicaSet.isEmpty { - params.append("replicaSet=\(percentEncodeQueryValue(replicaSet))") - } - if connection.mongoUseSrv { - params.append("mongoUseSrv=true") - } - if let uuidRepresentation = connection.additionalFields["mongoUuidRepresentation"], - !uuidRepresentation.isEmpty { - params.append("uuidRepresentation=\(percentEncodeQueryValue(uuidRepresentation))") - } + params.append(contentsOf: mongoParameters(connection)) return params.joined(separator: "&") } + // The standard option names, so the URL opens in other MongoDB tools as well as here. The fields come first and a + // typed parameter named like one is left out, as the driver reads keys without case. + private static func mongoParameters(_ connection: DatabaseConnection) -> [String] { + let fields: [(String, String?)] = [ + ("authSource", connection.mongoAuthSource), + ("authMechanism", connection.mongoAuthMechanism), + ("replicaSet", connection.mongoReplicaSet), + ("readPreference", connection.mongoReadPreference), + ("w", connection.mongoWriteConcern), + ("uuidRepresentation", connection.additionalFields["mongoUuidRepresentation"]?.nilIfEmpty) + ] + var options = fields.compactMap { key, value in value.flatMap { $0.isEmpty ? nil : (key, $0) } } + var taken = Set(options.map { $0.0.lowercased() }) + let typed = connection.additionalFields + .compactMap { key, value -> (String, String)? in + guard key.hasPrefix("mongoParam_"), !value.isEmpty else { return nil } + let name = String(key.dropFirst("mongoParam_".count)) + return name.isEmpty ? nil : (name, value) + } + .sorted { $0.0 < $1.0 } + for option in typed where taken.insert(option.0.lowercased()).inserted { + options.append(option) + } + return options.map { "\(percentEncodeMongoOption($0.0))=\(percentEncodeMongoOption($0.1))" } + } + + // Other MongoDB drivers split options on `;` and read `+` as a space, so only unreserved characters stay literal. + private static func percentEncodeMongoOption(_ value: String) -> String { + value.addingPercentEncoding(withAllowedCharacters: unreservedAllowed) ?? value + } + private static func colorHex(_ color: ConnectionColor) -> String? { switch color { case .none: return nil @@ -341,11 +381,4 @@ struct ConnectionURLFormatter { allowed.remove(charactersIn: ":@") return value.addingPercentEncoding(withAllowedCharacters: allowed) ?? value } - - private static func percentEncodeQueryValue(_ value: String) -> String { - value.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed)? - .replacingOccurrences(of: "&", with: "%26") - .replacingOccurrences(of: "=", with: "%3D") - ?? value - } } diff --git a/apple/TablePro/Core/Utilities/Connection/ConnectionURLParser.swift b/apple/TablePro/Core/Utilities/Connection/ConnectionURLParser.swift index ef1d1565e0..2d94281d0e 100644 --- a/apple/TablePro/Core/Utilities/Connection/ConnectionURLParser.swift +++ b/apple/TablePro/Core/Utilities/Connection/ConnectionURLParser.swift @@ -769,10 +769,11 @@ struct ConnectionURLParser { let params = queryString.split(separator: "&", omittingEmptySubsequences: true) for param in params { let parts = param.split(separator: "=", maxSplits: 1) - guard let key = parts.first else { continue } - let value = parts.count > 1 ? String(parts[1]) : nil - guard let value else { continue } - let keyStr = String(key).lowercased() + guard let encodedKey = parts.first, parts.count > 1 else { continue } + // Decoded once, as URLQueryItem decodes the standard form's query. + let key = String(encodedKey).removingPercentEncoding ?? String(encodedKey) + let value = String(parts[1]).removingPercentEncoding ?? String(parts[1]) + let keyStr = key.lowercased() if keyStr == "useprivatekey" { ext.usePrivateKey = value.lowercased() == "true" continue @@ -786,10 +787,10 @@ struct ConnectionURLParser { continue } if keyStr == "agentsocket" { - ext.agentSocket = value.removingPercentEncoding ?? value + ext.agentSocket = value continue } - applyQueryParam(key: String(key), value: value, to: &ext, dbType: dbType) + applyQueryParam(key: key, value: value, to: &ext, dbType: dbType) } return ext } diff --git a/apple/TablePro/Core/Utilities/Connection/TransientConnectionFactory.swift b/apple/TablePro/Core/Utilities/Connection/TransientConnectionFactory.swift index 3eb6b1cd6a..e9d4da2bfe 100644 --- a/apple/TablePro/Core/Utilities/Connection/TransientConnectionFactory.swift +++ b/apple/TablePro/Core/Utilities/Connection/TransientConnectionFactory.swift @@ -64,8 +64,14 @@ internal enum TransientConnectionFactory { connection.additionalFields.merge(parsed.additionalFields) { _, imported in imported } + // The same field mapping as Import from URL, so an opened link reads legacy UUIDs the way its sender set. for (key, value) in parsed.mongoQueryParams where !value.isEmpty { - if key != "authMechanism" && key != "replicaSet" { + switch key { + case "authMechanism", "replicaSet": + continue + case "uuidRepresentation": + connection.additionalFields["mongoUuidRepresentation"] = value + default: connection.additionalFields["mongoParam_\(key)"] = value } } diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index a16eaf6d41..85bbe4e652 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -1569,12 +1569,13 @@ final class MainContentCoordinator: ObservableObject { let editorLanguage = services.pluginManager.editorLanguage(for: connection.type) let pluginBuildsBrowse = (services.databaseManager.driver(for: connectionId) as? PluginDriverAdapter)? .queryBuildingPluginDriver != nil - /// A SQL engine whose plugin writes its own browse gets no table from a query's text, and the name a - /// query tab carries came from an earlier result or an older build, so it can name another table. - if tab.tabType != .table, editorLanguage == .sql, pluginBuildsBrowse { + /// An engine whose plugin writes its own browse, or whose language is not SQL, gets no table from a query's + /// text, and the name a query tab carries came from an earlier result or an older build, so it can name + /// another collection. A pipeline's `_id` can also be a group key that names an unrelated document. + let usesNoSQLBrowsing = editorLanguage != .sql || pluginBuildsBrowse + if tab.tabType != .table, usesNoSQLBrowsing { return (nil, false) } - let usesNoSQLBrowsing = editorLanguage != .sql || pluginBuildsBrowse if usesNoSQLBrowsing { let name = tab.tableContext.tableName return (name, name != nil) diff --git a/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift index 494479ed47..15ea81577c 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift @@ -1,5 +1,4 @@ import Foundation -import os import TableProDatabase import TableProModels import TableProNumberFormatting @@ -23,16 +22,6 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil ) } - private let searchesShownText = OSAllocatedUnfairLock(initialState: false) - private var matchesShownText: Bool { searchesShownText.withLock { $0 } } - - // Read once the session is open and off the main actor: a version not yet cached is a round trip to the server. - override func connect() async throws { - try await super.connect() - let supported = MongoDBCapabilities.parse(mongo.serverVersion).supportsRegexMatch - searchesShownText.withLock { $0 = supported } - } - // With no database named, the plugin opens the first one that is not a system database. override var currentDatabase: String? { mongo.currentDb.isEmpty ? nil : mongo.currentDb @@ -47,7 +36,7 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil let sorted = request.sortState.columns return builder.buildQuery( collection: request.table, - filterDocument: Self.filterDocument(for: request, builder: builder, matchesShownText: matchesShownText), + filterDocument: Self.filterDocument(for: request, builder: builder), sortColumns: sorted.indices.map { (columnIndex: $0, ascending: sorted[$0].ascending) }, columns: sorted.map(\.name), limit: limit, @@ -77,7 +66,7 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil // Bounded like the Mac's background count, so a slow count never holds the session the next page needs. func rowCount(_ request: TableReadRequest) async throws -> Int? { let builder = mongo.filterQueryBuilder(for: request.table, database: pinnedDatabase) - let filter = Self.filterDocument(for: request, builder: builder, matchesShownText: matchesShownText) + let filter = Self.filterDocument(for: request, builder: builder) let mongo = mongo return try await inSessionDatabase { try await mongo.documentCount(table: request.table, filterDocument: filter, background: true) @@ -123,11 +112,7 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil // A filter with no plugin form matches nothing rather than being dropped, which would show rows it excludes. A field // named like the plugin's raw-filter column would have its value read as a whole filter document, code included. - static func filterDocument( - for request: TableReadRequest, - builder: MongoDBQueryBuilder, - matchesShownText: Bool - ) -> String { + static func filterDocument(for request: TableReadRequest, builder: MongoDBQueryBuilder) -> String { var parts: [String] = [] let filters = request.filters.map { filter in filtersAsField(filter.columnName) ? filter.pluginFilter : nil @@ -139,17 +124,12 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil parts.append(builder.buildFilterDocument(from: filters.compactMap { $0 }, logicMode: logic)) } if let text = request.searchText, !text.isEmpty { - let columns = request.searchColumns.filter(filtersAsField) - let terms = columns.map { PluginQueryFilter(column: $0, op: "CONTAINS", value: text, isCaseSensitive: false) } - var matches = terms.isEmpty ? [] : [builder.buildFilterDocument(from: terms, logicMode: "or")] - if matchesShownText { - matches += columns.map { shownTextMatch(field: $0, text: text) } - } - switch matches.count { - case 0: parts.append(MongoDBQueryBuilder.impossibleFilter) - case 1: parts.append(matches[0]) - default: parts.append("{\"$or\": [\(matches.joined(separator: ", "))]}") + let terms = request.searchColumns.filter(filtersAsField).map { + PluginQueryFilter(column: $0, op: "CONTAINS", value: text, isCaseSensitive: false) } + parts.append(terms.isEmpty + ? MongoDBQueryBuilder.impossibleFilter + : builder.buildFilterDocument(from: terms, logicMode: "or")) } switch parts.count { case 0: return "{}" @@ -158,15 +138,6 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil } } - // A regex matches strings only, so a number, a date or an ObjectId is matched by its text as well, the way a SQL - // search casts each column. Arrays and documents convert to nothing and stay matched by the regex alone. - private static func shownTextMatch(field: String, text: String) -> String { - let path = MongoDBQueryBuilder.escapeJsonString("$" + field) - let pattern = MongoDBQueryBuilder.escapeJsonString(NSRegularExpression.escapedPattern(for: text)) - let input = "{\"$convert\": {\"input\": \"\(path)\", \"to\": \"string\", \"onError\": \"\", \"onNull\": \"\"}}" - return "{\"$expr\": {\"$regexMatch\": {\"input\": \(input), \"regex\": \"\(pattern)\", \"options\": \"i\"}}}" - } - private static func filtersAsField(_ name: String) -> Bool { name != MongoDBQueryBuilder.rawFilterColumn && MongoDBQueryBuilder.isPlainFieldName(name) } diff --git a/apple/TableProMobile/TableProMobile/Drivers/MongoDBDialPlan.swift b/apple/TableProMobile/TableProMobile/Drivers/MongoDBDialPlan.swift index 8e1c36957e..58c11a0458 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MongoDBDialPlan.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MongoDBDialPlan.swift @@ -41,6 +41,10 @@ nonisolated enum MongoDBDialPlan { throw DatabaseError(message: srvBehindTunnelMessage) } fields[hostsField] = nil + // The driver reads option keys without case, so an imported `directconnection=false` would still apply. + for key in fields.keys where key.lowercased() == directConnectionField.lowercased() { + fields[key] = nil + } fields[directConnectionField] = "true" return fields } diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift index cf1a7ca0f0..715cca4148 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift @@ -79,21 +79,26 @@ struct MongoDBOnIOSTests { @Test("From MongoDB 4.2, search matches a number, a date or an ObjectId by its text as well") func searchMatchesShownText() { let filter = MongoDBBackedDriver.filterDocument( - for: request(search: "4.2", searchColumns: ["age", "name"]), builder: MongoDBQueryBuilder(), matchesShownText: true + for: request(search: "4.2", searchColumns: ["age", "name"]), + builder: MongoDBQueryBuilder(matchesShownText: true) ) #expect(filter.contains(#"{"age": {"$regex": "4\\.2", "$options": "i"}}"#)) - #expect(filter.contains(#"{"$expr": {"$regexMatch": {"input": {"$convert": {"input": "$age", "to": "string", "onError": "", "onNull": ""}}, "regex": "4\\.2", "options": "i"}}}"#)) - #expect(filter.contains(#""input": "$name""#)) + #expect(filter.contains(MongoDBShownText.regexMatch(field: "age", pattern: "4\\.2", ignoresCase: true))) + #expect(filter.contains(MongoDBShownText.regexMatch(field: "name", pattern: "4\\.2", ignoresCase: true))) #expect(filter.hasPrefix(#"{"$or": ["#)) } @Test("Before 4.2, or on a server whose version is not known, search sends no $regexMatch") func searchWithoutRegexMatch() { let filter = MongoDBBackedDriver.filterDocument( - for: request(search: "42", searchColumns: ["age"]), builder: MongoDBQueryBuilder(), matchesShownText: false + for: request(search: "42", searchColumns: ["age"]), builder: MongoDBQueryBuilder() ) #expect(!filter.contains("$regexMatch")) + let unconnected = MongoDBPluginDriver(config: DriverConnectionConfig( + host: "db.example.com", port: 27_017, username: "u", password: "p", database: "app" + )) + #expect(unconnected.filterQueryBuilder(for: "users").matchesShownText == false) #expect(MongoDBCapabilities.parse("4.2.1").supportsRegexMatch) #expect(!MongoDBCapabilities.parse("4.0.28").supportsRegexMatch) #expect(!MongoDBCapabilities.parse(nil).supportsRegexMatch) diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBPluginConnectionTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBPluginConnectionTests.swift index 67a949b812..09b70e3b81 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBPluginConnectionTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBPluginConnectionTests.swift @@ -1,3 +1,4 @@ +import CLibMongoc import Foundation @testable import TableProMobile import TableProPluginKit @@ -89,10 +90,11 @@ struct MongoDBPluginConnectionTests { let file = try #require(try MongoDBClientCertificateFile.joining(fixture.ssl)) defer { file.remove() } - let parameters = MongoDBSSLMapping.uriParameters(for: fixture.ssl, certificateKeyFile: file.path) + let options = MongoDBSSLMapping.uriOptions(for: fixture.ssl, certificateKeyFile: file.path) - let encoded = try #require(file.path.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed)) - #expect(parameters.filter { $0.hasPrefix("tlsCertificateKeyFile=") } == ["tlsCertificateKeyFile=\(encoded)"]) + #expect(options.filter { $0.key == "tlsCertificateKeyFile" } == [ + MongoDBURIOption(key: "tlsCertificateKeyFile", value: file.path) + ]) } @Test("A certificate alone, or a key at the certificate's own path, is passed as it is") @@ -108,7 +110,9 @@ struct MongoDBPluginConnectionTests { for ssl in [alone, samePath, disabled] { #expect(try MongoDBClientCertificateFile.joining(ssl)?.path == nil) } - #expect(MongoDBSSLMapping.uriParameters(for: alone).contains("tlsCertificateKeyFile=/certs/client.pem")) + #expect(MongoDBSSLMapping.uriOptions(for: alone).contains( + MongoDBURIOption(key: "tlsCertificateKeyFile", value: "/certs/client.pem") + )) } @Test("A key file that cannot be read fails with its path") @@ -127,6 +131,144 @@ struct MongoDBPluginConnectionTests { } } +// libmongoc decodes every option value and splits options on a raw `&`, so each test reads the built URI back with it. +struct MongoDBConnectionURITests { + private func uri( + host: String = "db.example.com", + ssl: [MongoDBURIOption] = [], + authSource: String = "admin", + readPreference: String? = nil, + writeConcern: String? = nil, + authMechanism: String? = nil, + replicaSet: String? = nil, + extraParams: [String: String] = [:] + ) -> String { + MongoDBConnectionURI( + useSrv: false, host: host, port: 27_017, user: "u", password: "p@ss:w/rd", database: "app", + options: MongoDBConnectionURI.options( + connectTimeout: MongoDBConnectTimeout(milliseconds: 5_000), + authSource: authSource, + ssl: ssl, + readPreference: readPreference, + writeConcern: writeConcern, + authMechanism: authMechanism, + replicaSet: replicaSet, + extraParams: extraParams + ) + ).text + } + + private func parsed(_ text: String) throws -> OpaquePointer { + var error = bson_error_t() + let parsed = text.withCString { mongoc_uri_new_with_error($0, &error) } + return try #require(parsed, "libmongoc refused \(text)") + } + + private func option(_ uri: OpaquePointer, _ key: String) -> String? { + mongoc_uri_get_option_as_utf8(uri, key, nil).map { String(cString: $0) } + } + + @Test("A `&` or `=` in a field reaches libmongoc as part of its value, never as an option of its own") + func fieldValuesCannotStartAnOption() throws { + let caFile = "/Users/me/R&D/My Certs/c+a;b#1%.pem" + let text = uri( + ssl: MongoDBSSLMapping.uriOptions(for: SSLConfiguration(mode: .verifyIdentity, caCertificatePath: caFile)), + authSource: "adm&in", + replicaSet: "rs&tlsAllowInvalidCertificates=true" + ) + let parsed = try parsed(text) + defer { mongoc_uri_destroy(parsed) } + + #expect(String(cString: mongoc_uri_get_replica_set(parsed)) == "rs&tlsAllowInvalidCertificates=true") + #expect(String(cString: mongoc_uri_get_auth_source(parsed)) == "adm&in") + #expect(option(parsed, MONGOC_URI_TLSCAFILE) == caFile) + #expect(!mongoc_uri_has_option(parsed, MONGOC_URI_TLSALLOWINVALIDCERTIFICATES)) + #expect(text.contains("replicaSet=rs%26tlsAllowInvalidCertificates%3Dtrue")) + } + + // libmongoc refuses a read preference or mechanism it does not know, so these stay one value it can name. + @Test("A read preference, write concern or mechanism is encoded whole") + func dropdownValuesAreEncodedWhole() { + let text = uri( + readPreference: "secondary&appname=x", + writeConcern: "majority&journal=false", + authMechanism: "SCRAM-SHA-256&authSource=evil" + ) + #expect(text.contains("readPreference=secondary%26appname%3Dx")) + #expect(text.contains("w=majority%26journal%3Dfalse")) + #expect(text.contains("authMechanism=SCRAM-SHA-256%26authSource%3Devil")) + #expect(!text.contains("&appname=")) + } + + @Test("A typed parameter keeps its value whole, and a key with `&` in it cannot inject one") + func typedParametersAreEncoded() throws { + let parsed = try parsed(uri(extraParams: ["appname": "My App+1 & co/é", "tlsInsecure=true&x": "1"])) + defer { mongoc_uri_destroy(parsed) } + + #expect(option(parsed, MONGOC_URI_APPNAME) == "My App+1 & co/é") + #expect(!mongoc_uri_get_option_as_bool(parsed, MONGOC_URI_TLSINSECURE, false)) + } + + @Test("A typed parameter named like a field's key in any case never overrides the field") + func typedParameterKeysCompareWithoutCase() throws { + let options = MongoDBConnectionURI.options( + connectTimeout: MongoDBConnectTimeout(milliseconds: 5_000), + authSource: "admin", + ssl: [], + readPreference: "nearest", + writeConcern: nil, + authMechanism: nil, + replicaSet: "rs0", + extraParams: ["REPLICASET": "other", "authsource": "evil", "ReadPreference": "primary", "w": "majority"] + ) + let keys = options.map { $0.key.lowercased() } + + #expect(Set(keys).count == keys.count) + #expect(options.contains(MongoDBURIOption(key: "replicaSet", value: "rs0"))) + #expect(options.contains(MongoDBURIOption(key: "authSource", value: "admin"))) + #expect(options.contains(MongoDBURIOption(key: "readPreference", value: "nearest"))) + #expect(options.contains(MongoDBURIOption(key: "w", value: "majority"))) + } + + @Test("With TLS on, the SSL mode owns certificate checking, and no typed TLS parameter conflicts with it") + func typedTLSParametersYieldToTheSSLMode() throws { + let typed = ["tlsInsecure": "true", "tlsDisableOCSPEndpointCheck": "true"] + let required = MongoDBSSLMapping.uriOptions(for: SSLConfiguration(mode: .required)) + let relaxed = try parsed(uri(ssl: required, extraParams: typed)) + defer { mongoc_uri_destroy(relaxed) } + #expect(mongoc_uri_get_option_as_bool(relaxed, MONGOC_URI_TLSALLOWINVALIDCERTIFICATES, false)) + #expect(!mongoc_uri_has_option(relaxed, MONGOC_URI_TLSINSECURE)) + #expect(!mongoc_uri_has_option(relaxed, MONGOC_URI_TLSDISABLEOCSPENDPOINTCHECK)) + + let verified = MongoDBSSLMapping.uriOptions(for: SSLConfiguration(mode: .verifyIdentity)) + let strict = try parsed(uri(ssl: verified, extraParams: typed)) + defer { mongoc_uri_destroy(strict) } + #expect(!mongoc_uri_has_option(strict, MONGOC_URI_TLSINSECURE)) + #expect(mongoc_uri_get_option_as_bool(strict, MONGOC_URI_TLSDISABLEOCSPENDPOINTCHECK, false)) + + let plain = try parsed(uri(extraParams: ["tlsInsecure": "true"])) + defer { mongoc_uri_destroy(plain) } + #expect(mongoc_uri_get_option_as_bool(plain, MONGOC_URI_TLSINSECURE, false)) + } + + @Test("The user, password and database still read back as typed") + func userInfoAndDatabaseRoundTrip() throws { + let parsed = try parsed(uri()) + defer { mongoc_uri_destroy(parsed) } + + #expect(String(cString: mongoc_uri_get_username(parsed)) == "u") + #expect(String(cString: mongoc_uri_get_password(parsed)) == "p@ss:w/rd") + #expect(String(cString: mongoc_uri_get_database(parsed)) == "app") + } + + @Test("A host list MongoDBHostList cannot read keeps a typed port out of the options") + func unreadableHostListCannotStartTheOptions() { + let text = uri(host: "a.example.com:1/?tlsInsecure=true,b.example.com") + + #expect(text.hasPrefix("mongodb://u:p%40ss%3Aw%2Frd@a.example.com:1%2F%3FtlsInsecure%3Dtrue,b.example.com:27017/app?")) + } +} + struct MongoSessionKillsTests { @Test("A session is killed once until that kill ends, so a resent Stop queues no second one") func killsCoalescePerSession() { diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift new file mode 100644 index 0000000000..da481565d9 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift @@ -0,0 +1,102 @@ +import Foundation +@testable import TableProMobile +import TableProPluginKit +import Testing + +struct MongoDBQueryBuilderTests { + private func filter( + _ op: String, + _ value: String = "4.2", + column: String = "age", + isCaseSensitive: Bool = false, + elementScope: String? = nil, + builder: MongoDBQueryBuilder = MongoDBQueryBuilder(matchesShownText: true) + ) -> String { + builder.buildFilterDocument(from: [PluginQueryFilter( + column: column, op: op, value: value, isCaseSensitive: isCaseSensitive, + secondValue: nil, elementScope: elementScope + )]) + } + + private func regex(_ field: String, _ pattern: String, ignoresCase: Bool = true) -> String { + let options = ignoresCase ? #", "$options": "i""# : "" + return #"{"\#(field)": {"$regex": "\#(MongoDBQueryBuilder.escapeJsonString(pattern))"\#(options)}}"# + } + + @Test("Contains, starts with and ends with match a string by $regex or any value by its shown text", arguments: [ + ("CONTAINS", "4\\.2"), ("STARTS WITH", "^4\\.2"), ("ENDS WITH", "4\\.2$") + ]) + func patternsReadShownText(op: String, pattern: String) { + let shown = MongoDBShownText.regexMatch(field: "age", pattern: pattern, ignoresCase: true) + #expect(filter(op) == #"{"$or": [\#(regex("age", pattern)), \#(shown)]}"#) + } + + @Test("Not contains keeps only a document whose string and shown text both miss") + func notContainsReadsShownText() { + let shown = MongoDBShownText.regexMatch(field: "age", pattern: "4\\.2", ignoresCase: true) + #expect(filter("NOT CONTAINS") == #"{"$nor": [\#(regex("age", "4\\.2")), \#(shown)]}"#) + } + + @Test("A pattern ignores case as the filter asks, whatever kind the field was sampled as") + func patternCaseFollowsTheFilter() { + let typed = MongoDBQueryBuilder(columnKinds: ["age": .int32], matchesShownText: true) + #expect(filter("CONTAINS", builder: typed).contains(regex("age", "4\\.2"))) + let sensitive = filter("CONTAINS", isCaseSensitive: true, builder: typed) + #expect(sensitive.contains(regex("age", "4\\.2", ignoresCase: false))) + #expect(!sensitive.contains(#""options": "i""#)) + } + + @Test("A condition on one array element matches by $regex alone, since $expr reads only the top-level document") + func elementScopedPatternHasNoExpression() { + let scoped = filter("CONTAINS", "x", column: "tags.name", elementScope: "tags") + #expect(scoped == #"{"tags": {"$elemMatch": {"name": {"$regex": "x", "$options": "i"}}}}"#) + } + + @Test("A name no aggregation path can read matches by $regex alone, since the path would fail the whole find", + arguments: ["", "a.", ".b", "c..d", "e.$f"]) + func unreadablePathHasNoExpression(column: String) { + #expect(!filter("CONTAINS", "x", column: column).contains("$expr")) + #expect(MongoDBShownText.reads(field: "a.b")) + } + + @Test("Without $regexMatch a pattern is a $regex alone") + func patternWithoutShownText() { + #expect(filter("CONTAINS", builder: MongoDBQueryBuilder()) == regex("age", "4\\.2")) + #expect(!filter("NOT CONTAINS", builder: MongoDBQueryBuilder()).contains("$expr")) + } + + @Test("The shown text converts only values that always convert and formats an in-range date as the grid shows it") + func shownTextExpression() throws { + let expression = MongoDBShownText.expression(field: "a.b") + #expect(expression.contains(#""vars": {"v": "$a.b"}"#)) + #expect(expression.contains(#"["double", "int", "long", "decimal", "bool", "objectId"]"#)) + #expect(expression.contains(#"{"$date": {"$numberLong": "-62167219200000"}}"#)) + #expect(expression.contains(#"{"$date": {"$numberLong": "253402300799999"}}"#)) + #expect(expression.contains("%Y-%m-%dT%H:%M:%SZ")) + #expect(expression.contains("%Y-%m-%dT%H:%M:%S.%LZ")) + #expect(!expression.contains("$convert")) + + let match = MongoDBShownText.regexMatch(field: #"we"ird"#, pattern: #"a\.b"#, ignoresCase: false) + let parsed = try #require(try JSONSerialization.jsonObject(with: Data(match.utf8)) as? [String: Any]) + #expect(parsed["$expr"] != nil) + #expect(match.contains(#""$we\"ird""#)) + #expect(!match.contains("options")) + } + + @Test("Only a find in the session's database with no projection teaches the collection's kinds") + func storedDocumentsAreOnlyWholeFinds() { + func outcome(_ find: MongoScriptFindShape?) -> MongoScriptStatementResult { + var result = MongoScriptStatementResult() + result.collection = "orders" + result.find = find + result.producedDocuments = true + return result + } + let whole = outcome(MongoScriptFindShape(database: "app", returnsWholeDocuments: true, skip: 0)) + #expect(whole.returnsStoredDocuments(in: "app")) + #expect(!whole.returnsStoredDocuments(in: "other")) + #expect(!outcome(MongoScriptFindShape(database: "app", returnsWholeDocuments: false, skip: 0)) + .returnsStoredDocuments(in: "app")) + #expect(!outcome(nil).returnsStoredDocuments(in: "app")) + } +} diff --git a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift index 185f7af4ab..8486e6e5a7 100644 --- a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift +++ b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift @@ -103,6 +103,21 @@ struct FindMatcherTests { #expect(FindMatcher.nearestMatchIndex(to: 99, in: matches) == 0) #expect(FindMatcher.nearestMatchIndex(to: 0, in: []) == nil) } + + @Test("a server search sends only text columns unless the engine's patterns read any column") + func serverSearchableColumns() { + let typed: [ColumnType] = [ + .integer(rawType: "INTEGER"), .decimal(rawType: "FLOAT"), .timestamp(rawType: "TIMESTAMP"), + .boolean(rawType: "BOOLEAN"), .json(rawType: "JSON") + ] + for type in typed { + #expect(!FindMatcher.isServerSearchable(type)) + #expect(FindMatcher.isServerSearchable(type, patternReadsAnyColumn: true)) + } + #expect(FindMatcher.isServerSearchable(.text(rawType: "ObjectId"))) + #expect(!FindMatcher.isServerSearchable(.blob(rawType: "BLOB"), patternReadsAnyColumn: true)) + #expect(!FindMatcher.isServerSearchable(.spatial(rawType: "GEOMETRY"), patternReadsAnyColumn: true)) + } } struct TabFindStateTests { diff --git a/apple/TableProTests/Core/Database/ScopedDriverRoutingTests.swift b/apple/TableProTests/Core/Database/ScopedDriverRoutingTests.swift index 93794fec69..145cc6f85f 100644 --- a/apple/TableProTests/Core/Database/ScopedDriverRoutingTests.swift +++ b/apple/TableProTests/Core/Database/ScopedDriverRoutingTests.swift @@ -54,6 +54,22 @@ struct ScopedDriverRoutingTests { #expect(explicit?.schema == "staging", "An explicit schema still passes through") } + @Test("A filtered count runs with the page where the driver types filters by the rows it read") + func filteredCountRunsWithThePage() { + let mongo = Self.makeSession(type: .mongodb, browseDatabase: "app") + let mysql = Self.makeSession(type: .mysql, browseDatabase: "app") + defer { + DatabaseManager.shared.removeSession(for: mongo.id) + DatabaseManager.shared.removeSession(for: mysql.id) + } + + let scope = Self.scope(mongo, database: "app") + #expect(DatabaseManager.shared.rowCountRunsWithPage(scope, filtered: true)) + #expect(DatabaseManager.shared.executionRoute(for: scope) == .sessionDriver) + #expect(!DatabaseManager.shared.rowCountRunsWithPage(scope, filtered: false)) + #expect(!DatabaseManager.shared.rowCountRunsWithPage(Self.scope(mysql, database: "app"), filtered: true)) + } + @Test("A pin-capable engine keeps the user's SQL on the session driver") func pinCapableEngineUsesTheSessionDriver() throws { let connection = Self.makeSession(type: .mysql, browseDatabase: "inventory") diff --git a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift index 406b63e5aa..ee1976f948 100644 --- a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift @@ -292,6 +292,20 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(StructureEditEligibility.allows(.dropColumn, on: .table, matrix: built.structureEditing.structureEdits)) } + @Test("MongoDB keeps how its filters read fields when its plugin registers") + func mongoDBKeepsHowItsFiltersReadFields() { + let built = PluginMetadataRegistry.shared.buildMetadataSnapshot(from: MockMongoDBPlugin.self) + + #expect( + built.capabilities.patternFiltersReadAnyColumn == true, + "Search all rows would drop number, date and boolean fields" + ) + #expect( + built.capabilities.filtersTypedByReadRows == true, + "A pooled count would type a filter by nothing and disagree with the page" + ) + } + @Test("DynamoDB keeps its full-scan count when its plugin registers") func dynamoDBKeepsItsFullScanCount() { let registry = PluginMetadataRegistry.shared @@ -554,6 +568,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.supportsColumnSort == true) #expect(built.capabilities.supportsMatchAnyFilters == true) #expect(built.capabilities.columnsAreSampled == false) + #expect(built.capabilities.patternFiltersReadAnyColumn == false) + #expect(built.capabilities.filtersTypedByReadRows == false) #expect(built.capabilities.tlsImpliedPorts.isEmpty) #expect(built.capabilities.verifiesServerWithSystemTrust == false) #expect(built.capabilities.supportsPerConnectionCertificatePaths == true) diff --git a/apple/TableProTests/Core/Utilities/ConnectionURLFormatterTests.swift b/apple/TableProTests/Core/Utilities/ConnectionURLFormatterTests.swift index fc73e2675f..c71f915b71 100644 --- a/apple/TableProTests/Core/Utilities/ConnectionURLFormatterTests.swift +++ b/apple/TableProTests/Core/Utilities/ConnectionURLFormatterTests.swift @@ -632,6 +632,65 @@ struct ConnectionURLFormatterTests { #expect(url.contains("host1:27017,host2:27018,host3:27019")) } + @Test("An SRV connection copies as a mongodb+srv URL with no port, and opens as SRV again") + func mongoSrvRoundTrips() throws { + let conn = DatabaseConnection( + name: "", host: "cluster0.example.com", port: 27_017, database: "app", + username: "user", type: .mongodb, mongoUseSrv: true + ) + let url = ConnectionURLFormatter.format(conn, password: "pass", sshPassword: nil) + #expect(url == "mongodb+srv://user:pass@cluster0.example.com/app") + + guard case .success(let parsed) = ConnectionURLParser.parse(url) else { + Issue.record("\(url) did not parse") + return + } + #expect(TransientConnectionFactory.build(from: parsed).mongoUseSrv) + } + + @Test("A copied MongoDB URL keeps every option, encoded, and an opened one maps the UUID encoding to its field") + func mongoOptionsRoundTrip() throws { + let conn = DatabaseConnection( + name: "", host: "host", port: 27_017, database: "app", username: "user", type: .mongodb, + additionalFields: [ + "mongoAuthSource": "$external", "mongoReadPreference": "nearest", "mongoWriteConcern": "majority", + "mongoReplicaSet": "rs0", "mongoParam_appName": "My App+1;x", "mongoParam_REPLICASET": "other", + "mongoUuidRepresentation": "javaLegacy" + ] + ) + let url = ConnectionURLFormatter.format(conn, password: "pass", sshPassword: nil) + #expect(url.contains("authSource=%24external")) + #expect(url.contains("readPreference=nearest")) + #expect(url.contains("w=majority")) + #expect(url.contains("appName=My%20App%2B1%3Bx")) + #expect(url.contains("replicaSet=rs0")) + #expect(!url.contains("REPLICASET")) + #expect(!url.contains("mongoUseSrv")) + + guard case .success(let parsed) = ConnectionURLParser.parse(url) else { + Issue.record("\(url) did not parse") + return + } + let opened = TransientConnectionFactory.build(from: parsed) + #expect(opened.mongoAuthSource == "$external") + #expect(opened.additionalFields["mongoParam_appName"] == "My App+1;x") + #expect(opened.additionalFields["mongoUuidRepresentation"] == "javaLegacy") + #expect(opened.additionalFields["mongoParam_uuidRepresentation"] == nil) + } + + @Test("An SSH URL's MongoDB options are decoded once, as a standard URL's are") + func mongoSSHURLDecodesOptions() throws { + let url = "mongodb+ssh://me@bastion.example.com/u:p@10.0.0.5:27017/app" + + "?authSource=%24external&authMechanism=MONGODB-X509&appName=a%26b" + guard case .success(let parsed) = ConnectionURLParser.parse(url) else { + Issue.record("\(url) did not parse") + return + } + #expect(parsed.authSource == "$external") + #expect(parsed.mongoQueryParams["authMechanism"] == "MONGODB-X509") + #expect(parsed.mongoQueryParams["appName"] == "a&b") + } + // MARK: - DuckDB @Test("DuckDB local mode formats a file URL from the file path field") diff --git a/apple/TableProTests/Database/NativeDumpRegistryTests.swift b/apple/TableProTests/Database/NativeDumpRegistryTests.swift index 388b77108c..bd0025ea89 100644 --- a/apple/TableProTests/Database/NativeDumpRegistryTests.swift +++ b/apple/TableProTests/Database/NativeDumpRegistryTests.swift @@ -243,10 +243,74 @@ struct NativeDumpRegistryTests { let built = try command(.mongodb, connection: secured) let uri = try configurationValue("uri", in: configuration(of: built)) #expect(uri.contains("tls=true&tlsInsecure=true")) - #expect(uri.contains("tlsCertificateKeyFile=/certs/client.pem")) + #expect(uri.contains("tlsCertificateKeyFile=%2Fcerts%2Fclient.pem")) #expect(!built.arguments.contains("--ssl")) } + @Test("Every option value reaches the tool encoded, so a `&`, `;` or `+` in it never starts another option") + func mongoOptionValuesAreEncoded() throws { + var secured = connection(type: .mongodb, sslMode: .verifyCa, sslEnabled: true) + secured.sslConfig.caCertificatePath = "/Users/me/R&D/My Certs/c+a;b.pem" + secured.mongoReplicaSet = "rs;tlsInsecure=true" + secured.mongoAuthSource = "a&b" + let uri = try configurationValue("uri", in: configuration(of: command(.mongodb, connection: secured))) + #expect(uri.contains("replicaSet=rs%3BtlsInsecure%3Dtrue")) + #expect(uri.contains("authSource=a%26b")) + #expect(uri.contains("tlsCAFile=%2FUsers%2Fme%2FR%26D%2FMy%20Certs%2Fc%2Ba%3Bb.pem")) + #expect(!uri.contains("tlsInsecure=true")) + } + + @Test("A typed parameter named like a field's key in another case never overrides the field") + func mongoTypedParameterKeysCompareWithoutCase() throws { + var replicaSet = connection(type: .mongodb) + replicaSet.mongoReplicaSet = "rs0" + replicaSet.additionalFields["mongoParam_REPLICASET"] = "other" + replicaSet.additionalFields["mongoParam_appName"] = "Table Pro" + let uri = try configurationValue("uri", in: configuration(of: command(.mongodb, connection: replicaSet))) + #expect(uri.contains("replicaSet=rs0")) + #expect(!uri.contains("REPLICASET")) + #expect(uri.contains("appName=Table%20Pro")) + } + + @Test("X.509 and AWS authenticate against $external, which the tools' driver requires", arguments: [ + "MONGODB-X509", "MONGODB-AWS" + ]) + func mongoExternalMechanismsUseExternal(mechanism: String) throws { + var external = connection(type: .mongodb, sslMode: .required, sslEnabled: true) + external.mongoAuthMechanism = mechanism + let uri = try configurationValue("uri", in: configuration(of: command(.mongodb, connection: external))) + #expect(uri.contains("authSource=%24external")) + } + + @Test("A typed TLS parameter never overrides the SSL mode or sits beside the tlsInsecure it implies") + func mongoTypedTLSParametersYieldToTheSSLMode() throws { + var insecure = connection(type: .mongodb, sslMode: .required, sslEnabled: true) + insecure.additionalFields["mongoParam_tlsDisableOCSPEndpointCheck"] = "true" + let relaxed = try configurationValue("uri", in: configuration(of: command(.mongodb, connection: insecure))) + #expect(relaxed.contains("tlsInsecure=true")) + #expect(!relaxed.contains("tlsDisableOCSPEndpointCheck")) + + var verified = connection(type: .mongodb, sslMode: .verifyIdentity, sslEnabled: true) + verified.additionalFields["mongoParam_tlsInsecure"] = "true" + let strict = try configurationValue("uri", in: configuration(of: command(.mongodb, connection: verified))) + #expect(!strict.contains("tlsInsecure")) + } + + @Test("A bare IPv6 host reaches the tool in brackets") + func mongoBareIPv6HostIsBracketed() throws { + let local = connection(type: .mongodb, host: "fd00::5", port: 27_017) + let uri = try configurationValue("uri", in: configuration(of: command(.mongodb, connection: local))) + #expect(uri.hasPrefix("mongodb://alice@[fd00::5]:27017/")) + } + + @Test("A port typed into a host list is encoded, so it cannot open the options") + func mongoHostListPortIsEncoded() throws { + var replicaSet = connection(type: .mongodb, host: "a.example.com", port: 27_017) + replicaSet.additionalFields["mongoHosts"] = "a.example.com:27017,b.example.com:1/?tlsInsecure=true" + let uri = try configurationValue("uri", in: configuration(of: command(.mongodb, connection: replicaSet))) + #expect(uri.hasPrefix("mongodb://alice@a.example.com:27017,b.example.com:1%2F%3FtlsInsecure%3Dtrue/")) + } + @Test("MongoDB names the database on backup and renames the archive into the target on restore") func mongoScopesItsDatabase() throws { #expect(try command(.mongodb, kind: .backup).arguments.contains("--db=sales")) diff --git a/apple/TableProTests/Views/Main/PluginBrowsedQueryTabEditabilityTests.swift b/apple/TableProTests/Views/Main/PluginBrowsedQueryTabEditabilityTests.swift index 51419aa268..cf4e33a455 100644 --- a/apple/TableProTests/Views/Main/PluginBrowsedQueryTabEditabilityTests.swift +++ b/apple/TableProTests/Views/Main/PluginBrowsedQueryTabEditabilityTests.swift @@ -31,13 +31,24 @@ private final class BrowseBuildingDriver: PluginDriverStub, @unchecked Sendable } } +private final class ShellDriver: PluginDriverStub, @unchecked Sendable { + var supportsSchemas: Bool { false } + var supportsTransactions: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + + func ping() async throws {} +} + /// A query tab keeps the table name of whatever result it last showed, and an older build filled it from the -/// query's text. On a SQL engine whose plugin writes its own browse, the app no longer reads that text, so a name -/// left on the tab could point the grid's edits at a different table from the one the rows came from. +/// query's text. On an engine whose plugin writes its own browse, or whose language is not SQL, the app no longer reads +/// that text, so a name left on the tab could point the grid's edits at a different table from the one the rows came +/// from. @MainActor struct PluginBrowsedQueryTabEditabilityTests { private func withCoordinator( type: DatabaseType, + driver: any PluginDatabaseDriver = BrowseBuildingDriver(), selecting tab: QueryTab, alongside others: [QueryTab] = [], _ body: (MainContentCoordinator) -> Void @@ -45,7 +56,7 @@ struct PluginBrowsedQueryTabEditabilityTests { let connection = TestFixtures.makeConnection(type: type) var session = ConnectionSession( connection: connection, - driver: PluginDriverAdapter(connection: connection, pluginDriver: BrowseBuildingDriver()) + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) ) session.status = .connected DatabaseManager.shared.injectSession(session, for: connection.id) @@ -78,6 +89,36 @@ struct PluginBrowsedQueryTabEditabilityTests { } } + @Test("A MongoDB query tab carrying a stale collection name stays read-only, whatever its statement", arguments: [ + "db.orders.aggregate([{$group: {_id: \"$customerId\", n: {$sum: 1}}}])", + "db.getCollection(\"orders\").find({}, {renamed: \"$a\"})", + "db.customers.find()" + ]) + func staleMongoQueryTabNameIsNotTrusted(statement: String) { + for driver: any PluginDatabaseDriver in [BrowseBuildingDriver(), ShellDriver()] { + var tab = QueryTab(query: statement, tabType: .query) + tab.tableContext.tableName = "orders" + withCoordinator(type: .mongodb, driver: driver, selecting: tab) { coordinator in + let resolved = coordinator.resolveTableEditability(tab: tab, sql: tab.content.query) + + #expect(resolved.tableName == nil) + #expect(!resolved.isEditable) + } + } + } + + @Test("A MongoDB table tab stays editable on its own collection") + func mongoTableTabKeepsItsCollection() { + var tab = QueryTab(query: "", tabType: .table) + tab.tableContext.tableName = "orders" + withCoordinator(type: .mongodb, driver: ShellDriver(), selecting: tab) { coordinator in + let resolved = coordinator.resolveTableEditability(tab: tab, sql: "") + + #expect(resolved.tableName == "orders") + #expect(resolved.isEditable) + } + } + @Test("A table tab that is not selected resolves its own table, not the selected tab's") func backgroundTableTabResolvesItsOwnTable() { var background = QueryTab(query: "", tabType: .table) diff --git a/apple/scripts/probes/check-mongodb-shown-text.sh b/apple/scripts/probes/check-mongodb-shown-text.sh new file mode 100755 index 0000000000..2231c69041 --- /dev/null +++ b/apple/scripts/probes/check-mongodb-shown-text.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# +# The text a contains, starts with or ends with filter matches a MongoDB value by, asked of a real server. +# +# MongoDBShownText writes an aggregation expression by hand: it converts only the types it lists and formats a date +# only inside years 0 to 9999, because `$convert` raises past its onError on binary data and on a date outside that +# range (measured on 8.0), and one such value fails the whole find. The unit tests pin its text but cannot ask a server. +# So this compiles the expression's own source with swiftc against the libbson and libmongoc the plugin links, stores +# one value of each kind, and counts what each pattern matches: a number, a date and an ObjectId by the text the grid +# shows, and binary data, out-of-range dates and missing values by nothing, with no error. +# +# Usage: +# scripts/probes/check-mongodb-shown-text.sh [mongodb-uri] +# +# The default URI is mongodb://127.0.0.1:27017. Needs MongoDB 4.2 or later and the static libraries from +# scripts/download-libs.sh. Writes only to a database named tablepro_shown_text_, which it drops. Exits non-zero +# on a disagreement. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +URI="${1:-mongodb://127.0.0.1:27017}" +PLUGIN="$ROOT/Plugins/MongoDBDriverPlugin" +LIBS="$ROOT/Libs" + +for lib in "$LIBS/libmongoc.a" "$LIBS/libbson.a" "$LIBS/dylibs/libssl.3.dylib" "$LIBS/dylibs/libcrypto.3.dylib"; do + [ -e "$lib" ] || { + echo "missing $lib (run scripts/download-libs.sh)" >&2 + exit 3 + } +done + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +cat > "$WORK/builder.swift" <<'SWIFT' +enum MongoDBQueryBuilder { + static func escapeJsonString(_ value: String) -> String { + value.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "\"", with: "\\\"") + } +} +SWIFT + +cat > "$WORK/main.swift" <<'SWIFT' +import CLibMongoc +import Foundation + +let documents: [(Int, String)] = [ + (1, #""v":{"$numberInt":"42"}"#), + (2, #""v":{"$numberDouble":"1.5"}"#), + (3, #""v":true"#), + (4, #""v":{"$date":{"$numberLong":"1704164645000"}}"#), + (5, #""v":{"$date":{"$numberLong":"1704164645120"}}"#), + (6, #""v":{"$oid":"65a1b2c3d4e5f60718293a4b"}"#), + (7, #""v":{"$binary":{"base64":"AAAAAAAAAAAAAAAAAAAAAA==","subType":"04"}}"#), + (8, #""v":{"$date":{"$numberLong":"253402300800000"}}"#), + (9, #""v":[{"$date":{"$numberLong":"253402300800000"}},"a05Z"]"#), + (10, #""v":{"x":{"$date":{"$numberLong":"-62167219200001"}}}"#), + (11, #""v":{"$numberLong":"9007199254740993"}"#), + (12, #""v":{"$numberDecimal":"1.50"}"#), + (13, ""), + (14, #""v":null"#), + (15, #""v":"s 05Z""#), + (16, #""v":{"$date":{"$numberLong":"-62167219200000"}}"#), + (17, #""v":{"$binary":{"base64":"AAEC","subType":"00"}}"#) +] + +// A pattern as the filter builder escapes it, and the documents it must match. +let cases: [(String, Set)] = [ + ("42", [1]), + ("05Z", [4, 9, 15]), + ("05\\.120Z", [5]), + ("1\\.5", [2, 12]), + ("TRUE", [3]), + ("65A1B2", [6]), + ("^9007199254740993$", [11]), + ("^0000-01-01T00:00:00Z$", [16]), + ("^2024-01-02T03:04:05Z$", [4]), + ("AAAA", []) +] + +func bson(_ json: String) -> OpaquePointer? { + var error = bson_error_t() + return json.withCString { bson_new_from_json($0, -1, &error) } +} + +func errorText(_ error: inout bson_error_t) -> String { + withUnsafeBytes(of: &error.message) { String(cString: $0.bindMemory(to: CChar.self).baseAddress!) } +} + +@main +struct Check { + static func main() { + mongoc_init() + exit(run()) + } + + // Returns rather than exits, so the deferred drop runs. + static func run() -> Int32 { + let uri = CommandLine.arguments.count > 1 ? CommandLine.arguments[1] : "mongodb://127.0.0.1:27017" + guard let client = mongoc_client_new(uri) else { + print("FAIL cannot parse \(uri)") + return 2 + } + let name = "tablepro_shown_text_\(getpid())" + let database = mongoc_client_get_database(client, name) + let collection = mongoc_client_get_collection(client, name, "values") + defer { + _ = mongoc_database_drop(database, nil) + mongoc_collection_destroy(collection) + mongoc_database_destroy(database) + mongoc_client_destroy(client) + } + + var error = bson_error_t() + for (id, fields) in documents { + let json = fields.isEmpty ? #"{"_id":\#(id)}"# : #"{"_id":\#(id),\#(fields)}"# + guard let document = bson(json), mongoc_collection_insert_one(collection, document, nil, nil, &error) else { + print("FAIL insert \(id): \(errorText(&error))") + return 2 + } + bson_destroy(document) + } + + var failures = 0 + for (pattern, expected) in cases { + let regex = #"{"v":{"$regex":"\#(MongoDBQueryBuilder.escapeJsonString(pattern))","$options":"i"}}"# + let shown = MongoDBShownText.regexMatch(field: "v", pattern: pattern, ignoresCase: true) + for negated in [false, true] { + let filter = #"{"\#(negated ? "$nor" : "$or")":[\#(regex),\#(shown)]}"# + guard let matched = ids(collection, filter) else { + failures += 1 + continue + } + let want = negated ? Set(documents.map(\.0)).subtracting(expected) : expected + let label = "\(negated ? "not " : "")\(pattern)" + if matched == want { + print("ok \(label): \(matched.sorted())") + } else { + print("FAIL \(label): matched \(matched.sorted()), expected \(want.sorted())") + failures += 1 + } + } + } + return failures == 0 ? 0 : 1 + } + + static func ids(_ collection: OpaquePointer?, _ filter: String) -> Set? { + guard let query = bson(filter) else { + print("FAIL not JSON: \(filter.prefix(200))") + return nil + } + defer { bson_destroy(query) } + let cursor = mongoc_collection_find_with_opts(collection, query, nil, nil) + defer { mongoc_cursor_destroy(cursor) } + var found = Set() + var document: OpaquePointer? + while mongoc_cursor_next(cursor, &document) { + guard let document, let text = bson_as_relaxed_extended_json(document, nil) else { continue } + defer { bson_free(text) } + let parsed = try? JSONSerialization.jsonObject(with: Data(String(cString: text).utf8)) as? [String: Any] + if let id = parsed?["_id"] as? Int { found.insert(id) } + } + var error = bson_error_t() + if mongoc_cursor_error(cursor, &error) { + print("FAIL the server refused the filter: \(errorText(&error))") + return nil + } + return found + } +} +SWIFT + +echo "Building the shown text checker from $PLUGIN" +DEVELOPER_DIR="${DEVELOPER_DIR:-$(xcode-select -p)}" xcrun swiftc -swift-version 6 -parse-as-library -O \ + -module-name ShownTextCheck \ + -I "$PLUGIN/CLibMongoc" \ + -Xcc "-I$PLUGIN/CLibMongoc/include" \ + -Xcc "-I$PLUGIN/CLibMongoc/include/libbson-1.0" \ + -Xcc "-I$PLUGIN/CLibMongoc/include/libmongoc-1.0" \ + -L "$LIBS/dylibs" \ + -Xlinker -force_load -Xlinker "$LIBS/libmongoc.a" \ + -Xlinker -force_load -Xlinker "$LIBS/libbson.a" \ + -lssl.3 -lcrypto.3 -lresolv -lz \ + -Xlinker -rpath -Xlinker "$LIBS/dylibs" \ + "$PLUGIN/MongoDBShownText.swift" \ + "$WORK/builder.swift" \ + "$WORK/main.swift" \ + -o "$WORK/check" > "$WORK/build.log" 2>&1 || { + cat "$WORK/build.log" >&2 + exit 2 +} + +"$WORK/check" "$URI" From 6c01dd7b1724b18f8af5e430d01c20a95da08b0c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 22 Oct 2026 18:24:10 +0700 Subject: [PATCH 2/7] fix(mongodb): count with the page's filter and keep ssl aliases from overriding the SSL mode --- .../MongoDBConnectionURI.swift | 18 ++++++++- .../MongoDBPluginDriver.swift | 38 +++++++++++++++++-- .../MongoDBQueryBuilder.swift | 23 ++--------- .../Core/Coordinators/FindMatcher.swift | 5 ++- .../Database/MongoToolsConnectionString.swift | 13 ++++++- .../Core/Plugins/PluginMetadataRegistry.swift | 4 +- .../Drivers/MongoDBBackedDriver.swift | 25 ++++++++++-- .../MongoDBPluginConnectionTests.swift | 6 ++- .../Drivers/MongoDBQueryBuilderTests.swift | 33 ++++++++++++++++ .../Core/Coordinators/FindMatcherTests.swift | 5 ++- .../Database/NativeDumpRegistryTests.swift | 4 ++ 11 files changed, 138 insertions(+), 36 deletions(-) diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift index ee24ece5ce..63ac07b105 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift @@ -108,12 +108,28 @@ nonisolated struct MongoDBConnectionURI: Sendable { if ssl.contains(where: { $0.key == "tlsAllowInvalidCertificates" }) { taken.formUnion(["tlsdisableocspendpointcheck", "tlsdisablecertificaterevocationcheck"]) } - for key in extraParams.keys.sorted() where taken.insert(key.lowercased()).inserted { + for key in extraParams.keys.sorted() where taken.insert(canonicalKey(key)).inserted { options.append(MongoDBURIOption(key: key, value: extraParams[key] ?? "")) } return options } + // libmongoc still reads the old `ssl` names as their `tls` options, so a typed one would turn off a check the SSL + // mode asked for. + static let deprecatedAliases = [ + "ssl": "tls", + "sslallowinvalidcertificates": "tlsallowinvalidcertificates", + "sslallowinvalidhostnames": "tlsallowinvalidhostnames", + "sslcertificateauthorityfile": "tlscafile", + "sslclientcertificatekeyfile": "tlscertificatekeyfile", + "sslclientcertificatekeypassword": "tlscertificatekeyfilepassword" + ] + + static func canonicalKey(_ key: String) -> String { + let lowered = key.lowercased() + return deprecatedAliases[lowered] ?? lowered + } + static let fieldOwnedKeys = [ "connectTimeoutMS", "serverSelectionTimeoutMS", "authSource", "authMechanism", "replicaSet", diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 5683f3ef03..6b00b9879b 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -21,6 +21,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se private var identityKindsByCollection: [String: Set] = [:] private var binarySubtypesByCollection: [String: MongoDBBinarySubtypes] = [:] private var fieldKindsByCollection: [String: MongoDBFieldKinds] = [:] + private var pageFiltersByCollection: [String: (signature: String, document: String)] = [:] private static let logger = Logger(subsystem: "com.TablePro", category: "MongoDBPluginDriver") @@ -515,8 +516,9 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se logicMode: String, background: Bool ) async throws -> Int? { - let filterJson = filterQueryBuilder(for: table) - .buildFilterDocument(from: filters, logicMode: logicMode) + let signature = Self.filterSignature(filters, logicMode: logicMode) + let filterJson = pageFilter(signature: signature, table: table) + ?? filterQueryBuilder(for: table).buildFilterDocument(from: filters, logicMode: logicMode) return try await documentCount(table: table, filterDocument: filterJson, background: background) } @@ -785,12 +787,39 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se columnKinds: [String: PluginColumnKind] ) -> String? { let builder = filterQueryBuilder(for: table) - return builder.buildFilteredQuery( - collection: table, queryFilters: queryFilters, logicMode: logicMode, + let document = builder.buildFilterDocument(from: queryFilters, logicMode: logicMode) + recordPageFilter(document, signature: Self.filterSignature(queryFilters, logicMode: logicMode), table: table) + return builder.buildQuery( + collection: table, filterDocument: document, sortColumns: sortColumns, columns: columns, limit: limit, offset: offset ) } + // A page's own rows can change the kinds that type a filter value before its count is built, so the count reuses + // the document the page ran: a field holding both 1 and "1" would otherwise count the documents the page left out. + func recordPageFilter(_ document: String, signature: String, table: String, database: String? = nil) { + let key = columnKindKey(table, database: database) + columnKindLock.withLock { pageFiltersByCollection[key] = (signature, document) } + } + + func pageFilter(signature: String, table: String, database: String? = nil) -> String? { + let key = columnKindKey(table, database: database) + guard let page = columnKindLock.withLock({ pageFiltersByCollection[key] }), page.signature == signature else { + return nil + } + return page.document + } + + static func filterSignature(_ filters: [PluginQueryFilter], logicMode: String) -> String { + let rows = filters.map { filter in + [ + filter.column, filter.op, filter.value, filter.isCaseSensitive ? "1" : "0", + filter.secondValue ?? "\u{0}", filter.elementScope ?? "\u{0}" + ].joined(separator: "\u{1F}") + } + return ([logicMode] + rows).joined(separator: "\u{1E}") + } + func generateRowWrites( table: String, schema: String?, @@ -1156,6 +1185,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se fieldPathKindsByCollection = fieldPathKindsByCollection.filter { !isStale($0.key) } declaredSchemasByCollection = declaredSchemasByCollection.filter { !isStale($0.key) } identityKindsByCollection = identityKindsByCollection.filter { !isStale($0.key) } + pageFiltersByCollection = pageFiltersByCollection.filter { !isStale($0.key) } } } } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift index ae3f7f42ea..235881b0a9 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift @@ -54,24 +54,6 @@ nonisolated struct MongoDBQueryBuilder { ) } - /// Build: db.collection.find({filter}).sort({}).skip(offset).limit(limit) - func buildFilteredQuery( - collection: String, - queryFilters: [PluginQueryFilter], - logicMode: String = "and", - sortColumns: [(columnIndex: Int, ascending: Bool)] = [], - columns: [String] = [], - limit: Int = 200, - offset: Int = 0 - ) -> String? { - exactFilterDocument(from: queryFilters, logicMode: logicMode).map { - buildQuery( - collection: collection, filterDocument: $0, - sortColumns: sortColumns, columns: columns, limit: limit, offset: offset - ) - } - } - func buildQuery( collection: String, filterDocument: String, @@ -135,8 +117,9 @@ nonisolated struct MongoDBQueryBuilder { } /// Nil when a row cannot be built: a raw document that does not normalize, a name that reads as an operator, a - /// missing bound. Running the other rows without it would, under "match all", show rows the filter excludes. - func exactFilterDocument( + /// missing bound. Running the other rows without it would, under "match all", show rows the filter excludes, and + /// a query that failed to build would leave a refresh to reload the table unfiltered, so the caller matches nothing. + private func exactFilterDocument( from filters: [PluginQueryFilter], logicMode: String = "and" ) -> String? { diff --git a/apple/TablePro/Core/Coordinators/FindMatcher.swift b/apple/TablePro/Core/Coordinators/FindMatcher.swift index 766f9d7d36..5c9cfae430 100644 --- a/apple/TablePro/Core/Coordinators/FindMatcher.swift +++ b/apple/TablePro/Core/Coordinators/FindMatcher.swift @@ -20,9 +20,10 @@ enum FindMatcher { /// Columns a `LIKE` comparison can be built against without a cast. Postgres has no implicit /// integer to text conversion, so `id ILIKE '%abc%'` fails the whole query rather than matching /// nothing. Client-side matching is looser on purpose: it compares the text already on screen. - /// An engine whose pattern filters read any column by its shown text matches what the page does. + /// An engine whose pattern filters read a scalar column by its shown text searches those too. A nested value's + /// JSON is text only on screen. static func isServerSearchable(_ columnType: ColumnType?, patternReadsAnyColumn: Bool = false) -> Bool { - if patternReadsAnyColumn { return isSearchable(columnType) } + if patternReadsAnyColumn, columnType?.isJsonType != true { return isSearchable(columnType) } return switch columnType { case .text, .enumType, .set, nil: true default: false diff --git a/apple/TablePro/Core/Database/MongoToolsConnectionString.swift b/apple/TablePro/Core/Database/MongoToolsConnectionString.swift index b33521c85f..53aed2a96b 100644 --- a/apple/TablePro/Core/Database/MongoToolsConnectionString.swift +++ b/apple/TablePro/Core/Database/MongoToolsConnectionString.swift @@ -26,6 +26,17 @@ internal enum MongoToolsConnectionString { // With TLS on, the SSL mode decides certificate checking, and the driver refuses a revocation switch beside the // `tlsInsecure` that already turns it off. private static let ownedByTLS = ["tlsInsecure"] + + // The driver still reads the old `ssl` names as their `tls` options, so a typed one would override a field. + private static let deprecatedAliases = [ + "ssl": "tls", + "sslinsecure": "tlsinsecure", + "sslallowinvalidcertificates": "tlsallowinvalidcertificates", + "sslallowinvalidhostnames": "tlsallowinvalidhostnames", + "sslcertificateauthorityfile": "tlscafile", + "sslclientcertificatekeyfile": "tlscertificatekeyfile", + "sslclientcertificatekeypassword": "tlscertificatekeyfilepassword" + ] private static let impliedByInsecure = ["tlsDisableOCSPEndpointCheck", "tlsDisableCertificateRevocationCheck"] static func make(for connection: DatabaseConnection, host: String) -> String { @@ -140,7 +151,7 @@ internal enum MongoToolsConnectionString { return (name, value) } .sorted { $0.0 < $1.0 } - .filter { taken.insert($0.0.lowercased()).inserted } + .filter { taken.insert(deprecatedAliases[$0.0.lowercased()] ?? $0.0.lowercased()).inserted } .map { "\(encodedValue($0.0))=\(encodedValue($0.1))" } } diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift index 9e15a69745..bb72c1db7e 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -103,8 +103,8 @@ struct PluginMetadataSnapshot: Sendable { /// MongoDB collection lists the fields found in its first documents, so a field missing from /// one side's list says nothing about whether that side holds it. var columnsAreSampled: Bool = false - /// Whether a contains, starts with or ends with filter matches a column of any type by the text it shows, so a - /// search can send number and date columns too. MongoDB's builder converts the value before it matches. + /// Whether a contains, starts with or ends with filter matches a scalar column of any type by the text it shows, + /// so a search can send number and date columns too. MongoDB's builder converts the value before it matches. var patternFiltersReadAnyColumn: Bool = false /// Whether the driver types a filter's values by the kinds it learned from rows it read, so a count of a /// filtered table runs on the driver that read the page. A pooled driver read nothing and would type the same diff --git a/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift index 15ea81577c..7f78ee3286 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift @@ -34,9 +34,13 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil func pageQuery(_ request: TableReadRequest, limit: Int, offset: Int) -> String { let builder = mongo.filterQueryBuilder(for: request.table, database: pinnedDatabase) let sorted = request.sortState.columns + let filter = Self.filterDocument(for: request, builder: builder) + mongo.recordPageFilter( + filter, signature: Self.signature(of: request), table: request.table, database: pinnedDatabase + ) return builder.buildQuery( collection: request.table, - filterDocument: Self.filterDocument(for: request, builder: builder), + filterDocument: filter, sortColumns: sorted.indices.map { (columnIndex: $0, ascending: sorted[$0].ascending) }, columns: sorted.map(\.name), limit: limit, @@ -65,8 +69,11 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil // Bounded like the Mac's background count, so a slow count never holds the session the next page needs. func rowCount(_ request: TableReadRequest) async throws -> Int? { - let builder = mongo.filterQueryBuilder(for: request.table, database: pinnedDatabase) - let filter = Self.filterDocument(for: request, builder: builder) + let signature = Self.signature(of: request) + let filter = mongo.pageFilter(signature: signature, table: request.table, database: pinnedDatabase) + ?? Self.filterDocument( + for: request, builder: mongo.filterQueryBuilder(for: request.table, database: pinnedDatabase) + ) let mongo = mongo return try await inSessionDatabase { try await mongo.documentCount(table: request.table, filterDocument: filter, background: true) @@ -138,6 +145,18 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil } } + static func signature(of request: TableReadRequest) -> String { + let filters = request.filters.map { filter in + [ + filter.columnName, filter.filterOperator.rawValue, filter.value, filter.secondValue, + filter.isCaseSensitive ? "1" : "0" + ].joined(separator: "\u{1F}") + } + let search = [request.searchText ?? "\u{0}"] + request.searchColumns + return ([request.logicMode == .and ? "and" : "or", search.joined(separator: "\u{1F}")] + filters) + .joined(separator: "\u{1E}") + } + private static func filtersAsField(_ name: String) -> Bool { name != MongoDBQueryBuilder.rawFilterColumn && MongoDBQueryBuilder.isPlainFieldName(name) } diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBPluginConnectionTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBPluginConnectionTests.swift index 09b70e3b81..5fefd4cd4c 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBPluginConnectionTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBPluginConnectionTests.swift @@ -241,9 +241,13 @@ struct MongoDBConnectionURITests { #expect(!mongoc_uri_has_option(relaxed, MONGOC_URI_TLSDISABLEOCSPENDPOINTCHECK)) let verified = MongoDBSSLMapping.uriOptions(for: SSLConfiguration(mode: .verifyIdentity)) - let strict = try parsed(uri(ssl: verified, extraParams: typed)) + let aliases = ["sslAllowInvalidCertificates": "true", "SSLAllowInvalidHostnames": "true", "ssl": "false"] + let strict = try parsed(uri(ssl: verified, extraParams: typed.merging(aliases) { first, _ in first })) defer { mongoc_uri_destroy(strict) } #expect(!mongoc_uri_has_option(strict, MONGOC_URI_TLSINSECURE)) + #expect(!mongoc_uri_has_option(strict, MONGOC_URI_TLSALLOWINVALIDCERTIFICATES)) + #expect(!mongoc_uri_has_option(strict, MONGOC_URI_TLSALLOWINVALIDHOSTNAMES)) + #expect(mongoc_uri_get_tls(strict)) #expect(mongoc_uri_get_option_as_bool(strict, MONGOC_URI_TLSDISABLEOCSPENDPOINTCHECK, false)) let plain = try parsed(uri(extraParams: ["tlsInsecure": "true"])) diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift index da481565d9..4cc9668ca2 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift @@ -83,6 +83,39 @@ struct MongoDBQueryBuilderTests { #expect(!match.contains("options")) } + @Test("A row that cannot be built makes the filter match nothing rather than run the other rows") + func unbuildableRowMatchesNothing() { + let rows = [ + PluginQueryFilter(column: "$where", op: "=", value: "1"), + PluginQueryFilter(column: "n", op: "=", value: "5") + ] + let builder = MongoDBQueryBuilder() + #expect(builder.buildFilterDocument(from: rows, logicMode: "and") == MongoDBQueryBuilder.impossibleFilter) + #expect(builder.buildFilterDocument(from: rows, logicMode: "or") == MongoDBQueryBuilder.impossibleFilter) + let raw = PluginQueryFilter(column: MongoDBQueryBuilder.rawFilterColumn, op: "=", value: "{\"status\": \"a\"") + #expect(builder.buildFilterDocument(from: [raw, rows[1]]) == MongoDBQueryBuilder.impossibleFilter) + } + + @Test("A count reuses the filter its page was built with, whatever the page taught the driver since") + func countReusesThePageFilter() throws { + let driver = MongoDBPluginDriver(config: DriverConnectionConfig( + host: "db.example.com", port: 27_017, username: "u", password: "p", database: "app" + )) + let rows = [PluginQueryFilter(column: "a", op: "!=", value: "1")] + let query = try #require(driver.buildFilteredQuery( + table: "orders", schema: nil, queryFilters: rows, logicMode: "and", + sortColumns: [], columns: [], limit: 50, offset: 0, columnKinds: [:] + )) + let signature = MongoDBPluginDriver.filterSignature(rows, logicMode: "and") + let recorded = try #require(driver.pageFilter(signature: signature, table: "orders")) + + #expect(query.contains(recorded)) + #expect(driver.pageFilter(signature: signature, table: "customers") == nil) + #expect(driver.pageFilter(signature: MongoDBPluginDriver.filterSignature(rows, logicMode: "or"), table: "orders") == nil) + let other = [PluginQueryFilter(column: "a", op: "!=", value: "2")] + #expect(driver.pageFilter(signature: MongoDBPluginDriver.filterSignature(other, logicMode: "and"), table: "orders") == nil) + } + @Test("Only a find in the session's database with no projection teaches the collection's kinds") func storedDocumentsAreOnlyWholeFinds() { func outcome(_ find: MongoScriptFindShape?) -> MongoScriptStatementResult { diff --git a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift index 8486e6e5a7..78a7abe9d8 100644 --- a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift +++ b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift @@ -104,17 +104,18 @@ struct FindMatcherTests { #expect(FindMatcher.nearestMatchIndex(to: 0, in: []) == nil) } - @Test("a server search sends only text columns unless the engine's patterns read any column") + @Test("a server search sends only text columns unless the engine's patterns read every scalar column") func serverSearchableColumns() { let typed: [ColumnType] = [ .integer(rawType: "INTEGER"), .decimal(rawType: "FLOAT"), .timestamp(rawType: "TIMESTAMP"), - .boolean(rawType: "BOOLEAN"), .json(rawType: "JSON") + .boolean(rawType: "BOOLEAN") ] for type in typed { #expect(!FindMatcher.isServerSearchable(type)) #expect(FindMatcher.isServerSearchable(type, patternReadsAnyColumn: true)) } #expect(FindMatcher.isServerSearchable(.text(rawType: "ObjectId"))) + #expect(!FindMatcher.isServerSearchable(.json(rawType: "JSON"), patternReadsAnyColumn: true)) #expect(!FindMatcher.isServerSearchable(.blob(rawType: "BLOB"), patternReadsAnyColumn: true)) #expect(!FindMatcher.isServerSearchable(.spatial(rawType: "GEOMETRY"), patternReadsAnyColumn: true)) } diff --git a/apple/TableProTests/Database/NativeDumpRegistryTests.swift b/apple/TableProTests/Database/NativeDumpRegistryTests.swift index bd0025ea89..5fc9c0634d 100644 --- a/apple/TableProTests/Database/NativeDumpRegistryTests.swift +++ b/apple/TableProTests/Database/NativeDumpRegistryTests.swift @@ -292,8 +292,12 @@ struct NativeDumpRegistryTests { var verified = connection(type: .mongodb, sslMode: .verifyIdentity, sslEnabled: true) verified.additionalFields["mongoParam_tlsInsecure"] = "true" + verified.additionalFields["mongoParam_sslInsecure"] = "true" + verified.additionalFields["mongoParam_sslCertificateAuthorityFile"] = "/tmp/other.pem" let strict = try configurationValue("uri", in: configuration(of: command(.mongodb, connection: verified))) #expect(!strict.contains("tlsInsecure")) + #expect(!strict.contains("sslInsecure")) + #expect(!strict.contains("sslCertificateAuthorityFile")) } @Test("A bare IPv6 host reaches the tool in brackets") From eeae00c771ba2772977b005b5f7d008b816073a3 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 22 Oct 2026 18:49:27 +0700 Subject: [PATCH 3/7] fix(mongodb): decode URL query values once and copy only the options the driver sends --- .../MongoDBConnectionURI.swift | 5 +- .../MongoDBQueryBuilder.swift | 35 ++++++++---- .../Database/MongoToolsConnectionString.swift | 44 ++------------ .../Core/Database/MongoTypedParameters.swift | 57 +++++++++++++++++++ .../Connection/ConnectionURLFormatter.swift | 52 +++++++---------- .../Connection/ConnectionURLParser.swift | 40 +++++++------ .../Drivers/MongoDBQueryBuilderTests.swift | 5 +- .../ConnectionURLFormatterTests.swift | 30 ++++++++++ .../Utilities/ConnectionURLParserTests.swift | 16 ++++++ 9 files changed, 177 insertions(+), 107 deletions(-) create mode 100644 apple/TablePro/Core/Database/MongoTypedParameters.swift diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift index 63ac07b105..7b5fc7d60c 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnectionURI.swift @@ -108,8 +108,9 @@ nonisolated struct MongoDBConnectionURI: Sendable { if ssl.contains(where: { $0.key == "tlsAllowInvalidCertificates" }) { taken.formUnion(["tlsdisableocspendpointcheck", "tlsdisablecertificaterevocationcheck"]) } - for key in extraParams.keys.sorted() where taken.insert(canonicalKey(key)).inserted { - options.append(MongoDBURIOption(key: key, value: extraParams[key] ?? "")) + for (key, value) in extraParams.sorted(by: { $0.key < $1.key }) + where !value.isEmpty && taken.insert(canonicalKey(key)).inserted { + options.append(MongoDBURIOption(key: key, value: value)) } return options } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift index 235881b0a9..bc140658c4 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift @@ -116,28 +116,36 @@ nonisolated struct MongoDBQueryBuilder { exactFilterDocument(from: filters, logicMode: logicMode) ?? Self.impossibleFilter } - /// Nil when a row cannot be built: a raw document that does not normalize, a name that reads as an operator, a - /// missing bound. Running the other rows without it would, under "match all", show rows the filter excludes, and - /// a query that failed to build would leave a refresh to reload the table unfiltered, so the caller matches nothing. + /// Nil when a row cannot be built under "match all" (a raw document that does not normalize, a name that reads as + /// an operator, a missing bound), since the other rows alone would show rows the filter excludes. Under "match + /// any" the row only adds documents, so the rest still run. The caller matches nothing rather than return no + /// query, which a refresh would answer by reloading the table unfiltered. private func exactFilterDocument( from filters: [PluginQueryFilter], logicMode: String = "and" ) -> String? { guard !filters.isEmpty else { return "{}" } + let matchesAll = logicMode == "and" var clauses: [MongoDBFilterClause] = [] var scopeOrder: [String] = [] var scoped: [String: [PluginQueryFilter]] = [:] for filter in filters { guard filter.column != Self.rawFilterColumn else { - guard let raw = rawFilterClause(for: filter) else { return nil } - clauses.append(raw) + if let raw = rawFilterClause(for: filter) { + clauses.append(raw) + } else if matchesAll { + return nil + } continue } guard let scope = filter.elementScope, !scope.isEmpty else { - guard let clause = buildClause(for: filter, field: filter.column) else { return nil } - clauses.append(clause) + if let clause = buildClause(for: filter, field: filter.column) { + clauses.append(clause) + } else if matchesAll { + return nil + } continue } if scoped[scope] == nil { @@ -147,12 +155,15 @@ nonisolated struct MongoDBQueryBuilder { } for scope in scopeOrder { - guard let clause = elementMatchClause( - scope: scope, filters: scoped[scope] ?? [], logicMode: logicMode - ) else { return nil } - clauses.append(clause) + if let clause = elementMatchClause(scope: scope, filters: scoped[scope] ?? [], logicMode: logicMode) { + clauses.append(clause) + } else if matchesAll { + return nil + } } + guard !clauses.isEmpty else { return nil } + if clauses.count == 1 { return "{\(clauses[0].json)}" } @@ -238,7 +249,7 @@ nonisolated struct MongoDBQueryBuilder { let inner = filters.compactMap { filter -> MongoDBFilterClause? in buildClause(for: filter, field: Self.relativePath(filter.column, under: scope), readsShownText: false) } - guard !inner.isEmpty, inner.count == filters.count else { return nil } + guard !inner.isEmpty, logicMode != "and" || inner.count == filters.count else { return nil } let body: String if logicMode == "and" || inner.count == 1 { diff --git a/apple/TablePro/Core/Database/MongoToolsConnectionString.swift b/apple/TablePro/Core/Database/MongoToolsConnectionString.swift index 53aed2a96b..e30ddcac6b 100644 --- a/apple/TablePro/Core/Database/MongoToolsConnectionString.swift +++ b/apple/TablePro/Core/Database/MongoToolsConnectionString.swift @@ -18,27 +18,6 @@ internal enum MongoToolsConnectionString { charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" ) - private static let fieldOwnedParameterKeys: Set = [ - "authSource", "authMechanism", "replicaSet", - "tls", "tlsAllowInvalidCertificates", "tlsAllowInvalidHostnames", "tlsCAFile", "tlsCertificateKeyFile" - ] - - // With TLS on, the SSL mode decides certificate checking, and the driver refuses a revocation switch beside the - // `tlsInsecure` that already turns it off. - private static let ownedByTLS = ["tlsInsecure"] - - // The driver still reads the old `ssl` names as their `tls` options, so a typed one would override a field. - private static let deprecatedAliases = [ - "ssl": "tls", - "sslinsecure": "tlsinsecure", - "sslallowinvalidcertificates": "tlsallowinvalidcertificates", - "sslallowinvalidhostnames": "tlsallowinvalidhostnames", - "sslcertificateauthorityfile": "tlscafile", - "sslclientcertificatekeyfile": "tlscertificatekeyfile", - "sslclientcertificatekeypassword": "tlscertificatekeyfilepassword" - ] - private static let impliedByInsecure = ["tlsDisableOCSPEndpointCheck", "tlsDisableCertificateRevocationCheck"] - static func make(for connection: DatabaseConnection, host: String) -> String { let scheme = connection.usesMongoSrv ? "mongodb+srv" : "mongodb" let base = "\(scheme)://\(userInfo(connection))\(hosts(connection, host: host))/" @@ -135,24 +114,11 @@ internal enum MongoToolsConnectionString { } parameters.append(contentsOf: tlsParameters(for: connection)) let setKeys = parameters.compactMap { $0.split(separator: "=", maxSplits: 1).first.map(String.init) } - let tls = setKeys.contains("tls") ? ownedByTLS : [] - let implied = setKeys.contains("tlsInsecure") ? impliedByInsecure : [] - return parameters + extraParameters(connection, excluding: setKeys + fieldOwnedParameterKeys + tls + implied) - } - - // The driver reads keys without case, so a typed parameter named like a field's key in any case is dropped. - private static func extraParameters(_ connection: DatabaseConnection, excluding keys: [String]) -> [String] { - var taken = Set(keys.map { $0.lowercased() }) - return connection.additionalFields - .compactMap { key, value -> (String, String)? in - guard key.hasPrefix("mongoParam_") else { return nil } - let name = String(key.dropFirst("mongoParam_".count)) - guard !name.isEmpty else { return nil } - return (name, value) - } - .sorted { $0.0 < $1.0 } - .filter { taken.insert(deprecatedAliases[$0.0.lowercased()] ?? $0.0.lowercased()).inserted } - .map { "\(encodedValue($0.0))=\(encodedValue($0.1))" } + let typed = MongoTypedParameters.sent( + by: connection, besides: setKeys, + tls: setKeys.contains("tls"), skipsVerification: setKeys.contains("tlsInsecure") + ) + return parameters + typed.map { "\(encodedValue($0.key))=\(encodedValue($0.value))" } } private static func encodedHost(_ host: String) -> String { diff --git a/apple/TablePro/Core/Database/MongoTypedParameters.swift b/apple/TablePro/Core/Database/MongoTypedParameters.swift new file mode 100644 index 0000000000..33b26f4221 --- /dev/null +++ b/apple/TablePro/Core/Database/MongoTypedParameters.swift @@ -0,0 +1,57 @@ +// +// MongoTypedParameters.swift +// TablePro +// + +import Foundation + +// The parameters typed into a MongoDB connection (its `mongoParam_` fields) that reach the driver. The connection's own +// fields own their options, so a parameter named like one, in any case or under an old `ssl` name, never overrides it, +// as the plugin's MongoDBConnectionURI rules. Copy Connection String and the backup tools both follow this. +internal enum MongoTypedParameters { + private static let fieldOwnedKeys: Set = [ + "connecttimeoutms", "serverselectiontimeoutms", "authsource", "authmechanism", "replicaset", "uuidrepresentation", + "tls", "tlsallowinvalidcertificates", "tlsallowinvalidhostnames", "tlscafile", "tlscertificatekeyfile" + ] + + // libmongoc and the tools' Go driver still read these as the `tls` options. + private static let deprecatedAliases = [ + "ssl": "tls", + "sslinsecure": "tlsinsecure", + "sslallowinvalidcertificates": "tlsallowinvalidcertificates", + "sslallowinvalidhostnames": "tlsallowinvalidhostnames", + "sslcertificateauthorityfile": "tlscafile", + "sslclientcertificatekeyfile": "tlscertificatekeyfile", + "sslclientcertificatekeypassword": "tlscertificatekeyfilepassword" + ] + + static func canonicalKey(_ key: String) -> String { + let lowered = key.lowercased() + return deprecatedAliases[lowered] ?? lowered + } + + // `written` names the options the string already carries. With TLS on, the SSL mode decides certificate checking, + // and a mode that skips verification already turns off the revocation checks a driver refuses beside it. + static func sent( + by connection: DatabaseConnection, + besides written: [String], + tls: Bool, + skipsVerification: Bool + ) -> [(key: String, value: String)] { + var taken = fieldOwnedKeys.union(written.map(canonicalKey)) + if tls { + taken.insert("tlsinsecure") + } + if skipsVerification { + taken.formUnion(["tlsdisableocspendpointcheck", "tlsdisablecertificaterevocationcheck"]) + } + return connection.additionalFields + .compactMap { key, value -> (key: String, value: String)? in + guard key.hasPrefix("mongoParam_"), !value.isEmpty else { return nil } + let name = String(key.dropFirst("mongoParam_".count)) + return name.isEmpty ? nil : (name, value) + } + .sorted { $0.key < $1.key } + .filter { taken.insert(canonicalKey($0.key)).inserted } + } +} diff --git a/apple/TablePro/Core/Utilities/Connection/ConnectionURLFormatter.swift b/apple/TablePro/Core/Utilities/Connection/ConnectionURLFormatter.swift index 3f456fdffd..bfd0fd4e93 100644 --- a/apple/TablePro/Core/Utilities/Connection/ConnectionURLFormatter.swift +++ b/apple/TablePro/Core/Utilities/Connection/ConnectionURLFormatter.swift @@ -259,13 +259,7 @@ struct ConnectionURLFormatter { var params: [String] = [] if !connection.name.isEmpty { - let encoded = connection.name - .replacingOccurrences(of: " ", with: "+") - .addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed)? - .replacingOccurrences(of: "&", with: "%26") - .replacingOccurrences(of: "=", with: "%3D") - ?? connection.name - params.append("name=\(encoded)") + params.append("name=\(formEncode(connection.name))") } if ssh.enabled && ssh.authMethod == .privateKey { @@ -295,13 +289,7 @@ struct ConnectionURLFormatter { if let tagId = connection.tagIds.first, let tag = TagStorage.shared.tag(for: tagId) { - let encoded = tag.name - .replacingOccurrences(of: " ", with: "+") - .addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed)? - .replacingOccurrences(of: "&", with: "%26") - .replacingOccurrences(of: "=", with: "%3D") - ?? tag.name - params.append("env=\(encoded)") + params.append("env=\(formEncode(tag.name))") } params.append(contentsOf: mongoParameters(connection)) @@ -309,30 +297,32 @@ struct ConnectionURLFormatter { return params.joined(separator: "&") } - // The standard option names, so the URL opens in other MongoDB tools as well as here. The fields come first and a - // typed parameter named like one is left out, as the driver reads keys without case. + // The standard option names, so the URL opens in other MongoDB tools as well as here: the fields, then the typed + // parameters the driver sends beside them. private static func mongoParameters(_ connection: DatabaseConnection) -> [String] { - let fields: [(String, String?)] = [ + let fields: [(key: String, value: String?)] = [ ("authSource", connection.mongoAuthSource), ("authMechanism", connection.mongoAuthMechanism), ("replicaSet", connection.mongoReplicaSet), ("readPreference", connection.mongoReadPreference), ("w", connection.mongoWriteConcern), - ("uuidRepresentation", connection.additionalFields["mongoUuidRepresentation"]?.nilIfEmpty) + ("uuidRepresentation", connection.additionalFields["mongoUuidRepresentation"].flatMap { $0.isEmpty ? nil : $0 }) ] - var options = fields.compactMap { key, value in value.flatMap { $0.isEmpty ? nil : (key, $0) } } - var taken = Set(options.map { $0.0.lowercased() }) - let typed = connection.additionalFields - .compactMap { key, value -> (String, String)? in - guard key.hasPrefix("mongoParam_"), !value.isEmpty else { return nil } - let name = String(key.dropFirst("mongoParam_".count)) - return name.isEmpty ? nil : (name, value) - } - .sorted { $0.0 < $1.0 } - for option in typed where taken.insert(option.0.lowercased()).inserted { - options.append(option) - } - return options.map { "\(percentEncodeMongoOption($0.0))=\(percentEncodeMongoOption($0.1))" } + let written = fields.compactMap { key, value in value.map { (key: key, value: $0) } } + // An SRV connection always runs TLS, without verification while the SSL mode is still Disabled. + let ssl = connection.usesMongoSrv && !connection.sslConfig.isEnabled + ? SSLConfiguration(mode: .required) : connection.sslConfig + let typed = MongoTypedParameters.sent( + by: connection, besides: written.map(\.key), + tls: ssl.isEnabled, skipsVerification: ssl.mode == .preferred || ssl.mode == .required + ) + return (written + typed).map { "\(percentEncodeMongoOption($0.key))=\(percentEncodeMongoOption($0.value))" } + } + + // The parser reads `name` and `env` form-style, a space as `+`, so a literal `+` is written `%2B`. + private static func formEncode(_ value: String) -> String { + let allowed = unreservedAllowed.union(CharacterSet(charactersIn: " ")) + return (value.addingPercentEncoding(withAllowedCharacters: allowed) ?? value).replacingOccurrences(of: " ", with: "+") } // Other MongoDB drivers split options on `;` and read `+` as a space, so only unreserved characters stay literal. diff --git a/apple/TablePro/Core/Utilities/Connection/ConnectionURLParser.swift b/apple/TablePro/Core/Utilities/Connection/ConnectionURLParser.swift index 2d94281d0e..16bf4da860 100644 --- a/apple/TablePro/Core/Utilities/Connection/ConnectionURLParser.swift +++ b/apple/TablePro/Core/Utilities/Connection/ConnectionURLParser.swift @@ -193,7 +193,7 @@ struct ConnectionURLParser { case .failure(let error): return .failure(error) } - var ext = parseQueryItems(components.queryItems, dbType: dbType) + var ext = parseQueryItems(components.percentEncodedQueryItems, dbType: dbType) let rawPort = endpoint.port ?? components.port var sslMode = ext.resolvedSSLMode(for: dbType, port: rawPort ?? dbType.defaultPort) @@ -649,9 +649,7 @@ struct ConnectionURLParser { if let qs = queryString { queryItems = qs.split(separator: "&").map { param in let kv = param.split(separator: "=", maxSplits: 1) - let key = String(kv[0]) - let val = kv.count > 1 ? (String(kv[1]).removingPercentEncoding ?? String(kv[1])) : "" - return URLQueryItem(name: key, value: val) + return URLQueryItem(name: String(kv[0]), value: kv.count > 1 ? String(kv[1]) : "") } } @@ -753,6 +751,7 @@ struct ConnectionURLParser { } } + // Takes the items still percent-encoded, as every path hands them over, so applyQueryParam decodes each once. private static func parseQueryItems(_ queryItems: [URLQueryItem]?, dbType: DatabaseType? = nil) -> ExtendedParams { var ext = ExtendedParams() guard let queryItems else { return ext } @@ -770,10 +769,9 @@ struct ConnectionURLParser { for param in params { let parts = param.split(separator: "=", maxSplits: 1) guard let encodedKey = parts.first, parts.count > 1 else { continue } - // Decoded once, as URLQueryItem decodes the standard form's query. - let key = String(encodedKey).removingPercentEncoding ?? String(encodedKey) - let value = String(parts[1]).removingPercentEncoding ?? String(parts[1]) - let keyStr = key.lowercased() + let key = String(encodedKey) + let value = String(parts[1]) + let keyStr = (key.removingPercentEncoding ?? key).lowercased() if keyStr == "useprivatekey" { ext.usePrivateKey = value.lowercased() == "true" continue @@ -787,7 +785,7 @@ struct ConnectionURLParser { continue } if keyStr == "agentsocket" { - ext.agentSocket = value + ext.agentSocket = value.removingPercentEncoding ?? value continue } applyQueryParam(key: key, value: value, to: &ext, dbType: dbType) @@ -795,8 +793,13 @@ struct ConnectionURLParser { return ext } - private static func applyQueryParam(key rawKey: String, value: String, to ext: inout ExtendedParams, dbType: DatabaseType? = nil) { + // `key` and `encoded` arrive percent-encoded and are decoded here once. A value the app writes form-style reads `+` + // as a space, so a literal `+` in it is `%2B`. + private static func applyQueryParam(key encodedKey: String, value encoded: String, to ext: inout ExtendedParams, dbType: DatabaseType? = nil) { + let rawKey = encodedKey.removingPercentEncoding ?? encodedKey let key = rawKey.lowercased() + let value = encoded.removingPercentEncoding ?? encoded + let formValue = encoded.replacingOccurrences(of: "+", with: " ").removingPercentEncoding ?? value switch key { case "sslmode": if let mode = parseSSLMode(value) { @@ -817,29 +820,24 @@ struct ConnectionURLParser { case "statuscolor": ext.statusColor = value case "env", "enviroment", "environment": - ext.envTag = value.removingPercentEncoding ?? value + ext.envTag = formValue case "schema": ext.schema = value case "table": - ext.tableName = value.replacingOccurrences(of: "+", with: " ") - .removingPercentEncoding ?? value + ext.tableName = formValue case "name": - ext.connectionName = value.replacingOccurrences(of: "+", with: " ") - .removingPercentEncoding ?? value + ext.connectionName = formValue case "view": - ext.tableName = value.replacingOccurrences(of: "+", with: " ") - .removingPercentEncoding ?? value + ext.tableName = formValue ext.isView = true case "column": ext.filterColumn = value case "operation", "operator": ext.filterOperation = value case "value": - ext.filterValue = value.replacingOccurrences(of: "+", with: " ") - .removingPercentEncoding ?? value + ext.filterValue = formValue case "condition", "raw", "query": - ext.filterCondition = value.replacingOccurrences(of: "+", with: " ") - .removingPercentEncoding ?? value + ext.filterCondition = formValue case "tlsmode": if ext.sslMode == nil, let intValue = Int(value) { ext.sslMode = parseTlsModeInteger(intValue) diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift index 4cc9668ca2..8326f1f8e9 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift @@ -83,7 +83,7 @@ struct MongoDBQueryBuilderTests { #expect(!match.contains("options")) } - @Test("A row that cannot be built makes the filter match nothing rather than run the other rows") + @Test("Under match all a row that cannot be built matches nothing; under match any the other rows still run") func unbuildableRowMatchesNothing() { let rows = [ PluginQueryFilter(column: "$where", op: "=", value: "1"), @@ -91,7 +91,8 @@ struct MongoDBQueryBuilderTests { ] let builder = MongoDBQueryBuilder() #expect(builder.buildFilterDocument(from: rows, logicMode: "and") == MongoDBQueryBuilder.impossibleFilter) - #expect(builder.buildFilterDocument(from: rows, logicMode: "or") == MongoDBQueryBuilder.impossibleFilter) + #expect(builder.buildFilterDocument(from: rows, logicMode: "or") == #"{"n": 5}"#) + #expect(builder.buildFilterDocument(from: [rows[0]], logicMode: "or") == MongoDBQueryBuilder.impossibleFilter) let raw = PluginQueryFilter(column: MongoDBQueryBuilder.rawFilterColumn, op: "=", value: "{\"status\": \"a\"") #expect(builder.buildFilterDocument(from: [raw, rows[1]]) == MongoDBQueryBuilder.impossibleFilter) } diff --git a/apple/TableProTests/Core/Utilities/ConnectionURLFormatterTests.swift b/apple/TableProTests/Core/Utilities/ConnectionURLFormatterTests.swift index c71f915b71..5b269fb091 100644 --- a/apple/TableProTests/Core/Utilities/ConnectionURLFormatterTests.swift +++ b/apple/TableProTests/Core/Utilities/ConnectionURLFormatterTests.swift @@ -678,6 +678,36 @@ struct ConnectionURLFormatterTests { #expect(opened.additionalFields["mongoParam_uuidRepresentation"] == nil) } + @Test("A copied MongoDB URL leaves out a typed parameter the driver ignores beside a field") + func mongoIgnoredParametersAreNotCopied() { + let conn = DatabaseConnection( + name: "", host: "host", port: 27_017, database: "app", username: "user", type: .mongodb, + sslConfig: SSLConfiguration(mode: .verifyIdentity), + additionalFields: [ + "mongoParam_AUTHSource": "other", "mongoParam_tls": "false", "mongoParam_sslAllowInvalidCertificates": "true", + "mongoParam_tlsInsecure": "true", "mongoParam_uuidRepresentation": "javaLegacy", "mongoParam_appName": "x" + ] + ) + let url = ConnectionURLFormatter.format(conn, password: nil, sshPassword: nil) + for ignored in ["AUTHSource", "tls=", "sslAllowInvalidCertificates", "tlsInsecure", "uuidRepresentation"] { + #expect(!url.contains(ignored), "\(url)") + } + #expect(url.contains("appName=x")) + } + + @Test("A connection name with a plus copies and opens as itself") + func nameWithPlusRoundTrips() { + let conn = DatabaseConnection( + name: "C++ Shop & Co", host: "host", port: 5_432, database: "app", username: "user", type: .postgresql + ) + let url = ConnectionURLFormatter.format(conn, password: nil, sshPassword: nil) + guard case .success(let parsed) = ConnectionURLParser.parse(url) else { + Issue.record("\(url) did not parse") + return + } + #expect(parsed.connectionName == "C++ Shop & Co") + } + @Test("An SSH URL's MongoDB options are decoded once, as a standard URL's are") func mongoSSHURLDecodesOptions() throws { let url = "mongodb+ssh://me@bastion.example.com/u:p@10.0.0.5:27017/app" diff --git a/apple/TableProTests/Core/Utilities/ConnectionURLParserTests.swift b/apple/TableProTests/Core/Utilities/ConnectionURLParserTests.swift index eca925578d..d2fc6f94dd 100644 --- a/apple/TableProTests/Core/Utilities/ConnectionURLParserTests.swift +++ b/apple/TableProTests/Core/Utilities/ConnectionURLParserTests.swift @@ -943,6 +943,22 @@ struct ConnectionURLParserTests { #expect(parsed.isView == false) } + @Test("A query value is decoded once, and an encoded plus stays a plus, in a standard or an SSH URL", arguments: [ + "postgresql://user@host/db", + "postgresql+ssh://me@bastion/user@10.0.0.5/db" + ]) + func queryValuesDecodeOnce(base: String) throws { + let url = base + "?name=C%2B%2B+DB&value=a%2Bb&condition=n+%2B+1+%3D+2&schema=s%2525" + guard case .success(let parsed) = ConnectionURLParser.parse(url) else { + Issue.record("\(url) did not parse") + return + } + #expect(parsed.connectionName == "C++ DB") + #expect(parsed.filterValue == "a+b") + #expect(parsed.filterCondition == "n + 1 = 2") + #expect(parsed.schema == "s%25") + } + @Test("Connection name from name parameter in standard URL") func testConnectionNameFromStandardURL() { let result = ConnectionURLParser.parse("mysql://root@localhost/db?name=My+Database") From cde6035dffa4e2be00db70535d78ee0d4496b36c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 22 Oct 2026 19:00:14 +0700 Subject: [PATCH 4/7] fix(mongodb): keep a filtered page from retyping the fields it filtered on --- .../MongoDBPluginDriver.swift | 92 ++++++++++--------- .../MongoFilterFields.swift | 44 +++++++++ .../MongoDBDriverPlugin/MongoScriptHost.swift | 3 +- .../MongoScriptRuntime.swift | 1 + .../Drivers/MongoDBBackedDriver.swift | 25 +---- .../Drivers/MongoDBQueryBuilderTests.swift | 35 +++---- 6 files changed, 118 insertions(+), 82 deletions(-) create mode 100644 apple/Plugins/MongoDBDriverPlugin/MongoFilterFields.swift diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 6b00b9879b..8bf878aa0c 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -21,7 +21,6 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se private var identityKindsByCollection: [String: Set] = [:] private var binarySubtypesByCollection: [String: MongoDBBinarySubtypes] = [:] private var fieldKindsByCollection: [String: MongoDBFieldKinds] = [:] - private var pageFiltersByCollection: [String: (signature: String, document: String)] = [:] private static let logger = Logger(subsystem: "com.TablePro", category: "MongoDBPluginDriver") @@ -244,7 +243,8 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se from: documents, startTime: startTime, isTruncated: isTruncated, collection: collection, declared: declared, - recordsKinds: outcome.returnsStoredDocuments(in: self.currentDb) + recordsKinds: outcome.returnsStoredDocuments(in: self.currentDb), + filteredFields: outcome.find?.filteredFields ?? [] ) } ), @@ -516,9 +516,8 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se logicMode: String, background: Bool ) async throws -> Int? { - let signature = Self.filterSignature(filters, logicMode: logicMode) - let filterJson = pageFilter(signature: signature, table: table) - ?? filterQueryBuilder(for: table).buildFilterDocument(from: filters, logicMode: logicMode) + let filterJson = filterQueryBuilder(for: table) + .buildFilterDocument(from: filters, logicMode: logicMode) return try await documentCount(table: table, filterDocument: filterJson, background: background) } @@ -787,39 +786,13 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se columnKinds: [String: PluginColumnKind] ) -> String? { let builder = filterQueryBuilder(for: table) - let document = builder.buildFilterDocument(from: queryFilters, logicMode: logicMode) - recordPageFilter(document, signature: Self.filterSignature(queryFilters, logicMode: logicMode), table: table) return builder.buildQuery( - collection: table, filterDocument: document, + collection: table, + filterDocument: builder.buildFilterDocument(from: queryFilters, logicMode: logicMode), sortColumns: sortColumns, columns: columns, limit: limit, offset: offset ) } - // A page's own rows can change the kinds that type a filter value before its count is built, so the count reuses - // the document the page ran: a field holding both 1 and "1" would otherwise count the documents the page left out. - func recordPageFilter(_ document: String, signature: String, table: String, database: String? = nil) { - let key = columnKindKey(table, database: database) - columnKindLock.withLock { pageFiltersByCollection[key] = (signature, document) } - } - - func pageFilter(signature: String, table: String, database: String? = nil) -> String? { - let key = columnKindKey(table, database: database) - guard let page = columnKindLock.withLock({ pageFiltersByCollection[key] }), page.signature == signature else { - return nil - } - return page.document - } - - static func filterSignature(_ filters: [PluginQueryFilter], logicMode: String) -> String { - let rows = filters.map { filter in - [ - filter.column, filter.op, filter.value, filter.isCaseSensitive ? "1" : "0", - filter.secondValue ?? "\u{0}", filter.elementScope ?? "\u{0}" - ].joined(separator: "\u{1F}") - } - return ([logicMode] + rows).joined(separator: "\u{1E}") - } - func generateRowWrites( table: String, schema: String?, @@ -997,7 +970,8 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se self.buildPluginResult( from: MongoReadDocuments(dictionaries: documents.dictionaries, texts: []), startTime: Date(), isTruncated: isTruncated, collection: collection, - recordsKinds: outcome.returnsStoredDocuments(in: self.currentDb) + recordsKinds: outcome.returnsStoredDocuments(in: self.currentDb), + filteredFields: outcome.find?.filteredFields ?? [] ) } ) @@ -1020,7 +994,8 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se isTruncated: Bool = false, collection: String = "", declared: MongoDBCollectionSchema = .empty, - recordsKinds: Bool = false + recordsKinds: Bool = false, + filteredFields: Set = [] ) -> PluginQueryResult { let documents = read.dictionaries if documents.isEmpty { @@ -1036,9 +1011,9 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se for: sampled, documents: documents, representation: uuidRepresentation ) if recordsKinds { - rememberColumnKinds(sampledKinds, for: sampled, collection: collection) + rememberColumnKinds(sampledKinds, for: sampled, collection: collection, keeping: filteredFields) rememberIdentityKinds(of: documents, collection: collection) - rememberFieldPathKinds(from: documents, collection: collection) + rememberFieldPathKinds(from: documents, collection: collection, keeping: filteredFields) rememberBinarySubtypes(of: documents, collection: collection) rememberFieldKinds(of: documents, collection: collection) } @@ -1085,14 +1060,40 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se return result } - private func rememberColumnKinds(_ kinds: [BsonValueKind], for columns: [String], collection: String) { + private func rememberColumnKinds( + _ kinds: [BsonValueKind], + for columns: [String], + collection: String, + keeping filteredFields: Set = [] + ) { guard !collection.isEmpty else { return } var byName: [String: BsonValueKind] = [:] for (index, name) in columns.enumerated() where index < kinds.count { byName[name] = kinds[index] } let key = columnKindKey(collection) - columnKindLock.withLock { columnKindsByCollection[key] = byName } + columnKindLock.withLock { + columnKindsByCollection[key] = Self.kinds( + byName, keepingFrom: columnKindsByCollection[key] ?? [:], of: filteredFields + ) + } + } + + // A filtered page holds only the values its filter let through, so it says nothing about the fields it filtered + // on: a field holding 1 and "1" filtered by `!= 1` returns only strings. Those keep the kind they had, or the + // page's count, typed again after the page, would count other documents than the page shows. + static func kinds( + _ page: [String: BsonValueKind], + keepingFrom previous: [String: BsonValueKind], + of filteredFields: Set + ) -> [String: BsonValueKind] { + var merged = page + for field in filteredFields { + if let kept = previous[field] { + merged[field] = kept + } + } + return merged } /// Added to rather than replaced, so a value keeps its subtype after another page or another tab @@ -1139,13 +1140,21 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se /// build the filter. Sampling through `sampleFieldPaths` cannot do it: that call is routed /// through `MetadataConnectionPool`, so it lands on a different driver instance whose cache /// the filter path never reads. - private func rememberFieldPathKinds(from documents: [[String: Any]], collection: String) { + private func rememberFieldPathKinds( + from documents: [[String: Any]], + collection: String, + keeping filteredFields: Set = [] + ) { guard !collection.isEmpty, !documents.isEmpty else { return } let sampled = Array(documents.prefix(MongoStreamProjection.sampleSize)) let kinds = BsonDocumentFlattener.fieldPathKinds(from: sampled, representation: uuidRepresentation) guard !kinds.isEmpty else { return } let key = columnKindKey(collection) - columnKindLock.withLock { fieldPathKindsByCollection[key] = kinds } + columnKindLock.withLock { + fieldPathKindsByCollection[key] = Self.kinds( + kinds, keepingFrom: fieldPathKindsByCollection[key] ?? [:], of: filteredFields + ) + } } /// Kinds a filter can be typed against: the flat columns the grid shows, plus every nested @@ -1185,7 +1194,6 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se fieldPathKindsByCollection = fieldPathKindsByCollection.filter { !isStale($0.key) } declaredSchemasByCollection = declaredSchemasByCollection.filter { !isStale($0.key) } identityKindsByCollection = identityKindsByCollection.filter { !isStale($0.key) } - pageFiltersByCollection = pageFiltersByCollection.filter { !isStale($0.key) } } } } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoFilterFields.swift b/apple/Plugins/MongoDBDriverPlugin/MongoFilterFields.swift new file mode 100644 index 0000000000..a6f0dbc664 --- /dev/null +++ b/apple/Plugins/MongoDBDriverPlugin/MongoFilterFields.swift @@ -0,0 +1,44 @@ +// +// MongoFilterFields.swift +// MongoDBDriverPlugin +// + +import Foundation + +// The fields a find's filter document reads: every key that is not an operator, at any depth, and every `$`-prefixed +// path an expression names. Wider than exact on purpose, since it only decides which fields a page keeps its learned +// kinds for, and each dotted path also names its top-level field. +nonisolated enum MongoFilterFields { + static func named(in filterJson: String) -> Set { + guard let filter = MongoJsonValue.parse(filterJson) else { return [] } + var fields = Set() + collect(filter, into: &fields) + return fields + } + + private static func collect(_ value: Any, into fields: inout Set) { + if let object = value as? [String: Any] { + guard !MongoJsonValue.isWrapper(object) else { return } + for (key, child) in object { + if !key.hasPrefix("$") { + insert(key, into: &fields) + } + collect(child, into: &fields) + } + } else if let array = value as? [Any] { + for element in array { + collect(element, into: &fields) + } + } else if let text = value as? String, text.hasPrefix("$"), !text.hasPrefix("$$") { + insert(String(text.dropFirst()), into: &fields) + } + } + + private static func insert(_ path: String, into fields: inout Set) { + guard !path.isEmpty else { return } + fields.insert(path) + if let first = path.split(separator: ".", omittingEmptySubsequences: false).first { + fields.insert(String(first)) + } + } +} diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift b/apple/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift index b4690ba0ae..731f92ef51 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift @@ -224,7 +224,8 @@ nonisolated final class MongoScriptHost { let shape = MongoScriptFindShape( database: cursor.database, returnsWholeDocuments: cursor.options.projection == nil, - skip: cursor.options.skip ?? 0 + skip: cursor.options.skip ?? 0, + filteredFields: MongoFilterFields.named(in: cursor.filterJson) ) return (cursor.collection, shape) } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift b/apple/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift index 2d68fe336e..1ac7d25160 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoScriptRuntime.swift @@ -10,6 +10,7 @@ nonisolated struct MongoScriptFindShape: Equatable, Sendable { let database: String let returnsWholeDocuments: Bool let skip: Int + var filteredFields: Set = [] } /// What one statement of a script evaluated to. diff --git a/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift index 7f78ee3286..15ea81577c 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift @@ -34,13 +34,9 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil func pageQuery(_ request: TableReadRequest, limit: Int, offset: Int) -> String { let builder = mongo.filterQueryBuilder(for: request.table, database: pinnedDatabase) let sorted = request.sortState.columns - let filter = Self.filterDocument(for: request, builder: builder) - mongo.recordPageFilter( - filter, signature: Self.signature(of: request), table: request.table, database: pinnedDatabase - ) return builder.buildQuery( collection: request.table, - filterDocument: filter, + filterDocument: Self.filterDocument(for: request, builder: builder), sortColumns: sorted.indices.map { (columnIndex: $0, ascending: sorted[$0].ascending) }, columns: sorted.map(\.name), limit: limit, @@ -69,11 +65,8 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil // Bounded like the Mac's background count, so a slow count never holds the session the next page needs. func rowCount(_ request: TableReadRequest) async throws -> Int? { - let signature = Self.signature(of: request) - let filter = mongo.pageFilter(signature: signature, table: request.table, database: pinnedDatabase) - ?? Self.filterDocument( - for: request, builder: mongo.filterQueryBuilder(for: request.table, database: pinnedDatabase) - ) + let builder = mongo.filterQueryBuilder(for: request.table, database: pinnedDatabase) + let filter = Self.filterDocument(for: request, builder: builder) let mongo = mongo return try await inSessionDatabase { try await mongo.documentCount(table: request.table, filterDocument: filter, background: true) @@ -145,18 +138,6 @@ nonisolated final class MongoDBBackedDriver: PluginBackedDriver, DriverQueryBuil } } - static func signature(of request: TableReadRequest) -> String { - let filters = request.filters.map { filter in - [ - filter.columnName, filter.filterOperator.rawValue, filter.value, filter.secondValue, - filter.isCaseSensitive ? "1" : "0" - ].joined(separator: "\u{1F}") - } - let search = [request.searchText ?? "\u{0}"] + request.searchColumns - return ([request.logicMode == .and ? "and" : "or", search.joined(separator: "\u{1F}")] + filters) - .joined(separator: "\u{1E}") - } - private static func filtersAsField(_ name: String) -> Bool { name != MongoDBQueryBuilder.rawFilterColumn && MongoDBQueryBuilder.isPlainFieldName(name) } diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift index 8326f1f8e9..0ee7cca425 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift @@ -97,24 +97,25 @@ struct MongoDBQueryBuilderTests { #expect(builder.buildFilterDocument(from: [raw, rows[1]]) == MongoDBQueryBuilder.impossibleFilter) } - @Test("A count reuses the filter its page was built with, whatever the page taught the driver since") - func countReusesThePageFilter() throws { - let driver = MongoDBPluginDriver(config: DriverConnectionConfig( - host: "db.example.com", port: 27_017, username: "u", password: "p", database: "app" - )) - let rows = [PluginQueryFilter(column: "a", op: "!=", value: "1")] - let query = try #require(driver.buildFilteredQuery( - table: "orders", schema: nil, queryFilters: rows, logicMode: "and", - sortColumns: [], columns: [], limit: 50, offset: 0, columnKinds: [:] - )) - let signature = MongoDBPluginDriver.filterSignature(rows, logicMode: "and") - let recorded = try #require(driver.pageFilter(signature: signature, table: "orders")) + @Test("A filtered page keeps the kinds of the fields it filtered on, so its count is typed as the page was") + func filteredPageKeepsItsFieldsKinds() { + let filter = MongoDBQueryBuilder(columnKinds: ["a": .int32], matchesShownText: true).buildFilterDocument( + from: [ + PluginQueryFilter(column: "a", op: "!=", value: "1"), + PluginQueryFilter(column: "b.c", op: "CONTAINS", value: "x", isCaseSensitive: false) + ], + logicMode: "and" + ) + let fields = MongoFilterFields.named(in: filter) + #expect(fields.isSuperset(of: ["a", "b.c", "b"])) + #expect(!fields.contains("1")) - #expect(query.contains(recorded)) - #expect(driver.pageFilter(signature: signature, table: "customers") == nil) - #expect(driver.pageFilter(signature: MongoDBPluginDriver.filterSignature(rows, logicMode: "or"), table: "orders") == nil) - let other = [PluginQueryFilter(column: "a", op: "!=", value: "2")] - #expect(driver.pageFilter(signature: MongoDBPluginDriver.filterSignature(other, logicMode: "and"), table: "orders") == nil) + let page: [String: BsonValueKind] = ["a": .string, "b": .document, "n": .string] + let previous: [String: BsonValueKind] = ["a": .int32, "n": .int32] + let merged = MongoDBPluginDriver.kinds(page, keepingFrom: previous, of: fields) + #expect(merged["a"] == .int32) + #expect(merged["n"] == .string) + #expect(merged["b"] == .document) } @Test("Only a find in the session's database with no projection teaches the collection's kinds") From cab92389a831883e275824b625b02da22e738280 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 22 Oct 2026 19:11:13 +0700 Subject: [PATCH 5/7] fix(mongodb): type filters apart from writes and keep unknown kinds a filtered page read --- .../MongoDBPluginDriver.swift | 77 +++++++++---------- .../Drivers/MongoDBQueryBuilderTests.swift | 6 +- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 8bf878aa0c..7a3c2fd0ca 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -16,7 +16,9 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se private let columnKindLock = NSLock() private let rawFilterNormalizer = MongoDBRawFilterNormalizer() private var columnKindsByCollection: [String: [String: BsonValueKind]] = [:] - private var fieldPathKindsByCollection: [String: [String: BsonValueKind]] = [:] + // What filters are typed by: top-level fields and nested paths. Kept apart from the kinds writes are typed by, since + // a filtered page leaves the fields it filtered on alone here and must still type the next write by what it read. + private var filterKindsByCollection: [String: [String: BsonValueKind]] = [:] private var declaredSchemasByCollection: [String: MongoDBCollectionSchema] = [:] private var identityKindsByCollection: [String: Set] = [:] private var binarySubtypesByCollection: [String: MongoDBBinarySubtypes] = [:] @@ -373,6 +375,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se for: sampled, documents: docs, representation: uuidRepresentation ) rememberColumnKinds(kinds, for: sampled, collection: table) + rememberFilterKinds(from: docs, columns: sampled, kinds: kinds, collection: table) let sampledColumns = sampled.enumerated().map { index, name in PluginColumnInfo( @@ -1011,9 +1014,11 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se for: sampled, documents: documents, representation: uuidRepresentation ) if recordsKinds { - rememberColumnKinds(sampledKinds, for: sampled, collection: collection, keeping: filteredFields) + rememberColumnKinds(sampledKinds, for: sampled, collection: collection) + rememberFilterKinds( + from: documents, columns: sampled, kinds: sampledKinds, collection: collection, keeping: filteredFields + ) rememberIdentityKinds(of: documents, collection: collection) - rememberFieldPathKinds(from: documents, collection: collection, keeping: filteredFields) rememberBinarySubtypes(of: documents, collection: collection) rememberFieldKinds(of: documents, collection: collection) } @@ -1060,28 +1065,44 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se return result } - private func rememberColumnKinds( - _ kinds: [BsonValueKind], - for columns: [String], - collection: String, - keeping filteredFields: Set = [] - ) { + private func rememberColumnKinds(_ kinds: [BsonValueKind], for columns: [String], collection: String) { guard !collection.isEmpty else { return } var byName: [String: BsonValueKind] = [:] for (index, name) in columns.enumerated() where index < kinds.count { byName[name] = kinds[index] } let key = columnKindKey(collection) + columnKindLock.withLock { columnKindsByCollection[key] = byName } + } + + /// Recorded from the documents a browse already fetched, on the session driver that will + /// build the filter. Sampling through `sampleFieldPaths` cannot do it: that call is routed + /// through `MetadataConnectionPool`, so it lands on a different driver instance whose cache + /// the filter path never reads. + private func rememberFilterKinds( + from documents: [[String: Any]], + columns: [String], + kinds: [BsonValueKind], + collection: String, + keeping filteredFields: Set = [] + ) { + guard !collection.isEmpty, !documents.isEmpty else { return } + let sampled = Array(documents.prefix(MongoStreamProjection.sampleSize)) + var learned = BsonDocumentFlattener.fieldPathKinds(from: sampled, representation: uuidRepresentation) + for (index, name) in columns.enumerated() where index < kinds.count { + learned[name] = kinds[index] + } + let key = columnKindKey(collection) columnKindLock.withLock { - columnKindsByCollection[key] = Self.kinds( - byName, keepingFrom: columnKindsByCollection[key] ?? [:], of: filteredFields + filterKindsByCollection[key] = Self.kinds( + learned, keepingFrom: filterKindsByCollection[key] ?? [:], of: filteredFields ) } } // A filtered page holds only the values its filter let through, so it says nothing about the fields it filtered - // on: a field holding 1 and "1" filtered by `!= 1` returns only strings. Those keep the kind they had, or the - // page's count, typed again after the page, would count other documents than the page shows. + // on: a field holding 1 and "1" filtered by `!= 1` returns only strings. Those keep what was known of them, an + // unknown kind included, or the page's count, typed again after the page, would count other documents. static func kinds( _ page: [String: BsonValueKind], keepingFrom previous: [String: BsonValueKind], @@ -1089,9 +1110,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se ) -> [String: BsonValueKind] { var merged = page for field in filteredFields { - if let kept = previous[field] { - merged[field] = kept - } + merged[field] = previous[field] } return merged } @@ -1136,27 +1155,6 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se columnKindLock.withLock { identityKindsByCollection[key, default: []].formUnion(held) } } - /// Recorded from the documents a browse already fetched, on the session driver that will - /// build the filter. Sampling through `sampleFieldPaths` cannot do it: that call is routed - /// through `MetadataConnectionPool`, so it lands on a different driver instance whose cache - /// the filter path never reads. - private func rememberFieldPathKinds( - from documents: [[String: Any]], - collection: String, - keeping filteredFields: Set = [] - ) { - guard !collection.isEmpty, !documents.isEmpty else { return } - let sampled = Array(documents.prefix(MongoStreamProjection.sampleSize)) - let kinds = BsonDocumentFlattener.fieldPathKinds(from: sampled, representation: uuidRepresentation) - guard !kinds.isEmpty else { return } - let key = columnKindKey(collection) - columnKindLock.withLock { - fieldPathKindsByCollection[key] = Self.kinds( - kinds, keepingFrom: fieldPathKindsByCollection[key] ?? [:], of: filteredFields - ) - } - } - /// Kinds a filter can be typed against: the flat columns the grid shows, plus every nested /// path the picker offers. Without the nested half, a filter on a nested Date or ObjectId is /// compared as a string and MongoDB's type bracketing returns nothing. @@ -1164,8 +1162,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se let key = columnKindKey(collection, database: database) return columnKindLock.withLock { (declaredSchemasByCollection[key]?.valueKinds ?? [:]) - .merging(fieldPathKindsByCollection[key] ?? [:]) { _, sampled in sampled } - .merging(columnKindsByCollection[key] ?? [:]) { _, top in top } + .merging(filterKindsByCollection[key] ?? [:]) { _, sampled in sampled } } } @@ -1191,7 +1188,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se let isStale = { (key: String) in MongoCollectionCacheKey.names(key, collection: table) } columnKindLock.withLock { columnKindsByCollection = columnKindsByCollection.filter { !isStale($0.key) } - fieldPathKindsByCollection = fieldPathKindsByCollection.filter { !isStale($0.key) } + filterKindsByCollection = filterKindsByCollection.filter { !isStale($0.key) } declaredSchemasByCollection = declaredSchemasByCollection.filter { !isStale($0.key) } identityKindsByCollection = identityKindsByCollection.filter { !isStale($0.key) } } diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift index 0ee7cca425..4986f10c61 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBQueryBuilderTests.swift @@ -115,7 +115,11 @@ struct MongoDBQueryBuilderTests { let merged = MongoDBPluginDriver.kinds(page, keepingFrom: previous, of: fields) #expect(merged["a"] == .int32) #expect(merged["n"] == .string) - #expect(merged["b"] == .document) + #expect(merged["b"] == nil, "A field the filter read stays unknown when it was unknown before the page") + + let restored = MongoDBPluginDriver.kinds(page, keepingFrom: [:], of: fields) + #expect(restored["a"] == nil) + #expect(restored["n"] == .string) } @Test("Only a find in the session's database with no projection teaches the collection's kinds") From d7a80f64edec5000f0673b8f6d4d13249c9fa865 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 22 Oct 2026 19:22:48 +0700 Subject: [PATCH 6/7] fix(mongodb): keep the kinds of paths under a filtered array --- .../MongoDBPluginDriver.swift | 19 +++++++++++++++---- .../Drivers/MongoDBQueryBuilderTests.swift | 12 ++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 7a3c2fd0ca..2c605ed6e7 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -1101,20 +1101,31 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se } // A filtered page holds only the values its filter let through, so it says nothing about the fields it filtered - // on: a field holding 1 and "1" filtered by `!= 1` returns only strings. Those keep what was known of them, an - // unknown kind included, or the page's count, typed again after the page, would count other documents. + // on or the paths under them, which an `$elemMatch` names relative to its array: a field holding 1 and "1" + // filtered by `!= 1` returns only strings. Those keep what was known of them, an unknown kind included, or the + // page's count, typed again after the page, would count other documents. static func kinds( _ page: [String: BsonValueKind], keepingFrom previous: [String: BsonValueKind], of filteredFields: Set ) -> [String: BsonValueKind] { + guard !filteredFields.isEmpty else { return page } var merged = page - for field in filteredFields { - merged[field] = previous[field] + for path in Set(page.keys).union(previous.keys) where isUnder(path, anyOf: filteredFields) { + merged[path] = previous[path] } return merged } + private static func isUnder(_ path: String, anyOf fields: Set) -> Bool { + var candidate = Substring(path) + while true { + if fields.contains(String(candidate)) { return true } + guard let dot = candidate.lastIndex(of: ".") else { return false } + candidate = candidate[.. Date: Thu, 22 Oct 2026 19:28:43 +0700 Subject: [PATCH 7/7] docs(changelog): add the MongoDB fixes --- apple/changelog.d/3786.fixed.1.md | 1 + apple/changelog.d/3786.fixed.2.md | 1 + apple/changelog.d/3786.fixed.3.md | 1 + apple/changelog.d/3786.fixed.4.md | 1 + apple/changelog.d/3786.fixed.5.md | 1 + apple/changelog.d/3786.fixed.md | 1 + apple/changelog.d/3786.security.md | 1 + 7 files changed, 7 insertions(+) create mode 100644 apple/changelog.d/3786.fixed.1.md create mode 100644 apple/changelog.d/3786.fixed.2.md create mode 100644 apple/changelog.d/3786.fixed.3.md create mode 100644 apple/changelog.d/3786.fixed.4.md create mode 100644 apple/changelog.d/3786.fixed.5.md create mode 100644 apple/changelog.d/3786.fixed.md create mode 100644 apple/changelog.d/3786.security.md diff --git a/apple/changelog.d/3786.fixed.1.md b/apple/changelog.d/3786.fixed.1.md new file mode 100644 index 0000000000..a0d077113e --- /dev/null +++ b/apple/changelog.d/3786.fixed.1.md @@ -0,0 +1 @@ +A MongoDB option holding `&`, such as a CA file path or replica set name, broke or changed the connection. diff --git a/apple/changelog.d/3786.fixed.2.md b/apple/changelog.d/3786.fixed.2.md new file mode 100644 index 0000000000..7a2dc7174d --- /dev/null +++ b/apple/changelog.d/3786.fixed.2.md @@ -0,0 +1 @@ +A query tab saved by an old version could save edits to MongoDB aggregation rows into the wrong documents. diff --git a/apple/changelog.d/3786.fixed.3.md b/apple/changelog.d/3786.fixed.3.md new file mode 100644 index 0000000000..a324975a71 --- /dev/null +++ b/apple/changelog.d/3786.fixed.3.md @@ -0,0 +1 @@ +A filtered MongoDB table could show a row total that did not match its rows. diff --git a/apple/changelog.d/3786.fixed.4.md b/apple/changelog.d/3786.fixed.4.md new file mode 100644 index 0000000000..b5e31bd811 --- /dev/null +++ b/apple/changelog.d/3786.fixed.4.md @@ -0,0 +1 @@ +MongoDB backup and restore failed with X.509 or AWS sign-in, a bare IPv6 host, or extra TLS options. diff --git a/apple/changelog.d/3786.fixed.5.md b/apple/changelog.d/3786.fixed.5.md new file mode 100644 index 0000000000..db895b3a58 --- /dev/null +++ b/apple/changelog.d/3786.fixed.5.md @@ -0,0 +1 @@ +Copy Connection String dropped a MongoDB connection's SRV setting and options, and a `+` in a pasted URL became a space. diff --git a/apple/changelog.d/3786.fixed.md b/apple/changelog.d/3786.fixed.md new file mode 100644 index 0000000000..f27b244be6 --- /dev/null +++ b/apple/changelog.d/3786.fixed.md @@ -0,0 +1 @@ +Search All Rows on a MongoDB collection skipped numbers, dates and ObjectIds, and failed on iOS beside binary data. diff --git a/apple/changelog.d/3786.security.md b/apple/changelog.d/3786.security.md new file mode 100644 index 0000000000..1b0aefe812 --- /dev/null +++ b/apple/changelog.d/3786.security.md @@ -0,0 +1 @@ +A typed MongoDB parameter could turn off the certificate check the SSL mode asked for.