diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 34aae47217..95b0b558d8 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -278,6 +278,11 @@ let package = Package( dependencies: ["TableProSpannerCore"], path: "Tests/TableProSpannerCoreTests" ), + .testTarget( + name: "TableProTeradataCoreTests", + dependencies: ["TableProTeradataCore"], + path: "Tests/TableProTeradataCoreTests" + ), .testTarget( name: "TableProTrinoCoreTests", dependencies: ["TableProTrinoCore"], diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift index d96c5be4f0..bc2ce87ab9 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift @@ -15,14 +15,16 @@ public struct BigtableColumn: Hashable, Sendable, Comparable { family + ":" + BigtableBytes.display(qualifier) } + // By scalar: a qualifier that starts with a combining mark joins the colon into one Character. public init(name: String) throws { - guard let colon = name.firstIndex(of: ":"), colon != name.startIndex else { + guard let colon = name.unicodeScalars.firstIndex(of: ":"), colon != name.startIndex else { throw BigtableError.configuration(String( format: String(localized: "\"%@\" is not a column. Name it as family:qualifier."), name )) } - self.family = String(name[.. Bool { diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift index 08b8966201..c05591c4f0 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift @@ -157,7 +157,7 @@ public struct BigtableRequest: Sendable, Equatable { /// Text that would not read back as one bare word goes in JSON quotes. static func token(_ text: String, reserved: String) -> String { - let plain = !text.isEmpty && !text.hasPrefix("\"") && text.unicodeScalars.allSatisfy { scalar in + let plain = !text.isEmpty && text.unicodeScalars.first != "\"" && text.unicodeScalars.allSatisfy { scalar in !CharacterSet.whitespacesAndNewlines.contains(scalar) && !reserved.unicodeScalars.contains(scalar) } if plain { return text } @@ -173,22 +173,23 @@ public struct BigtableRequest: Sendable, Equatable { static let missingTable = BigtableError.configuration(String(localized: "Name the table before the key, as in GET events/user#42.")) } -/// Reads a request line one token at a time. +// Read by scalar, as `token` writes: a key that starts with a combining mark joins the quote or slash before it +// into one Character. private struct RequestScanner { - private var text: Substring + private var text: Substring.UnicodeScalarView private var query: [String: String] = [:] init(_ text: String) { - self.text = Substring(text) + self.text = Substring(text).unicodeScalars } var isAtEnd: Bool { - text.allSatisfy(\.isWhitespace) + text.allSatisfy(\.properties.isWhitespace) } - mutating func word(stoppingAt stop: Character? = nil) -> String? { + mutating func word(stoppingAt stop: Unicode.Scalar? = nil) -> String? { skipSpace() - let word = text.prefix { !$0.isWhitespace && $0 != stop } + let word = text.prefix { !$0.properties.isWhitespace && $0 != stop } guard !word.isEmpty else { return nil } text = text.dropFirst(word.count) return String(word) @@ -196,7 +197,7 @@ private struct RequestScanner { mutating func rest() -> String { let remaining = String(text).trimmingCharacters(in: .whitespacesAndNewlines) - text = "" + text.removeAll() return remaining } @@ -204,7 +205,7 @@ private struct RequestScanner { /// no slash is a table alone, as in `SCAN events`. mutating func target(tableOnly: Bool) throws -> (table: String, key: Data?) { skipSpace() - let head = text.prefix { !$0.isWhitespace && $0 != "\"" && $0 != "?" } + let head = text.prefix { !$0.properties.isWhitespace && $0 != "\"" && $0 != "?" } let table: String if let slash = head.firstIndex(of: "/") { table = String(text[.. Void + ) { + completionHandler(nil) + } + func finish() { guard let task = lock.withLock({ task }) else { return } transport?.unregister(task) diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift index 0272332c25..2abb610a77 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift @@ -58,11 +58,15 @@ public enum MongoStatementClassifier { // Server-side JavaScript, plus the two stages that write to a collection the statement names. private static let codeExecutionKeys: Set = ["$where", "$function", "$accumulator", "$out", "$merge"] + // The server accepts its own operators from any client, and rewrites `mapReduce` into two of them that run a + // function: `$_internalJsEmit` and `$_internalJsReduce`. Nothing a person writes names one. + private static let internalOperatorPrefix = "$_internal" + private static let codeExecutionCalls: Set = ["mapreduce", "eval"] // For text the grammar rejects, where a false alarm only stops an external client sending code nobody could read. private static let codeExecutionMarkers: [String] = [ - "$where", "$function", "$accumulator", "mapreduce", ".eval(", "$out", "$merge" + "$where", "$function", "$accumulator", "$_internal", "mapreduce", ".eval(", "$out", "$merge" ] private static func statementVerdict( @@ -78,6 +82,7 @@ public enum MongoStatementClassifier { private static func verdict(of statement: MongoStatement) -> MongoStatementVerdict { let reachesCode = !statement.objectKeys.isDisjoint(with: codeExecutionKeys) + || statement.objectKeys.contains { $0.hasPrefix(internalOperatorPrefix) } || statement.calls.contains { call in codeExecutionCalls.contains(call.name.lowercased()) || call.commandName.map { codeExecutionCalls.contains($0.lowercased()) } == true diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnfilteredWrite.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnfilteredWrite.swift index 835d11631c..9582c3d733 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnfilteredWrite.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnfilteredWrite.swift @@ -16,6 +16,8 @@ public enum SQLUnfilteredWrite: Sendable, Equatable { private static let period = UInt16(UnicodeScalar(".").value) + private static let whereLength = "WHERE".utf16.count + private static let servedStatements: Set = ["SELECT", "INSERT", "UPDATE", "DELETE", "MERGE"] // `:where` and `$where` are bind parameters on SQLite. @@ -30,58 +32,65 @@ public enum SQLUnfilteredWrite: Sendable, Equatable { } let depth = tokens[start].depth var found: [SQLUnfilteredWrite] = [] - let verb: Int? + let verbs: [Int] switch first { case "UPDATE", "DELETE": - verb = start + verbs = [start] case "WITH": - // A CTE may be named `delete` or `merge`, and on PostgreSQL its body may itself write. A list that does - // not parse is searched whole, which can only find a write that is not there. - let list = commonTableExpressions(in: tokens, with: start) - for body in list?.bodies ?? [] { + // A CTE may be named `delete` or `merge`, and on PostgreSQL its body may itself write. Every body is read + // whether or not the list around it can be followed. When it cannot, every word at the statement's depth + // is a candidate, which can only find a write that is not there. + for body in commonTableExpressionBodies(in: tokens, depth: depth) { found += writes(inStatement: source.substring(with: body) as NSString, grammar: grammar) } - verb = tokens.indices.dropFirst(list?.main ?? start + 1).first { index in - tokens[index].depth == depth && servedStatements.contains(tokens[index].word ?? "") - } + verbs = mainStatement(in: tokens, with: start).map { [$0] } + ?? tokens.indices.dropFirst(start + 1).filter { tokens[$0].depth == depth } case "ALTER": // ClickHouse writes rows with `ALTER TABLE t UPDATE ... WHERE` and `ALTER TABLE t DELETE WHERE`. guard tokens.indices.contains(start + 1), tokens[start + 1].word == "TABLE" else { return [] } - verb = tokens.indices.dropFirst(start + 2).first { index in + let verb = tokens.indices.dropFirst(start + 2).first { index in let word = tokens[index].word return tokens[index].depth == depth && (word == "UPDATE" || word == "DELETE") && tokens[index - 1].word != "ON" } + verbs = verb.map { [$0] } ?? [] default: - verb = nil + verbs = [] } - if let verb, let write = write(at: verb, in: tokens), !hasOwnWhere(tokens, depth: depth, in: source) { - found.append(write) + let writes = verbs.compactMap { write(at: $0, in: tokens) } + if !writes.isEmpty, !hasOwnWhere(tokens, depth: depth, in: source) { + found += writes } return found } - private static func commonTableExpressions( - in tokens: [TSQLTokens.Token], - with start: Int - ) -> (bodies: [NSRange], main: Int)? { - let depth = tokens[start].depth + private static let bodyOpeners: Set = ["AS", "MATERIALIZED"] + + // Found by scanning rather than by walking the list, so a body is read when a name is several tokens, as + // `U&"gone"` is, or a SEARCH or CYCLE clause follows an earlier body. + private static func commonTableExpressionBodies(in tokens: [TSQLTokens.Token], depth: Int) -> [NSRange] { + tokens.indices.dropFirst().compactMap { index in + guard tokens[index].depth == depth, bodyOpeners.contains(tokens[index - 1].word ?? ""), + let close = close(of: index, in: tokens) + else { return nil } + let inside = tokens[index].location + 1 + return NSRange(location: inside, length: tokens[close].location - inside) + } + } + + // Nil when no statement begins right after the last body: a name T-SQL reads as several words where the engine + // reads one, as at a euro sign, or PostgreSQL's `SEARCH ... SET name` and `CYCLE ... USING name` after a body. + private static func mainStatement(in tokens: [TSQLTokens.Token], with start: Int) -> Int? { var index = start + 1 - var bodies: [NSRange] = [] func word(_ expected: String) -> Bool { guard tokens.indices.contains(index), tokens[index].word == expected else { return false } index += 1 return true } - func parentheses() -> NSRange? { - guard tokens.indices.contains(index), tokens[index].kind == .open, - let close = tokens.indices.dropFirst(index + 1).first(where: { - tokens[$0].kind == .close && tokens[$0].depth == depth - }) - else { return nil } - let inside = tokens[index].location + 1 + func parentheses() -> Bool { + guard let close = close(of: index, in: tokens) else { return false } index = close + 1 - return NSRange(location: inside, length: tokens[close].location - inside) + return true } _ = word("RECURSIVE") while tokens.indices.contains(index) { @@ -90,12 +99,19 @@ public enum SQLUnfilteredWrite: Sendable, Equatable { guard word("AS") else { return nil } _ = word("NOT") _ = word("MATERIALIZED") - guard let body = parentheses() else { return nil } - bodies.append(body) + guard parentheses() else { return nil } guard tokens.indices.contains(index), tokens[index].kind == .comma else { break } index += 1 } - return tokens.indices.contains(index) ? (bodies, index) : nil + guard tokens.indices.contains(index), servedStatements.contains(tokens[index].word ?? "") else { return nil } + return index + } + + private static func close(of open: Int, in tokens: [TSQLTokens.Token]) -> Int? { + guard tokens.indices.contains(open), tokens[open].kind == .open else { return nil } + return tokens.indices.dropFirst(open + 1).first { + tokens[$0].kind == .close && tokens[$0].depth == tokens[open].depth + } } private static func write(at index: Int, in tokens: [TSQLTokens.Token]) -> SQLUnfilteredWrite? { @@ -115,13 +131,23 @@ public enum SQLUnfilteredWrite: Sendable, Equatable { private static func hasOwnWhere(_ tokens: [TSQLTokens.Token], depth: Int, in source: NSString) -> Bool { tokens.indices.contains { index in - guard tokens[index].depth == depth, tokens[index].word == "WHERE" else { return false } - guard index > 0 else { return true } - let location = tokens[index].location - if location > 0, parameterPrefixes.contains(source.character(at: location - 1)) { return false } - // `t.where` is a column. - let previous = tokens[index - 1] - return previous.kind != .other || source.character(at: previous.location) != period + tokens[index].depth == depth && tokens[index].word == "WHERE" && !isName(tokens, at: index, in: source) } } + + // The tokens end a word where T-SQL does. SQLite, PostgreSQL and MySQL read every character outside ASCII as part + // of a name, a no-break space included, and MySQL lets a digit begin one. Taking a real filter for a name only + // asks about a statement that had one. + private static func isName(_ tokens: [TSQLTokens.Token], at index: Int, in source: NSString) -> Bool { + guard index > 0 else { return false } + let location = tokens[index].location + let before = source.character(at: location - 1) + if parameterPrefixes.contains(before) || SqlDollarQuote.isIdentifierContinuation(before) { return true } + let end = location + whereLength + if end < source.length, source.character(at: end) >= 0x80 { return true } + // `t.where` is a column, and so is PostgreSQL's `RETURNING id AS where`. + let previous = tokens[index - 1] + if previous.word == "AS" { return true } + return previous.kind == .other && source.character(at: previous.location) == period + } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift index 8ed1b5ee0a..576e9b5f0a 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift @@ -73,6 +73,9 @@ public enum SQLUnterminatedStatements { return tokens[index].depth == 0 || (next?.word != "HINT" && next?.word != "PLAN") case "END": return next?.word == "CONVERSATION" + case "BULK": + // `ADMINISTER BULK OPERATIONS` is a permission, and neither `BULK INSERT` nor `OPENROWSET(BULK ...)` goes on so. + return next?.word != "OPERATIONS" default: return true } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift index ddc18c1b2c..ed5608fe9f 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift @@ -5,6 +5,7 @@ public final class TeradataConnection { private let config: TeradataConnectionConfig private let socketLock = NSLock() private var _socket: (any TeradataTransport)? + private var _hasLostConnection = false private var socket: (any TeradataTransport)? { get { socketLock.withLock { _socket } } set { socketLock.withLock { _socket = newValue } } @@ -23,8 +24,17 @@ public final class TeradataConnection { self.config = config } + init(config: TeradataConnectionConfig, transport: any TeradataTransport) { + self.config = config + self._socket = transport + confidentialityBypassed = true + } + public var isConnected: Bool { socket != nil } + // Set when a request failed partway, so the session it ran on was given up rather than ended. + public var hasLostConnection: Bool { socketLock.withLock { _hasLostConnection } } + public func connect() throws { try validateLogMech() let deadline = TeradataConnectDeadline(milliseconds: config.connectTimeoutMilliseconds) @@ -36,7 +46,10 @@ public final class TeradataConnection { connectDeadline = nil throw error } - self.socket = socket + socketLock.withLock { + _socket = socket + _hasLostConnection = false + } var established = false defer { if !established { @@ -78,20 +91,36 @@ public final class TeradataConnection { let body = TeradataMessages.optionsParcel(returnStatementInfo: false).encoded() + TeradataMessages.requestParcel(sql: sql).encoded() + TeradataMessages.responseParcel().encoded() - try sendEncrypted(kind: .start, body: body, requestNumber: requestNumber) + let parcels: [Parcel] + do { + parcels = try exchange(body, requestNumber: requestNumber) + } catch { + // The reply still on its way would be read as the next request's. + dropTransport() + throw error + } + return try TeradataResultParser.parse(parcels) + } + private func exchange(_ body: [UInt8], requestNumber: UInt32) throws -> [Parcel] { + try sendEncrypted(kind: .start, body: body, requestNumber: requestNumber) var parcels: [Parcel] = [] - var responseComplete = false - while !responseComplete { + while true { let batch = try readReply() parcels.append(contentsOf: batch) - if Self.responseIsComplete(batch) { - responseComplete = true - } else { - try sendContinue(requestNumber: requestNumber) - } + if Self.responseIsComplete(batch) { return parcels } + try sendContinue(requestNumber: requestNumber) } - return try TeradataResultParser.parse(parcels) + } + + private func dropTransport() { + let dropped = socketLock.withLock { + let transport = _socket + _socket = nil + _hasLostConnection = transport != nil + return transport + } + dropped?.close() } public func disconnect() { diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift index e737c16353..70cd2da0a6 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift @@ -167,6 +167,17 @@ private final class TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Se transport?.register(task) } + // A redirect would carry the API key to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } + func finish() { guard let task = lock.withLock({ task }) else { return } transport?.unregister(task) diff --git a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift index 36757a8d44..7442a5c272 100644 --- a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift @@ -116,6 +116,8 @@ struct BigtableBytesTests { let column = try BigtableColumn(name: "cf:a:b") #expect(column.family == "cf" && column.qualifier == Data("a:b".utf8)) #expect(try BigtableColumn(name: "cf:b64:/wA=").qualifier == Data([0xFF, 0x00])) + let marked = try BigtableColumn(name: "cf:\u{301}a:b") + #expect(marked.family == "cf" && marked.qualifier == Data("\u{301}a:b".utf8)) #expect(throws: BigtableError.self) { try BigtableColumn(name: "nocolon") } #expect(throws: BigtableError.self) { try BigtableColumn(name: ":q") } } diff --git a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift index f8bc8ba3f0..e9bf7ce67b 100644 --- a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift @@ -72,6 +72,23 @@ struct BigtableStatementTests { #expect(try BigtableStatement.parse(BigtableStatement.encode(spec)) == .browse(spec)) } + @Test("A key or column that starts or ends with a joining scalar reads back from its own text") + func roundTripBesideJoiningScalars() throws { + for text in ["\u{301}audit?", "\u{301}a/b", "\u{301}audit cf:a", "\u{301}audit", "\"\u{301}audit", "a b\u{600}", "a\u{600}", "x \u{600}\\"] { + let key = Data(text.utf8) + for request in [ + BigtableRequest(verb: .delete, table: "events", key: key), + BigtableRequest(verb: .delete, table: "events", key: key, column: "cf:" + text), + BigtableRequest(verb: .put, table: "events", key: key, body: #"{"cf:a":"v"}"#), + BigtableRequest(verb: .scan, table: "events", key: key, limit: 5) + ] { + #expect(try BigtableRequest.parse(request.text) == request, "\(request.text)") + } + } + let column = BigtableRequest(verb: .delete, table: "events", key: Data("k".utf8), column: "\u{200D}") + #expect(try BigtableRequest.parse(column.text) == column) + } + @Test("A write body sets cells, deletes nulled ones under PUT, and writes numbers and booleans as text") func mutations() throws { let put = BigtableRequest(verb: .put, table: "t", key: Data("k".utf8), body: #"{"cf:name": null, "cf:n": 42, "cf:on": true, "cf:raw": "b64:AAE="}"#) diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/MongoStatementClassifierTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/MongoStatementClassifierTests.swift index 4ea8d18902..45b98fab3a 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/MongoStatementClassifierTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/MongoStatementClassifierTests.swift @@ -63,6 +63,23 @@ struct MongoStatementClassifierTests { #expect(!verdict(#"db.users.find({where: "this.a > 1"})"#).reachesCode) } + @Test("The server's own operators reach code, the two it rewrites mapReduce into among them", arguments: [ + #"db.users.aggregate([{$project: {e: {$_internalJsEmit: {this: "$$ROOT", eval: "function () {}"}}}}])"#, + #"db.users.aggregate([{$group: {_id: null, v: {$_internalJsReduce: {data: {k: 1, v: "$a"}, eval: "function () {}"}}}}])"#, + #"db.users.aggregate([{$project: {e: {"\u0024_internalJsEmit": {this: "$$ROOT", eval: "function () {}"}}}}])"#, + #"db.users.find({$expr: {$_internalJsEmit: {this: "$$ROOT", eval: "function () {}"}}})"# + ]) + func internalOperatorsReachCode(text: String) { + #expect(verdict(text) == MongoStatementVerdict(tier: .read, reachesCode: true)) + } + + @Test("An internal operator reaches code in text the grammar rejects, and only as a key in text it accepts") + func internalOperatorSpelling() { + #expect(verdict(#"var p = [{$project: {e: {$_internalJsEmit: {this: "$$ROOT", eval: "function () {}"}}}}]"#) + == MongoStatementVerdict(tier: .write, reachesCode: true)) + #expect(!verdict(#"db.users.find({_internal: "$_internalJsEmit"})"#).reachesCode) + } + @Test("Several statements take the worst tier, and any one reaching code marks the text") func worstStatementWins() { #expect(tier("db.a.find({})\ndb.b.insertOne({})") == .write) diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLUnfilteredWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLUnfilteredWriteTests.swift index 1bf04160cc..c6bf2af551 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLUnfilteredWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLUnfilteredWriteTests.swift @@ -80,6 +80,61 @@ struct SQLUnfilteredWriteTests { #expect(first("WITH a AS (SELECT 1), b AS (UPDATE t SET c = 1 RETURNING c) SELECT * FROM b") == .update) } + private static let walk = "WITH RECURSIVE a(id) AS (SELECT 1 UNION ALL SELECT id + 1 FROM a WHERE id < 2)" + + @Test("A writing common table expression is read wherever it stands in the list", arguments: [ + "WITH U&\"gone\" AS (DELETE FROM items RETURNING *) SELECT 1", + "WITH gone\u{20AC} AS (DELETE FROM items RETURNING *) SELECT 1", + "WITH \u{20AC}as (id) AS (DELETE FROM items RETURNING id) SELECT 1", + "\(walk) SEARCH DEPTH FIRST BY id SET ord, gone AS (DELETE FROM items RETURNING *) SELECT 1", + "\(walk) CYCLE id SET looped USING path, gone AS MATERIALIZED (DELETE FROM items RETURNING *) SELECT 1", + ]) + func writingExpressionAnywhereInTheList(sql: String) { + #expect(first(sql) == .delete) + } + + @Test("A name the engine reads as one word and T-SQL as several is not the statement", arguments: [ + "WITH merge\u{20AC} AS (SELECT 1) DELETE FROM t", + "WITH update\u{20AC} AS (SELECT 1) DELETE FROM t", + "WITH c\u{A0}select AS (SELECT 1) DELETE FROM t", + "WITH \u{20AC}as AS (SELECT 1), merge AS (SELECT 1) DELETE FROM t", + "WITH 1merge AS (SELECT 1) DELETE FROM t", + ]) + func nameOfSeveralTokens(sql: String) { + #expect(SQLUnfilteredWrite.writes(in: sql, grammar: Self.grammar("SQLite")).contains(.delete)) + #expect(SQLUnfilteredWrite.writes(in: sql, grammar: Self.mySQL).contains(.delete)) + #expect(first(sql + " WHERE id = 1", Self.grammar("SQLite")) == nil) + } + + @Test("A SEARCH or CYCLE clause is not the statement") + func searchAndCycleClauses() { + #expect(first("\(Self.walk) SEARCH BREADTH FIRST BY id SET merge DELETE FROM items") == .delete) + #expect(first("\(Self.walk) CYCLE id SET c USING merge DELETE FROM items") == .delete) + #expect(first("\(Self.walk) CYCLE id SET c USING p DELETE FROM items WHERE id IN (SELECT id FROM a)") == nil) + #expect(first("\(Self.walk) SEARCH DEPTH FIRST BY id SET ord SELECT * FROM a") == nil) + } + + @Test("A WHERE the engine reads as a name, or as part of one, is not the statement's", arguments: [ + ("DELETE FROM orders AS o\u{A0}WHERE\u{A0}id\u{A0}IS\u{A0}NULL", "SQLite"), + ("DELETE FROM t AS \u{20AC}where", "SQLite"), + ("DELETE FROM t AS where\u{20AC}", "SQLite"), + ("DELETE FROM t x\u{200B}where", "PostgreSQL"), + ("DELETE FROM t RETURNING id AS where", "PostgreSQL"), + ("DELETE FROM t RETURNING id AS /* label */ where", "PostgreSQL"), + ("DELETE FROM t AS 1where", "MySQL"), + ("DELETE x\u{660}where FROM t AS x\u{660}where", "SQL Server"), + ]) + func whereInsideAName(sql: String, typeId: String) { + #expect(first(sql, Self.grammar(typeId)) == .delete) + } + + @Test("A name outside ASCII that ends before the WHERE leaves it the statement's own") + func whereAfterANameOutsideASCII() { + #expect(first("DELETE FROM t AS \u{20AC} WHERE id = 1", Self.grammar("SQLite")) == nil) + #expect(first("DELETE FROM \"\u{20AC}\"WHERE id = 1 RETURNING id AS where") == nil) + #expect(first("UPDATE t SET a = 1, b = 2 WHERE \u{E9}t\u{E9} = 1") == nil) + } + @Test("Only T-SQL's maintenance command is spelled UPDATE STATISTICS without a SET") func tableNamedStatistics() { #expect(first("UPDATE statistics SET value = 1") == .update) diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLUnterminatedStatementsTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLUnterminatedStatementsTests.swift index c2c461f661..ef2a4f647d 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLUnterminatedStatementsTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLUnterminatedStatementsTests.swift @@ -64,6 +64,7 @@ struct SQLUnterminatedStatementsTests { ("DELETE FROM t SELECT 1 WHERE 1 = 1", ["DELETE FROM t", "SELECT 1 WHERE 1 = 1"]), ("CREATE TYPE dbo.t FROM int DROP TABLE x", ["CREATE TYPE dbo.t FROM int", "DROP TABLE x"]), ("SELECT 1 LINENO 5 DELETE FROM t", ["SELECT 1", "LINENO 5", "DELETE FROM t"]), + ("SELECT 1 BULK INSERT t FROM 'f'", ["SELECT 1", "BULK", "INSERT t FROM 'f'"]), ]) func juxtaposedStatements(sql: String, expected: [String]) { #expect(inner(sql) == expected) @@ -159,6 +160,8 @@ struct SQLUnterminatedStatementsTests { ), ("CREATE TABLE t (a int REFERENCES p (id) ON DELETE SET NULL ON UPDATE SET DEFAULT)", "CREATE"), ("GRANT SELECT, UPDATE ON [t] TO [u]", "GRANT"), + ("GRANT ADMINISTER BULK OPERATIONS TO u", "GRANT"), + ("DENY ADMINISTER DATABASE BULK OPERATIONS TO u", "DENY"), ("ALTER TABLE [t] ADD FOREIGN KEY ([a]) REFERENCES [p] ([id]) ON DELETE CASCADE ON UPDATE SET NULL", "ALTER"), ( "MERGE t USING s ON t.id = s.id WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id);", @@ -170,6 +173,7 @@ struct SQLUnterminatedStatementsTests { #expect(found.first?.uppercased().hasPrefix(keyword) == true) #expect(!found.contains { $0.uppercased().hasPrefix("DELETE") }) #expect(!found.contains { $0.uppercased().hasPrefix("INSERT") }) + #expect(!found.contains { $0.uppercased().hasPrefix("BULK") }) } @Test("A procedure, function or trigger runs nothing written after its header", arguments: [ @@ -190,6 +194,14 @@ struct SQLUnterminatedStatementsTests { #expect(inner(sql) == ["INSERT INTO log", "SELECT id FROM () AS d", "DELETE FROM t OUTPUT deleted.id"]) } + @Test("The BULK of an OPENROWSET comes on its own, whatever stands before it", arguments: [ + "SELECT BulkColumn FROM OPENROWSET(BULK 'f', SINGLE_CLOB) AS d", + "SELECT BulkColumn FROM OPENROWSET ( /* */ bulk N'f', SINGLE_CLOB) AS d", + ]) + func openRowsetBulkComesOnItsOwn(sql: String) { + #expect(begins("BULK", in: sql)) + } + @Test("A subquery does not end the statement around it, so its WHERE stays with it") func subqueryKeepsTheOuterWhere() { let sql = "DELETE t FROM t JOIN (SELECT id FROM s) AS x ON x.id = t.id WHERE x.id > 1" diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataConnectionLossTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataConnectionLossTests.swift new file mode 100644 index 0000000000..f006e0246d --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataConnectionLossTests.swift @@ -0,0 +1,77 @@ +@testable import TableProTeradataCore +import Testing + +private final class ScriptedTransport: TeradataTransport { + private var replies: [[UInt8]] + private var timeouts: [Bool] + private var stream: [UInt8] = [] + private(set) var sent = 0 + private(set) var closed = false + + // A nil reply is a receive that times out, as `recv` does after 20 seconds. + init(replies: [[UInt8]?]) { + self.replies = replies.map { $0 ?? [] } + timeouts = replies.map { $0 == nil } + } + + func send(_ bytes: [UInt8]) throws { + sent += 1 + } + + func receive(_ count: Int) throws -> [UInt8] { + if stream.isEmpty, !replies.isEmpty { + let timesOut = timeouts.removeFirst() + stream = replies.removeFirst() + if timesOut { throw TeradataWireError.truncated("recv errno 35") } + } + guard stream.count >= count else { throw TeradataWireError.truncated("closed") } + defer { stream.removeFirst(count) } + return Array(stream.prefix(count)) + } + + func finishConnecting() {} + + func cancel() {} + + func close() { + closed = true + } +} + +struct TeradataConnectionLossTests { + private static let config = TeradataConnectionConfig(host: "td.example", username: "dbc", password: "dbc") + + private static func reply(_ parcels: [Parcel]) -> [UInt8] { + LanMessage(kind: .start, body: parcels.flatMap { $0.encoded() }).encoded() + } + + @Test("A request that times out gives the session up, so its late reply is never read as the next one's") + func timeoutGivesTheSessionUp() throws { + let lateReply = Self.reply([Parcel(.success), Parcel(.endRequest)]) + let transport = ScriptedTransport(replies: [nil, lateReply]) + let connection = TeradataConnection(config: Self.config, transport: transport) + + #expect(throws: TeradataWireError.self) { try connection.execute("SELECT * FROM mallory.notes") } + #expect(transport.closed) + #expect(!connection.isConnected) + #expect(connection.hasLostConnection) + + #expect(throws: TeradataWireError.self) { try connection.execute("SELECT iban FROM finance.payees") } + #expect(transport.sent == 1) + } + + @Test("An error the server answers with ends the request and keeps the session") + func serverErrorKeepsTheSession() throws { + let failure = Parcel(.failure, body: [UInt8](repeating: 0, count: 8) + [0x0E, 0x3F, 0x00, 0x01, 0x78]) + let transport = ScriptedTransport(replies: [Self.reply([failure]), Self.reply([Parcel(.endRequest)])]) + let connection = TeradataConnection(config: Self.config, transport: transport) + + #expect(throws: TeradataWireError.self) { try connection.execute("SELECT * FROM missing") } + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sent == 2) + #expect(!transport.closed) + } +} diff --git a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift index 75b0173664..5bbd4314cd 100644 --- a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift +++ b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift @@ -70,6 +70,7 @@ internal final class BigQueryPluginDriver: PluginDatabaseDriver, @unchecked Send .dataCompare, .cellExpressions, .uuidCellText, + .rowWrites, ] } diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift index 0d5acfefb4..3a26349f24 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift @@ -285,12 +285,9 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked ) urlConfig.timeoutIntervalForResource = HttpQueryTimeout.sessionResourceTimeout + let delegate = tlsDelegate ?? PluginRedirectRefusingDelegate() lock.withLock { - if let tlsDelegate { - session = URLSession(configuration: urlConfig, delegate: tlsDelegate, delegateQueue: nil) - } else { - session = URLSession(configuration: urlConfig) - } + session = URLSession(configuration: urlConfig, delegate: delegate, delegateQueue: nil) } let versionText: String? @@ -925,7 +922,7 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked // MARK: - TLS Delegate -nonisolated final class ClickHouseTLSDelegate: NSObject, URLSessionDelegate, @unchecked Sendable { +nonisolated final class ClickHouseTLSDelegate: PluginRedirectRefusingDelegate, @unchecked Sendable { private enum Strategy { case systemTrust case skipVerify diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift index a0c432ad4c..c491bb2332 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift @@ -136,7 +136,7 @@ nonisolated extension ClickHousePluginDriver { /// With `send_progress_in_http_headers` the status goes out with the first progress header, so /// a query that fails after it answers HTTP 200 and names the failure in this header. static func isFailure(_ response: HTTPURLResponse) -> Bool { - response.statusCode >= 400 || response.value(forHTTPHeaderField: exceptionCodeHeader) != nil + !(200 ..< 300).contains(response.statusCode) || response.value(forHTTPHeaderField: exceptionCodeHeader) != nil } static func headerFields(_ response: HTTPURLResponse?) -> [String: String] { diff --git a/apple/Plugins/ClickHouseDriverTests/ClickHouseHTTPStub.swift b/apple/Plugins/ClickHouseDriverTests/ClickHouseHTTPStub.swift index c026a3ab9f..dd2826ae15 100644 --- a/apple/Plugins/ClickHouseDriverTests/ClickHouseHTTPStub.swift +++ b/apple/Plugins/ClickHouseDriverTests/ClickHouseHTTPStub.swift @@ -59,6 +59,12 @@ final class ClickHouseHTTPStubProtocol: URLProtocol, @unchecked Sendable { headerFields: reply.headers ) else { return } + // As URLSession's own loader does: offer the redirect, then hand over the 3xx if the delegate keeps it. + if (300 ..< 400).contains(reply.statusCode), let location = reply.headers["Location"].flatMap(URL.init) { + var redirected = request + redirected.url = location + client?.urlProtocol(self, wasRedirectedTo: redirected, redirectResponse: response) + } client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) client?.urlProtocol(self, didLoad: Data(reply.body.utf8)) client?.urlProtocolDidFinishLoading(self) diff --git a/apple/Plugins/ClickHouseDriverTests/ClickHouseRedirectTests.swift b/apple/Plugins/ClickHouseDriverTests/ClickHouseRedirectTests.swift new file mode 100644 index 0000000000..4312e7a225 --- /dev/null +++ b/apple/Plugins/ClickHouseDriverTests/ClickHouseRedirectTests.swift @@ -0,0 +1,21 @@ +import Foundation +import TableProPluginKit +import Testing + +struct ClickHouseRedirectTests { + @Test("A redirect to another server is refused, so the connection and its credentials stay where they point") + func redirectElsewhereIsRefused() async throws { + let elsewhere = ClickHouseStubServer() + let target = "http://\(elsewhere.host):8123/" + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 307, body: "", headers: ["Location": target]) + } + let driver = server.unconnectedDriver() + defer { driver.disconnect() } + + try? await driver.connect() + await #expect(throws: (any Error).self) { try await driver.execute(query: "SELECT 1") } + #expect(!server.requests.isEmpty) + #expect(elsewhere.requests.isEmpty) + } +} diff --git a/apple/Plugins/CloudflareD1DriverPlugin/D1HttpClient.swift b/apple/Plugins/CloudflareD1DriverPlugin/D1HttpClient.swift index f3ef39574f..1a5edb6df6 100644 --- a/apple/Plugins/CloudflareD1DriverPlugin/D1HttpClient.swift +++ b/apple/Plugins/CloudflareD1DriverPlugin/D1HttpClient.swift @@ -354,7 +354,7 @@ final class D1HttpClient: @unchecked Sendable { throw D1HttpError(message: "Invalid response from server") } - if httpResponse.statusCode >= 400 { + if !(200 ..< 300).contains(httpResponse.statusCode) { try handleHttpError(statusCode: httpResponse.statusCode, data: data, response: httpResponse) } @@ -404,7 +404,7 @@ final class D1HttpClient: @unchecked Sendable { } } -private final class D1TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { +private final class D1TaskTracker: PluginRedirectRefusingDelegate, @unchecked Sendable { private weak var client: D1HttpClient? private let lock = NSLock() private var task: URLSessionTask? diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBAccessMode.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBAccessMode.swift index bc85c0febe..5bbdd843dd 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBAccessMode.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBAccessMode.swift @@ -2,8 +2,7 @@ // DuckDBAccessMode.swift // DuckDBDriverPlugin // -// Which access mode a connection opens its file with. No CDuckDB import, so TableProTests -// can exercise the gate without loading the plugin bundle. +// Which access mode a connection opens its file with. // import Foundation @@ -25,18 +24,20 @@ enum DuckDBAccessMode: String { extension DuckDBAccessMode { static let fieldId = "duckdbReadOnly" - /// Read-only applies to DuckDB's own storage and nothing else. Measured: `access_mode` of - /// `READ_ONLY` against `:memory:`, or against the Parquet, CSV and JSON readers that - /// `DuckDBFileKinds.readOnlyData` covers, fails the open outright with - /// `Catalog Error: Cannot launch in-memory database in read-only mode!`. Those paths hold no - /// write lock to give up in the first place, and DuckDB never writes back to them, so the - /// setting is ignored for them rather than turned into a connection that will not open. + /// Read-only applies to a database file, whatever it is called. Measured: `access_mode` of + /// `READ_ONLY` against an empty path, `:memory:`, or a Parquet, CSV or JSON reader fails the + /// open outright with `Catalog Error: Cannot launch in-memory database in read-only mode!`. + /// Those paths hold no write lock to give up in the first place, and DuckDB never writes back + /// to them, so the setting is ignored for them rather than turned into a connection that will + /// not open. static func resolve(fields: [String: String], path: String) -> DuckDBAccessMode { guard fields[fieldId] == "true", carriesAccessMode(path: path) else { return .readWrite } return .readOnly } static func carriesAccessMode(path: String) -> Bool { - DuckDBFileKinds.database.contains((path as NSString).pathExtension.lowercased()) + // `:memory:x` is a file named so. + guard !path.isEmpty, path != ":memory:" else { return false } + return DuckDBFileKinds.holdsDatabase(atPath: path) || !DuckDBFileKinds.opensAsReader(path) } } diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBFileKinds.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBFileKinds.swift index 2dd7d0cfed..bd6f3cdf5a 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBFileKinds.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBFileKinds.swift @@ -27,6 +27,34 @@ enum DuckDBFileKinds { /// or `.csv` path is a typo, and DuckDB would answer it by creating an empty database /// under that name, which reads as a working connection to nothing. static func canBeCreated(atPath path: String) -> Bool { - !readOnlyData.contains((path as NSString).pathExtension.lowercased()) + !opensAsReader(path) + } + + // Measured on the shipped libduckdb (scripts/probes/check-duckdb-access-mode.sh): DuckDB picks a reader by name, + // in any case and with or without gzip or zstd around it, and opens every other path as a database. + static func opensAsReader(_ path: String) -> Bool { + var name = (path as NSString).lastPathComponent.lowercased() as NSString + if compressionExtensions.contains(name.pathExtension) { + name = name.deletingPathExtension as NSString + } + return readerExtensions.contains(name.pathExtension) + } + + private static let readerExtensions = Set(readOnlyData + ["jsonl"]) + + private static let compressionExtensions: Set = ["gz", "zst"] + + private static let duckDBMagic = Data("DUCK".utf8) + private static let duckDBMagicOffset = 8 + private static let sqliteMagic = Data("SQLite format 3\u{0}".utf8) + + // DuckDB opens its own storage and a SQLite file by their header, whatever they are called. + static func holdsDatabase(atPath path: String) -> Bool { + guard let handle = FileHandle(forReadingAtPath: path) else { return false } + defer { try? handle.close() } + guard let header = try? handle.read(upToCount: sqliteMagic.count) else { return false } + if header.starts(with: sqliteMagic) { return true } + let magicRange = duckDBMagicOffset..<(duckDBMagicOffset + duckDBMagic.count) + return header.count >= magicRange.upperBound && header.subdata(in: magicRange) == duckDBMagic } } diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchConnection.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchConnection.swift index 6802e13f9e..347a2ba7fa 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchConnection.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchConnection.swift @@ -459,7 +459,7 @@ internal final class ElasticsearchConnection: NSObject, @unchecked Sendable { } } -extension ElasticsearchConnection: URLSessionDelegate { +extension ElasticsearchConnection: URLSessionTaskDelegate { func urlSession( _ session: URLSession, didReceive challenge: URLAuthenticationChallenge, @@ -474,4 +474,15 @@ extension ElasticsearchConnection: URLSessionDelegate { } completionHandler(.useCredential, URLCredential(trust: trust)) } + + // A redirect would carry the credentials to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } } diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift b/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift index e02ff6437d..ecf8bf1dc5 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift @@ -373,21 +373,17 @@ internal final class EtcdHttpClient: @unchecked Sendable { ) urlConfig.timeoutIntervalForResource = HttpQueryTimeout.sessionResourceTimeout - let delegate: URLSessionDelegate? + let delegate: PluginRedirectRefusingDelegate switch tlsMode { case "Required": // Encryption without certificate verification — matches UI "Required (skip verify)" delegate = InsecureTlsDelegate() default: - delegate = try verifyingDelegate(tlsMode: tlsMode) + delegate = try verifyingDelegate(tlsMode: tlsMode) ?? PluginRedirectRefusingDelegate() } lock.withLock { - if let delegate { - session = URLSession(configuration: urlConfig, delegate: delegate, delegateQueue: nil) - } else { - session = URLSession(configuration: urlConfig) - } + session = URLSession(configuration: urlConfig, delegate: delegate, delegateQueue: nil) } do { @@ -414,7 +410,7 @@ internal final class EtcdHttpClient: @unchecked Sendable { !config.username.isEmpty } - private func verifyingDelegate(tlsMode: String) throws -> URLSessionDelegate? { + private func verifyingDelegate(tlsMode: String) throws -> PluginRedirectRefusingDelegate? { let caPath = config.additionalFields["etcdCaCertPath"] guard let serverTrust = try EtcdServerTrust.make(tlsMode: tlsMode, caCertificatePath: caPath) else { return nil @@ -821,7 +817,8 @@ internal final class EtcdHttpClient: @unchecked Sendable { guard let httpResponse = response as? HTTPURLResponse else { throw EtcdError.serverError(String(localized: "Invalid response type")) } - guard httpResponse.statusCode >= 400 else { return data } + // A refused redirect arrives as its 3xx. + guard !(200 ..< 300).contains(httpResponse.statusCode) else { return data } let fault = EtcdServerFault.decode(httpStatus: httpResponse.statusCode, body: data) let recovery = EtcdRequestRecovery.action( @@ -1082,7 +1079,7 @@ internal final class EtcdHttpClient: @unchecked Sendable { // MARK: - TLS Delegates - private final class InsecureTlsDelegate: NSObject, URLSessionDelegate { + private final class InsecureTlsDelegate: PluginRedirectRefusingDelegate { func urlSession( _ session: URLSession, didReceive challenge: URLAuthenticationChallenge, @@ -1097,7 +1094,7 @@ internal final class EtcdHttpClient: @unchecked Sendable { } } - private final class EtcdTlsDelegate: NSObject, URLSessionDelegate { + private final class EtcdTlsDelegate: PluginRedirectRefusingDelegate { private let serverTrust: EtcdServerTrust private let clientCredential: URLCredential? diff --git a/apple/Plugins/LibSQLDriverPlugin/HranaHttpClient.swift b/apple/Plugins/LibSQLDriverPlugin/HranaHttpClient.swift index 0354ba0970..1d49a8696f 100644 --- a/apple/Plugins/LibSQLDriverPlugin/HranaHttpClient.swift +++ b/apple/Plugins/LibSQLDriverPlugin/HranaHttpClient.swift @@ -298,7 +298,7 @@ final class HranaHttpClient: @unchecked Sendable { throw HranaHttpError(message: "Invalid response from server") } - if httpResponse.statusCode >= 400 { + if !(200 ..< 300).contains(httpResponse.statusCode) { try handleHttpError(statusCode: httpResponse.statusCode, data: data, response: httpResponse) } @@ -357,7 +357,7 @@ final class HranaHttpClient: @unchecked Sendable { } } -private final class HranaTaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { +private final class HranaTaskTracker: PluginRedirectRefusingDelegate, @unchecked Sendable { private weak var client: HranaHttpClient? private let lock = NSLock() private var task: URLSessionTask? diff --git a/apple/Plugins/Shared/PluginRedirectRefusingDelegate.swift b/apple/Plugins/Shared/PluginRedirectRefusingDelegate.swift new file mode 100644 index 0000000000..21beb34d79 --- /dev/null +++ b/apple/Plugins/Shared/PluginRedirectRefusingDelegate.swift @@ -0,0 +1,15 @@ +import Foundation + +// URLSession repeats a driver's credential headers to wherever a redirect points. No engine a driver speaks to +// redirects, so the redirect comes back as the response, as in DynamoDB, BigQuery, Spanner and Trino. +nonisolated class PluginRedirectRefusingDelegate: NSObject, URLSessionTaskDelegate, @unchecked Sendable { + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } +} diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift index b3b1838a9e..925e4c7fd6 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift @@ -82,11 +82,15 @@ final class SnowflakeConnection: @unchecked Sendable { let configuration = URLSessionConfiguration.ephemeral configuration.timeoutIntervalForRequest = Self.requestTimeout configuration.timeoutIntervalForResource = 600 - self.session = URLSession(configuration: configuration) + self.session = URLSession( + configuration: configuration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil + ) let connectConfiguration = URLSessionConfiguration.ephemeral connectConfiguration.timeoutIntervalForRequest = TimeInterval(PluginConnectTimeout.maximumMilliseconds) / 1_000 connectConfiguration.timeoutIntervalForResource = TimeInterval(PluginConnectTimeout.maximumMilliseconds) / 1_000 - self.connectSession = URLSession(configuration: connectConfiguration) + self.connectSession = URLSession( + configuration: connectConfiguration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil + ) self._currentDatabase = params.database.isEmpty ? nil : params.database self._currentSchema = params.schema.isEmpty ? nil : params.schema diff --git a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift index 67b75b893c..cd645015f0 100644 --- a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift +++ b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift @@ -37,7 +37,7 @@ internal final class SpannerPluginDriver: PluginDatabaseDriver, @unchecked Senda } var capabilities: PluginCapabilities { - [.multiSchema, .cancelQuery, .transactions, .truncateTable, .cellExpressions, .uuidCellText] + [.multiSchema, .cancelQuery, .transactions, .truncateTable, .cellExpressions, .uuidCellText, .rowWrites] } var serverVersion: String? { diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealRPCClient.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealRPCClient.swift index 5635b7b81b..57ac7e3fa3 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealRPCClient.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealRPCClient.swift @@ -63,10 +63,9 @@ public final class SurrealRPCClient: NSObject, @unchecked Sendable { let configuration = URLSessionConfiguration.ephemeral configuration.httpAdditionalHeaders = [:] configuration.timeoutIntervalForRequest = deadline.remainingSeconds() - let delegate = config.skipTLSVerify ? self : nil lock.withLock { connectDeadline = deadline - session = URLSession(configuration: configuration, delegate: delegate, delegateQueue: nil) + session = URLSession(configuration: configuration, delegate: self, delegateQueue: nil) } } @@ -331,7 +330,7 @@ public final class SurrealRPCClient: NSObject, @unchecked Sendable { } } -extension SurrealRPCClient: URLSessionDelegate { +extension SurrealRPCClient: URLSessionTaskDelegate { public func urlSession( _ session: URLSession, didReceive challenge: URLAuthenticationChallenge, @@ -345,4 +344,15 @@ extension SurrealRPCClient: URLSessionDelegate { } completionHandler(.useCredential, URLCredential(trust: trust)) } + + // A redirect would carry the credentials to wherever it points. + public func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift index ecb0231e7e..ef3166e4c1 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift @@ -13,6 +13,9 @@ final class TeradataAsyncConnection: @unchecked Sendable { queue.sync { connection.isConnected } } + // Read outside the queue, so asking never waits behind a running statement. + var hasLostConnection: Bool { connection.hasLostConnection } + func connect() async throws { try await run { try $0.connect() } } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift index b417b59570..427fbb96cd 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift @@ -124,11 +124,12 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { static let logger = Logger(subsystem: "com.TablePro", category: "TeradataPluginDriver") var serverVersion: String? { _serverVersion } + var hasLostConnection: Bool { connection?.hasLostConnection ?? false } var supportsSchemas: Bool { false } var supportsTransactions: Bool { true } var capabilities: PluginCapabilities { - [.cancelQuery, .transactions, .alterTableDDL, .dataCompare, .cellExpressions, .uuidCellText] + [.cancelQuery, .transactions, .alterTableDDL, .dataCompare, .cellExpressions, .uuidCellText, .rowWrites] } init(config: DriverConnectionConfig) { diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift index 9700105918..e9c842a0f7 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift @@ -62,6 +62,13 @@ extension TeradataPluginDriver { return statements.isEmpty ? nil : statements } + func generateIdentityPreservingInsert( + table: String, schema: String?, columns: [String], primaryKeyColumns: [String], rows: [[PluginCellValue]] + ) -> [(statement: String, parameters: [PluginCellValue])]? { + let target = TeradataSchemaQueries.qualifiedName(database: effectiveDatabaseForSchema(schema), table: table) + return rows.map { (TeradataRowEditSQL.insert(target: target, columns: columns, values: $0), []) } + } + private func quote(_ name: String) -> String { TeradataSchemaQueries.quoteIdentifier(name) } diff --git a/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift b/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift index 24e1cc8e7e..3ac366d3a6 100644 --- a/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift +++ b/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift @@ -342,7 +342,7 @@ internal final class TypesenseConnection: NSObject, @unchecked Sendable { } } -extension TypesenseConnection: URLSessionDelegate { +extension TypesenseConnection: URLSessionTaskDelegate { func urlSession( _ session: URLSession, didReceive challenge: URLAuthenticationChallenge, @@ -357,4 +357,15 @@ extension TypesenseConnection: URLSessionDelegate { } completionHandler(.useCredential, URLCredential(trust: trust)) } + + // A redirect would carry the API key to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Verification.swift b/apple/TablePro/Core/Database/DatabaseManager+Verification.swift index 5a5d04fe93..4c199d5042 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Verification.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Verification.swift @@ -64,7 +64,8 @@ extension DatabaseManager { /// and stamps the connection fresh each time. internal func verifyBeforeUse(_ connectionId: UUID) async { guard let session = activeSessions[connectionId], let driver = session.driver else { return } - guard supportsHealthChecks(connectionId) else { return } + // A driver that reports its connection gone answered unasked, so an opted-out engine is reconnected too. + guard supportsHealthChecks(connectionId) || driver.hasLostConnection else { return } /// A session already known to be in trouble has a reconnect of its own, either the /// monitor's or the one `ensureConnected` is about to run. Checking it again would only /// race them. diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index 14cc6a3c49..26782cdda8 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -199,6 +199,15 @@ enum QueryClassifier { if SQLProceduralBlock.runs(statement, grammar: grammar) { return proceduralBlockDeletesEverything(statement, grammar: grammar) } + // With ANALYZE the engine runs what it explains, so the explained statement is read as the tier reads it. + let projection = StatementProjection(statement: statement, grammar: grammar) + let keyword = SQLLeadingComments.leadingCodeKeyword(projection.body) + if explainPrefixes.contains(keyword) { + guard let inner = explainedInnerStatement(projection, keyword: keyword), inner.executesStatement else { + return false + } + return statementDeletesEverything(inner.statement, grammar: grammar, databaseType: databaseType) + } return SQLUnfilteredWrite.writes(in: statement, grammar: grammar).contains(.delete) } @@ -240,7 +249,22 @@ private extension QueryClassifier { static let executingExplainOptions: [String] = ["ANALYZE", "ANALYSE"] static func explainExecutesStatement(_ options: String) -> Bool { - executingExplainOptions.contains { options.contains($0) } + executingExplainOptions.contains { option in + options.components(separatedBy: option).dropFirst().contains { !turnsOptionOff($0) } + } + } + + static let optionOffValues = ["FALSE", "OFF", "0"] + + // PostgreSQL turns an option off with FALSE, OFF or 0 after it, so `EXPLAIN (ANALYZE FALSE)` only plans. + static func turnsOptionOff(_ rest: String) -> Bool { + let value = StatementBlank.trimmingLeading(rest[...]) + guard value.startIndex != rest.startIndex else { return false } + return optionOffValues.contains { off in + guard value.hasPrefix(off) else { return false } + guard let next = value.dropFirst(off.count).first else { return true } + return !(next.isLetter || next.isNumber || next == "_" || next == "$") + } } static let whereClauseRegex = try? NSRegularExpression(pattern: "\\sWHERE\\s", options: []) @@ -272,7 +296,7 @@ private extension QueryClassifier { static let filesystemOrCodeKeywords: Set = [ "COPY", "ATTACH", "DETACH", "DO", "LOAD", "INSTALL", "IMPORT", "EXPORT", - "BACKUP", "RESTORE", "DUMP", "SOURCE", "UNLOAD" + "BACKUP", "RESTORE", "DUMP", "SOURCE", "UNLOAD", "BULK" ] static let writeKeywords: Set = [ @@ -301,7 +325,8 @@ private extension QueryClassifier { "INTO OUTFILE", "INTO DUMPFILE", "TO PROGRAM", "FROM PROGRAM", "VACUUM INTO", "LOAD_FILE(", "PG_READ_FILE(", "PG_READ_BINARY_FILE(", "PG_LS_DIR(", "LO_IMPORT(", "LO_EXPORT(", "PG_FILE_WRITE(", "LOAD_EXTENSION(", - "READFILE(", "WRITEFILE(", "FN_GET_AUDIT_FILE(", "XP_CMDSHELL", "SP_OACREATE" + "READFILE(", "WRITEFILE(", "FN_GET_AUDIT_FILE(", "XP_CMDSHELL", "SP_OACREATE", + "OPENROWSET(", "OPENDATASOURCE(" ] // A comment, a line break or a run of spaces between two words, or a space before `(`, is nothing to the server: @@ -392,6 +417,10 @@ private extension QueryClassifier { return QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) } + if keyword == "CREATE" || keyword == "REPLACE", replacesStoredData(projection.code) { + return QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) + } + if keyword == "ALTER" { // `ALTER TABLE t TRUNCATE PARTITION ALL` empties the table, measured on MySQL 8.4. let dropsData = body.range(of: " DROP ", options: .literal) != nil @@ -446,6 +475,44 @@ private extension QueryClassifier { private static let routineDefinitionLookahead = 3 + private static let replacedDataKinds: Set = ["TABLE", "SCHEMA", "DATABASE"] + + private static let replacedObjectKinds: Set = [ + "VIEW", "FUNCTION", "PROCEDURE", "PROC", "TRIGGER", "PACKAGE", "TYPE", "SEQUENCE", "MACRO", "INDEX", "SYNONYM", + ] + + // Modifiers such as `GLOBAL TEMPORARY` or DuckDB's `UNLOGGED` stand before the kind, two words at most. + private static let replacedKindLookahead = 3 + + /// `CREATE OR REPLACE TABLE t` drops `t` and its rows before it makes the new one on Snowflake, BigQuery, DuckDB, + /// MariaDB, ClickHouse and Databricks, and a replaced schema or database takes every table in it. ClickHouse's + /// `REPLACE TABLE` is the same statement. A replaced view or routine loses no rows. + static func replacesStoredData(_ code: String) -> Bool { + let code = code as NSString + var cursor = skipBlanks(in: code, from: 0) + func nextWord() -> String? { + let end = endOfWord(in: code, from: cursor) + guard end > cursor else { return nil } + let word = code.substring(with: NSRange(location: cursor, length: end - cursor)).uppercased() + cursor = skipBlanks(in: code, from: end) + return word + } + switch nextWord() { + case "CREATE": + guard nextWord() == "OR", nextWord() == "REPLACE" else { return false } + case "REPLACE": + break + default: + return false + } + for _ in 0 ..< replacedKindLookahead { + guard let word = nextWord() else { return false } + if replacedDataKinds.contains(word) { return true } + if replacedObjectKinds.contains(word) { return false } + } + return false + } + static func commonTableExpressionClassification(_ body: String) -> QueryClassification { for keyword in ["DROP", "TRUNCATE"] where containsWord(body, keyword) { return QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) diff --git a/apple/TableProMobile/project.yml b/apple/TableProMobile/project.yml index 9558606aa5..176f350bbe 100644 --- a/apple/TableProMobile/project.yml +++ b/apple/TableProMobile/project.yml @@ -72,6 +72,7 @@ targets: excludes: - Info.plist - ../Plugins/Shared/PluginConnectTimeout.swift + - ../Plugins/Shared/PluginRedirectRefusingDelegate.swift # The MongoDB plugin driver runs behind PluginBackedDriver too; the iOS CLibMongoc bridge stands in for the # plugin's own, and the principal class stays on the Mac. - path: ../Plugins/MongoDBDriverPlugin diff --git a/apple/TableProTests/Core/Database/ConnectionVerificationTests.swift b/apple/TableProTests/Core/Database/ConnectionVerificationTests.swift index b22550e6e9..b8a2cf01a7 100644 --- a/apple/TableProTests/Core/Database/ConnectionVerificationTests.swift +++ b/apple/TableProTests/Core/Database/ConnectionVerificationTests.swift @@ -184,6 +184,19 @@ struct ConnectionVerificationTests { await cleanUp(connection.id) } + @Test("an engine that opts out of health checks is still checked once its driver reports the connection lost") + func optedOutEngineWithALostDriverIsChecked() async { + let driver = MockDatabaseDriver() + driver.hasLostConnection = true + let connection = makeSession(driver: driver, typeId: DatabaseType.sqlite.rawValue) + DatabaseManager.shared.markSessionVerified(connection.id) + + await DatabaseManager.shared.verifyBeforeUse(connection.id) + + #expect(driver.pingCallCount == 1) + await cleanUp(connection.id) + } + @Test("a check that fails reconnects the connection") func aFailedCheckReconnects() async { let driver = MockDatabaseDriver() diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierCreateOrReplaceTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierCreateOrReplaceTests.swift new file mode 100644 index 0000000000..18a8d60320 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierCreateOrReplaceTests.swift @@ -0,0 +1,56 @@ +import Foundation +@testable import TablePro +import Testing + +struct QueryClassifierCreateOrReplaceTests { + @Test("Replacing a table, schema or database is destructive", arguments: [ + (DatabaseType.snowflake, "CREATE OR REPLACE TABLE orders (id INT)"), + (.snowflake, "create or replace table orders as select * from orders where 1 = 0"), + (.snowflake, "CREATE OR REPLACE TRANSIENT TABLE orders (id INT)"), + (.snowflake, "CREATE OR REPLACE SCHEMA sales"), + (.snowflake, "CREATE OR REPLACE DATABASE prod"), + (.bigQuery, "CREATE OR REPLACE TABLE shop.orders (id INT64)"), + (.duckdb, "CREATE OR REPLACE TEMP TABLE orders (id INT)"), + (.duckdb, "CREATE OR REPLACE UNLOGGED TABLE orders (id INT)"), + (.duckdb, "CREATE OR REPLACE TABLE function (id INT)"), + (.snowflake, "CREATE OR REPLACE GLOBAL TEMPORARY TABLE orders (id INT)"), + (.mariadb, "CREATE OR REPLACE TABLE orders (id INT)"), + (.mariadb, "CREATE /* swap */ OR REPLACE\nTABLE orders (id INT)"), + (.mariadb, "CREATE OR REPLACE DATABASE shop"), + (.clickhouse, "CREATE OR REPLACE TABLE orders (id Int8) ENGINE = Memory"), + (.clickhouse, "REPLACE TABLE orders (id Int8) ENGINE = Memory"), + ]) + func replacingStoredDataIsDestructive(databaseType: DatabaseType, sql: String) { + #expect(QueryClassifier.classifyTier(sql, databaseType: databaseType) == .destructive) + #expect(QueryClassifier.isDangerousQuery(sql, databaseType: databaseType)) + } + + @Test("Replacing a view or a routine, or creating a table, stays a write", arguments: [ + (DatabaseType.postgresql, "CREATE OR REPLACE VIEW v AS SELECT 1"), + (.postgresql, "CREATE OR REPLACE FUNCTION f() RETURNS int AS $$ SELECT 1 $$ LANGUAGE sql"), + (.snowflake, "CREATE OR REPLACE VIEW \"TABLE\" AS SELECT 1"), + (.snowflake, "CREATE TABLE orders (id INT)"), + (.snowflake, "CREATE OR REPLACE STREAM s ON TABLE orders"), + (.oracle, "CREATE OR REPLACE TYPE ids AS TABLE OF NUMBER"), + (.duckdb, "CREATE OR REPLACE MACRO m(x) AS TABLE SELECT x"), + (.mysql, "REPLACE INTO orders (id) VALUES (1)"), + ]) + func losingNoRowsStaysAWrite(databaseType: DatabaseType, sql: String) { + #expect(QueryClassifier.classifyTier(sql, databaseType: databaseType) == .write) + } + + @Test("A caller that may not send DROP TABLE may not send CREATE OR REPLACE TABLE") + func gateRefusesItWithoutTheDestructiveConfirmation() { + let statement = ExternalStatementGate.Statement( + sql: "CREATE OR REPLACE TABLE orders (id INT)", + connectionId: UUID(), + databaseType: .snowflake, + externalAccess: .readWrite, + loadsExtensions: false, + allowsDestructive: false + ) + #expect(throws: ExternalStatementGateError.self) { + try ExternalStatementGate.classify(statement) + } + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index aeb52f8d45..15c1c9ec33 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -120,6 +120,35 @@ struct QueryClassifierFailClosedTests { #expect(QueryClassifier.isWriteQuery(statement, databaseType: .mysql)) } + @Test("T-SQL's BULK loads a file the server can reach", arguments: [ + "BULK INSERT dbo.staging FROM '\\\\198.51.100.7\\share\\rows.txt'", + "SELECT 1 BULK INSERT dbo.staging FROM 'C:\\rows.txt'", + "SELECT BulkColumn FROM OPENROWSET(BULK 'C:\\inetpub\\wwwroot\\web.config', SINGLE_CLOB) AS f", + "INSERT INTO dbo.staging SELECT BulkColumn FROM OPENROWSET ( BULK 'C:\\web.config', SINGLE_BLOB) AS f", + ]) + func bulkLoadsAreFlagged(statement: String) { + #expect(QueryClassifier.reachesFilesystemOrExecutesCode(statement, databaseType: .mssql)) + #expect(QueryClassifier.isWriteQuery(statement, databaseType: .mssql)) + } + + @Test("T-SQL's ad hoc data sources open the file or the server the statement names", arguments: [ + "SELECT * FROM OPENROWSET('Microsoft.ACE.OLEDB.12.0', 'Text;Database=C:\\logs', 'SELECT * FROM app.txt')", + "SELECT * FROM OPENDATASOURCE('Microsoft.ACE.OLEDB.12.0', 'Data Source=C:\\a.xlsx')...[Sheet1$]", + ]) + func adHocDataSourcesAreFlagged(statement: String) { + #expect(QueryClassifier.reachesFilesystemOrExecutesCode(statement, databaseType: .mssql)) + } + + @Test("A word that only resembles T-SQL's BULK reaches no file", arguments: [ + "GRANT ADMINISTER BULK OPERATIONS TO loader", + "ALTER DATABASE app SET RECOVERY BULK_LOGGED", + "SELECT [bulk], bulk_id, 'BULK INSERT' FROM dbo.orders", + "SELECT [openrowset] FROM dbo.orders", + ]) + func bulkLookalikesAreNotFlagged(statement: String) { + #expect(!QueryClassifier.reachesFilesystemOrExecutesCode(statement, databaseType: .mssql)) + } + @Test("Ordinary reads stay safe and never claim to reach the filesystem") func readsStaySafe() { let reads = [ diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierUnfilteredDeleteTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierUnfilteredDeleteTests.swift index f36a93b4a9..e9a8bbfbae 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierUnfilteredDeleteTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierUnfilteredDeleteTests.swift @@ -7,6 +7,9 @@ struct QueryClassifierUnfilteredDeleteTests { "WITH old AS (SELECT id FROM archive WHERE id > 0) DELETE FROM orders", "DELETE FROM orders USING (SELECT id FROM archive WHERE id > 0) a", "DELETE FROM orders RETURNING (SELECT 1 WHERE true)", + "WITH merge\u{20AC} AS (SELECT 1) DELETE FROM orders", + "WITH U&\"gone\" AS (DELETE FROM orders RETURNING *) SELECT count(*) FROM gone", + "WITH RECURSIVE walk(id) AS (SELECT 1) SEARCH DEPTH FIRST BY id SET ord, gone AS (DELETE FROM orders) SELECT 1", ]) func deleteWithoutItsOwnWhereIsDangerous(sql: String) { #expect(QueryClassifier.isDangerousQuery(sql, databaseType: .postgresql)) @@ -21,6 +24,43 @@ struct QueryClassifierUnfilteredDeleteTests { #expect(!QueryClassifier.isDangerousQuery(sql, databaseType: .postgresql)) } + @Test("A DELETE whose WHERE the engine reads as a name still asks first") + func whereReadAsANameIsDangerous() { + let alias = "DELETE FROM orders AS o\u{A0}WHERE\u{A0}id\u{A0}IS\u{A0}NULL" + #expect(QueryClassifier.isDangerousQuery(alias, databaseType: .sqlite)) + #expect(QueryClassifier.isDangerousQuery("DELETE FROM orders RETURNING id AS where", databaseType: .postgresql)) + #expect(QueryClassifier.isDangerousQuery("DELETE FROM orders AS 1where", databaseType: .mysql)) + #expect(!QueryClassifier.isDangerousQuery("DELETE FROM orders AS o WHERE id IS NULL", databaseType: .sqlite)) + } + + @Test("An EXPLAIN that runs a DELETE with no WHERE asks first, as the DELETE does", arguments: [ + ("EXPLAIN ANALYZE DELETE FROM orders", DatabaseType.postgresql), + ("EXPLAIN ANALYSE DELETE FROM orders", .postgresql), + ("EXPLAIN (ANALYZE, FORMAT JSON) DELETE FROM orders", .postgresql), + ("EXPLAIN ANALYZE WITH gone AS (DELETE FROM orders RETURNING *) SELECT * FROM gone", .postgresql), + ("ANALYZE DELETE FROM orders", .mariadb), + ("ANALYZE FORMAT=JSON DELETE FROM orders", .mariadb), + ("EXPLAIN (ANALYZE TRUE) DELETE FROM orders", .postgresql), + ("EXPLAIN (ANALYZE OFF, ANALYZE) DELETE FROM orders", .postgresql), + ("EXPLAIN (ANALYZE OFFSET) DELETE FROM orders", .postgresql), + ]) + func explainThatRunsAnUnfilteredDeleteIsDangerous(sql: String, databaseType: DatabaseType) { + #expect(QueryClassifier.isDangerousQuery(sql, databaseType: databaseType)) + } + + @Test("An EXPLAIN that only plans the DELETE, or runs one with a WHERE, does not", arguments: [ + "EXPLAIN DELETE FROM orders", + "EXPLAIN (FORMAT JSON) DELETE FROM orders", + "EXPLAIN ANALYZE DELETE FROM orders WHERE id = 1", + "EXPLAIN (ANALYZE FALSE) DELETE FROM orders", + "EXPLAIN (ANALYZE OFF, FORMAT JSON) DELETE FROM orders", + "EXPLAIN (ANALYZE 0) DELETE FROM orders", + "ANALYZE orders", + ]) + func explainThatDeletesNothingUnfilteredIsNotDangerous(sql: String) { + #expect(!QueryClassifier.isDangerousQuery(sql, databaseType: .postgresql)) + } + @Test("On SQL Server a WHERE of the statement after the DELETE is not the DELETE's") func sqlServerFollowingStatementWhere() { #expect(QueryClassifier.isDangerousQuery("DELETE FROM [orders] SELECT 1 WHERE 1 = 1", databaseType: .mssql)) diff --git a/apple/changelog.d/3907.fixed.1.md b/apple/changelog.d/3907.fixed.1.md new file mode 100644 index 0000000000..5da836bb9a --- /dev/null +++ b/apple/changelog.d/3907.fixed.1.md @@ -0,0 +1 @@ +CREATE OR REPLACE TABLE, SCHEMA or DATABASE replacing data without the confirmation DROP gets. diff --git a/apple/changelog.d/3907.fixed.2.md b/apple/changelog.d/3907.fixed.2.md new file mode 100644 index 0000000000..1329b70e54 --- /dev/null +++ b/apple/changelog.d/3907.fixed.2.md @@ -0,0 +1 @@ +T-SQL BULK INSERT and OPENROWSET, and MongoDB $_internalJs operators, passing the external file and code refusals. diff --git a/apple/changelog.d/3907.fixed.3.md b/apple/changelog.d/3907.fixed.3.md new file mode 100644 index 0000000000..133900ff8a --- /dev/null +++ b/apple/changelog.d/3907.fixed.3.md @@ -0,0 +1 @@ +DuckDB's Open the File Read-Only being ignored for a database not named .duckdb or .ddb. diff --git a/apple/changelog.d/3907.fixed.4.md b/apple/changelog.d/3907.fixed.4.md new file mode 100644 index 0000000000..544e89f5b0 --- /dev/null +++ b/apple/changelog.d/3907.fixed.4.md @@ -0,0 +1 @@ +Spanner, BigQuery and Teradata saving a grid change they could not key to the default schema's table. diff --git a/apple/changelog.d/3907.fixed.5.md b/apple/changelog.d/3907.fixed.5.md new file mode 100644 index 0000000000..02abc9831d --- /dev/null +++ b/apple/changelog.d/3907.fixed.5.md @@ -0,0 +1 @@ +HTTP drivers such as Typesense following a server's redirect to another address with the connection's key. diff --git a/apple/changelog.d/3907.fixed.6.md b/apple/changelog.d/3907.fixed.6.md new file mode 100644 index 0000000000..ff30f90824 --- /dev/null +++ b/apple/changelog.d/3907.fixed.6.md @@ -0,0 +1 @@ +Bigtable saving a grid change to another row when the key starts with a combining mark. diff --git a/apple/changelog.d/3907.fixed.7.md b/apple/changelog.d/3907.fixed.7.md new file mode 100644 index 0000000000..20cbef3d28 --- /dev/null +++ b/apple/changelog.d/3907.fixed.7.md @@ -0,0 +1 @@ +Teradata showing each result one statement late after a statement hit the 20-second receive timeout. diff --git a/apple/changelog.d/3907.fixed.md b/apple/changelog.d/3907.fixed.md new file mode 100644 index 0000000000..09d461ca54 --- /dev/null +++ b/apple/changelog.d/3907.fixed.md @@ -0,0 +1 @@ +A DELETE with no WHERE running unconfirmed behind EXPLAIN ANALYZE, an unusual CTE, or a WHERE glued to a name. diff --git a/apple/project.yml b/apple/project.yml index 3e4a737bbe..dbfc3b4caa 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -91,6 +91,7 @@ targetTemplates: - Plugins/Shared/PluginConnectTimeout.swift - Plugins/Shared/PluginQueryTimeout.swift - Plugins/Shared/PluginURLSessionTaskBox.swift + - Plugins/Shared/PluginRedirectRefusingDelegate.swift dependencies: - target: TableProPluginKit embed: false diff --git a/apple/scripts/probes/check-duckdb-access-mode.sh b/apple/scripts/probes/check-duckdb-access-mode.sh new file mode 100755 index 0000000000..8b75e25807 --- /dev/null +++ b/apple/scripts/probes/check-duckdb-access-mode.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# +# Measures which paths libduckdb opens with access_mode=READ_ONLY, the facts DuckDBFileKinds +# and DuckDBAccessMode hard-code. A database file honours it whatever it is called; a reader +# (CSV, TSV, JSON, JSON Lines, Parquet, also gzip or zstd compressed), an empty path and +# :memory: refuse it. Run after bumping libduckdb. +# +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HEADERS="$ROOT/Plugins/DuckDBDriverPlugin/CDuckDB/include" +LIB="$ROOT/Libs/libduckdb.a" + +if [ ! -f "$LIB" ]; then + echo "error: $LIB is missing. Run scripts/download-libs.sh first." >&2 + exit 1 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +cat > "$WORK/probe.c" <<'PROBE' +#include +#include +#include "duckdb.h" + +static int open_db(const char *path, int read_only, duckdb_database *db, char **error) { + duckdb_config config = NULL; + if (read_only) { + duckdb_create_config(&config); + duckdb_set_config(config, "access_mode", "READ_ONLY"); + } + duckdb_state state = duckdb_open_ext(path, db, config, error); + if (config) duckdb_destroy_config(&config); + return state == DuckDBSuccess; +} + +int main(int argc, char **argv) { + if (argc > 2 && strcmp(argv[1], "create") == 0) { + duckdb_database db; + duckdb_connection con; + char *error = NULL; + if (!open_db(argv[2], 0, &db, &error)) return 2; + duckdb_connect(db, &con); + duckdb_query(con, "CREATE TABLE t AS SELECT 1 AS a", NULL); + duckdb_query(con, "COPY (SELECT 1 AS a) TO 'data.parquet' (FORMAT parquet)", NULL); + duckdb_disconnect(&con); + duckdb_close(&db); + return 0; + } + for (int i = 1; i < argc; i++) { + duckdb_database db; + char *error = NULL; + if (!open_db(argv[i], 1, &db, &error)) { + printf("%s\trefused\n", argv[i]); + if (error) duckdb_free(error); + continue; + } + duckdb_connection con; + duckdb_connect(db, &con); + duckdb_result result; + int wrote = duckdb_query(con, "CREATE TABLE written(a INTEGER)", &result) == DuckDBSuccess; + duckdb_destroy_result(&result); + printf("%s\t%s\n", argv[i], wrote ? "WRITABLE" : "read-only"); + duckdb_disconnect(&con); + duckdb_close(&db); + } + return 0; +} +PROBE + +cc -O1 -I "$HEADERS" "$WORK/probe.c" "$LIB" -lc++ -o "$WORK/probe" + +cd "$WORK" +./probe create warehouse.duckdb +cp warehouse.duckdb warehouse.db +cp warehouse.duckdb warehouse +cp warehouse.duckdb warehouse.duckdb.bak +/usr/bin/sqlite3 app.sqlite "CREATE TABLE s(a); INSERT INTO s VALUES (1);" +cp app.sqlite app.db +printf 'a,b\n1,2\n' > rows.csv +printf 'a\tb\n1\t2\n' > rows.tsv +printf '[{"a":1}]' > rows.json +printf '{"a":1}\n' > rows.jsonl +cp rows.jsonl rows.ndjson +cp rows.csv UPPER.CSV +gzip -k rows.csv +gzip -k rows.jsonl + +databases=(warehouse.duckdb warehouse.db warehouse warehouse.duckdb.bak app.sqlite app.db) +readers=(rows.csv rows.tsv rows.json rows.jsonl rows.ndjson data.parquet UPPER.CSV rows.csv.gz rows.jsonl.gz "" ":memory:") + +./probe "${databases[@]}" "${readers[@]}" > measured.tsv +cat measured.tsv + +failed=0 +for path in "${databases[@]}"; do + grep -qxF "$path read-only" measured.tsv || { echo "MISMATCH: $path should open read-only" >&2; failed=1; } +done +for path in "${readers[@]}"; do + grep -qxF "$path refused" measured.tsv || { echo "MISMATCH: $path should refuse READ_ONLY" >&2; failed=1; } +done +if [ "$failed" -ne 0 ]; then + echo "FAIL: update DuckDBFileKinds.opensAsReader and DuckDBAccessMode.carriesAccessMode." >&2 + exit 1 +fi +echo "OK: libduckdb honours READ_ONLY on every database file and refuses it on every reader."