From 8be0b896630f216a609a813e992cb95a09745d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:28:43 +0700 Subject: [PATCH 1/6] feat(plugin-memcached): add a Memcached driver over the text protocol --- .github/plugin-registry.json | 3 + apple/Packages/TableProCore/Package.swift | 13 + .../SQLDDLFallbackPolicy.swift | 1 + .../TableProCoreTypes/DatabaseType.swift | 4 +- .../MemcachedBrowseSpec.swift | 163 +++++++ .../MemcachedCommand.swift | 269 +++++++++++ .../MemcachedKeyListing.swift | 95 ++++ .../MemcachedReplyReader.swift | 150 +++++++ .../MemcachedStatements.swift | 29 ++ .../SQLLexicalProfile.swift | 1 + .../MemcachedCommandParserTests.swift | 103 +++++ .../MemcachedKeyListingTests.swift | 114 +++++ .../MemcachedReplyReaderTests.swift | 77 ++++ .../DatabaseTypeTests.swift | 10 +- .../Plugins/MemcachedDriverPlugin/Info.plist | 12 + .../MemcachedConnection.swift | 282 ++++++++++++ .../MemcachedError.swift | 88 ++++ .../MemcachedPlugin.swift | 70 +++ .../MemcachedPluginDriver.swift | 416 ++++++++++++++++++ .../MemcachedStatementGenerator.swift | 132 ++++++ ...inMetadataRegistry+MemcachedDefaults.swift | 64 +++ ...ginMetadataRegistry+RegistryDefaults.swift | 2 +- .../MemcachedDashboardProvider.swift | 85 ++++ .../ServerDashboardQueryProviderFactory.swift | 5 + .../Execution/ExternalStatementGate.swift | 1 + .../SQL/QueryClassifier+Memcached.swift | 27 ++ .../Core/Utilities/SQL/QueryClassifier.swift | 2 + .../Models/Connection/DatabaseType.swift | 1 + ...rnalStatementGatePrivateRequestTests.swift | 2 + ...PluginMetadataRegistryTypeCountTests.swift | 2 +- .../MemcachedDashboardProviderTests.swift | 51 +++ ...erDashboardQueryProviderFactoryTests.swift | 2 +- .../Query/AllTablesListingTests.swift | 2 +- .../SQL/QueryClassifierHardeningTests.swift | 16 + .../Models/Query/QueryTabBaseQueryTests.swift | 2 +- .../Main/QueryParameterDetectionTests.swift | 2 +- apple/project.yml | 16 + apple/scripts/probes/check-memcached.sh | 56 +++ apple/scripts/probes/memcached-probe.swift | 157 +++++++ 39 files changed, 2519 insertions(+), 8 deletions(-) create mode 100644 apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedReplyReaderTests.swift create mode 100644 apple/Plugins/MemcachedDriverPlugin/Info.plist create mode 100644 apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift create mode 100644 apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift create mode 100644 apple/Plugins/MemcachedDriverPlugin/MemcachedPlugin.swift create mode 100644 apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift create mode 100644 apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift create mode 100644 apple/TablePro/Core/Plugins/PluginMetadataRegistry+MemcachedDefaults.swift create mode 100644 apple/TablePro/Core/ServerDashboard/Providers/MemcachedDashboardProvider.swift create mode 100644 apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift create mode 100644 apple/TableProTests/Core/ServerDashboard/MemcachedDashboardProviderTests.swift create mode 100755 apple/scripts/probes/check-memcached.sh create mode 100644 apple/scripts/probes/memcached-probe.swift diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index f6bf29a547..b2fbb77120 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -49,6 +49,9 @@ "LibSQLDriverPlugin": { "name": "libSQL / Turso Driver" }, + "MemcachedDriverPlugin": { + "name": "Memcached Driver" + }, "MSSQLDriver": { "name": "MSSQL Driver" }, diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 94fe92c270..958f65b149 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -56,6 +56,7 @@ let package = Package( .library(name: "TableProMySQLCore", targets: ["TableProMySQLCore"]), .library(name: "TableProPostgreSQLCore", targets: ["TableProPostgreSQLCore"]), .library(name: "TableProRedisCore", targets: ["TableProRedisCore"]), + .library(name: "TableProMemcachedCore", targets: ["TableProMemcachedCore"]), .library(name: "TableProTestSupport", targets: ["TableProTestSupport"]) ], targets: [ @@ -269,6 +270,12 @@ let package = Package( path: "Sources/TableProPostgreSQLCore", swiftSettings: pluginCoreSettings ), + .target( + name: "TableProMemcachedCore", + dependencies: [], + path: "Sources/TableProMemcachedCore", + swiftSettings: pluginCoreSettings + ), .target( name: "TableProRedisCore", dependencies: [], @@ -440,6 +447,12 @@ let package = Package( path: "Tests/TableProPostgreSQLCoreTests", swiftSettings: pluginCoreSettings ), + .testTarget( + name: "TableProMemcachedCoreTests", + dependencies: ["TableProMemcachedCore"], + path: "Tests/TableProMemcachedCoreTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProRedisCoreTests", dependencies: ["TableProRedisCore"], diff --git a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift index d6962fdd63..715ba2b64e 100644 --- a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift +++ b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift @@ -36,6 +36,7 @@ public enum SQLDDLFallbackPolicy { "Elasticsearch", "Typesense", "Weaviate", + "Memcached", "Kafka", "etcd", "Redis", diff --git a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift index bb002a60ef..4eb015773a 100644 --- a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift +++ b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift @@ -42,6 +42,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let kafka = DatabaseType(rawValue: "Kafka") public static let cloudflareR2SQL = DatabaseType(rawValue: "Cloudflare R2 SQL") public static let weaviate = DatabaseType(rawValue: "Weaviate") + public static let memcached = DatabaseType(rawValue: "Memcached") public static let firestore = DatabaseType(rawValue: "Firestore") public static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") public static let bigtable = DatabaseType(rawValue: "Bigtable") @@ -50,7 +51,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { .mysql, .mariadb, .tidb, .databend, .oceanbase, .postgresql, .sqlite, .redis, .mongodb, .clickhouse, .mssql, .oracle, .dameng, .duckdb, .cassandra, .redshift, .etcd, .cloudflareD1, .dynamodb, .bigquery, .spanner, .snowflake, .libsql, .beancount, - .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .firestore, .firebaseRealtimeDatabase, .bigtable + .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .memcached, .firestore, .firebaseRealtimeDatabase, .bigtable ] /// Icon name for this database type — asset catalog name (e.g. "mysql-icon") or SF Symbol fallback @@ -87,6 +88,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .kafka: return "kafka-icon" case .cloudflareR2SQL: return "cloudflare-r2-sql-icon" case .weaviate: return "weaviate-icon" + case .memcached: return "memorychip" case .firestore, .firebaseRealtimeDatabase: return "firebase-icon" case .bigtable: return "bigtable-icon" default: return "externaldrive" diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift new file mode 100644 index 0000000000..5408d8d0be --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift @@ -0,0 +1,163 @@ +import Foundation + +public enum MemcachedColumn: String, CaseIterable, Sendable { + case key = "Key" + case value = "Value" + case ttl = "TTL" + case flags = "Flags" + + public var typeName: String { + switch self { + case .key, .value: return "String" + case .ttl, .flags: return "Int64" + } + } +} + +public enum MemcachedKeyComparison: String, Codable, CaseIterable, Sendable { + case equal = "=" + case notEqual = "!=" + case contains = "CONTAINS" + case notContains = "NOT CONTAINS" + case startsWith = "STARTS WITH" + case endsWith = "ENDS WITH" + case inList = "IN" + case notInList = "NOT IN" + case regex = "REGEX" +} + +public enum MemcachedFilterRefusal: Error, Equatable, Sendable { + case notKeyColumn(String) + case unsupportedOperator(String) + case emptyList(String) + case invalidPattern(String) +} + +/// A page of the key grid. The grid hands the driver this spec as a tagged statement, so a page, its sort and its +/// filter reach the driver the way every other statement does. +public struct MemcachedBrowseSpec: Codable, Equatable, Sendable { + public struct KeyCondition: Codable, Equatable, Sendable { + public let comparison: MemcachedKeyComparison + public let value: String + public let isCaseSensitive: Bool + + public init(comparison: MemcachedKeyComparison, value: String, isCaseSensitive: Bool) { + self.comparison = comparison + self.value = value + self.isCaseSensitive = isCaseSensitive + } + } + + public static let tag = "MEMCACHED_BROWSE:" + public static let refusalTag = "MEMCACHED_REFUSAL:" + + public let limit: Int + public let offset: Int + public let ascending: Bool + public let conditions: [KeyCondition] + public let matchesAll: Bool + + public init(limit: Int, offset: Int, ascending: Bool, conditions: [KeyCondition] = [], matchesAll: Bool = true) { + self.limit = limit + self.offset = offset + self.ascending = ascending + self.conditions = conditions + self.matchesAll = matchesAll + } + + public var statement: String { + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + return Self.tag + ((try? encoder.encode(self)) ?? Data()).base64EncodedString() + } + + public static func parse(_ statement: String) -> MemcachedBrowseSpec? { + guard statement.hasPrefix(tag), let data = Data(base64Encoded: String(statement.dropFirst(tag.count))) else { + return nil + } + return try? JSONDecoder().decode(MemcachedBrowseSpec.self, from: data) + } + + public static func refusalStatement(_ message: String) -> String { + refusalTag + Data(message.utf8).base64EncodedString() + } + + public static func refusalMessage(_ statement: String) -> String? { + guard statement.hasPrefix(refusalTag), + let data = Data(base64Encoded: String(statement.dropFirst(refusalTag.count))) else { return nil } + return String(bytes: data, encoding: .utf8) + } + + public static func condition( + column: String, op: String, value: String, isCaseSensitive: Bool + ) throws(MemcachedFilterRefusal) -> KeyCondition { + guard column == MemcachedColumn.key.rawValue else { throw .notKeyColumn(column) } + let name = op.trimmingCharacters(in: .whitespaces).uppercased() + guard let comparison = MemcachedKeyComparison(rawValue: name) else { throw .unsupportedOperator(op) } + let condition = KeyCondition(comparison: comparison, value: value, isCaseSensitive: isCaseSensitive) + _ = try KeyTest(condition) + return condition + } + + /// The keys this page shows, out of every key listed: filtered, sorted by key, then cut to the page. + public func page(of keys: [MemcachedKeyEntry]) throws(MemcachedFilterRefusal) -> [MemcachedKeyEntry] { + let matching = try filter(keys) + let sorted = matching.sorted { ascending ? $0.key < $1.key : $0.key > $1.key } + guard offset < sorted.count else { return [] } + return Array(sorted[offset ..< min(sorted.count, offset + limit)]) + } + + public func filter(_ keys: [MemcachedKeyEntry]) throws(MemcachedFilterRefusal) -> [MemcachedKeyEntry] { + guard !conditions.isEmpty else { return keys } + var tests: [KeyTest] = [] + for condition in conditions { tests.append(try KeyTest(condition)) } + return keys.filter { entry in + matchesAll ? tests.allSatisfy { $0.matches(entry.key) } : tests.contains { $0.matches(entry.key) } + } + } +} + +private struct KeyTest { + let condition: MemcachedBrowseSpec.KeyCondition + let items: [String] + let regex: NSRegularExpression? + + init(_ condition: MemcachedBrowseSpec.KeyCondition) throws(MemcachedFilterRefusal) { + self.condition = condition + switch condition.comparison { + case .inList, .notInList: + items = condition.value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty } + guard !items.isEmpty else { throw .emptyList(condition.comparison.rawValue) } + regex = nil + case .regex: + items = [] + let options: NSRegularExpression.Options = condition.isCaseSensitive ? [] : [.caseInsensitive] + guard let compiled = try? NSRegularExpression(pattern: condition.value, options: options) else { + throw .invalidPattern(condition.value) + } + regex = compiled + default: + items = [] + regex = nil + } + } + + func matches(_ key: String) -> Bool { + let options: String.CompareOptions = condition.isCaseSensitive ? [.literal] : [.caseInsensitive] + let value = condition.value + switch condition.comparison { + case .equal: return key.compare(value, options: options) == .orderedSame + case .notEqual: return key.compare(value, options: options) != .orderedSame + case .contains: return key.range(of: value, options: options) != nil + case .notContains: return key.range(of: value, options: options) == nil + case .startsWith: return key.range(of: value, options: options.union(.anchored)) != nil + case .endsWith: return key.range(of: value, options: options.union([.anchored, .backwards])) != nil + case .inList: return items.contains { key.compare($0, options: options) == .orderedSame } + case .notInList: return !items.contains { key.compare($0, options: options) == .orderedSame } + case .regex: + let range = NSRange(key.startIndex..., in: key) + return regex?.firstMatch(in: key, range: range) != nil + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift new file mode 100644 index 0000000000..c670df9240 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift @@ -0,0 +1,269 @@ +import Foundation + +public enum MemcachedStorageVerb: String, CaseIterable, Sendable { + case set, add, replace, append, prepend +} + +/// One statement of the console, in the order the text protocol takes its arguments. A value is the last argument +/// of the line instead of a data block, so a statement stays one line the review sheet can show. +public enum MemcachedCommand: Equatable, Sendable { + case get(keys: [String], withCAS: Bool) + case getAndTouch(exptime: Int64, keys: [String], withCAS: Bool) + case store(MemcachedStorageVerb, key: String, flags: UInt32, exptime: Int64, value: Data) + case compareAndSwap(key: String, flags: UInt32, exptime: Int64, unique: UInt64, value: Data) + case touch(key: String, exptime: Int64) + case delete(key: String) + case increment(key: String, delta: UInt64) + case decrement(key: String, delta: UInt64) + case flushAll(delay: Int64?) + case stats(arguments: [String]) + case version + case metadump(selector: String) + + public var replyShape: MemcachedReplyShape { + switch self { + case .get, .getAndTouch: return .values + case .stats, .metadump: return .lines + default: return .status + } + } + + /// The bytes the text protocol takes for this command, data block included. + public var wireBytes: [UInt8] { + func line(_ words: [String]) -> [UInt8] { Array((words.joined(separator: " ") + "\r\n").utf8) } + func block(_ head: [String], _ value: Data) -> [UInt8] { + line(head + [String(value.count)]) + Array(value) + Array("\r\n".utf8) + } + switch self { + case .get(let keys, let withCAS): + return line([withCAS ? "gets" : "get"] + keys) + case .getAndTouch(let exptime, let keys, let withCAS): + return line([withCAS ? "gats" : "gat", String(exptime)] + keys) + case .store(let verb, let key, let flags, let exptime, let value): + return block([verb.rawValue, key, String(flags), String(exptime)], value) + case .compareAndSwap(let key, let flags, let exptime, let unique, let value): + return line(["cas", key, String(flags), String(exptime), String(value.count), String(unique)]) + + Array(value) + Array("\r\n".utf8) + case .touch(let key, let exptime): + return line(["touch", key, String(exptime)]) + case .delete(let key): + return line(["delete", key]) + case .increment(let key, let delta): + return line(["incr", key, String(delta)]) + case .decrement(let key, let delta): + return line(["decr", key, String(delta)]) + case .flushAll(let delay): + return line(["flush_all"] + (delay.map { [String($0)] } ?? [])) + case .stats(let arguments): + return line(["stats"] + arguments) + case .version: + return line(["version"]) + case .metadump(let selector): + return line(["lru_crawler", "metadump", selector]) + } + } +} + +public enum MemcachedCommandError: Error, Equatable, Sendable { + case empty + case unknownCommand(String) + case wrongArguments(command: String, usage: String) + case invalidKey(String) + case invalidNumber(String) + case unterminatedQuote +} + +public enum MemcachedCommandParser { + /// The longest key memcached accepts, in bytes. + public static let maximumKeyLength = 250 + + public static func parse(_ statement: String) throws(MemcachedCommandError) -> MemcachedCommand { + let words = try tokenize(statement) + guard let first = words.first else { throw .empty } + let name = Self.text(first).lowercased() + let arguments = Array(words.dropFirst()) + func text(_ index: Int) -> String { Self.text(arguments[index]) } + func usage(_ form: String) -> MemcachedCommandError { .wrongArguments(command: name, usage: form) } + + switch name { + case "get", "gets": + guard !arguments.isEmpty else { throw usage("\(name) [...]") } + return .get(keys: try keys(arguments[...]), withCAS: name == "gets") + case "gat", "gats": + guard arguments.count >= 2 else { throw usage("\(name) [...]") } + return .getAndTouch( + exptime: try integer(text(0)), keys: try keys(arguments.dropFirst()), withCAS: name == "gats" + ) + case "set", "add", "replace", "append", "prepend": + guard arguments.count == 4, let verb = MemcachedStorageVerb(rawValue: name) else { + throw usage("\(name) ") + } + return .store( + verb, key: try key(arguments[0]), flags: try flags(text(1)), exptime: try integer(text(2)), + value: Data(arguments[3]) + ) + case "cas": + guard arguments.count == 5 else { throw usage("cas ") } + return .compareAndSwap( + key: try key(arguments[0]), flags: try flags(text(1)), exptime: try integer(text(2)), + unique: try unsigned(text(3)), value: Data(arguments[4]) + ) + case "touch": + guard arguments.count == 2 else { throw usage("touch ") } + return .touch(key: try key(arguments[0]), exptime: try integer(text(1))) + case "delete": + guard arguments.count == 1 else { throw usage("delete ") } + return .delete(key: try key(arguments[0])) + case "incr", "decr": + guard arguments.count == 2 else { throw usage("\(name) ") } + let key = try key(arguments[0]) + let delta = try unsigned(text(1)) + return name == "incr" ? .increment(key: key, delta: delta) : .decrement(key: key, delta: delta) + case "flush_all": + guard arguments.count <= 1 else { throw usage("flush_all []") } + return .flushAll(delay: try arguments.isEmpty ? nil : integer(text(0))) + case "stats": + let words = arguments.indices.map(text) + guard words.allSatisfy(isPlainWord) else { throw usage("stats [ []]") } + return .stats(arguments: words) + case "version": + guard arguments.isEmpty else { throw usage("version") } + return .version + case "lru_crawler": + guard arguments.count == 2, text(0).lowercased() == "metadump", isPlainWord(text(1)) else { + throw usage("lru_crawler metadump ") + } + return .metadump(selector: text(1)) + default: + throw .unknownCommand(name) + } + } + + /// Writes a value so `parse` reads it back byte for byte. + public static func quoted(_ value: String) -> String { + escaped(Array(value.utf8), keepingUTF8: true) + } + + /// Writes bytes that may not be UTF-8, every byte past ASCII as `\xHH`. + public static func quoted(bytes: Data) -> String { + escaped(Array(bytes), keepingUTF8: false) + } + + private static func escaped(_ bytes: [UInt8], keepingUTF8: Bool) -> String { + var out: [UInt8] = [0x22] + for byte in bytes { + switch byte { + case 0x22: out += Array(#"\""#.utf8) + case 0x5C: out += Array(#"\\"#.utf8) + case 0x0A: out += Array(#"\n"#.utf8) + case 0x0D: out += Array(#"\r"#.utf8) + case 0x09: out += Array(#"\t"#.utf8) + case 0x20 ..< 0x7F: out.append(byte) + case 0x80...: out += keepingUTF8 ? [byte] : Array(String(format: #"\x%02X"#, byte).utf8) + default: out += Array(String(format: #"\x%02X"#, byte).utf8) + } + } + out.append(0x22) + return text(out) + } + + /// Splits a line on whitespace. Double quotes take `\n`, `\r`, `\t`, `\xHH`, `\"` and `\\`; single quotes + /// take everything as written. + static func tokenize(_ text: String) throws(MemcachedCommandError) -> [[UInt8]] { + var words: [[UInt8]] = [] + var current: [UInt8] = [] + var inWord = false + var scalars = Array(text.unicodeScalars)[...] + while let scalar = scalars.popFirst() { + switch scalar { + case " ", "\t", "\n", "\r": + if inWord { words.append(current); current = []; inWord = false } + case "'": + inWord = true + guard let end = scalars.firstIndex(of: "'") else { throw .unterminatedQuote } + current += Array(String(String.UnicodeScalarView(scalars[.., + into current: inout [UInt8] + ) throws(MemcachedCommandError) { + while let scalar = scalars.popFirst() { + switch scalar { + case "\"": + return + case "\\": + guard let escaped = scalars.popFirst() else { throw .unterminatedQuote } + switch escaped { + case "n": current.append(0x0A) + case "r": current.append(0x0D) + case "t": current.append(0x09) + case "x": + let digits = String(String.UnicodeScalarView(scalars.prefix(2))) + guard digits.count == 2, let byte = UInt8(digits, radix: 16) else { + current += Array("\\x".utf8) + continue + } + current.append(byte) + scalars = scalars.dropFirst(2) + default: current += Array(String(escaped).utf8) + } + default: + current += Array(String(scalar).utf8) + } + } + throw .unterminatedQuote + } + + /// A key is 1 to 250 bytes with no space or control character, the limits memcached's text protocol sets. + public static func isValidKey(_ bytes: [UInt8]) -> Bool { + !bytes.isEmpty && bytes.count <= maximumKeyLength && !bytes.contains { $0 <= 0x20 || $0 == 0x7F } + } + + private static func key(_ bytes: [UInt8]) throws(MemcachedCommandError) -> String { + let text = Self.text(bytes) + guard isValidKey(bytes) else { throw .invalidKey(text) } + return text + } + + private static func keys(_ words: ArraySlice<[UInt8]>) throws(MemcachedCommandError) -> [String] { + var keys: [String] = [] + for word in words { keys.append(try key(word)) } + return keys + } + + /// Bytes as text for a message or a number, never for a value: a byte that is not UTF-8 reads as Latin-1. + static func text(_ bytes: [UInt8]) -> String { + String(bytes: bytes, encoding: .utf8) ?? String(bytes: bytes, encoding: .isoLatin1) ?? "" + } + + private static func isPlainWord(_ word: String) -> Bool { + isValidKey(Array(word.utf8)) + } + + private static func integer(_ text: String) throws(MemcachedCommandError) -> Int64 { + guard let value = Int64(text) else { throw .invalidNumber(text) } + return value + } + + private static func unsigned(_ text: String) throws(MemcachedCommandError) -> UInt64 { + guard let value = UInt64(text) else { throw .invalidNumber(text) } + return value + } + + private static func flags(_ text: String) throws(MemcachedCommandError) -> UInt32 { + guard let value = UInt32(text) else { throw .invalidNumber(text) } + return value + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift new file mode 100644 index 0000000000..cd44dc0707 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift @@ -0,0 +1,95 @@ +import Foundation + +public struct MemcachedKeyEntry: Equatable, Sendable { + public let key: String + /// Unix time the item expires at, or nil for an item that never expires. + public let expiresAt: Int64? + public let flags: UInt32? + + public init(key: String, expiresAt: Int64?, flags: UInt32?) { + self.key = key + self.expiresAt = expiresAt + self.flags = flags + } + + /// Seconds left at `serverTime`, nil for no expiry. An item past its time reads as gone. + public func secondsLeft(at serverTime: Int64) -> Int64? { + expiresAt.map { $0 - serverTime } + } + + public func isExpired(at serverTime: Int64) -> Bool { + (secondsLeft(at: serverTime) ?? 1) <= 0 + } +} + +/// How the key list was read. A cachedump is a per-slab sample capped by the server, never the whole cache. +public enum MemcachedKeySource: String, Sendable, Equatable { + case metadump + case cachedump +} + +public enum MemcachedKeyListing { + /// One line of `lru_crawler metadump`: `key=a%3A1 exp=-1 la=1793499867 cas=2 fetch=yes cls=1 size=71 flags=5`. + /// The key is URL-encoded, and `exp` is a Unix time or -1 for never. + public static func metadumpEntry(_ line: String) -> MemcachedKeyEntry? { + var fields: [Substring: Substring] = [:] + for part in line.split(separator: " ") { + guard let equals = part.firstIndex(of: "=") else { continue } + fields[part[.. 0 ? $0 : nil }, + flags: fields["flags"].flatMap { UInt32($0) } + ) + } + + /// One line of `stats cachedump`: `ITEM user:42 [5 b; 1793500167 s]`. An expiry of 0 means never. + public static func cachedumpEntry(_ line: String) -> MemcachedKeyEntry? { + guard line.hasPrefix("ITEM "), let open = line.lastIndex(of: "[") else { return nil } + let key = line[line.index(line.startIndex, offsetBy: 5) ..< open].trimmingCharacters(in: .whitespaces) + guard !key.isEmpty else { return nil } + let inside = line[line.index(after: open)...].split(whereSeparator: { $0 == ";" || $0 == " " || $0 == "]" }) + let expiry = inside.count >= 3 ? Int64(inside[2]) : nil + return MemcachedKeyEntry(key: key, expiresAt: expiry.flatMap { $0 > 0 ? $0 : nil }, flags: nil) + } + + /// The slab classes holding items, read from `stats items` lines such as `STAT items:1:number 3`. + public static func slabClasses(fromItemStats lines: [String]) -> [Int] { + var classes: [Int] = [] + for (name, value) in statPairs(lines) where name.hasSuffix(":number") && (Int(value) ?? 0) > 0 { + let parts = name.split(separator: ":") + if parts.count == 3, parts[0] == "items", let id = Int(parts[1]), !classes.contains(id) { + classes.append(id) + } + } + return classes.sorted() + } + + /// `STAT ` lines as pairs, in the server's order. + public static func statPairs(_ lines: [String]) -> [(name: String, value: String)] { + lines.compactMap { line in + let parts = line.split(separator: " ", maxSplits: 2, omittingEmptySubsequences: true) + guard parts.count >= 2, parts[0] == "STAT" else { return nil } + return (String(parts[1]), parts.count == 3 ? String(parts[2]) : "") + } + } + + private static func percentDecoded(_ text: Substring) -> String? { + var bytes: [UInt8] = [] + var iterator = Array(text.utf8)[...] + while let byte = iterator.popFirst() { + guard byte == UInt8(ascii: "%") else { + bytes.append(byte) + continue + } + guard iterator.count >= 2, + let decoded = UInt8(String(bytes: iterator.prefix(2), encoding: .ascii) ?? "", radix: 16) else { return nil } + bytes.append(decoded) + iterator = iterator.dropFirst(2) + } + return String(bytes: bytes, encoding: .utf8) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift new file mode 100644 index 0000000000..57a9282892 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift @@ -0,0 +1,150 @@ +import Foundation + +public enum MemcachedReplyShape: Sendable, Equatable { + /// `VALUE` blocks closed by `END`, the reply to `get` and its kin. + case values + /// Lines closed by `END`, the reply to `stats` and `lru_crawler metadump`. + case lines + /// One status line such as `STORED`, `DELETED`, `VERSION 1.6.45` or a counter's new value. + case status +} + +public struct MemcachedValue: Equatable, Sendable { + public let key: String + public let flags: UInt32 + public let data: Data + public let unique: UInt64? + + public init(key: String, flags: UInt32, data: Data, unique: UInt64? = nil) { + self.key = key + self.flags = flags + self.data = data + self.unique = unique + } +} + +public enum MemcachedReply: Equatable, Sendable { + case values([MemcachedValue]) + /// `truncated` means the reader stopped at its line limit before `END`; the rest is still on the wire. + case lines([String], truncated: Bool) + case status(String) +} + +public struct MemcachedServerError: Error, Equatable, Sendable { + public enum Kind: String, Sendable { + case error = "ERROR" + case clientError = "CLIENT_ERROR" + case serverError = "SERVER_ERROR" + case busy = "BUSY" + } + + public let kind: Kind + public let message: String + + public init(kind: Kind, message: String) { + self.kind = kind + self.message = message + } + + /// `CLIENT_ERROR unauthenticated` is what a server started with an auth file says to any command before login. + public var needsAuthentication: Bool { + kind == .clientError && message.lowercased().contains("unauthenticated") + } +} + +public enum MemcachedProtocolError: Error, Equatable, Sendable { + case malformedLine(String) + case lineTooLong +} + +/// Reads one reply from a byte stream fed in pieces. A reply can be large, so lines already read are kept rather +/// than scanned again when more bytes arrive. +public struct MemcachedReplyReader: Sendable { + public let shape: MemcachedReplyShape + private let lineLimit: Int + private var buffer: [UInt8] = [] + private var cursor = 0 + private var values: [MemcachedValue] = [] + private var lines: [String] = [] + private var pendingValue: (key: String, flags: UInt32, length: Int, unique: UInt64?)? + + private static let maximumLineLength = 64 * 1_024 + + public init(shape: MemcachedReplyShape, lineLimit: Int = .max) { + self.shape = shape + self.lineLimit = lineLimit + } + + public mutating func append(_ bytes: Bytes) where Bytes.Element == UInt8 { + if cursor > 1 << 16 { + buffer.removeFirst(cursor) + cursor = 0 + } + buffer.append(contentsOf: bytes) + } + + /// The reply once it is whole, or nil while more bytes are needed. + public mutating func reply() throws -> MemcachedReply? { + while true { + if let pending = pendingValue { + guard buffer.count - cursor >= pending.length + 2 else { return nil } + let data = Data(buffer[cursor ..< cursor + pending.length]) + guard buffer[cursor + pending.length] == 0x0D, buffer[cursor + pending.length + 1] == 0x0A else { + throw MemcachedProtocolError.malformedLine("VALUE \(pending.key)") + } + cursor += pending.length + 2 + values.append(MemcachedValue(key: pending.key, flags: pending.flags, data: data, unique: pending.unique)) + pendingValue = nil + continue + } + guard let line = try nextLine() else { return nil } + if let error = Self.serverError(line) { throw error } + switch shape { + case .status: + return .status(line) + case .lines: + if line == "END" { return .lines(lines, truncated: false) } + if line == "OK", lines.isEmpty { continue } + lines.append(line) + if lines.count >= lineLimit { return .lines(lines, truncated: true) } + case .values: + if line == "END" { return .values(values) } + pendingValue = try Self.valueHeader(line) + } + } + } + + /// `lru_crawler metadump` ends its key lines with a bare LF and only `END` with CRLF, so a line ends at LF and + /// a CR before it is dropped. Data blocks are read by length, never as lines. + private mutating func nextLine() throws -> String? { + guard let newline = buffer[cursor...].firstIndex(of: 0x0A) else { + if buffer.count - cursor > Self.maximumLineLength { throw MemcachedProtocolError.lineTooLong } + return nil + } + let end = newline > cursor && buffer[newline - 1] == 0x0D ? newline - 1 : newline + let line = MemcachedCommandParser.text(Array(buffer[cursor ..< end])) + cursor = newline + 1 + return line + } + + static func serverError(_ line: String) -> MemcachedServerError? { + for kind in [MemcachedServerError.Kind.clientError, .serverError, .busy, .error] { + let name = kind.rawValue + if line == name { return MemcachedServerError(kind: kind, message: "") } + if line.hasPrefix(name + " ") { + return MemcachedServerError(kind: kind, message: String(line.dropFirst(name.count + 1))) + } + } + return nil + } + + private static func valueHeader(_ line: String) throws -> (key: String, flags: UInt32, length: Int, unique: UInt64?) { + let parts = line.split(separator: " ", omittingEmptySubsequences: true) + guard parts.count == 4 || parts.count == 5, parts[0] == "VALUE", + let flags = UInt32(parts[2]), let length = Int(parts[3]), length >= 0 else { + throw MemcachedProtocolError.malformedLine(line) + } + let unique = parts.count == 5 ? UInt64(parts[4]) : nil + return (String(parts[1]), flags, length, unique) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift new file mode 100644 index 0000000000..e520906283 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift @@ -0,0 +1,29 @@ +import Foundation + +/// The console lines a grid edit becomes. Each one parses back into the command it stands for. +public enum MemcachedStatements { + /// Memcached reads an exptime above 30 days as a Unix time rather than seconds from now. + public static let relativeExptimeLimit: Int64 = 60 * 60 * 24 * 30 + + /// No TTL, or 0, keeps the item until it is evicted. A TTL past 30 days is sent as the time it ends. + public static func exptime(forTTL ttl: Int64?, now: Int64) -> Int64 { + guard let ttl, ttl != 0 else { return 0 } + return ttl > relativeExptimeLimit ? now + ttl : ttl + } + + public static func set(key: String, flags: UInt32, exptime: Int64, value: String) -> String { + "set \(key) \(flags) \(exptime) \(MemcachedCommandParser.quoted(value))" + } + + public static func set(key: String, flags: UInt32, exptime: Int64, bytes: Data) -> String { + "set \(key) \(flags) \(exptime) \(MemcachedCommandParser.quoted(bytes: bytes))" + } + + public static func touch(key: String, exptime: Int64) -> String { + "touch \(key) \(exptime)" + } + + public static func delete(key: String) -> String { + "delete \(key)" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index 4241c8d5ce..620339432b 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -319,6 +319,7 @@ public struct SQLLexicalProfile: Sendable, Hashable { "Elasticsearch": commandLineFamily, "Typesense": commandLineFamily, "Weaviate": commandLineFamily, + "Memcached": commandLineFamily, "Kafka": commandLineFamily, "Firestore": commandLineFamily, "Firebase Realtime Database": commandLineFamily, diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift new file mode 100644 index 0000000000..737c955f83 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift @@ -0,0 +1,103 @@ +import Foundation +import TableProMemcachedCore +import Testing + +struct MemcachedCommandParserTests { + @Test("A storage command takes the value as its last argument and sends it as a data block") + func storeIsDataBlock() throws { + let command = try MemcachedCommandParser.parse(#"set user:1 5 300 "hello world""#) + #expect(command == .store(.set, key: "user:1", flags: 5, exptime: 300, value: Data("hello world".utf8))) + #expect(String(bytes: command.wireBytes, encoding: .utf8) == "set user:1 5 300 11\r\nhello world\r\n") + } + + @Test("A quoted value round-trips byte for byte", arguments: [ + "plain", "two words", "line\nbreak\r\n", "tab\there", "quote \" and \\ slash", "bell\u{07}", "日本語", + ]) + func quotedRoundTrips(value: String) throws { + let command = try MemcachedCommandParser.parse("set k 0 0 \(MemcachedCommandParser.quoted(value))") + #expect(command == .store(.set, key: "k", flags: 0, exptime: 0, value: Data(value.utf8))) + } + + @Test("Bytes that are not UTF-8 round-trip through \\x escapes") + func bytesRoundTrip() throws { + let bytes = Data([0x00, 0xFF, 0xC3, 0x28, 0x41, 0x22]) + let command = try MemcachedCommandParser.parse("set k 0 0 \(MemcachedCommandParser.quoted(bytes: bytes))") + #expect(command == .store(.set, key: "k", flags: 0, exptime: 0, value: bytes)) + } + + @Test("Single quotes keep backslashes as written") + func singleQuotesAreLiteral() throws { + let command = try MemcachedCommandParser.parse(#"append k 0 0 'a\nb'"#) + #expect(command == .store(.append, key: "k", flags: 0, exptime: 0, value: Data(#"a\nb"#.utf8))) + } + + @Test("Reads, counters and admin commands parse", arguments: [ + ("get a b", MemcachedCommand.get(keys: ["a", "b"], withCAS: false)), + ("GETS a", .get(keys: ["a"], withCAS: true)), + ("gat 60 a b", .getAndTouch(exptime: 60, keys: ["a", "b"], withCAS: false)), + ("touch a 0", .touch(key: "a", exptime: 0)), + ("delete a", .delete(key: "a")), + ("incr hits 3", .increment(key: "hits", delta: 3)), + ("decr hits 1", .decrement(key: "hits", delta: 1)), + ("flush_all", .flushAll(delay: nil)), + ("flush_all 10", .flushAll(delay: 10)), + ("stats", .stats(arguments: [])), + ("stats cachedump 1 0", .stats(arguments: ["cachedump", "1", "0"])), + ("version", .version), + ("lru_crawler metadump all", .metadump(selector: "all")), + ]) + func parses(text: String, expected: MemcachedCommand) throws { + #expect(try MemcachedCommandParser.parse(text) == expected) + } + + @Test("cas puts the unique after the byte count") + func casWire() throws { + let command = try MemcachedCommandParser.parse("cas k 0 0 42 v") + #expect(String(bytes: command.wireBytes, encoding: .utf8) == "cas k 0 0 1 42\r\nv\r\n") + } + + @Test("A key with a space or over 250 bytes is refused before it reaches the server") + func invalidKeys() { + #expect(throws: MemcachedCommandError.invalidKey("a b")) { try MemcachedCommandParser.parse(#"get "a b""#) } + let long = String(repeating: "k", count: 251) + #expect(throws: MemcachedCommandError.invalidKey(long)) { try MemcachedCommandParser.parse("delete \(long)") } + } + + @Test("A wrong argument count names the usage") + func wrongArguments() { + #expect(throws: MemcachedCommandError.wrongArguments( + command: "set", usage: "set " + )) { try MemcachedCommandParser.parse("set k 0 0") } + #expect(throws: MemcachedCommandError.unknownCommand("shutdown")) { try MemcachedCommandParser.parse("shutdown") } + #expect(throws: MemcachedCommandError.unterminatedQuote) { try MemcachedCommandParser.parse(#"set k 0 0 "x"#) } + #expect(throws: MemcachedCommandError.invalidNumber("-1")) { try MemcachedCommandParser.parse("set k -1 0 v") } + } + + @Test("A stats argument cannot smuggle a second command") + func statsArgumentsArePlain() { + #expect(throws: MemcachedCommandError.self) { try MemcachedCommandParser.parse("stats \"items\r\nflush_all\"") } + } +} + +struct MemcachedStatementsTests { + @Test("A TTL past 30 days is sent as the Unix time it ends, since memcached reads it that way") + func longTTLIsAbsolute() { + #expect(MemcachedStatements.exptime(forTTL: nil, now: 1_000) == 0) + #expect(MemcachedStatements.exptime(forTTL: 0, now: 1_000) == 0) + #expect(MemcachedStatements.exptime(forTTL: 300, now: 1_000) == 300) + #expect(MemcachedStatements.exptime(forTTL: MemcachedStatements.relativeExptimeLimit, now: 1_000) + == MemcachedStatements.relativeExptimeLimit) + #expect(MemcachedStatements.exptime(forTTL: MemcachedStatements.relativeExptimeLimit + 1, now: 1_000) + == 1_000 + MemcachedStatements.relativeExptimeLimit + 1) + } + + @Test("Each write statement parses back into its command") + func statementsParse() throws { + #expect(try MemcachedCommandParser.parse(MemcachedStatements.set(key: "k", flags: 3, exptime: 60, value: "a \"b\"\n")) + == .store(.set, key: "k", flags: 3, exptime: 60, value: Data("a \"b\"\n".utf8))) + #expect(try MemcachedCommandParser.parse(MemcachedStatements.set(key: "k", flags: 0, exptime: 0, bytes: Data([0xFF]))) + == .store(.set, key: "k", flags: 0, exptime: 0, value: Data([0xFF]))) + #expect(try MemcachedCommandParser.parse(MemcachedStatements.touch(key: "k", exptime: 10)) == .touch(key: "k", exptime: 10)) + #expect(try MemcachedCommandParser.parse(MemcachedStatements.delete(key: "k")) == .delete(key: "k")) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift new file mode 100644 index 0000000000..0b8b699fec --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift @@ -0,0 +1,114 @@ +import Foundation +import TableProMemcachedCore +import Testing + +struct MemcachedKeyListingTests { + @Test("A metadump line gives the decoded key, its expiry and flags") + func metadump() { + let entry = MemcachedKeyListing.metadumpEntry( + "key=a%3A1 exp=1793500167 la=1793499867 cas=2 fetch=yes cls=1 size=71 flags=5 " + ) + #expect(entry == MemcachedKeyEntry(key: "a:1", expiresAt: 1_793_500_167, flags: 5)) + #expect(entry?.secondsLeft(at: 1_793_499_867) == 300) + } + + @Test("exp=-1 means the item never expires") + func metadumpNeverExpires() { + let entry = MemcachedKeyListing.metadumpEntry("key=%22sp%22 exp=-1 la=1 cas=4 fetch=no cls=1 size=64 flags=0") + #expect(entry == MemcachedKeyEntry(key: "\"sp\"", expiresAt: nil, flags: 0)) + #expect(entry?.isExpired(at: .max) == false) + } + + @Test("A cachedump line gives the key and expiry; 0 means never") + func cachedump() { + #expect(MemcachedKeyListing.cachedumpEntry("ITEM k1 [2 b; 0 s]") == MemcachedKeyEntry( + key: "k1", expiresAt: nil, flags: nil + )) + #expect(MemcachedKeyListing.cachedumpEntry("ITEM user:42 [5 b; 1793500167 s]") == MemcachedKeyEntry( + key: "user:42", expiresAt: 1_793_500_167, flags: nil + )) + #expect(MemcachedKeyListing.cachedumpEntry("STAT pid 1") == nil) + } + + @Test("Slab classes come from the item counts in stats items") + func slabClasses() { + let lines = [ + "STAT items:1:number 3", "STAT items:1:age 10", "STAT items:12:number 1", "STAT items:5:number 0", + ] + #expect(MemcachedKeyListing.slabClasses(fromItemStats: lines) == [1, 12]) + } + + @Test("Stat pairs keep a value with spaces whole") + func statPairs() { + let pairs = MemcachedKeyListing.statPairs(["STAT version 1.6.45", "STAT slab_reassign_last_busy_status none x"]) + #expect(pairs.map(\.name) == ["version", "slab_reassign_last_busy_status"]) + #expect(pairs.last?.value == "none x") + } +} + +struct MemcachedBrowseSpecTests { + private let keys = ["user:2", "user:10", "session:a", "User:3"].map { + MemcachedKeyEntry(key: $0, expiresAt: nil, flags: nil) + } + + @Test("A spec survives its tagged statement") + func roundTrip() throws { + let condition = try MemcachedBrowseSpec.condition(column: "Key", op: "starts with", value: "user:", isCaseSensitive: true) + let spec = MemcachedBrowseSpec(limit: 50, offset: 100, ascending: false, conditions: [condition]) + #expect(MemcachedBrowseSpec.parse(spec.statement) == spec) + #expect(MemcachedBrowseSpec.parse("get a") == nil) + } + + @Test("A page is filtered, sorted by key, then cut") + func page() throws { + let condition = try MemcachedBrowseSpec.condition(column: "Key", op: "STARTS WITH", value: "user:", isCaseSensitive: false) + let spec = MemcachedBrowseSpec(limit: 2, offset: 1, ascending: true, conditions: [condition]) + #expect(try spec.page(of: keys).map(\.key) == ["user:10", "user:2"]) + #expect(try MemcachedBrowseSpec(limit: 10, offset: 9, ascending: true).page(of: keys).isEmpty) + } + + @Test("Key comparisons", arguments: [ + ("=", "user:2", ["user:2"]), + ("!=", "user:2", ["User:3", "session:a", "user:10"]), + ("CONTAINS", "sion", ["session:a"]), + ("ENDS WITH", ":a", ["session:a"]), + ("IN", "user:2, session:a", ["session:a", "user:2"]), + ("NOT IN", "user:2,session:a", ["User:3", "user:10"]), + ("REGEX", "^user:[0-9]$", ["user:2"]), + ]) + func comparisons(op: String, value: String, expected: [String]) throws { + let condition = try MemcachedBrowseSpec.condition(column: "Key", op: op, value: value, isCaseSensitive: true) + let spec = MemcachedBrowseSpec(limit: 100, offset: 0, ascending: true, conditions: [condition]) + #expect(try spec.page(of: keys).map(\.key) == expected) + } + + @Test("OR keeps a key any condition matches") + func orLogic() throws { + let first = try MemcachedBrowseSpec.condition(column: "Key", op: "=", value: "user:2", isCaseSensitive: true) + let second = try MemcachedBrowseSpec.condition(column: "Key", op: "=", value: "User:3", isCaseSensitive: true) + let spec = MemcachedBrowseSpec(limit: 100, offset: 0, ascending: true, conditions: [first, second], matchesAll: false) + #expect(try spec.page(of: keys).map(\.key) == ["User:3", "user:2"]) + } + + @Test("Only Key filters, and a bad operator or pattern is refused") + func refusals() { + #expect(throws: MemcachedFilterRefusal.notKeyColumn("Value")) { + try MemcachedBrowseSpec.condition(column: "Value", op: "=", value: "x", isCaseSensitive: true) + } + #expect(throws: MemcachedFilterRefusal.unsupportedOperator(">")) { + try MemcachedBrowseSpec.condition(column: "Key", op: ">", value: "x", isCaseSensitive: true) + } + #expect(throws: MemcachedFilterRefusal.invalidPattern("[")) { + try MemcachedBrowseSpec.condition(column: "Key", op: "REGEX", value: "[", isCaseSensitive: true) + } + #expect(throws: MemcachedFilterRefusal.emptyList("IN")) { + try MemcachedBrowseSpec.condition(column: "Key", op: "IN", value: " , ", isCaseSensitive: true) + } + } + + @Test("A refusal statement carries its message") + func refusalStatement() { + let statement = MemcachedBrowseSpec.refusalStatement("Memcached can only filter by Key.") + #expect(MemcachedBrowseSpec.refusalMessage(statement) == "Memcached can only filter by Key.") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedReplyReaderTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedReplyReaderTests.swift new file mode 100644 index 0000000000..2949dfbcaa --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedReplyReaderTests.swift @@ -0,0 +1,77 @@ +import Foundation +import TableProMemcachedCore +import Testing + +struct MemcachedReplyReaderTests { + private func read(_ shape: MemcachedReplyShape, _ chunks: [String], lineLimit: Int = .max) throws -> MemcachedReply? { + var reader = MemcachedReplyReader(shape: shape, lineLimit: lineLimit) + var reply: MemcachedReply? + for chunk in chunks { + reader.append(Array(chunk.utf8)) + reply = try reader.reply() + } + return reply + } + + @Test("Values split across reads, with a CRLF inside the data, come back whole") + func valuesAcrossChunks() throws { + let reply = try read(.values, ["VALUE a 5 4\r\nx\r", "\ny\r\nVALUE b 0 0 7\r\n\r\nEN", "D\r\n"]) + #expect(reply == .values([ + MemcachedValue(key: "a", flags: 5, data: Data("x\r\ny".utf8)), + MemcachedValue(key: "b", flags: 0, data: Data(), unique: 7), + ])) + } + + @Test("A reply missing its END is not whole yet") + func waitsForEnd() throws { + #expect(try read(.values, ["VALUE a 0 1\r\nx\r\n"]) == nil) + #expect(try read(.lines, ["STAT pid 1\r\n"]) == nil) + } + + @Test("Stats lines, and the OK some servers send before a metadump, are read up to END") + func lines() throws { + #expect(try read(.lines, ["STAT pid 1\r\nSTAT uptime 20\r\nEND\r\n"]) + == .lines(["STAT pid 1", "STAT uptime 20"], truncated: false)) + #expect(try read(.lines, ["OK\r\nkey=a exp=-1\r\nEND\r\n"]) == .lines(["key=a exp=-1"], truncated: false)) + } + + @Test("A metadump ends its key lines with a bare LF and only END with CRLF") + func metadumpLineEndings() throws { + #expect(try read(.lines, ["OK\r\nkey=a exp=-1 \nkey=b", " exp=-1 \nEND\r\n"]) + == .lines(["key=a exp=-1 ", "key=b exp=-1 "], truncated: false)) + } + + @Test("The line limit stops a long listing early and says so") + func lineLimit() throws { + #expect(try read(.lines, ["key=a\r\nkey=b\r\nkey=c\r\n"], lineLimit: 2) == .lines(["key=a", "key=b"], truncated: true)) + } + + @Test("Error lines throw with the server's message", arguments: [ + ("ERROR\r\n", MemcachedServerError(kind: .error, message: "")), + ("CLIENT_ERROR unauthenticated\r\n", MemcachedServerError(kind: .clientError, message: "unauthenticated")), + ("SERVER_ERROR out of memory storing object\r\n", MemcachedServerError( + kind: .serverError, message: "out of memory storing object" + )), + ("CLIENT_ERROR lru crawler disabled\r\n", MemcachedServerError(kind: .clientError, message: "lru crawler disabled")), + ]) + func errors(line: String, expected: MemcachedServerError) { + #expect(throws: expected) { try read(.lines, [line]) } + } + + @Test("An unauthenticated refusal is recognized") + func needsAuthentication() { + #expect(MemcachedServerError(kind: .clientError, message: "unauthenticated").needsAuthentication) + #expect(!MemcachedServerError(kind: .clientError, message: "bad data chunk").needsAuthentication) + } + + @Test("A status reply is its one line") + func status() throws { + #expect(try read(.status, ["STO", "RED\r\n"]) == .status("STORED")) + #expect(try read(.status, ["VERSION 1.6.45\r\n"]) == .status("VERSION 1.6.45")) + } + + @Test("A malformed VALUE header is a protocol error") + func malformedHeader() { + #expect(throws: MemcachedProtocolError.self) { try read(.values, ["VALUE a x 1\r\n"]) } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift b/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift index ca8b6a75bd..3190e32036 100644 --- a/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift @@ -61,7 +61,7 @@ struct DatabaseTypeTests { @Test("allKnownTypes contains all expected types") func allKnownTypesComplete() { - #expect(DatabaseType.allKnownTypes.count == 33) + #expect(DatabaseType.allKnownTypes.count == 34) #expect(DatabaseType.allKnownTypes.contains(.mysql)) #expect(DatabaseType.allKnownTypes.contains(.tidb)) #expect(DatabaseType.allKnownTypes.contains(.databend)) @@ -78,6 +78,7 @@ struct DatabaseTypeTests { #expect(DatabaseType.allKnownTypes.contains(.kafka)) #expect(DatabaseType.allKnownTypes.contains(.cloudflareR2SQL)) #expect(DatabaseType.allKnownTypes.contains(.weaviate)) + #expect(DatabaseType.allKnownTypes.contains(.memcached)) #expect(DatabaseType.allKnownTypes.contains(.firestore)) #expect(DatabaseType.allKnownTypes.contains(.firebaseRealtimeDatabase)) #expect(DatabaseType.allKnownTypes.contains(.bigtable)) @@ -105,6 +106,13 @@ struct DatabaseTypeTests { #expect(DatabaseType.weaviate.pluginTypeId == "Weaviate") } + @Test("Memcached resolves its icon and plugin type id") + func memcachedIdentity() { + #expect(DatabaseType.memcached.rawValue == "Memcached") + #expect(DatabaseType.memcached.iconName == "memorychip") + #expect(DatabaseType.memcached.pluginTypeId == "Memcached") + } + @Test("Both Firebase types resolve the Firebase icon and keep their own plugin type id") func firebaseIdentity() { #expect(DatabaseType.firestore.rawValue == "Firestore") diff --git a/apple/Plugins/MemcachedDriverPlugin/Info.plist b/apple/Plugins/MemcachedDriverPlugin/Info.plist new file mode 100644 index 0000000000..a9240bbf3f --- /dev/null +++ b/apple/Plugins/MemcachedDriverPlugin/Info.plist @@ -0,0 +1,12 @@ + + + + + TableProPluginKitVersion + TABLEPRO_PLUGINKIT_VERSION + TableProProvidesDatabaseTypeIds + + Memcached + + + diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift new file mode 100644 index 0000000000..011a9c7e25 --- /dev/null +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift @@ -0,0 +1,282 @@ +import Foundation +import NIOConcurrencyHelpers +import NIOCore +import NIOPosix +import NIOSSL +import TableProMemcachedCore +import TableProPluginKit + +/// Feeds the bytes of the one request in flight into its reader and resumes the caller once the reply is whole. +/// Every path that resumes goes through the lock and clears the continuation first, so it resumes once. +private final class MemcachedReplyHandler: ChannelInboundHandler, @unchecked Sendable { + typealias InboundIn = ByteBuffer + + private var reader: MemcachedReplyReader? + private var continuation: CheckedContinuation? + private var failure: Error? + private let lock = NIOLock() + + func expect( + _ shape: MemcachedReplyShape, + lineLimit: Int, + continuation: CheckedContinuation + ) -> Bool { + let refusal: Error? = lock.withLock { + if let failure { return failure } + reader = MemcachedReplyReader(shape: shape, lineLimit: lineLimit) + self.continuation = continuation + return nil + } + guard let refusal else { return true } + continuation.resume(throwing: refusal) + return false + } + + func channelRead(context: ChannelHandlerContext, data: NIOAny) { + let buffer = unwrapInboundIn(data) + let outcome: (CheckedContinuation, Result)? = lock.withLock { + guard continuation != nil else { return nil } + reader?.append(buffer.readableBytesView) + let result: Result + do { + guard let reply = try reader?.reply() else { return nil } + result = .success(reply) + } catch { + result = .failure(error) + } + defer { continuation = nil; reader = nil } + return continuation.map { ($0, result) } + } + guard let (waiting, result) = outcome else { return } + waiting.resume(with: result) + } + + func errorCaught(context: ChannelHandlerContext, error: Error) { + finish(with: error) + context.close(promise: nil) + } + + func channelInactive(context: ChannelHandlerContext) { + finish(with: MemcachedError.connectionClosed) + } + + func finish(with error: Error) { + let waiting = lock.withLock { () -> CheckedContinuation? in + if failure == nil { failure = error } + defer { continuation = nil; reader = nil } + return continuation + } + waiting?.resume(throwing: error) + } +} + +/// Resumes a connect's caller once, whichever of the connect and the cancellation arrives first. +private final class ConnectGate: @unchecked Sendable { + private var continuation: CheckedContinuation? + private var settled = false + private let lock = NIOLock() + + func arm(_ continuation: CheckedContinuation) { + let alreadySettled: Bool = lock.withLock { + if settled { return true } + self.continuation = continuation + return false + } + if alreadySettled { continuation.resume(throwing: CancellationError()) } + } + + func deliver(_ channel: Channel) -> Bool { + guard let waiting = take() else { return false } + waiting.resume(returning: channel) + return true + } + + func fail(_ error: Error) { take()?.resume(throwing: error) } + func cancel() { take()?.resume(throwing: CancellationError()) } + + private func take() -> CheckedContinuation? { + lock.withLock { + guard !settled else { return nil } + settled = true + defer { continuation = nil } + return continuation + } + } +} + +struct MemcachedEndpoint: Sendable { + let host: String + let port: Int + let ssl: SSLConfiguration + let username: String + let password: String + let connectTimeout: TimeAmount +} + +/// One TCP connection speaking the text protocol, one request at a time. The text protocol has no request ids, +/// so replies are matched to requests by order alone, and a queue keeps two callers from interleaving. +actor MemcachedConnection { + private let endpoint: MemcachedEndpoint + private var channel: Channel? + private var handler: MemcachedReplyHandler? + private var wireSlotIsTaken = false + private var waitingForWireSlot: [CheckedContinuation] = [] + + init(endpoint: MemcachedEndpoint) { + self.endpoint = endpoint + } + + func open() async throws { + guard channel == nil else { return } + let sslContext = try Self.makeSSLContext(endpoint.ssl) + let serverHostname = endpoint.ssl.verifiesHostname ? endpoint.host : nil + let replyHandler = MemcachedReplyHandler() + let bootstrap = ClientBootstrap(group: NIOSingletons.posixEventLoopGroup) + .connectTimeout(endpoint.connectTimeout) + .channelOption(ChannelOptions.socketOption(.so_keepalive), value: 1) + .channelInitializer { channel in + do { + if let sslContext { + let tls = try NIOSSLClientHandler(context: sslContext, serverHostname: serverHostname) + try channel.pipeline.syncOperations.addHandler(tls) + } + try channel.pipeline.syncOperations.addHandler(replyHandler) + return channel.eventLoop.makeSucceededVoidFuture() + } catch { + return channel.eventLoop.makeFailedFuture(error) + } + } + + do { + channel = try await Self.connect(bootstrap, host: endpoint.host, port: endpoint.port) + } catch is CancellationError { + throw CancellationError() + } catch { + throw MemcachedError.connectFailed(Self.describeConnect(error)) + } + handler = replyHandler + + do { + try await authenticate() + } catch { + await close() + throw error + } + } + + /// A server started with `-Y ` takes the user and password as the data of the first `set`. + private func authenticate() async throws { + guard !endpoint.username.isEmpty else { return } + let login = MemcachedCommand.store( + .set, key: "tablepro", flags: 0, exptime: 0, value: Data("\(endpoint.username) \(endpoint.password)".utf8) + ) + do { + guard case .status("STORED") = try await send(login) else { throw MemcachedError.authenticationFailed } + } catch let error as MemcachedServerError where error.kind == .clientError { + throw MemcachedError.authenticationFailed + } + } + + private static func connect(_ bootstrap: ClientBootstrap, host: String, port: Int) async throws -> Channel { + let gate = ConnectGate() + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + gate.arm(continuation) + bootstrap.connect(host: host, port: port).whenComplete { result in + switch result { + case .success(let channel): + if !gate.deliver(channel) { channel.close(promise: nil) } + case .failure(let error): + gate.fail(error) + } + } + } + } onCancel: { + gate.cancel() + } + } + + func close() async { + let existing = channel + handler?.finish(with: MemcachedError.notConnected) + channel = nil + handler = nil + try? await existing?.close() + } + + var isOpen: Bool { channel?.isActive == true } + + /// Sends one command and reads its reply. Cancelling closes the channel: once a request is written the text + /// protocol has no way to skip its reply, so the connection cannot be reused for the next one. + func send(_ command: MemcachedCommand, lineLimit: Int = .max) async throws -> MemcachedReply { + await claimWireSlot() + defer { releaseWireSlot() } + + guard let channel, let handler, channel.isActive else { throw MemcachedError.notConnected } + try Task.checkCancellation() + + let bytes = command.wireBytes + var buffer = channel.allocator.buffer(capacity: bytes.count) + buffer.writeBytes(bytes) + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + guard handler.expect(command.replyShape, lineLimit: lineLimit, continuation: continuation) else { return } + channel.writeAndFlush(buffer).whenFailure { error in + handler.finish(with: MemcachedError.connectFailed(error.localizedDescription)) + } + } + } onCancel: { + channel.close(promise: nil) + } + } + + private func claimWireSlot() async { + guard wireSlotIsTaken else { + wireSlotIsTaken = true + return + } + await withCheckedContinuation { waitingForWireSlot.append($0) } + } + + private func releaseWireSlot() { + guard !waitingForWireSlot.isEmpty else { + wireSlotIsTaken = false + return + } + waitingForWireSlot.removeFirst().resume() + } + + private static func makeSSLContext(_ ssl: SSLConfiguration) throws -> NIOSSLContext? { + guard ssl.isEnabled else { return nil } + var configuration = TLSConfiguration.makeClientConfiguration() + switch ssl.mode { + case .verifyIdentity: configuration.certificateVerification = .fullVerification + case .verifyCa: configuration.certificateVerification = .noHostnameVerification + default: configuration.certificateVerification = .none + } + if ssl.verifiesCertificate, !ssl.caCertificatePath.isEmpty { + configuration.trustRoots = .certificates(try NIOSSLCertificate.fromPEMFile(ssl.caCertificatePath)) + } + if !ssl.clientCertificatePath.isEmpty { + let chain = try NIOSSLCertificate.fromPEMFile(ssl.clientCertificatePath) + configuration.certificateChain = chain.map { .certificate($0) } + } + if !ssl.clientKeyPath.isEmpty { + configuration.privateKey = .privateKey(try NIOSSLPrivateKey(file: ssl.clientKeyPath, format: .pem)) + } + return try NIOSSLContext(configuration: configuration) + } + + private static func describeConnect(_ error: Error) -> String { + if let channelError = error as? ChannelError, case .connectTimeout = channelError { + return String(localized: "the connection timed out") + } + if let ioError = error as? IOError, ioError.errnoCode == ECONNREFUSED { + return String(localized: "the connection was refused") + } + if error is NIOSSLError || error is NIOSSLExtraError { + return String(localized: "the TLS handshake failed") + } + return error.localizedDescription + } +} diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift new file mode 100644 index 0000000000..43c25be009 --- /dev/null +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift @@ -0,0 +1,88 @@ +import Foundation +import TableProMemcachedCore +import TableProPluginKit + +enum MemcachedError: Error, PluginDriverError { + case notConnected + case connectionClosed + case connectFailed(String) + case authenticationRequired + case authenticationFailed + case server(MemcachedServerError) + case command(MemcachedCommandError) + case filter(MemcachedFilterRefusal) + case refused(String) + case unexpectedReply(String) + + var pluginErrorMessage: String { + switch self { + case .notConnected: + return String(localized: "Not connected to Memcached") + case .connectionClosed: + return String(localized: "Memcached closed the connection") + case .connectFailed(let reason): + return String(format: String(localized: "Could not connect to Memcached: %@"), reason) + case .authenticationRequired: + return String(localized: "This Memcached server requires a username and password") + case .authenticationFailed: + return String(localized: "Memcached rejected the username or password") + case .server(let error): + return error.message.isEmpty ? error.kind.rawValue : "\(error.kind.rawValue): \(error.message)" + case .command(let error): + return Self.describe(error) + case .filter(let refusal): + return Self.describe(refusal) + case .refused(let message): + return message + case .unexpectedReply(let line): + return String(format: String(localized: "Unexpected reply from Memcached: %@"), line) + } + } + + var errorDescription: String? { pluginErrorMessage } + + static func wrap(_ error: Error) -> Error { + switch error { + case let error as MemcachedServerError: + return error.needsAuthentication ? MemcachedError.authenticationRequired : MemcachedError.server(error) + case let error as MemcachedCommandError: + return MemcachedError.command(error) + case let error as MemcachedFilterRefusal: + return MemcachedError.filter(error) + case is MemcachedProtocolError: + return MemcachedError.unexpectedReply(String(describing: error)) + default: + return error + } + } + + private static func describe(_ error: MemcachedCommandError) -> String { + switch error { + case .empty: + return String(localized: "Enter a Memcached command") + case .unknownCommand(let name): + return String(format: String(localized: "Unsupported Memcached command: %@"), name) + case .wrongArguments(_, let usage): + return String(format: String(localized: "Usage: %@"), usage) + case .invalidKey(let key): + return String(format: String(localized: "'%@' is not a valid key: a key is 1 to 250 bytes with no spaces or control characters"), key) + case .invalidNumber(let text): + return String(format: String(localized: "'%@' is not a valid number here"), text) + case .unterminatedQuote: + return String(localized: "A quoted value is missing its closing quote") + } + } + + private static func describe(_ refusal: MemcachedFilterRefusal) -> String { + switch refusal { + case .notKeyColumn: + return String(localized: "Memcached can only filter by Key") + case .unsupportedOperator(let op): + return String(format: String(localized: "Memcached cannot filter keys with %@"), op) + case .emptyList(let op): + return String(format: String(localized: "Enter at least one value to filter with %@"), op) + case .invalidPattern(let pattern): + return String(format: String(localized: "'%@' is not a valid regular expression"), pattern) + } + } +} diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPlugin.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPlugin.swift new file mode 100644 index 0000000000..16344b92ff --- /dev/null +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPlugin.swift @@ -0,0 +1,70 @@ +import Foundation +import TableProPluginKit + +final class MemcachedPlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Memcached Driver" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Memcached support over the text protocol" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "Memcached" + static let databaseDisplayName = "Memcached" + static let iconName = "memorychip" + static let defaultPort = 11_211 + static let isDownloadable = true + + static let navigationModel: NavigationModel = .standard + static let pathFieldRole: PathFieldRole = .database + static let requiresAuthentication = false + static let urlSchemes: [String] = ["memcached"] + static let brandColorHex = "#2A7AB0" + static let queryLanguageName = "Memcached" + static let editorLanguage: EditorLanguage = .bash + static let supportsForeignKeys = false + static let supportsSchemaEditing = false + static let supportsDatabaseSwitching = false + static let supportsImport = false + static let supportsSSL = true + static let tableEntityName = "Keys" + static let supportsForeignKeyDisable = false + static let supportsReadOnlyMode = true + static let databaseGroupingStrategy: GroupingStrategy = .flat + static let defaultGroupName = "default" + static let defaultPrimaryKeyColumn: String? = "Key" + static let structureColumnFields: [StructureColumnField] = [.name, .type, .nullable] + static let sqlDialect: SQLDialectDescriptor? = nil + static let caseSensitivityStyle: SQLDialectDescriptor.CaseSensitivityStyle = .driverManaged + static let columnTypesByCategory: [String: [String]] = ["String": ["String"], "Integer": ["Int64"]] + + /// The one table the sidebar shows. Memcached has no namespaces, so every key lives in it. + static let keysTable = "keys" + + static var statementCompletions: [CompletionEntry] { + [ + CompletionEntry(label: "get", insertText: "get "), + CompletionEntry(label: "gets", insertText: "gets "), + CompletionEntry(label: "gat", insertText: "gat "), + CompletionEntry(label: "set", insertText: "set "), + CompletionEntry(label: "add", insertText: "add "), + CompletionEntry(label: "replace", insertText: "replace "), + CompletionEntry(label: "append", insertText: "append "), + CompletionEntry(label: "prepend", insertText: "prepend "), + CompletionEntry(label: "cas", insertText: "cas "), + CompletionEntry(label: "touch", insertText: "touch "), + CompletionEntry(label: "delete", insertText: "delete "), + CompletionEntry(label: "incr", insertText: "incr "), + CompletionEntry(label: "decr", insertText: "decr "), + CompletionEntry(label: "flush_all", insertText: "flush_all"), + CompletionEntry(label: "stats", insertText: "stats"), + CompletionEntry(label: "stats items", insertText: "stats items"), + CompletionEntry(label: "stats slabs", insertText: "stats slabs"), + CompletionEntry(label: "stats settings", insertText: "stats settings"), + CompletionEntry(label: "version", insertText: "version"), + CompletionEntry(label: "lru_crawler metadump all", insertText: "lru_crawler metadump all"), + ] + } + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + MemcachedPluginDriver(config: config) + } +} diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift new file mode 100644 index 0000000000..c538b10414 --- /dev/null +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -0,0 +1,416 @@ +import Foundation +import NIOCore +import os +import TableProMemcachedCore +import TableProPluginKit + +/// The text protocol has no command that lists keys. `lru_crawler metadump` (1.4.31 and later) walks every item, +/// so it lists them all; an older server, or one started with the crawler off, only offers `stats cachedump`, +/// which the server caps per slab class, so that list is a sample and the table says so. +final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "MemcachedPluginDriver") + + /// Keys one listing reads before it stops, so a cache holding millions of items cannot exhaust memory. + static let listingLimit = 1_000_000 + private static let valuesPerRequest = 100 + + private let endpoint: MemcachedEndpoint + private let lock = NSLock() + private var connection: MemcachedConnection? + private var _serverVersion: String? + private var keySource: MemcachedKeySource = .metadump + + init(config: DriverConnectionConfig) { + endpoint = MemcachedEndpoint( + host: config.host, + port: config.port, + ssl: config.ssl, + username: config.username, + password: config.password, + connectTimeout: .seconds(10) + ) + } + + var serverVersion: String? { lock.withLock { _serverVersion } } + var supportsTransactions: Bool { false } + var capabilities: PluginCapabilities { [] } + + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + + func quoteIdentifier(_ name: String) -> String { name } + func escapeStringLiteral(_ value: String) -> String { value } + + // MARK: - Connection + + func connect() async throws { + let opened = MemcachedConnection(endpoint: endpoint) + do { + try await opened.open() + let version = try await Self.version(on: opened) + let source = try await Self.keySource(on: opened) + lock.withLock { + connection = opened + _serverVersion = version + keySource = source + } + } catch { + await opened.close() + throw MemcachedError.wrap(error) + } + } + + func disconnect() { + let existing = lock.withLock { () -> MemcachedConnection? in + defer { connection = nil } + return connection + } + guard let existing else { return } + Task { await existing.close() } + } + + func ping() async throws { + _ = try await Self.version(on: try liveConnection()) + } + + private func liveConnection() throws -> MemcachedConnection { + guard let connection = lock.withLock({ connection }) else { throw MemcachedError.notConnected } + return connection + } + + private static func version(on connection: MemcachedConnection) async throws -> String { + guard case .status(let line) = try await connection.send(.version), line.hasPrefix("VERSION ") else { + throw MemcachedError.unexpectedReply("version") + } + return String(line.dropFirst("VERSION ".count)) + } + + /// `stats settings` reports `lru_crawler yes` when the crawler runs. A server too old to report it has none. + private static func keySource(on connection: MemcachedConnection) async throws -> MemcachedKeySource { + guard case .lines(let lines, _) = try await connection.send(.stats(arguments: ["settings"])) else { + return .cachedump + } + let crawler = MemcachedKeyListing.statPairs(lines).first { $0.name == "lru_crawler" }?.value + return crawler == "yes" ? .metadump : .cachedump + } + + // MARK: - Execution + + func execute(query: String) async throws -> PluginQueryResult { + let started = Date() + let statement = query.trimmingCharacters(in: .whitespacesAndNewlines) + do { + if statement.lowercased() == "select 1" { + try await ping() + return result(columns: ["ok"], types: ["Int32"], rows: [[.text("1")]], started: started) + } + if let message = MemcachedBrowseSpec.refusalMessage(statement) { + throw MemcachedError.refused(message) + } + if let spec = MemcachedBrowseSpec.parse(statement) { + return try await browse(spec, started: started) + } + return try await run(try MemcachedCommandParser.parse(statement), started: started) + } catch { + throw MemcachedError.wrap(error) + } + } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await execute(query: query) + } + + private func run(_ command: MemcachedCommand, started: Date) async throws -> PluginQueryResult { + let connection = try liveConnection() + if case .metadump = command { + return try await metadumpResult(command, started: started) + } + switch try await connection.send(command) { + case .values(let values): + let withCAS: Bool + switch command { + case .get(_, let cas), .getAndTouch(_, _, let cas): withCAS = cas + default: withCAS = false + } + let rows: [PluginRow] = values.map { value in + var row: PluginRow = [.text(value.key), Self.cell(value.data), .text(String(value.flags))] + if withCAS { row.append(PluginCellValue.fromOptional(value.unique.map(String.init))) } + return row + } + return result( + columns: ["Key", "Value", "Flags"] + (withCAS ? ["CAS"] : []), + types: ["String", "String", "Int64"] + (withCAS ? ["Int64"] : []), + rows: rows, started: started + ) + case .lines(let lines, _): + let rows: [PluginRow] = MemcachedKeyListing.statPairs(lines).map { [.text($0.name), .text($0.value)] } + return result(columns: ["Name", "Value"], types: ["String", "String"], rows: rows, started: started) + case .status(let line): + let affected = ["STORED", "DELETED", "TOUCHED", "OK"].contains(line) || UInt64(line) != nil ? 1 : 0 + return result( + columns: ["Result"], types: ["String"], rows: [[.text(line)]], affected: affected, started: started + ) + } + } + + /// A metadump cannot share a connection with other commands, so it runs on its own. + private func metadumpResult(_ command: MemcachedCommand, started: Date) async throws -> PluginQueryResult { + let lines = try await withListingConnection { connection in + guard case .lines(let lines, _) = try await connection.send(command, lineLimit: Self.listingLimit) else { + return [String]() + } + return lines + } + let rows: [PluginRow] = lines.compactMap(MemcachedKeyListing.metadumpEntry).map { entry in + [ + .text(entry.key), + PluginCellValue.fromOptional(entry.expiresAt.map(String.init)), + PluginCellValue.fromOptional(entry.flags.map(String.init)), + ] + } + return result(columns: ["Key", "Expires", "Flags"], types: ["String", "Int64", "Int64"], rows: rows, started: started) + } + + // MARK: - Browsing + + private func browse(_ spec: MemcachedBrowseSpec, started: Date) async throws -> PluginQueryResult { + let connection = try liveConnection() + let serverTime = try await Self.serverTime(on: connection) + let listed = try await listKeys().filter { !$0.isExpired(at: serverTime) } + let page = try spec.page(of: listed) + + var values: [String: MemcachedValue] = [:] + var start = page.startIndex + while start < page.endIndex { + try Task.checkCancellation() + let end = min(start + Self.valuesPerRequest, page.endIndex) + let keys = page[start ..< end].map(\.key) + if case .values(let read) = try await connection.send(.get(keys: keys, withCAS: false)) { + for value in read { values[value.key] = value } + } + start = end + } + + let rows: [PluginRow] = page.compactMap { entry in + guard let value = values[entry.key] else { return nil } + return [ + .text(entry.key), + Self.cell(value.data), + PluginCellValue.fromOptional(entry.secondsLeft(at: serverTime).map(String.init)), + .text(String(value.flags)), + ] + } + return result( + columns: MemcachedColumn.allCases.map(\.rawValue), + types: MemcachedColumn.allCases.map(\.typeName), + rows: rows, started: started + ) + } + + private func listKeys() async throws -> [MemcachedKeyEntry] { + if lock.withLock({ keySource }) == .metadump { + do { + return try await withListingConnection { connection in + let reply = try await connection.send(.metadump(selector: "all"), lineLimit: Self.listingLimit) + guard case .lines(let lines, _) = reply else { return [] } + return lines.compactMap(MemcachedKeyListing.metadumpEntry) + } + } catch let error as MemcachedServerError where error.kind != .busy && !error.needsAuthentication { + Self.logger.info("metadump refused, listing with cachedump") + lock.withLock { keySource = .cachedump } + } + } + return try await cachedumpKeys() + } + + private func cachedumpKeys() async throws -> [MemcachedKeyEntry] { + let connection = try liveConnection() + guard case .lines(let itemStats, _) = try await connection.send(.stats(arguments: ["items"])) else { return [] } + var entries: [MemcachedKeyEntry] = [] + for slab in MemcachedKeyListing.slabClasses(fromItemStats: itemStats) { + try Task.checkCancellation() + let reply = try await connection.send(.stats(arguments: ["cachedump", String(slab), "0"])) + guard case .lines(let lines, _) = reply else { continue } + entries += lines.compactMap(MemcachedKeyListing.cachedumpEntry) + if entries.count >= Self.listingLimit { break } + } + return entries + } + + /// Opens a connection for one listing and closes it after, even when the listing stopped early with the rest + /// of the reply still on the wire. + private func withListingConnection( + _ body: (MemcachedConnection) async throws -> T + ) async throws -> T { + let listing = MemcachedConnection(endpoint: endpoint) + do { + try await listing.open() + let value = try await body(listing) + await listing.close() + return value + } catch { + await listing.close() + throw error + } + } + + private static func serverTime(on connection: MemcachedConnection) async throws -> Int64 { + guard case .lines(let lines, _) = try await connection.send(.stats(arguments: [])), + let time = MemcachedKeyListing.statPairs(lines).first(where: { $0.name == "time" }).flatMap({ Int64($0.value) }) + else { return Int64(Date().timeIntervalSince1970) } + return time + } + + private func stat(_ name: String) async throws -> String? { + guard case .lines(let lines, _) = try await liveConnection().send(.stats(arguments: [])) else { return nil } + return MemcachedKeyListing.statPairs(lines).first { $0.name == name }?.value + } + + // MARK: - Schema + + func fetchDatabases() async throws -> [String] { [MemcachedPlugin.defaultGroupName] } + + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + let bytes = try await stat("bytes").flatMap { Int64($0) } + return PluginDatabaseMetadata(name: database, tableCount: 1, sizeBytes: bytes) + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + let count = try await stat("curr_items").flatMap { Int($0) } + return [PluginTableInfo( + name: MemcachedPlugin.keysTable, type: "TABLE", rowCount: count, comment: listingComment + )] + } + + private var listingComment: String { + switch lock.withLock({ keySource }) { + case .metadump: + return String(localized: "Keys listed with lru_crawler metadump") + case .cachedump: + return String(localized: "Keys listed with stats cachedump, which returns a capped sample per slab class") + } + } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + [ + PluginColumnInfo(name: "Key", dataType: "String", isNullable: false, isPrimaryKey: true), + PluginColumnInfo(name: "Value", dataType: "String", isNullable: true), + PluginColumnInfo(name: "TTL", dataType: "Int64", isNullable: true), + PluginColumnInfo(name: "Flags", dataType: "Int64", isNullable: false), + ] + } + + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { + [MemcachedPlugin.keysTable: try await fetchColumns(table: MemcachedPlugin.keysTable, schema: schema)] + } + + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { + try await stat("curr_items").flatMap { Int($0) } + } + + func fetchTableDDL(table: String, schema: String?) async throws -> String { + "// Memcached keys\n// \(listingComment)\n// Use 'get ' to read a value" + } + + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + throw MemcachedError.refused(String(localized: "Memcached has no views")) + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + let count = try await stat("curr_items").flatMap { Int64($0) } + return PluginTableMetadata(tableName: table, rowCount: count, comment: listingComment, engine: "Memcached") + } + + // MARK: - Grid + + func buildBrowseQuery( + table: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int + ) -> String? { + MemcachedBrowseSpec(limit: limit, offset: offset, ascending: Self.ascending(sortColumns)).statement + } + + func buildFilteredQuery( + table: String, + schema: String?, + queryFilters filters: [PluginQueryFilter], + logicMode: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int, + columnKinds: [String: PluginColumnKind] + ) -> String? { + do { + var conditions: [MemcachedBrowseSpec.KeyCondition] = [] + for filter in filters { + conditions.append(try MemcachedBrowseSpec.condition( + column: filter.column, op: filter.op, value: filter.value, isCaseSensitive: filter.isCaseSensitive + )) + } + return MemcachedBrowseSpec( + limit: limit, offset: offset, ascending: Self.ascending(sortColumns), + conditions: conditions, matchesAll: logicMode.lowercased() != "or" + ).statement + } catch { + return MemcachedBrowseSpec.refusalStatement(MemcachedError.filter(error).pluginErrorMessage) + } + } + + private static func ascending(_ sortColumns: [(columnIndex: Int, ascending: Bool)]) -> Bool { + sortColumns.first { $0.columnIndex == 0 }?.ascending ?? true + } + + func defaultExportQuery(table: String) -> String? { + MemcachedBrowseSpec(limit: Self.listingLimit, offset: 0, ascending: true).statement + } + + func truncateTableStatements(table: String, schema: String?, cascade: Bool) -> [String]? { + ["flush_all"] + } + + func dropObjectStatement(name: String, objectType: String, schema: String?, cascade: Bool) -> String? { nil } + + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try MemcachedStatementGenerator(columns: columns).generateRowWrites( + from: changes, + insertedRowData: insertedRowData, + deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices + ) + } + + // MARK: - Helpers + + private static func cell(_ data: Data) -> PluginCellValue { + String(data: data, encoding: .utf8).map(PluginCellValue.text) ?? .bytes(data) + } + + private func result( + columns: [String], types: [String], rows: [PluginRow], affected: Int = 0, started: Date + ) -> PluginQueryResult { + PluginQueryResult( + columns: columns, + columnTypeNames: types, + rows: rows, + rowsAffected: affected, + executionTime: Date().timeIntervalSince(started) + ) + } +} diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift new file mode 100644 index 0000000000..b717538998 --- /dev/null +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift @@ -0,0 +1,132 @@ +import Foundation +import TableProMemcachedCore +import TableProPluginKit + +/// Turns grid edits into console commands: `set` for a new row or a new value or flags, `touch` for a TTL alone, +/// `delete` for a removed row. The key names the item, so it is never rewritten in place. +struct MemcachedStatementGenerator { + let columns: [String] + var now: () -> Int64 = { Int64(Date().timeIntervalSince1970) } + + func generateRowWrites( + from changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) throws -> [PluginRowWrite] { + var writes: [PluginRowWrite] = [] + for change in changes { + let statement: String? + switch change.type { + case .insert: + guard insertedRowIndices.contains(change.rowIndex) else { continue } + statement = try insertStatement(for: change, row: insertedRowData[change.rowIndex]) + case .update: + statement = try updateStatement(for: change) + case .delete: + guard deletedRowIndices.contains(change.rowIndex) else { continue } + statement = MemcachedStatements.delete(key: try originalKey(of: change)) + } + if let statement { writes.append(PluginRowWrite(statement: statement, rowIndices: [change.rowIndex])) } + } + return writes + } + + private func insertStatement(for change: PluginRowChange, row: [PluginCellValue]?) throws -> String { + var cells: [MemcachedColumn: PluginCellValue] = [:] + if let row { + for column in MemcachedColumn.allCases { + if let index = columns.firstIndex(of: column.rawValue), index < row.count { cells[column] = row[index] } + } + } else { + for cell in change.cellChanges { + if let column = MemcachedColumn(rawValue: cell.columnName) { cells[column] = cell.newValue } + } + } + guard let key = cells[.key]?.asText, !key.isEmpty else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String(localized: "A new key needs a name.")) + } + try validate(key: key, rowIndex: change.rowIndex) + return setStatement( + key: key, + flags: try number(cells[.flags], of: .flags, rowIndex: change.rowIndex) ?? 0, + ttl: try number(cells[.ttl], of: .ttl, rowIndex: change.rowIndex), + value: cells[.value] ?? .null + ) + } + + private func updateStatement(for change: PluginRowChange) throws -> String? { + guard !change.cellChanges.isEmpty else { return nil } + let key = try originalKey(of: change) + var edited: [MemcachedColumn: PluginCellValue] = [:] + for cell in change.cellChanges { + guard let column = MemcachedColumn(rawValue: cell.columnName) else { continue } + guard column != .key else { + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String(localized: "A key cannot be renamed. Add a row with the new key instead.") + ) + } + edited[column] = cell.newValue + } + let ttl = try number(edited[.ttl] ?? original(.ttl, of: change), of: .ttl, rowIndex: change.rowIndex) + guard edited[.value] != nil || edited[.flags] != nil else { + return edited[.ttl] == nil ? nil : MemcachedStatements.touch(key: key, exptime: exptime(forTTL: ttl)) + } + return setStatement( + key: key, + flags: try number(edited[.flags] ?? original(.flags, of: change), of: .flags, rowIndex: change.rowIndex) ?? 0, + ttl: ttl, + value: edited[.value] ?? original(.value, of: change) ?? .null + ) + } + + private func setStatement(key: String, flags: Int64, ttl: Int64?, value: PluginCellValue) -> String { + let flags = UInt32(flags) + if case .bytes(let data) = value { + return MemcachedStatements.set(key: key, flags: flags, exptime: exptime(forTTL: ttl), bytes: data) + } + return MemcachedStatements.set(key: key, flags: flags, exptime: exptime(forTTL: ttl), value: value.asText ?? "") + } + + private func exptime(forTTL ttl: Int64?) -> Int64 { + MemcachedStatements.exptime(forTTL: ttl, now: now()) + } + + private func number(_ cell: PluginCellValue?, of column: MemcachedColumn, rowIndex: Int) throws -> Int64? { + guard let text = cell?.asText?.trimmingCharacters(in: .whitespaces), !text.isEmpty else { return nil } + let limit: ClosedRange = column == .flags ? 0 ... Int64(UInt32.max) : Int64.min ... Int64.max + guard let value = Int64(text), limit.contains(value) else { + throw PluginRowWriteRefusal( + rowIndex: rowIndex, + reason: String(format: String(localized: "%@ must be a whole number."), column.rawValue) + ) + } + return value + } + + private func validate(key: String, rowIndex: Int) throws { + guard MemcachedCommandParser.isValidKey(Array(key.utf8)) else { + throw PluginRowWriteRefusal( + rowIndex: rowIndex, + reason: String(localized: "A key is 1 to 250 bytes with no spaces or control characters.") + ) + } + } + + private func originalKey(of change: PluginRowChange) throws -> String { + guard let key = original(.key, of: change)?.asText, MemcachedCommandParser.isValidKey(Array(key.utf8)) else { + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String(localized: "This row's key cannot be addressed from the grid.") + ) + } + return key + } + + private func original(_ column: MemcachedColumn, of change: PluginRowChange) -> PluginCellValue? { + guard let index = columns.firstIndex(of: column.rawValue), + let row = change.originalRow, index < row.count else { return nil } + return row[index] + } +} diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MemcachedDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MemcachedDefaults.swift new file mode 100644 index 0000000000..c77d8f12a1 --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MemcachedDefaults.swift @@ -0,0 +1,64 @@ +import Foundation +import TableProPluginKit + +extension PluginMetadataRegistry { + func memcachedPluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { + [ + ("Memcached", PluginMetadataSnapshot( + displayName: "Memcached", iconName: "memorychip", defaultPort: 11_211, + requiresAuthentication: false, supportsForeignKeys: false, supportsSchemaEditing: false, + isDownloadable: true, primaryUrlScheme: "memcached", parameterStyle: .questionMark, + navigationModel: .standard, explainVariants: [], pathFieldRole: .database, + supportsHealthMonitor: true, urlSchemes: ["memcached"], postConnectActions: [], + brandColorHex: "#2A7AB0", + queryLanguageName: "Memcached", editorLanguage: .bash, + connectionMode: .network, supportsDatabaseSwitching: false, + capabilities: PluginMetadataSnapshot.CapabilityFlags( + supportsSchemaSwitching: false, + supportsImport: false, + supportsExport: true, + supportsSSH: true, + supportsSSL: true, + supportsCascadeDrop: false, + supportsForeignKeyDisable: false, + supportsReadOnlyMode: true, + supportsQueryProgress: false, + requiresReconnectForDatabaseSwitch: false, + supportsDropDatabase: false, + supportsOpportunisticTLS: false, + supportsColumnSort: false + ), + schema: PluginMetadataSnapshot.SchemaInfo( + defaultSchemaName: "", + defaultGroupName: "default", + tableEntityName: "Keys", + containerEntityName: "Server", + defaultPrimaryKeyColumn: "Key", + immutableColumns: ["Key"], + systemDatabaseNames: [], + systemSchemaNames: [], + fileExtensions: [], + databaseGroupingStrategy: .flat, + structureColumnFields: [.name, .type, .nullable] + ), + editor: PluginMetadataSnapshot.EditorConfig( + sqlDialect: nil, + statementCompletions: memcachedCompletions, + columnTypesByCategory: ["String": ["String"], "Integer": ["Int64"]] + ), + connection: PluginMetadataSnapshot.ConnectionConfig( + additionalConnectionFields: [], + category: .keyValue, + tagline: String(localized: "In-memory key-value cache"), + hidesBuiltInDatabase: true + ) + )) + ] + } +} + +private let memcachedCompletions: [CompletionEntry] = [ + "get", "gets", "gat", "set", "add", "replace", "append", "prepend", "cas", "touch", "delete", "incr", "decr", +].map { CompletionEntry(label: $0, insertText: "\($0) ") } + [ + "flush_all", "stats", "stats items", "stats slabs", "stats settings", "version", "lru_crawler metadump all", +].map { CompletionEntry(label: $0, insertText: $0) } diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 1df518f949..c9b00d26a5 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -1046,7 +1046,7 @@ extension PluginMetadataRegistry { + cloudPluginDefaults() + elasticsearchPluginDefaults() + surrealDBPluginDefaults() + kafkaPluginDefaults() + typesensePluginDefaults() + r2SQLPluginDefaults() + weaviatePluginDefaults() + hanaPluginDefaults() + firebasePluginDefaults() + bigtablePluginDefaults() - + trinoPluginDefaults() + + trinoPluginDefaults() + memcachedPluginDefaults() } // swiftlint:enable function_body_length } diff --git a/apple/TablePro/Core/ServerDashboard/Providers/MemcachedDashboardProvider.swift b/apple/TablePro/Core/ServerDashboard/Providers/MemcachedDashboardProvider.swift new file mode 100644 index 0000000000..80cd0c9a99 --- /dev/null +++ b/apple/TablePro/Core/ServerDashboard/Providers/MemcachedDashboardProvider.swift @@ -0,0 +1,85 @@ +// +// MemcachedDashboardProvider.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +/// Server metrics for Memcached, read from `stats`, which the driver returns as a Name and Value grid. +struct MemcachedDashboardProvider: ServerDashboardQueryProvider { + let supportedPanels: Set = [.serverMetrics] + + func fetchMetrics(execute: (String) async throws -> QueryResult) async throws -> [DashboardMetric] { + let stats = Self.stats(from: try await execute("stats")) + func number(_ name: String) -> Int64? { stats[name].flatMap { Int64($0) } } + + var metrics: [DashboardMetric] = [] + if let used = number("bytes") { + let limit = number("limit_maxbytes") ?? 0 + metrics.append(DashboardMetric( + id: "memory", + label: String(localized: "Used Memory"), + value: ByteSizeFormatting.string(bytes: used), + unit: limit > 0 ? Self.share(used, of: limit) : "", + icon: "memorychip" + )) + } + if let items = stats["curr_items"] { + metrics.append(DashboardMetric( + id: "items", label: String(localized: "Items"), value: items, unit: "", icon: "key" + )) + } + if let hits = number("get_hits"), let misses = number("get_misses") { + let lookups = Double(hits + misses) + metrics.append(DashboardMetric( + id: "hit_ratio", + label: String(localized: "Hit Ratio"), + value: lookups > 0 ? String(format: "%.1f", Double(hits) / lookups * 100) : "0", + unit: "%", + icon: "bolt" + )) + } + if let evictions = stats["evictions"] { + metrics.append(DashboardMetric( + id: "evictions", label: String(localized: "Evictions"), value: evictions, unit: "", icon: "arrow.uturn.left" + )) + } + if let connections = stats["curr_connections"] { + metrics.append(DashboardMetric( + id: "connections", + label: String(localized: "Connections"), + value: connections, + unit: "", + icon: "person.2" + )) + } + if let uptime = number("uptime") { + metrics.append(DashboardMetric( + id: "uptime", + label: String(localized: "Uptime"), + value: DurationFormatting.string(seconds: Int(uptime)), + unit: "", + icon: "clock" + )) + } + return metrics + } + + func acceptsProcessId(_ processId: String) -> Bool { + false + } + + private static func stats(from result: QueryResult) -> [String: String] { + var stats: [String: String] = [:] + for row in result.rows where row.count >= 2 { + if let name = row[0].asText { stats[name] = row[1].asText } + } + return stats + } + + private static func share(_ used: Int64, of limit: Int64) -> String { + let share = (Double(used) / Double(limit)).formatted(.percent.precision(.fractionLength(0))) + return String(format: String(localized: "%@ used"), share) + } +} diff --git a/apple/TablePro/Core/ServerDashboard/ServerDashboardQueryProviderFactory.swift b/apple/TablePro/Core/ServerDashboard/ServerDashboardQueryProviderFactory.swift index 03c6fe62f4..2c34f93633 100644 --- a/apple/TablePro/Core/ServerDashboard/ServerDashboardQueryProviderFactory.swift +++ b/apple/TablePro/Core/ServerDashboard/ServerDashboardQueryProviderFactory.swift @@ -48,6 +48,8 @@ enum ServerDashboardQueryProviderFactory { return MongoDBDashboardProvider() case .redis: return RedisDashboardProvider(serverVersion: serverVersion) + case .memcached: + return MemcachedDashboardProvider() } } } @@ -64,6 +66,7 @@ private enum DashboardEngine { case oracle case mongodb case redis + case memcached init?(_ databaseType: DatabaseType) { switch databaseType { @@ -87,6 +90,8 @@ private enum DashboardEngine { self = .mongodb case .redis: self = .redis + case .memcached: + self = .memcached default: return nil } diff --git a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift index dbc89e62f4..e75a0d2393 100644 --- a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift +++ b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift @@ -176,6 +176,7 @@ internal enum ExternalStatementGate { "ELASTICSEARCH_EXPORT:", "ELASTICSEARCH_SEARCH:", "ELASTICSEARCH_WRITE:", "ETCD_COUNT:", "ETCD_RANGE:", "ETCD_REFUSAL:", "FIREBASE_BROWSE:", + "MEMCACHED_BROWSE:", "MEMCACHED_REFUSAL:", "SPANNER_BROWSE:", "TYPESENSE_EXPORT:", "TYPESENSE_SEARCH:", "TYPESENSE_WRITE:", "WEAVIATE_EXPORT:", "WEAVIATE_SEARCH:", "WEAVIATE_WRITE:", diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift new file mode 100644 index 0000000000..7118db953e --- /dev/null +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift @@ -0,0 +1,27 @@ +// +// QueryClassifier+Memcached.swift +// TablePro +// + +import Foundation + +extension QueryClassifier { + private static let memcachedReadCommands: Set = ["get", "gets", "version"] + private static let memcachedDestructiveCommands: Set = ["delete", "flush_all"] + + /// `stats reset` clears the counters and `stats detail on` changes what the server records; every other + /// `stats` form only reads, as does `lru_crawler metadump`. `gat` reads a value and moves its expiry, so it writes. + static func memcachedClassification(_ trimmed: String) -> QueryClassification { + if trimmed.hasPrefix("MEMCACHED_BROWSE:") || trimmed.hasPrefix("MEMCACHED_REFUSAL:") { return .safe } + let words = trimmed.split(whereSeparator: \.isWhitespace).map { $0.lowercased() } + guard let command = words.first else { return .safe } + if command == "stats" { + let changesServer = words.dropFirst().first.map { $0 == "reset" || $0 == "detail" } ?? false + return changesServer ? QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) : .safe + } + if command == "lru_crawler", words.dropFirst().first == "metadump" { return .safe } + if memcachedReadCommands.contains(command) { return .safe } + let tier: QueryTier = memcachedDestructiveCommands.contains(command) ? .destructive : .write + return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: false) + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index 16b070d1cb..5942570922 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -1013,6 +1013,8 @@ private extension QueryClassifier { return worstRequest(typesenseClassification) case .weaviate: return worstRequest(weaviateClassification) + case .memcached: + return worstRequest(memcachedClassification) case .firestore, .firebaseRealtimeDatabase: return worstRequest { firebaseClassification($0, databaseType: databaseType) } case .bigtable: diff --git a/apple/TablePro/Models/Connection/DatabaseType.swift b/apple/TablePro/Models/Connection/DatabaseType.swift index bce99781a3..cb33d8b8aa 100644 --- a/apple/TablePro/Models/Connection/DatabaseType.swift +++ b/apple/TablePro/Models/Connection/DatabaseType.swift @@ -53,6 +53,7 @@ extension DatabaseType { static let trino = DatabaseType(rawValue: "Trino") static let presto = DatabaseType(rawValue: "Presto") static let weaviate = DatabaseType(rawValue: "Weaviate") + static let memcached = DatabaseType(rawValue: "Memcached") static let bigtable = DatabaseType(rawValue: "Bigtable") static let firestore = DatabaseType(rawValue: "Firestore") static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") diff --git a/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift b/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift index 1e7e0924a5..b841a3bb27 100644 --- a/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift +++ b/apple/TableProTests/Core/Execution/ExternalStatementGatePrivateRequestTests.swift @@ -75,6 +75,8 @@ struct ExternalStatementGatePrivateRequestTests { (.weaviate, "WEAVIATE_SEARCH:"), (.weaviate, "WEAVIATE_WRITE:"), (.weaviate, "WEAVIATE_EXPORT:"), + (.memcached, "MEMCACHED_BROWSE:"), + (.memcached, "MEMCACHED_REFUSAL:"), (.etcd, "ETCD_RANGE:"), (.etcd, "ETCD_COUNT:"), (.etcd, "ETCD_REFUSAL:"), diff --git a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift index e5d6ddaacd..5c33e970c2 100644 --- a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift @@ -10,7 +10,7 @@ struct PluginMetadataRegistryTypeCountTests { "CockroachDB", "Dameng", "Databend", "DuckDB", "DynamoDB", "Elasticsearch", "etcd", "Kafka", "KingbaseES", "libSQL", "MariaDB", "MongoDB", "MySQL", "Oracle", "PGlite", "PostgreSQL", "Redis", "Redshift", "SAP HANA", "ScyllaDB", "Snowflake", "Spanner", "SQL Server", "SQLite", "SurrealDB", "Teradata", "TiDB", - "Trino", "Presto", "Turso", "Typesense", "Weaviate", "OceanBase", "Firestore", "Firebase Realtime Database", "Bigtable" + "Trino", "Presto", "Turso", "Typesense", "Weaviate", "Memcached", "OceanBase", "Firestore", "Firebase Realtime Database", "Bigtable" ] private static func builtInTypeIds() -> Set { diff --git a/apple/TableProTests/Core/ServerDashboard/MemcachedDashboardProviderTests.swift b/apple/TableProTests/Core/ServerDashboard/MemcachedDashboardProviderTests.swift new file mode 100644 index 0000000000..ce576feba2 --- /dev/null +++ b/apple/TableProTests/Core/ServerDashboard/MemcachedDashboardProviderTests.swift @@ -0,0 +1,51 @@ +// +// MemcachedDashboardProviderTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +struct MemcachedDashboardProviderTests { + private static func stats(_ pairs: [(String, String)]) -> QueryResult { + QueryResult( + columns: ["Name", "Value"], + columnTypes: [], + rows: pairs.map { [.text($0.0), .text($0.1)] }, + rowsAffected: 0, + executionTime: 0, + error: nil + ) + } + + @Test("Metrics come from one stats call") + func metricsFromStats() async throws { + var commands: [String] = [] + let metrics = try await MemcachedDashboardProvider().fetchMetrics { command in + commands.append(command) + return Self.stats([ + ("uptime", "3600"), ("curr_connections", "4"), ("curr_items", "12"), ("bytes", "1048576"), + ("limit_maxbytes", "67108864"), ("get_hits", "3"), ("get_misses", "1"), ("evictions", "0"), + ]) + } + #expect(commands == ["stats"]) + let byId = Dictionary(uniqueKeysWithValues: metrics.map { ($0.id, $0) }) + #expect(byId["items"]?.value == "12") + #expect(byId["hit_ratio"]?.value == "75.0") + #expect(byId["connections"]?.value == "4") + #expect(byId["evictions"]?.value == "0") + #expect(byId["memory"] != nil) + #expect(byId["uptime"] != nil) + } + + @Test("No lookups yet reads as a 0 hit ratio, and missing stats are left out") + func sparseStats() async throws { + let metrics = try await MemcachedDashboardProvider().fetchMetrics { _ in + Self.stats([("get_hits", "0"), ("get_misses", "0")]) + } + #expect(metrics.map(\.id) == ["hit_ratio"]) + #expect(metrics.first?.value == "0") + } +} diff --git a/apple/TableProTests/Core/ServerDashboard/ServerDashboardQueryProviderFactoryTests.swift b/apple/TableProTests/Core/ServerDashboard/ServerDashboardQueryProviderFactoryTests.swift index 7cfbc7f656..268e5f3732 100644 --- a/apple/TableProTests/Core/ServerDashboard/ServerDashboardQueryProviderFactoryTests.swift +++ b/apple/TableProTests/Core/ServerDashboard/ServerDashboardQueryProviderFactoryTests.swift @@ -34,7 +34,7 @@ struct ServerDashboardQueryProviderFactoryTests { func serverEnginesHaveOne() { let types: [DatabaseType] = [ .postgresql, .kingbasees, .mysql, .mariadb, .redshift, .cockroachdb, .mssql, .clickhouse, .oracle, .mongodb, - .redis, + .redis, .memcached, ] for type in types { #expect(ServerDashboardQueryProviderFactory.supportsDashboard(for: type), "\(type.rawValue)") diff --git a/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift b/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift index d5dabb8050..3e6dfbd3e4 100644 --- a/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift +++ b/apple/TableProTests/Core/Services/Query/AllTablesListingTests.swift @@ -28,7 +28,7 @@ struct AllTablesListingTests { } @Test("An engine that only highlights like MongoDB or Redis gets no shell command", arguments: [ - DatabaseType.elasticsearch, .typesense, .weaviate, .etcd, .cassandra + DatabaseType.elasticsearch, .typesense, .weaviate, .etcd, .memcached, .cassandra ]) func noListingForOtherEngines(databaseType: DatabaseType) { #expect(AllTablesListing.statement(databaseType: databaseType, pluginListing: nil) == nil) diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 1261179ed8..1d70afa92b 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -899,6 +899,22 @@ struct QueryClassifierNonSqlTests { #expect(!QueryClassifier.isWriteQuery(search, databaseType: .weaviate)) } + @Test("Memcached reads are safe, stores write, and delete and flush_all are destructive") + func memcachedTiers() { + #expect(!QueryClassifier.isWriteQuery("MEMCACHED_BROWSE:e30=", databaseType: .memcached)) + #expect(!QueryClassifier.isWriteQuery("get user:1 user:2", databaseType: .memcached)) + #expect(!QueryClassifier.isWriteQuery("stats items", databaseType: .memcached)) + #expect(!QueryClassifier.isWriteQuery("lru_crawler metadump all", databaseType: .memcached)) + #expect(QueryClassifier.classifyTier("stats reset", databaseType: .memcached) == .write) + #expect(QueryClassifier.classifyTier("lru_crawler disable", databaseType: .memcached) == .write) + #expect(QueryClassifier.classifyTier("set k 0 0 \"v\"", databaseType: .memcached) == .write) + #expect(QueryClassifier.classifyTier("gat 60 k", databaseType: .memcached) == .write) + #expect(QueryClassifier.classifyTier("touch k 0", databaseType: .memcached) == .write) + #expect(QueryClassifier.classifyTier("delete k", databaseType: .memcached) == .destructive) + #expect(QueryClassifier.classifyTier("FLUSH_ALL", databaseType: .memcached) == .destructive) + #expect(QueryClassifier.classifyTier("get k; flush_all", databaseType: .memcached) == .destructive) + } + @Test("Bigtable request lines are tiered by their verb, and a PUT that nulls a cell is destructive") func bigtableTiers() { #expect(!QueryClassifier.isWriteQuery("BIGTABLE_BROWSE:e30=", databaseType: .bigtable)) diff --git a/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift b/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift index 3e05253cff..d3dff2c4db 100644 --- a/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift +++ b/apple/TableProTests/Models/Query/QueryTabBaseQueryTests.swift @@ -51,7 +51,7 @@ struct QueryTabBaseQueryTests { /// Their plugins build the browse query. Without one the tab must not be handed another engine's /// command: either the plugin builds it or the build throws. @Test("An engine that highlights like MongoDB or Redis is not browsed with their commands", arguments: [ - DatabaseType.elasticsearch, .typesense, .weaviate, .etcd + DatabaseType.elasticsearch, .typesense, .weaviate, .etcd, .memcached ]) func otherEnginesNeverGetShellBrowseQueries(databaseType: DatabaseType) { let query = try? QueryTab.buildBaseTableQuery(tableName: "products", databaseType: databaseType) diff --git a/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift b/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift index 1cd4d7e40c..e5b239c795 100644 --- a/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift +++ b/apple/TableProTests/Views/Main/QueryParameterDetectionTests.swift @@ -14,7 +14,7 @@ extension GlobalStateSuites { arguments: [ (DatabaseType.mysql, true), (.postgresql, true), (.sqlite, true), (.elasticsearch, false), (.typesense, false), (.weaviate, false), (.mongodb, false), - (.redis, false), (.etcd, false), (.surrealdb, false), (.kafka, false) + (.redis, false), (.etcd, false), (.memcached, false), (.surrealdb, false), (.kafka, false) ] ) func onlySQLBindsNamedParameters(type: DatabaseType, binds: Bool) { diff --git a/apple/project.yml b/apple/project.yml index cbf751b9d2..001863705b 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -1069,6 +1069,21 @@ targets: - package: swift-nio-ssl product: NIOSSL + MemcachedDriverPlugin: + templates: [DriverPlugin] + templateAttributes: + folder: MemcachedDriverPlugin + principalClass: MemcachedPlugin + dependencies: + - package: TableProCore + product: TableProMemcachedCore + - package: swift-nio + product: NIOCore + - package: swift-nio + product: NIOPosix + - package: swift-nio-ssl + product: NIOSSL + KingbaseESDriverPlugin: templates: [DriverPlugin, PostgreSQLLinkage] templateAttributes: @@ -1302,6 +1317,7 @@ aggregateTargets: - KafkaDriverPlugin - KingbaseESDriverPlugin - LibSQLDriverPlugin + - MemcachedDriverPlugin - MSSQLDriver - MongoDBDriver - OracleDriver diff --git a/apple/scripts/probes/check-memcached.sh b/apple/scripts/probes/check-memcached.sh new file mode 100755 index 0000000000..ad430b9403 --- /dev/null +++ b/apple/scripts/probes/check-memcached.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Run the Memcached driver against a live server: connect, the auth file login, key listing through +# lru_crawler metadump and through the stats cachedump fallback, browse pages with values and TTLs, +# key filters, grid writes (set, touch, delete), console commands and stats. The probe writes keys +# under tablepro_probe: and deletes them after. +# +# Usage: +# scripts/probes/check-memcached.sh [host] [port] +# +# MEMCACHED_USER and MEMCACHED_PASSWORD log in to a server started with -Y . Start one with: +# docker run -d --name tablepro-memcached-probe -p 11211:11211 memcached:1.6 +# +# Exits 3 when no server answers. + +set -euo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-11211}" +HERE="$(cd "$(dirname "$0")" && pwd)" +APPLE="$(cd "$HERE/../.." && pwd)" + +if ! nc -z -w 3 "$HOST" "$PORT" 2>/dev/null; then + echo "No Memcached server answers at $HOST:$PORT." >&2 + exit 3 +fi + +WORK="${TMPDIR:-/tmp}/tablepro-memcached-probe" +rm -rf "$WORK/Sources" +mkdir -p "$WORK/Sources/probe" +cp "$HERE/memcached-probe.swift" "$WORK/Sources/probe/main.swift" +for file in "$APPLE"/Plugins/MemcachedDriverPlugin/*.swift; do + ln -s "$file" "$WORK/Sources/probe/$(basename "$file")" +done +cat > "$WORK/Package.swift" < [[String?]] { + result.rows.map { $0.map(\.asText) } +} + +let env = ProcessInfo.processInfo.environment +func refusal(user: String, password: String) async -> String? { + let attempt = MemcachedPluginDriver(config: DriverConnectionConfig( + host: env["MEMCACHED_HOST"] ?? "127.0.0.1", port: Int(env["MEMCACHED_PORT"] ?? "") ?? 11_211, + username: user, password: password, database: "" + )) + do { + try await attempt.connect() + attempt.disconnect() + return nil + } catch { + return error.localizedDescription + } +} +if let user = env["MEMCACHED_USER"], !user.isEmpty { + let missing = await refusal(user: "", password: "") + check(missing?.contains("requires a username") == true, "no credentials: \(missing ?? "connected")") + let wrong = await refusal(user: user, password: "wrong") + check(wrong?.contains("rejected") == true, "wrong password: \(wrong ?? "connected")") +} +let driver = MemcachedPluginDriver(config: DriverConnectionConfig( + host: env["MEMCACHED_HOST"] ?? "127.0.0.1", + port: Int(env["MEMCACHED_PORT"] ?? "") ?? 11_211, + username: env["MEMCACHED_USER"] ?? "", + password: env["MEMCACHED_PASSWORD"] ?? "", + database: "" +)) +try await driver.connect() +print("version", driver.serverVersion ?? "nil") +check(driver.serverVersion != nil, "connects and reads the version") + +let prefix = "tablepro_probe:" +for index in 0 ..< 30 { + _ = try await driver.execute(query: "set \(prefix)\(String(format: "%02d", index)) \(index) 0 \"value \(index)\"") +} +_ = try await driver.execute(query: "set \(prefix)ttl 0 600 \"expires\"") +_ = try await driver.execute(query: "set \(prefix)multi 0 0 \"line one\\nline two \\\"quoted\\\"\"") + +let tables = try await driver.fetchTables(schema: nil) +print("listing", tables.first?.comment ?? "") +check(tables.map(\.name) == ["keys"], "one keys table") + +let condition = try MemcachedBrowseSpec.condition(column: "Key", op: "STARTS WITH", value: prefix, isCaseSensitive: true) +let firstPage = try await driver.execute(query: MemcachedBrowseSpec( + limit: 10, offset: 0, ascending: true, conditions: [condition] +).statement) +check(firstPage.columns == ["Key", "Value", "TTL", "Flags"], "browse columns") +check(texts(firstPage).first == ["\(prefix)00", "value 0", nil, "0"], "first row sorted by key") +check(firstPage.rows.count == 10, "page of 10") + +let thirdPage = try await driver.execute(query: MemcachedBrowseSpec( + limit: 10, offset: 20, ascending: true, conditions: [condition] +).statement) +check(texts(thirdPage).first?.first == "\(prefix)20", "offset 20 starts at key 20") +check(texts(thirdPage).first?[3] == "20", "flags read back") + +let ttlRow = try await driver.execute(query: MemcachedBrowseSpec(limit: 10, offset: 0, ascending: true, conditions: [ + try MemcachedBrowseSpec.condition(column: "Key", op: "=", value: "\(prefix)ttl", isCaseSensitive: true), +]).statement) +let ttl = texts(ttlRow).first?[2].flatMap { Int($0) } ?? 0 +check(ttl > 590 && ttl <= 600, "TTL counts down from 600 (\(ttl))") + +let multi = try await driver.execute(query: "get \(prefix)multi") +check(texts(multi).first?[1] == "line one\nline two \"quoted\"", "escaped value round-trips") + +let refusal = driver.buildFilteredQuery( + table: "keys", schema: nil, queryFilters: [PluginQueryFilter(column: "Value", op: "=", value: "x")], + logicMode: "and", sortColumns: [], columns: [], limit: 10, offset: 0, columnKinds: [:] +) ?? "" +do { + _ = try await driver.execute(query: refusal) + check(false, "a Value filter is refused") +} catch { + check("\(error.localizedDescription)".contains("Key"), "a Value filter is refused: \(error.localizedDescription)") +} + +let columns = ["Key", "Value", "TTL", "Flags"] +let original: [PluginCellValue] = [.text("\(prefix)05"), .text("value 5"), .null, .text("5")] +let writes = try driver.generateRowWrites( + table: "keys", schema: nil, columns: columns, primaryKeyColumns: ["Key"], + changes: [ + PluginRowChange(rowIndex: 0, type: .update, cellChanges: [(1, "Value", .text("value 5"), .text("edited"))], originalRow: original), + PluginRowChange(rowIndex: 1, type: .update, cellChanges: [(2, "TTL", .null, .text("120"))], originalRow: [ + .text("\(prefix)06"), .text("value 6"), .null, .text("6"), + ]), + PluginRowChange(rowIndex: 2, type: .delete, cellChanges: [], originalRow: [.text("\(prefix)07"), .text("value 7"), .null, .text("7")]), + PluginRowChange(rowIndex: 3, type: .insert, cellChanges: [], originalRow: nil), + ], + insertedRowData: [3: [.text("\(prefix)new"), .text("inserted"), .text("60"), .text("9")]], + deletedRowIndices: [2], insertedRowIndices: [3] +) ?? [] +for write in writes { _ = try await driver.execute(query: write.statement) } +let afterWrites = try await driver.execute(query: "get \(prefix)05 \(prefix)06 \(prefix)07 \(prefix)new") +let byKey = Dictionary(uniqueKeysWithValues: texts(afterWrites).map { ($0[0] ?? "", $0) }) +check(byKey["\(prefix)05"]?[1] == "edited" && byKey["\(prefix)05"]?[2] == "5", "set keeps flags") +check(byKey["\(prefix)06"]?[1] == "value 6", "touch keeps the value") +check(byKey["\(prefix)07"] == nil, "delete removes the key") +check(byKey["\(prefix)new"]?[1] == "inserted" && byKey["\(prefix)new"]?[2] == "9", "insert sets value and flags") +// A metadump walks the LRU while the server moves items between its segments, so an item bumped by the touch can +// be missed by one walk and listed by the next. +var ttls: [Int] = [] +for _ in 0 ..< 5 where ttls.count < 2 { + let touched = try await driver.execute(query: MemcachedBrowseSpec(limit: 10, offset: 0, ascending: true, conditions: [ + try MemcachedBrowseSpec.condition(column: "Key", op: "IN", value: "\(prefix)06,\(prefix)new", isCaseSensitive: true), + ]).statement) + ttls = texts(touched).map { Int($0[2] ?? "") ?? -1 } +} +check(ttls.count == 2 && ttls[0] > 110 && ttls[0] <= 120 && ttls[1] > 50 && ttls[1] <= 60, "TTLs from touch and insert \(ttls)") + +let incr = try await driver.execute(query: "set \(prefix)counter 0 0 \"41\"") +check(texts(incr).first?.first == "STORED", "set replies STORED") +let counted = try await driver.execute(query: "incr \(prefix)counter 1") +check(texts(counted).first?.first == "42", "incr replies the new value") +let stats = try await driver.execute(query: "stats") +check(stats.columns == ["Name", "Value"] && texts(stats).contains { $0.first == "curr_items" }, "stats as a grid") +do { + let metadump = try await driver.execute(query: "lru_crawler metadump all") + check(metadump.rows.count >= 30, "metadump lists the probe's keys (\(metadump.rows.count) rows)") +} catch { + check(tables.first?.comment?.contains("cachedump") == true, "metadump refused where the listing uses cachedump: \(error.localizedDescription)") +} +do { + _ = try await driver.execute(query: "set bad key 0 0 v") + check(false, "a bad statement is refused") +} catch { + check(true, "a bad statement is refused: \(error.localizedDescription)") +} +let notStored = try await driver.execute(query: "add \(prefix)00 0 0 \"x\"") +check(texts(notStored).first?.first == "NOT_STORED", "add on an existing key replies NOT_STORED") + +let keys = try await driver.execute(query: MemcachedBrowseSpec( + limit: 100, offset: 0, ascending: true, conditions: [condition] +).statement) +for row in texts(keys) { _ = try await driver.execute(query: "delete \(row[0] ?? "")") } +try await driver.ping() +driver.disconnect() + +print(failures == 0 ? "ALL PASS" : "\(failures) FAILED") +exit(failures == 0 ? 0 : 1) From 2f3ccc6bf69730f397ea0c144b8ff3672c71a34d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:33:31 +0700 Subject: [PATCH 2/6] fix(plugin-memcached): log in only when the server asks, quote keys, read stats replies by form --- .../MemcachedCommand.swift | 19 +++- .../MemcachedReplyReader.swift | 4 +- .../MemcachedStatements.swift | 8 +- .../MemcachedCommandParserTests.swift | 25 +++++ .../MemcachedReplyReaderTests.swift | 8 +- .../MemcachedConnection.swift | 25 ++++- .../MemcachedError.swift | 3 + .../MemcachedPluginDriver.swift | 8 ++ .../SQL/QueryClassifier+Memcached.swift | 29 ++++-- .../TablePro/Resources/Localizable.xcstrings | 94 +++++++++++++++++++ .../SQL/QueryClassifierHardeningTests.swift | 6 ++ apple/scripts/probes/memcached-probe.swift | 19 +++- 12 files changed, 225 insertions(+), 23 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift index c670df9240..78e4a26736 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift @@ -23,11 +23,23 @@ public enum MemcachedCommand: Equatable, Sendable { public var replyShape: MemcachedReplyShape { switch self { case .get, .getAndTouch: return .values - case .stats, .metadump: return .lines + case .stats(let arguments): return Self.statsChangingServer(arguments) ? .status : .lines + case .metadump: return .lines default: return .status } } + /// `stats reset`, `stats detail on|off` and `stats sizes_enable|sizes_disable` change the server and answer one + /// status line with no `END`. Every other `stats` form reads. + public static func statsChangingServer(_ arguments: [String]) -> Bool { + let words = arguments.map { $0.lowercased() } + switch words.first { + case "reset", "sizes_enable", "sizes_disable": return true + case "detail": return words.count > 1 && words[1] != "dump" + default: return false + } + } + /// The bytes the text protocol takes for this command, data block included. public var wireBytes: [UInt8] { func line(_ words: [String]) -> [UInt8] { Array((words.joined(separator: " ") + "\r\n").utf8) } @@ -144,6 +156,11 @@ public enum MemcachedCommandParser { escaped(Array(value.utf8), keepingUTF8: true) } + /// A key as one argument: as written unless it holds a quote or backslash, which the tokenizer would read. + public static func argument(_ key: String) -> String { + key.contains(where: { $0 == "\"" || $0 == "'" || $0 == "\\" }) ? quoted(key) : key + } + /// Writes bytes that may not be UTF-8, every byte past ASCII as `\xHH`. public static func quoted(bytes: Data) -> String { escaped(Array(bytes), keepingUTF8: false) diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift index 57a9282892..e90c657a49 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift @@ -69,6 +69,8 @@ public struct MemcachedReplyReader: Sendable { private var pendingValue: (key: String, flags: UInt32, length: Int, unique: UInt64?)? private static let maximumLineLength = 64 * 1_024 + /// Memcached's largest item size setting, `-I 1024m`. A longer data block is not a Memcached reply. + public static let maximumValueLength = 1 << 30 public init(shape: MemcachedReplyShape, lineLimit: Int = .max) { self.shape = shape @@ -141,7 +143,7 @@ public struct MemcachedReplyReader: Sendable { private static func valueHeader(_ line: String) throws -> (key: String, flags: UInt32, length: Int, unique: UInt64?) { let parts = line.split(separator: " ", omittingEmptySubsequences: true) guard parts.count == 4 || parts.count == 5, parts[0] == "VALUE", - let flags = UInt32(parts[2]), let length = Int(parts[3]), length >= 0 else { + let flags = UInt32(parts[2]), let length = Int(parts[3]), (0 ... maximumValueLength).contains(length) else { throw MemcachedProtocolError.malformedLine(line) } let unique = parts.count == 5 ? UInt64(parts[4]) : nil diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift index e520906283..e496880a26 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift @@ -12,18 +12,18 @@ public enum MemcachedStatements { } public static func set(key: String, flags: UInt32, exptime: Int64, value: String) -> String { - "set \(key) \(flags) \(exptime) \(MemcachedCommandParser.quoted(value))" + "set \(MemcachedCommandParser.argument(key)) \(flags) \(exptime) \(MemcachedCommandParser.quoted(value))" } public static func set(key: String, flags: UInt32, exptime: Int64, bytes: Data) -> String { - "set \(key) \(flags) \(exptime) \(MemcachedCommandParser.quoted(bytes: bytes))" + "set \(MemcachedCommandParser.argument(key)) \(flags) \(exptime) \(MemcachedCommandParser.quoted(bytes: bytes))" } public static func touch(key: String, exptime: Int64) -> String { - "touch \(key) \(exptime)" + "touch \(MemcachedCommandParser.argument(key)) \(exptime)" } public static func delete(key: String) -> String { - "delete \(key)" + "delete \(MemcachedCommandParser.argument(key))" } } diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift index 737c955f83..b213c4fffc 100644 --- a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift @@ -101,3 +101,28 @@ struct MemcachedStatementsTests { #expect(try MemcachedCommandParser.parse(MemcachedStatements.delete(key: "k")) == .delete(key: "k")) } } + +struct MemcachedKeyQuotingTests { + @Test("A key holding a quote or backslash is quoted, so the statement addresses that key", arguments: [ + "user'1", "a\"b\"", "back\\slash", "plain:key", + ]) + func keysRoundTrip(key: String) throws { + #expect(try MemcachedCommandParser.parse(MemcachedStatements.delete(key: key)) == .delete(key: key)) + #expect(try MemcachedCommandParser.parse(MemcachedStatements.touch(key: key, exptime: 5)) == .touch(key: key, exptime: 5)) + #expect(try MemcachedCommandParser.parse(MemcachedStatements.set(key: key, flags: 0, exptime: 0, value: "v")) + == .store(.set, key: key, flags: 0, exptime: 0, value: Data("v".utf8))) + } + + @Test("A plain key is written as it is") + func plainKey() { + #expect(MemcachedStatements.delete(key: "user:1") == "delete user:1") + } + + @Test("stats forms that change the server answer one status line", arguments: [ + ("stats reset", MemcachedReplyShape.status), ("stats detail on", .status), ("stats detail off", .status), + ("stats sizes_enable", .status), ("stats detail dump", .lines), ("stats items", .lines), ("stats", .lines), + ]) + func statsShapes(statement: String, shape: MemcachedReplyShape) throws { + #expect(try MemcachedCommandParser.parse(statement).replyShape == shape) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedReplyReaderTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedReplyReaderTests.swift index 2949dfbcaa..fde57f9fb4 100644 --- a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedReplyReaderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedReplyReaderTests.swift @@ -70,8 +70,10 @@ struct MemcachedReplyReaderTests { #expect(try read(.status, ["VERSION 1.6.45\r\n"]) == .status("VERSION 1.6.45")) } - @Test("A malformed VALUE header is a protocol error") - func malformedHeader() { - #expect(throws: MemcachedProtocolError.self) { try read(.values, ["VALUE a x 1\r\n"]) } + @Test("A malformed VALUE header, or one announcing more than memcached can store, is a protocol error", arguments: [ + "VALUE a x 1\r\n", "VALUE a 0 9223372036854775807\r\n", "VALUE a 0 -1\r\n", + ]) + func malformedHeader(header: String) { + #expect(throws: MemcachedProtocolError.self) { try read(.values, [header]) } } } diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift index 011a9c7e25..866c5be7fc 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift @@ -121,6 +121,12 @@ actor MemcachedConnection { private var handler: MemcachedReplyHandler? private var wireSlotIsTaken = false private var waitingForWireSlot: [CheckedContinuation] = [] + private var requestTimeout: TimeAmount? + + /// A reply that has not finished by then closes the connection, since the text protocol cannot skip it. + func setRequestTimeout(seconds: Int) { + requestTimeout = seconds > 0 ? .seconds(Int64(seconds)) : nil + } init(endpoint: MemcachedEndpoint) { self.endpoint = endpoint @@ -164,9 +170,15 @@ actor MemcachedConnection { } } - /// A server started with `-Y ` takes the user and password as the data of the first `set`. + /// A server started with `-Y ` refuses every command until a `set` carries the user and password. + /// Any other server would store them as an ordinary item, so they are sent only after a refusal. private func authenticate() async throws { - guard !endpoint.username.isEmpty else { return } + do { + _ = try await send(.version) + return + } catch let error as MemcachedServerError where error.needsAuthentication { + guard !endpoint.username.isEmpty else { throw MemcachedError.authenticationRequired } + } let login = MemcachedCommand.store( .set, key: "tablepro", flags: 0, exptime: 0, value: Data("\(endpoint.username) \(endpoint.password)".utf8) ) @@ -208,7 +220,7 @@ actor MemcachedConnection { /// Sends one command and reads its reply. Cancelling closes the channel: once a request is written the text /// protocol has no way to skip its reply, so the connection cannot be reused for the next one. - func send(_ command: MemcachedCommand, lineLimit: Int = .max) async throws -> MemcachedReply { + func send(_ command: MemcachedCommand, lineLimit: Int = 1_000_000) async throws -> MemcachedReply { await claimWireSlot() defer { releaseWireSlot() } @@ -218,6 +230,13 @@ actor MemcachedConnection { let bytes = command.wireBytes var buffer = channel.allocator.buffer(capacity: bytes.count) buffer.writeBytes(bytes) + let deadline = requestTimeout.map { timeout in + channel.eventLoop.scheduleTask(in: timeout) { + handler.finish(with: MemcachedError.timedOut) + channel.close(promise: nil) + } + } + defer { deadline?.cancel() } return try await withTaskCancellationHandler { try await withCheckedThrowingContinuation { continuation in guard handler.expect(command.replyShape, lineLimit: lineLimit, continuation: continuation) else { return } diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift index 43c25be009..813c3d6258 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift @@ -5,6 +5,7 @@ import TableProPluginKit enum MemcachedError: Error, PluginDriverError { case notConnected case connectionClosed + case timedOut case connectFailed(String) case authenticationRequired case authenticationFailed @@ -20,6 +21,8 @@ enum MemcachedError: Error, PluginDriverError { return String(localized: "Not connected to Memcached") case .connectionClosed: return String(localized: "Memcached closed the connection") + case .timedOut: + return String(localized: "Memcached did not answer in time") case .connectFailed(let reason): return String(format: String(localized: "Could not connect to Memcached: %@"), reason) case .authenticationRequired: diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift index c538b10414..4891e4a8ec 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -19,6 +19,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var connection: MemcachedConnection? private var _serverVersion: String? private var keySource: MemcachedKeySource = .metadump + private var queryTimeout = 0 init(config: DriverConnectionConfig) { endpoint = MemcachedEndpoint( @@ -48,6 +49,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let opened = MemcachedConnection(endpoint: endpoint) do { try await opened.open() + await opened.setRequestTimeout(seconds: lock.withLock { queryTimeout }) let version = try await Self.version(on: opened) let source = try await Self.keySource(on: opened) lock.withLock { @@ -74,6 +76,11 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { _ = try await Self.version(on: try liveConnection()) } + func applyQueryTimeout(_ seconds: Int) async throws { + lock.withLock { queryTimeout = seconds } + await lock.withLock({ connection })?.setRequestTimeout(seconds: seconds) + } + private func liveConnection() throws -> MemcachedConnection { guard let connection = lock.withLock({ connection }) else { throw MemcachedError.notConnected } return connection @@ -246,6 +253,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let listing = MemcachedConnection(endpoint: endpoint) do { try await listing.open() + await listing.setRequestTimeout(seconds: lock.withLock { queryTimeout }) let value = try await body(listing) await listing.close() return value diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift index 7118db953e..6792887c72 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift @@ -8,20 +8,29 @@ import Foundation extension QueryClassifier { private static let memcachedReadCommands: Set = ["get", "gets", "version"] private static let memcachedDestructiveCommands: Set = ["delete", "flush_all"] + private static let memcachedReadingStats: Set = ["items", "slabs", "settings", "conns", "sizes", "cachedump"] - /// `stats reset` clears the counters and `stats detail on` changes what the server records; every other - /// `stats` form only reads, as does `lru_crawler metadump`. `gat` reads a value and moves its expiry, so it writes. + /// Only the `stats` forms that read are safe: `stats reset` clears the counters and `stats detail on` changes + /// what the server records. `gat` reads a value and moves its expiry, so it writes. The driver strips quotes + /// before it sends a word, so they are stripped here too. static func memcachedClassification(_ trimmed: String) -> QueryClassification { if trimmed.hasPrefix("MEMCACHED_BROWSE:") || trimmed.hasPrefix("MEMCACHED_REFUSAL:") { return .safe } - let words = trimmed.split(whereSeparator: \.isWhitespace).map { $0.lowercased() } + let words = trimmed.split(whereSeparator: \.isWhitespace).map { + $0.lowercased().filter { $0 != "\"" && $0 != "'" } + } guard let command = words.first else { return .safe } - if command == "stats" { - let changesServer = words.dropFirst().first.map { $0 == "reset" || $0 == "detail" } ?? false - return changesServer ? QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) : .safe + let subcommand = words.dropFirst().first + switch command { + case "stats": + guard let subcommand, !memcachedReadingStats.contains(subcommand) else { return .safe } + if subcommand == "detail", words.dropFirst(2).first == "dump" { return .safe } + return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + case "lru_crawler" where subcommand == "metadump": + return .safe + default: + if memcachedReadCommands.contains(command) { return .safe } + let tier: QueryTier = memcachedDestructiveCommands.contains(command) ? .destructive : .write + return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: false) } - if command == "lru_crawler", words.dropFirst().first == "metadump" { return .safe } - if memcachedReadCommands.contains(command) { return .safe } - let tier: QueryTier = memcachedDestructiveCommands.contains(command) ? .destructive : .write - return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: false) } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 4b999ea5d9..e3d437c638 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -656,6 +656,18 @@ } } }, + "'%@' is not a valid key: a key is 1 to 250 bytes with no spaces or control characters" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "'%@' is not a valid number here" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "'%@' is not a valid regular expression" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "'%@' is not a valid regular expression." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -4303,6 +4315,8 @@ } }, "%@ must be a whole number." : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "fr" : { "stringUnit" : { @@ -15771,6 +15785,14 @@ } } }, + "A key cannot be renamed. Add a row with the new key instead." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "A key is 1 to 250 bytes with no spaces or control characters." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A key needs a name." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -16247,6 +16269,10 @@ } } }, + "A quoted value is missing its closing quote" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A quoted value is missing its closing quote." : { "extractionState" : "manual", "generatesSymbol" : false @@ -52367,6 +52393,10 @@ } } }, + "Could not connect to Memcached: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Could not connect to the database: %@" : { }, @@ -71638,6 +71668,10 @@ } } }, + "Enter a Memcached command" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Enter a name for the new table." : { "localizations" : { "fr" : { @@ -71823,6 +71857,10 @@ }, "Enter at least one value for %@." : { + }, + "Enter at least one value to filter with %@" : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Enter at least one value to filter with %@." : { "extractionState" : "manual", @@ -98756,6 +98794,14 @@ } } }, + "Keys listed with lru_crawler metadump" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Keys listed with stats cachedump, which returns a capped sample per slab class" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Keys only" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -108450,6 +108496,30 @@ } } }, + "Memcached can only filter by Key" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Memcached cannot filter keys with %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Memcached closed the connection" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Memcached did not answer in time" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Memcached has no views" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Memcached rejected the username or password" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Memory" : { "extractionState" : "manual", "generatesSymbol" : false @@ -121453,6 +121523,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Not connected to Memcached" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Not connected to MongoDB" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -197897,6 +197971,10 @@ } } }, + "This Memcached server requires a username and password" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This operation is not supported" : { "localizations" : { "fr" : { @@ -198542,6 +198620,10 @@ } } }, + "This row's key cannot be addressed from the grid." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This row's key cannot be matched, so it cannot be checked before writing." : { "localizations" : { "fr" : { @@ -204595,6 +204677,10 @@ } } }, + "Unexpected reply from Memcached: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Unexpected response from the SSO portal for profile \"%@\"." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -206446,6 +206532,10 @@ } } }, + "Unsupported Memcached command: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Unsupported MongoDB method: %@" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -207569,6 +207659,10 @@ } } }, + "Usage: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Use" : { "extractionState" : "stale", "localizations" : { diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 1d70afa92b..4a52593683 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -906,6 +906,12 @@ struct QueryClassifierNonSqlTests { #expect(!QueryClassifier.isWriteQuery("stats items", databaseType: .memcached)) #expect(!QueryClassifier.isWriteQuery("lru_crawler metadump all", databaseType: .memcached)) #expect(QueryClassifier.classifyTier("stats reset", databaseType: .memcached) == .write) + #expect(QueryClassifier.classifyTier("stats \"reset\"", databaseType: .memcached) == .write) + #expect(QueryClassifier.classifyTier("stats 'detail' on", databaseType: .memcached) == .write) + #expect(QueryClassifier.classifyTier("stats sizes_enable", databaseType: .memcached) == .write) + #expect(!QueryClassifier.isWriteQuery("stats detail dump", databaseType: .memcached)) + #expect(!QueryClassifier.isWriteQuery("stats cachedump 1 0", databaseType: .memcached)) + #expect(!QueryClassifier.isWriteQuery("stats", databaseType: .memcached)) #expect(QueryClassifier.classifyTier("lru_crawler disable", databaseType: .memcached) == .write) #expect(QueryClassifier.classifyTier("set k 0 0 \"v\"", databaseType: .memcached) == .write) #expect(QueryClassifier.classifyTier("gat 60 k", databaseType: .memcached) == .write) diff --git a/apple/scripts/probes/memcached-probe.swift b/apple/scripts/probes/memcached-probe.swift index 13b5b1992d..014012c9d3 100644 --- a/apple/scripts/probes/memcached-probe.swift +++ b/apple/scripts/probes/memcached-probe.swift @@ -30,6 +30,9 @@ func refusal(user: String, password: String) async -> String? { return error.localizedDescription } } +if env["MEMCACHED_USER"]?.isEmpty ?? true { + _ = await refusal(user: "probe-user", password: "probe-password") +} if let user = env["MEMCACHED_USER"], !user.isEmpty { let missing = await refusal(user: "", password: "") check(missing?.contains("requires a username") == true, "no credentials: \(missing ?? "connected")") @@ -143,13 +146,27 @@ do { } catch { check(true, "a bad statement is refused: \(error.localizedDescription)") } +let leaked = try await driver.execute(query: "get tablepro") +check(leaked.rows.isEmpty, "credentials given to an open server are not stored as an item") +let reset = try await driver.execute(query: "stats reset") +check(texts(reset).first?.first == "RESET", "stats reset answers one status line") +let quotedKey = "\(prefix)it's \"quoted\"" +let quotedWrites = try driver.generateRowWrites( + table: "keys", schema: nil, columns: columns, primaryKeyColumns: ["Key"], + changes: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], + insertedRowData: [0: [.text(quotedKey.replacingOccurrences(of: " ", with: "_")), .text("q"), .null, .null]], + deletedRowIndices: [], insertedRowIndices: [0] +) ?? [] +for write in quotedWrites { _ = try await driver.execute(query: write.statement) } +let quotedRead = try await driver.execute(query: "get \(MemcachedCommandParser.argument(quotedKey.replacingOccurrences(of: " ", with: "_")))") +check(texts(quotedRead).first?[1] == "q", "a key with quotes is written and read back") let notStored = try await driver.execute(query: "add \(prefix)00 0 0 \"x\"") check(texts(notStored).first?.first == "NOT_STORED", "add on an existing key replies NOT_STORED") let keys = try await driver.execute(query: MemcachedBrowseSpec( limit: 100, offset: 0, ascending: true, conditions: [condition] ).statement) -for row in texts(keys) { _ = try await driver.execute(query: "delete \(row[0] ?? "")") } +for row in texts(keys) { _ = try await driver.execute(query: MemcachedStatements.delete(key: row[0] ?? "")) } try await driver.ping() driver.disconnect() From ae4992c106ac1643c13a4d2b9f31605d1e0b9bb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:36:56 +0700 Subject: [PATCH 3/6] fix(plugin-memcached): keep an item's expiry on edit and bound pages and TTLs --- .../MemcachedBrowseSpec.swift | 6 +- .../MemcachedStatements.swift | 12 +++- .../MemcachedCommandParserTests.swift | 7 ++ .../MemcachedKeyListingTests.swift | 7 ++ .../MemcachedPluginDriver.swift | 18 ++++- .../MemcachedStatementGenerator.swift | 70 +++++++++++++------ .../TablePro/Resources/Localizable.xcstrings | 8 +++ apple/scripts/probes/memcached-probe.swift | 16 +++++ 8 files changed, 118 insertions(+), 26 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift index 5408d8d0be..bb65220078 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift @@ -75,7 +75,9 @@ public struct MemcachedBrowseSpec: Codable, Equatable, Sendable { guard statement.hasPrefix(tag), let data = Data(base64Encoded: String(statement.dropFirst(tag.count))) else { return nil } - return try? JSONDecoder().decode(MemcachedBrowseSpec.self, from: data) + guard let spec = try? JSONDecoder().decode(MemcachedBrowseSpec.self, from: data), + spec.limit >= 0, spec.offset >= 0 else { return nil } + return spec } public static func refusalStatement(_ message: String) -> String { @@ -104,7 +106,7 @@ public struct MemcachedBrowseSpec: Codable, Equatable, Sendable { let matching = try filter(keys) let sorted = matching.sorted { ascending ? $0.key < $1.key : $0.key > $1.key } guard offset < sorted.count else { return [] } - return Array(sorted[offset ..< min(sorted.count, offset + limit)]) + return Array(sorted[offset ..< offset + min(limit, sorted.count - offset)]) } public func filter(_ keys: [MemcachedKeyEntry]) throws(MemcachedFilterRefusal) -> [MemcachedKeyEntry] { diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift index e496880a26..dd91bb1722 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift @@ -5,10 +5,16 @@ public enum MemcachedStatements { /// Memcached reads an exptime above 30 days as a Unix time rather than seconds from now. public static let relativeExptimeLimit: Int64 = 60 * 60 * 24 * 30 - /// No TTL, or 0, keeps the item until it is evicted. A TTL past 30 days is sent as the time it ends. - public static func exptime(forTTL ttl: Int64?, now: Int64) -> Int64 { + /// The text protocol reads an exptime as a 32-bit number. + public static let maximumExptime = Int64(Int32.max) + + /// No TTL, or 0, keeps the item until it is evicted. A TTL past 30 days is sent as the time it ends. Nil when + /// the end falls past what the protocol can carry. + public static func exptime(forTTL ttl: Int64?, now: Int64) -> Int64? { guard let ttl, ttl != 0 else { return 0 } - return ttl > relativeExptimeLimit ? now + ttl : ttl + guard ttl > relativeExptimeLimit else { return ttl >= -maximumExptime ? ttl : nil } + let (end, overflow) = now.addingReportingOverflow(ttl) + return !overflow && end <= maximumExptime ? end : nil } public static func set(key: String, flags: UInt32, exptime: Int64, value: String) -> String { diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift index b213c4fffc..610ddcea26 100644 --- a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedCommandParserTests.swift @@ -91,6 +91,13 @@ struct MemcachedStatementsTests { == 1_000 + MemcachedStatements.relativeExptimeLimit + 1) } + @Test("A TTL whose end the protocol cannot carry is refused rather than overflowing", arguments: [ + Int64.max, Int64(Int32.max), Int64.min, + ]) + func hugeTTL(ttl: Int64) { + #expect(MemcachedStatements.exptime(forTTL: ttl, now: 1_793_500_000) == nil) + } + @Test("Each write statement parses back into its command") func statementsParse() throws { #expect(try MemcachedCommandParser.parse(MemcachedStatements.set(key: "k", flags: 3, exptime: 60, value: "a \"b\"\n")) diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift index 0b8b699fec..ff68599232 100644 --- a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift @@ -59,6 +59,13 @@ struct MemcachedBrowseSpecTests { #expect(MemcachedBrowseSpec.parse("get a") == nil) } + @Test("A spec with a negative bound is not a page, and a huge limit stops at the last key") + func bounds() throws { + #expect(MemcachedBrowseSpec.parse(MemcachedBrowseSpec(limit: -1, offset: 0, ascending: true).statement) == nil) + #expect(MemcachedBrowseSpec.parse(MemcachedBrowseSpec(limit: 10, offset: -5, ascending: true).statement) == nil) + #expect(try MemcachedBrowseSpec(limit: .max, offset: 1, ascending: true).page(of: keys).count == 3) + } + @Test("A page is filtered, sorted by key, then cut") func page() throws { let condition = try MemcachedBrowseSpec.condition(column: "Key", op: "STARTS WITH", value: "user:", isCaseSensitive: false) diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift index 4891e4a8ec..59617e2793 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -20,6 +20,9 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var _serverVersion: String? private var keySource: MemcachedKeySource = .metadump private var queryTimeout = 0 + private var shownExpiries: [String: MemcachedKnownExpiry] = [:] + private var serverClockOffset: Int64 = 0 + private static let shownExpiryLimit = 100_000 init(config: DriverConnectionConfig) { endpoint = MemcachedEndpoint( @@ -199,6 +202,12 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { start = end } + lock.withLock { + serverClockOffset = serverTime - Int64(Date().timeIntervalSince1970) + if shownExpiries.count + page.count > Self.shownExpiryLimit { shownExpiries.removeAll() } + for entry in page { shownExpiries[entry.key] = MemcachedKnownExpiry(expiresAt: entry.expiresAt) } + } + let rows: [PluginRow] = page.compactMap { entry in guard let value = values[entry.key] else { return nil } return [ @@ -396,7 +405,14 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { - try MemcachedStatementGenerator(columns: columns).generateRowWrites( + let offset = lock.withLock { serverClockOffset } + let expiries = lock.withLock { shownExpiries } + let generator = MemcachedStatementGenerator( + columns: columns, + now: { Int64(Date().timeIntervalSince1970) + offset }, + knownExpiry: { expiries[$0] } + ) + return try generator.generateRowWrites( from: changes, insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift index b717538998..a0e40fe779 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift @@ -2,11 +2,18 @@ import Foundation import TableProMemcachedCore import TableProPluginKit +/// When a key the grid showed expires, as the server put it. `expiresAt` nil means it never does. +struct MemcachedKnownExpiry: Sendable { + let expiresAt: Int64? +} + /// Turns grid edits into console commands: `set` for a new row or a new value or flags, `touch` for a TTL alone, /// `delete` for a removed row. The key names the item, so it is never rewritten in place. struct MemcachedStatementGenerator { let columns: [String] + /// The server's clock, so an absolute expiry is compared with the time the server keeps. var now: () -> Int64 = { Int64(Date().timeIntervalSince1970) } + var knownExpiry: (String) -> MemcachedKnownExpiry? = { _ in nil } func generateRowWrites( from changes: [PluginRowChange], @@ -46,11 +53,17 @@ struct MemcachedStatementGenerator { guard let key = cells[.key]?.asText, !key.isEmpty else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String(localized: "A new key needs a name.")) } - try validate(key: key, rowIndex: change.rowIndex) + guard MemcachedCommandParser.isValidKey(Array(key.utf8)) else { + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String(localized: "A key is 1 to 250 bytes with no spaces or control characters.") + ) + } + let ttl = try number(cells[.ttl], of: .ttl, rowIndex: change.rowIndex) return setStatement( key: key, flags: try number(cells[.flags], of: .flags, rowIndex: change.rowIndex) ?? 0, - ttl: try number(cells[.ttl], of: .ttl, rowIndex: change.rowIndex), + exptime: try exptime(forTTL: ttl, rowIndex: change.rowIndex), value: cells[.value] ?? .null ) } @@ -69,28 +82,54 @@ struct MemcachedStatementGenerator { } edited[column] = cell.newValue } - let ttl = try number(edited[.ttl] ?? original(.ttl, of: change), of: .ttl, rowIndex: change.rowIndex) - guard edited[.value] != nil || edited[.flags] != nil else { - return edited[.ttl] == nil ? nil : MemcachedStatements.touch(key: key, exptime: exptime(forTTL: ttl)) + if let ttl = edited[.ttl], edited[.value] == nil, edited[.flags] == nil { + let seconds = try number(ttl, of: .ttl, rowIndex: change.rowIndex) + return MemcachedStatements.touch(key: key, exptime: try exptime(forTTL: seconds, rowIndex: change.rowIndex)) + } + guard edited[.value] != nil || edited[.flags] != nil else { return nil } + let exptime: Int64 + if let ttl = edited[.ttl] { + exptime = try self.exptime(forTTL: try number(ttl, of: .ttl, rowIndex: change.rowIndex), rowIndex: change.rowIndex) + } else { + exptime = try currentExptime(of: key, change: change) } return setStatement( key: key, flags: try number(edited[.flags] ?? original(.flags, of: change), of: .flags, rowIndex: change.rowIndex) ?? 0, - ttl: ttl, + exptime: exptime, value: edited[.value] ?? original(.value, of: change) ?? .null ) } - private func setStatement(key: String, flags: Int64, ttl: Int64?, value: PluginCellValue) -> String { + /// A `set` replaces the expiry too, so one that leaves TTL alone sends the time the item already ends at, + /// rather than the TTL the grid read, which has been counting down since. + private func currentExptime(of key: String, change: PluginRowChange) throws -> Int64 { + guard let known = knownExpiry(key) else { + return try exptime(forTTL: try number(original(.ttl, of: change), of: .ttl, rowIndex: change.rowIndex), rowIndex: change.rowIndex) + } + guard let expiresAt = known.expiresAt else { return 0 } + guard expiresAt > now() else { + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String(localized: "This item expired while the grid was open. Refresh to see whether it still exists.") + ) + } + return expiresAt + } + + private func setStatement(key: String, flags: Int64, exptime: Int64, value: PluginCellValue) -> String { let flags = UInt32(flags) if case .bytes(let data) = value { - return MemcachedStatements.set(key: key, flags: flags, exptime: exptime(forTTL: ttl), bytes: data) + return MemcachedStatements.set(key: key, flags: flags, exptime: exptime, bytes: data) } - return MemcachedStatements.set(key: key, flags: flags, exptime: exptime(forTTL: ttl), value: value.asText ?? "") + return MemcachedStatements.set(key: key, flags: flags, exptime: exptime, value: value.asText ?? "") } - private func exptime(forTTL ttl: Int64?) -> Int64 { - MemcachedStatements.exptime(forTTL: ttl, now: now()) + private func exptime(forTTL ttl: Int64?, rowIndex: Int) throws -> Int64 { + guard let exptime = MemcachedStatements.exptime(forTTL: ttl, now: now()) else { + throw PluginRowWriteRefusal(rowIndex: rowIndex, reason: String(localized: "This TTL is too large for Memcached.")) + } + return exptime } private func number(_ cell: PluginCellValue?, of column: MemcachedColumn, rowIndex: Int) throws -> Int64? { @@ -105,15 +144,6 @@ struct MemcachedStatementGenerator { return value } - private func validate(key: String, rowIndex: Int) throws { - guard MemcachedCommandParser.isValidKey(Array(key.utf8)) else { - throw PluginRowWriteRefusal( - rowIndex: rowIndex, - reason: String(localized: "A key is 1 to 250 bytes with no spaces or control characters.") - ) - } - } - private func originalKey(of change: PluginRowChange) throws -> String { guard let key = original(.key, of: change)?.asText, MemcachedCommandParser.isValidKey(Array(key.utf8)) else { throw PluginRowWriteRefusal( diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index e3d437c638..a7a132bb14 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -197119,6 +197119,10 @@ } } }, + "This item expired while the grid was open. Refresh to see whether it still exists." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This item no longer exists." : { "localizations" : { "fr" : { @@ -199446,6 +199450,10 @@ } } }, + "This TTL is too large for Memcached." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This user can grant this privilege to others." : { "localizations" : { "fr" : { diff --git a/apple/scripts/probes/memcached-probe.swift b/apple/scripts/probes/memcached-probe.swift index 014012c9d3..7474b31efa 100644 --- a/apple/scripts/probes/memcached-probe.swift +++ b/apple/scripts/probes/memcached-probe.swift @@ -81,6 +81,22 @@ let ttlRow = try await driver.execute(query: MemcachedBrowseSpec(limit: 10, offs let ttl = texts(ttlRow).first?[2].flatMap { Int($0) } ?? 0 check(ttl > 590 && ttl <= 600, "TTL counts down from 600 (\(ttl))") +let keepExpiry = try driver.generateRowWrites( + table: "keys", schema: nil, columns: ["Key", "Value", "TTL", "Flags"], primaryKeyColumns: ["Key"], + changes: [PluginRowChange(rowIndex: 0, type: .update, cellChanges: [(1, "Value", .text("expires"), .text("changed"))], + originalRow: [.text("\(prefix)ttl"), .text("expires"), .text("600"), .text("0")])], + insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] +) ?? [] +let keepStatement = keepExpiry.first?.statement ?? "" +let keepExptime = Int64(keepStatement.split(separator: " ").dropFirst(3).first ?? "") ?? 0 +check(keepExptime > MemcachedStatements.relativeExptimeLimit, "a value edit sends the absolute expiry it read (\(keepStatement))") +for write in keepExpiry { _ = try await driver.execute(query: write.statement) } +let keptRow = try await driver.execute(query: MemcachedBrowseSpec(limit: 10, offset: 0, ascending: true, conditions: [ + try MemcachedBrowseSpec.condition(column: "Key", op: "=", value: "\(prefix)ttl", isCaseSensitive: true), +]).statement) +let keptTTL = texts(keptRow).first?[2].flatMap { Int($0) } ?? 0 +check(texts(keptRow).first?[1] == "changed" && keptTTL > 0 && keptTTL <= ttl, "the value changed and the TTL kept counting down (\(keptTTL))") + let multi = try await driver.execute(query: "get \(prefix)multi") check(texts(multi).first?[1] == "line one\nline two \"quoted\"", "escaped value round-trips") From 9820ad61ff92005bb00e570ce680e7c529c9406b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:39:45 +0700 Subject: [PATCH 4/6] fix(plugin-memcached): bound the login by the connect timeout and show non-stat lines --- .../MemcachedDriverPlugin/MemcachedConnection.swift | 3 +++ .../MemcachedDriverPlugin/MemcachedPluginDriver.swift | 7 ++++++- apple/scripts/probes/memcached-probe.swift | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift index 866c5be7fc..583a7e8213 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift @@ -162,6 +162,9 @@ actor MemcachedConnection { } handler = replyHandler + let requestTimeoutAfterOpen = requestTimeout + requestTimeout = endpoint.connectTimeout + defer { requestTimeout = requestTimeoutAfterOpen } do { try await authenticate() } catch { diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift index 59617e2793..fb5fa15397 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -154,7 +154,12 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { rows: rows, started: started ) case .lines(let lines, _): - let rows: [PluginRow] = MemcachedKeyListing.statPairs(lines).map { [.text($0.name), .text($0.value)] } + let pairs = MemcachedKeyListing.statPairs(lines) + guard pairs.count == lines.count else { + let rows: [PluginRow] = lines.map { [.text($0)] } + return result(columns: ["Line"], types: ["String"], rows: rows, started: started) + } + let rows: [PluginRow] = pairs.map { [.text($0.name), .text($0.value)] } return result(columns: ["Name", "Value"], types: ["String", "String"], rows: rows, started: started) case .status(let line): let affected = ["STORED", "DELETED", "TOUCHED", "OK"].contains(line) || UInt64(line) != nil ? 1 : 0 diff --git a/apple/scripts/probes/memcached-probe.swift b/apple/scripts/probes/memcached-probe.swift index 7474b31efa..3345deff80 100644 --- a/apple/scripts/probes/memcached-probe.swift +++ b/apple/scripts/probes/memcached-probe.swift @@ -164,6 +164,11 @@ do { } let leaked = try await driver.execute(query: "get tablepro") check(leaked.rows.isEmpty, "credentials given to an open server are not stored as an item") +let slabs = try await driver.execute(query: "stats items") +if let slab = texts(slabs).compactMap({ $0[0] }).first(where: { $0.hasSuffix(":number") })?.split(separator: ":").dropFirst().first { + let dump = try await driver.execute(query: "stats cachedump \(slab) 5") + check(dump.columns == ["Line"] && texts(dump).first?.first??.hasPrefix("ITEM ") == true, "stats cachedump shows its ITEM lines") +} let reset = try await driver.execute(query: "stats reset") check(texts(reset).first?.first == "RESET", "stats reset answers one status line") let quotedKey = "\(prefix)it's \"quoted\"" From 850f7e64a050368a09d151db4cab23234999eb29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:40:04 +0700 Subject: [PATCH 5/6] docs(changelog): add the Memcached fragment --- apple/changelog.d/3980.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 apple/changelog.d/3980.added.md diff --git a/apple/changelog.d/3980.added.md b/apple/changelog.d/3980.added.md new file mode 100644 index 0000000000..981626415e --- /dev/null +++ b/apple/changelog.d/3980.added.md @@ -0,0 +1 @@ +Memcached support: browse keys with values and TTLs, edit them in the grid, and run text protocol commands. From fbb6e111ef13212f5f75f2e12b611daedb7c7be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Nam=20Long?= <26674308+J2TeamNNL@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:05:15 +0700 Subject: [PATCH 6/6] fix(plugin-memcached): add the dashboard and picker strings to the catalog --- apple/TablePro/Resources/Localizable.xcstrings | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index a7a132bb14..2f520c010e 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -74347,6 +74347,9 @@ }, "Everything selected was already in your library." : { + }, + "Evictions" : { + }, "Exact value" : { "localizations" : { @@ -89027,6 +89030,9 @@ } } } + }, + "Hit Ratio" : { + }, "Holds a bidirectional control, which can change the order the text is shown in" : { @@ -92273,6 +92279,9 @@ } } } + }, + "In-memory key-value cache" : { + }, "Inactive" : { "localizations" : {