diff --git a/packages/opencode/src/altimate/tools/sql-classify.ts b/packages/opencode/src/altimate/tools/sql-classify.ts index a79357eb34..48cdc9077b 100644 --- a/packages/opencode/src/altimate/tools/sql-classify.ts +++ b/packages/opencode/src/altimate/tools/sql-classify.ts @@ -5,7 +5,7 @@ // Parser failures are never reads; lexical checks retain hard-deny protection. import { hasMongoWriteStage, normalizeMqlCommand } from "@altimateai/drivers" -import { isModeledDialect, maskSql, statementsOf, tokenizeSql } from "./sql-tokenize" +import { DOLLAR_TAG_ANYWHERE, isModeledDialect, lexesDollarQuotes, maskSql, statementsOf, tokenizeSql } from "./sql-tokenize" import { sqlFunctionCalls, sqlLiteral } from "./sql-functions" import { hasSideEffectFunction, isReadOnlyNestedFunction } from "./sql-function-policy" @@ -32,6 +32,7 @@ const READ_CATEGORIES = new Set(["query"]) * Only code tokens change; quoted content and the original statement boundaries * remain authoritative when checking the parser's result. */ function parserSql(sql: string, dialect?: string): string { + if (/\bEXPLAIN\b/i.test(sql)) sql = blankExplainOptions(sql, dialect) const duckdb = dialect?.toLowerCase() === "duckdb" if (!duckdb && dialect?.toLowerCase() !== "sqlite") return sql const compatible = tokenizeSql(sql, dialect).tokens @@ -141,14 +142,78 @@ function isReadOnlyMetadata(statement: string): boolean { } // EXPLAIN is a read only when the explained statement is one: `EXPLAIN ANALYZE` // executes the statement, so `EXPLAIN ANALYZE DELETE …` must stay a write. -const EXPLAIN_PREFIX = /^\s*EXPLAIN\b(\s+(ANALYZE|VERBOSE|QUERY\s+PLAN|FORMAT\s+\w+|\([^)]*\)))*\s+/i +// Options are accepted only where the dialect defines them: Snowflake USING JSON|TEXT|TABULAR, +// Databricks/Spark/Hive EXTENDED|CODEGEN|COST|FORMATTED, and the parenthesized list of +// documented options that PostgreSQL and DuckDB share. Any other option, or a list in another dialect, leaves a write. +const POSTGRES_BOOLEAN = "(?:\\s+(?:TRUE|FALSE|ON|OFF|YES|NO|T|F|Y|N|0|1))?" +const POSTGRES_OPTION = + "(?:FORMAT\\s+(?:TEXT|XML|JSON|YAML|HTML|GRAPHVIZ)|SERIALIZE(?:\\s+(?:NONE|TEXT|BINARY))?|" + + `(?:ANALYZE|ANALYSE|VERBOSE|COSTS|IO|SETTINGS|GENERIC_PLAN|BUFFERS|WAL|TIMING|SUMMARY|MEMORY)${POSTGRES_BOOLEAN})` +// Linear: the items are separated by a comma, so no whitespace run can be split two ways. +const POSTGRES_OPTION_LIST = `\\(\\s*${POSTGRES_OPTION}(?:\\s*,\\s*${POSTGRES_OPTION})*\\s*\\)` + +function explainPrefix(dialect?: string): RegExp { + const d = dialect?.toLowerCase() ?? "" + const words = ["ANALYZE", "VERBOSE", "QUERY\\s+PLAN", "FORMAT\\s+\\w+"] + if (d === "snowflake") words.push("USING\\s+(?:JSON|TEXT|TABULAR)") + if (/^(databricks|spark|hive)$/.test(d)) words.push("EXTENDED", "CODEGEN", "COST", "FORMATTED") + // PostgreSQL and DuckDB (same grammar); other dialects have no such list. PostgreSQL takes either the parenthesized list or the legacy words, never both. + const list = /^(postgres|postgresql|duckdb)$/.test(d) ? `\\s*${POSTGRES_OPTION_LIST}\\s*|` : "" + return new RegExp(`^\\s*EXPLAIN\\b(?:${list}(?:\\s+(?:${words.join("|")}))*\\s+)`, "i") +} + +/** Length of the EXPLAIN keyword and its options, beyond a bare `EXPLAIN `. */ +function optionsLength(statement: string, prefix: RegExpExecArray): number { + return prefix[0].length - (/^\s*EXPLAIN\s+/i.exec(statement)?.[0].length ?? 0) +} + +/** The parser lacks the dialect's EXPLAIN options. Blank them in place, so the parser reads plain + * EXPLAIN over the original text, comments and statement boundaries unchanged. Never used for execution. */ +function blankExplainOptions(sql: string, dialect?: string): string { + // A lexer without dollar quotes can read `$$--$$` as a comment that hides statements the parser + // skips too, so the two statement counts would agree on a wrong answer. Leave such text unadapted: + // the parser rejects the options and the statement is a write. + if (DOLLAR_TAG_ANYWHERE.test(sql) && !lexesDollarQuotes(dialect)) return sql + const lexical = tokenizeSql(sql, dialect) + if (!lexical.complete) return sql + // Same length as the text: comments are blanks, quoted text a filler no option can match. + const scan = lexical.tokens + .map(({ kind, text }) => + kind === "code" ? text : kind === "comment" ? " ".repeat(text.length) : "\u0001".repeat(text.length), + ) + .join("") + const prefix = explainPrefix(dialect) + const parts: string[] = [] + let copied = 0 + let start = 0 + for (const statement of scan.split(";")) { + const match = prefix.exec(statement) + if (match && optionsLength(statement, match) > 0 && statement.slice(match[0].length).trim()) { + const from = start + /^\s*/.exec(match[0])![0].length + "EXPLAIN".length + const to = start + match[0].length + parts.push(sql.slice(copied, from), " ".repeat(to - from)) + copied = to + } + start += statement.length + 1 + } + return parts.length ? parts.join("") + sql.slice(copied) : sql +} + +/** EXPLAIN with options is read-only only as the sole statement of its text. */ +function hasExplainOptions(statement: string, dialect?: string): boolean { + const prefix = explainPrefix(dialect).exec(statement) + return !!prefix && optionsLength(statement, prefix) > 0 +} function isReadOnlyOther(statement: string, dialect?: string): boolean { if (isReadOnlyMetadata(statement)) return true - const explained = statement.replace(EXPLAIN_PREFIX, "") + const explained = statement.replace(explainPrefix(dialect), "") if (explained === statement) return false return ( - (/^\s*(SELECT|WITH)\b/i.test(explained) && !hasQueryWriteClause(explained, dialect)) || + (/^\s*(SELECT|WITH)\b/i.test(explained) && + !hasQueryWriteClause(explained, dialect) && + // DuckDB runs COPY inside a CTE, and an adapted EXPLAIN reaches the parser without its options. + !/\bCOPY\b/i.test(maskSql(tokenizeSql(explained, dialect).tokens))) || isReadOnlyMetadata(explained) ) } @@ -161,6 +226,7 @@ function otherIsReadOnly(sql: string, parsed: Array<{ category: string }>, diale if (!tokenizeSql(sql, dialect).complete) return false const statements = statementsOf(sql, dialect) if (statements.length !== parsed.length) return false + if (statements.length > 1 && statements.some((statement) => hasExplainOptions(statement, dialect))) return false return parsed.every( (statement, i) => (READ_CATEGORIES.has(statement.category) && !hasQueryWriteClause(statements[i], dialect)) || diff --git a/packages/opencode/src/altimate/tools/sql-tokenize.ts b/packages/opencode/src/altimate/tools/sql-tokenize.ts index c980b3734e..48b9d92808 100644 --- a/packages/opencode/src/altimate/tools/sql-tokenize.ts +++ b/packages/opencode/src/altimate/tools/sql-tokenize.ts @@ -1,7 +1,15 @@ type SqlToken = { kind: "code" | "comment" | "literal" | "identifier"; text: string } // PostgreSQL, DuckDB and Redshift also allow any non-ASCII character in identifiers and dollar tags. -const DOLLAR_TAG = /\$(?:[A-Za-z_\u0080-\uffff][\w\u0080-\uffff]*)?\$/y +const DOLLAR_TAG_SOURCE = "\\$(?:[A-Za-z_\\u0080-\\uffff][\\w\\u0080-\\uffff]*)?\\$" +const DOLLAR_TAG = new RegExp(DOLLAR_TAG_SOURCE, "y") +/** Text that opens a dollar quote, anywhere. */ +export const DOLLAR_TAG_ANYWHERE = new RegExp(DOLLAR_TAG_SOURCE) + +/** Dialects whose lexer reads dollar quotes (ClickHouse heredocs read alike). */ +export function lexesDollarQuotes(dialect?: string): boolean { + return /^(postgres|postgresql|redshift|duckdb|snowflake|clickhouse)$/i.test(dialect ?? "") +} const IDENTIFIER_CHAR = /[\w$\u0080-\uffff]/ /** A supported warehouse dialect these lexing rules model; anything else (none, "generic", @@ -36,7 +44,7 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] // Where nesting is not known either way, a /* inside a comment makes its end uncertain. const nestingKnown = nestedComments || /^(mysql|mariadb|sqlite|oracle|trino)$/i.test(dialect ?? "") // ClickHouse heredocs ($tag$...$tag$) read like PostgreSQL dollar quotes. - const dollarQuotes = /^(postgres|redshift|duckdb|snowflake|clickhouse)$/i.test(dialect ?? "") + const dollarQuotes = lexesDollarQuotes(dialect) // Quoting is dialect-specific: brackets can contain executable array expressions, // and PostgreSQL backticks can be operators. Unknown dialects keep both visible. const backtickIdentifiers = /^(mysql|mariadb|sqlite|bigquery|clickhouse|spark|databricks|hive)$/i.test(dialect ?? "") diff --git a/packages/opencode/test/altimate/sql-classify-explain.test.ts b/packages/opencode/test/altimate/sql-classify-explain.test.ts new file mode 100644 index 0000000000..516d3b1bc1 --- /dev/null +++ b/packages/opencode/test/altimate/sql-classify-explain.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, test } from "bun:test" +import { classifyAndCheck } from "../../src/altimate/tools/sql-classify" + +const type = (sql: string, dialect: string) => classifyAndCheck(sql, dialect).queryType + +describe("EXPLAIN with options", () => { + const readInner = ["SELECT 1", "SELECT * FROM t WHERE a = 'x;y'", "WITH a AS (SELECT 1) SELECT * FROM a"] + const writeInner = [ + "INSERT INTO t VALUES (1)", + "DELETE FROM t", + "UPDATE t SET a = 1", + "MERGE INTO t USING s ON t.a = s.a WHEN MATCHED THEN DELETE", + "CREATE TABLE t2 AS SELECT 1", + "SELECT 1 INTO t2", + "WITH a AS (DELETE FROM t RETURNING *) SELECT * FROM a", + ] + + for (const format of ["JSON", "TEXT", "TABULAR"]) { + test(`Snowflake EXPLAIN USING ${format} is a read only for a read`, () => { + for (const inner of readInner) expect(type(`EXPLAIN USING ${format} ${inner}`, "snowflake")).toBe("read") + for (const inner of writeInner) expect(type(`EXPLAIN USING ${format} ${inner}`, "snowflake")).toBe("write") + }) + } + + test("Snowflake EXPLAIN USING tolerates case, whitespace and comments", () => { + for (const sql of [ + "explain using json select 1", + "ExPlAiN USING\n\tTeXt select 1", + "/* lead */ EXPLAIN /* a */ USING /* b */ JSON /* c */ SELECT 1 -- tail", + "EXPLAIN USING JSON SELECT 1;", + ]) { + expect(classifyAndCheck(sql, "snowflake")).toEqual({ queryType: "read", blocked: false }) + } + expect(type("explain using json /* c */ delete from t", "snowflake")).toBe("write") + }) + + test("Snowflake EXPLAIN USING with more than one statement is not a read", () => { + expect(type("EXPLAIN USING JSON SELECT 1", "snowflake")).toBe("read") + for (const sql of [ + "EXPLAIN USING JSON SELECT 1; SELECT 2", + "EXPLAIN USING JSON SELECT 1; DELETE FROM t", + "EXPLAIN USING JSON SELECT 1; DROP TABLE t", + "SELECT 1; EXPLAIN USING JSON DELETE FROM t", + "SELECT 1; EXPLAIN USING JSON SELECT 2", + ]) { + expect(type(sql, "snowflake")).toBe("write") + } + }) + + test("Snowflake EXPLAIN USING with an unknown format or no statement is not a read", () => { + expect(type("EXPLAIN USING TEXT SELECT 1", "snowflake")).toBe("read") + for (const sql of ["EXPLAIN USING XML SELECT 1", "EXPLAIN USING JSON", "EXPLAIN USING JSON ;"]) { + expect(type(sql, "snowflake")).toBe("write") + } + }) + + test("USING is not an EXPLAIN option outside Snowflake", () => { + expect(type("EXPLAIN USING JSON SELECT 1", "snowflake")).toBe("read") + for (const dialect of ["postgres", "mysql", "bigquery", "databricks"]) { + expect(type("EXPLAIN USING JSON SELECT 1", dialect)).toBe("write") + } + }) + + test("Databricks EXPLAIN modes of a read are reads, of a write are writes", () => { + for (const mode of ["EXTENDED", "CODEGEN", "COST", "FORMATTED"]) { + expect(type(`EXPLAIN ${mode} SELECT 1`, "databricks")).toBe("read") + expect(type(`explain ${mode.toLowerCase()} select 1`, "databricks")).toBe("read") + expect(type(`EXPLAIN ${mode} DELETE FROM t`, "databricks")).toBe("write") + expect(type(`EXPLAIN ${mode} INSERT INTO t SELECT 1`, "databricks")).toBe("write") + expect(type(`EXPLAIN ${mode} SELECT 1; DELETE FROM t`, "databricks")).toBe("write") + } + expect(type("EXPLAIN EXTENDED SELECT 1", "snowflake")).toBe("write") + }) + + test("Postgres EXPLAIN option lists follow the explained statement", () => { + for (const options of ["(FORMAT JSON)", "(ANALYZE, FORMAT JSON)", "(VERBOSE, COSTS OFF)"]) { + expect(type(`EXPLAIN ${options} SELECT 1`, "postgres")).toBe("read") + expect(type(`EXPLAIN ${options} DELETE FROM t`, "postgres")).toBe("write") + expect(type(`EXPLAIN ${options} INSERT INTO t VALUES (1)`, "postgres")).toBe("write") + expect(type(`EXPLAIN ${options} SELECT 1; DELETE FROM t`, "postgres")).toBe("write") + } + }) + + test("Postgres option lists accept only documented options", () => { + for (const options of ["(BOGUS TRUE)", "(ANALYZE, (FORMAT JSON)", "(FORMAT XML2)", "(ANALYZE,)", "()", "(ANALYZE) (VERBOSE)"]) { + expect(type(`EXPLAIN ${options} SELECT 1`, "postgres")).toBe("write") + } + for (const options of ["(ANALYZE TRUE, BUFFERS off, SERIALIZE BINARY, FORMAT YAML)", "( analyze , format json )"]) { + expect(type(`EXPLAIN ${options} SELECT 1`, "postgres")).toBe("read") + } + expect(type("EXPLAIN(FORMAT JSON)SELECT 1", "postgres")).toBe("read") + expect(type("EXPLAIN(FORMAT JSON)DELETE FROM t", "postgres")).toBe("write") + }) + + test("EXPLAIN (ANALYZE) of a write stays a write", () => { + for (const sql of [ + "EXPLAIN (ANALYZE) DELETE FROM t", + "EXPLAIN (ANALYZE TRUE, FORMAT JSON) UPDATE t SET a = 1", + "EXPLAIN (ANALYZE) INSERT INTO t SELECT 1", + "EXPLAIN (ANALYZE) SELECT 1 INTO t2", + "EXPLAIN (ANALYZE) WITH a AS (DELETE FROM t RETURNING *) SELECT * FROM a", + "EXPLAIN(ANALYZE)DELETE FROM t", + ]) { + expect(type(sql, "postgres")).toBe("write") + } + }) + + test("a parenthesized option list is not an option outside Postgres", () => { + for (const dialect of ["snowflake", "mysql", "bigquery", "sqlserver", "databricks", "redshift"]) { + for (const options of ["(FORMAT JSON)", "(BOGUS TRUE)", "(ANALYZE)"]) { + expect(type(`EXPLAIN ${options} SELECT 1`, dialect)).toBe("write") + } + } + expect(type("EXPLAIN (FORMAT JSON) SELECT 1", undefined as unknown as string)).toBe("write") + }) + + test("DuckDB and the postgresql alias share the Postgres option list", () => { + for (const dialect of ["duckdb", "postgresql", "postgres"]) { + expect(type("EXPLAIN (ANALYZE, FORMAT JSON) SELECT 1", dialect)).toBe("read") + expect(type("EXPLAIN (FORMAT HTML) SELECT 1", dialect)).toBe("read") + expect(type("EXPLAIN (ANALYZE, FORMAT JSON) DELETE FROM t", dialect)).toBe("write") + expect(type("EXPLAIN (BOGUS TRUE) SELECT 1", dialect)).toBe("write") + } + }) + + test("Postgres boolean spellings and the IO option are accepted", () => { + for (const options of ["(IO TRUE)", "(ANALYZE yes)", "(BUFFERS t, COSTS n)", "(ANALYZE 1, TIMING off)"]) { + expect(type(`EXPLAIN ${options} SELECT 1`, "postgres")).toBe("read") + expect(type(`EXPLAIN ${options} DELETE FROM t`, "postgres")).toBe("write") + } + }) + + test("EXPLAIN of a query with a COPY in a CTE is a write", () => { + const copy = "WITH copied AS (COPY (SELECT 1 AS x) TO '/tmp/out.csv') SELECT * FROM copied" + for (const sql of [`EXPLAIN (ANALYZE) ${copy}`, `EXPLAIN (FORMAT JSON) ${copy}`, `EXPLAIN ANALYZE ${copy}`, `EXPLAIN ${copy}`]) { + expect(type(sql, "duckdb")).toBe("write") + expect(type(sql, "postgres")).toBe("write") + } + expect(type("EXPLAIN (ANALYZE) SELECT 'copy' AS \"copy\"", "duckdb")).toBe("read") + }) + + test("an optioned EXPLAIN batch of reads is still a write", () => { + for (const sql of ["EXPLAIN ANALYZE SELECT 1; SELECT 2", "EXPLAIN (FORMAT JSON) SELECT 1; SELECT 2", "SELECT 1; EXPLAIN (ANALYZE) SELECT 2"]) { + expect(type(sql, "postgres")).toBe("write") + } + expect(type("EXPLAIN COST SELECT 1; SELECT 2", "databricks")).toBe("write") + }) + + test("the postgresql alias reads dollar quotes, so hidden statements cannot slip through", () => { + for (const sql of [ + "EXPLAIN (ANALYZE) SELECT $$--$$ AS a; CLUSTER t", + "EXPLAIN (ANALYZE) SELECT $$--$$ AS a; CHECKPOINT", + "EXPLAIN (ANALYZE) SELECT $$--$$ AS a; NOTIFY channel", + "EXPLAIN VERBOSE SELECT $$--$$ AS a; CLUSTER t", + "EXPLAIN ANALYZE SELECT $$--$$ AS a; CHECKPOINT", + "EXPLAIN VERBOSE SELECT $t$--$t$ AS a; NOTIFY channel", + "EXPLAIN SELECT $$--$$ AS a; CHECKPOINT", + "EXPLAIN (ANALYZE) SELECT $t1$--$t1$ AS a; CLUSTER t", + ]) { + expect(type(sql, "postgresql")).toBe("write") + expect(type(sql, "postgres")).toBe("write") + } + expect(type("EXPLAIN (FORMAT JSON) SELECT 1", "postgresql")).toBe("read") + }) + + test("nested optioned EXPLAIN is a write", () => { + expect(type("EXPLAIN USING JSON EXPLAIN USING JSON SELECT 1", "snowflake")).toBe("write") + expect(type("EXPLAIN COST EXPLAIN COST SELECT 1", "databricks")).toBe("write") + expect(type("EXPLAIN (FORMAT JSON) EXPLAIN (FORMAT JSON) SELECT 1", "postgres")).toBe("write") + }) + + test("statements after text the lexer reads as a comment are not hidden from the parser", () => { + for (const dialect of ["databricks", "spark", "hive"]) { + for (const quote of ["$$", "$t$"]) { + const literal = `${quote}--${quote}` + expect(type(`EXPLAIN COST SELECT ${literal} AS a; DELETE FROM t`, dialect)).toBe("write") + expect(type(`EXPLAIN COST SELECT $t1$--$t1$ AS a; CHECKPOINT`, dialect)).toBe("write") + expect(type(`EXPLAIN SELECT ${literal} AS a; DELETE FROM t`, dialect)).toBe("write") + } + } + expect(type("EXPLAIN COST SELECT 1 -- note; not a statement", "databricks")).toBe("read") + }) + + test("a long option list is classified in linear time", () => { + // Quadratic matching took seconds here; the budget leaves room for a loaded runner. + const budget = 500 + const timed = (sql: string, dialect: string, expected: "read" | "write") => { + const start = performance.now() + expect(type(sql, dialect)).toBe(expected) + expect(performance.now() - start).toBeLessThan(budget) + } + timed("EXPLAIN (" + " ".repeat(64_000) + "FORMAT JSON) SELECT 1", "postgres", "read") + timed("EXPLAIN (" + "\n".repeat(64_000) + "FORMAT JSON) SELECT 1", "postgres", "read") + timed("EXPLAIN (" + "(((".repeat(64_000) + "FORMAT JSON) SELECT 1", "postgres", "write") + timed("EXPLAIN (" + "ANALYZE, ".repeat(8_000) + "FORMAT JSON) SELECT 1", "postgres", "read") + }) + + test("a long batch of optioned statements scales linearly", () => { + // Four times the statements costs about four times as much; rebuilding the text per statement costs sixteen. + const best = (statement: string, dialect: string, count: number) => { + const sql = statement.repeat(count) + return Math.min( + ...[0, 1, 2].map(() => { + const start = performance.now() + expect(type(sql, dialect)).toBe("write") + return performance.now() - start + }), + ) + } + for (const [statement, dialect] of [ + ["EXPLAIN USING JSON SELECT 1;", "snowflake"], + ["EXPLAIN (FORMAT JSON) SELECT 1;", "postgres"], + ]) { + expect(best(statement, dialect, 64_000) / best(statement, dialect, 16_000)).toBeLessThan(9) + } + }) + + test("plain EXPLAIN is unchanged", () => { + for (const dialect of ["snowflake", "postgres", "bigquery", "redshift", "mysql"]) { + expect(type("EXPLAIN SELECT 1", dialect)).toBe("read") + expect(type("EXPLAIN DELETE FROM t", dialect)).toBe("write") + } + expect(type("EXPLAIN ANALYZE DELETE FROM t", "postgres")).toBe("write") + }) +})