From 43332441ecaea835f2f2bf2326cc6853b25f6356 Mon Sep 17 00:00:00 2001 From: Maher Khalil Date: Wed, 7 Oct 2026 14:43:43 -0400 Subject: [PATCH] feat: add parameterized sql execution and shared jdbc cleanup - Compile SelectQueryStruct into SQL templates and ordered bindings - Add RawPreparedRDBMSSelectWrapper for prepared execution - Centralize JDBC cleanup and failure aggregation in ConnectionUtils --- .../auth/utils/AbstractSecurityUtils.java | 36 +- .../sql/ParameterizedSqlInterpreter.java | 1491 +++++++++++++++++ .../selectors/AbstractQuerySelector.java | 13 +- .../RawPreparedRDBMSSelectWrapper.java | 456 +++++ .../wrappers/RawRDBMSSelectWrapper.java | 386 +++-- .../rdf/engine/wrappers/WrapperManager.java | 70 +- src/prerna/util/ConnectionUtils.java | 444 ++--- src/prerna/util/QueryExecutionUtility.java | 55 +- src/prerna/util/sql/AbstractSqlQueryUtil.java | 12 +- src/prerna/util/sql/AnsiSqlQueryUtil.java | 7 +- .../util/sql/MicrosoftSqlServerQueryUtil.java | 5 + src/prerna/util/sql/PostgresQueryUtil.java | 5 + .../utils/AbstractSecurityUtilsUnitTests.java | 40 +- .../utils/SecurityEngineUtilsUnitTests.java | 7 +- .../ModelInferenceLogsUtilsUnitTests.java | 20 +- .../ParameterizedSqlInterpreterUnitTests.java | 873 ++++++++++ .../AbstractQuerySelectorUnitTests.java | 48 + ...awPreparedRDBMSSelectWrapperUnitTests.java | 452 +++++ .../RawRDBMSSelectWrapperUnitTests.java | 197 +++ .../wrappers/WrapperManagerUnitTests.java | 157 ++ .../prerna/util/ConnectionUtilsUnitTests.java | 247 ++- .../util/QueryExecutionUtilityUnitTests.java | 70 + .../sql/AbstractSqlQueryUtilUnitTests.java | 32 +- .../util/sql/AnsiSqlQueryUtilUnitTests.java | 16 + .../MicrosoftSqlServerQueryUtilUnitTests.java | 16 + .../util/sql/PostgresJdbcUnitTests.java | 6 + .../util/sql/PostgresQueryUtilUnitTests.java | 12 + .../SqlServerParameterizedQueryUnitTests.java | 78 + 28 files changed, 4790 insertions(+), 461 deletions(-) create mode 100644 src/prerna/query/interpreters/sql/ParameterizedSqlInterpreter.java create mode 100644 src/prerna/rdf/engine/wrappers/RawPreparedRDBMSSelectWrapper.java create mode 100644 test/prerna/query/interpreters/sql/ParameterizedSqlInterpreterUnitTests.java create mode 100644 test/prerna/query/querystruct/selectors/AbstractQuerySelectorUnitTests.java create mode 100644 test/prerna/rdf/engine/wrappers/RawPreparedRDBMSSelectWrapperUnitTests.java create mode 100644 test/prerna/rdf/engine/wrappers/RawRDBMSSelectWrapperUnitTests.java create mode 100644 test/prerna/rdf/engine/wrappers/WrapperManagerUnitTests.java create mode 100644 test/prerna/util/sql/SqlServerParameterizedQueryUnitTests.java diff --git a/src/prerna/auth/utils/AbstractSecurityUtils.java b/src/prerna/auth/utils/AbstractSecurityUtils.java index 697591866b6..4677a3ae0a3 100644 --- a/src/prerna/auth/utils/AbstractSecurityUtils.java +++ b/src/prerna/auth/utils/AbstractSecurityUtils.java @@ -3183,7 +3183,7 @@ static String getUserFilters(User user) { } /** - * Get a vector of the user ids + * Get an array of the user ids * * @param user * @return @@ -3200,6 +3200,22 @@ static Collection getUserFiltersQs(User user) { return filters; } + /** + * Original login IDs for JDBC binding. Do not SQL-escape these values. + * + * @param user the user for whom to get filter values + * @return a collection of the user's login IDs + */ + static Collection getUserFilterValues(User user) { + List values = new ArrayList<>(); + if (user != null) { + for (AuthProvider login : user.getLogins()) { + values.add(user.getAccessToken(login).getId()); + } + } + return values; + } + /** * Keep the rows whose effective permission is one of the given levels. The * effective permission is the better (lower) of the user's own grant and their @@ -3304,6 +3320,24 @@ static IQueryFilter getCreatedByFilter(String createdByCol, String createdByType return anyCreator; } + /** + * Creator filters for JDBC binding. Pass original values without SQL escaping. + */ + static IQueryFilter getPreparedCreatedByFilter(String createdByCol, String createdByTypeCol, + Collection> creators) { + if (creators == null || creators.isEmpty()) { + throw new IllegalArgumentException("A creator filter needs at least one creator"); + } + OrQueryFilter anyCreator = new OrQueryFilter(); + for (Pair creator : creators) { + AndQueryFilter thisCreator = new AndQueryFilter(); + thisCreator.addFilter(SimpleQueryFilter.makeColToValFilter(createdByCol, "==", creator.getValue0())); + thisCreator.addFilter(SimpleQueryFilter.makeColToValFilter(createdByTypeCol, "==", creator.getValue1())); + anyCreator.addFilter(thisCreator); + } + return anyCreator; + } + /** * Read creator filters from reactor input: maps with the creator's login * {@code id} and login {@code type}. diff --git a/src/prerna/query/interpreters/sql/ParameterizedSqlInterpreter.java b/src/prerna/query/interpreters/sql/ParameterizedSqlInterpreter.java new file mode 100644 index 00000000000..f3a7b2b301c --- /dev/null +++ b/src/prerna/query/interpreters/sql/ParameterizedSqlInterpreter.java @@ -0,0 +1,1491 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.query.interpreters.sql; + +import static prerna.query.interpreters.IQueryInterpreter.getAllSearchComparators; +import static prerna.query.interpreters.IQueryInterpreter.getNegSearchComparators; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Connection; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import prerna.algorithm.api.SemossDataType; +import prerna.date.SemossDate; +import prerna.engine.api.IRDBMSEngine; +import prerna.query.querystruct.HardSelectQueryStruct; +import prerna.query.querystruct.SelectQueryStruct; +import prerna.query.querystruct.filters.AndQueryFilter; +import prerna.query.querystruct.filters.BetweenQueryFilter; +import prerna.query.querystruct.filters.FunctionQueryFilter; +import prerna.query.querystruct.filters.IQueryFilter; +import prerna.query.querystruct.filters.OrQueryFilter; +import prerna.query.querystruct.filters.SimpleQueryFilter; +import prerna.query.querystruct.joins.BasicRelationship; +import prerna.query.querystruct.joins.IRelation; +import prerna.query.querystruct.joins.SubqueryRelationship; +import prerna.query.querystruct.selectors.AbstractQuerySelector; +import prerna.query.querystruct.selectors.IQuerySelector; +import prerna.query.querystruct.selectors.IQuerySort; +import prerna.query.querystruct.selectors.QueryArithmeticSelector; +import prerna.query.querystruct.selectors.QueryColumnOrderBySelector; +import prerna.query.querystruct.selectors.QueryColumnSelector; +import prerna.query.querystruct.selectors.QueryConstantSelector; +import prerna.query.querystruct.selectors.QueryFunctionSelector; +import prerna.query.querystruct.selectors.QueryIfSelector; +import prerna.rdf.engine.wrappers.RawPreparedRDBMSSelectWrapper; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.QueryExecutionUtility.ParameterizedQuery; +import prerna.util.sql.RdbmsTypeEnum; + +/** + * Compiles a {@link SelectQueryStruct} into SQL templates and ordered JDBC + * values, using the supplied engine's metadata and SQL dialect. Compilation + * opens no JDBC connection and leaves the input query structure unchanged. Each + * call creates a fresh compilation context, so an interpreter instance can be + * reused. + * + *

+ * SQL is assembled from {@link Fragment} objects that keep text and bindings + * together. Concatenating fragments concatenates their parameter lists in the + * same order; repeating an expression repeats its bindings. This preserves JDBC + * placeholder order even when query-structure traversal differs from SQL clause + * order. Values are bound separately and are never SQL-escaped or interpolated. + * + *

+ * The resulting {@link CompiledQuery} contains independent data and count + * plans. Execution methods pass that plan to + * {@link RawPreparedRDBMSSelectWrapper}, which owns statement execution, + * streaming, counting, reset, and resource cleanup. Unsupported query shapes + * are rejected rather than delegated to literal SQL generation. + */ +public final class ParameterizedSqlInterpreter { + + /** + * Supplies SQL dialect rules and physical metadata mappings. Compilation itself + * does not acquire a connection; execution delegates connection handling to the + * wrapper. + */ + private final IRDBMSEngine engine; + + /** + * Creates a compiler using an engine's SQL dialect and physical metadata + * mappings. + * + * @param engine non-null database whose metadata and dialect govern compilation + * @throws NullPointerException if the engine is null + */ + public ParameterizedSqlInterpreter(IRDBMSEngine engine) { + this.engine = Objects.requireNonNull(engine, "engine"); + } + + /** + * Builds data and count templates without executing SQL or changing the query. + * Each template carries the values in its own placeholder order. Compilation + * normalizes supported scalar values and snapshots the parameter lists. + * + * @param query non-null structured SELECT, including its filters and subqueries + * @return the compiled data query and a count query that includes SQL + * pagination + * @throws NullPointerException if the query or required metadata is null + * @throws IllegalArgumentException if a query shape or value is unsupported or + * invalid + */ + public CompiledQuery compile(SelectQueryStruct query) { + return new Compilation(engine, Objects.requireNonNull(query, "query")).compile(); + } + + /** + * Compiles and executes a query using an engine-acquired connection. The + * returned wrapper manages the read lifecycle and must be closed by the caller. + * Execution uses no JDBC statement timeout. + * + * @param query structured SELECT to compile and execute + * @return an executed wrapper that streams the result rows + * @throws Exception if compilation, execution, or resource initialization fails + */ + public RawPreparedRDBMSSelectWrapper execute(SelectQueryStruct query) throws Exception { + return RawPreparedRDBMSSelectWrapper.executeParameterized(engine, compile(query), 0); + } + + /** + * Compiles and executes inside the caller's existing connection and + * transaction. Closing the wrapper releases its statements and results without + * committing, rolling back, changing auto-commit, or closing the supplied + * connection. + * + * @param connection non-null connection whose lifecycle remains with the + * caller + * @param query structured SELECT to compile and execute + * @param timeoutSeconds non-negative JDBC timeout for data and count + * statements; zero means no statement timeout + * @return an executed streaming wrapper that the caller must close + * @throws Exception if compilation, execution, or resource initialization fails + */ + public RawPreparedRDBMSSelectWrapper execute(Connection connection, SelectQueryStruct query, int timeoutSeconds) + throws Exception { + return RawPreparedRDBMSSelectWrapper.executeParameterized(engine, connection, compile(query), timeoutSeconds); + } + + /** + * The executable result of compilation: one plan for rows and another for their + * count. Each {@link ParameterizedQuery} carries SQL with placeholders, ordered + * bindings, and a JDBC maximum-row limit. The compiler expresses pagination in + * SQL and leaves that JDBC limit at zero. + * + *

+ * The count plan wraps the relevant SELECT, preserving distinct/grouping and + * SQL pagination. An unpaged count omits unnecessary ordering. Its bindings + * belong to its own SQL template and must not be replaced with the data query's + * list. Both parameter lists are copied into unmodifiable lists; their elements + * are not recursively copied. + * + * @param query data-returning SQL template and its bindings + * @param countQuery count SQL template and its independently ordered bindings + */ + public record CompiledQuery(ParameterizedQuery query, ParameterizedQuery countQuery) { + /** + * Validates both plans and snapshots their parameter lists, including null + * values. + * + * @param query data-returning plan to retain + * @param countQuery count plan to retain + * @throws NullPointerException if either plan is null + * @throws IllegalArgumentException if SQL is blank, bindings are null, or a row + * limit is negative + */ + public CompiledQuery { + query = snapshot(query); + countQuery = snapshot(countQuery); + } + + /** + * Copies a validated plan's parameter list so later list edits do not change + * the retained bindings. This is a shallow copy: mutable parameter objects + * themselves are not cloned here. + * + * @param query plan with nonblank SQL, a binding list, and a non-negative row + * limit + * @return a plan with the same SQL/limit and an unmodifiable copy of the + * bindings + * @throws NullPointerException if the plan is null + * @throws IllegalArgumentException if the plan's SQL, bindings, or limit is + * invalid + */ + private static ParameterizedQuery snapshot(ParameterizedQuery query) { + Objects.requireNonNull(query, "query"); + if (query.sql() == null || query.sql().isBlank() || query.parameters() == null || query.limit() < 0) { + throw new IllegalArgumentException("SQL, parameters, and a non-negative row limit are required"); + } + return new ParameterizedQuery(query.sql(), + Collections.unmodifiableList(new ArrayList<>(query.parameters())), query.limit()); + } + + /** + * Returns the data query's SQL template, without substituting parameter values. + * + * @return SQL containing the placeholders to prepare through JDBC + */ + public String sql() { + return query.sql(); + } + + /** + * Returns the data query's retained bindings. List index zero corresponds to + * JDBC parameter one; callers must preserve this order when binding. + * + * @return the unmodifiable parameter list, which can contain null values + */ + public List parameters() { + return query.parameters(); + } + } + + /** + * A piece of SQL and the values belonging to its placeholders, in textual + * order. Fragments can represent a constant, expression, clause, or complete + * subquery. They are assembled together so SQL rewrites do not lose or reorder + * bindings. + * + *

+ * For example, joining {@code column = } with a value fragment produces + * {@code column = ?} and a one-element binding list. Appending that fragment + * twice also appends its value twice. The record itself does not validate SQL + * or copy the supplied list; assembly methods create the required combined + * lists. + * + * @param sql SQL text, possibly empty or containing placeholders + * @param values values for this fragment's placeholders in left-to-right order + */ + private record Fragment(String sql, List values) { + /** + * Creates a fragment containing compiler-generated or validated SQL structure. + * User values belong in {@link #value(Object)}, not in this text. + * + * @param sql SQL structure with no bind placeholders of its own + * @return the text with an empty binding list + */ + static Fragment text(String sql) { + return new Fragment(sql, List.of()); + } + + /** + * Creates one JDBC placeholder and retains its corresponding value separately. + * + * @param value already-normalized value to bind, including null + * @return the SQL text {@code ?} and its one-element binding list + */ + static Fragment value(Object value) { + return new Fragment("?", Collections.singletonList(value)); + } + + /** + * Appends another fragment without modifying either input. Bindings from the + * right-hand fragment follow this fragment's bindings, matching the SQL order. + * + * @param other fragment to append + * @return combined SQL and a newly combined binding list + */ + Fragment then(Fragment other) { + List combined = new ArrayList<>(values); + combined.addAll(other.values); + return new Fragment(sql + other.sql, combined); + } + + /** + * Appends SQL structure that contributes no additional parameter values. + * + * @param text compiler-generated or validated SQL text to append + * @return the combined fragment with the existing binding order preserved + */ + Fragment then(String text) { + return then(text(text)); + } + + /** + * Joins fragments with a SQL separator and flattens their bindings in list + * order. An empty input produces empty SQL and no values. + * + * @param fragments ordered fragments to combine + * @param separator SQL between fragments, such as a comma or AND; contains no + * bindings + * @return the joined SQL and its ordered bindings + */ + static Fragment join(List fragments, String separator) { + StringBuilder sql = new StringBuilder(); + List values = new ArrayList<>(); + for (int i = 0; i < fragments.size(); i++) { + Fragment fragment = fragments.get(i); + sql.append(i == 0 ? "" : separator).append(fragment.sql); + values.addAll(fragment.values); + } + return new Fragment(sql.toString(), values); + } + } + + /** + * Mutable state for compiling one SELECT scope. The inherited interpreter + * provides metadata resolution helpers only; this class builds its own SQL and + * bindings. Child scopes have independent table aliases and projections but + * share the identity-based recursion guard with their ancestors. + */ + private static final class Compilation extends SqlInterpreter { + /** + * The input SELECT for this scope; compilation reads it without modifying it. + */ + private final SelectQueryStruct select; + + /** + * Queries on the current compilation path, compared by object identity and + * shared with child scopes. A repeated active object indicates a cycle; an + * object may be reused in another branch once its earlier compilation finishes. + */ + private final Set activeQueries; + + /** + * Maps conceptual table or derived-source names to generated SQL aliases in + * this scope. Insertion order determines aliases such as {@code t0} and + * {@code t1}. + */ + private final Map tableAliases = new LinkedHashMap<>(); + + /** + * Validated physical table paths used when rendering ordinary FROM/JOIN + * sources. + */ + private final Map physicalTables = new LinkedHashMap<>(); + + /** + * Child compilation scopes used to resolve derived-table columns and their + * types. + */ + private final Map derivedTables = new LinkedHashMap<>(); + + /** + * Compiled derived sources, retained with bindings until inserted into + * FROM/JOIN SQL. + */ + private final Map derivedQueries = new LinkedHashMap<>(); + + /** + * Result labels mapped to selectors for projected-alias sorting and + * derived-column lookup. + */ + private final Map projections = new LinkedHashMap<>(); + + /** + * Creates a root scope with a fresh recursion guard based on object identity. + * + * @param engine database supplying the SQL dialect and metadata mappings + * @param select query structure to compile in this scope + */ + Compilation(IRDBMSEngine engine, SelectQueryStruct select) { + this(engine, select, Collections.newSetFromMap(new IdentityHashMap<>())); + } + + /** + * Creates a scope sharing its ancestors' recursion guard. Alias and projection + * maps remain local so nested queries cannot resolve columns through outer + * scopes. + * + * @param engine database supplying the SQL dialect and metadata mappings + * @param select query structure to compile in this scope + * @param activeQueries identity-based set of queries currently being compiled + */ + Compilation(IRDBMSEngine engine, SelectQueryStruct select, Set activeQueries) { + super(engine); + this.select = select; + this.activeQueries = activeQueries; + } + + /** + * Compiles this scope as a top-level query, retaining its requested output + * ordering. + * + * @return independent data and count plans with ordered bindings + */ + CompiledQuery compile() { + return compile(false); + } + + /** + * Guards compilation against query-object cycles such as A containing B + * containing A. The active marker is removed on success or failure, allowing a + * completed subquery to be reused elsewhere in the query structure. + * + * @param nested true when embedding this SELECT in another query; unnecessary + * ordering is omitted from unpaged subqueries + * @return the compiled data and count plans for this scope + * @throws IllegalArgumentException if this query is already active or its + * structure is unsupported + */ + private CompiledQuery compile(boolean nested) { + // Set.add returns false if this exact query is already being compiled + // by an ancestor, for example A -> B -> A. Stop before recursing again. + if (!activeQueries.add(select)) { + throw unsupported("cyclic query structures"); + } + try { + return compose(nested); + } finally { + // Runs on success or failure, including before returning. Removing the + // query allows the same subquery to be reused in another branch. + activeQueries.remove(select); + } + } + + /** + * Builds SELECT, FROM/JOIN, filters, grouping, ordering, and pagination as + * fragments. Derived sources are compiled first for column resolution, then + * their bindings are inserted at their actual position in the final SQL. The + * count plan is built from structured fragments rather than by parsing the + * completed SQL string. + * + * @param nested true for an embedded SELECT whose unpaged ordering can be + * omitted + * @return data and count plans, each with bindings matching its SQL placeholder + * order + * @throws IllegalArgumentException if query structure, aliases, or expressions + * are unsupported + */ + private CompiledQuery compose(boolean nested) { + Objects.requireNonNull(queryUtil, "query utility"); + if (select instanceof HardSelectQueryStruct || select.getCustomFrom() != null + || select.getCustomFromAliasName() != null || select.getSelectors().isEmpty() + || select.getQueryAll() || select.getFrame() != null || !select.getPragmap().isEmpty() + || select.from != null || select.body != null || select.filter != null + || !select.nselectors.isEmpty() || !select.joins.isEmpty() || !select.aliasHash.isEmpty()) { + throw unsupported("raw, frame, query-all, or legacy query structure"); + } + // Register derived sources before resolving any outer columns. Their SQL is + // inserted with its bindings later, at its actual position in FROM. + for (IRelation relation : select.getRelations()) { + if (relation instanceof SubqueryRelationship subquery) { + String name = identifier(subquery.getQueryAlias()); + if (derivedTables.containsKey(name)) { + throw unsupported("duplicate derived-table aliases"); + } + Compilation child = child(subquery.getQs()); + ParameterizedQuery query = child.compile(true).query(); + derivedTables.put(name, child); + derivedQueries.put(name, new Fragment(query.sql(), query.parameters())); + } + } + List selectors = new ArrayList<>(); + List labels = new ArrayList<>(); + for (IQuerySelector selector : select.getSelectors()) { + Fragment expression = projectedExpression(selector); + String label = selector instanceof AbstractQuerySelector abstractSelector + ? abstractSelector.getExplicitAlias() + : null; + if (label == null || label.isEmpty()) { + // Constant-derived aliases would leak values into the SQL template. + label = expression.values.isEmpty() ? selector.getAlias() : "value_" + (selectors.size() + 1); + } + if (labels.contains(label)) { + throw unsupported("duplicate result aliases"); + } + labels.add(label); + projections.put(label, selector); + selectors.add(expression.then(" AS " + alias(label))); + } + Fragment where = filters(combinedFilters(), " AND "); + List groups = new ArrayList<>(); + for (IQuerySelector group : select.getGroupBy()) { + groups.add(expression(group)); + } + Fragment having = filters(select.getHavingFilters().getFilters(), " AND "); + List orders = new ArrayList<>(); + for (IQuerySort sort : select.getCombinedOrderBy()) { + if (!(sort instanceof QueryColumnOrderBySelector column)) { + throw unsupported("custom ordering"); + } + String direction = column.getSortDirString(); + if (!Set.of("", "ASC", "DESC", "ASCENDING", "DESCENDING").contains(direction)) { + throw unsupported("sort direction"); + } + Fragment ordered = column.isPrimKeyColumn() && projections.containsKey(column.getTable()) + ? Fragment.text(alias(column.getTable())) + : expression(column); + orders.add(ordered.then(" " + column.getSortDir().name())); + } + Fragment result = Fragment.text(select.isDistinct() ? "SELECT DISTINCT " : "SELECT ") + .then(Fragment.join(selectors, ", ")).then(" FROM ").then(from()); + if (!where.sql.isEmpty()) { + result = result.then(" WHERE ").then(where); + } + if (!groups.isEmpty()) { + result = result.then(" GROUP BY ").then(Fragment.join(groups, ", ")); + } + if (!having.sql.isEmpty()) { + result = result.then(" HAVING ").then(having); + } + Fragment base = result; + boolean paged = select.getLimit() > 0 || select.getOffset() > 0; + if (!orders.isEmpty() && (!nested || paged)) { + result = result.then(" ORDER BY ").then(Fragment.join(orders, ", ")); + } + Fragment countBody = base; + if (paged) { + if (queryUtil.getDbType() == RdbmsTypeEnum.TERADATA || queryUtil.getDbType() == RdbmsTypeEnum.SYNAPSE) { + countBody = windowPage(base, labels); + result = nested ? countBody : countBody.then(" ORDER BY parameterized_row_number"); + } else { + result = paginate(result, orders.isEmpty(), labels.get(0)); + countBody = result; + } + } + Fragment count = Fragment.text("SELECT COUNT(*) FROM (").then(countBody).then(") parameterized_count"); + return new CompiledQuery(new ParameterizedQuery(result.sql, result.values, 0), + new ParameterizedQuery(count.sql, count.values, 0)); + } + + /** + * Selects filters according to the query's precedence flags. Ignoring filters + * returns none; override mode excludes overlapping implicit filters and omits + * frame/panel filters. Otherwise all explicit, implicit, frame, and panel + * filters participate. The input filter collections and values are left + * unchanged. + * + * @return filters to combine with AND in the WHERE clause + */ + private List combinedFilters() { + if (select.ignoreFilters) { + return List.of(); + } + List combined = new ArrayList<>(select.getExplicitFilters().getFilters()); + if (select.isOverrideImplicit()) { + Set explicitColumns = select.getExplicitFilters().getAllFilteredColumns(); + for (IQueryFilter filter : select.getImplicitFilters().getFilters()) { + if (Collections.disjoint(filter.getAllUsedColumns(), explicitColumns)) { + combined.add(filter); + } + } + } else { + combined.addAll(select.getImplicitFilters().getFilters()); + combined.addAll(select.getFrameImplicitFilters().getFilters()); + combined.addAll(select.getPanelImplicitFilters().getFilters()); + } + return combined; + } + + /** + * Creates an independent alias/metadata scope for an embedded query while + * sharing this compilation's active-query set. Outer table aliases are not + * inherited. + * + * @param query non-null subquery to compile + * @return a fresh child scope using the same engine and recursion guard + * @throws NullPointerException if the subquery is null + */ + private Compilation child(SelectQueryStruct query) { + return new Compilation((IRDBMSEngine) engine, Objects.requireNonNull(query, "subquery"), activeQueries); + } + + /** + * Appends the dialect's pagination syntax using the current SELECT's + * limit/offset. User-supplied bounds become bindings in the order required by + * the dialect. For offset-only queries, some dialects require an additional + * unlimited-limit value. Required fallback ordering uses the first projected + * label. Dialects requiring a ROW_NUMBER rewrite are handled separately by + * {@link #windowPage(Fragment, List)}. + * + * @param query SELECT fragment before pagination, including any explicit + * ORDER BY + * @param needsOrder true when no explicit ORDER BY was rendered + * @param firstLabel first projected label to use when the dialect requires + * ordering + * @return the query with pagination SQL and its additional ordered bindings + */ + private Fragment paginate(Fragment query, boolean needsOrder, String firstLabel) { + long limit = select.getLimit(); + long offset = Math.max(0, select.getOffset()); + RdbmsTypeEnum dialect = queryUtil.getDbType(); + if (dialect == RdbmsTypeEnum.TRINO || dialect == RdbmsTypeEnum.ATHENA) { + if (offset > 0) { + query = query.then(" OFFSET ").then(Fragment.value(offset)); + } + return limit > 0 ? query.then(" LIMIT ").then(Fragment.value(limit)) : query; + } + if (dialect == RdbmsTypeEnum.HIVE) { + query = query.then(" LIMIT "); + if (offset > 0) { + query = query.then(Fragment.value(offset)).then(", "); + } + return query.then(Fragment.value(limit > 0 ? limit : Long.MAX_VALUE)); + } + if (dialect == RdbmsTypeEnum.SQL_SERVER) { + if (needsOrder) { + query = query.then(" ORDER BY " + alias(firstLabel)); + } + query = query.then(" OFFSET ").then(Fragment.value(offset)).then(" ROWS"); + return limit > 0 ? query.then(" FETCH NEXT ").then(Fragment.value(limit)).then(" ROWS ONLY") : query; + } + if (dialect == RdbmsTypeEnum.ORACLE || dialect == RdbmsTypeEnum.DB2 || dialect == RdbmsTypeEnum.DERBY) { + if (offset > 0) { + query = query.then(" OFFSET ").then(Fragment.value(offset)).then(" ROWS"); + } + return limit > 0 ? query.then(" FETCH NEXT ").then(Fragment.value(limit)).then(" ROWS ONLY") : query; + } + if (limit > 0) { + if (dialect == RdbmsTypeEnum.IMPALA && offset > 0 && needsOrder) { + query = query.then(" ORDER BY " + alias(firstLabel)); + } + query = query.then(" LIMIT ").then(Fragment.value(limit)); + } else if (dialect == RdbmsTypeEnum.MYSQL || dialect == RdbmsTypeEnum.MARIADB) { + query = query.then(" LIMIT 18446744073709551615"); + } else if (dialect == RdbmsTypeEnum.SQLITE) { + query = query.then(" LIMIT -1"); + } else if (dialect == RdbmsTypeEnum.BIG_QUERY || dialect == RdbmsTypeEnum.SAP_HANA + || dialect == RdbmsTypeEnum.IMPALA) { + if (dialect == RdbmsTypeEnum.IMPALA && needsOrder) { + query = query.then(" ORDER BY " + alias(firstLabel)); + } + query = query.then(" LIMIT ").then(Fragment.value(Long.MAX_VALUE)); + } + return offset > 0 ? query.then(" OFFSET ").then(Fragment.value(offset)) : query; + } + + /** + * Implements pagination for Teradata and Synapse using a derived ROW_NUMBER + * column and bound row-range predicates. Ordering references projected labels; + * the outer SELECT excludes the generated row number from the returned columns. + * Without an explicit sort, the first projected label supplies the window + * ordering. + * + * @param base SELECT fragment before output ordering and pagination + * @param labels projected result labels in SELECT order + * @return the page-producing SELECT with the original bindings followed by row + * bounds + * @throws IllegalArgumentException if a label is reserved or sorting uses an + * unselected column + * @throws ArithmeticException if offset plus limit overflows a long + */ + private Fragment windowPage(Fragment base, List labels) { + if (labels.contains("parameterized_row_number")) { + throw unsupported("reserved pagination alias"); + } + List order = new ArrayList<>(); + for (IQuerySort sort : select.getCombinedOrderBy()) { + QueryColumnOrderBySelector column = (QueryColumnOrderBySelector) sort; + int position = column.isPrimKeyColumn() ? labels.indexOf(column.getTable()) : -1; + for (int i = 0; position < 0 && i < select.getSelectors().size(); i++) { + if (Objects.equals(select.getSelectors().get(i).getQueryStructName(), + column.getQueryStructName())) { + position = i; + break; + } + } + if (position < 0) { + throw unsupported("pagination sorting by an unselected column on this dialect"); + } + order.add("parameterized_source." + alias(labels.get(position)) + " " + column.getSortDir().name()); + } + if (order.isEmpty()) { + order.add("parameterized_source." + alias(labels.get(0))); + } + Fragment numbered = Fragment.text("SELECT parameterized_source.*, ROW_NUMBER() OVER (ORDER BY " + + String.join(", ", order) + ") AS parameterized_row_number FROM (").then(base) + .then(") parameterized_source"); + List projection = labels.stream().map(label -> "parameterized_page." + alias(label)).toList(); + Fragment result = Fragment.text("SELECT " + String.join(", ", projection) + " FROM (").then(numbered) + .then(") parameterized_page WHERE parameterized_row_number > ") + .then(Fragment.value(Math.max(0, select.getOffset()))); + if (select.getLimit() > 0) { + result = result.then(" AND parameterized_row_number <= ") + .then(Fragment.value(Math.addExact(Math.max(0, select.getOffset()), select.getLimit()))); + } + return result; + } + + /** + * Compiles a SELECT expression, adding an explicit type cast for H2 constant + * projections. H2 must resolve derived-table column types during preparation, + * before JDBC binds values; count queries also wrap the SELECT as a derived + * table. The cast supplies a type without embedding the constant value in SQL. + * + * @param selector expression to project before its result alias is appended + * @return the expression and bindings, with an H2 type hint where required + */ + private Fragment projectedExpression(IQuerySelector selector) { + Fragment expression = expression(selector); + if (queryUtil.getDbType() != RdbmsTypeEnum.H2_DB || !(selector instanceof QueryConstantSelector)) { + return expression; + } + // H2 resolves derived-table columns during prepare, before JDBC can supply + // parameter types. Count queries wrap projections in derived tables too. + Object value = expression.values.getFirst(); + String type; + if (value instanceof Boolean) { + type = "BOOLEAN"; + } else if (value instanceof Byte || value instanceof Short || value instanceof Integer) { + type = "INTEGER"; + } else if (value instanceof Long) { + type = "BIGINT"; + } else if (value instanceof Float || value instanceof Double) { + type = "DOUBLE"; + } else if (value instanceof BigDecimal decimal) { + int scale = Math.max(0, decimal.scale()); + type = "DECIMAL(" + Math.max(1, Math.max(decimal.precision() - decimal.scale(), 0) + scale) + "," + + scale + ")"; + } else if (value instanceof java.sql.Date) { + type = "DATE"; + } else if (value instanceof Timestamp) { + type = "TIMESTAMP(9)"; + } else if (value instanceof OffsetDateTime) { + type = "TIMESTAMP(9) WITH TIME ZONE"; + } else { + type = "VARCHAR"; + } + return Fragment.text("CAST(").then(expression).then(" AS " + type + ")"); + } + + /** + * Recursively renders a selector as an SQL expression without its result alias. + * Columns resolve through physical metadata or a child query's projected + * labels; bare concepts retain the legacy primary-key mapping. Constants become + * bindings, and supported arithmetic, CASE, and function expressions compose + * child fragments in SQL order. Function names, options, and operators are + * validated as structure. + * + * @param selector column, constant, arithmetic, conditional, or supported + * function selector + * @return the SQL expression and its ordered parameter values + * @throws IllegalArgumentException if a selector, identifier, function, or + * option is unsupported + */ + private Fragment expression(IQuerySelector selector) { + if (selector instanceof QueryColumnSelector column) { + if (column.getTableAlias() != null && !column.getTableAlias().isEmpty()) { + throw unsupported("explicit table aliases"); + } + String table = table(column.getTable()); + Compilation derived = derivedTables.get(column.getTable()); + if (derived != null) { + if (!derived.projections.containsKey(column.getColumn())) { + throw unsupported("unknown derived-table column " + column.getColumn()); + } + return Fragment.text(table + "." + alias(column.getColumn())); + } + String name = column.isPrimKeyColumn() ? getPrimKey4Table(column.getTable()) + : getPhysicalPropertyNameFromConceptualName(column.getTable(), identifier(column.getColumn())); + return Fragment.text(table + "." + sqlIdentifier(name)); + } + if (selector instanceof QueryConstantSelector constant) { + return Fragment.value(value(constant.getConstant(), null)); + } + if (selector instanceof QueryArithmeticSelector arithmetic) { + String operator = arithmetic.getMathExpr(); + if (!Set.of("+", "-", "*", "/").contains(operator)) { + throw unsupported("arithmetic operator"); + } + Fragment left = expression(arithmetic.getLeftSelector()); + Fragment right = expression(arithmetic.getRightSelector()); + if (operator.equals("/")) { + right = Fragment.text("NULLIF(").then(right).then(", 0)"); + } + return Fragment.text("(CAST(").then(left).then(" AS DECIMAL) " + operator + " CAST(").then(right) + .then(" AS DECIMAL))"); + } + if (selector instanceof QueryIfSelector conditional) { + Fragment result = Fragment.text("CASE WHEN ").then(filter(conditional.getCondition())).then(" THEN ") + .then(expression(conditional.getPrecedent())); + if (conditional.getAntecedent() != null) { + result = result.then(" ELSE ").then(expression(conditional.getAntecedent())); + } + return result.then(" END"); + } + if (selector instanceof QueryFunctionSelector function) { + String name = Objects.requireNonNull(function.getFunction(), "function").toUpperCase(Locale.ROOT); + boolean distinct = function.isDistinct() || name.startsWith("UNIQUE"); + if (name.startsWith("UNIQUE")) { + name = name.substring("UNIQUE".length()); + } + if (name.equals("MEAN") || name.equals("AVERAGE")) { + name = "AVG"; + } + if (!Set.of("COUNT", "SUM", "MIN", "MAX", "AVG", "LOWER", "UPPER", "COALESCE", "CAST", "REGEXLIKE", + "PATINDEX").contains(name) || !function.getAdditionalFunctionParams().isEmpty() + || (function.getColCast() != null && !function.getColCast().isEmpty()) + || function.getSeparator() != null) { + throw unsupported("function or function options"); + } + int size = function.getInnerSelector().size(); + boolean validArity = switch (name) { + case "COALESCE" -> size >= 2; + case "REGEXLIKE" -> size == 2 || size == 3; + case "PATINDEX" -> size == 2; + default -> size == 1; + }; + if (!validArity || (distinct + && Set.of("LOWER", "UPPER", "COALESCE", "CAST", "REGEXLIKE", "PATINDEX").contains(name))) { + throw unsupported("function arity or distinct option"); + } + List arguments = new ArrayList<>(); + for (IQuerySelector argument : function.getInnerSelector()) { + arguments.add(expression(argument)); + } + if (name.equals("CAST")) { + return Fragment.text("CAST(").then(arguments.getFirst()) + .then(" AS " + castType(function.getDataType()) + ")"); + } + return Fragment.text(queryUtil.getSqlFunctionSyntax(name) + "(" + (distinct ? "DISTINCT " : "")) + .then(Fragment.join(arguments, ", ")).then(")"); + } + throw unsupported("selector " + selector.getSelectorType()); + } + + /** + * Normalizes and validates a CAST target against the supported SQL type + * spellings and optional numeric size/precision arguments. Type names are SQL + * structure and cannot be represented by JDBC value placeholders. The database + * still validates whether a supported spelling and its numeric arguments are + * valid for that dialect. + * + * @param type non-null requested SQL type, such as VARCHAR or DECIMAL(18, 2) + * @return the trimmed, upper-case type expression + * @throws NullPointerException if the type is null + * @throws IllegalArgumentException if the type spelling or argument syntax is + * unsupported + */ + private String castType(String type) { + String normalized = Objects.requireNonNull(type, "cast type").trim().toUpperCase(Locale.ROOT); + if (!normalized.matches( + "(?:INT|INTEGER|SMALLINT|BIGINT|INT64|FLOAT64|DECIMAL|NUMERIC|NUMBER|REAL|FLOAT|DOUBLE|BOOLEAN|BOOL|BIT|DATE|TIMESTAMP|DATETIME|DATETIME2|TEXT|STRING|CHAR|VARCHAR|VARCHAR2|NVARCHAR)(?:\\([0-9]+(?:\\s*,\\s*[0-9]+)?\\))?")) { + throw unsupported("cast type"); + } + return normalized; + } + + /** + * Compiles filters in list order and joins them into a clause body. No WHERE or + * HAVING keyword is added; an empty list produces an empty fragment. + * + * @param filters predicates to compile in their existing order + * @param separator compiler-supplied SQL connector, usually AND or OR with + * spaces + * @return the clause body and its ordered bindings + */ + private Fragment filters(List filters, String separator) { + List parts = new ArrayList<>(); + for (IQueryFilter filter : filters) { + parts.add(filter(filter)); + } + return Fragment.join(parts, separator); + } + + /** + * Compiles one predicate, recursively handling boolean groups, BETWEEN, + * supported boolean functions, and comparisons. Literal-left comparisons are + * reversed so the column can be processed on the left. Single-column subqueries + * receive their own compilation scope; scalar values are bound separately from + * SQL structure. Column-to-value membership and search semantics are delegated + * to {@link #membership(IQuerySelector, String, NounMetadata)}. + * + * @param filter structured predicate to compile + * @return predicate SQL with values in placeholder order + * @throws IllegalArgumentException if the predicate, comparator, or operands + * are unsupported + */ + private Fragment filter(IQueryFilter filter) { + if (filter instanceof AndQueryFilter and) { + return group(and.getFilterList(), " AND ", true); + } + if (filter instanceof OrQueryFilter or) { + return group(or.getFilterList(), " OR ", false); + } + if (filter instanceof BetweenQueryFilter between) { + SemossDataType type = dataType(between.getColumn()); + return expression(between.getColumn()).then(" BETWEEN ") + .then(Fragment.value(value(between.getStart(), type))).then(" AND ") + .then(Fragment.value(value(between.getEnd(), type))); + } + if (filter instanceof FunctionQueryFilter function) { + if (!"RegexLike".equalsIgnoreCase(function.getFunctionSelector().getFunction())) { + throw unsupported("non-boolean filter function"); + } + return expression(function.getFunctionSelector()); + } + if (!(filter instanceof SimpleQueryFilter simple)) { + throw unsupported("filter " + filter.getQueryFilterType()); + } + NounMetadata left = simple.getLComparison(); + NounMetadata right = simple.getRComparison(); + String comparator = Objects.requireNonNull(simple.getComparator(), "comparator").trim(); + // Use current noun types, not SimpleQueryFilter's potentially cached type. + if (left.getNounType() == PixelDataType.COLUMN && right.getNounType() == PixelDataType.COLUMN) { + return expression((IQuerySelector) left.getValue()).then(" " + comparison(comparator) + " ") + .then(expression((IQuerySelector) right.getValue())); + } + if (right.getNounType() == PixelDataType.COLUMN && left.getNounType() != PixelDataType.COLUMN) { + NounMetadata swap = left; + left = right; + right = swap; + comparator = IQueryFilter.getReverseNumericalComparator(comparator); + } + if (left.getNounType() == PixelDataType.COLUMN && right.getNounType() == PixelDataType.QUERY_STRUCT) { + SelectQueryStruct nested = (SelectQueryStruct) right.getValue(); + if (nested.getSelectors().size() != 1) { + throw unsupported("subquery filters with multiple projected columns"); + } + ParameterizedQuery query = child(nested).compile(true).query(); + String operator = switch (comparator) { + case "=", "==" -> "IN"; + case "!=", "<>" -> "NOT IN"; + default -> comparison(comparator); + }; + return expression((IQuerySelector) left.getValue()).then(" " + operator + " (") + .then(new Fragment(query.sql(), query.parameters())).then(")"); + } + if (isScalar(left) && isScalar(right)) { + return Fragment + .value(value(left.getValue(), SemossDataType.convertFromSemossDataType(left.getNounType()))) + .then(" " + comparison(comparator) + " ").then(Fragment.value(value(right.getValue(), + SemossDataType.convertFromSemossDataType(right.getNounType())))); + } + if (left.getNounType() == PixelDataType.COLUMN && Set.of("~", "!~", "~*", "!~*").contains(comparator)) { + if (queryUtil.getDbType() != RdbmsTypeEnum.POSTGRES || !isScalar(right)) { + throw unsupported("regex comparator for this dialect or operand"); + } + return expression((IQuerySelector) left.getValue()).then(" " + comparator + " ") + .then(Fragment.value(value(right.getValue(), SemossDataType.STRING))); + } + if (left.getNounType() != PixelDataType.COLUMN + || SemossDataType.convertFromSemossDataType(right.getNounType()) == null + && right.getNounType() != PixelDataType.NULL_VALUE) { + throw unsupported("subquery, task, lambda, or non-scalar filter operands"); + } + return membership((IQuerySelector) left.getValue(), comparator, right); + } + + /** + * Checks whether a noun's declared type represents a scalar value or null. This + * classifies the operand type only; value conversion separately validates the + * actual Java object before creating a binding. + * + * @param noun operand whose current noun type is inspected + * @return true for a recognized scalar type or explicit null type + */ + private boolean isScalar(NounMetadata noun) { + return noun.getNounType() == PixelDataType.NULL_VALUE + || SemossDataType.convertFromSemossDataType(noun.getNounType()) != null; + } + + /** + * Compiles a parenthesized boolean group. Empty AND groups are true and empty + * OR groups are false, expressed as constant SQL predicates without bindings. + * + * @param children predicates within the group + * @param separator compiler-supplied AND or OR connector, including spaces + * @param emptyResult truth value to render when there are no child predicates + * @return the grouped predicate or its empty-group truth value + */ + private Fragment group(List children, String separator, boolean emptyResult) { + return children.isEmpty() ? Fragment.text(emptyResult ? "1=1" : "1=0") + : Fragment.text("(").then(filters(children, separator)).then(")"); + } + + /** + * Compiles a column/expression comparison against one value or a collection. + * Equality uses IN/NOT IN; ordered comparisons require one non-null value; + * search comparators produce bound LIKE patterns with dialect-specific pattern + * handling. Search normalization changes the bound pattern without + * interpolating it into SQL. + * + *

+ * Nulls become IS NULL/IS NOT NULL conditions, including legacy null-token + * handling for non-string metadata types. Empty collections have explicit + * true/false semantics rather than generating empty IN lists. Mixed null/value + * predicates combine the null check with the non-null comparison. Multiple + * search patterns retain the existing interpreter's OR semantics, including + * negative searches. + * + * @param selector left-hand column or expression + * @param comparator equality, ordered, or supported search comparator + * @param noun right-hand scalar or collection, including its declared + * type + * @return the combined predicate and normalized ordered bindings + * @throws IllegalArgumentException if values cannot be converted or the + * comparison is unsupported + */ + private Fragment membership(IQuerySelector selector, String comparator, NounMetadata noun) { + boolean search = getAllSearchComparators().contains(comparator); + boolean negative = search ? getNegSearchComparators().contains(comparator) + : comparator.equals("!=") || comparator.equals("<>"); + boolean equality = comparator.equals("==") || comparator.equals("=") || negative; + if (!search) { + comparison(comparator); + } + Fragment column = expression(selector); + SemossDataType type = dataType(selector); + SemossDataType bindingType = type == null ? SemossDataType.convertFromSemossDataType(noun.getNounType()) + : type; + Collection inputs = noun.getValue() instanceof Collection values ? values + : Collections.singletonList(noun.getValue()); + List values = new ArrayList<>(); + boolean includeNull = false; + for (Object input : inputs) { + boolean nullToken = type != null && SemossDataType.isNotString(type) + && ("null".equals(input) || "nan".equals(input) + || ((comparator.equals("==") || comparator.equals("=")) && "".equals(input))); + if (input == null || nullToken) { + includeNull = true; + } else { + Object normalized = value(input, search ? null : bindingType); + values.add(normalized); + if (search && Set.of("n", "nu", "nul", "null").contains(normalized)) { + includeNull = true; + } + } + } + if (includeNull && !search && !equality) { + throw unsupported("null with an ordered comparator"); + } + Fragment predicate; + if (values.isEmpty()) { + predicate = Fragment.text(negative ? "1=1" : "1=0"); + } else if (search) { + List patterns = new ArrayList<>(); + for (Object input : values) { + String pattern = input.toString().toLowerCase(Locale.ROOT); + // These dialects interpret backslash as LIKE's default escape character. + if (switch (queryUtil.getDbType()) { + case H2_DB, POSTGRES, MYSQL, MARIADB, REDSHIFT, BIG_QUERY, HIVE, IMPALA, SPARK, DATABRICKS -> true; + default -> false; + }) { + pattern = pattern.replace("\\", "\\\\"); + } + if (comparator.equals("?like") || comparator.equals("?nlike") || comparator.endsWith("ends")) { + pattern = "%" + pattern; + } + if (comparator.equals("?like") || comparator.equals("?nlike") || comparator.endsWith("begins")) { + pattern += "%"; + } + Fragment operand = type == SemossDataType.STRING || type == SemossDataType.FACTOR ? column + : Fragment.text("CAST(").then(column).then(" AS " + searchCastType() + ")"); + patterns.add(Fragment.text("LOWER(").then(operand).then(negative ? ") NOT LIKE " : ") LIKE ") + .then(Fragment.value(pattern))); + } + // Preserve the existing interpreter's OR semantics for multiple patterns. + predicate = Fragment.text("(").then(Fragment.join(patterns, " OR ")).then(")"); + } else if (equality) { + List parameters = values.stream().map(Fragment::value).toList(); + predicate = column.then(negative ? " NOT IN (" : " IN (").then(Fragment.join(parameters, ", ")) + .then(")"); + } else { + if (values.size() != 1) { + throw unsupported("multiple values with an ordered comparator"); + } + predicate = column.then(" " + comparison(comparator) + " ").then(Fragment.value(values.get(0))); + } + if (!includeNull) { + return predicate; + } + Fragment nullCheck = column.then(negative ? " IS NOT NULL" : " IS NULL"); + return values.isEmpty() ? nullCheck + : Fragment.text("(").then(nullCheck).then(negative ? " AND " : " OR ").then(predicate).then(")"); + } + + /** + * Determines the type used to normalize values compared with a selector. + * Derived columns resolve through their child's projected selector. Other + * selectors use an explicit type when present, then non-basic engine metadata + * for columns, including the legacy primary-key mapping for bare concepts. + * + * @param selector expression whose comparison-value type is needed + * @return the resolved SEMOSS type, or null when no recognized type is + * available + */ + private SemossDataType dataType(IQuerySelector selector) { + if (selector instanceof QueryColumnSelector column && derivedTables.containsKey(column.getTable())) { + Compilation derived = derivedTables.get(column.getTable()); + IQuerySelector projected = derived.projections.get(column.getColumn()); + return projected == null ? null : derived.dataType(projected); + } + String type = selector.getDataType(); + if (type == null && selector instanceof QueryColumnSelector column && !engine.isBasic()) { + String property = column.isPrimKeyColumn() ? getPrimKey4Table(column.getTable()) + : getPhysicalPropertyNameFromConceptualName(column.getTable(), column.getColumn()); + String table = getPhysicalTableNameFromConceptualName(column.getTable()); + type = engine.getDataTypes("http://semoss.org/ontologies/Concept/" + property + "/" + table); + if (type == null) { + type = engine + .getDataTypes("http://semoss.org/ontologies/Relation/Contains/" + property + "/" + table); + } + } + return SemossDataType.convertStringToDataType(type); + } + + /** + * Registers a source on first use and returns its generated alias in this + * scope. Ordinary tables also resolve to validated physical paths; derived + * sources keep their compiled child query instead. Registration does not append + * SQL or bindings. + * + * @param name conceptual table name or registered derived-source name + * @return the stable generated alias for this scope, such as t0 + * @throws IllegalArgumentException if a conceptual or physical identifier is + * unsupported + */ + private String table(String name) { + if (!tableAliases.containsKey(name)) { + qualifiedIdentifier(name); + if (!derivedTables.containsKey(name)) { + physicalTables.put(name, qualifiedIdentifier(getPhysicalTableNameFromConceptualName(name))); + } + tableAliases.put(name, "t" + tableAliases.size()); + } + return tableAliases.get(name); + } + + /** + * Renders a FROM/JOIN source with its generated alias. A physical source + * contributes only table text; a derived source contributes its parenthesized + * SELECT and all of that child query's bindings at this position in the parent + * SQL. + * + * @param name conceptual table name or registered derived-source name + * @return the source declaration and any nested-query bindings + */ + private Fragment tableDeclaration(String name) { + String tableAlias = table(name); + Fragment derived = derivedQueries.get(name); + return derived == null ? Fragment.text(physicalTables.get(name) + " " + tableAlias) + : Fragment.text("(").then(derived).then(") " + tableAlias); + } + + /** + * Parses an explicit join endpoint. This compiler requires both table and + * column; it does not infer a join column from a bare table name or OWL + * relationship. Identifier validation occurs when the resulting selector is + * rendered. + * + * @param endpoint join endpoint in table__column form + * @return a column selector for the endpoint + * @throws IllegalArgumentException if the endpoint is null or lacks the + * required two-part form + */ + private QueryColumnSelector joinColumn(String endpoint) { + if (endpoint == null || endpoint.split("__", -1).length != 2) { + throw unsupported("inferred joins; specify table__column endpoints"); + } + return new QueryColumnSelector(endpoint); + } + + /** + * Normalizes supported join spellings, including Pixel's dot-separated form, + * into SQL keywords. Currently accepts inner, left, and right joins. + * + * @param type non-null join spelling from the relationship + * @return INNER JOIN, LEFT JOIN, or RIGHT JOIN + * @throws NullPointerException if the join type is null + * @throws IllegalArgumentException if the join type is unsupported + */ + private String joinType(String type) { + return switch (Objects.requireNonNull(type, "join type").toLowerCase(Locale.ROOT).replace('.', ' ') + .trim()) { + case "inner", "inner join" -> "INNER JOIN"; + case "left", "left outer", "left join", "left outer join" -> "LEFT JOIN"; + case "right", "right outer", "right join", "right outer join" -> "RIGHT JOIN"; + default -> throw unsupported("join type"); + }; + } + + /** + * Builds the FROM/JOIN clause body from registered sources and explicit + * relations. Relations must form an ordered tree: each join connects an + * included source to a new source. Derived-table joins can contribute several + * AND-connected conditions. Their child bindings are inserted with the source + * before the ON-clause bindings. Unjoined sources, repeated sources, and + * unsupported relationship shapes fail instead of producing an implicit + * Cartesian product or inferred join. + * + * @return the clause body without the FROM keyword, with bindings in SQL order + * @throws IllegalArgumentException if sources or relationships cannot form the + * supported join tree + */ + private Fragment from() { + List joins = new ArrayList<>(select.getRelations()); + if (joins.isEmpty()) { + if (tableAliases.size() != 1) { + throw unsupported("missing join between tables"); + } + return tableDeclaration(tableAliases.keySet().iterator().next()); + } + String root; + IRelation first = joins.getFirst(); + if (first instanceof BasicRelationship basic) { + root = joinColumn(basic.getFromConcept()).getTable(); + } else if (first instanceof SubqueryRelationship subquery && !subquery.getJoinOnDetails().isEmpty()) { + String[] on = subquery.getJoinOnDetails().getFirst(); + if (on.length != 3) { + throw unsupported("join condition"); + } + String left = joinColumn(on[0]).getTable(); + root = left.equals(subquery.getQueryAlias()) ? joinColumn(on[1]).getTable() : left; + } else { + throw unsupported("relationship without join conditions"); + } + Set included = new LinkedHashSet<>(); + included.add(root); + Fragment result = tableDeclaration(root); + for (IRelation relation : joins) { + String joinedTable; + String kind; + List conditions = new ArrayList<>(); + if (relation instanceof BasicRelationship basic) { + QueryColumnSelector left = joinColumn(basic.getFromConcept()); + QueryColumnSelector right = joinColumn(basic.getToConcept()); + if (!included.contains(left.getTable()) || included.contains(right.getTable())) { + throw unsupported("joins must form an ordered tree without self joins"); + } + joinedTable = right.getTable(); + kind = joinType(basic.getJoinType()); + conditions.add(expression(left) + .then(" " + comparison(basic.getComparator() == null ? "=" : basic.getComparator()) + " ") + .then(expression(right))); + } else if (relation instanceof SubqueryRelationship subquery) { + joinedTable = subquery.getQueryAlias(); + kind = joinType(subquery.getJoinType()); + if (included.contains(joinedTable) || subquery.getJoinOnDetails().isEmpty()) { + throw unsupported("repeated derived-table alias or missing join condition"); + } + for (String[] on : subquery.getJoinOnDetails()) { + if (on.length != 3) { + throw unsupported("join condition"); + } + QueryColumnSelector left = joinColumn(on[0]); + QueryColumnSelector right = joinColumn(on[1]); + if (!(left.getTable().equals(joinedTable) && included.contains(right.getTable())) + && !(right.getTable().equals(joinedTable) && included.contains(left.getTable()))) { + throw unsupported("disconnected derived-table join"); + } + conditions.add(expression(left).then(" " + comparison(on[2] == null ? "=" : on[2]) + " ") + .then(expression(right))); + } + } else { + throw unsupported("relationship type"); + } + included.add(joinedTable); + result = result.then(" " + kind + " ").then(tableDeclaration(joinedTable)).then(" ON ") + .then(Fragment.join(conditions, " AND ")); + } + if (!included.containsAll(tableAliases.keySet())) { + throw unsupported("unjoined table"); + } + return result; + } + + /** + * Validates and renders a dot-separated table path. Ordinary dialects validate + * each segment and quote keywords as needed. BigQuery permits dashes in path + * segments and quotes the entire project/dataset/table path with backticks. + * + * @param name non-null conceptual or physical table path + * @return the validated path in dialect-appropriate identifier syntax + * @throws NullPointerException if the name is null + * @throws IllegalArgumentException if any segment has unsupported characters or + * is empty + */ + private String qualifiedIdentifier(String name) { + Objects.requireNonNull(name, "identifier"); + if (queryUtil.getDbType() == RdbmsTypeEnum.BIG_QUERY) { + // Project IDs commonly contain dashes; quote the complete table path. + for (String part : name.split("\\.", -1)) { + if (!part.matches("[A-Za-z_][A-Za-z0-9_$-]*")) { + throw unsupported("BigQuery table path"); + } + } + return "`" + name + "`"; + } + List parts = new ArrayList<>(); + for (String part : name.split("\\.", -1)) { + parts.add(sqlIdentifier(part)); + } + return String.join(".", parts); + } + + /** + * Validates one unqualified identifier and quotes it when the query utility + * marks it as a keyword. This accepts SQL structure only, not arbitrary SQL + * expressions. + * + * @param name column or table-path segment to render + * @return the validated identifier, quoted when required for a keyword + * @throws IllegalArgumentException if the identifier has an unsupported + * spelling + */ + private String sqlIdentifier(String name) { + identifier(name); + return queryUtil.isSelectorKeyword(name) ? alias(name) : name; + } + + /** + * Validates and quotes one result alias for use in projections and references. + * Uses explicit backticks for dialects that require them and the query + * utility's quoting convention for the remaining dialects. + * + * @param name unqualified alias to render + * @return the quoted alias + * @throws IllegalArgumentException if the alias has an unsupported spelling + */ + private String alias(String name) { + identifier(name); + return switch (queryUtil.getDbType()) { + case BIG_QUERY, HIVE, IMPALA, SPARK, DATABRICKS -> "`" + name + "`"; + default -> queryUtil.getEscapeKeyword(name); + }; + } + + /** + * Chooses the dialect's text type for applying LIKE to a non-string expression. + * The returned text is compiler-selected SQL structure, not a bound value. + * + * @return a text CAST target appropriate to the current dialect + */ + private String searchCastType() { + return switch (queryUtil.getDbType()) { + case BIG_QUERY, HIVE, IMPALA, SPARK, DATABRICKS -> "STRING"; + case MYSQL, MARIADB -> "CHAR"; + case SQL_SERVER, SYNAPSE -> "NVARCHAR(4000)"; + default -> queryUtil.getVarcharDataTypeName(); + }; + } + } + + /** + * Validates a single simple identifier before it is used as SQL structure. + * Names must begin with a letter or underscore, followed by letters, digits, + * underscores, or dollar signs. Qualified paths are validated one segment at a + * time elsewhere. + * + * @param name unquoted schema, table, column, or alias segment + * @return the supplied name unchanged after validation + * @throws IllegalArgumentException if the name is null or has unsupported + * characters + */ + private static String identifier(String name) { + if (name == null || !name.matches("[A-Za-z_][A-Za-z0-9_$]*")) { + throw unsupported("identifier; use simple schema, table, column, and alias names"); + } + return name; + } + + /** + * Maps supported equality and ordered comparators to SQL operators. Pixel + * equality and inequality spellings normalize to SQL equivalents. Search/regex + * comparators are handled separately when compiling filters. + * + * @param comparator equality or ordered comparison operator + * @return the normalized SQL operator + * @throws IllegalArgumentException if the operator is unsupported + */ + private static String comparison(String comparator) { + return switch (comparator) { + case "=", "==" -> "="; + case "!=", "<>" -> "<>"; + case "<", "<=", ">", ">=" -> comparator; + default -> throw unsupported("comparator"); + }; + } + + /** + * Normalizes one value for JDBC binding without SQL escaping. Enums use their + * text; a type hint converts textual numbers, booleans, dates, and timestamps. + * Supported Java/SEMOSS temporal objects become JDBC values, and BigInteger + * becomes BigDecimal. Collections are expanded by the caller, not bound as + * scalar objects. + * + *

+ * ZonedDateTime and SemossDate conversions retain local date/time fields; + * OffsetDateTime retains its offset, and Instant is converted with + * Timestamp.from. Copies of supported mutable JDBC date/timestamp inputs are + * made during this normalization, independently of the later parameter-list + * snapshot. + * + * @param input scalar to normalize; null remains null + * @param type optional comparison type used for textual and SEMOSS date + * conversion + * @return a supported JDBC value, with untyped strings left unescaped + * @throws IllegalArgumentException if the value is unsupported or cannot be + * converted to its type + */ + private static Object value(Object input, SemossDataType type) { + if (input == null) { + return null; + } + if (input instanceof Enum enumeration) { + input = enumeration.toString(); + } + if (input instanceof String text && type != null) { + return switch (type) { + case INT -> Long.valueOf(text); + case DOUBLE -> new BigDecimal(text); + case BOOLEAN -> { + if (!text.equalsIgnoreCase("true") && !text.equalsIgnoreCase("false")) { + throw new IllegalArgumentException("Expected a boolean filter value"); + } + yield Boolean.valueOf(text); + } + case DATE -> java.sql.Date.valueOf(text); + case TIMESTAMP -> Timestamp.valueOf(text.replace('T', ' ')); + default -> text; + }; + } + if (input instanceof SemossDate date) { + LocalDateTime local = Objects.requireNonNull(date.getZonedDateTime(), "date").toLocalDateTime(); + return type == SemossDataType.DATE + || type == null && !date.patternHasTime() && local.toLocalTime().equals(LocalTime.MIDNIGHT) + ? java.sql.Date.valueOf(local.toLocalDate()) + : Timestamp.valueOf(local); + } + if (input instanceof Timestamp timestamp) { + return Timestamp.valueOf(timestamp.toLocalDateTime()); + } + if (input instanceof java.sql.Date date) { + return java.sql.Date.valueOf(date.toLocalDate()); + } + if (input instanceof LocalDate date) { + return java.sql.Date.valueOf(date); + } + if (input instanceof LocalDateTime date) { + return Timestamp.valueOf(date); + } + if (input instanceof ZonedDateTime date) { + return Timestamp.valueOf(date.toLocalDateTime()); + } + if (input instanceof OffsetDateTime date) { + return date; + } + if (input instanceof Instant instant) { + return Timestamp.from(instant); + } + if (input instanceof java.util.Date date) { + return new Timestamp(date.getTime()); + } + if (input instanceof BigInteger integer) { + return new BigDecimal(integer); + } + if (input instanceof String || input instanceof Boolean || input instanceof Byte || input instanceof Short + || input instanceof Integer || input instanceof Long || input instanceof Float + || input instanceof Double || input instanceof BigDecimal) { + return input; + } + throw unsupported("non-scalar parameter value"); + } + + /** + * Creates a consistent compilation error for a feature that cannot be rendered + * through the prepared-query path. Callers throw the result rather than falling + * back to literal SQL generation. + * + * @param feature description of the unsupported query feature or value + * @return the exception to throw + */ + private static IllegalArgumentException unsupported(String feature) { + return new IllegalArgumentException("Parameterized SQL does not support " + feature); + } +} diff --git a/src/prerna/query/querystruct/selectors/AbstractQuerySelector.java b/src/prerna/query/querystruct/selectors/AbstractQuerySelector.java index 69deacb4d3e..6ad45979c7a 100644 --- a/src/prerna/query/querystruct/selectors/AbstractQuerySelector.java +++ b/src/prerna/query/querystruct/selectors/AbstractQuerySelector.java @@ -32,13 +32,13 @@ public abstract class AbstractQuerySelector implements IQuerySelector, Serializable { protected String alias; - + /** * Default constructor */ public AbstractQuerySelector() { // we want the alias to be an empty string - // since we dont want to get null pointers + // since we dont want to get null pointers // when we to the equals when we merge selectors this.alias = ""; } @@ -54,7 +54,14 @@ public void setAlias(String alias) { } } } - + + /** + * Returns only the caller-supplied alias, before any value-derived default. + */ + public String getExplicitAlias() { + return this.alias; + } + @Override public String toString() { return this.getQueryStructName(); diff --git a/src/prerna/rdf/engine/wrappers/RawPreparedRDBMSSelectWrapper.java b/src/prerna/rdf/engine/wrappers/RawPreparedRDBMSSelectWrapper.java new file mode 100644 index 00000000000..c67d0526b3f --- /dev/null +++ b/src/prerna/rdf/engine/wrappers/RawPreparedRDBMSSelectWrapper.java @@ -0,0 +1,456 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.rdf.engine.wrappers; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Objects; + +import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.query.interpreters.sql.ParameterizedSqlInterpreter.CompiledQuery; +import prerna.usertracking.UserQueryTrackingThread; +import prerna.util.ConnectionUtils; +import prerna.util.QueryExecutionUtility; +import prerna.util.QueryExecutionUtility.ParameterizedQuery; + +/** + * Streams a compiled SELECT through JDBC prepared statements, retaining the SQL + * templates and ordered bindings for data, count, and reset operations. + * Inherits row conversion, headers, types, and the iterator contract from + * {@link RawRDBMSSelectWrapper}; this class owns the prepared execution + * lifecycle. + * + *

+ * Use the factories to choose engine-acquired or caller-owned connections and + * close the returned wrapper after use. Ownership is fixed at construction: + * inherited connection-close flags do not change it. Neither execution nor + * cleanup changes auto-commit or commits a transaction. + */ +public class RawPreparedRDBMSSelectWrapper extends RawRDBMSSelectWrapper { + + /** + * Data-query SQL template, ordered JDBC bindings, and maximum-row limit. + * Retained for deferred execution and reset so neither operation needs to + * reconstruct values from SQL. + */ + private final ParameterizedQuery parameterizedQuery; + + /** + * Compiler-generated count template with its own bindings. SQL pagination is + * included, so it counts the returned page rather than an unrestricted result + * set. + */ + private final ParameterizedQuery parameterizedCountQuery; + + /** + * Connection supplied by the caller. The wrapper closes its own statements and + * results but never commits, rolls back, changes auto-commit, or closes this + * connection. It is also reused for count queries and reset. + */ + private final Connection borrowedConnection; + + /** + * True for engine-acquired execution, where the wrapper manages the read + * lifecycle and releases pooled connections. False for a borrowed connection, + * regardless of its auto-commit mode. Shared non-pooled engine connections + * remain open. + */ + private final boolean ownsReadTransaction; + + /** + * True only when this wrapper owns the read and its connection has auto-commit + * disabled. Cleanup then rolls back to end that read transaction. This records + * the existing mode; execution does not change auto-commit or commit reads. + */ + private boolean manualReadTransaction; + + /** + * Whether numRows contains a successfully computed count, including zero. + * Cleared on execution/reset so an old count is not reused for a new execution. + */ + private boolean parameterizedCountKnown; + + /** + * JDBC statement timeout applied to data and count statements. Zero means no + * statement timeout; this is not a limit on how long a caller may retain the + * wrapper. + */ + private final int parameterizedTimeoutSeconds; + + /** + * Accepts only the retained SQL template so bindings cannot become detached. + * + * @param query SQL template already associated with this wrapper + * @throws IllegalStateException if a different template is supplied + */ + @Override + public void setQuery(String query) { + if (!parameterizedQuery.sql().equals(query)) { + throw new IllegalStateException("Create a new compiled wrapper to change its SQL or bindings"); + } + super.setQuery(query); + } + + /** + * Accepts only the original engine, preserving connection ownership and + * dialect. + * + * @param engine engine already associated with this wrapper + * @throws IllegalStateException if a different engine is supplied + */ + @Override + public void setEngine(IDatabaseEngine engine) { + if (this.engine != engine) { + throw new IllegalStateException("A compiled wrapper must retain its database and connection ownership"); + } + super.setEngine(engine); + } + + /** + * Retains the compiled plans and fixes ownership before the wrapper is exposed. + * No JDBC resources are acquired until {@link #execute()} is called. + * + * @param database engine supplying connections and result metadata + * conventions + * @param connection caller-owned connection, or null to acquire from the + * engine + * @param compiled data and count plans with their ordered values + * @param timeoutSeconds non-negative JDBC statement timeout; zero disables it + * @throws NullPointerException if the database or compiled plans are null + * @throws IllegalArgumentException if the timeout is negative + */ + private RawPreparedRDBMSSelectWrapper(IRDBMSEngine database, Connection connection, CompiledQuery compiled, + int timeoutSeconds) { + this.engine = Objects.requireNonNull(database, "database"); + Objects.requireNonNull(compiled, "compiled query"); + if (timeoutSeconds < 0) { + throw new IllegalArgumentException("Query timeout must be non-negative"); + } + this.parameterizedQuery = compiled.query(); + this.parameterizedCountQuery = compiled.countQuery(); + this.query = parameterizedQuery.sql(); + this.borrowedConnection = connection; + this.ownsReadTransaction = connection == null; + this.parameterizedTimeoutSeconds = timeoutSeconds; + this.closedConnection = true; + } + + /** + * Creates a prepared wrapper without acquiring a connection or executing SQL. + * Calling execute starts the engine-owned read lifecycle; close or exhaustion + * ends any owned manual transaction and releases pooled connections. Do not + * interleave unrelated work on a shared engine connection while the wrapper is + * open. + * + * @param database non-null engine supplying connections and result + * metadata conventions + * @param compiled non-null data and count plans with their ordered + * parameter values + * @param timeoutSeconds non-negative timeout for data and count statements; + * zero means no JDBC statement timeout + * @return an unexecuted wrapper retaining both compiled plans + * @throws NullPointerException if the database or compiled plan is null + * @throws IllegalArgumentException if the timeout is negative + */ + public static RawPreparedRDBMSSelectWrapper prepareParameterized(IRDBMSEngine database, CompiledQuery compiled, + int timeoutSeconds) { + return new RawPreparedRDBMSSelectWrapper(database, null, compiled, timeoutSeconds); + } + + /** + * Executes a compiled query using an engine-acquired connection. The wrapper + * owns read cleanup and must be closed by the caller after consuming the + * results. + * + * @param database engine supplying the connection and result metadata + * conventions + * @param query compiled data and count plans with their ordered + * bindings + * @param timeoutSeconds non-negative JDBC statement timeout; zero means no + * timeout + * @return an executed wrapper that streams the result rows + * @throws Exception if arguments are invalid or execution/resource + * initialization fails + */ + public static RawPreparedRDBMSSelectWrapper executeParameterized(IRDBMSEngine database, CompiledQuery query, + int timeoutSeconds) throws Exception { + RawPreparedRDBMSSelectWrapper wrapper = prepareParameterized(database, query, timeoutSeconds); + wrapper.execute(); + return wrapper; + } + + /** + * Executes a compiled query using the caller's connection. The returned wrapper + * closes only its statements and results, including on failure. It never + * commits, rolls back, changes auto-commit, or closes the supplied connection. + * + * @param database engine supplying result metadata conventions and + * tracking identity + * @param connection non-null connection whose transaction and lifecycle + * remain with the caller + * @param query compiled data and count plans with their ordered + * bindings + * @param timeoutSeconds non-negative JDBC statement timeout; zero means no + * timeout + * @return an executed streaming wrapper that the caller must close + * @throws Exception if arguments are invalid or execution/resource + * initialization fails + */ + public static RawPreparedRDBMSSelectWrapper executeParameterized(IRDBMSEngine database, Connection connection, + CompiledQuery query, int timeoutSeconds) throws Exception { + Objects.requireNonNull(connection, "connection"); + RawPreparedRDBMSSelectWrapper wrapper = new RawPreparedRDBMSSelectWrapper(database, connection, query, + timeoutSeconds); + wrapper.execute(); + return wrapper; + } + + /** + * Closes the previous execution, clears row/count state, and prepares and binds + * the retained data query. Acquires an engine connection or reuses the borrowed + * one according to ownership. Used for both initial execution and reset; + * cleanup also runs if preparation, binding, execution, or metadata + * initialization fails. + * + * @throws Exception if execution or resource initialization/cleanup fails + */ + @Override + public void execute() throws Exception { + close(); + this.conn = null; + this.stmt = null; + this.rs = null; + this.currRow = null; + this.manualReadTransaction = false; + this.parameterizedCountKnown = false; + this.numRows = 0; + this.closedConnection = false; + UserQueryTrackingThread tracker = WrapperManager.preparedQueryTracker(engine, parameterizedQuery.sql()); + try { + if (tracker != null) { + tracker.setStartTimeNow(); + } + IRDBMSEngine database = (IRDBMSEngine) this.engine; + this.conn = ownsReadTransaction ? database.getConnection() : borrowedConnection; + // Observe the existing mode without changing auto-commit. A borrowed + // connection never becomes our transaction to finish, even in manual mode. + this.manualReadTransaction = ownsReadTransaction && !this.conn.getAutoCommit(); + this.databaseZoneId = database.getDatabaseZoneId(); + PreparedStatement prepared = conn.prepareStatement(parameterizedQuery.sql()); + this.stmt = prepared; + prepared.setMaxRows(parameterizedQuery.limit()); + prepared.setQueryTimeout(parameterizedTimeoutSeconds); + bindParameterized(prepared, parameterizedQuery); + this.rs = prepared.executeQuery(); + setVariables(); + } catch (Exception | Error failure) { + if (tracker != null) { + tracker.setFailed(); + } + closeAfterParameterizedFailure(failure); + throw failure; + } finally { + if (tracker != null) { + WrapperManager.finishPreparedQueryTracking(tracker); + } + } + } + + /** + * Binds a plan's values in list order using one-based JDBC parameter positions. + * The same operation is used for data and count plans with their respective + * lists. + * + * @param statement prepared statement whose placeholders match the plan + * @param bindings plan containing the ordered values, including nulls + * @throws SQLException if the driver rejects a parameter binding + */ + private void bindParameterized(PreparedStatement statement, ParameterizedQuery bindings) throws SQLException { + for (int i = 0; i < bindings.parameters().size(); i++) { + statement.setObject(i + 1, bindings.parameters().get(i)); + } + } + + /** + * Executes and caches the independent count plan, including a legitimate zero. + * Uses the live execution connection when available. After engine-owned + * execution has closed, counting acquires a separate engine-managed read; a + * borrowed connection remains the caller's responsibility in every case. + * + * @return the count including SQL pagination, capped by a positive JDBC row + * limit + * @throws IllegalArgumentException if the count cannot be obtained + */ + @Override + public long getNumRows() { + if (!this.parameterizedCountKnown) { + // The compiler supplies an independent template and bindings for counting. + String countSql = parameterizedCountQuery.sql(); + QueryExecutionUtility.StatementBinder binder = statement -> { + statement.setQueryTimeout(parameterizedTimeoutSeconds); + bindParameterized(statement, parameterizedCountQuery); + }; + UserQueryTrackingThread tracker = WrapperManager.preparedQueryTracker(engine, countSql); + try { + if (tracker != null) { + tracker.setStartTimeNow(); + } + long count; + if (closedConnection && ownsReadTransaction) { + count = QueryExecutionUtility.queryOne((IRDBMSEngine) engine, countSql, binder, + result -> result.getLong(1)); + } else { + Connection countConnection = ownsReadTransaction ? conn : borrowedConnection; + count = QueryExecutionUtility.queryOne(countConnection, countSql, binder, + result -> result.getLong(1)); + } + this.numRows = parameterizedQuery.limit() > 0 ? Math.min(count, parameterizedQuery.limit()) : count; + this.parameterizedCountKnown = true; + } catch (Exception | Error failure) { + if (tracker != null) { + tracker.setFailed(); + } + closeAfterParameterizedFailure(failure); + if (failure instanceof Error error) { + throw error; + } + throw new IllegalArgumentException("Error counting parameterized query", failure); + } finally { + WrapperManager.finishPreparedQueryTracking(tracker); + } + } + return this.numRows; + } + + /** + * Attempts prepared-query cleanup while retaining the original failure. A + * distinct cleanup failure is attached as a suppressed exception instead of + * replacing it. + * + * @param failure original execution or counting failure to preserve + */ + private void closeAfterParameterizedFailure(Throwable failure) { + try { + close(); + } catch (RuntimeException | Error cleanup) { + // failure is the non-null original exception being propagated by the caller. + ConnectionUtils.jdbcCleanupFailure(failure, cleanup); + } + } + + /** + * Closes this execution's result set and statement. An owned manual read is + * ended with rollback, and an owned pooled connection is released. Borrowed + * connections and their transactions are left to the caller. Every cleanup step + * is attempted, and subsequent failures are suppressed onto the first failure. + * + * @throws IllegalStateException if resource or transaction cleanup fails with a + * non-Error failure; the first failure is the + * cause + */ + @Override + public void close() { + if (this.closedConnection) { + return; + } + this.closedConnection = true; + this.currRow = null; + Throwable failure = ConnectionUtils.closeAllConnections(null, this.stmt, this.rs); + if (this.manualReadTransaction && this.conn != null) { + failure = ConnectionUtils.jdbcCleanupFailure(failure, ConnectionUtils.rollbackConnection(this.conn)); + } + if (this.ownsReadTransaction && this.conn != null) { + failure = ConnectionUtils.jdbcCleanupFailure(failure, + ConnectionUtils.closeConnectionIfPooling((IRDBMSEngine) engine, this.conn)); + } + if (failure instanceof Error error) { + throw error; + } + if (failure != null) { + throw new IllegalStateException("Unable to close parameterized query", failure); + } + } + + /** + * Reads ahead through the inherited JDBC row converter. Any cursor, conversion, + * or exhaustion cleanup failure closes this execution before it propagates. + * + * @return true if a buffered row is available; false before execution or after + * close + * @throws IllegalArgumentException if a JDBC row read or exhaustion cleanup + * fails + */ + @Override + public boolean hasNext() { + if (this.closedConnection) { + return false; + } + try { + if (this.currRow == null) { + this.currRow = getNextRow(); + } + return this.currRow != null; + } catch (SQLException failure) { + closeAfterParameterizedFailure(failure); + throw new IllegalArgumentException("Error reading parameterized query", failure); + } catch (RuntimeException | Error failure) { + closeAfterParameterizedFailure(failure); + throw failure; + } + } + + /** + * Uses the shared column mappings, failing execution if metadata is + * unavailable. The enclosing execute call then releases the resources it + * acquired. + * + * @throws IllegalArgumentException if the driver cannot provide result metadata + */ + @Override + protected void setVariables() { + try { + readResultMetadata(); + } catch (SQLException failure) { + throw new IllegalArgumentException("Unable to read prepared result metadata", failure); + } + } + + /** + * Re-executes the retained data plan with its original bindings and timeout. + * Any previous result is closed and the cached row count is cleared first. + * + * @throws Exception if cleanup, execution, or result initialization fails + */ + @Override + public void reset() throws Exception { + execute(); + } + +} diff --git a/src/prerna/rdf/engine/wrappers/RawRDBMSSelectWrapper.java b/src/prerna/rdf/engine/wrappers/RawRDBMSSelectWrapper.java index 4930f4bec59..690c788aa89 100644 --- a/src/prerna/rdf/engine/wrappers/RawRDBMSSelectWrapper.java +++ b/src/prerna/rdf/engine/wrappers/RawRDBMSSelectWrapper.java @@ -64,7 +64,6 @@ import prerna.query.querystruct.SelectQueryStruct; import prerna.usertracking.UserQueryTrackingThread; import prerna.util.ConnectionUtils; -import prerna.util.Constants; import prerna.util.sql.AbstractSqlQueryUtil; import prerna.util.sql.RdbmsTypeEnum; @@ -108,7 +107,7 @@ public void execute() throws Exception { setVariables(); this.databaseZoneId = engine.getDatabaseZoneId(); } catch (Exception e) { - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to execute RDBMS select query", e); if (this.useEngineConnection) { ConnectionUtils.closeAllConnections(null, stmt, rs); } else { @@ -155,7 +154,7 @@ public boolean hasNext() { } } catch (SQLException e) { - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to read the next RDBMS result row", e); if (e.getMessage() != null && !e.getMessage().isEmpty()) { throw new IllegalArgumentException( "Error occurred grabbing next row for query. Detailed message = " + e.getMessage()); @@ -166,201 +165,213 @@ public boolean hasNext() { return false; } - private IHeadersDataRow getNextRow() throws SQLException { - if (rs.next()) { - Object[] row = new Object[numColumns]; - // iterate through all the columns to get the appropriate data types - for (int colNum = 1; colNum <= numColumns; colNum++) { - Object val = null; - int type = colTypes[colNum - 1]; - if (type == Types.INTEGER) { - val = rs.getInt(colNum); - } else if (type == Types.BIGINT) { - val = rs.getLong(colNum); - } else if (type == Types.FLOAT || type == Types.DOUBLE || type == Types.NUMERIC || type == Types.DECIMAL - || type == Types.REAL) { - val = rs.getDouble(colNum); - } else if (type == Types.DATE) { - try { - Date dVal = rs.getDate(colNum); - if (dVal == null) { - val = null; - } else { - val = new SemossDate(dVal.toInstant(), this.databaseZoneId, "yyyy-MM-dd"); - } - } catch (Exception e) { - // some rdbms do not actually support dates - // and just return a string - // ex: SQLite - try { - String dateValStr = rs.getString(colNum); - // does the string represent a long? - try { - long dateLong = Long.parseLong(dateValStr); - val = new SemossDate(dateLong, "yyyy-MM-dd", this.databaseZoneId); - } catch (NumberFormatException nfee) { - val = new SemossDate(dateValStr, "yyyy-MM-dd", this.databaseZoneId); - } - } catch (Exception e2) { - // out of luck... - classLogger.error(Constants.STACKTRACE, e); - classLogger.error(Constants.STACKTRACE, e2); - } + /** + * Advances the result set and converts the row using the shared JDBC mappings. + * Calls {@link #close()} when no more rows are available. + * + * @return the next row, or null after exhaustion + * @throws SQLException if advancing, reading, or exhaustion cleanup fails + */ + protected IHeadersDataRow getNextRow() throws SQLException { + if (!rs.next()) { + close(); + return null; + } + + Object[] row = new Object[numColumns]; + // iterate through all the columns to get the appropriate data types + for (int colNum = 1; colNum <= numColumns; colNum++) { + Object val = null; + int type = colTypes[colNum - 1]; + if (type == Types.INTEGER) { + val = rs.getInt(colNum); + } else if (type == Types.BIGINT) { + val = rs.getLong(colNum); + } else if (type == Types.FLOAT || type == Types.DOUBLE || type == Types.NUMERIC || type == Types.DECIMAL + || type == Types.REAL) { + val = rs.getDouble(colNum); + } else if (type == Types.DATE) { + try { + Date dVal = rs.getDate(colNum); + if (dVal == null) { + val = null; + } else { + val = new SemossDate(dVal.toInstant(), this.databaseZoneId, "yyyy-MM-dd"); } - } else if (type == Types.TIMESTAMP) { + } catch (Exception e) { + // some rdbms do not actually support dates + // and just return a string + // ex: SQLite try { - Timestamp dVal = rs.getTimestamp(colNum); - if (dVal == null) { - val = null; - } else { - val = new SemossDate(dVal, this.databaseZoneId, "yyyy-MM-dd HH:mm:ss"); - } - } catch (Exception e) { - // some rdbms do not actually support dates - // and just return a string - // ex: SQLite + String dateValStr = rs.getString(colNum); + // does the string represent a long? try { - String dateValStr = rs.getString(colNum); - val = new SemossDate(dateValStr, "yyyy-MM-dd HH:mm:ss", this.databaseZoneId); - } catch (Exception e2) { - // out of luck... - classLogger.error(Constants.STACKTRACE, e); - classLogger.error(Constants.STACKTRACE, e2); + long dateLong = Long.parseLong(dateValStr); + val = new SemossDate(dateLong, "yyyy-MM-dd", this.databaseZoneId); + } catch (NumberFormatException nfee) { + val = new SemossDate(dateValStr, "yyyy-MM-dd", this.databaseZoneId); } + } catch (Exception e2) { + // out of luck... + classLogger.error("Failed to read DATE value from result column {}", colNum, e); + classLogger.error("Failed to convert string fallback to DATE for result column {}", colNum, e2); } - } else if (type == Types.CLOB) { - val = rs.getClob(colNum); - try { - val = AbstractSqlQueryUtil.flushClobToString((java.sql.Clob) val); - } catch (Exception e) { - classLogger.error(Constants.STACKTRACE, e); - if (!rs.wasNull()) { - val = rs.getString(colNum); - } + } + } else if (type == Types.TIMESTAMP) { + try { + Timestamp dVal = rs.getTimestamp(colNum); + if (dVal == null) { + val = null; + } else { + val = new SemossDate(dVal, this.databaseZoneId, "yyyy-MM-dd HH:mm:ss"); } - } else if (type == Types.BLOB) { - val = rs.getBlob(colNum); + } catch (Exception e) { + // some rdbms do not actually support dates + // and just return a string + // ex: SQLite try { - val = AbstractSqlQueryUtil.flushBlobToString((java.sql.Blob) val); - } catch (IOException e) { - classLogger.error(Constants.STACKTRACE, e); - } catch (NullPointerException e) { - if (!rs.wasNull()) { - val = rs.getString(colNum); - } + String dateValStr = rs.getString(colNum); + val = new SemossDate(dateValStr, "yyyy-MM-dd HH:mm:ss", this.databaseZoneId); + } catch (Exception e2) { + // out of luck... + classLogger.error("Failed to read TIMESTAMP value from result column {}", colNum, e); + classLogger.error("Failed to convert string fallback to TIMESTAMP for result column {}", colNum, + e2); } - } else if (type == Types.BINARY) { - try (InputStream is = rs.getBinaryStream(colNum); - ByteArrayOutputStream baos = new ByteArrayOutputStream();) { - if (is != null) { - byte[] buffer = new byte[1024]; - int length; - - while ((length = is.read(buffer)) != -1) { - baos.write(buffer, 0, length); - } - - val = baos.toString("UTF-8"); - } - } catch (IOException e) { - classLogger.error(Constants.STACKTRACE, e); + } + } else if (type == Types.CLOB) { + val = rs.getClob(colNum); + try { + val = AbstractSqlQueryUtil.flushClobToString((java.sql.Clob) val); + } catch (Exception e) { + classLogger.error("Failed to convert CLOB to text for result column {}", colNum, e); + if (!rs.wasNull()) { + val = rs.getString(colNum); } - } else if (type == Types.ARRAY) { - Array arrVal = rs.getArray(colNum); - if (arrVal != null) { - val = arrVal.getArray(); + } + } else if (type == Types.BLOB) { + val = rs.getBlob(colNum); + try { + val = AbstractSqlQueryUtil.flushBlobToString((java.sql.Blob) val); + } catch (IOException e) { + classLogger.error("Failed to convert BLOB to text for result column {}", colNum, e); + } catch (NullPointerException e) { + if (!rs.wasNull()) { + val = rs.getString(colNum); } - } else if (type == Types.VARBINARY) { - byte[] bytes = rs.getBytes(colNum); - if (bytes != null) { - try { - val = new String(bytes, "UTF-8"); - } catch (UnsupportedEncodingException e) { - classLogger.error(Constants.STACKTRACE, e); + } + } else if (type == Types.BINARY) { + try (InputStream is = rs.getBinaryStream(colNum); + ByteArrayOutputStream baos = new ByteArrayOutputStream();) { + if (is != null) { + byte[] buffer = new byte[1024]; + int length; + + while ((length = is.read(buffer)) != -1) { + baos.write(buffer, 0, length); } + + val = baos.toString("UTF-8"); } - } else if (type == Types.BOOLEAN || type == Types.BIT) { + } catch (IOException e) { + classLogger.error("Failed to read binary stream for result column {}", colNum, e); + } + } else if (type == Types.ARRAY) { + Array arrVal = rs.getArray(colNum); + if (arrVal != null) { + val = arrVal.getArray(); + } + } else if (type == Types.VARBINARY) { + byte[] bytes = rs.getBytes(colNum); + if (bytes != null) { try { - val = rs.getBoolean(colNum); - } catch (SQLDataException e) { - // sometimes, this is stored as an integer or string - // as an example, opensearch - try { - val = rs.getInt(colNum); - if (val != null) { - if (((int) val) == 0) { - val = false; - } else { - val = true; - } - } - } catch (SQLDataException e2) { - val = rs.getString(colNum); - if (val != null) { - if (Integer.parseInt(val + "") == 0) { - val = false; - } else { - val = true; - } - } - } + val = new String(bytes, "UTF-8"); + } catch (UnsupportedEncodingException e) { + classLogger.error("Failed to decode VARBINARY as UTF-8 for result column {}", colNum, e); } } - // just grab the object and see what happens... - else if (type == Types.OTHER) { + } else if (type == Types.BOOLEAN || type == Types.BIT) { + try { + val = rs.getBoolean(colNum); + } catch (SQLDataException e) { + // sometimes, this is stored as an integer or string + // as an example, opensearch try { - val = rs.getObject(colNum); - } catch (Exception e) { - classLogger.error(Constants.STACKTRACE, e); + val = rs.getInt(colNum); + if (val != null) { + if (((int) val) == 0) { + val = false; + } else { + val = true; + } + } + } catch (SQLDataException e2) { + val = rs.getString(colNum); + if (val != null) { + if (Integer.parseInt(val + "") == 0) { + val = false; + } else { + val = true; + } + } } - } else { - val = rs.getString(colNum); } - - // need to account for null values - if (rs.wasNull()) { - val = null; + } + // just grab the object and see what happens... + else if (type == Types.OTHER) { + try { + val = rs.getObject(colNum); + } catch (Exception e) { + classLogger.error("Failed to read JDBC OTHER value from result column {}", colNum, e); } - - row[colNum - 1] = val; + } else { + val = rs.getString(colNum); } - // return the header row - return new HeadersDataRow(headers, rawHeaders, row, row); - } else { - try { - close(); - } catch (IOException e) { - classLogger.error(Constants.STACKTRACE, e); + // need to account for null values + if (rs.wasNull()) { + val = null; } + + row[colNum - 1] = val; } - // no more results - // return null - return null; + // return the header row + return new HeadersDataRow(headers, rawHeaders, row, row); } protected void setVariables() { try { - // get the result set metadata - ResultSetMetaData rsmd = rs.getMetaData(); - numColumns = rsmd.getColumnCount(); - - // create the arrays to store the column types, - // the physical variable names and the display variable names - colTypes = new int[numColumns]; - types = new SemossDataType[numColumns]; - rawHeaders = new String[numColumns]; - headers = new String[numColumns]; - - for (int colIndex = 1; colIndex <= numColumns; colIndex++) { - rawHeaders[colIndex - 1] = rsmd.getColumnName(colIndex); - headers[colIndex - 1] = rsmd.getColumnLabel(colIndex); - colTypes[colIndex - 1] = rsmd.getColumnType(colIndex); - types[colIndex - 1] = SemossDataType.convertStringToDataType(rsmd.getColumnTypeName(colIndex)); - } + readResultMetadata(); } catch (SQLException e) { - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to read RDBMS result metadata", e); + } + } + + /** + * Populates headers and column types from the current result set. Keeping the + * JDBC work separate lets subclasses choose how to handle metadata failures + * without duplicating the column mappings. + * + * @throws SQLException if the driver cannot provide result metadata + */ + protected void readResultMetadata() throws SQLException { + // get the result set metadata + ResultSetMetaData rsmd = rs.getMetaData(); + numColumns = rsmd.getColumnCount(); + + // create the arrays to store the column types, + // the physical variable names and the display variable names + colTypes = new int[numColumns]; + types = new SemossDataType[numColumns]; + rawHeaders = new String[numColumns]; + headers = new String[numColumns]; + + for (int colIndex = 1; colIndex <= numColumns; colIndex++) { + rawHeaders[colIndex - 1] = rsmd.getColumnName(colIndex); + headers[colIndex - 1] = rsmd.getColumnLabel(colIndex); + colTypes[colIndex - 1] = rsmd.getColumnType(colIndex); + types[colIndex - 1] = SemossDataType.convertStringToDataType(rsmd.getColumnTypeName(colIndex)); } } @@ -383,7 +394,7 @@ public void setCloseConenctionAfterExecution(boolean closeConnectionAfterExecuti } @Override - public void close() throws IOException { + public void close() { if (this.closedConnection) { return; } @@ -417,7 +428,7 @@ public long getNumRows() { try { this.conn = activeEngine.getConnection(); } catch (SQLException e) { - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to acquire database connection for result counting", e); } } } @@ -431,7 +442,7 @@ public long getNumRows() { query = parser.processQuery(this.query); } } catch (Exception e) { - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to rewrite SELECT for result counting; using original SQL", e); query = this.query; } if (query.endsWith(";")) { @@ -470,7 +481,7 @@ public long getNumRows() { if (queryT != null) { queryT.setFailed(); } - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to execute RDBMS row-count query", e); } finally { if (this.dataSource != null) { ConnectionUtils.closeAllConnections(connection, statement, resultSet); @@ -524,7 +535,7 @@ public static RawRDBMSSelectWrapper directExecutionViaConnection(Connection conn wrapper.setVariables(); return wrapper; } catch (Exception e) { - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to execute SQL on the supplied connection", e); if (closeIfFail) { ConnectionUtils.closeAllConnections(wrapper.conn, wrapper.stmt, wrapper.rs); } else { @@ -535,13 +546,18 @@ public static RawRDBMSSelectWrapper directExecutionViaConnection(Connection conn } /** - * This method allows me to perform the execution of a query on a given - * connection without having to go through a formal RDBMSNativeEngine construct - * i.e. the naked engine ;) - * - * @param conn - * @param query - * @throws Exception + * Compiles a query through the engine's existing interpreter and executes its + * SQL on the supplied connection. This is the legacy Statement-based execution + * path. + * + * @param database engine supplying the interpreter and query-tracking + * identity + * @param conn existing connection on which to execute the rendered SQL + * @param qs structured SELECT to render with the engine's interpreter + * @param closeIfFail whether execution failure should also close the supplied + * connection + * @return an executed wrapper exposing the result rows + * @throws Exception if compilation, execution, or result initialization fails */ public static RawRDBMSSelectWrapper directExecutionViaConnection(IRDBMSEngine database, Connection conn, SelectQueryStruct qs, boolean closeIfFail) throws Exception { @@ -567,7 +583,8 @@ public static RawRDBMSSelectWrapper directExecutionViaConnection(IRDBMSEngine da return wrapper; } catch (Exception e) { queryT.setFailed(); - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to execute structured query for database {} on the supplied connection", engineId, + e); if (closeIfFail) { ConnectionUtils.closeAllConnections(wrapper.conn, wrapper.stmt, wrapper.rs); } else { @@ -614,7 +631,8 @@ public static RawRDBMSSelectWrapper directExecutionPreparedStatement(IRDBMSEngin return wrapper; } catch (Exception e) { queryT.setFailed(); - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to execute prepared statement for database {} on the supplied connection", + engineId, e); if (closeIfFail) { ConnectionUtils.closeAllConnections(wrapper.conn, wrapper.stmt, wrapper.rs); } else { @@ -641,7 +659,7 @@ public static RawRDBMSSelectWrapper flushRsToWrapper(ResultSet rs) throws Except wrapper.setVariables(); return wrapper; } catch (Exception e) { - classLogger.error(Constants.STACKTRACE, e); + classLogger.error("Failed to initialize RDBMS wrapper from the supplied result set", e); throw e; } } diff --git a/src/prerna/rdf/engine/wrappers/WrapperManager.java b/src/prerna/rdf/engine/wrappers/WrapperManager.java index 8a698e1d184..5a0f4c41537 100644 --- a/src/prerna/rdf/engine/wrappers/WrapperManager.java +++ b/src/prerna/rdf/engine/wrappers/WrapperManager.java @@ -38,6 +38,7 @@ import prerna.ds.RawGemlinSelectWrapper; import prerna.engine.api.IConstructWrapper; import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IRDBMSEngine; import prerna.engine.api.IRawSelectWrapper; import prerna.engine.api.ISelectWrapper; import prerna.engine.impl.json.JsonWrapper; @@ -46,6 +47,7 @@ import prerna.om.ThreadStore; import prerna.query.interpreters.GremlinInterpreter; import prerna.query.interpreters.IQueryInterpreter; +import prerna.query.interpreters.sql.ParameterizedSqlInterpreter; import prerna.query.querystruct.SelectQueryStruct; import prerna.query.querystruct.evaluator.QueryStructExpressionIterator; import prerna.usertracking.UserQueryTrackingThread; @@ -94,7 +96,73 @@ public static WrapperManager getInstance() { return manager; } - // TODO >>>timb: REST - here add another engine type REMOTE or REST + /** + * Compiles structured SQL and executes it in a prepared wrapper with no JDBC + * statement timeout. The caller must close the returned wrapper. + * + * @param engine relational engine supplying the dialect and connections + * @param qs structured SELECT containing the values to bind + * @return an executed wrapper exposing the usual rows, headers, and types + * @throws Exception if compilation, execution, or result initialization fails + */ + public RawPreparedRDBMSSelectWrapper getPreparedWrapper(IRDBMSEngine engine, SelectQueryStruct qs) + throws Exception { + return getPreparedWrapper(engine, qs, false, 0); + } + + /** + * Creates a prepared wrapper retaining SQL templates and bindings for data, + * count, and reset. Deferred execution acquires no connection until execute is + * called; otherwise execution starts immediately. The caller must close the + * wrapper after use. + * + * @param engine relational engine supplying the dialect and connections + * @param qs structured SELECT containing the values to bind + * @param delayExecution true to compile now and execute later + * @param timeoutSeconds non-negative timeout for data and count statements; + * zero disables the JDBC statement timeout + * @return a prepared wrapper, executed unless delayExecution is true + * @throws Exception if arguments are invalid or compilation/execution fails + */ + public RawPreparedRDBMSSelectWrapper getPreparedWrapper(IRDBMSEngine engine, SelectQueryStruct qs, + boolean delayExecution, int timeoutSeconds) throws Exception { + var compiled = new ParameterizedSqlInterpreter(engine).compile(qs); + var wrapper = RawPreparedRDBMSSelectWrapper.prepareParameterized(engine, compiled, timeoutSeconds); + if (!delayExecution) { + wrapper.execute(); + } + return wrapper; + } + + /** + * Track templates at actual execution time, including reset and deferred reads. + */ + static UserQueryTrackingThread preparedQueryTracker(IDatabaseEngine engine, String template) { + String engineId = engine.getEngineId(); + if (engineId == null) { + return null; + } + getInstance(); + if (ignoreDatabases.contains(engineId) || engineId.endsWith(Constants.OWL_ENGINE_SUFFIX) + || engineId.endsWith(Constants.RDBMS_INSIGHTS_ENGINE_SUFFIX)) { + return null; + } + var tracker = new UserQueryTrackingThread(ThreadStore.getUser(), engineId); + tracker.setQuery(template); + return tracker; + } + + static void finishPreparedQueryTracking(UserQueryTrackingThread tracker) { + if (tracker != null) { + try { + tracker.setEndTimeNow(); + Thread.ofVirtual().start(tracker); + } catch (RuntimeException failure) { + classLogger.warn("Unable to finish prepared query tracking", failure); + } + } + } + public IRawSelectWrapper getRawWrapper(IDatabaseEngine engine, SelectQueryStruct qs) throws Exception { return getRawWrapper(engine, qs, false); } diff --git a/src/prerna/util/ConnectionUtils.java b/src/prerna/util/ConnectionUtils.java index 271e820c408..3f4d0da6398 100644 --- a/src/prerna/util/ConnectionUtils.java +++ b/src/prerna/util/ConnectionUtils.java @@ -29,7 +29,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.sql.SQLException; import java.sql.Statement; import org.apache.logging.log4j.LogManager; @@ -38,313 +37,336 @@ import prerna.engine.api.IRDBMSEngine; /** - * Utility methods for safely closing JDBC resources ({@link Connection}, - * {@link Statement}, and {@link ResultSet}). - *

- * Each {@code close*} method silently swallows and logs any exception raised - * while closing a resource, so callers can invoke them from a {@code finally} - * block without additional try/catch handling. Resources are closed in the - * standard reverse order of acquisition: result set, then statement, then - * connection. + * Closes JDBC resources and completes explicitly requested transactions while + * retaining failures for the caller. Methods return null on success; otherwise + * they return the first failure with later failures suppressed. Cleanup + * continues after both exceptions and errors so one failed close does not leak + * the remaining resources. Callers may propagate the returned failure or retain + * log-only behavior by ignoring it. + * *

- * The {@code *IfPooling} variants are connection-pool aware: the underlying - * {@link Connection} is only closed (returned to the pool) when the supplied - * {@link IRDBMSEngine} reports {@link IRDBMSEngine#isConnectionPooling()} as - * {@code true}. This prevents a non-pooled engine's long-lived connection from - * being closed prematurely while still ensuring pooled connections are released - * back to the pool. + * Resources close in acquisition-reverse order: result set, statement, then + * connection. Pool-aware methods release connections only when the engine + * reports pooling. A statement's connection is resolved before closing the + * statement, since some drivers reject getConnection after close. */ public class ConnectionUtils { private static final Logger classLogger = LogManager.getLogger(ConnectionUtils.class); /** - * Returns a pooled connection while allowing the transaction owner to preserve - * a release failure. Non-pooled engine connections remain open. The existing - * closeAllConnectionsIfPooling methods retain their log-and-swallow contract. + * Logs a new failure and combines it with an existing failure. A null cleanup + * result represents success and is neither logged nor suppressed. + * + * @param failure original failure to preserve, or null + * @param cleanup newly caught failure, or null if the operation succeeded + * @param message description of the failed JDBC operation + * @return the original failure with cleanup suppressed, or cleanup if no + * original failure exists; null if both arguments are null */ - public static void closeConnectionIfPooling(IRDBMSEngine engine, Connection connection) throws SQLException { - if (engine != null && engine.isConnectionPooling() && connection != null) { - connection.close(); + public static Throwable jdbcCleanupFailure(Throwable failure, Throwable cleanup, String message) { + if (cleanup != null) { + classLogger.error(message, cleanup); } + return jdbcCleanupFailure(failure, cleanup); } /** - * Close a result set, statement, and connection that belong to a pooling-aware - * engine. The result set and statement are always closed. If {@code con} is - * {@code null} and the engine is pooling, the connection backing the statement - * is resolved via {@link Statement#getConnection()}. The connection is only - * closed (released to the pool) when the engine reports connection pooling. + * Combines failures that have already been logged by a JDBC helper. Preserves + * the original throwable and avoids illegal self-suppression. * - * @param engine the engine the resources belong to; may be {@code null} - * @param con the connection to close; may be {@code null} to derive it from - * {@code stmt} - * @param stmt the statement to close; may be {@code null} - * @param rs the result set to close; may be {@code null} + * @param failure original failure to preserve, or null + * @param cleanup returned cleanup failure, or null on successful cleanup + * @return the first failure with any distinct later failure suppressed, or null */ - public static void closeAllConnectionsIfPooling(IRDBMSEngine engine, Connection con, Statement stmt, ResultSet rs) { - if (rs != null) { - try { - rs.close(); - } catch (Exception e) { - classLogger.error("Error closing result set", e); - } + public static Throwable jdbcCleanupFailure(Throwable failure, Throwable cleanup) { + if (cleanup == null) { + return failure; } - if (stmt != null) { - try { - if (engine != null && engine.isConnectionPooling() && con == null) { - con = stmt.getConnection(); - } - stmt.close(); - } catch (Exception e) { - classLogger.error("Error closing statement", e); - } + if (failure == null) { + return cleanup; } - if (engine != null && engine.isConnectionPooling()) { - try { - if (con != null) { - con.close(); - } - } catch (Exception e) { - classLogger.error("Error closing connection", e); - } + if (failure != cleanup) { + failure.addSuppressed(cleanup); } + return failure; } /** - * Close a connection and a variable number of statements for a pooling-aware - * engine. Each statement is closed in turn. If {@code con} is {@code null} and - * the engine is pooling, the connection is resolved from the first statement - * that provides one. The connection is only closed (released to the pool) when - * the engine reports connection pooling. + * Releases a pooled connection, leaving non-pooled engine connections open. * - * @param engine the engine the resources belong to; may be {@code null} - * @param con the connection to close; may be {@code null} to derive it from - * a statement - * @param stmts the statements to close; may be {@code null} + * @param engine engine that owns the connection; may be null + * @param connection connection to release; may be null + * @return a logged failure, or null if no operation failed */ - public static void closeAllConnectionsIfPooling(IRDBMSEngine engine, Connection con, Statement... stmts) { + public static Throwable closeConnectionIfPooling(IRDBMSEngine engine, Connection connection) { + return closeAllConnectionsIfPooling(engine, connection, (Statement[]) null); + } + + /** + * Closes a result set and statement, then releases their connection if pooled. + * If no connection is supplied, resolves it from the statement before closing + * that statement. A failed lookup does not prevent closing the statement. + * + * @param engine engine that owns the resources; may be null + * @param con connection to release, or null to derive it from stmt + * @param stmt statement to close; may be null + * @param rs result set to close; may be null + * @return the first logged failure with later failures suppressed, or null + */ + public static Throwable closeAllConnectionsIfPooling(IRDBMSEngine engine, Connection con, Statement stmt, + ResultSet rs) { + Throwable failure = closeResultSet(rs); + return closePooledStatements(engine, con, new Statement[] { stmt }, failure); + } + + /** + * Closes statements in order, then releases one pooled connection. When con is + * null, tries to resolve it from each non-null statement until one provides a + * connection. Lookup and close failures are recorded independently. + * + * @param engine engine that owns the resources; may be null + * @param con connection to release, or null to derive it from a statement + * @param stmts statements to close; the array and its entries may be null + * @return the first logged failure with later failures suppressed, or null + */ + public static Throwable closeAllConnectionsIfPooling(IRDBMSEngine engine, Connection con, Statement... stmts) { + return closePooledStatements(engine, con, stmts, null); + } + + /** + * Performs pooled statement cleanup while continuing an existing failure chain. + * + * @param engine engine used to determine connection ownership; may be null + * @param con connection to release, or null to derive it from a statement + * @param stmts statements to close; the array and its entries may be null + * @param failure previously captured failure, or null + * @return the first failure with subsequent failures suppressed, or null + */ + private static Throwable closePooledStatements(IRDBMSEngine engine, Connection con, Statement[] stmts, + Throwable failure) { + boolean pooling = false; + try { + pooling = engine != null && engine.isConnectionPooling(); + } catch (Exception | Error cleanup) { + failure = jdbcCleanupFailure(failure, cleanup, "Error determining JDBC connection pooling"); + } if (stmts != null) { for (Statement stmt : stmts) { - try { - if (engine != null && engine.isConnectionPooling() && con == null) { + if (stmt == null) { + continue; + } + if (pooling && con == null) { + try { con = stmt.getConnection(); + } catch (Exception | Error cleanup) { + failure = jdbcCleanupFailure(failure, cleanup, "Error resolving JDBC statement connection"); } - stmt.close(); - } catch (Exception e) { - classLogger.error("Error closing statement", e); } + failure = jdbcCleanupFailure(failure, closeStatement(stmt)); } } - if (engine != null && engine.isConnectionPooling()) { - try { - if (con != null) { - con.close(); - } - } catch (Exception e) { - classLogger.error("Error closing connection", e); - } + if (pooling) { + failure = jdbcCleanupFailure(failure, closeConnection(con)); } + return failure; } /** - * Close a statement and result set for a pooling-aware engine, deriving the - * connection from the statement. The result set and statement are always - * closed. The statement's connection is only closed (released to the pool) when - * the engine reports connection pooling. + * Closes a result set and statement, deriving the connection from the statement + * before close and releasing it only for a pooled engine. * - * @param engine the engine the resources belong to; may be {@code null} - * @param stmt the statement to close; may be {@code null} - * @param rs the result set to close; may be {@code null} + * @param engine engine that owns the resources; may be null + * @param stmt statement to close; may be null + * @param rs result set to close; may be null + * @return the first logged failure with later failures suppressed, or null */ - public static void closeAllConnectionsIfPooling(IRDBMSEngine engine, Statement stmt, ResultSet rs) { - if (rs != null) { - try { - rs.close(); - } catch (Exception e) { - classLogger.error("Error closing result set", e); - } - } - if (stmt != null) { - try { - stmt.close(); - } catch (Exception e) { - classLogger.error("Error closing statement", e); - } - } - if (engine != null && engine.isConnectionPooling()) { - try { - if (stmt != null && stmt.getConnection() != null) { - stmt.getConnection().close(); - } - } catch (Exception e) { - classLogger.error("Error closing connection", e); - } - } + public static Throwable closeAllConnectionsIfPooling(IRDBMSEngine engine, Statement stmt, ResultSet rs) { + return closeAllConnectionsIfPooling(engine, null, stmt, rs); } /** - * Close a result set, statement, and connection unconditionally, in that order. - * Each resource is closed only if non-{@code null}, and any exception is logged - * rather than propagated. + * Closes a result set, statement, and connection in that order. Every supplied + * resource is attempted, even when an earlier close fails. * - * @param con the connection to close; may be {@code null} - * @param stmt the statement to close; may be {@code null} - * @param rs the result set to close; may be {@code null} + * @param con connection to close; may be null + * @param stmt statement to close; may be null + * @param rs result set to close; may be null + * @return the first logged failure with later failures suppressed, or null */ - public static void closeAllConnections(Connection con, Statement stmt, ResultSet rs) { - if (rs != null) { - try { - rs.close(); - } catch (Exception e) { - classLogger.error("Error closing result set", e); - } - } - if (stmt != null) { - try { - stmt.close(); - } catch (Exception e) { - classLogger.error("Error closing statement", e); - } - } - if (con != null) { - try { - con.close(); - } catch (Exception e) { - classLogger.error("Error closing connection", e); - } - } + public static Throwable closeAllConnections(Connection con, Statement stmt, ResultSet rs) { + Throwable failure = closeResultSet(rs); + failure = jdbcCleanupFailure(failure, closeStatement(stmt)); + return jdbcCleanupFailure(failure, closeConnection(con)); } /** - * Close a statement and connection unconditionally. + * Closes a statement followed by its connection. * - * @param con the connection to close; may be {@code null} - * @param ps the statement to close; may be {@code null} + * @param con connection to close; may be null + * @param ps statement to close; may be null + * @return the first logged failure with later failures suppressed, or null */ - public static void closeAllConnections(Connection con, Statement ps) { - closeAllConnections(con, ps, null); + public static Throwable closeAllConnections(Connection con, Statement ps) { + return closeAllConnections(con, ps, null); } /** - * Close a statement and result set unconditionally. + * Closes a result set followed by its statement without closing a connection. * - * @param stmt the statement to close; may be {@code null} - * @param rs the result set to close; may be {@code null} + * @param stmt statement to close; may be null + * @param rs result set to close; may be null + * @return the first logged failure with later failures suppressed, or null */ - public static void closeAllConnections(Statement stmt, ResultSet rs) { - closeAllConnections(null, stmt, rs); + public static Throwable closeAllConnections(Statement stmt, ResultSet rs) { + return closeAllConnections(null, stmt, rs); } /** - * Close a result set, logging any exception. + * Closes a result set and captures any failure. * - * @param rs the result set to close; may be {@code null} + * @param rs result set to close; may be null + * @return a logged failure, or null if no operation failed */ - public static void closeResultSet(ResultSet rs) { - closeAllConnections(null, null, rs); + public static Throwable closeResultSet(ResultSet rs) { + return closeResource(rs, "Error closing JDBC result set"); } /** - * Close a statement, logging any exception. + * Closes a statement and captures any failure. * - * @param stmt the statement to close; may be {@code null} + * @param stmt statement to close; may be null + * @return a logged failure, or null if no operation failed */ - public static void closeStatement(Statement stmt) { - closeAllConnections(null, stmt, null); + public static Throwable closeStatement(Statement stmt) { + return closeResource(stmt, "Error closing JDBC statement"); } /** - * Close a connection, logging any exception. + * Closes a connection and captures any failure without checking pool ownership. * - * @param con the connection to close; may be {@code null} + * @param con connection to close; may be null + * @return a logged failure, or null if no operation failed */ - public static void closeConnection(Connection con) { - closeAllConnections(con, null, null); + public static Throwable closeConnection(Connection con) { + return closeResource(con, "Error closing JDBC connection"); } /** - * Close a connection for a pooling-aware engine. The connection is only closed - * (released to the pool) when the engine reports connection pooling. + * Uses JDBC's standard AutoCloseable contract to capture resource-close + * failures in one place without introducing a separate cleanup interface. * - * @param engine the engine the connection belongs to; may be {@code null} - * @param con the connection to close; may be {@code null} + * @param resource result set, statement, or connection to close; may be null + * @param message description identifying the resource whose close failed + * @return a logged failure, or null if no operation failed */ - public static void closeAllConnectionsIfPooling(IRDBMSEngine engine, Connection con) { - closeAllConnectionsIfPooling(engine, con, null, null); + private static Throwable closeResource(AutoCloseable resource, String message) { + if (resource != null) { + try { + resource.close(); + } catch (Exception | Error cleanup) { + return jdbcCleanupFailure(null, cleanup, message); + } + } + return null; } /** - * Close a connection and statement for a pooling-aware engine. The connection - * is only closed (released to the pool) when the engine reports connection - * pooling. + * Releases a connection only when its engine reports pooling. * - * @param engine the engine the resources belong to; may be {@code null} - * @param con the connection to close; may be {@code null} - * @param stmt the statement to close; may be {@code null} + * @param engine engine that owns the connection; may be null + * @param con connection to release; may be null + * @return a logged failure, or null if no operation failed */ - public static void closeAllConnectionsIfPooling(IRDBMSEngine engine, Connection con, Statement stmt) { - closeAllConnectionsIfPooling(engine, con, stmt, null); + public static Throwable closeAllConnectionsIfPooling(IRDBMSEngine engine, Connection con) { + return closeConnectionIfPooling(engine, con); } /** - * Close a statement for a pooling-aware engine, deriving the connection from - * the statement. The statement's connection is only closed (released to the - * pool) when the engine reports connection pooling. + * Closes a statement and releases a pooled connection. If con is null, resolves + * the connection from the statement before closing it. * - * @param engine the engine the statement belongs to; may be {@code null} - * @param stmt the statement to close; may be {@code null} + * @param engine engine that owns the resources; may be null + * @param con connection to release, or null to derive it from stmt + * @param stmt statement to close; may be null + * @return the first logged failure with later failures suppressed, or null */ - public static void closeAllConnectionsIfPooling(IRDBMSEngine engine, Statement stmt) { - closeAllConnectionsIfPooling(engine, stmt, null); + public static Throwable closeAllConnectionsIfPooling(IRDBMSEngine engine, Connection con, Statement stmt) { + return closeAllConnectionsIfPooling(engine, con, stmt, null); } /** - * Commit all pending transactions on the connection. No commit is issued if the - * connection is already in auto-commit mode. + * Closes a statement and releases its connection if the engine reports pooling. * - * @param conn the connection to commit - * @throws IllegalArgumentException if the commit fails + * @param engine engine that owns the statement; may be null + * @param stmt statement to close; may be null + * @return the first logged failure with later failures suppressed, or null */ - public static void commitConnection(Connection conn) { - try { - if (!conn.getAutoCommit()) { - conn.commit(); + public static Throwable closeAllConnectionsIfPooling(IRDBMSEngine engine, Statement stmt) { + return closeAllConnectionsIfPooling(engine, stmt, null); + } + + /** + * Commits pending work when auto-commit is disabled. Callers must inspect the + * returned failure before treating the transaction as successfully committed. + * This method does not roll back, change auto-commit, or close the connection. + * + * @param conn connection whose work should be committed; may be null + * @return a logged commit/state-read failure, or null if no operation failed + */ + public static Throwable commitConnection(Connection conn) { + if (conn != null) { + try { + if (!conn.getAutoCommit()) { + conn.commit(); + } + } catch (Exception | Error failure) { + return jdbcCleanupFailure(null, failure, "Error committing JDBC transaction"); + } + } + return null; + } + + /** + * Rolls back an owned transaction without changing auto-commit or closing the + * connection. The caller must determine transaction ownership and whether a + * rollback is required; no auto-commit check is made here. + * + * @param conn connection whose transaction should end; may be null + * @return a logged rollback failure, or null if no operation failed + */ + public static Throwable rollbackConnection(Connection conn) { + if (conn != null) { + try { + conn.rollback(); + } catch (Exception | Error failure) { + return jdbcCleanupFailure(null, failure, "Error rolling back JDBC transaction"); } - } catch (SQLException e) { - classLogger.error("Error committing the transaction to the database", e); - throw new IllegalArgumentException( - "An error occurred commiting the transaction to the database. See logs for details."); } + return null; } /** - * Close a variable number of statements for a pooling-aware engine. Each - * statement is closed in turn, and the connection backing each statement is - * only closed (released to the pool) when the engine reports connection - * pooling. + * Closes each statement and releases its own connection if pooling is enabled. + * Unlike the single-connection varargs overload, statements may belong to + * different connections. Each connection is resolved before its statement + * closes. * - * @param engine the engine the statements belong to; may be {@code null} - * @param statements the statements to close; may be {@code null} + * @param engine engine that owns the statements; may be null + * @param statements statements to close; the array and its entries may be null + * @return the first logged failure with later failures suppressed, or null */ - public static void closeAllDbConnectionsIfPooling(IRDBMSEngine engine, Statement... statements) { + public static Throwable closeAllDbConnectionsIfPooling(IRDBMSEngine engine, Statement... statements) { + Throwable failure = null; if (statements != null) { for (Statement stmt : statements) { if (stmt != null) { - try { - stmt.close(); - } catch (Exception e) { - classLogger.error("Error closing statement", e); - } - if (engine != null && engine.isConnectionPooling()) { - try { - stmt.getConnection().close(); - } catch (Exception e) { - classLogger.error("Error closing connection", e); - } - } + // Each statement may have its own connection; the batch helper releases only + // one. + failure = closePooledStatements(engine, null, new Statement[] { stmt }, failure); } } } + return failure; } } diff --git a/src/prerna/util/QueryExecutionUtility.java b/src/prerna/util/QueryExecutionUtility.java index 6a3af2d0edb..a3496bb772f 100644 --- a/src/prerna/util/QueryExecutionUtility.java +++ b/src/prerna/util/QueryExecutionUtility.java @@ -135,7 +135,9 @@ public interface RowMapper { T map(ResultSet result) throws Exception; } - /** Executes one prepared DML statement in an owned write transaction. */ + /** + * Executes one prepared DML statement in an owned write transaction. + */ public static int executeUpdate(IRDBMSEngine engine, String sql, StatementBinder binder) throws Exception { return write(engine, connection -> executeUpdate(connection, sql, binder)); } @@ -326,23 +328,15 @@ private static T withJdbcConnection(IRDBMSEngine engine, Connection connecti connection.commit(); } else if (manualTransaction) { rollbackAttempted = true; - try { - connection.rollback(); - } catch (Exception | Error cleanup) { - classLogger.error("Error completing JDBC read transaction", cleanup); - throw cleanup; - } + failure = ConnectionUtils.rollbackConnection(connection); } - transactionEnded = true; + transactionEnded = failure == null; } catch (Exception | Error e) { failure = e; if (!rollbackAttempted && (manualTransaction || changeAttempted)) { - try { - connection.rollback(); - transactionEnded = true; - } catch (Exception | Error cleanup) { - failure = jdbcCleanupFailure(failure, cleanup, "Error rolling back JDBC transaction"); - } + Throwable cleanup = ConnectionUtils.rollbackConnection(connection); + transactionEnded = cleanup == null; + failure = ConnectionUtils.jdbcCleanupFailure(failure, cleanup); } } // Never enable auto-commit after a failed rollback, or guess an unread state. @@ -350,14 +344,11 @@ private static T withJdbcConnection(IRDBMSEngine engine, Connection connecti try { connection.setAutoCommit(originalAutoCommit); } catch (Exception | Error cleanup) { - failure = jdbcCleanupFailure(failure, cleanup, "Error restoring JDBC auto-commit"); + failure = ConnectionUtils.jdbcCleanupFailure(failure, cleanup, "Error restoring JDBC auto-commit"); } } - try { - ConnectionUtils.closeConnectionIfPooling(engine, connection); - } catch (Exception | Error cleanup) { - failure = jdbcCleanupFailure(failure, cleanup, "Error releasing JDBC connection"); - } + failure = ConnectionUtils.jdbcCleanupFailure(failure, + ConnectionUtils.closeConnectionIfPooling(engine, connection)); if (failure instanceof Exception) { throw (Exception) failure; } @@ -367,17 +358,6 @@ private static T withJdbcConnection(IRDBMSEngine engine, Connection connecti return result; } - private static Throwable jdbcCleanupFailure(Throwable failure, Throwable cleanup, String message) { - classLogger.error(message, cleanup); - if (failure == null) { - return cleanup; - } - if (failure != cleanup) { - failure.addSuppressed(cleanup); - } - return failure; - } - /** * Executes a query and returns the first column of its first row as a string. * The value is cast to {@link String}, not converted with {@code toString()}. @@ -657,6 +637,19 @@ public static List> flushRsToMap(IDatabaseEngine engine, Sel } } + /** + * Compiles a structured JDBC query with separate bindings and materializes the + * usual wrapper-provided headers and value types. Closes the owned read on all + * paths. + */ + public static List> flushPreparedRsToMap(IRDBMSEngine engine, SelectQueryStruct qs) { + try (IRawSelectWrapper wrapper = WrapperManager.getInstance().getPreparedWrapper(engine, qs)) { + return flushWrapperToMap(wrapper); + } catch (Exception e) { + throw new IllegalArgumentException("Error executing prepared query", e); + } + } + /** * Executes a query and maps each row's headers to values, optionally parsing * selected columns as JSON objects. Conversion follows diff --git a/src/prerna/util/sql/AbstractSqlQueryUtil.java b/src/prerna/util/sql/AbstractSqlQueryUtil.java index fbb63700452..6f36b240a0a 100644 --- a/src/prerna/util/sql/AbstractSqlQueryUtil.java +++ b/src/prerna/util/sql/AbstractSqlQueryUtil.java @@ -301,9 +301,9 @@ public static Connection makeConnection(RdbmsTypeEnum type, String connectionUrl * Replace a conneciton url to a file based db (H2, SQLite) with * "@BaseFolder@/db/@ENGINE@" *

- * In {@link RDBMSNativeEngine#open(Properties) method we call - * {@link #fillFileParameterizedConnectionUrl(String, String, String)} to turn - * back into a useable conneciton url + * In {@link RDBMSNativeEngine#open(Properties) method we call {@link + * #fillFileParameterizedConnectionUrl(String, String, String)} to turn back + * into a useable conneciton url *

* * @param connectionUrl @@ -884,6 +884,12 @@ public abstract String createUpdatePreparedStatementString(String tableName, Str */ public abstract IQueryFilter getSearchRegexFilter(String columnQs, String searchTerm); + /** + * Build the dialect's search predicate with original values for prepared + * binding. + */ + public abstract IQueryFilter getPreparedSearchRegexFilter(String columnQs, String searchTerm); + /** * Create the syntax to merge 2 tables together * diff --git a/src/prerna/util/sql/AnsiSqlQueryUtil.java b/src/prerna/util/sql/AnsiSqlQueryUtil.java index 4f3eba321c4..4165e3be463 100644 --- a/src/prerna/util/sql/AnsiSqlQueryUtil.java +++ b/src/prerna/util/sql/AnsiSqlQueryUtil.java @@ -765,11 +765,16 @@ public String createUpdatePreparedStatementString(String tableName, String[] col @Override public IQueryFilter getSearchRegexFilter(String columnQs, String searchTerm) { + return getPreparedSearchRegexFilter(columnQs, escapeForSQLStatement(searchTerm)); + } + + @Override + public IQueryFilter getPreparedSearchRegexFilter(String columnQs, String searchTerm) { FunctionQueryFilter filter = new FunctionQueryFilter(); QueryFunctionSelector regexFunction = new QueryFunctionSelector(); regexFunction.setFunction(QueryFunctionHelper.REGEXP_LIKE); regexFunction.addInnerSelector(new QueryColumnSelector(columnQs)); - regexFunction.addInnerSelector(new QueryConstantSelector(escapeForSQLStatement(searchTerm))); + regexFunction.addInnerSelector(new QueryConstantSelector(searchTerm)); regexFunction.addInnerSelector(new QueryConstantSelector("i")); filter.setFunctionSelector(regexFunction); return filter; diff --git a/src/prerna/util/sql/MicrosoftSqlServerQueryUtil.java b/src/prerna/util/sql/MicrosoftSqlServerQueryUtil.java index 56f7104a8f7..fee60d7e589 100644 --- a/src/prerna/util/sql/MicrosoftSqlServerQueryUtil.java +++ b/src/prerna/util/sql/MicrosoftSqlServerQueryUtil.java @@ -270,6 +270,11 @@ public String getRegexLikeFunctionSyntax() { return "PATINDEX"; } + @Override + public IQueryFilter getPreparedSearchRegexFilter(String columnQs, String searchTerm) { + return getSearchRegexFilter(columnQs, searchTerm); + } + @Override public IQueryFilter getSearchRegexFilter(String columnQs, String searchTerm) { // WHERE PATINDEX ('%pattern%',expression) != 0 diff --git a/src/prerna/util/sql/PostgresQueryUtil.java b/src/prerna/util/sql/PostgresQueryUtil.java index 992cb0fa55a..61aa6064cb6 100644 --- a/src/prerna/util/sql/PostgresQueryUtil.java +++ b/src/prerna/util/sql/PostgresQueryUtil.java @@ -419,6 +419,11 @@ public boolean allowIfExistsAddConstraint() { return false; } + @Override + public IQueryFilter getPreparedSearchRegexFilter(String columnQs, String searchTerm) { + return getSearchRegexFilter(columnQs, searchTerm); + } + @Override public IQueryFilter getSearchRegexFilter(String columnQs, String searchTerm) { QueryFunctionSelector fun = new QueryFunctionSelector(); diff --git a/test/prerna/auth/utils/AbstractSecurityUtilsUnitTests.java b/test/prerna/auth/utils/AbstractSecurityUtilsUnitTests.java index 9771909b8e3..e6ece98f94d 100644 --- a/test/prerna/auth/utils/AbstractSecurityUtilsUnitTests.java +++ b/test/prerna/auth/utils/AbstractSecurityUtilsUnitTests.java @@ -37,6 +37,7 @@ import java.io.File; import java.io.IOException; import java.sql.SQLException; +import java.sql.Timestamp; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -46,6 +47,7 @@ import java.util.Map; import java.util.Properties; +import org.javatuples.Pair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -61,11 +63,13 @@ import prerna.engine.api.IEngine; import prerna.engine.api.IRDBMSEngine; import prerna.project.api.IProject; +import prerna.query.interpreters.sql.ParameterizedSqlInterpreter; import prerna.query.querystruct.SelectQueryStruct; import prerna.query.querystruct.selectors.QueryColumnSelector; import prerna.util.Constants; import prerna.util.DIHelper; import prerna.util.SystemEngineRegistry; +import prerna.util.Utility; public class AbstractSecurityUtilsUnitTests extends AbstractSecurityUtilsUnitTestsSetup { @@ -638,7 +642,7 @@ void testHash_successful() { void testCalculateEndDate() { ZonedDateTime now = ZonedDateTime.now(); String nowString = now.toString(); - java.sql.Timestamp ts = AbstractSecurityUtils.calculateEndDate(nowString); + Timestamp ts = AbstractSecurityUtils.calculateEndDate(nowString); // probably figure out a better way to test the timestamp is in utc. assertNotNull(ts); } @@ -660,4 +664,38 @@ void testEndDateIsExpired_false() throws Exception { assertFalse(AbstractSecurityUtils.endDateIsExpired(semossDate)); } + @Test + void preparedIdentityFiltersRetainOriginalValuesWithoutSqlSanitizers() { + try (var sanitizerGuard = org.mockito.Mockito.mockStatic(Utility.class, invocation -> { + String method = invocation.getMethod().getName(); + if (method.equals("inputSQLSanitizer") || method.equals("inputSanitizer")) { + throw new AssertionError("Prepared queries must not call Utility." + method); + } + return invocation.callRealMethod(); + })) { + String id = "O'Brien\\path"; + User user = new User(); + AccessToken token = new AccessToken(); + token.setId(id); + token.setProvider(AuthProvider.MICROSOFT); + user.setPrimaryLogin(AuthProvider.MICROSOFT); + user.setAccessToken(token); + assertEquals(List.of(id), AbstractSecurityUtils.getUserFilterValues(user)); + assertTrue(AbstractSecurityUtils.getUserFilterValues(null).isEmpty()); + assertTrue(AbstractSecurityUtils.getUserFilterValues(new User()).isEmpty()); + var creators = List.of(Pair.with(id, "provider'type"), Pair.with("other", "MICROSOFT")); + var qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector("ENGINE__ENGINEID", "id")); + qs.addExplicitFilter(AbstractSecurityUtils.getPreparedCreatedByFilter("ENGINE__CREATEDBY", + "ENGINE__CREATEDBYTYPE", creators)); + var compiled = new ParameterizedSqlInterpreter(securityDb).compile(qs); + assertEquals(List.of(id, "provider'type", "other", "MICROSOFT"), compiled.parameters()); + assertFalse(compiled.sql().contains(id)); + assertThrows(IllegalArgumentException.class, () -> AbstractSecurityUtils + .getPreparedCreatedByFilter("ENGINE__CREATEDBY", "ENGINE__CREATEDBYTYPE", List.of())); + assertThrows(IllegalArgumentException.class, () -> AbstractSecurityUtils + .getPreparedCreatedByFilter("ENGINE__CREATEDBY", "ENGINE__CREATEDBYTYPE", null)); + } + } + } diff --git a/test/prerna/auth/utils/SecurityEngineUtilsUnitTests.java b/test/prerna/auth/utils/SecurityEngineUtilsUnitTests.java index 5207085a9f2..784b5190b6f 100644 --- a/test/prerna/auth/utils/SecurityEngineUtilsUnitTests.java +++ b/test/prerna/auth/utils/SecurityEngineUtilsUnitTests.java @@ -1106,13 +1106,13 @@ void testEngineFavorite() throws IllegalAccessException { null, null, null, null); assertEquals(1, list.size()); Map userEngine = list.getFirst(); - assertEquals("0", userEngine.get("database_favorite").toString()); + assertEquals("0", userEngine.get("engine_favorite").toString()); SecurityEngineUtils.setEngineFavorite(user, "testId", true); assertEquals("1", SecurityEngineUtils.getUserEngineList(user, null, null, false, null, null, null, null, null, null) - .getFirst().get("database_favorite").toString()); + .getFirst().get("engine_favorite").toString()); } /// @@ -1446,7 +1446,7 @@ void testGetUserDiscoverableEngineList() throws IllegalAccessException { "0"); assertEquals(1, vals.size()); - assertEquals("testId2", vals.getFirst().get("database_id")); + assertEquals("testId2", vals.getFirst().get("engine_id")); } /// @@ -1538,7 +1538,6 @@ void testUpdateEngineToolApp() { List> engineList = SecurityEngineUtils.getUserEngineList(user, null, null); assertEquals(1, engineList.size()); - assertEquals("testToolApp", engineList.getFirst().get("database_tool_app")); assertEquals("testToolApp", engineList.getFirst().get("engine_tool_app")); } diff --git a/test/prerna/engine/impl/model/inferencetracking/ModelInferenceLogsUtilsUnitTests.java b/test/prerna/engine/impl/model/inferencetracking/ModelInferenceLogsUtilsUnitTests.java index 6a35e75b709..a30f477704c 100644 --- a/test/prerna/engine/impl/model/inferencetracking/ModelInferenceLogsUtilsUnitTests.java +++ b/test/prerna/engine/impl/model/inferencetracking/ModelInferenceLogsUtilsUnitTests.java @@ -171,7 +171,7 @@ void tearDown() throws Exception { @Test void initModelInferenceLogsDatabase() throws Exception { try (MockedStatic schema = Mockito.mockStatic(AbstractOwlCreator.class); - MockedStatic connUtil = Mockito.mockStatic(ConnectionUtils.class)) { + MockedStatic connUtil = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { when(engine.getQueryUtil()).thenReturn(absQueryUtil); when(engine.getOWLEngineFactory()).thenReturn(owlFactory); when(owlFactory.getWriteOWL()).thenReturn(owlEngine); @@ -221,7 +221,7 @@ void userIsMessageAuthor() throws Exception { @Test void recordFeedback() throws Exception { try (MockedStatic staticWrapperManager = Mockito.mockStatic(WrapperManager.class); - MockedStatic staticConnUtils = Mockito.mockStatic(ConnectionUtils.class)) { + MockedStatic staticConnUtils = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { staticWrapperManager.when(() -> WrapperManager.getInstance()).thenReturn(wrapperManager); when(wrapperManager.getRawWrapper(eq(engine), any(SelectQueryStruct.class))).thenReturn(rawWrapper); @@ -323,7 +323,7 @@ void getProjectUsageFromModelInferenceLogs() { @Test void doCreateNewConversation() throws Exception { try (MockedStatic statticUUID = Mockito.mockStatic(UUID.class); - MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class)) { + MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { statticUUID.when(() -> UUID.randomUUID()).thenReturn(FIXED_UUID); when(conn.prepareStatement( @@ -369,7 +369,7 @@ void doCheckRoomExists() throws Exception { when(rs.next()).thenReturn(true).thenReturn(false); when(rs.getInt(1)).thenReturn(1); - try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class)) { + try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { assertTrue(ModelInferenceLogsUtils.doCheckRoomExists("1")); assertFalse(ModelInferenceLogsUtils.doCheckRoomExists("1")); } @@ -385,7 +385,7 @@ void doModelIsRegistered() throws Exception { when(rs.next()).thenReturn(true).thenReturn(false); when(rs.getInt(1)).thenReturn(1); - try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class)) { + try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { assertTrue(ModelInferenceLogsUtils.doModelIsRegistered("1")); assertFalse(ModelInferenceLogsUtils.doModelIsRegistered("1")); } @@ -394,7 +394,7 @@ void doModelIsRegistered() throws Exception { @Test void doCreateNewAgent() throws Exception { - try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class)) { + try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { when(conn.prepareStatement( "INSERT INTO AGENT (AGENT_ID, AGENT_NAME, DESCRIPTION, AGENT_TYPE, AUTHOR, DATE_CREATED) VALUES (?, ?, ?, ?, ?, ?)")) @@ -518,7 +518,7 @@ void doVerifyConversation() { @Test void removeFeedback() throws Exception { try (MockedStatic staticWrapperManager = Mockito.mockStatic(WrapperManager.class); - MockedStatic staticConnUtils = Mockito.mockStatic(ConnectionUtils.class)) { + MockedStatic staticConnUtils = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { staticWrapperManager.when(() -> WrapperManager.getInstance()).thenReturn(wrapperManager); when(wrapperManager.getRawWrapper(eq(engine), any(SelectQueryStruct.class))).thenReturn(rawWrapper); @@ -585,7 +585,7 @@ void getRoomOptions() throws Exception { @Test void setRoomOptions() throws Exception { - try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class)) { + try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { when(conn.prepareStatement("UPDATE ROOM SET OPTIONS = ? WHERE USER_ID = ? AND ROOM_ID = ?")).thenReturn(ps); when(engine.getQueryUtil()).thenReturn(absQueryUtil); @@ -610,7 +610,7 @@ void setRoomOptions() throws Exception { @Test void setRoomWorkspaceId() throws Exception { - try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class)) { + try (MockedStatic connUtils = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { when(conn.prepareStatement("UPDATE ROOM SET WORKSPACE_ID = ? WHERE USER_ID = ? AND ROOM_ID = ?")) .thenReturn(ps); @@ -690,7 +690,7 @@ void getTotalUsageForUser() throws Exception { try (MockedStatic staticSecEngineUtils = Mockito.mockStatic(SecurityEngineUtils.class); MockedStatic staticRDBMSWrapper = Mockito .mockStatic(RawRDBMSSelectWrapper.class); - MockedStatic connUtilStatic = Mockito.mockStatic(ConnectionUtils.class)) { + MockedStatic connUtilStatic = Mockito.mockStatic(ConnectionUtils.class, Mockito.CALLS_REAL_METHODS)) { staticSecEngineUtils.when(() -> SecurityEngineUtils.getModelEngineIdsWithRestrictions(user, "engineId")) .thenReturn(engineList); diff --git a/test/prerna/query/interpreters/sql/ParameterizedSqlInterpreterUnitTests.java b/test/prerna/query/interpreters/sql/ParameterizedSqlInterpreterUnitTests.java new file mode 100644 index 00000000000..d32564026c2 --- /dev/null +++ b/test/prerna/query/interpreters/sql/ParameterizedSqlInterpreterUnitTests.java @@ -0,0 +1,873 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.query.interpreters.sql; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import prerna.algorithm.api.SemossDataType; +import prerna.auth.AuthProvider; +import prerna.date.SemossDate; +import prerna.engine.api.IRDBMSEngine; +import prerna.query.querystruct.HardSelectQueryStruct; +import prerna.query.querystruct.SelectQueryStruct; +import prerna.query.querystruct.filters.AndQueryFilter; +import prerna.query.querystruct.filters.BetweenQueryFilter; +import prerna.query.querystruct.filters.OrQueryFilter; +import prerna.query.querystruct.filters.SimpleQueryFilter; +import prerna.query.querystruct.joins.SubqueryRelationship; +import prerna.query.querystruct.selectors.QueryArithmeticSelector; +import prerna.query.querystruct.selectors.QueryColumnSelector; +import prerna.query.querystruct.selectors.QueryConstantSelector; +import prerna.query.querystruct.selectors.QueryFunctionSelector; +import prerna.query.querystruct.selectors.QueryIfSelector; +import prerna.query.querystruct.selectors.QueryOpaqueSelector; +import prerna.query.querystruct.selectors.QueryTypedColumnSelector; +import prerna.rdf.engine.wrappers.RawPreparedRDBMSSelectWrapper; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.JdbcTestDatabase; +import prerna.util.QueryExecutionUtility; +import prerna.util.sql.RdbmsTypeEnum; +import prerna.util.sql.SqlQueryUtilFactory; + +public class ParameterizedSqlInterpreterUnitTests { + + private IRDBMSEngine engine() { + IRDBMSEngine engine = mock(IRDBMSEngine.class); + when(engine.isBasic()).thenReturn(true); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(RdbmsTypeEnum.H2_DB)); + when(engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + return engine; + } + + private static SelectQueryStruct query() { + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector("ITEMS__ID", "id")); + qs.addSelector(new QueryColumnSelector("ITEMS__V", "value")); + qs.addOrderBy("ITEMS__ID"); + return qs; + } + + @Test + void capturesValuesInSqlOrderAndRecompilesWithoutChangingTheInput() throws Exception { + IRDBMSEngine engine = engine(); + var interpreter = new ParameterizedSqlInterpreter(engine); + SelectQueryStruct qs = query(); + String value = "O'Brien ? \u03bb"; + List supplied = new ArrayList<>(List.of(value, "second")); + qs.addSelector(new QueryConstantSelector("selected ?", "constant")); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "==", supplied)); + BetweenQueryFilter between = new BetweenQueryFilter(); + between.setColumn(new QueryColumnSelector("ITEMS__ID")); + between.setStart(2); + between.setEnd(8); + qs.addExplicitFilter(between); + qs.setLimit(3); + qs.setOffSet(1); + var compiled = interpreter.compile(qs); + assertEquals(List.of("selected ?", value, "second", 2, 8, 3L, 1L), compiled.parameters()); + assertTrue(compiled.sql().contains("CAST(? AS VARCHAR) AS \"constant\"")); + assertTrue(compiled.sql().contains(" IN (?, ?)")); + assertTrue(compiled.sql().endsWith("LIMIT ? OFFSET ?")); + assertFalse(compiled.sql().contains(value)); + assertEquals(0, compiled.query().limit()); + assertEquals(compiled, interpreter.compile(qs)); + assertEquals(List.of(value, "second"), supplied); + supplied.set(0, "replacement"); + assertEquals(value, compiled.parameters().get(1)); + assertEquals(compiled.sql(), interpreter.compile(qs).sql()); + assertThrows(UnsupportedOperationException.class, () -> compiled.parameters().add("extra")); + verify(engine, never()).getConnection(); + } + + @Test + void constantDerivedAliasesRemainIndependentOfValuesIncludingNull() { + var interpreter = new ParameterizedSqlInterpreter(engine()); + SelectQueryStruct qs = query(); + QueryConstantSelector constant = new QueryConstantSelector("first"); + qs.addSelector(constant); + var first = interpreter.compile(qs); + constant.setConstant("second"); + assertEquals(first.sql(), interpreter.compile(qs).sql()); + assertFalse(first.sql().contains("first")); + constant.setConstant(null); + assertEquals(first.sql(), interpreter.compile(qs).sql()); + assertNull(interpreter.compile(qs).parameters().get(0)); + } + + @Test + void recursiveFragmentsRepeatBindingsWhenAnExpressionAppearsTwice() { + SelectQueryStruct qs = query(); + var coalesce = QueryFunctionSelector.makeCol2ValCoalesceSelector("ITEMS__V", "fallback ?", "label"); + qs.addSelector(coalesce); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter(coalesce, "==", "match", PixelDataType.CONST_STRING)); + var compiled = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertEquals(List.of("fallback ?", "fallback ?", "match"), compiled.parameters()); + assertEquals(3, compiled.sql().chars().filter(c -> c == '?').count()); + } + + @Test + void metadataMappingsAndPrimaryKeysUsePhysicalIdentifiersAndTypedBindings() { + IRDBMSEngine engine = engine(); + when(engine.isBasic()).thenReturn(false); + when(engine.getPhysicalUriFromPixelSelector("People")).thenReturn("http://example/PEOPLE"); + when(engine.getPhysicalUriFromPixelSelector("People__age")).thenReturn("http://example/AGE/PEOPLE"); + when(engine.getLegacyPrimKey4Table("http://example/PEOPLE")).thenReturn("PERSON_ID"); + when(engine.getDataTypes("http://semoss.org/ontologies/Concept/AGE/PEOPLE")).thenReturn("TYPE:INT"); + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector("People")); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("People__age", ">", "21")); + var compiled = new ParameterizedSqlInterpreter(engine).compile(qs); + assertTrue(compiled.sql().contains("t0.PERSON_ID AS \"People\" FROM PEOPLE t0")); + assertTrue(compiled.sql().contains("t0.AGE > ?")); + assertEquals(List.of(21L), compiled.parameters()); + } + + @Test + void preservesCombinedFiltersAndOverrideRules() { + var interpreter = new ParameterizedSqlInterpreter(engine()); + SelectQueryStruct qs = query(); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", "==", 1)); + qs.addImplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", "==", 2)); + qs.getFrameImplicitFilters().addFilters(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "!=", "frame")); + qs.getPanelImplicitFilters().addFilters(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "!=", "panel")); + assertEquals(List.of(1, 2, "frame", "panel"), interpreter.compile(qs).parameters()); + qs.setOverrideImplicit(true); + assertEquals(List.of(1), interpreter.compile(qs).parameters()); + qs.ignoreFilters = true; + assertTrue(interpreter.compile(qs).parameters().isEmpty()); + } + + @ParameterizedTest + @CsvSource({ "==,IS NULL, OR ,IN", "!=,IS NOT NULL, AND ,NOT IN" }) + void nullAndMembershipFiltersPreserveMixedValues(String comparator, String nullPredicate, String junction, + String membership) { + SelectQueryStruct qs = query(); + qs.addExplicitFilter( + SimpleQueryFilter.makeColToValFilter("ITEMS__V", comparator, Arrays.asList(null, "", "null", null))); + var compiled = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertEquals(List.of("", "null"), compiled.parameters()); + assertTrue(compiled.sql().contains(nullPredicate)); + assertTrue(compiled.sql().contains(junction.trim())); + assertTrue(compiled.sql().contains(membership + " (?, ?)")); + } + + @ParameterizedTest + @CsvSource({ "==,1=0", "!=,1=1" }) + void emptyMembershipHasPortableMeaning(String comparator, String predicate) { + SelectQueryStruct qs = query(); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", comparator, List.of())); + var compiled = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertTrue(compiled.sql().contains(predicate)); + assertTrue(compiled.parameters().isEmpty()); + } + + @ParameterizedTest + @CsvSource({ "?like,%mixed%", "?nlike,%mixed%", "?begins,mixed%", "?nbegins,mixed%", "?ends,%mixed", + "?nends,%mixed" }) + void searchPatternsAreBoundAndCaseNormalized(String comparator, String pattern) { + SelectQueryStruct qs = query(); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", comparator, "MiXeD")); + var compiled = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertEquals(List.of(pattern), compiled.parameters()); + assertFalse(compiled.sql().contains("MiXeD")); + assertTrue(compiled.sql().contains(comparator.startsWith("?n") ? "NOT LIKE ?" : "LIKE ?")); + } + + @Test + void typedDatesNumbersAndBooleansAreJdbcValues() { + SelectQueryStruct qs = query(); + for (var item : List.of(new Object[] { SemossDataType.DATE, "2026-10-01", java.sql.Date.valueOf("2026-10-01") }, + new Object[] { SemossDataType.TIMESTAMP, "2026-10-01T12:13:14", + Timestamp.valueOf("2026-10-01 12:13:14") }, + new Object[] { SemossDataType.DOUBLE, "1.25", new BigDecimal("1.25") }, + new Object[] { SemossDataType.BOOLEAN, "true", true })) { + qs.getExplicitFilters().clear(); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + new QueryTypedColumnSelector("ITEMS__V", (SemossDataType) item[0]), "==", item[1], + PixelDataType.CONST_STRING)); + assertEquals(List.of(item[2]), new ParameterizedSqlInterpreter(engine()).compile(qs).parameters()); + } + } + + @Test + void reversesValueToColumnComparisonsAndPreservesBooleanGrouping() { + SelectQueryStruct qs = query(); + OrQueryFilter or = new OrQueryFilter(); + or.addFilter(new SimpleQueryFilter(new NounMetadata(3, PixelDataType.CONST_INT), "<", + new NounMetadata(new QueryColumnSelector("ITEMS__ID"), PixelDataType.COLUMN))); + AndQueryFilter and = new AndQueryFilter(); + and.addFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "==", "a")); + and.addFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", "<=", 2)); + or.addFilter(and); + qs.addExplicitFilter(or); + var compiled = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertEquals(List.of(3, "a", 2), compiled.parameters()); + assertTrue(compiled.sql().contains("(t0.ID > ? OR (t0.V IN (?) AND t0.ID <= ?))")); + } + + @ParameterizedTest + @ValueSource(strings = { "raw", "opaque", "function", "subquery", "identifier", "comparator", "sort", "unjoined" }) + void unsupportedStructureFailsBeforeGettingAConnection(String feature) throws Exception { + SelectQueryStruct qs = query(); + switch (feature) { + case "raw" -> qs.setCustomFrom("SELECT 1"); + case "opaque" -> qs.addSelector(new QueryOpaqueSelector()); + case "function" -> + qs.addSelector(QueryFunctionSelector.makeFunctionSelector("arbitrary", "ITEMS__ID", "value")); + case "subquery" -> qs.addExplicitFilter( + new SimpleQueryFilter(new NounMetadata(new QueryColumnSelector("ITEMS__ID"), PixelDataType.COLUMN), + "==", new NounMetadata(query(), PixelDataType.QUERY_STRUCT))); + case "identifier" -> qs.addSelector(new QueryColumnSelector("ITEMS__invalid name")); + case "comparator" -> qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", "unknown", 1)); + case "sort" -> qs.addOrderBy("ITEMS__V", "unknown"); + case "unjoined" -> qs.addSelector(new QueryColumnSelector("OTHER__ID")); + } + IRDBMSEngine engine = engine(); + assertThrows(IllegalArgumentException.class, () -> new ParameterizedSqlInterpreter(engine).execute(qs)); + verify(engine, never()).getConnection(); + } + + @Test + void supportsDifferentDialectsAndRejectsRawQueryStructs() { + IRDBMSEngine engine = engine(); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(RdbmsTypeEnum.SQLITE)); + assertNotNull(new ParameterizedSqlInterpreter(engine).compile(query())); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(RdbmsTypeEnum.H2_DB)); + assertThrows(IllegalArgumentException.class, + () -> new ParameterizedSqlInterpreter(engine).compile(new HardSelectQueryStruct())); + } + + @ParameterizedTest + @EnumSource(RdbmsTypeEnum.class) + void compilesTemplatesAndPaginationForEverySqlDialect(RdbmsTypeEnum dialect) throws Exception { + IRDBMSEngine engine = engine(); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(dialect)); + SelectQueryStruct qs = query(); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "==", "user input")); + qs.setLimit(10); + qs.setOffSet(2); + var compiled = new ParameterizedSqlInterpreter(engine).compile(qs); + assertFalse(compiled.sql().contains("user input")); + assertEquals("user input", compiled.parameters().get(0)); + assertEquals(compiled.parameters().size(), compiled.sql().chars().filter(c -> c == '?').count()); + assertEquals(compiled.countQuery().parameters().size(), + compiled.countQuery().sql().chars().filter(c -> c == '?').count()); + List pagination = switch (dialect) { + case SQL_SERVER, ORACLE, DB2, DERBY, HIVE, TRINO, ATHENA -> List.of(2L, 10L); + case TERADATA, SYNAPSE -> List.of(2L, 12L); + default -> List.of(10L, 2L); + }; + assertEquals(pagination, compiled.parameters().subList(1, 3)); + String ending = switch (dialect) { + case SQL_SERVER, ORACLE, DB2, DERBY -> "OFFSET ? ROWS FETCH NEXT ? ROWS ONLY"; + case HIVE -> "LIMIT ?, ?"; + case TRINO, ATHENA -> "OFFSET ? LIMIT ?"; + case TERADATA, SYNAPSE -> + "parameterized_row_number > ? AND parameterized_row_number <= ? ORDER BY parameterized_row_number"; + default -> "LIMIT ? OFFSET ?"; + }; + assertTrue(compiled.sql().endsWith(ending), compiled.sql()); + String quote = switch (dialect) { + case BIG_QUERY, HIVE, IMPALA, SPARK, DATABRICKS, MYSQL, MARIADB -> "`"; + default -> "\""; + }; + assertTrue(compiled.sql().contains(" AS " + quote + "id" + quote)); + verify(engine, never()).getConnection(); + } + + @Test + void sqlServerPaginationAddsRequiredOrderWithoutMutatingInputAndCountDropsUnusedOrder() { + IRDBMSEngine engine = engine(); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(RdbmsTypeEnum.SQL_SERVER)); + SelectQueryStruct qs = query(); + qs.getOrderBy().clear(); + qs.setLimit(5); + var compiled = new ParameterizedSqlInterpreter(engine).compile(qs); + assertTrue(compiled.sql().endsWith("ORDER BY \"id\" OFFSET ? ROWS FETCH NEXT ? ROWS ONLY")); + assertEquals(List.of(0L, 5L), compiled.parameters()); + assertTrue(qs.getOrderBy().isEmpty()); + qs.setLimit(-1); + qs.addOrderBy("ITEMS__ID"); + compiled = new ParameterizedSqlInterpreter(engine).compile(qs); + assertTrue(compiled.sql().contains("ORDER BY")); + assertFalse(compiled.countQuery().sql().contains("ORDER BY")); + } + + @ParameterizedTest + @EnumSource(RdbmsTypeEnum.class) + void dialectsCompileLimitOnlyOffsetOnlyAndSearchCasts(RdbmsTypeEnum dialect) { + IRDBMSEngine engine = engine(); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(dialect)); + var interpreter = new ParameterizedSqlInterpreter(engine); + SelectQueryStruct qs = query(); + qs.getOrderBy().clear(); + qs.setOffSet(4); + var offset = interpreter.compile(qs); + assertTrue(offset.parameters().contains(4L)); + assertEquals(offset.parameters().size(), offset.sql().chars().filter(c -> c == '?').count()); + qs.setOffSet(-1); + qs.setLimit(7); + var limit = interpreter.compile(qs); + assertTrue(limit.parameters().contains(7L)); + assertEquals(limit.parameters().size(), limit.sql().chars().filter(c -> c == '?').count()); + qs.setLimit(-1); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", "?like", "12")); + var search = interpreter.compile(qs); + assertEquals(List.of("%12%"), search.parameters()); + assertTrue(search.sql().contains("CAST(t0.ID AS ")); + if (List.of(RdbmsTypeEnum.BIG_QUERY, RdbmsTypeEnum.HIVE, RdbmsTypeEnum.IMPALA, RdbmsTypeEnum.SPARK, + RdbmsTypeEnum.DATABRICKS).contains(dialect)) { + assertTrue(search.sql().contains(" AS STRING)")); + } + } + + @ParameterizedTest + @CsvSource({ "Mean,AVG", "Average,AVG", "UniqueAvg,AVG", "UniqueCount,COUNT", "UniqueSum,SUM" }) + void semossFunctionNamesUseDialectSyntaxAndDistinct(String function, String expected) { + SelectQueryStruct qs = query(); + qs.addSelector(QueryFunctionSelector.makeFunctionSelector(function, "ITEMS__ID", "n")); + var compiled = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertTrue(compiled.sql() + .contains(expected + "(" + (function.startsWith("Unique") ? "DISTINCT " : "") + "t0.ID)")); + } + + @Test + void snapshotsMutableDatesAndNormalizesSupportedJavaScalarTypes() { + var timestamp = Timestamp.valueOf("2026-10-01 12:13:14.123456789"); + var date = java.sql.Date.valueOf("2026-10-01"); + var local = timestamp.toLocalDateTime(); + var inputs = List.of(timestamp, date, local.toLocalDate(), local, ZonedDateTime.of(local, ZoneOffset.UTC), + OffsetDateTime.of(local, ZoneOffset.UTC), Instant.parse("2026-10-01T12:13:14Z"), + new java.util.Date(1000), new BigInteger("9223372036854775808")); + SelectQueryStruct qs = query(); + for (int i = 0; i < inputs.size(); i++) { + qs.addSelector(new QueryConstantSelector(inputs.get(i), "c" + i)); + } + var compiled = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertEquals(timestamp, compiled.parameters().get(0)); + assertNotSame(timestamp, compiled.parameters().get(0)); + assertNotSame(date, compiled.parameters().get(1)); + assertEquals(new BigDecimal("9223372036854775808"), compiled.parameters().get(8)); + timestamp.setTime(0); + date.setTime(0); + assertEquals(Timestamp.valueOf(local), compiled.parameters().get(0)); + assertEquals(java.sql.Date.valueOf(local.toLocalDate()), compiled.parameters().get(1)); + } + + @Test + void invalidValuesAndUnsafeOptionsAreRejectedBeforeExecution() throws Exception { + IRDBMSEngine engine = engine(); + var interpreter = new ParameterizedSqlInterpreter(engine); + SelectQueryStruct qs = query(); + qs.addSelector(new QueryConstantSelector(new Object(), "invalid")); + assertThrows(IllegalArgumentException.class, () -> interpreter.execute(qs)); + SelectQueryStruct multiple = query(); + multiple.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", ">", List.of(1, 2))); + assertThrows(IllegalArgumentException.class, () -> interpreter.execute(multiple)); + SelectQueryStruct nullOrdered = query(); + nullOrdered.addExplicitFilter( + SimpleQueryFilter.makeColToValFilter("ITEMS__ID", ">", null, PixelDataType.NULL_VALUE)); + assertThrows(IllegalArgumentException.class, () -> interpreter.execute(nullOrdered)); + SelectQueryStruct invalidBoolean = query(); + invalidBoolean.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + new QueryTypedColumnSelector("ITEMS__ACTIVE", SemossDataType.BOOLEAN), "==", "perhaps", + PixelDataType.CONST_STRING)); + assertThrows(IllegalArgumentException.class, () -> interpreter.execute(invalidBoolean)); + verify(engine, never()).getConnection(); + } + + @Test + void rejectsMalformedJoinsAndProjectionAliases() { + var interpreter = new ParameterizedSqlInterpreter(engine()); + SelectQueryStruct duplicate = query(); + duplicate.addSelector(new QueryColumnSelector("ITEMS__TEAM", "id")); + assertThrows(IllegalArgumentException.class, () -> interpreter.compile(duplicate)); + SelectQueryStruct inferred = query(); + inferred.addRelation("ITEMS", "TEAMS", "inner.join"); + assertThrows(IllegalArgumentException.class, () -> interpreter.compile(inferred)); + SelectQueryStruct self = query(); + self.addRelation("ITEMS__ID", "ITEMS__TEAM", "inner.join"); + assertThrows(IllegalArgumentException.class, () -> interpreter.compile(self)); + } + + @Test + void semossDatesAndTypedNullTokensPreserveDateAndNullMeaning() { + var interpreter = new ParameterizedSqlInterpreter(engine()); + SelectQueryStruct qs = query(); + qs.addSelector(new QueryConstantSelector(new SemossDate(LocalDate.of(2026, 10, 1), ZoneOffset.UTC), "day")); + qs.addSelector(new QueryConstantSelector(new SemossDate(LocalDateTime.of(2026, 10, 1, 12, 0), ZoneOffset.UTC), + "time")); + qs.addExplicitFilter( + SimpleQueryFilter.makeColToValFilter(new QueryTypedColumnSelector("ITEMS__ID", SemossDataType.INT), + "==", List.of("null", "nan", "", "2"), PixelDataType.CONST_STRING)); + var compiled = interpreter.compile(qs); + assertEquals(List.of(java.sql.Date.valueOf("2026-10-01"), Timestamp.valueOf("2026-10-01 12:00:00"), 2L), + compiled.parameters()); + assertTrue(compiled.sql().contains("t0.ID IS NULL OR t0.ID IN (?)")); + SelectQueryStruct midnight = query(); + midnight.addSelector(new QueryConstantSelector( + new SemossDate(LocalDateTime.of(2026, 10, 1, 0, 0), ZoneOffset.UTC), "midnight")); + assertInstanceOf(Timestamp.class, interpreter.compile(midnight).parameters().get(0)); + + qs.addImplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "==", "implicit")); + qs.setOverrideImplicit(true); + assertEquals("implicit", interpreter.compile(qs).parameters().get(3)); + } + + @Test + void emptyBooleanGroupsAndColumnComparisonsExecuteWithoutValueParameters() throws Exception { + try (var db = new JdbcTestDatabase()) { + when(db.engine.isBasic()).thenReturn(true); + when(db.engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + db.execute("CREATE TABLE ITEMS (ID INT, V VARCHAR(20))", "INSERT INTO ITEMS VALUES (1, 'one')"); + var interpreter = new ParameterizedSqlInterpreter(db.engine); + SelectQueryStruct qs = query(); + qs.addExplicitFilter(new AndQueryFilter()); + qs.addExplicitFilter( + new SimpleQueryFilter(new NounMetadata(new QueryColumnSelector("ITEMS__ID"), PixelDataType.COLUMN), + "==", new NounMetadata(new QueryColumnSelector("ITEMS__ID"), PixelDataType.COLUMN))); + try (var wrapper = interpreter.execute(db.connection, qs, 2)) { + assertEquals(1, wrapper.getNumRows()); + } + qs.addExplicitFilter(new OrQueryFilter()); + try (var wrapper = interpreter.execute(qs)) { + assertFalse(wrapper.hasNext()); + assertEquals(0, wrapper.getNumRows()); + } + } + } + + @ParameterizedTest + @EnumSource(value = RdbmsTypeEnum.class, names = { "SYNAPSE", "TERADATA" }) + void windowPaginationPreservesDistinctRowsHeadersAndCount(RdbmsTypeEnum dialect) throws Exception { + try (var db = new JdbcTestDatabase()) { + when(db.engine.isBasic()).thenReturn(true); + when(db.engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(dialect)); + when(db.engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + db.execute("CREATE TABLE ITEMS (ID INT, V VARCHAR(20))", + "INSERT INTO ITEMS VALUES (1, 'one'),(2, 'two'),(2, 'two'),(3, 'three')"); + SelectQueryStruct qs = query(); + qs.setDistinct(true); + qs.setOffSet(1); + qs.setLimit(1); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", ">", 0)); + var interpreter = new ParameterizedSqlInterpreter(db.engine); + try (var wrapper = interpreter.execute(qs)) { + assertArrayEquals(new String[] { "id", "value" }, wrapper.getHeaders()); + assertEquals(1, wrapper.getNumRows()); + assertArrayEquals(new Object[] { 2, "two" }, wrapper.next().getValues()); + assertFalse(wrapper.hasNext()); + } + qs.addOrderBy("ITEMS__UNSELECTED"); + assertThrows(IllegalArgumentException.class, () -> interpreter.compile(qs)); + qs.getOrderBy().clear(); + qs.setOffSet(Long.MAX_VALUE); + assertThrows(ArithmeticException.class, () -> interpreter.compile(qs)); + } + } + + @ParameterizedTest + @ValueSource(strings = { "left.outer.join", "right.outer.join" }) + void explicitOuterJoinsExecuteAndRetainUnmatchedRows(String kind) throws Exception { + try (var db = new JdbcTestDatabase()) { + when(db.engine.isBasic()).thenReturn(true); + when(db.engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + db.execute("CREATE TABLE ITEMS(ID INT, V VARCHAR(20), TEAM INT)", "CREATE TABLE TEAMS(ID INT)", + "INSERT INTO ITEMS VALUES (1,'one',10)", "INSERT INTO TEAMS VALUES (10),(20)"); + SelectQueryStruct qs = query(); + qs.addRelation("ITEMS__TEAM", "TEAMS__ID", kind); + try (var wrapper = new ParameterizedSqlInterpreter(db.engine).execute(qs)) { + assertEquals(kind.startsWith("left") ? 1 : 2, wrapper.getNumRows()); + } + } + } + + @Test + void bigQueryQuotesCompleteProjectPathsAndRejectsNonIdentifierFragments() { + IRDBMSEngine engine = engine(); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(RdbmsTypeEnum.BIG_QUERY)); + var interpreter = new ParameterizedSqlInterpreter(engine); + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector("my-project.analytics.ITEMS__ID", "id")); + assertTrue(interpreter.compile(qs).sql().contains("FROM `my-project.analytics.ITEMS` t0")); + SelectQueryStruct invalid = new SelectQueryStruct(); + invalid.addSelector(new QueryColumnSelector("my-project.analytics.invalid name__ID", "id")); + assertThrows(IllegalArgumentException.class, () -> interpreter.compile(invalid)); + } + + @Test + void h2ExecutesLookupDynamicListJoinGroupAndBoundPagination() throws Exception { + try (var db = new JdbcTestDatabase()) { + when(db.engine.isBasic()).thenReturn(true); + when(db.engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + roundTrip(db.engine); + } + } + + /** Shared assertions for the database execution fixtures. */ + public static void roundTrip(IRDBMSEngine engine) throws Exception { + boolean sqlServer = engine.getQueryUtil().getDbType() == RdbmsTypeEnum.SQL_SERVER; + try (var statement = engine.getConnection().createStatement()) { + statement.execute("CREATE TABLE ITEMS (ID INT PRIMARY KEY, V " + + (sqlServer ? "NVARCHAR(100)" : "VARCHAR(100)") + ", TEAM INT, D DATE, TS " + + (sqlServer ? "DATETIME2" : "TIMESTAMP") + ", ACTIVE " + (sqlServer ? "BIT" : "BOOLEAN") + ")"); + statement.execute("CREATE TABLE TEAMS (ID INT PRIMARY KEY, LABEL VARCHAR(100))"); + statement.execute("INSERT INTO TEAMS VALUES (10, 'group')"); + } + QueryExecutionUtility.executeBatch(engine, "INSERT INTO ITEMS VALUES (?, ?, ?, ?, ?, ?)", List.of(1, 2, 3), + (ps, id) -> { + ps.setInt(1, id); + ps.setString(2, id == 1 ? "O'Brien ? \u03bb" : id == 2 ? null : ""); + ps.setInt(3, 10); + ps.setDate(4, java.sql.Date.valueOf("2026-10-01")); + ps.setTimestamp(5, Timestamp.valueOf("2026-10-01 12:13:14")); + ps.setBoolean(6, true); + }); + var interpreter = new ParameterizedSqlInterpreter(engine); + SelectQueryStruct lookup = query(); + lookup.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "==", "O'Brien ? \u03bb")); + try (var wrapper = interpreter.execute(lookup)) { + assertEquals(1, wrapper.getNumRows()); + assertArrayEquals(new String[] { "id", "value" }, wrapper.getHeaders()); + assertEquals("O'Brien ? \u03bb", wrapper.next().getValues()[1]); + assertFalse(wrapper.hasNext()); + wrapper.reset(); + assertTrue(wrapper.hasNext()); + } + SelectQueryStruct list = query(); + list.addExplicitFilter( + SimpleQueryFilter.makeColToValFilter("ITEMS__ID", "==", List.of(1, 2, 3), PixelDataType.CONST_INT)); + list.setLimit(2); + list.setOffSet(1); + try (var wrapper = interpreter.execute(list)) { + assertEquals(2, wrapper.getNumRows()); + assertArrayEquals(new Object[] { 2, null }, wrapper.next().getValues()); + assertArrayEquals(new Object[] { 3, "" }, wrapper.next().getValues()); + assertFalse(wrapper.hasNext()); + } + SelectQueryStruct joined = query(); + joined.addRelation("ITEMS__TEAM", "TEAMS__ID", "inner.join"); + joined.addSelector(new QueryColumnSelector("TEAMS__LABEL", "team")); + try (var wrapper = interpreter.execute(joined)) { + assertEquals(3, wrapper.getNumRows()); + assertEquals("group", wrapper.next().getValues()[2]); + } + SelectQueryStruct grouped = new SelectQueryStruct(); + grouped.addSelector(new QueryColumnSelector("ITEMS__TEAM", "team")); + var count = QueryFunctionSelector.makeFunctionSelector("Count", "ITEMS__ID", "n"); + grouped.addSelector(count); + grouped.addGroupBy(new QueryColumnSelector("ITEMS__TEAM")); + grouped.addHavingFilter(SimpleQueryFilter.makeColToValFilter(count, ">", 1, PixelDataType.CONST_INT)); + try (var wrapper = interpreter.execute(grouped)) { + assertEquals(1, wrapper.getNumRows()); + assertEquals(3L, ((Number) wrapper.next().getValues()[1]).longValue()); + } + for (String comparator : List.of("?like", "?begins", "?ends")) { + SelectQueryStruct search = query(); + search.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", comparator, + comparator.equals("?like") ? "BRIEN" : comparator.equals("?begins") ? "O'" : "\u03bb")); + try (var wrapper = interpreter.execute(search)) { + assertEquals(1, wrapper.next().getValues()[0]); + assertFalse(wrapper.hasNext()); + assertEquals(1, wrapper.getNumRows()); + } + } + QueryExecutionUtility.executeUpdate(engine, "UPDATE ITEMS SET V=? WHERE ID=3", + ps -> ps.setString(1, "path\\file")); + SelectQueryStruct slash = query(); + slash.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "?like", "\\")); + try (var wrapper = interpreter.execute(slash)) { + assertEquals(3, wrapper.next().getValues()[0]); + assertFalse(wrapper.hasNext()); + } + QueryExecutionUtility.executeUpdate(engine, "UPDATE ITEMS SET V=? WHERE ID=3", ps -> ps.setString(1, "")); + SelectQueryStruct nullable = query(); + nullable.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "==", Arrays.asList(null, ""))); + try (var wrapper = interpreter.execute(nullable)) { + assertEquals(2, wrapper.getNumRows()); + assertEquals(2, wrapper.next().getValues()[0]); + assertEquals(3, wrapper.next().getValues()[0]); + assertFalse(wrapper.hasNext()); + } + SelectQueryStruct computed = query(); + QueryIfSelector conditional = new QueryIfSelector(); + conditional.setCondition(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", "==", 1)); + conditional.setPrecedent(new QueryConstantSelector("first ?")); + conditional.setAntecedent(new QueryConstantSelector("other")); + conditional.setAlias("category"); + computed.addSelector(conditional); + QueryArithmeticSelector arithmetic = new QueryArithmeticSelector(); + arithmetic.setLeftSelector(new QueryColumnSelector("ITEMS__ID")); + arithmetic.setRightSelector(new QueryConstantSelector(2)); + arithmetic.setMathExpr("/"); + arithmetic.setAlias("ratio"); + computed.addSelector(arithmetic); + try (var wrapper = interpreter.execute(computed)) { + assertEquals(3, wrapper.getNumRows()); + Object[] row = wrapper.next().getValues(); + assertEquals("first ?", row[2]); + assertEquals(0.5, ((Number) row[3]).doubleValue(), 0.000001); + assertEquals("other", wrapper.next().getValues()[2]); + } + // Compare results with the existing interpreter for a structured lookup. + SelectQueryStruct parity = query(); + parity.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", "==", 1)); + var legacy = engine.getQueryUtil().getInterpreter(engine); + legacy.setQueryStruct(parity); + List legacyRows = QueryExecutionUtility.queryList(engine, legacy.composeQuery(), ps -> { + }, rs -> rs.getString(2)); + assertEquals(1, legacyRows.size()); + try (var wrapper = interpreter.execute(parity)) { + assertEquals(legacyRows.get(0), wrapper.next().getValues()[1]); + assertFalse(wrapper.hasNext()); + } + SelectQueryStruct dates = query(); + dates.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__D", "==", LocalDate.of(2026, 10, 1), + PixelDataType.CONST_DATE)); + dates.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__TS", "==", + LocalDateTime.of(2026, 10, 1, 12, 13, 14), PixelDataType.CONST_TIMESTAMP)); + dates.addExplicitFilter( + SimpleQueryFilter.makeColToValFilter("ITEMS__ACTIVE", "==", true, PixelDataType.BOOLEAN)); + try (var wrapper = interpreter.execute(dates)) { + assertEquals(3, wrapper.getNumRows()); + } + } + + @Test + void derivedAndMembershipSubqueriesKeepSqlOrderAndIndependentAliases() throws Exception { + SelectQueryStruct derived = new SelectQueryStruct(); + derived.addSelector(new QueryColumnSelector("TEAMS__ID", "team_id")); + var inner = new QueryConstantSelector("inner value"); + inner.setAlias("note"); + derived.addSelector(inner); + derived.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("TEAMS__LABEL", "==", "group")); + derived.addOrderBy("team_id"); + SelectQueryStruct membership = new SelectQueryStruct(); + membership.addSelector(new QueryColumnSelector("TEAMS__ID")); + membership.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("TEAMS__LABEL", "!=", "excluded")); + SelectQueryStruct outer = query(); + var constant = new QueryConstantSelector("outer value"); + constant.setAlias("constant"); + outer.addSelector(constant); + outer.addSelector(new QueryColumnSelector("derived__note", "note")); + outer.addRelation(new SubqueryRelationship(derived, "derived", "left.outer.join", + new String[] { "derived__team_id", "ITEMS__TEAM", "=" })); + outer.addExplicitFilter(SimpleQueryFilter.makeColToSubQuery("ITEMS__TEAM", "==", membership)); + outer.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("derived__note", "==", "inner value")); + outer.setLimit(2); + outer.setOffSet(1); + var compiled = new ParameterizedSqlInterpreter(engine()).compile(outer); + assertEquals(List.of("outer value", "inner value", "group", "excluded", "inner value", 2L, 1L), + compiled.parameters()); + assertEquals(compiled.parameters(), compiled.countQuery().parameters()); + assertEquals(1, compiled.sql().split("ORDER BY", -1).length - 1); + assertEquals(1, derived.getOrderBy().size()); + try (var db = new JdbcTestDatabase()) { + when(db.engine.isBasic()).thenReturn(true); + db.execute("CREATE TABLE ITEMS (ID INT, V VARCHAR, TEAM INT)", "CREATE TABLE TEAMS (ID INT, LABEL VARCHAR)", + "INSERT INTO ITEMS VALUES (1, 'one', 10), (2, 'two', 10), (3, 'three', 10)", + "INSERT INTO TEAMS VALUES (10, 'group')"); + try (var wrapper = new ParameterizedSqlInterpreter(db.engine).execute(outer)) { + assertEquals(2, wrapper.getNumRows()); + assertArrayEquals(new Object[] { 2, "two", "outer value", "inner value" }, wrapper.next().getValues()); + } + } + } + + @ParameterizedTest + @ValueSource(strings = { "==", "!=", ">" }) + void membershipAndScalarSubqueriesUseTheRequestedComparison(String comparator) { + var nested = new SelectQueryStruct(); + nested.addSelector(new QueryColumnSelector("TEAMS__ID")); + nested.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("TEAMS__LABEL", "==", "group")); + var qs = query(); + qs.addExplicitFilter(SimpleQueryFilter.makeColToSubQuery("ITEMS__TEAM", comparator, nested)); + var result = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertTrue(result.sql().contains( + comparator.equals("==") ? " IN (SELECT" : comparator.equals("!=") ? " NOT IN (SELECT" : " > (SELECT")); + assertEquals(List.of("group"), result.parameters()); + } + + @ParameterizedTest + @ValueSource(strings = { "INT", "INTEGER", "DECIMAL(12, 2)", "NVARCHAR(200)", "TIMESTAMP" }) + void castsUseValidatedStructuralTypesAndBoundValues(String type) { + var cast = QueryFunctionSelector.makeFunctionSelector("Cast", new QueryConstantSelector("7"), "converted"); + cast.setDataType(type); + var qs = query(); + qs.addSelector(cast); + var result = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertTrue(result.sql().contains("CAST(? AS " + type + ")")); + assertEquals(List.of("7"), result.parameters()); + } + + @ParameterizedTest + @ValueSource(strings = { "INT extra", "CUSTOM_TYPE", "VARCHAR(size)", "DECIMAL(-1)", "INT(1.5)" }) + void unrecognizedCastTypesAreRejected(String type) { + var cast = QueryFunctionSelector.makeFunctionSelector("Cast", "ITEMS__V", "converted"); + cast.setDataType(type); + var qs = query(); + qs.addSelector(cast); + assertThrows(IllegalArgumentException.class, () -> new ParameterizedSqlInterpreter(engine()).compile(qs)); + } + + @Test + void enumValuesUseTheirStoredTextAndConstantPredicatesStayBound() { + var qs = query(); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "==", AuthProvider.MICROSOFT)); + qs.addExplicitFilter(new SimpleQueryFilter(new NounMetadata(1, PixelDataType.CONST_INT), "==", + new NounMetadata(0, PixelDataType.CONST_INT))); + var result = new ParameterizedSqlInterpreter(engine()).compile(qs); + assertEquals(List.of(AuthProvider.MICROSOFT.toString(), 1, 0), result.parameters()); + assertTrue(result.sql().contains("? = ?")); + } + + @Test + void typedDerivedColumnsUseTheirProjectionTypeWithoutLookingUpFictitiousTables() { + var nested = new SelectQueryStruct(); + var cast = QueryFunctionSelector.makeFunctionSelector("Cast", "ITEMS__V", "number"); + cast.setDataType("INT"); + nested.addSelector(cast); + var qs = query(); + qs.addRelation(new SubqueryRelationship(nested, "derived", "inner.join", + new String[] { "ITEMS__ID", "derived__number", "=" })); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("derived__number", "==", "17")); + assertEquals(List.of(17L), new ParameterizedSqlInterpreter(engine()).compile(qs).parameters()); + } + + @ParameterizedTest + @ValueSource(strings = { "unknown_column", "duplicate_alias", "missing_condition", "malformed_condition", + "disconnected", "cyclic", "invalid_endpoint", "invalid_alias", "empty_root_condition" }) + void invalidDerivedRelationshipsFailDuringCompilation(String shape) { + var nested = new SelectQueryStruct(); + nested.addSelector(new QueryColumnSelector("TEAMS__ID", "team_id")); + var qs = query(); + var relation = new SubqueryRelationship(nested, "derived", "left.outer.join", + new String[] { "ITEMS__TEAM", "derived__team_id", "=" }); + qs.addRelation(relation); + switch (shape) { + case "unknown_column" -> qs.addSelector(new QueryColumnSelector("derived__missing", "missing")); + case "duplicate_alias" -> qs.addRelation(new SubqueryRelationship(nested, "derived", "inner.join", + new String[] { "ITEMS__TEAM", "derived__team_id", "=" })); + case "missing_condition" -> relation.setJoinOnDetails(List.of()); + case "malformed_condition" -> relation.addJoinOn(new String[] { "ITEMS__ID" }); + case "disconnected" -> relation.addJoinOn("OTHER__ID", "derived__team_id", "="); + case "cyclic" -> relation.setQs(qs); + case "invalid_endpoint" -> relation.addJoinOn("ITEMS", "derived__team_id", "="); + case "invalid_alias" -> relation.setQueryAlias("invalid alias"); + case "empty_root_condition" -> relation.setJoinOnDetails(Collections.singletonList(new String[] {})); + } + assertThrows(IllegalArgumentException.class, () -> new ParameterizedSqlInterpreter(engine()).compile(qs)); + } + + @Test + void cyclicMembershipSubqueriesFailWithoutRecursingIndefinitely() { + var qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector("ITEMS__ID")); + qs.addExplicitFilter(SimpleQueryFilter.makeColToSubQuery("ITEMS__ID", "==", qs)); + assertThrows(IllegalArgumentException.class, () -> new ParameterizedSqlInterpreter(engine()).compile(qs)); + } + + @ParameterizedTest + @ValueSource(strings = { "~", "!~", "~*", "!~*" }) + void postgresRegexOperatorsBindTheirPatternsAndOtherDialectsRejectThem(String operator) { + var qs = query(); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", operator, "O'Brien")); + var engine = engine(); + assertThrows(IllegalArgumentException.class, () -> new ParameterizedSqlInterpreter(engine).compile(qs)); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(RdbmsTypeEnum.POSTGRES)); + var result = new ParameterizedSqlInterpreter(engine).compile(qs); + assertEquals(List.of("O'Brien"), result.parameters()); + assertTrue(result.sql().contains(" " + operator + " ?")); + } + + @Test + void h2CountsTypedConstantProjectionsWithoutTruncatingStringsOrDecimalScale() throws Exception { + try (var db = new JdbcTestDatabase()) { + when(db.engine.isBasic()).thenReturn(true); + when(db.engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + db.execute("CREATE TABLE ITEMS (ID INT, V VARCHAR)", "INSERT INTO ITEMS VALUES (1, 'one')"); + var qs = query(); + var constants = Arrays.asList(true, (byte) 1, (short) 2, 3, 4L, 1.5f, 2.5d, new BigDecimal("0.000001"), + new BigDecimal("1E+10"), "long ".repeat(2000), null); + for (int i = 0; i < constants.size(); i++) { + qs.addSelector(new QueryConstantSelector(constants.get(i), "c" + i)); + } + var compiled = new ParameterizedSqlInterpreter(db.engine).compile(qs); + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(db.engine, compiled, 0)) { + assertEquals(1, wrapper.getNumRows()); + Object[] row = wrapper.next().getValues(); + assertEquals(true, row[2]); + assertEquals("long ".repeat(2000), row[11]); + assertNull(row[12]); + } + try (var ps = db.connection.prepareStatement(compiled.sql())) { + for (int i = 0; i < compiled.parameters().size(); i++) { + ps.setObject(i + 1, compiled.parameters().get(i)); + } + try (var rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertEquals(new BigDecimal("0.000001"), rs.getBigDecimal(10)); + assertEquals(new BigDecimal("10000000000"), rs.getBigDecimal(11)); + } + } + } + } +} diff --git a/test/prerna/query/querystruct/selectors/AbstractQuerySelectorUnitTests.java b/test/prerna/query/querystruct/selectors/AbstractQuerySelectorUnitTests.java new file mode 100644 index 00000000000..58e11a18929 --- /dev/null +++ b/test/prerna/query/querystruct/selectors/AbstractQuerySelectorUnitTests.java @@ -0,0 +1,48 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.query.querystruct.selectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import org.junit.jupiter.api.Test; + +class AbstractQuerySelectorUnitTests { + + @Test + void explicitAliasCanBeDistinguishedFromAValueDerivedDefault() { + QueryConstantSelector selector = new QueryConstantSelector("user value"); + assertEquals("", selector.getExplicitAlias()); + assertNotEquals(selector.getExplicitAlias(), selector.getAlias()); + selector.setAlias("display"); + assertEquals("display", selector.getExplicitAlias()); + selector.setConstant("different value"); + assertEquals("display", selector.getExplicitAlias()); + assertEquals("display", selector.getAlias()); + } +} diff --git a/test/prerna/rdf/engine/wrappers/RawPreparedRDBMSSelectWrapperUnitTests.java b/test/prerna/rdf/engine/wrappers/RawPreparedRDBMSSelectWrapperUnitTests.java new file mode 100644 index 00000000000..207043f7b6b --- /dev/null +++ b/test/prerna/rdf/engine/wrappers/RawPreparedRDBMSSelectWrapperUnitTests.java @@ -0,0 +1,452 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.rdf.engine.wrappers; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import com.zaxxer.hikari.HikariDataSource; + +import prerna.engine.api.IRDBMSEngine; +import prerna.engine.api.IRawSelectWrapper; +import prerna.query.interpreters.sql.ParameterizedSqlInterpreter; +import prerna.query.interpreters.sql.ParameterizedSqlInterpreter.CompiledQuery; +import prerna.query.querystruct.SelectQueryStruct; +import prerna.query.querystruct.filters.SimpleQueryFilter; +import prerna.query.querystruct.selectors.QueryColumnSelector; +import prerna.util.JdbcTestDatabase; +import prerna.util.QueryExecutionUtility.ParameterizedQuery; +import prerna.util.sql.RdbmsTypeEnum; +import prerna.util.sql.SqlQueryUtilFactory; + +class RawPreparedRDBMSSelectWrapperUnitTests { + + private CompiledQuery compiled(IRDBMSEngine engine) { + when(engine.isBasic()).thenReturn(true); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(RdbmsTypeEnum.H2_DB)); + when(engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector("ITEMS__ID", "id")); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__ID", ">=", 1)); + qs.addOrderBy("ITEMS__ID"); + return new ParameterizedSqlInterpreter(engine).compile(qs); + } + + @Test + void deferredExecutionCountResetAndEarlyCloseRetainBindingsAndMetadata() throws Exception { + try (var db = new JdbcTestDatabase()) { + db.execute("CREATE TABLE ITEMS (ID INT)", "INSERT INTO ITEMS VALUES (1),(2)"); + var query = compiled(db.engine); + db.manual(); + try (RawRDBMSSelectWrapper wrapper = RawPreparedRDBMSSelectWrapper.prepareParameterized(db.engine, query, + 2)) { + assertInstanceOf(RawPreparedRDBMSSelectWrapper.class, wrapper); + assertFalse(wrapper.hasNext()); + assertThrows(NoSuchElementException.class, wrapper::next); + verify(db.engine, never()).getConnection(); + assertEquals(query.sql(), wrapper.getQuery()); + wrapper.execute(); + assertArrayEquals(new String[] { "id" }, wrapper.getHeaders()); + assertEquals(2, wrapper.getNumRows()); + assertEquals(2, wrapper.getNumRecords()); + assertEquals(1, wrapper.next().getValues()[0]); + wrapper.reset(); + assertEquals(1, wrapper.next().getValues()[0]); + assertEquals(2, wrapper.next().getValues()[0]); + assertFalse(wrapper.hasNext()); + wrapper.reset(); + assertTrue(wrapper.hasNext()); + } + verify(db.connection, times(3)).rollback(); + verify(db.connection, never()).commit(); + verify(db.connection, never()).close(); + verify(db.connection, never()).setAutoCommit(anyBoolean()); + } + } + + @Test + void borrowedConnectionKeepsUncommittedWorkAcrossExhaustionCountAndReset() throws Exception { + try (var db = new JdbcTestDatabase()) { + db.execute("CREATE TABLE ITEMS (ID INT)"); + var query = compiled(db.engine); + db.manual(); + db.execute("INSERT INTO ITEMS VALUES (1)"); + try (IRawSelectWrapper wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(db.engine, + db.connection, query, 0)) { + assertEquals(1, wrapper.getNumRows()); + assertEquals(1, wrapper.next().getValues()[0]); + assertFalse(wrapper.hasNext()); + wrapper.reset(); + assertEquals(1, wrapper.getNumRows()); + } + verify(db.connection, never()).commit(); + verify(db.connection, never()).rollback(); + verify(db.connection, never()).close(); + verify(db.connection, never()).setAutoCommit(anyBoolean()); + db.connection.rollback(); + assertEquals(0, db.count("ITEMS")); + } + } + + @ParameterizedTest + @ValueSource(strings = { "prepare", "bind", "execute", "metadata", "cursor", "value", "resultClose", + "statementClose", "rollback", "connectionClose" }) + void failuresReleaseOwnedResourcesAndPropagate(String stage) throws Exception { + IRDBMSEngine engine = mock(IRDBMSEngine.class); + Connection connection = mock(Connection.class); + PreparedStatement statement = mock(PreparedStatement.class); + ResultSet result = mock(ResultSet.class); + ResultSetMetaData metadata = mock(ResultSetMetaData.class); + when(engine.getConnection()).thenReturn(connection); + when(engine.isConnectionPooling()).thenReturn(true); + when(connection.prepareStatement(anyString())).thenReturn(statement); + when(statement.executeQuery()).thenReturn(result); + when(result.getMetaData()).thenReturn(metadata); + when(metadata.getColumnCount()).thenReturn(1); + when(metadata.getColumnType(1)).thenReturn(Types.INTEGER); + when(metadata.getColumnTypeName(1)).thenReturn("INTEGER"); + when(result.next()).thenReturn(true, false); + SQLException failure = new SQLException(stage); + switch (stage) { + case "prepare" -> when(connection.prepareStatement(anyString())).thenThrow(failure); + case "bind" -> doThrow(failure).when(statement).setObject(eq(1), any()); + case "execute" -> when(statement.executeQuery()).thenThrow(failure); + case "metadata" -> when(metadata.getColumnCount()).thenThrow(failure); + case "cursor" -> when(result.next()).thenThrow(failure); + case "value" -> when(result.getInt(1)).thenThrow(failure); + case "resultClose" -> doThrow(failure).when(result).close(); + case "statementClose" -> doThrow(failure).when(statement).close(); + case "rollback" -> doThrow(failure).when(connection).rollback(); + case "connectionClose" -> doThrow(failure).when(connection).close(); + } + var query = compiled(engine); + Exception propagated = assertThrows(Exception.class, () -> { + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(engine, query, 0)) { + while (wrapper.hasNext()) { + wrapper.next(); + } + } + }); + Throwable cause = propagated; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + assertSame(failure, cause); + verify(connection).rollback(); + verify(connection).close(); + verify(connection, never()).commit(); + if (!stage.equals("prepare")) { + verify(statement).close(); + } + if (!List.of("prepare", "bind", "execute").contains(stage)) { + verify(result).close(); + } + } + + @Test + void cleanupKeepsOriginalFailureAndAttemptsAllResources() throws Exception { + IRDBMSEngine engine = mock(IRDBMSEngine.class); + Connection connection = mock(Connection.class); + PreparedStatement statement = mock(PreparedStatement.class); + when(engine.getConnection()).thenReturn(connection); + when(engine.isConnectionPooling()).thenReturn(true); + when(connection.prepareStatement(anyString())).thenReturn(statement); + SQLException original = new SQLException("execution"); + when(statement.executeQuery()).thenThrow(original); + doThrow(new SQLException("statement close")).when(statement).close(); + doThrow(new SQLException("rollback")).when(connection).rollback(); + doThrow(new SQLException("release")).when(connection).close(); + assertSame(original, assertThrows(SQLException.class, + () -> RawPreparedRDBMSSelectWrapper.executeParameterized(engine, compiled(engine), 0))); + assertEquals(1, original.getSuppressed().length); + assertInstanceOf(IllegalStateException.class, original.getSuppressed()[0]); + assertEquals(2, original.getSuppressed()[0].getCause().getSuppressed().length); + verify(connection).rollback(); + verify(connection).close(); + } + + @Test + void zeroCountIsCachedAndCanBeComputedAfterExhaustion() throws Exception { + try (var db = new JdbcTestDatabase()) { + db.execute("CREATE TABLE ITEMS (ID INT)"); + var query = compiled(db.engine); + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(db.engine, query, 0)) { + assertFalse(wrapper.hasNext()); + clearInvocations(db.connection); + assertEquals(0, wrapper.getNumRows()); + assertEquals(0, wrapper.getNumRows()); + verify(db.connection).prepareStatement(query.countQuery().sql()); + } + } + } + + @Test + void compiledQueriesSnapshotListsAndApplyJdbcMaximumRows() throws Exception { + try (var db = new JdbcTestDatabase()) { + db.execute("CREATE TABLE ITEMS (ID INT)", "INSERT INTO ITEMS VALUES (1),(2)"); + var initial = compiled(db.engine); + List parameters = new ArrayList<>(initial.parameters()); + var limited = new CompiledQuery(new ParameterizedQuery(initial.sql(), parameters, 1), initial.countQuery()); + parameters.set(0, 100); + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(db.engine, limited, 0)) { + assertEquals(1, wrapper.getNumRows()); + assertEquals(1, wrapper.next().getValues()[0]); + assertFalse(wrapper.hasNext()); + } + } + } + + @Test + void pooledExecutionCountAfterExhaustionAndResetReleaseEveryLease() throws Exception { + try (var pool = new HikariDataSource()) { + pool.setJdbcUrl("jdbc:h2:mem:prepared_pool_" + UUID.randomUUID()); + pool.setAutoCommit(false); + pool.setMaximumPoolSize(2); + try (var connection = pool.getConnection(); var statement = connection.createStatement()) { + statement.execute("CREATE TABLE ITEMS(ID INT)"); + statement.execute("INSERT INTO ITEMS VALUES (1)"); + connection.commit(); + } + IRDBMSEngine engine = mock(IRDBMSEngine.class); + when(engine.isConnectionPooling()).thenReturn(true); + List leases = new ArrayList<>(); + when(engine.getConnection()).thenAnswer(call -> { + var connection = spy(pool.getConnection()); + leases.add(connection); + return connection; + }); + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(engine, compiled(engine), 0)) { + assertEquals(1, wrapper.next().getValues()[0]); + assertFalse(wrapper.hasNext()); + assertEquals(0, pool.getHikariPoolMXBean().getActiveConnections()); + assertEquals(1, wrapper.getNumRows()); + assertEquals(0, pool.getHikariPoolMXBean().getActiveConnections()); + wrapper.reset(); + assertEquals(1, wrapper.next().getValues()[0]); + } + assertEquals(3, leases.size()); + for (Connection connection : leases) { + verify(connection).rollback(); + verify(connection).close(); + verify(connection, never()).commit(); + verify(connection, never()).setAutoCommit(anyBoolean()); + } + assertEquals(0, pool.getHikariPoolMXBean().getActiveConnections()); + } + } + + @Test + void countFailureClosesStreamingResourcesAndOwnedReadTransaction() throws Exception { + try (var db = new JdbcTestDatabase()) { + db.execute("CREATE TABLE ITEMS (ID INT)", "INSERT INTO ITEMS VALUES (1)"); + var query = compiled(db.engine); + var brokenCount = new CompiledQuery(query.query(), + new ParameterizedQuery("SELECT * FROM MISSING_TABLE", List.of(), 0)); + db.manual(); + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(db.engine, brokenCount, 0)) { + assertTrue(wrapper.hasNext()); + assertThrows(IllegalArgumentException.class, wrapper::getNumRows); + assertFalse(wrapper.hasNext()); + } + verify(db.connection).rollback(); + verify(db.connection, never()).commit(); + } + } + + @Test + void borrowedExecutionFailureLeavesCallerTransactionIntact() throws Exception { + try (var db = new JdbcTestDatabase()) { + var query = compiled(db.engine); // ITEMS deliberately does not exist. + db.manual(); + assertThrows(SQLException.class, + () -> RawPreparedRDBMSSelectWrapper.executeParameterized(db.engine, db.connection, query, 0)); + verify(db.connection, never()).rollback(); + verify(db.connection, never()).close(); + verify(db.connection, never()).commit(); + verify(db.engine, never()).getConnection(); + } + } + + @Test + void compiledWrappersRetainTheirTemplateAndResourceOwner() throws Exception { + IRDBMSEngine engine = mock(IRDBMSEngine.class); + var query = compiled(engine); + try (var wrapper = RawPreparedRDBMSSelectWrapper.prepareParameterized(engine, query, 0)) { + wrapper.setQuery(query.sql()); + wrapper.setEngine(engine); + assertThrows(IllegalStateException.class, () -> wrapper.setQuery("SELECT 2")); + assertThrows(IllegalStateException.class, () -> wrapper.setEngine(mock(IRDBMSEngine.class))); + assertEquals(query.sql(), wrapper.getQuery()); + assertSame(engine, wrapper.getEngine()); + } + assertThrows(IllegalArgumentException.class, + () -> RawPreparedRDBMSSelectWrapper.prepareParameterized(engine, query, -1)); + assertThrows(IllegalArgumentException.class, + () -> new CompiledQuery(new ParameterizedQuery("", List.of(), 0), query.countQuery())); + verify(engine, never()).getConnection(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + void uncheckedReadFailuresPreserveTheCauseAndReleaseResources(boolean error) throws Exception { + try (var db = new JdbcTestDatabase()) { + db.execute("CREATE TABLE ITEMS (ID INT)", "INSERT INTO ITEMS VALUES (1)"); + var query = compiled(db.engine); + db.manual(); + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(db.engine, query, 0)) { + wrapper.rs = spy(wrapper.rs); + Throwable failure = error ? new AssertionError("driver error") + : new IllegalStateException("driver failure"); + when(wrapper.rs.next()).thenThrow(failure); + assertSame(failure, assertThrows(failure.getClass(), wrapper::hasNext)); + assertFalse(wrapper.hasNext()); + verify(wrapper.rs).close(); + } + verify(db.connection).rollback(); + } + } + + @Test + void earlyCloseDiscardsBufferedRowsWithoutTakingOwnershipOfBorrowedConnection() throws Exception { + try (var db = new JdbcTestDatabase()) { + db.execute("CREATE TABLE ITEMS (ID INT)", "INSERT INTO ITEMS VALUES (1)"); + var query = compiled(db.engine); + db.manual(); + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(db.engine, db.connection, query, 0)) { + wrapper.setCloseConenctionAfterExecution(true); + assertTrue(wrapper.hasNext()); + wrapper.close(); + wrapper.close(); + assertFalse(wrapper.hasNext()); + assertThrows(NoSuchElementException.class, wrapper::next); + assertEquals(1, wrapper.getNumRows()); + wrapper.reset(); + assertEquals(1, wrapper.next().getValues()[0]); + } + verify(db.engine, never()).getConnection(); + verify(db.connection, never()).close(); + verify(db.connection, never()).rollback(); + verify(db.connection, never()).commit(); + } + } + + @Test + void countAndResetUseIndependentBindingsAndStatementTimeouts() throws Exception { + try (var db = new JdbcTestDatabase()) { + db.execute("CREATE TABLE ITEMS (ID INT)", "INSERT INTO ITEMS VALUES (1),(2)"); + var initial = compiled(db.engine); + // Distinct data/count values make accidentally reusing the data binder + // observable. + var query = new CompiledQuery(initial.query(), + new ParameterizedQuery("SELECT COUNT(*) FROM ITEMS WHERE ID >= ?", List.of(2), 0)); + List statements = new ArrayList<>(); + doAnswer(call -> { + var statement = spy((PreparedStatement) call.callRealMethod()); + statements.add(statement); + return statement; + }).when(db.connection).prepareStatement(anyString()); + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(db.engine, query, 7)) { + assertEquals(1, wrapper.getNumRows()); + assertEquals(1, wrapper.next().getValues()[0]); + assertEquals(2, wrapper.next().getValues()[0]); + assertFalse(wrapper.hasNext()); + db.execute("INSERT INTO ITEMS VALUES (3)"); + wrapper.reset(); + assertEquals(2, wrapper.getNumRows()); + } + assertEquals(4, statements.size()); + for (var statement : statements) { + verify(statement).setQueryTimeout(7); + verify(statement).close(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = { "result", "statement", "rollback", "connection" }) + void cleanupErrorsAtExhaustionPropagateAfterAllResourcesAreAttempted(String stage) throws Exception { + IRDBMSEngine engine = mock(IRDBMSEngine.class); + Connection connection = mock(Connection.class); + PreparedStatement statement = mock(PreparedStatement.class); + ResultSet result = mock(ResultSet.class); + when(engine.getConnection()).thenReturn(connection); + when(engine.isConnectionPooling()).thenReturn(true); + when(connection.prepareStatement(anyString())).thenReturn(statement); + when(statement.executeQuery()).thenReturn(result); + when(result.getMetaData()).thenReturn(mock(ResultSetMetaData.class)); + AssertionError failure = new AssertionError(stage); + switch (stage) { + case "result" -> doThrow(failure).when(result).close(); + case "statement" -> doThrow(failure).when(statement).close(); + case "rollback" -> doThrow(failure).when(connection).rollback(); + case "connection" -> doThrow(failure).when(connection).close(); + } + try (var wrapper = RawPreparedRDBMSSelectWrapper.executeParameterized(engine, compiled(engine), 0)) { + assertSame(failure, assertThrows(AssertionError.class, wrapper::hasNext)); + assertFalse(wrapper.hasNext()); + } + verify(result).close(); + verify(statement).close(); + verify(connection).rollback(); + verify(connection).close(); + } +} diff --git a/test/prerna/rdf/engine/wrappers/RawRDBMSSelectWrapperUnitTests.java b/test/prerna/rdf/engine/wrappers/RawRDBMSSelectWrapperUnitTests.java new file mode 100644 index 00000000000..858695352ab --- /dev/null +++ b/test/prerna/rdf/engine/wrappers/RawRDBMSSelectWrapperUnitTests.java @@ -0,0 +1,197 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.rdf.engine.wrappers; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.ZoneOffset; +import java.util.Map; +import java.util.NoSuchElementException; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import prerna.algorithm.api.SemossDataType; +import prerna.date.SemossDate; +import prerna.engine.api.IRDBMSEngine; +import prerna.util.JdbcTestDatabase; + +class RawRDBMSSelectWrapperUnitTests { + + @Test + void legacyConnectionExecutionStillMapsRowsAndCounts() throws Exception { + try (var db = new JdbcTestDatabase(); + var wrapper = RawRDBMSSelectWrapper.directExecutionViaConnection(db.connection, "SELECT 42 AS ANSWER", + false)) { + assertEquals(RawRDBMSSelectWrapper.class, wrapper.getClass()); + assertArrayEquals(new String[] { "ANSWER" }, wrapper.getHeaders()); + assertArrayEquals(new SemossDataType[] { SemossDataType.INT }, wrapper.getTypes()); + assertEquals(1, wrapper.getMetaData().getColumnCount()); + assertEquals(1, wrapper.getNumRows()); + assertEquals(1, wrapper.getNumRecords()); + assertFalse(wrapper.flushable()); + assertNull(wrapper.flush()); + assertTrue(wrapper.hasNext()); + assertTrue(wrapper.hasNext()); // Reading ahead must not skip a row. + assertEquals(42, wrapper.next().getValues()[0]); + assertFalse(wrapper.hasNext()); + assertThrows(NoSuchElementException.class, wrapper::next); + assertFalse(db.connection.isClosed()); + } + } + + @Test + void sharedConverterPreservesJdbcValueTypesAndNulls() throws Exception { + String sql = "SELECT 42, CAST(5000000000 AS BIGINT), CAST(1.25 AS DECIMAL(8,2)), " + + "DATE '2026-01-02', TIMESTAMP '2026-01-02 03:04:05', CAST('text' AS CLOB), " + + "CAST(X'68656c6c6f' AS BLOB), CAST(X'616263' AS BINARY(3)), " + + "CAST(X'646566' AS VARBINARY), TRUE, ARRAY[1,2], 'label', CAST(NULL AS INTEGER)"; + try (var db = new JdbcTestDatabase(); + var wrapper = RawRDBMSSelectWrapper.directExecutionViaConnection(db.connection, sql, false)) { + wrapper.databaseZoneId = ZoneOffset.UTC; + Object[] row = wrapper.next().getValues(); + assertEquals(42, row[0]); + assertEquals(5000000000L, row[1]); + assertEquals(1.25d, row[2]); + assertEquals("2026-01-02", assertInstanceOf(SemossDate.class, row[3]).toString()); + assertEquals("2026-01-02 03:04:05", assertInstanceOf(SemossDate.class, row[4]).toString()); + assertEquals("text", row[5]); + assertEquals("hello", row[6]); + assertEquals("abc", row[7]); + assertEquals("def", row[8]); + assertEquals(true, row[9]); + assertArrayEquals(new Object[] { 1, 2 }, (Object[]) row[10]); + assertEquals("label", row[11]); + assertNull(row[12]); + assertFalse(wrapper.hasNext()); + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + void engineExecutionKeepsTheLegacyConnectionPolicy(boolean closeConnection) throws Exception { + try (var db = new JdbcTestDatabase(); var wrapper = new RawRDBMSSelectWrapper()) { + when(db.engine.execQuery("SELECT 7 AS VALUE_COLUMN")).thenAnswer(call -> { + Statement statement = db.connection.createStatement(); + return Map.of(IRDBMSEngine.STATEMENT_OBJECT, statement, IRDBMSEngine.ENGINE_CONNECTION_OBJECT, + db.connection, IRDBMSEngine.RESULTSET_OBJECT, + statement.executeQuery("SELECT 7 AS VALUE_COLUMN")); + }); + wrapper.setEngine(db.engine); + wrapper.setQuery("SELECT 7 AS VALUE_COLUMN"); + wrapper.setCloseConenctionAfterExecution(closeConnection); + wrapper.execute(); + assertEquals(7, wrapper.next().getValues()[0]); + verify(db.engine).execQuery("SELECT 7 AS VALUE_COLUMN"); + wrapper.close(); + assertEquals(closeConnection, db.connection.isClosed()); + } + } + + @Test + void ordinaryWrappersStillAllowChangingTheQueryAndEngine() throws Exception { + try (var wrapper = new RawRDBMSSelectWrapper()) { + var first = mock(IRDBMSEngine.class); + var second = mock(IRDBMSEngine.class); + wrapper.setQuery("SELECT 1"); + wrapper.setEngine(first); + wrapper.setQuery("SELECT 2"); + wrapper.setEngine(second); + assertEquals("SELECT 2", wrapper.getQuery()); + assertSame(second, wrapper.getEngine()); + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + void directExecutionFailureRespectsCloseIfFail(boolean closeIfFail) throws Exception { + Connection connection = mock(Connection.class); + Statement statement = mock(Statement.class); + SQLException failure = new SQLException("query failed"); + when(connection.createStatement()).thenReturn(statement); + when(statement.executeQuery("SELECT 1")).thenThrow(failure); + assertSame(failure, assertThrows(SQLException.class, + () -> RawRDBMSSelectWrapper.directExecutionViaConnection(connection, "SELECT 1", closeIfFail))); + verify(statement).close(); + verify(connection, times(closeIfFail ? 1 : 0)).close(); + } + + @Test + void metadataFailuresKeepLegacyLogAndContinueBehavior() throws Exception { + ResultSet result = mock(ResultSet.class); + when(result.getMetaData()).thenThrow(new SQLException("metadata unavailable")); + try (var wrapper = assertDoesNotThrow(() -> RawRDBMSSelectWrapper.flushRsToWrapper(result))) { + assertNull(wrapper.getHeaders()); + assertFalse(wrapper.hasNext()); + } + verify(result).close(); + } + + @Test + void exhaustionKeepsLegacyLogAndContinueBehaviorForCloseFailures() throws Exception { + try (var db = new JdbcTestDatabase(); + var wrapper = RawRDBMSSelectWrapper.directExecutionViaConnection(db.connection, "SELECT 1 WHERE 1=0", + false)) { + wrapper.rs = spy(wrapper.rs); + doThrow(new SQLException("close failed")).when(wrapper.rs).close(); + assertFalse(assertDoesNotThrow(wrapper::hasNext)); + } + } + + @Test + void cursorFailuresKeepLegacyErrorMessage() throws Exception { + try (var db = new JdbcTestDatabase(); + var wrapper = RawRDBMSSelectWrapper.directExecutionViaConnection(db.connection, "SELECT 1", false)) { + wrapper.rs = spy(wrapper.rs); + when(wrapper.rs.next()).thenThrow(new SQLException("cursor failed")); + assertTrue(assertThrows(IllegalArgumentException.class, wrapper::hasNext).getMessage() + .contains("cursor failed")); + verify(db.connection, never()).close(); + } + } +} diff --git a/test/prerna/rdf/engine/wrappers/WrapperManagerUnitTests.java b/test/prerna/rdf/engine/wrappers/WrapperManagerUnitTests.java new file mode 100644 index 00000000000..82d2e19c395 --- /dev/null +++ b/test/prerna/rdf/engine/wrappers/WrapperManagerUnitTests.java @@ -0,0 +1,157 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.rdf.engine.wrappers; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.startsWith; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.SQLException; +import java.time.ZoneOffset; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; + +import prerna.query.querystruct.SelectQueryStruct; +import prerna.query.querystruct.filters.SimpleQueryFilter; +import prerna.query.querystruct.selectors.QueryColumnSelector; +import prerna.query.querystruct.selectors.QueryOpaqueSelector; +import prerna.usertracking.UserQueryTrackingThread; +import prerna.util.Constants; +import prerna.util.JdbcTestDatabase; + +class WrapperManagerUnitTests { + + private SelectQueryStruct query() { + SelectQueryStruct query = new SelectQueryStruct(); + query.addSelector(new QueryColumnSelector("ITEMS__V", "label")); + query.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "==", "O'Brien")); + return query; + } + + @Test + void deferredReadsCountAndResetTrackTemplatesOnlyWhenExecuted() throws Exception { + try (var db = new JdbcTestDatabase(); var trackers = mockConstruction(UserQueryTrackingThread.class)) { + db.execute("CREATE TABLE ITEMS (V VARCHAR)", "INSERT INTO ITEMS VALUES ('O''Brien')"); + when(db.engine.isBasic()).thenReturn(true); + when(db.engine.getEngineId()).thenReturn("catalog-tracking-test"); + when(db.engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + try (var wrapper = WrapperManager.getInstance().getPreparedWrapper(db.engine, query(), true, 2)) { + assertInstanceOf(RawPreparedRDBMSSelectWrapper.class, wrapper); + verify(db.engine, never()).getConnection(); + assertTrue(trackers.constructed().isEmpty()); + wrapper.execute(); + assertEquals(1, wrapper.getNumRows()); + assertEquals(1, wrapper.getNumRows()); // Cached counts are not separate queries. + assertEquals("O'Brien", wrapper.next().getValues()[0]); + wrapper.reset(); + assertEquals(3, trackers.constructed().size()); + for (var tracker : trackers.constructed()) { + verify(tracker).setQuery(argThat(sql -> sql.contains("?") && !sql.contains("O'Brien"))); + verify(tracker).setStartTimeNow(); + verify(tracker).setEndTimeNow(); + verify(tracker, never()).setFailed(); + verify(tracker, timeout(1000)).run(); + } + } + } + } + + @ParameterizedTest + @ValueSource(strings = { "prepare", "count" }) + void failuresAreTrackedAndReleaseReadResources(String stage) throws Exception { + try (var db = new JdbcTestDatabase(); var trackers = mockConstruction(UserQueryTrackingThread.class)) { + db.execute("CREATE TABLE ITEMS (V VARCHAR)"); + when(db.engine.isBasic()).thenReturn(true); + when(db.engine.getEngineId()).thenReturn("catalog-tracking-test"); + db.manual(); + if (stage.equals("prepare")) { + doThrow(new SQLException("prepare failed")).when(db.connection).prepareStatement(anyString()); + assertThrows(SQLException.class, + () -> WrapperManager.getInstance().getPreparedWrapper(db.engine, query())); + } else { + try (var wrapper = WrapperManager.getInstance().getPreparedWrapper(db.engine, query())) { + assertInstanceOf(RawPreparedRDBMSSelectWrapper.class, wrapper); + doThrow(new SQLException("count failed")).when(db.connection) + .prepareStatement(startsWith("SELECT COUNT")); + assertThrows(IllegalArgumentException.class, wrapper::getNumRows); + } + } + verify(trackers.constructed().getLast()).setFailed(); + verify(trackers.constructed().getLast()).setEndTimeNow(); + verify(db.connection, atLeastOnce()).rollback(); + } + } + + @ParameterizedTest + @NullSource + @ValueSource(strings = { Constants.SECURITY_DB, Constants.USER_TRACKING_DB, "test_" + Constants.OWL_ENGINE_SUFFIX, + "test_" + Constants.RDBMS_INSIGHTS_ENGINE_SUFFIX }) + void internalQueriesDoNotCreateTrackingLoops(String engineId) throws Exception { + try (var db = new JdbcTestDatabase(); var trackers = mockConstruction(UserQueryTrackingThread.class)) { + when(db.engine.getEngineId()).thenReturn(engineId); + assertNull(WrapperManager.preparedQueryTracker(db.engine, "SELECT ?")); + assertTrue(trackers.constructed().isEmpty()); + } + } + + @Test + void unsupportedStructuresFailBeforeAcquiringAConnection() throws Exception { + try (var db = new JdbcTestDatabase()) { + SelectQueryStruct invalid = query(); + invalid.addSelector(new QueryOpaqueSelector()); + assertThrows(IllegalArgumentException.class, + () -> WrapperManager.getInstance().getPreparedWrapper(db.engine, invalid)); + verify(db.engine, never()).getConnection(); + } + } + + @Test + void trackingCompletionFailureDoesNotDiscardASuccessfulRead() throws Exception { + var tracker = mock(UserQueryTrackingThread.class); + doThrow(new IllegalStateException("tracking unavailable")).when(tracker).setEndTimeNow(); + assertDoesNotThrow(() -> WrapperManager.finishPreparedQueryTracking(tracker)); + assertDoesNotThrow(() -> WrapperManager.finishPreparedQueryTracking(null)); + } +} diff --git a/test/prerna/util/ConnectionUtilsUnitTests.java b/test/prerna/util/ConnectionUtilsUnitTests.java index c617e062488..8a45c1eb42f 100644 --- a/test/prerna/util/ConnectionUtilsUnitTests.java +++ b/test/prerna/util/ConnectionUtilsUnitTests.java @@ -27,40 +27,271 @@ *******************************************************************************/ package prerna.util; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import prerna.engine.api.IRDBMSEngine; class ConnectionUtilsUnitTests { - @Test - void pooledReleaseClosesConnectionAndPropagatesFailure() throws Exception { + private IRDBMSEngine engine(boolean pooling) { IRDBMSEngine engine = mock(IRDBMSEngine.class); + when(engine.isConnectionPooling()).thenReturn(pooling); + return engine; + } + + @Test + void combinesFailuresWithoutLosingTheOriginalOrSuppressingSuccessOrItself() { + SQLException original = new SQLException("work failed"); + SQLException cleanup = new SQLException("cleanup failed"); + assertNull(ConnectionUtils.jdbcCleanupFailure(null, null)); + assertNull(ConnectionUtils.jdbcCleanupFailure(null, null, "no failure")); + assertSame(original, ConnectionUtils.jdbcCleanupFailure(null, original, "work failed")); + assertSame(original, ConnectionUtils.jdbcCleanupFailure(original, null)); + assertSame(original, ConnectionUtils.jdbcCleanupFailure(original, original)); + assertArrayEquals(new Throwable[0], original.getSuppressed()); + assertSame(original, ConnectionUtils.jdbcCleanupFailure(original, cleanup)); + assertArrayEquals(new Throwable[] { cleanup }, original.getSuppressed()); + } + + @Test + void attemptsEveryCloseAndReturnsFailuresInResourceOrder() throws Exception { + Connection connection = mock(Connection.class); + Statement statement = mock(Statement.class); + ResultSet result = mock(ResultSet.class); + SQLException resultFailure = new SQLException("result close"); + AssertionError statementFailure = new AssertionError("statement close"); + IllegalStateException connectionFailure = new IllegalStateException("connection close"); + doThrow(resultFailure).when(result).close(); + doThrow(statementFailure).when(statement).close(); + doThrow(connectionFailure).when(connection).close(); + + assertSame(resultFailure, ConnectionUtils.closeAllConnections(connection, statement, result)); + assertArrayEquals(new Throwable[] { statementFailure, connectionFailure }, resultFailure.getSuppressed()); + var order = inOrder(result, statement, connection); + order.verify(result).close(); + order.verify(statement).close(); + order.verify(connection).close(); + } + + @ParameterizedTest + @ValueSource(strings = { "connection", "connectionAlias", "statement", "statementResult", "connectionStatement", + "connectionStatementResult", "varargs", "multipleConnections" }) + void everyPooledOverloadReturnsTheReleaseFailure(String variant) throws Exception { Connection connection = mock(Connection.class); - when(engine.isConnectionPooling()).thenReturn(true); + Statement statement = mock(Statement.class); + ResultSet result = mock(ResultSet.class); + IRDBMSEngine engine = engine(true); SQLException failure = new SQLException("release failed"); doThrow(failure).when(connection).close(); - assertSame(failure, - assertThrows(SQLException.class, () -> ConnectionUtils.closeConnectionIfPooling(engine, connection))); + AtomicBoolean statementClosed = new AtomicBoolean(); + when(statement.getConnection()).thenAnswer(call -> { + assertFalse(statementClosed.get(), "connection must be obtained before statement close"); + return connection; + }); + doAnswer(call -> { + statementClosed.set(true); + return null; + }).when(statement).close(); + + Throwable actual = switch (variant) { + case "connection" -> ConnectionUtils.closeConnectionIfPooling(engine, connection); + case "connectionAlias" -> ConnectionUtils.closeAllConnectionsIfPooling(engine, connection); + case "statement" -> ConnectionUtils.closeAllConnectionsIfPooling(engine, statement); + case "statementResult" -> ConnectionUtils.closeAllConnectionsIfPooling(engine, statement, result); + case "connectionStatement" -> ConnectionUtils.closeAllConnectionsIfPooling(engine, null, statement); + case "connectionStatementResult" -> + ConnectionUtils.closeAllConnectionsIfPooling(engine, connection, statement, result); + case "varargs" -> + ConnectionUtils.closeAllConnectionsIfPooling(engine, null, new Statement[] { null, statement }); + case "multipleConnections" -> ConnectionUtils.closeAllDbConnectionsIfPooling(engine, null, statement); + default -> throw new AssertionError(variant); + }; + assertSame(failure, actual); verify(connection).close(); + if (!variant.equals("connection") && !variant.equals("connectionAlias")) { + verify(statement).close(); + } + } + + @Test + void pooledResultFailureRetainsSubsequentStatementAndReleaseFailures() throws Exception { + Connection connection = mock(Connection.class); + Statement statement = mock(Statement.class); + ResultSet result = mock(ResultSet.class); + SQLException resultFailure = new SQLException("result close"); + SQLException statementFailure = new SQLException("statement close"); + SQLException connectionFailure = new SQLException("connection close"); + doThrow(resultFailure).when(result).close(); + doThrow(statementFailure).when(statement).close(); + doThrow(connectionFailure).when(connection).close(); + assertSame(resultFailure, + ConnectionUtils.closeAllConnectionsIfPooling(engine(true), connection, statement, result)); + assertArrayEquals(new Throwable[] { statementFailure, connectionFailure }, resultFailure.getSuppressed()); + verify(result).close(); + verify(statement).close(); + verify(connection).close(); + } + + @Test + void lookupFailureStillClosesTheStatementAndTriesTheNextConnection() throws Exception { + Connection connection = mock(Connection.class); + Statement first = mock(Statement.class); + Statement second = mock(Statement.class); + SQLException lookup = new SQLException("connection lookup"); + SQLException closeFirst = new SQLException("first close"); + SQLException release = new SQLException("release"); + when(first.getConnection()).thenThrow(lookup); + doThrow(closeFirst).when(first).close(); + when(second.getConnection()).thenReturn(connection); + doThrow(release).when(connection).close(); + assertSame(lookup, ConnectionUtils.closeAllConnectionsIfPooling(engine(true), null, + new Statement[] { null, first, second, null })); + assertArrayEquals(new Throwable[] { closeFirst, release }, lookup.getSuppressed()); + var order = inOrder(first, second, connection); + order.verify(first).getConnection(); + order.verify(first).close(); + order.verify(second).getConnection(); + order.verify(second).close(); + order.verify(connection).close(); + } + + @Test + void multipleDatabaseConnectionsAreAllReleasedAfterAnError() throws Exception { + Connection firstConnection = mock(Connection.class); + Connection secondConnection = mock(Connection.class); + Statement first = mock(Statement.class); + Statement second = mock(Statement.class); + when(first.getConnection()).thenReturn(firstConnection); + when(second.getConnection()).thenReturn(secondConnection); + AssertionError failure = new AssertionError("first statement"); + SQLException release = new SQLException("first connection"); + SQLException later = new SQLException("second statement"); + doThrow(failure).when(first).close(); + doThrow(release).when(firstConnection).close(); + doThrow(later).when(second).close(); + assertSame(failure, ConnectionUtils.closeAllDbConnectionsIfPooling(engine(true), first, null, second)); + assertArrayEquals(new Throwable[] { release, later }, failure.getSuppressed()); + verify(firstConnection).close(); + verify(secondConnection).close(); } @Test - void sharedConnectionRemainsOpen() throws Exception { + void poolingPolicyFailureDoesNotPreventResultAndStatementCleanup() throws Exception { IRDBMSEngine engine = mock(IRDBMSEngine.class); Connection connection = mock(Connection.class); - ConnectionUtils.closeConnectionIfPooling(engine, connection); + Statement statement = mock(Statement.class); + ResultSet result = mock(ResultSet.class); + AssertionError failure = new AssertionError("pooling policy"); + when(engine.isConnectionPooling()).thenThrow(failure); + assertSame(failure, ConnectionUtils.closeAllConnectionsIfPooling(engine, connection, statement, result)); + verify(result).close(); + verify(statement).close(); + verifyNoInteractions(connection); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + void sharedOrUnknownEngineLeavesConnectionsOpen(boolean nullEngine) throws Exception { + IRDBMSEngine engine = nullEngine ? null : engine(false); + Connection connection = mock(Connection.class); + Statement statement = mock(Statement.class); + ResultSet result = mock(ResultSet.class); + assertNull(ConnectionUtils.closeConnectionIfPooling(engine, connection)); + assertNull(ConnectionUtils.closeAllConnectionsIfPooling(engine, connection, statement, result)); + verify(result).close(); + verify(statement).close(); + verify(statement, never()).getConnection(); verifyNoInteractions(connection); } + + @Test + void convenienceOverloadsReturnFailuresInsteadOfDiscardingThem() throws Exception { + Statement statement = mock(Statement.class); + ResultSet result = mock(ResultSet.class); + SQLException failure = new SQLException("statement close"); + doThrow(failure).when(statement).close(); + assertSame(failure, ConnectionUtils.closeAllConnections((Connection) null, statement)); + assertSame(failure, ConnectionUtils.closeAllConnections(statement, result)); + verify(result).close(); + } + + @Test + void nullResourcesAndSuccessfulClosesReturnNull() throws Exception { + assertNull(ConnectionUtils.closeAllConnections(null, null, null)); + assertNull(ConnectionUtils.closeAllConnectionsIfPooling(null, null, null, null)); + assertNull(ConnectionUtils.closeAllConnectionsIfPooling(engine(true), null, (Statement[]) null)); + assertNull(ConnectionUtils.closeAllDbConnectionsIfPooling(null, (Statement[]) null)); + assertNull(ConnectionUtils.closeAllDbConnectionsIfPooling(null, new Statement[] { null })); + assertNull(ConnectionUtils.commitConnection(null)); + assertNull(ConnectionUtils.rollbackConnection(null)); + Connection connection = mock(Connection.class); + Statement statement = mock(Statement.class); + ResultSet result = mock(ResultSet.class); + assertNull(ConnectionUtils.closeAllConnections(connection, statement, result)); + verify(connection).close(); + verify(statement).close(); + verify(result).close(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + void commitRespectsAutoCommitWithoutClosingOrChangingTheConnection(boolean autoCommit) throws Exception { + Connection connection = mock(Connection.class); + when(connection.getAutoCommit()).thenReturn(autoCommit); + assertNull(ConnectionUtils.commitConnection(connection)); + verify(connection, org.mockito.Mockito.times(autoCommit ? 0 : 1)).commit(); + verify(connection, never()).close(); + verify(connection, never()).setAutoCommit(org.mockito.ArgumentMatchers.anyBoolean()); + } + + @ParameterizedTest + @CsvSource({ "commit,false", "commit,true", "rollback,false", "rollback,true", "readState,false", + "readState,true" }) + void transactionHelpersReturnTheOriginalExceptionOrError(String operation, boolean error) throws Exception { + Connection connection = mock(Connection.class); + Throwable failure = error ? new AssertionError(operation) : new SQLException(operation); + if (operation.equals("commit")) { + doThrow(failure).when(connection).commit(); + } else if (operation.equals("rollback")) { + doThrow(failure).when(connection).rollback(); + } else { + when(connection.getAutoCommit()).thenThrow(failure); + } + assertSame(failure, operation.equals("rollback") ? ConnectionUtils.rollbackConnection(connection) + : ConnectionUtils.commitConnection(connection)); + verify(connection, never()).close(); + verify(connection, never()).setAutoCommit(org.mockito.ArgumentMatchers.anyBoolean()); + } + + @Test + void rollbackLeavesConnectionOwnershipAndModeToTheCaller() throws Exception { + Connection connection = mock(Connection.class); + assertNull(ConnectionUtils.rollbackConnection(connection)); + verify(connection).rollback(); + verify(connection, never()).getAutoCommit(); + verify(connection, never()).commit(); + verify(connection, never()).close(); + } } diff --git a/test/prerna/util/QueryExecutionUtilityUnitTests.java b/test/prerna/util/QueryExecutionUtilityUnitTests.java index d3442484bc4..c211205866c 100644 --- a/test/prerna/util/QueryExecutionUtilityUnitTests.java +++ b/test/prerna/util/QueryExecutionUtilityUnitTests.java @@ -31,6 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; @@ -50,8 +51,10 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.UUID; import org.junit.jupiter.api.AfterEach; @@ -60,7 +63,12 @@ import prerna.date.SemossDate; import prerna.engine.api.IRDBMSEngine; +import prerna.query.querystruct.SelectQueryStruct; +import prerna.query.querystruct.filters.SimpleQueryFilter; +import prerna.query.querystruct.selectors.QueryColumnSelector; import prerna.util.QueryExecutionUtility.ParameterizedQuery; +import prerna.util.sql.RdbmsTypeEnum; +import prerna.util.sql.SqlQueryUtilFactory; class QueryExecutionUtilityUnitTests { @@ -188,4 +196,66 @@ void rejectsInvalidArgumentsBeforeOpeningAStatement() { } verifyNoInteractions(engine); } + + @Test + void structuredPreparedReadsKeepOriginalValuesAndMapAliases() throws Exception { + when(engine.isBasic()).thenReturn(true); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(RdbmsTypeEnum.H2_DB)); + when(engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + try (var setup = connection.createStatement()) { + setup.execute("CREATE TABLE ITEMS (V VARCHAR, DETAIL CLOB)"); + setup.execute("INSERT INTO ITEMS VALUES ('O''Brien', 'long text')"); + } + var query = new SelectQueryStruct(); + query.addSelector(new QueryColumnSelector("ITEMS__V", "display_name")); + query.addSelector(new QueryColumnSelector("ITEMS__DETAIL", "detail")); + query.addExplicitFilter(SimpleQueryFilter.makeColToValFilter("ITEMS__V", "==", "O'Brien")); + var rows = QueryExecutionUtility.flushPreparedRsToMap(engine, query); + assertEquals(Map.of("display_name", "O'Brien", "detail", "long text"), rows.getFirst()); + verify(statement).setObject(1, "O'Brien"); + assertTrue(result.isClosed()); + assertTrue(statement.isClosed()); + assertFalse(connection.isClosed()); + } + + @Test + void structuredPreparedFailureRetainsCauseAndReleasesStatement() throws Exception { + when(engine.isBasic()).thenReturn(true); + when(engine.getQueryUtil()).thenReturn(SqlQueryUtilFactory.initialize(RdbmsTypeEnum.H2_DB)); + var query = new SelectQueryStruct(); + query.addSelector(new QueryColumnSelector("MISSING__V")); + var failure = assertThrows(IllegalArgumentException.class, + () -> QueryExecutionUtility.flushPreparedRsToMap(engine, query)); + assertEquals("Error executing prepared query", failure.getMessage()); + assertInstanceOf(SQLException.class, failure.getCause()); + assertFalse(connection.isClosed()); + } + + @Test + void returnedReleaseFailureIsReportedAfterSuccessfulRead() throws Exception { + when(engine.isConnectionPooling()).thenReturn(true); + connection.setAutoCommit(false); + SQLException failure = new SQLException("release failed"); + doThrow(failure).doCallRealMethod().when(connection).close(); + var actual = assertThrows(IllegalArgumentException.class, () -> QueryExecutionUtility.flushRsToMap(engine, + new ParameterizedQuery("SELECT 1 AS N", List.of(), 0), 0)); + assertSame(failure, actual.getCause()); + verify(connection).rollback(); + verify(connection, never()).commit(); + assertTrue(statement.isClosed()); + } + + @Test + void returnedCleanupErrorDoesNotReplaceTheOriginalWorkFailure() throws Exception { + when(engine.isConnectionPooling()).thenReturn(true); + connection.setAutoCommit(false); + SQLException original = new SQLException("read failed"); + AssertionError release = new AssertionError("release failed"); + doThrow(release).doCallRealMethod().when(connection).close(); + assertSame(original, assertThrows(SQLException.class, () -> QueryExecutionUtility.read(engine, c -> { + throw original; + }))); + assertSame(release, original.getSuppressed()[0]); + verify(connection).rollback(); + } } diff --git a/test/prerna/util/sql/AbstractSqlQueryUtilUnitTests.java b/test/prerna/util/sql/AbstractSqlQueryUtilUnitTests.java index 59132f225ec..ee477761f98 100644 --- a/test/prerna/util/sql/AbstractSqlQueryUtilUnitTests.java +++ b/test/prerna/util/sql/AbstractSqlQueryUtilUnitTests.java @@ -33,9 +33,17 @@ import java.sql.Connection; import java.sql.DriverManager; +import java.sql.Types; import java.util.UUID; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import prerna.engine.api.IRDBMSEngine; +import prerna.query.interpreters.sql.ParameterizedSqlInterpreter; +import prerna.query.querystruct.SelectQueryStruct; +import prerna.query.querystruct.selectors.QueryColumnSelector; class AbstractSqlQueryUtilUnitTests { @@ -59,14 +67,32 @@ void nullableStringVariantsPreserveTheirDistinctEmptyAndWhitespaceContracts() th dialect.setStringEmptyAsNullable(ps, 5, ""); dialect.setStringEmptyAsNullable(ps, 6, " "); dialect.setStringEmptyAsNullable(ps, 7, " café 世界 "); - verify(ps).setNull(1, java.sql.Types.VARCHAR); + verify(ps).setNull(1, Types.VARCHAR); verify(ps).setString(2, ""); verify(ps).setString(3, " "); - verify(ps).setNull(4, java.sql.Types.VARCHAR); - verify(ps).setNull(5, java.sql.Types.VARCHAR); + verify(ps).setNull(4, Types.VARCHAR); + verify(ps).setNull(5, Types.VARCHAR); verify(ps).setString(6, " "); verify(ps).setString(7, " café 世界 "); verifyNoMoreInteractions(ps); } } + + @Test + void dialectSearchContractProducesBoundPatternsThroughTheInterpreter() { + for (var type : RdbmsTypeEnum.values()) { + AbstractSqlQueryUtil util = SqlQueryUtilFactory.initialize(type); + var engine = mock(IRDBMSEngine.class); + Mockito.when(engine.isBasic()).thenReturn(true); + Mockito.when(engine.getQueryUtil()).thenReturn(util); + var query = new SelectQueryStruct(); + query.addSelector(new QueryColumnSelector("ITEMS__V", "label")); + query.addExplicitFilter(util.getPreparedSearchRegexFilter("ITEMS__V", "o'brien")); + var compiled = new ParameterizedSqlInterpreter(engine).compile(query); + Assertions.assertFalse(compiled.sql().contains("o'brien")); + Assertions.assertTrue(compiled.parameters().stream() + .anyMatch(value -> value instanceof String pattern && pattern.contains("o'brien"))); + } + } + } diff --git a/test/prerna/util/sql/AnsiSqlQueryUtilUnitTests.java b/test/prerna/util/sql/AnsiSqlQueryUtilUnitTests.java index 63015014753..7b1fe8228a2 100644 --- a/test/prerna/util/sql/AnsiSqlQueryUtilUnitTests.java +++ b/test/prerna/util/sql/AnsiSqlQueryUtilUnitTests.java @@ -46,6 +46,9 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import prerna.query.querystruct.filters.FunctionQueryFilter; +import prerna.query.querystruct.selectors.QueryConstantSelector; + class AnsiSqlQueryUtilUnitTests { @Test @@ -81,4 +84,17 @@ void legacyClobInputsAndEncodingContractsRemainCompatible() throws Exception { new PostgresQueryUtil().handleInsertionOfClob(statement, null, 1, gson); verify(statement).setNull(1, Types.LONGVARCHAR); } + + @Test + void preparedRegexKeepsRawPatternWhileLegacyFilterEscapesIt() { + var util = new H2QueryUtil(); + String pattern = "O'Brien\\path"; + var prepared = (FunctionQueryFilter) util.getPreparedSearchRegexFilter("ITEMS__V", pattern); + var legacy = (FunctionQueryFilter) util.getSearchRegexFilter("ITEMS__V", pattern); + assertEquals(pattern, + ((QueryConstantSelector) prepared.getFunctionSelector().getInnerSelector().get(1)).getConstant()); + assertEquals(util.escapeForSQLStatement(pattern), + ((QueryConstantSelector) legacy.getFunctionSelector().getInnerSelector().get(1)).getConstant()); + } + } diff --git a/test/prerna/util/sql/MicrosoftSqlServerQueryUtilUnitTests.java b/test/prerna/util/sql/MicrosoftSqlServerQueryUtilUnitTests.java index ca99ae568cc..6c41caa2aff 100644 --- a/test/prerna/util/sql/MicrosoftSqlServerQueryUtilUnitTests.java +++ b/test/prerna/util/sql/MicrosoftSqlServerQueryUtilUnitTests.java @@ -35,8 +35,13 @@ import java.sql.PreparedStatement; import java.sql.Types; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import prerna.query.querystruct.filters.SimpleQueryFilter; +import prerna.query.querystruct.selectors.QueryConstantSelector; +import prerna.query.querystruct.selectors.QueryFunctionSelector; + class MicrosoftSqlServerQueryUtilUnitTests { @Test @@ -57,4 +62,15 @@ void maxColumnBindingsPreserveExactTextBytesAndTypedNulls() throws Exception { assertTrue(util.getClobDataTypeName().toUpperCase().contains("VARCHAR")); assertTrue(util.getBlobDataTypeName().toUpperCase().contains("VARBINARY")); } + + @Test + void preparedSearchBindsPatindexPatternWithoutSqlEscaping() { + var util = new MicrosoftSqlServerQueryUtil(); + var filter = (SimpleQueryFilter) util.getPreparedSearchRegexFilter("ITEMS__V", "O'Brien\\path"); + var function = (QueryFunctionSelector) filter.getLComparison().getValue(); + Assertions.assertEquals("%O'Brien\\path%", + ((QueryConstantSelector) function.getInnerSelector().getFirst()).getConstant()); + Assertions.assertEquals("PATINDEX", function.getFunction()); + } + } diff --git a/test/prerna/util/sql/PostgresJdbcUnitTests.java b/test/prerna/util/sql/PostgresJdbcUnitTests.java index 9187c40b904..656ea0f2ed8 100644 --- a/test/prerna/util/sql/PostgresJdbcUnitTests.java +++ b/test/prerna/util/sql/PostgresJdbcUnitTests.java @@ -40,6 +40,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.List; @@ -51,6 +52,7 @@ import org.testcontainers.containers.wait.strategy.Wait; import prerna.engine.api.IRDBMSEngine; +import prerna.query.interpreters.sql.ParameterizedSqlInterpreterUnitTests; import prerna.util.QueryExecutionUtility; class PostgresJdbcUnitTests { @@ -71,6 +73,9 @@ void disposablePostgresRoundTripsAndEndsManualReadTransactions() throws Exceptio connection.setAutoCommit(false); IRDBMSEngine engine = mock(IRDBMSEngine.class); when(engine.getConnection()).thenReturn(connection); + when(engine.isBasic()).thenReturn(true); + when(engine.getQueryUtil()).thenReturn(new PostgresQueryUtil()); + when(engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); String insert = "INSERT INTO STATEMENT_VALUES VALUES (?, ?)"; assertEquals(1, QueryExecutionUtility.executeUpdate(engine, insert, ps -> { ps.setInt(1, 1); @@ -92,6 +97,7 @@ void disposablePostgresRoundTripsAndEndsManualReadTransactions() throws Exceptio int count = QueryExecutionUtility.queryOne(engine, "SELECT COUNT(*) FROM BIND_VALUES", ps -> { }, rs -> rs.getInt(1)); assertEquals(5, count); + ParameterizedSqlInterpreterUnitTests.roundTrip(engine); assertEquals(TransactionState.IDLE, connection.unwrap(BaseConnection.class).getTransactionState()); assertFalse(connection.getAutoCommit()); try (Connection observer = DriverManager.getConnection(connection.getMetaData().getURL(), "postgres", diff --git a/test/prerna/util/sql/PostgresQueryUtilUnitTests.java b/test/prerna/util/sql/PostgresQueryUtilUnitTests.java index 2e6fcf72876..0d6792bbd84 100644 --- a/test/prerna/util/sql/PostgresQueryUtilUnitTests.java +++ b/test/prerna/util/sql/PostgresQueryUtilUnitTests.java @@ -32,8 +32,11 @@ import java.sql.PreparedStatement; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import prerna.query.querystruct.filters.SimpleQueryFilter; + class PostgresQueryUtilUnitTests { @Test @@ -46,4 +49,13 @@ void postgresUsesTextAndBytesForNonNullValues() throws Exception { verify(ps).setString(1, " text "); verify(ps).setBytes(2, bytes); } + + @Test + void preparedRegexPreservesPatternSyntaxAndUsesBoundComparison() { + var util = new PostgresQueryUtil(); + var filter = (SimpleQueryFilter) util.getPreparedSearchRegexFilter("ITEMS__V", "O'Brien\\path"); + Assertions.assertEquals("o'brien\\path", filter.getRComparison().getValue()); + Assertions.assertEquals("~", filter.getComparator()); + } + } diff --git a/test/prerna/util/sql/SqlServerParameterizedQueryUnitTests.java b/test/prerna/util/sql/SqlServerParameterizedQueryUnitTests.java new file mode 100644 index 00000000000..ca3ef1ba05a --- /dev/null +++ b/test/prerna/util/sql/SqlServerParameterizedQueryUnitTests.java @@ -0,0 +1,78 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util.sql; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.time.Duration; +import java.time.ZoneOffset; + +import org.junit.jupiter.api.Test; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import com.microsoft.sqlserver.jdbc.SQLServerDataSource; + +import prerna.engine.api.IRDBMSEngine; +import prerna.query.interpreters.sql.ParameterizedSqlInterpreterUnitTests; + +class SqlServerParameterizedQueryUnitTests { + + @Test + void disposableSqlServerExecutesCompiledQueriesAndReleasesReadTransactions() throws Exception { + assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Disposable SQL Server requires Docker"); + String password = "Local-Jdbc-Test-42!"; + try (var server = new GenericContainer<>("mcr.microsoft.com/mssql/server:2022-latest") + .withEnv("ACCEPT_EULA", "Y").withEnv("MSSQL_PID", "Developer").withEnv("MSSQL_SA_PASSWORD", password) + .withExposedPorts(1433).withCreateContainerCmdModifier(command -> command.withPlatform("linux/amd64")) + .waitingFor(Wait.forLogMessage(".*SQL Server is now ready for client connections.*", 1)) + .withStartupTimeout(Duration.ofMinutes(3))) { + server.start(); + var dataSource = new SQLServerDataSource(); + dataSource.setURL("jdbc:sqlserver://" + server.getHost() + ":" + server.getMappedPort(1433) + + ";encrypt=true;trustServerCertificate=true"); + dataSource.setLoginTimeout(20); + dataSource.setSocketTimeout(30_000); + try (var connection = dataSource.getConnection("sa", password)) { + IRDBMSEngine engine = mock(IRDBMSEngine.class); + when(engine.getConnection()).thenReturn(connection); + when(engine.isBasic()).thenReturn(true); + when(engine.getQueryUtil()).thenReturn(new MicrosoftSqlServerQueryUtil()); + when(engine.getDatabaseZoneId()).thenReturn(ZoneOffset.UTC); + ParameterizedSqlInterpreterUnitTests.roundTrip(engine); + assertTrue(connection.getAutoCommit()); + assertFalse(connection.isClosed()); + } + } + } +}