From dc46b13cf4404d34ea8b2178de64fbb93f82ff9c Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Fri, 25 Sep 2026 22:02:14 -0400 Subject: [PATCH 1/7] feat(automation): prototype bounded loop containers --- .../automation-loop-phase-1-2026-09-25.md | 63 ++++++ .../automation/AddAutomationStepReactor.java | 4 + .../automation/AutomationConstants.java | 19 ++ .../automation/AutomationDatabaseUtility.java | 147 +++++++++++++- .../AutomationDefinitionService.java | 33 ++-- .../AutomationDefinitionValidator.java | 159 +++++++++++++++- .../automation/AutomationNodeCatalog.java | 19 ++ .../automation/AutomationNodeType.java | 1 + .../automation/AutomationProjectUtils.java | 2 +- .../AutomationRunExecutionService.java | 179 ++++++++++++++++++ .../reactor/automation/AutomationRuntime.java | 77 +++++++- .../automation/AutomationSourceRenderer.java | 4 +- .../scheduler/SchedulerOwlCreator.java | 3 + ...utomationDefinitionValidatorUnitTests.java | 63 ++++++ .../AutomationNodeCatalogUnitTests.java | 14 ++ ...utomationRunExecutionServiceUnitTests.java | 43 +++++ .../AutomationRuntimeUnitTests.java | 20 ++ 17 files changed, 819 insertions(+), 31 deletions(-) create mode 100644 outputs/review/automation-loop-phase-1-2026-09-25.md diff --git a/outputs/review/automation-loop-phase-1-2026-09-25.md b/outputs/review/automation-loop-phase-1-2026-09-25.md new file mode 100644 index 0000000000..6c81c68486 --- /dev/null +++ b/outputs/review/automation-loop-phase-1-2026-09-25.md @@ -0,0 +1,63 @@ +# Automation loop phase 1 + +## Status + +This work is parked for later review on dedicated backend and frontend branches. It is not intended to ship until the execution contract and UX are validated with representative automations. + +## Current approach + +- A loop is a first-class `control.loop` container in the parent automation graph. +- The parent graph stays acyclic. Iteration is owned by the Java execution service rather than represented with a circular control edge. +- The loop owns a nested acyclic body graph with ordinary automation nodes and control edges. +- The body has one entry node and may contain decisions and multiple routes. +- Phase 1 executes iterations sequentially with explicit maximum-iteration and body-size limits. +- Each iteration receives an isolated copy of the parent scope plus a loop context containing its index, total, first/last flags, current item when singular, and current batch. +- Body executions receive durable child history rows tied to the loop node and iteration index. +- Cancellation is checked between iterations and between body nodes. +- Nested loops and durable agent input waits inside a loop are intentionally unsupported in phase 1. + +## Current authoring UX + +- The loop appears as a distinct container card on the main automation canvas. +- Its collapsed state summarizes the repeated steps. +- Its expanded state contains a nested mini-canvas for the loop body. +- Users can insert steps at a route, edit body nodes, and use decisions inside the body. +- Expanding the loop increases its visual width and updates the surrounding React Flow node internals. + +## Deliberately unresolved + +### Input consumption + +Batching must not become the only loop contract. The current implementation supports `batchSize`, but this should be treated as an optional execution strategy while the following are tested: + +1. one item per iteration; +2. fixed-size batches; +3. a server-owned cursor or paged data reference; +4. streaming/provider-backed iteration for large tabular results. + +The final contract should let the user express the intent to repeat work without requiring them to understand frames, cursors, or backend storage. + +### Additional loop types + +The container/body design is intended to support later modes without circular parent graphs: + +- `forEach` for a collection; +- counted `for` ranges; +- bounded `while` conditions; +- paged or cursor-driven data iteration. + +Every mode must remain bounded, cancellable, observable, and resumable before it is enabled. + +## Before resuming implementation + +- Decide the canonical loop-input contract after testing lists, database results, and large provider-backed data. +- Define retry, resume, and partial-failure behavior at the iteration level. +- Confirm whether collected results are inline, summarized, or stored as run-owned data references. +- Validate branch convergence and output availability inside the nested body. +- Improve expanded-container layout so neighboring parent nodes never overlap. +- Test history rendering for parent loop rows and per-iteration child rows. +- Add end-to-end coverage for cancellation, limits, branching, and empty inputs. + +## Scope boundaries + +The parked work does not introduce circular parent-graph edges, parallel iteration, nested loops, agent approval waits, or a requirement that all loops consume batches. diff --git a/src/prerna/reactor/automation/AddAutomationStepReactor.java b/src/prerna/reactor/automation/AddAutomationStepReactor.java index c8da531468..9f15e38be8 100644 --- a/src/prerna/reactor/automation/AddAutomationStepReactor.java +++ b/src/prerna/reactor/automation/AddAutomationStepReactor.java @@ -117,6 +117,10 @@ private NounMetadata addStep(String projectId, AutomationDefinitionService.Defin customSource == null ? AutomationConstants.NODE_CODE_MODE_GENERATED : AutomationConstants.NODE_CODE_MODE_CUSTOM); node.put(AutomationConstants.NODE_FIELD_CONFIG, config); + if (AutomationConstants.NODE_CONTROL_LOOP.equals(nodeType)) { + node.put(AutomationConstants.NODE_FIELD_BODY, + Map.of(AutomationConstants.DOC_NODES, List.of(), AutomationConstants.DOC_EDGES, List.of())); + } node.put("position", Map.of("x", 240, "y", 80 + nodes.size() * 180)); List> updatedNodes = new ArrayList<>(nodes); diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index bb1744ffb6..22ced1655b 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -102,6 +102,12 @@ private AutomationConstants() { public static final String WORKSPACE_ID = "WORKSPACE_ID"; public static final String MODEL_MESSAGE_ID = "MODEL_MESSAGE_ID"; public static final String AGENT_RUN_ID = "AGENT_RUN_ID"; + /** Owning loop node for a dynamically materialized loop-body execution row. */ + public static final String PARENT_NODE_ID = "PARENT_NODE_ID"; + /** Zero-based loop iteration for a dynamically materialized body execution row. */ + public static final String ITERATION_INDEX = "ITERATION_INDEX"; + /** Canonical graph node represented by a dynamic execution row. */ + public static final String SOURCE_NODE_ID = "SOURCE_NODE_ID"; // -- AUTOMATION_RUN_WAITS columns --------------------------------------------- @@ -170,6 +176,7 @@ private AutomationConstants() { public static final String NODE_CONTROL_WAIT = "control.wait"; public static final String NODE_CONTROL_IF = "control.if"; public static final String NODE_CONTROL_JEV = "control.jev"; + public static final String NODE_CONTROL_LOOP = "control.loop"; public static final String NODE_DEVELOPER_PYTHON = "developer.python"; // -- Node config keys (node.config map fields, shared across executors) @@ -200,6 +207,11 @@ private AutomationConstants() { public static final String CONFIG_MAX_TURNS = "maxTurns"; public static final String CONFIG_MAX_REFLECTIONS = "maxReflections"; public static final String CONFIG_WAIT = "wait"; + public static final String CONFIG_LOOP_MODE = "mode"; + public static final String CONFIG_LOOP_ITEMS = "items"; + public static final String CONFIG_LOOP_BATCH_SIZE = "batchSize"; + public static final String CONFIG_LOOP_MAX_ITERATIONS = "maxIterations"; + public static final String LOOP_MODE_FOR_EACH = "forEach"; public static final String CONFIG_WAIT_TIMEOUT_MS = "waitTimeoutMs"; public static final String CONFIG_APP_ID = "appId"; @@ -215,6 +227,11 @@ private AutomationConstants() { public static final int DEFAULT_LIST_RUNS_LIMIT = 25; public static final int WAIT_MIN_SECONDS = 0; public static final int WAIT_MAX_SECONDS = 3600; + public static final int LOOP_MIN_BATCH_SIZE = 1; + public static final int LOOP_MAX_BATCH_SIZE = 1_000; + public static final int LOOP_DEFAULT_MAX_ITERATIONS = 1_000; + public static final int LOOP_MAX_ITERATIONS = 10_000; + public static final int LOOP_MAX_BODY_NODES = 1_000; public static final int NODE_SOURCE_MAX_BYTES = 100_000; public static final int NODE_OUTPUT_MAX_BYTES = 5 * 1024 * 1024; public static final int RUN_INPUTS_MAX_BYTES = 5 * 1024 * 1024; @@ -245,6 +262,8 @@ private AutomationConstants() { public static final String NODE_FIELD_TYPE = "type"; public static final String NODE_FIELD_LABEL = "label"; public static final String NODE_FIELD_CONFIG = "config"; + /** Nested acyclic graph owned by a container node such as {@code control.loop}. */ + public static final String NODE_FIELD_BODY = "body"; public static final String NODE_FIELD_OUTPUT_VAR = "outputVar"; public static final String EDGE_FIELD_SOURCE = "source"; public static final String EDGE_FIELD_TARGET = "target"; diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index 7cd5a0eebe..48bb1f2728 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -139,6 +139,8 @@ public final class AutomationDatabaseUtility { private static final Logger classLogger = LogManager.getLogger(AutomationDatabaseUtility.class); + /** Keeps dynamically materialized body rows after the stable parent graph rows. */ + private static final int LOOP_EXECUTION_ORDER_OFFSET = 1_000_000; /** Prefix identifying the automation tables inside the scheduler OWL schema. */ private static final String AUTOMATION_TABLE_PREFIX = "AUTOMATION_"; @@ -234,6 +236,19 @@ private AutomationDatabaseUtility() { (RUN_ID, NODE_ID, NODE_LABEL, EXECUTION_ORDER, STATUS, ROOM_ID, WORKSPACE_ID) \ VALUES (?, ?, ?, ?, ?, ?, ?)"""; + private static final String INSERT_LOOP_NODE_OUTPUT = """ + INSERT INTO AUTOMATION_NODE_OUTPUTS \ + (RUN_ID, NODE_ID, NODE_LABEL, EXECUTION_ORDER, STATUS, ROOM_ID, WORKSPACE_ID, \ + PARENT_NODE_ID, ITERATION_INDEX, SOURCE_NODE_ID) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""; + + private static final String INCREMENT_RUN_TOTAL_NODES = + "UPDATE AUTOMATION_RUNS SET TOTAL_NODES = TOTAL_NODES + ? WHERE RUN_ID = ?"; + + private static final String SKIP_PENDING_NODE_OUTPUT = """ + UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, ERROR_MESSAGE = ? \ + WHERE RUN_ID = ? AND NODE_ID = ? AND STATUS = ?"""; + private static final String UPDATE_NODE_OUTPUT_SUCCESS = """ UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ?, COMPLETED_AT = ?, \ DURATION_MS = ?, OUTPUT_VAR_NAME = ?, OUTPUT_VALUE = ?, OUTPUT_PREVIEW = ?, \ @@ -771,6 +786,64 @@ private static void insertAllNodeOutputs(Connection conn, String runId, List insertLoopIterationNodes(String runId, String loopNodeId, int iterationIndex, + List> orderedNodes, Map traceRoomIds) { + IRDBMSEngine schedulerDb = requireSchedulerDb("initialize an automation loop iteration"); + Map executionNodeIds = new LinkedHashMap<>(); + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement total = conn.prepareStatement(INCREMENT_RUN_TOTAL_NODES); + PreparedStatement output = conn.prepareStatement(INSERT_LOOP_NODE_OUTPUT)) { + total.setInt(1, orderedNodes.size()); + total.setString(2, runId); + requireSingleRow(total.executeUpdate(), "extend the loop run node count", runId, loopNodeId); + for (int index = 0; index < orderedNodes.size(); index++) { + Map node = orderedNodes.get(index); + String sourceNodeId = (String) node.get(NODE_FIELD_ID); + String executionNodeId = UUID.randomUUID().toString(); + executionNodeIds.put(sourceNodeId, executionNodeId); + int parameter = 1; + output.setString(parameter++, runId); + output.setString(parameter++, executionNodeId); + output.setString(parameter++, (String) node.get(NODE_FIELD_LABEL)); + output.setInt(parameter++, LOOP_EXECUTION_ORDER_OFFSET + + iterationIndex * Math.max(1, orderedNodes.size()) + index); + output.setString(parameter++, NODE_STATUS_PENDING); + setNullableString(output, parameter++, traceRoomIds == null ? null : traceRoomIds.get(sourceNodeId)); + setNullableString(output, parameter++, AutomationRuntime.configuredAgentWorkspaceId(node)); + output.setString(parameter++, loopNodeId); + output.setInt(parameter++, iterationIndex); + output.setString(parameter++, sourceNodeId); + output.addBatch(); + } + output.executeBatch(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return executionNodeIds; + } catch (Exception e) { + rollback(conn, e); + throw new IllegalStateException("Unable to initialize the automation loop iteration.", e); + } finally { + closeConnection(schedulerDb, conn); + } + } + /** * Sets the cluster-safe cancellation flag on a run. Called by * {@code CancelAutomationRunReactor} regardless of which pod receives the @@ -1166,6 +1239,43 @@ public static void skipPendingNodes(String runId, String reason) { } } + /** + * Marks only the unselected nodes in one materialized loop iteration skipped. + * + * @param runId owning run + * @param nodeIds runtime node IDs that remain pending + * @param reason persisted skip explanation + */ + public static void skipPendingNodes(String runId, List nodeIds, String reason) { + if (nodeIds == null || nodeIds.isEmpty()) { + return; + } + IRDBMSEngine schedulerDb = requireSchedulerDb("persist skipped automation loop nodes"); + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(SKIP_PENDING_NODE_OUTPUT)) { + for (String nodeId : nodeIds) { + ps.setString(1, NODE_STATUS_SKIPPED); + setNullableString(ps, 2, reason); + ps.setString(3, runId); + ps.setString(4, nodeId); + ps.setString(5, NODE_STATUS_PENDING); + ps.addBatch(); + } + ps.executeBatch(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + } catch (SQLException e) { + rollback(conn, e); + throw new IllegalStateException("Unable to persist skipped automation loop nodes.", e); + } finally { + closeConnection(schedulerDb, conn); + } + } + /** * Updates a node output after successful execution. */ @@ -1333,6 +1443,12 @@ public static List> getNodeOutputsForRun(String runId) { qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + WORKSPACE_ID, WORKSPACE_ID)); qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + MODEL_MESSAGE_ID, MODEL_MESSAGE_ID)); qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + AGENT_RUN_ID, AGENT_RUN_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + AutomationConstants.PARENT_NODE_ID, + AutomationConstants.PARENT_NODE_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + AutomationConstants.ITERATION_INDEX, + AutomationConstants.ITERATION_INDEX)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + AutomationConstants.SOURCE_NODE_ID, + AutomationConstants.SOURCE_NODE_ID)); qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + ERROR_MESSAGE, ERROR_MESSAGE)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter(TABLE_NODE_OUTPUTS + "__" + RUN_ID, "==", runId, @@ -1402,12 +1518,17 @@ public static boolean hasAgentRunTrace(String projectId, String automationRunId, */ public static List> buildNodeResults(List> nodeOutputs) { List> nodeResults = new ArrayList<>(); + Map> parentResults = new LinkedHashMap<>(); + Map>>> loopIterations = new LinkedHashMap<>(); if (nodeOutputs == null) { return nodeResults; } for (Map output : nodeOutputs) { - Map nodeResult = new HashMap<>(); - nodeResult.put(AutomationConstants.NODE_ID, output.get(AutomationConstants.NODE_ID)); + Map nodeResult = new LinkedHashMap<>(); + Object parentNodeId = output.get(AutomationConstants.PARENT_NODE_ID); + Object sourceNodeId = output.get(AutomationConstants.SOURCE_NODE_ID); + nodeResult.put(AutomationConstants.NODE_ID, + sourceNodeId == null ? output.get(AutomationConstants.NODE_ID) : sourceNodeId); nodeResult.put(AutomationConstants.NODE_LABEL, output.get(AutomationConstants.NODE_LABEL)); nodeResult.put(AutomationConstants.STATUS, output.get(AutomationConstants.STATUS)); nodeResult.put(AutomationConstants.DURATION_MS, output.get(AutomationConstants.DURATION_MS)); @@ -1432,7 +1553,27 @@ public static List> buildNodeResults(List new LinkedHashMap<>()) + .computeIfAbsent(iteration, ignored -> new ArrayList<>()).add(nodeResult); + } + } + for (Map.Entry>>> parent : loopIterations.entrySet()) { + Map parentResult = parentResults.get(parent.getKey()); + if (parentResult == null) { + continue; + } + List> iterations = new ArrayList<>(); + for (Map.Entry>> iteration : parent.getValue().entrySet()) { + iterations.add(Map.of("index", iteration.getKey(), "nodeResults", iteration.getValue())); + } + parentResult.put("iterations", iterations); } return nodeResults; } diff --git a/src/prerna/reactor/automation/AutomationDefinitionService.java b/src/prerna/reactor/automation/AutomationDefinitionService.java index 19482f91c2..4dce906656 100644 --- a/src/prerna/reactor/automation/AutomationDefinitionService.java +++ b/src/prerna/reactor/automation/AutomationDefinitionService.java @@ -105,8 +105,8 @@ private static DefinitionFiles loadFromFolder(Path folder) { .parseAndValidateForAuthoring(definition); validateUniqueNodeSourceFileNames(validated); Map sources = new LinkedHashMap<>(); - for (Map node : validated.nodes()) { - if (isJavaOwnedRoutingNode(node)) { + for (Map node : AutomationRuntime.allNodes(validated)) { + if (isJavaOwnedNode(node)) { continue; } String nodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); @@ -238,7 +238,7 @@ public static List getArtifactPaths(String projectId) { .parseAndValidateForAuthoring(definition.definition()); List paths = new ArrayList<>(); paths.add(definitionPath(assetsFolder)); - for (Map node : validated.nodes()) { + for (Map node : AutomationRuntime.allNodes(validated)) { if (requiresPythonSource(node)) { paths.add(nodeSourcePath(assetsFolder, node)); } @@ -262,7 +262,7 @@ private static Map defaultNodeSources( private static Map withoutTriggerSources(Map sources, AutomationDefinitionValidator.ValidatedDefinition definition) { Map result = new LinkedHashMap<>(sources); - for (Map node : definition.nodes()) { + for (Map node : AutomationRuntime.allNodes(definition)) { if (!requiresPythonSource(node)) { result.remove(node.get(AutomationConstants.NODE_FIELD_ID)); } @@ -274,7 +274,7 @@ private static Map validateAndCompleteNodeSources( AutomationDefinitionValidator.ValidatedDefinition definition, Map nodeSources) { Map supplied = nodeSources == null ? Map.of() : nodeSources; Map> nodesById = new LinkedHashMap<>(); - for (Map node : definition.nodes()) { + for (Map node : AutomationRuntime.allNodes(definition)) { String id = (String) node.get(AutomationConstants.NODE_FIELD_ID); nodesById.put(id, node); } @@ -282,15 +282,15 @@ private static Map validateAndCompleteNodeSources( if (!nodesById.containsKey(entry.getKey())) { throw new IllegalArgumentException("Python source was supplied for an unknown node: " + entry.getKey()); } - if (isJavaOwnedRoutingNode(nodesById.get(entry.getKey()))) { + if (isJavaOwnedNode(nodesById.get(entry.getKey()))) { throw new IllegalArgumentException( - "Routing node '" + entry.getKey() + "' is evaluated by Java and cannot have Python source."); + "Control node '" + entry.getKey() + "' is evaluated by Java and cannot have Python source."); } validateNodeSource(entry.getKey(), nodesById.get(entry.getKey()), entry.getValue()); } Map result = new LinkedHashMap<>(); for (Map.Entry> entry : nodesById.entrySet()) { - if (isJavaOwnedRoutingNode(entry.getValue())) { + if (isJavaOwnedNode(entry.getValue())) { continue; } String source = supplied.get(entry.getKey()); @@ -350,9 +350,9 @@ private static String normalizeGeneratedCodeModes(AutomationDefinitionValidator. boolean changed = false; changed |= definition.definition().remove(AutomationConstants.DOC_NODE_SOURCES) != null; changed |= definition.definition().remove(AutomationConstants.DOC_GLOBALS) != null; - for (Map node : definition.nodes()) { + for (Map node : AutomationRuntime.allNodes(definition)) { String nodeType = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); - if (isJavaOwnedRoutingNode(node)) { + if (isJavaOwnedNode(node)) { if (!AutomationConstants.NODE_CODE_MODE_GENERATED .equals(node.get(AutomationConstants.NODE_FIELD_CODE_MODE))) { node.put(AutomationConstants.NODE_FIELD_CODE_MODE, AutomationConstants.NODE_CODE_MODE_GENERATED); @@ -424,7 +424,7 @@ private static void writeAggregate(Path folder, DefinitionFiles files) throws IO validateUniqueNodeSourceFileNames(definition); Files.createDirectories(nodesFolder(folder)); writeReplace(definitionPath(folder), prettyJson(files.definition())); - for (Map node : definition.nodes()) { + for (Map node : AutomationRuntime.allNodes(definition)) { if (!requiresPythonSource(node)) { continue; } @@ -563,7 +563,7 @@ private static String slugify(String value) { private static void validateUniqueNodeSourceFileNames( AutomationDefinitionValidator.ValidatedDefinition definition) { Map nodeIdsByFileName = new LinkedHashMap<>(); - for (Map node : definition.nodes()) { + for (Map node : AutomationRuntime.allNodes(definition)) { if (!requiresPythonSource(node)) { continue; } @@ -579,7 +579,7 @@ private static void validateUniqueNodeSourceFileNames( private static boolean requiresPythonSource(Map node) { return !AutomationConstants.NODE_START.equals(node.get(AutomationConstants.NODE_FIELD_TYPE)) - && !isJavaOwnedRoutingNode(node); + && !isJavaOwnedNode(node); } /** @@ -587,12 +587,13 @@ private static boolean requiresPythonSource(Map node) { * runtime and therefore has no persisted Python source file. * * @param node automation node - * @return {@code true} for a Java-owned routing node + * @return {@code true} for a Java-owned control node */ - private static boolean isJavaOwnedRoutingNode(Map node) { + private static boolean isJavaOwnedNode(Map node) { Object nodeType = node.get(AutomationConstants.NODE_FIELD_TYPE); return AutomationConstants.NODE_CONTROL_IF.equals(nodeType) - || AutomationConstants.NODE_CONTROL_JEV.equals(nodeType); + || AutomationConstants.NODE_CONTROL_JEV.equals(nodeType) + || AutomationConstants.NODE_CONTROL_LOOP.equals(nodeType); } private static String emptyDefinition() { diff --git a/src/prerna/reactor/automation/AutomationDefinitionValidator.java b/src/prerna/reactor/automation/AutomationDefinitionValidator.java index 4564c7a23e..5fe4daeb66 100644 --- a/src/prerna/reactor/automation/AutomationDefinitionValidator.java +++ b/src/prerna/reactor/automation/AutomationDefinitionValidator.java @@ -125,10 +125,11 @@ private static ValidatedDefinition validate(Map definition, bool List> nodes = requireMapList(graph.get(AutomationConstants.DOC_NODES), "graph.nodes"); List> edges = requireMapList(graph.get(AutomationConstants.DOC_EDGES), "graph.edges"); - Map nodeTypes = validateNodes(nodes); + Map nodeTypes = validateNodes(nodes, true); Map> branchPorts = branchPorts(nodes); validateEdges(edges, nodeTypes, branchPorts); validateControlPath(edges, nodeTypes, branchPorts, requireExecutableGraph); + validateLoopBodies(nodes, nodeTypes.keySet(), requireExecutableGraph); validateTriggerBindings(definition.get(AutomationConstants.DOC_TRIGGER_BINDINGS)); String snapshot = AutomationRuntimeUtils.GSON.toJson(canonicalize(definition)); @@ -143,7 +144,7 @@ private static void validateVersion(Object value) { } } - private static Map validateNodes(List> nodes) { + private static Map validateNodes(List> nodes, boolean requireStart) { Map nodeTypes = new LinkedHashMap<>(); Set outputVariables = new HashSet<>(); int startCount = 0; @@ -195,10 +196,10 @@ private static Map validateNodes(List> nodes && !AutomationConstants.NODE_CODE_MODE_CUSTOM.equals(codeMode)) { throw new IllegalArgumentException("Node '" + nodeId + "' has unsupported codeMode: " + codeMode + "."); } - if (isRoutingNode(typedNode) + if (isJavaOwnedNode(typedNode) && !AutomationConstants.NODE_CODE_MODE_GENERATED.equals(codeMode)) { throw new IllegalArgumentException( - "Routing node '" + nodeId + "' must use generated mode; routing is evaluated by Java."); + "Control node '" + nodeId + "' must use generated mode; it is evaluated by Java."); } if (AutomationConstants.NODE_CODE_MODE_CUSTOM.equals(codeMode) && !(config instanceof Map)) { throw new IllegalArgumentException("Custom node '" + nodeId + "' must declare a config object."); @@ -208,9 +209,12 @@ private static Map validateNodes(List> nodes validateGeneratedAppPixel(nodeId, nodeType, nodeConfig); } } - if (startCount != 1) { + if (requireStart && startCount != 1) { throw new IllegalArgumentException("Python automation definition must contain exactly one start node."); } + if (!requireStart && startCount != 0) { + throw new IllegalArgumentException("A loop body cannot contain a trigger.start node."); + } return nodeTypes; } @@ -269,6 +273,7 @@ private static void validateNodeConfig(String nodeId, AutomationNodeType nodeTyp case CONTROL_WAIT -> validateWaitConfig(nodeId, config); case CONTROL_IF -> validateBranchConfig(nodeId, config); case CONTROL_JEV -> validateJevBranchConfig(nodeId, config); + case CONTROL_LOOP -> validateLoopConfig(nodeId, config); case STORAGE_LIST, TRIGGER_START, DEVELOPER_PYTHON -> { // These node types have no additional required configuration here. } @@ -279,6 +284,41 @@ private static boolean isRoutingNode(AutomationNodeType nodeType) { return nodeType == AutomationNodeType.CONTROL_IF || nodeType == AutomationNodeType.CONTROL_JEV; } + private static boolean isJavaOwnedNode(AutomationNodeType nodeType) { + return isRoutingNode(nodeType) || nodeType == AutomationNodeType.CONTROL_LOOP; + } + + private static void validateLoopConfig(String nodeId, Map config) { + Object mode = config.getOrDefault(AutomationConstants.CONFIG_LOOP_MODE, + AutomationConstants.LOOP_MODE_FOR_EACH); + if (!AutomationConstants.LOOP_MODE_FOR_EACH.equals(mode)) { + throw new IllegalArgumentException("Loop node '" + nodeId + "' config.mode must be '" + + AutomationConstants.LOOP_MODE_FOR_EACH + "'."); + } + Object items = config.get(AutomationConstants.CONFIG_LOOP_ITEMS); + boolean scopeReference = items instanceof String value + && value.trim().matches("\\$\\{[A-Za-z_][A-Za-z0-9_]*}"); + if (!(items instanceof List) && !scopeReference) { + throw new IllegalArgumentException("Loop node '" + nodeId + + "' config.items must be an array or an exact ${scope_value} reference."); + } + validateBoundedInteger(nodeId, config, AutomationConstants.CONFIG_LOOP_BATCH_SIZE, + AutomationConstants.LOOP_MIN_BATCH_SIZE, AutomationConstants.LOOP_MAX_BATCH_SIZE); + validateBoundedInteger(nodeId, config, AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 1, + AutomationConstants.LOOP_MAX_ITERATIONS); + } + + private static void validateBoundedInteger(String nodeId, Map config, String key, int minimum, + int maximum) { + Object value = config.get(key); + if (!(value instanceof Number number) || !Double.isFinite(number.doubleValue()) + || number.doubleValue() != Math.rint(number.doubleValue()) || number.intValue() < minimum + || number.intValue() > maximum) { + throw new IllegalArgumentException("Node '" + nodeId + "' config." + key + + " must be a whole number from " + minimum + " through " + maximum + "."); + } + } + /** * Require generated database reads to declare a finite integer row bound. The * limit is enforced again by the server-owned {@code SqlQuery} execution path. @@ -740,6 +780,115 @@ private static void validateControlPath(List> edges, Map> nodes, Set parentNodeIds, + boolean requireExecutableGraph) { + Set allNodeIds = new HashSet<>(parentNodeIds); + Set parentOutputVariables = new HashSet<>(); + for (Map node : nodes) { + Object outputVariable = node.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); + if (outputVariable instanceof String value) { + parentOutputVariables.add(value); + } + } + for (Map node : nodes) { + if (!AutomationConstants.NODE_CONTROL_LOOP.equals(node.get(AutomationConstants.NODE_FIELD_TYPE))) { + continue; + } + String loopNodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); + Object rawBody = node.get(AutomationConstants.NODE_FIELD_BODY); + if (rawBody == null && !requireExecutableGraph) { + continue; + } + Map body = requireMap(rawBody, "Loop node '" + loopNodeId + "' body"); + List> bodyNodes = requireMapList(body.get(AutomationConstants.DOC_NODES), + "Loop node '" + loopNodeId + "' body.nodes"); + List> bodyEdges = requireMapList(body.get(AutomationConstants.DOC_EDGES), + "Loop node '" + loopNodeId + "' body.edges"); + if (bodyNodes.isEmpty()) { + if (requireExecutableGraph) { + throw new IllegalArgumentException("Loop node '" + loopNodeId + "' requires at least one body node."); + } + continue; + } + if (bodyNodes.size() > AutomationConstants.LOOP_MAX_BODY_NODES) { + throw new IllegalArgumentException("Loop node '" + loopNodeId + "' body exceeds the maximum of " + + AutomationConstants.LOOP_MAX_BODY_NODES + " nodes."); + } + for (Map bodyNode : bodyNodes) { + String bodyType = String.valueOf(bodyNode.get(AutomationConstants.NODE_FIELD_TYPE)); + if (AutomationConstants.NODE_CONTROL_LOOP.equals(bodyType)) { + throw new IllegalArgumentException( + "Loop node '" + loopNodeId + "' cannot contain a nested loop in phase 1."); + } + if (AutomationConstants.NODE_AGENT_RUN.equals(bodyType)) { + throw new IllegalArgumentException("Loop node '" + loopNodeId + + "' cannot contain an agent.run node because durable input waits inside iterations are not supported."); + } + } + Map bodyNodeTypes = validateNodes(bodyNodes, false); + for (String bodyNodeId : bodyNodeTypes.keySet()) { + if (!allNodeIds.add(bodyNodeId)) { + throw new IllegalArgumentException( + "Automation definition has duplicate node id: " + bodyNodeId + "."); + } + } + for (Map bodyNode : bodyNodes) { + Object outputVariable = bodyNode.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); + if (outputVariable instanceof String value && parentOutputVariables.contains(value)) { + throw new IllegalArgumentException("Loop node '" + loopNodeId + "' body outputVar '" + value + + "' cannot shadow a parent graph outputVar."); + } + } + Map> bodyBranchPorts = branchPorts(bodyNodes); + validateEdges(bodyEdges, bodyNodeTypes, bodyBranchPorts); + validateLoopBodyControlPath(loopNodeId, bodyEdges, bodyNodeTypes, bodyBranchPorts, + requireExecutableGraph); + } + } + + private static void validateLoopBodyControlPath(String loopNodeId, List> edges, + Map nodeTypes, Map> branchPorts, boolean requireExecutableGraph) { + Map incoming = new LinkedHashMap<>(); + for (String nodeId : nodeTypes.keySet()) { + incoming.put(nodeId, 0); + } + for (Map edge : edges) { + if (AutomationConstants.EDGE_KIND_CONTROL.equals(edge.get(AutomationConstants.EDGE_FIELD_KIND))) { + incoming.compute((String) edge.get(AutomationConstants.EDGE_FIELD_TARGET), + (ignored, count) -> count + 1); + } + } + List roots = incoming.entrySet().stream().filter(entry -> entry.getValue() == 0).map(Map.Entry::getKey) + .toList(); + if (requireExecutableGraph && roots.size() != 1) { + throw new IllegalArgumentException( + "Loop node '" + loopNodeId + "' body must have exactly one entry node."); + } + + String virtualStart = "__loop_body_start__"; + while (nodeTypes.containsKey(virtualStart)) { + virtualStart += "_"; + } + Map augmentedTypes = new LinkedHashMap<>(); + augmentedTypes.put(virtualStart, AutomationConstants.NODE_START); + augmentedTypes.putAll(nodeTypes); + List> augmentedEdges = new ArrayList<>(edges); + if (requireExecutableGraph) { + augmentedEdges.add(Map.of(AutomationConstants.NODE_FIELD_ID, virtualStart + "-edge", + AutomationConstants.EDGE_FIELD_KIND, AutomationConstants.EDGE_KIND_CONTROL, + AutomationConstants.EDGE_FIELD_SOURCE, virtualStart, AutomationConstants.EDGE_FIELD_SOURCE_PORT, + AutomationConstants.CONTROL_PORT_OUT, AutomationConstants.EDGE_FIELD_TARGET, roots.get(0), + AutomationConstants.EDGE_FIELD_TARGET_PORT, AutomationConstants.CONTROL_PORT_IN)); + } + validateControlPath(augmentedEdges, augmentedTypes, branchPorts, requireExecutableGraph); + } + private static void validateTriggerBindings(Object value) { if (value == null) { return; diff --git a/src/prerna/reactor/automation/AutomationNodeCatalog.java b/src/prerna/reactor/automation/AutomationNodeCatalog.java index 28056e0b00..ed99f9f455 100644 --- a/src/prerna/reactor/automation/AutomationNodeCatalog.java +++ b/src/prerna/reactor/automation/AutomationNodeCatalog.java @@ -205,6 +205,25 @@ private static List createDefinitions() { controlInputs(), List.of(port("case:", "Route", PortKind.CONTROL, PortDirection.OUTPUT, null), port("else", "Fallback", PortKind.CONTROL, PortDirection.OUTPUT, null)))); + definitions.add(definition(AutomationNodeType.CONTROL_LOOP, "Loop over items", + "Run a nested sequence once for each item or batch in a list.", + orderedConfig("mode", AutomationConstants.LOOP_MODE_FOR_EACH, "items", List.of(), "batchSize", 1, + "maxIterations", AutomationConstants.LOOP_DEFAULT_MAX_ITERATIONS), + List.of(field("mode", ConfigFieldType.STRING, "Mode", true, + AutomationConstants.LOOP_MODE_FOR_EACH), + field("items", ConfigFieldType.JSON, "Items", true, List.of()), + boundedIntegerField("batchSize", "Batch size", 1, + AutomationConstants.LOOP_MIN_BATCH_SIZE, AutomationConstants.LOOP_MAX_BATCH_SIZE), + boundedIntegerField("maxIterations", "Maximum iterations", + AutomationConstants.LOOP_DEFAULT_MAX_ITERATIONS, 1, + AutomationConstants.LOOP_MAX_ITERATIONS)), + List.of(outputField("processed", OutputFieldType.NUMBER, "Processed", + "Number of items processed.", true), + outputField("iterations", OutputFieldType.NUMBER, "Iterations", + "Number of loop-body executions.", true), + outputField("results", OutputFieldType.OBJECT, "Results", + "Ordered outputs collected from each iteration.", true)), + controlInputs(), controlAndResultOutputs())); definitions.add(new AutomationNodeDefinition(AutomationNodeType.DEVELOPER_PYTHON, "Python", "Run custom Python for advanced transformations.", AutomationConstants.NODE_CODE_MODE_CUSTOM, Map.of(), List.of(), List.of(), controlInputs(), controlAndResultOutputs())); diff --git a/src/prerna/reactor/automation/AutomationNodeType.java b/src/prerna/reactor/automation/AutomationNodeType.java index 7f9fe5af37..5276a7f3b2 100644 --- a/src/prerna/reactor/automation/AutomationNodeType.java +++ b/src/prerna/reactor/automation/AutomationNodeType.java @@ -85,6 +85,7 @@ public enum AutomationNodeType { CONTROL_IF(AutomationConstants.NODE_CONTROL_IF, Category.CONTROL, null, Permission.NONE, false, false), CONTROL_JEV(AutomationConstants.NODE_CONTROL_JEV, Category.CONTROL, IEngine.CATALOG_TYPE.MODEL, Permission.VIEW, false, false), + CONTROL_LOOP(AutomationConstants.NODE_CONTROL_LOOP, Category.CONTROL, null, Permission.NONE, true, false), DEVELOPER_PYTHON(AutomationConstants.NODE_DEVELOPER_PYTHON, Category.DEVELOPER, null, Permission.NONE, true, true); /** diff --git a/src/prerna/reactor/automation/AutomationProjectUtils.java b/src/prerna/reactor/automation/AutomationProjectUtils.java index d60a31cb5a..91465f6f93 100644 --- a/src/prerna/reactor/automation/AutomationProjectUtils.java +++ b/src/prerna/reactor/automation/AutomationProjectUtils.java @@ -221,7 +221,7 @@ public static AutomationDefinitionService.DefinitionFiles createStarterDefinitio @SuppressWarnings("unchecked") public static void validateDefinitionReferences(AutomationDefinitionValidator.ValidatedDefinition definition, User user) { - for (Map node : definition.nodes()) { + for (Map node : AutomationRuntime.allNodes(definition)) { String nodeType = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); AutomationNodeType typedNode = AutomationNodeType.fromType(nodeType); Object rawConfig = node.get(AutomationConstants.NODE_FIELD_CONFIG); diff --git a/src/prerna/reactor/automation/AutomationRunExecutionService.java b/src/prerna/reactor/automation/AutomationRunExecutionService.java index 7b820e2f53..f974c94035 100644 --- a/src/prerna/reactor/automation/AutomationRunExecutionService.java +++ b/src/prerna/reactor/automation/AutomationRunExecutionService.java @@ -28,11 +28,14 @@ package prerna.reactor.automation; import java.io.File; +import java.lang.reflect.Array; import java.sql.Timestamp; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -238,6 +241,8 @@ private Map executeInControlOrder(Insight executionInsight, Stri nodeResult = executeConditionNode(runId, node, scope); } else if (AutomationConstants.NODE_CONTROL_JEV.equals(type)) { nodeResult = executeJevDecisionNode(executionInsight, runId, node, scope); + } else if (AutomationConstants.NODE_CONTROL_LOOP.equals(type)) { + nodeResult = executeLoopNode(executionInsight, projectId, runId, node, nodeSources, scope); } else { nodeResult = executeNodeSource(executionInsight, projectId, runId, node, nodeSources.get(nodeId), scope, traceRoomIds.get(nodeId), @@ -276,6 +281,174 @@ private Map executeInControlOrder(Insight executionInsight, Stri return result; } + /** + * Executes a bounded, sequential foreach container. The parent graph remains + * acyclic; each iteration materializes independent child history rows and runs + * the loop body's own validated acyclic control path against an isolated scope. + */ + @SuppressWarnings("unchecked") + private Map executeLoopNode(Insight executionInsight, String projectId, String runId, + Map loopNode, Map nodeSources, Map parentScope) { + String loopNodeId = (String) loopNode.get(AutomationConstants.NODE_FIELD_ID); + Timestamp started = Utility.getSqlTimestampUTC(LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC)); + long startedMs = System.currentTimeMillis(); + AutomationDatabaseUtility.markNodeRunning(runId, loopNodeId); + streamNodeProgress(runId, loopNode, AutomationConstants.NODE_STATUS_RUNNING, null, null, null); + try { + Map config = (Map) loopNode.get(AutomationConstants.NODE_FIELD_CONFIG); + List items = loopItems(config.get(AutomationConstants.CONFIG_LOOP_ITEMS), parentScope, loopNodeId); + int batchSize = ((Number) config.get(AutomationConstants.CONFIG_LOOP_BATCH_SIZE)).intValue(); + int maximumIterations = ((Number) config.get(AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS)).intValue(); + int iterationCount = items.isEmpty() ? 0 : (items.size() + batchSize - 1) / batchSize; + if (iterationCount > maximumIterations) { + throw new IllegalArgumentException("Loop node '" + loopNodeId + "' requires " + iterationCount + + " iterations, exceeding its maxIterations value of " + maximumIterations + "."); + } + + List> bodyNodes = AutomationRuntime.nodesForGraph( + AutomationRuntime.loopBodyNodes(loopNode), AutomationRuntime.loopBodyEdges(loopNode)); + List> bodyEdges = AutomationRuntime.loopBodyEdges(loopNode); + Map> bodyTargets = AutomationRuntime.controlTargets(bodyEdges); + String bodyEntry = AutomationRuntime.graphEntryNodeId(bodyNodes, bodyEdges); + String loopOutputVar = (String) loopNode.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); + List> iterationResults = new ArrayList<>(); + + for (int iteration = 0; iteration < iterationCount; iteration++) { + if (AutomationPythonRunRegistry.isCancellationRequested(runId)) { + throw new IllegalStateException("Run cancelled by user"); + } + int from = iteration * batchSize; + int to = Math.min(items.size(), from + batchSize); + List batch = new ArrayList<>(items.subList(from, to)); + Map loopContext = new LinkedHashMap<>(); + loopContext.put("index", iteration); + loopContext.put("batch", batch); + loopContext.put("total", iterationCount); + loopContext.put("isFirst", iteration == 0); + loopContext.put("isLast", iteration == iterationCount - 1); + if (batch.size() == 1) { + loopContext.put("item", batch.get(0)); + } + + Map iterationScope = new LinkedHashMap<>(parentScope); + iterationScope.put(loopOutputVar, loopContext); + Map bodyTraceRoomIds = allocateTraceRoomIds(bodyNodes); + Map executionNodeIds = AutomationDatabaseUtility.insertLoopIterationNodes(runId, + loopNodeId, iteration, bodyNodes, bodyTraceRoomIds); + Map> bodyNodesById = new LinkedHashMap<>(); + for (Map bodyNode : bodyNodes) { + bodyNodesById.put((String) bodyNode.get(AutomationConstants.NODE_FIELD_ID), bodyNode); + } + + Set visited = new HashSet<>(); + Map outputs = new LinkedHashMap<>(); + String current = bodyEntry; + while (current != null) { + if (AutomationPythonRunRegistry.isCancellationRequested(runId)) { + throw new IllegalStateException("Run cancelled by user"); + } + if (!visited.add(current)) { + throw new IllegalStateException("Loop body revisited node '" + current + "'."); + } + Map canonicalNode = bodyNodesById.get(current); + Map executionNode = new LinkedHashMap<>(canonicalNode); + executionNode.put(AutomationConstants.NODE_FIELD_ID, executionNodeIds.get(current)); + executionNode.put(AutomationConstants.SOURCE_NODE_ID, current); + executionNode.put(AutomationConstants.PARENT_NODE_ID, loopNodeId); + executionNode.put(AutomationConstants.ITERATION_INDEX, iteration); + String type = (String) canonicalNode.get(AutomationConstants.NODE_FIELD_TYPE); + Map nodeResult; + if (AutomationConstants.NODE_CONTROL_IF.equals(type)) { + nodeResult = executeConditionNode(runId, executionNode, iterationScope); + } else if (AutomationConstants.NODE_CONTROL_JEV.equals(type)) { + nodeResult = executeJevDecisionNode(executionInsight, runId, executionNode, iterationScope); + } else { + nodeResult = executeNodeSource(executionInsight, projectId, runId, executionNode, + nodeSources.get(current), iterationScope, bodyTraceRoomIds.get(current), null); + } + if (!AutomationConstants.NODE_STATUS_SUCCESS.equals(nodeResult.get(AutomationConstants.STATUS))) { + throw new IllegalStateException("Loop body node '" + current + "' did not complete successfully."); + } + String outputVar = (String) canonicalNode.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); + if (outputVar != null) { + Object value = nodeResult.get(AutomationConstants.RESULT_OUTPUT_VALUE); + iterationScope.put(outputVar, value); + outputs.put(outputVar, value); + } + String selectedPort = AutomationConstants.CONTROL_PORT_OUT; + if (AutomationConstants.NODE_CONTROL_IF.equals(type) + || AutomationConstants.NODE_CONTROL_JEV.equals(type)) { + Map decision = (Map) nodeResult + .get(AutomationConstants.RESULT_OUTPUT_VALUE); + selectedPort = (String) decision.get("branch"); + } + current = bodyTargets.getOrDefault(current, Map.of()).get(selectedPort); + } + List skipped = bodyNodesById.keySet().stream().filter(nodeId -> !visited.contains(nodeId)) + .map(executionNodeIds::get).toList(); + AutomationDatabaseUtility.skipPendingNodes(runId, skipped, "Loop branch was not selected"); + Map iterationResult = new LinkedHashMap<>(); + iterationResult.put("index", iteration); + iterationResult.put("outputs", outputs); + iterationResults.add(iterationResult); + AutomationRuntimeUtils.toBoundedRuntimeJson(iterationResults, + AutomationConstants.NODE_OUTPUT_MAX_BYTES, + "Automation loop '" + loopNodeId + "' collected results"); + } + + Map value = new LinkedHashMap<>(); + value.put("processed", items.size()); + value.put("iterations", iterationCount); + value.put("results", iterationResults); + String output = AutomationRuntimeUtils.toBoundedRuntimeJson(value, + AutomationConstants.NODE_OUTPUT_MAX_BYTES, "Automation loop '" + loopNodeId + "' output"); + Map prospectiveScope = new LinkedHashMap<>(parentScope); + prospectiveScope.put(loopOutputVar, value); + AutomationRuntimeUtils.toBoundedRuntimeJson(prospectiveScope, AutomationConstants.RUN_SCOPE_MAX_BYTES, + "Automation run scope"); + long duration = System.currentTimeMillis() - startedMs; + String preview = AutomationRuntimeUtils.generatePreview(output); + AutomationDatabaseUtility.updateNodeSuccess(runId, loopNodeId, started, duration, loopOutputVar, output, + preview, null, null); + AutomationPythonRunRegistry.nodeCompleted(runId); + streamNodeProgress(runId, loopNode, AutomationConstants.NODE_STATUS_SUCCESS, duration, preview, null); + return nodeResult(loopNodeId, AutomationConstants.NODE_STATUS_SUCCESS, value, null); + } catch (Exception e) { + long duration = System.currentTimeMillis() - startedMs; + String message = safeMessage(e); + AutomationDatabaseUtility.updateNodeFailed(runId, loopNodeId, started, duration, message); + streamNodeProgress(runId, loopNode, AutomationConstants.NODE_STATUS_FAILED, duration, null, message); + throw e instanceof RuntimeException runtimeException ? runtimeException : new RuntimeException(e); + } + } + + /** Resolves a loop's literal list or exact scope reference without string coercion. */ + static List loopItems(Object configuredItems, Map scope, String loopNodeId) { + Object resolved = configuredItems; + if (configuredItems instanceof String value) { + Matcher reference = EXACT_SCOPE_REFERENCE.matcher(value.trim()); + if (reference.matches()) { + String name = reference.group(1); + if (!scope.containsKey(name)) { + throw new IllegalArgumentException( + "Loop node '" + loopNodeId + "' references unavailable scope value: " + name); + } + resolved = scope.get(name); + } + } + if (resolved instanceof Collection collection) { + return new ArrayList<>(collection); + } + if (resolved != null && resolved.getClass().isArray()) { + List items = new ArrayList<>(Array.getLength(resolved)); + for (int index = 0; index < Array.getLength(resolved); index++) { + items.add(Array.get(resolved, index)); + } + return items; + } + throw new IllegalArgumentException("Loop node '" + loopNodeId + "' items must resolve to an array or list."); + } + /** * Evaluates one typed routing question through the configured TypeSafe/Jev * engine. The model may select only a configured route; responses below the @@ -1585,6 +1758,12 @@ private void streamNodeProgress(String runId, Map node, String s data.put(AutomationConstants.NODE_ID, node.get(AutomationConstants.NODE_FIELD_ID)); data.put(AutomationConstants.NODE_LABEL, node.get(AutomationConstants.NODE_FIELD_LABEL)); data.put(AutomationConstants.STATUS, status); + for (String field : List.of(AutomationConstants.SOURCE_NODE_ID, AutomationConstants.PARENT_NODE_ID, + AutomationConstants.ITERATION_INDEX)) { + if (node.get(field) != null) { + data.put(field, node.get(field)); + } + } if (durationMs != null) { data.put(AutomationConstants.DURATION_MS, durationMs); } diff --git a/src/prerna/reactor/automation/AutomationRuntime.java b/src/prerna/reactor/automation/AutomationRuntime.java index 0ca069bb12..c2eff6daf2 100644 --- a/src/prerna/reactor/automation/AutomationRuntime.java +++ b/src/prerna/reactor/automation/AutomationRuntime.java @@ -56,8 +56,14 @@ private AutomationRuntime() { } static List> nodesForRun(AutomationDefinitionValidator.ValidatedDefinition definition) { + return nodesForGraph(definition.nodes(), definition.edges()); + } + + /** Returns graph nodes in run-history order with runtime defaults applied. */ + static List> nodesForGraph(List> graphNodes, + List> graphEdges) { List> nodes = new ArrayList<>(); - for (Map original : controlOrderedNodes(definition)) { + for (Map original : controlOrderedNodes(graphNodes, graphEdges)) { Map node = new LinkedHashMap<>(original); node.putIfAbsent(AutomationConstants.NODE_FIELD_LABEL, node.get(AutomationConstants.NODE_FIELD_ID)); Object nodeType = node.get(AutomationConstants.NODE_FIELD_TYPE); @@ -72,6 +78,42 @@ static List> nodesForRun(AutomationDefinitionValidator.Valid return nodes; } + /** + * Returns canonical nodes from the parent graph and every loop body. Loop-body + * nodes remain ordinary source-owning nodes even though they are materialized + * into run history only when an iteration executes. + */ + static List> allNodes(AutomationDefinitionValidator.ValidatedDefinition definition) { + List> nodes = new ArrayList<>(); + for (Map node : definition.nodes()) { + nodes.add(node); + nodes.addAll(loopBodyNodes(node)); + } + return nodes; + } + + /** Returns the canonical nodes owned by one loop body. */ + @SuppressWarnings("unchecked") + static List> loopBodyNodes(Map loopNode) { + Object body = loopNode.get(AutomationConstants.NODE_FIELD_BODY); + if (!(body instanceof Map bodyMap) + || !(((Map) bodyMap).get(AutomationConstants.DOC_NODES) instanceof List nodes)) { + return List.of(); + } + return (List>) (List) nodes; + } + + /** Returns the canonical edges owned by one loop body. */ + @SuppressWarnings("unchecked") + static List> loopBodyEdges(Map loopNode) { + Object body = loopNode.get(AutomationConstants.NODE_FIELD_BODY); + if (!(body instanceof Map bodyMap) + || !(((Map) bodyMap).get(AutomationConstants.DOC_EDGES) instanceof List edges)) { + return List.of(); + } + return (List>) (List) edges; + } + private static String defaultOutputVariable(Map node) { String type = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); String id = (String) node.get(AutomationConstants.NODE_FIELD_ID); @@ -87,15 +129,21 @@ private static String defaultOutputVariable(Map node) { * remains Java-owned. */ static List> controlOrderedNodes(AutomationDefinitionValidator.ValidatedDefinition definition) { + return controlOrderedNodes(definition.nodes(), definition.edges()); + } + + /** Returns a deterministic topological order for an already validated graph. */ + static List> controlOrderedNodes(List> graphNodes, + List> graphEdges) { Map> nodes = new LinkedHashMap<>(); Map incoming = new LinkedHashMap<>(); - for (Map node : definition.nodes()) { + for (Map node : graphNodes) { String id = (String) node.get(AutomationConstants.NODE_FIELD_ID); nodes.put(id, node); incoming.put(id, 0); } Map> outgoing = new HashMap<>(); - for (Map edge : definition.edges()) { + for (Map edge : graphEdges) { if (!AutomationConstants.EDGE_KIND_CONTROL.equals(edge.get(AutomationConstants.EDGE_FIELD_KIND))) { continue; } @@ -139,8 +187,13 @@ static String startNodeId(AutomationDefinitionValidator.ValidatedDefinition defi static Map> controlTargets( AutomationDefinitionValidator.ValidatedDefinition definition) { + return controlTargets(definition.edges()); + } + + /** Indexes each control edge by source node and source port. */ + static Map> controlTargets(List> edges) { Map> targets = new LinkedHashMap<>(); - for (Map edge : definition.edges()) { + for (Map edge : edges) { if (!AutomationConstants.EDGE_KIND_CONTROL.equals(edge.get(AutomationConstants.EDGE_FIELD_KIND))) { continue; } @@ -152,6 +205,22 @@ static Map> controlTargets( return targets; } + /** Returns the sole zero-incoming node in an already validated loop body. */ + static String graphEntryNodeId(List> nodes, List> edges) { + Map incoming = new LinkedHashMap<>(); + for (Map node : nodes) { + incoming.put((String) node.get(AutomationConstants.NODE_FIELD_ID), 0); + } + for (Map edge : edges) { + if (AutomationConstants.EDGE_KIND_CONTROL.equals(edge.get(AutomationConstants.EDGE_FIELD_KIND))) { + incoming.compute((String) edge.get(AutomationConstants.EDGE_FIELD_TARGET), + (ignored, count) -> count + 1); + } + } + return incoming.entrySet().stream().filter(entry -> entry.getValue() == 0).map(Map.Entry::getKey).findFirst() + .orElseThrow(() -> new IllegalArgumentException("Loop body has no entry node.")); + } + /** * Runs one node module with the workflow scope supplied by the Java scheduler. */ diff --git a/src/prerna/reactor/automation/AutomationSourceRenderer.java b/src/prerna/reactor/automation/AutomationSourceRenderer.java index 3ab641a303..9eb6bd34b6 100644 --- a/src/prerna/reactor/automation/AutomationSourceRenderer.java +++ b/src/prerna/reactor/automation/AutomationSourceRenderer.java @@ -62,8 +62,8 @@ public static String renderNode(Map node) { : Map.of(); String source = switch (nodeType) { case TRIGGER_START -> triggerSource(); - case CONTROL_IF, CONTROL_JEV -> - throw new IllegalArgumentException("Routing nodes are evaluated by Java and do not have Python source."); + case CONTROL_IF, CONTROL_JEV, CONTROL_LOOP -> + throw new IllegalArgumentException("Java-owned control nodes do not have Python source."); case DATABASE_QUERY -> databaseQuerySource(config); case DATABASE_INSERT -> databaseWriteSource(config, "insertData"); case DATABASE_UPDATE -> databaseWriteSource(config, "updateData"); diff --git a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java index 45bb63eb0c..ac0ba1348f 100644 --- a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java +++ b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java @@ -331,6 +331,9 @@ public void createColumnsAndTypes(AbstractSqlQueryUtil queryUtil) { Pair.with(AutomationConstants.WORKSPACE_ID, AutomationConstants.VARCHAR_50), Pair.with(AutomationConstants.MODEL_MESSAGE_ID, AutomationConstants.VARCHAR_50), Pair.with(AutomationConstants.AGENT_RUN_ID, AutomationConstants.VARCHAR_50), + Pair.with(AutomationConstants.PARENT_NODE_ID, AutomationConstants.VARCHAR_255), + Pair.with(AutomationConstants.ITERATION_INDEX, AutomationConstants.INTEGER), + Pair.with(AutomationConstants.SOURCE_NODE_ID, AutomationConstants.VARCHAR_255), Pair.with(AutomationConstants.ERROR_MESSAGE, CLOB_DATATYPE))); addTable(AutomationConstants.TABLE_AUTOMATION_RUN_WAITS, Arrays.asList( diff --git a/test/prerna/reactor/automation/AutomationDefinitionValidatorUnitTests.java b/test/prerna/reactor/automation/AutomationDefinitionValidatorUnitTests.java index 3895000a14..6e4e8fbf84 100644 --- a/test/prerna/reactor/automation/AutomationDefinitionValidatorUnitTests.java +++ b/test/prerna/reactor/automation/AutomationDefinitionValidatorUnitTests.java @@ -328,4 +328,67 @@ void rejectsAnUnknownFormatVersion() { assertThrows(IllegalArgumentException.class, () -> AutomationDefinitionValidator.parseAndValidateForAuthoring(json)); } + + @Test + void acceptsABoundedLoopWithAnAcyclicBody() { + Map bodyNode = workNode(AutomationConstants.NODE_DEVELOPER_PYTHON, Map.of()); + bodyNode.put(AutomationConstants.NODE_FIELD_ID, "body-python"); + bodyNode.put(AutomationConstants.NODE_FIELD_OUTPUT_VAR, "body_result"); + bodyNode.put(AutomationConstants.NODE_FIELD_CODE_MODE, AutomationConstants.NODE_CODE_MODE_CUSTOM); + Map loop = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_FOR_EACH, + AutomationConstants.CONFIG_LOOP_ITEMS, List.of(1, 2, 3), + AutomationConstants.CONFIG_LOOP_BATCH_SIZE, 1, + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + loop.put(AutomationConstants.NODE_FIELD_BODY, + Map.of(AutomationConstants.DOC_NODES, List.of(bodyNode), AutomationConstants.DOC_EDGES, List.of())); + + AutomationDefinitionValidator.parseAndValidate(definition(Map.of(), loop)); + } + + @Test + void permitsAnIncompleteLoopBodyOnlyWhileAuthoring() { + Map loop = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_FOR_EACH, + AutomationConstants.CONFIG_LOOP_ITEMS, List.of(), + AutomationConstants.CONFIG_LOOP_BATCH_SIZE, 1, + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + String json = definition(Map.of(), loop); + + AutomationDefinitionValidator.parseAndValidateForAuthoring(json); + assertThrows(IllegalArgumentException.class, () -> AutomationDefinitionValidator.parseAndValidate(json)); + } + + @Test + void rejectsNestedLoops() { + Map nested = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_FOR_EACH, + AutomationConstants.CONFIG_LOOP_ITEMS, List.of(1), + AutomationConstants.CONFIG_LOOP_BATCH_SIZE, 1, + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + nested.put(AutomationConstants.NODE_FIELD_ID, "nested-loop"); + Map loop = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_FOR_EACH, + AutomationConstants.CONFIG_LOOP_ITEMS, List.of(1), + AutomationConstants.CONFIG_LOOP_BATCH_SIZE, 1, + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + loop.put(AutomationConstants.NODE_FIELD_BODY, + Map.of(AutomationConstants.DOC_NODES, List.of(nested), AutomationConstants.DOC_EDGES, List.of())); + + assertThrows(IllegalArgumentException.class, + () -> AutomationDefinitionValidator.parseAndValidate(definition(Map.of(), loop))); + } + + @Test + void rejectsAnUnboundedLoop() { + Map loop = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_FOR_EACH, + AutomationConstants.CONFIG_LOOP_ITEMS, List.of(1), + AutomationConstants.CONFIG_LOOP_BATCH_SIZE, 1, + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, + AutomationConstants.LOOP_MAX_ITERATIONS + 1)); + + assertThrows(IllegalArgumentException.class, + () -> AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), loop))); + } } diff --git a/test/prerna/reactor/automation/AutomationNodeCatalogUnitTests.java b/test/prerna/reactor/automation/AutomationNodeCatalogUnitTests.java index 7e13c9e8bb..d0ec0e5d25 100644 --- a/test/prerna/reactor/automation/AutomationNodeCatalogUnitTests.java +++ b/test/prerna/reactor/automation/AutomationNodeCatalogUnitTests.java @@ -108,6 +108,20 @@ void jevDecisionAdvertisesTypeSafeRoutingConfiguration() { definition.defaultConfig().get(AutomationConstants.CONFIG_QUESTION_TYPE)); } + @Test + void loopAdvertisesBoundedSequentialConfiguration() { + AutomationNodeDefinition definition = find(AutomationConstants.NODE_CONTROL_LOOP); + assertNotNull(definition); + assertEquals(AutomationConstants.LOOP_MODE_FOR_EACH, + definition.defaultConfig().get(AutomationConstants.CONFIG_LOOP_MODE)); + Map> fieldsByKey = definition.configFields().stream() + .collect(java.util.stream.Collectors.toMap(AutomationNodeDefinition.ConfigField::key, + AutomationNodeDefinition.ConfigField::toMap)); + assertEquals(AutomationConstants.LOOP_MAX_ITERATIONS, + fieldsByKey.get(AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS).get("maximum")); + assertEquals(1, fieldsByKey.get(AutomationConstants.CONFIG_LOOP_BATCH_SIZE).get("minimum")); + } + /** Writes need edit rights on the engine; a read only needs view. */ @Test void writeNodesRequireEditPermission() { diff --git a/test/prerna/reactor/automation/AutomationRunExecutionServiceUnitTests.java b/test/prerna/reactor/automation/AutomationRunExecutionServiceUnitTests.java index beb3030ac9..1c584b4089 100644 --- a/test/prerna/reactor/automation/AutomationRunExecutionServiceUnitTests.java +++ b/test/prerna/reactor/automation/AutomationRunExecutionServiceUnitTests.java @@ -29,7 +29,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -111,4 +113,45 @@ void exposesRunInsightOnlyWhileItRemainsInTheInsightStore() { assertNull(AutomationRunExecutionService.getAvailableExecutionInsightId(runId)); } + + @Test + void resolvesLoopItemsWithoutCoercingNativeValues() { + List items = List.of(Map.of("id", 1), Map.of("id", 2)); + assertEquals(items, AutomationRunExecutionService.loopItems("${records}", Map.of("records", items), "loop")); + assertEquals(List.of("a", "b"), + AutomationRunExecutionService.loopItems(new String[] { "a", "b" }, Map.of(), "loop")); + } + + @Test + void rejectsMissingOrNonCollectionLoopItems() { + assertThrows(IllegalArgumentException.class, + () -> AutomationRunExecutionService.loopItems("${missing}", Map.of(), "loop")); + assertThrows(IllegalArgumentException.class, + () -> AutomationRunExecutionService.loopItems(42, Map.of(), "loop")); + } + + @Test + @SuppressWarnings("unchecked") + void groupsMaterializedLoopHistoryUnderItsParentNode() { + Map parent = new LinkedHashMap<>(); + parent.put(AutomationConstants.RUN_ID, "run"); + parent.put(AutomationConstants.NODE_ID, "loop"); + parent.put(AutomationConstants.NODE_LABEL, "Loop"); + parent.put(AutomationConstants.STATUS, AutomationConstants.NODE_STATUS_SUCCESS); + Map child = new LinkedHashMap<>(); + child.put(AutomationConstants.RUN_ID, "run"); + child.put(AutomationConstants.NODE_ID, "execution-id"); + child.put(AutomationConstants.SOURCE_NODE_ID, "body-python"); + child.put(AutomationConstants.PARENT_NODE_ID, "loop"); + child.put(AutomationConstants.ITERATION_INDEX, 0); + child.put(AutomationConstants.NODE_LABEL, "Python"); + child.put(AutomationConstants.STATUS, AutomationConstants.NODE_STATUS_SUCCESS); + + List> results = AutomationDatabaseUtility.buildNodeResults(List.of(parent, child)); + + assertEquals(1, results.size()); + List> iterations = (List>) results.get(0).get("iterations"); + List> nodes = (List>) iterations.get(0).get("nodeResults"); + assertEquals("body-python", nodes.get(0).get(AutomationConstants.NODE_ID)); + } } diff --git a/test/prerna/reactor/automation/AutomationRuntimeUnitTests.java b/test/prerna/reactor/automation/AutomationRuntimeUnitTests.java index 478064e49f..9217253670 100644 --- a/test/prerna/reactor/automation/AutomationRuntimeUnitTests.java +++ b/test/prerna/reactor/automation/AutomationRuntimeUnitTests.java @@ -115,4 +115,24 @@ void rejectsNestedOrMissingRunBindings() { .definesRunEntryPoint("class Job:\n def run(self, scope):\n return {}\n")); assertFalse(AutomationDefinitionService.definesRunEntryPoint("def outer():\n run = 1\n return run\n")); } + + @Test + void includesLoopBodyNodesWhenCollectingSourceOwners() { + Map loop = new LinkedHashMap<>(); + loop.put(AutomationConstants.NODE_FIELD_ID, "loop"); + loop.put(AutomationConstants.NODE_FIELD_TYPE, AutomationConstants.NODE_CONTROL_LOOP); + loop.put(AutomationConstants.NODE_FIELD_BODY, + Map.of(AutomationConstants.DOC_NODES, + List.of(Map.of(AutomationConstants.NODE_FIELD_ID, "body-node", + AutomationConstants.NODE_FIELD_TYPE, AutomationConstants.NODE_DEVELOPER_PYTHON)), + AutomationConstants.DOC_EDGES, List.of())); + Map document = Map.of(AutomationConstants.DOC_GRAPH, + Map.of(AutomationConstants.DOC_NODES, List.of(triggerNode(Map.of()), loop), + AutomationConstants.DOC_EDGES, List.of())); + AutomationDefinitionValidator.ValidatedDefinition definition = new AutomationDefinitionValidator.ValidatedDefinition( + document, List.of(triggerNode(Map.of()), loop), List.of(), "", ""); + + assertEquals(List.of("start", "loop", "body-node"), AutomationRuntime.allNodes(definition).stream() + .map(node -> String.valueOf(node.get(AutomationConstants.NODE_FIELD_ID))).toList()); + } } From 3500820cd8dfaaf7162ccc81499269a301bccf57 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Fri, 2 Oct 2026 15:35:25 -0400 Subject: [PATCH 2/7] fix(automation): import runtime after package refactor --- .../reactor/automation/project/AutomationProjectService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/prerna/reactor/automation/project/AutomationProjectService.java b/src/prerna/reactor/automation/project/AutomationProjectService.java index 1329aeb05a..2ebc380e55 100644 --- a/src/prerna/reactor/automation/project/AutomationProjectService.java +++ b/src/prerna/reactor/automation/project/AutomationProjectService.java @@ -46,6 +46,7 @@ import prerna.engine.impl.model.inferencetracking.ModelInferenceLogsUtils; import prerna.project.api.IProject; import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.AutomationRuntime; import prerna.reactor.automation.definition.AutomationDefinitionService; import prerna.reactor.automation.definition.AutomationDefinitionValidator; import prerna.reactor.automation.definition.AutomationNodeType; From 4b14e44a1faefbff890d8b5e71e7af4a480d3324 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Mon, 5 Oct 2026 09:11:21 -0400 Subject: [PATCH 3/7] feat(automation): support configurable loop workflows --- src/prerna/reactor/automation/AGENTS.md | 12 +- .../automation/AutomationConstants.java | 4 + .../AutomationDefinitionValidator.java | 46 ++++-- .../definition/AutomationNodeCatalog.java | 25 +++- .../definition/AutomationNodeDefinition.java | 3 +- .../definition/AutomationSourceRenderer.java | 35 ++++- src/prerna/reactor/automation/run/AGENTS.md | 7 +- .../run/AutomationRunExecutionService.java | 141 ++++++++++++++---- .../automation/run/AutomationRunStore.java | 106 ++++++------- .../utils/AutomationRuntimeUtils.java | 58 ++++--- ...utomationDefinitionValidatorUnitTests.java | 39 +++++ .../AutomationNodeCatalogUnitTests.java | 18 +++ .../AutomationSourceRendererUnitTests.java | 15 ++ ...utomationRunExecutionServiceUnitTests.java | 2 + .../AutomationRuntimeUtilsUnitTests.java | 11 ++ 15 files changed, 370 insertions(+), 152 deletions(-) diff --git a/src/prerna/reactor/automation/AGENTS.md b/src/prerna/reactor/automation/AGENTS.md index c430113c75..8bfe2ce769 100644 --- a/src/prerna/reactor/automation/AGENTS.md +++ b/src/prerna/reactor/automation/AGENTS.md @@ -83,11 +83,13 @@ expression evaluator. The first match selects its `case:` edge; other `questionType: "choice"` selects among arbitrary described routes; `questionType: "noul"` maps the model's Yes probability to exactly one `{ answer: true }` route or one `{ answer: false }` route. Both modes retain stable route IDs for `case:` edges and select `else` when confidence is -below the configured threshold. `control.loop` owns a nested acyclic graph and currently supports -bounded sequential `forEach`/batch execution. Java owns iteration, cancellation, and server-side -limits; body nodes continue to use the ordinary node executors and appear under their parent loop -in run history. Nested loops, agent waits inside a loop, arbitrary fan-out from one port, and -parallel execution are rejected before execution. Nonselected branch nodes are retained in history +below the configured threshold. `control.loop` owns a nested acyclic graph and supports bounded, +sequential `forEach`, fixed-count `repeat`, and condition-based `while` execution. Java owns each +pass, cancellation, and server-side limits; body nodes continue to use the ordinary node executors +and appear under their parent loop in run history. Loop context exposes the current item/group or +pass number. A `while` loop also exposes only the prior pass's named body outputs. Nested loops, +agent waits inside a loop, arbitrary fan-out from one port, and parallel execution are rejected +before execution. Nonselected branch nodes are retained in history as `SKIPPED`. Trigger globals use the canonical `trigger.start.config.globals` list: each entry is `{ name, defaultValue, description? }`, with a non-private Python-identifier name. `trigger.start.config.pythonSource` holds the optional diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 514b771254..3a0eb2f154 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -210,8 +210,12 @@ private AutomationConstants() { public static final String CONFIG_LOOP_MODE = "mode"; public static final String CONFIG_LOOP_ITEMS = "items"; public static final String CONFIG_LOOP_BATCH_SIZE = "batchSize"; + public static final String CONFIG_LOOP_COUNT = "count"; + public static final String CONFIG_LOOP_CONDITION = "condition"; public static final String CONFIG_LOOP_MAX_ITERATIONS = "maxIterations"; public static final String LOOP_MODE_FOR_EACH = "forEach"; + public static final String LOOP_MODE_REPEAT = "repeat"; + public static final String LOOP_MODE_WHILE = "while"; public static final String CONFIG_WAIT_TIMEOUT_MS = "waitTimeoutMs"; public static final String CONFIG_APP_ID = "appId"; diff --git a/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java b/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java index b273da47e5..7057b86799 100644 --- a/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java +++ b/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java @@ -304,7 +304,8 @@ private static void validateNodeConfig(String nodeId, AutomationNodeType nodeTyp case CONTROL_IF -> validateBranchConfig(nodeId, config); case CONTROL_JEV -> validateJevBranchConfig(nodeId, config); case CONTROL_LOOP -> validateLoopConfig(nodeId, config); - case STORAGE_LIST, TRIGGER_START, DEVELOPER_PYTHON -> { + case STORAGE_LIST -> validateOptionalStringListOrPlaceholder(nodeId, config, "extensions"); + case TRIGGER_START, DEVELOPER_PYTHON -> { // These node types have no additional required configuration here. } } @@ -321,21 +322,38 @@ private static boolean isJavaOwnedNode(AutomationNodeType nodeType) { private static void validateLoopConfig(String nodeId, Map config) { Object mode = config.getOrDefault(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_FOR_EACH); - if (!AutomationConstants.LOOP_MODE_FOR_EACH.equals(mode)) { - throw new IllegalArgumentException("Loop node '" + nodeId + "' config.mode must be '" - + AutomationConstants.LOOP_MODE_FOR_EACH + "'."); - } - Object items = config.get(AutomationConstants.CONFIG_LOOP_ITEMS); - boolean scopeReference = items instanceof String value - && value.trim().matches("\\$\\{[A-Za-z_][A-Za-z0-9_]*}"); - if (!(items instanceof List) && !scopeReference) { - throw new IllegalArgumentException("Loop node '" + nodeId - + "' config.items must be an array or an exact ${scope_value} reference."); - } - validateBoundedInteger(nodeId, config, AutomationConstants.CONFIG_LOOP_BATCH_SIZE, - AutomationConstants.LOOP_MIN_BATCH_SIZE, AutomationConstants.LOOP_MAX_BATCH_SIZE); validateBoundedInteger(nodeId, config, AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 1, AutomationConstants.LOOP_MAX_ITERATIONS); + if (AutomationConstants.LOOP_MODE_FOR_EACH.equals(mode)) { + Object items = config.get(AutomationConstants.CONFIG_LOOP_ITEMS); + boolean scopeReference = items instanceof String value && isScopePlaceholder(value.trim()); + if (!(items instanceof List) && !scopeReference) { + throw new IllegalArgumentException("Loop node '" + nodeId + + "' config.items must be an array or an exact scope reference such as ${files} or ${download.files}."); + } + validateBoundedInteger(nodeId, config, AutomationConstants.CONFIG_LOOP_BATCH_SIZE, + AutomationConstants.LOOP_MIN_BATCH_SIZE, AutomationConstants.LOOP_MAX_BATCH_SIZE); + return; + } + if (AutomationConstants.LOOP_MODE_REPEAT.equals(mode)) { + validateBoundedInteger(nodeId, config, AutomationConstants.CONFIG_LOOP_COUNT, 1, + AutomationConstants.LOOP_MAX_ITERATIONS); + int count = ((Number) config.get(AutomationConstants.CONFIG_LOOP_COUNT)).intValue(); + int maximumIterations = ((Number) config.get(AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS)).intValue(); + if (count > maximumIterations) { + throw new IllegalArgumentException("Loop node '" + nodeId + + "' config.count cannot exceed config.maxIterations."); + } + return; + } + if (AutomationConstants.LOOP_MODE_WHILE.equals(mode)) { + String condition = requireNonblankString(config.get(AutomationConstants.CONFIG_LOOP_CONDITION), + "Loop node '" + nodeId + "' config.condition"); + AutomationConditionEvaluator.validate(condition); + return; + } + throw new IllegalArgumentException("Loop node '" + nodeId + + "' config.mode must be 'forEach', 'repeat', or 'while'."); } private static void validateBoundedInteger(String nodeId, Map config, String key, int minimum, diff --git a/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java b/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java index 02265c90e1..e9de3cf66f 100644 --- a/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java +++ b/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java @@ -214,15 +214,19 @@ private static List createDefinitions() { controlInputs(), List.of(port("case:", "Route", PortKind.CONTROL, PortDirection.OUTPUT, null), port("else", "Fallback", PortKind.CONTROL, PortDirection.OUTPUT, null)))); - definitions.add(definition(AutomationNodeType.CONTROL_LOOP, "Loop over items", - "Run a nested sequence once for each item or batch in a list.", + definitions.add(definition(AutomationNodeType.CONTROL_LOOP, "Repeat steps", + "Run a nested sequence for each item, a fixed number of times, or while a condition is true.", orderedConfig("mode", AutomationConstants.LOOP_MODE_FOR_EACH, "items", List.of(), "batchSize", 1, - "maxIterations", AutomationConstants.LOOP_DEFAULT_MAX_ITERATIONS), + "count", 1, "condition", "", "maxIterations", + AutomationConstants.LOOP_DEFAULT_MAX_ITERATIONS), List.of(field("mode", ConfigFieldType.STRING, "Mode", true, AutomationConstants.LOOP_MODE_FOR_EACH), - field("items", ConfigFieldType.JSON, "Items", true, List.of()), - boundedIntegerField("batchSize", "Batch size", 1, + field("items", ConfigFieldType.JSON, "Items", false, List.of()), + optionalBoundedIntegerField("batchSize", "Batch size", 1, AutomationConstants.LOOP_MIN_BATCH_SIZE, AutomationConstants.LOOP_MAX_BATCH_SIZE), + optionalBoundedIntegerField("count", "Number of times", 1, 1, + AutomationConstants.LOOP_MAX_ITERATIONS), + field("condition", ConfigFieldType.TEXT, "Continue while", false, ""), boundedIntegerField("maxIterations", "Maximum iterations", AutomationConstants.LOOP_DEFAULT_MAX_ITERATIONS, 1, AutomationConstants.LOOP_MAX_ITERATIONS)), @@ -230,7 +234,7 @@ private static List createDefinitions() { "Number of items processed.", true), outputField("iterations", OutputFieldType.NUMBER, "Iterations", "Number of loop-body executions.", true), - outputField("results", OutputFieldType.OBJECT, "Results", + outputField("results", OutputFieldType.OBJECT_LIST, "Results", "Ordered outputs collected from each iteration.", true)), controlInputs(), controlAndResultOutputs())); definitions.add(new AutomationNodeDefinition(AutomationNodeType.DEVELOPER_PYTHON, "Python", @@ -257,6 +261,10 @@ private static AutomationNodeDefinition storageDefinition(AutomationNodeType nod List fields = new ArrayList<>(); fields.add(engineField(IEngine.CATALOG_TYPE.STORAGE)); fields.add(field("path", ConfigFieldType.STRING, "Storage path", pathRequired, "")); + if (nodeType == AutomationNodeType.STORAGE_LIST) { + defaultConfig.put("extensions", List.of()); + fields.add(field("extensions", ConfigFieldType.STRING_LIST, "File types", false, List.of())); + } if (destinationRequired) { fields.add(field("destination", ConfigFieldType.STRING, "Destination", requireDestination, destinationDefault)); @@ -345,6 +353,11 @@ private static ConfigField boundedIntegerField(String key, String label, int def return new ConfigField(key, ConfigFieldType.INTEGER, label, true, defaultValue, minimum, maximum, null); } + private static ConfigField optionalBoundedIntegerField(String key, String label, int defaultValue, Integer minimum, + Integer maximum) { + return new ConfigField(key, ConfigFieldType.INTEGER, label, false, defaultValue, minimum, maximum, null); + } + private static Port port(String id, String label, PortKind kind, PortDirection direction, String dataType) { return new Port(id, label, kind, direction, dataType); } diff --git a/src/prerna/reactor/automation/definition/AutomationNodeDefinition.java b/src/prerna/reactor/automation/definition/AutomationNodeDefinition.java index cba80053ce..cbea868676 100644 --- a/src/prerna/reactor/automation/definition/AutomationNodeDefinition.java +++ b/src/prerna/reactor/automation/definition/AutomationNodeDefinition.java @@ -79,7 +79,8 @@ String getValue() { /** Supported structured result value shapes exposed to authoring clients. */ public enum OutputFieldType { - BOOLEAN("boolean"), NUMBER("number"), OBJECT("object"), STRING("string"), STRING_LIST("string[]"); + BOOLEAN("boolean"), NUMBER("number"), OBJECT("object"), OBJECT_LIST("object[]"), STRING("string"), + STRING_LIST("string[]"); private final String value; diff --git a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java index 13e0f74855..1a22f56888 100644 --- a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java +++ b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java @@ -74,7 +74,7 @@ public static String renderNode(Map node) { case MODEL_EMBEDDINGS -> modelEmbeddingsSource(config); case MODEL_VISION -> modelVisionSource(config); case MODEL_NER -> modelNerSource(config); - case STORAGE_LIST -> storageSource(config, "list", "STORAGE_PATH"); + case STORAGE_LIST -> storageListSource(config); case STORAGE_READ -> storageReadSource(config); case STORAGE_UPLOAD -> storageUploadSource(config); case STORAGE_DOWNLOAD -> storageDownloadSource(config); @@ -334,6 +334,39 @@ def run(scope): """.formatted(value(config, "engineId"), valueOrDefault(config, "path", ""), method, argument); } + private static String storageListSource(Map config) { + return """ + # List storage paths through the existing SEMOSS storage SDK. + from ai_server import StorageEngine + + ENGINE_ID = %s + STORAGE_PATH = %s + FILE_TYPES = %s + + def _normalized_suffixes(values): + return tuple( + "." + str(value).strip().lower().lstrip(".") + for value in values + if str(value).strip() + ) + + def run(scope): + storage = StorageEngine(engine_id=scope.resolve(ENGINE_ID)) + paths = storage.list(scope.resolve(STORAGE_PATH)) + file_types = scope.resolve(FILE_TYPES) + if not file_types: + return paths + if isinstance(file_types, str): + file_types = [file_types] + suffixes = _normalized_suffixes(file_types) + return [ + path for path in paths + if isinstance(path, str) and path.rstrip("/").lower().endswith(suffixes) + ] + """.formatted(value(config, "engineId"), valueOrDefault(config, "path", ""), + valueOrDefault(config, "extensions", List.of())); + } + private static String storageReadSource(Map config) { return """ # Read storage content through the existing SEMOSS reactor; the Python SDK has no read method. diff --git a/src/prerna/reactor/automation/run/AGENTS.md b/src/prerna/reactor/automation/run/AGENTS.md index abe1b9b98d..7e4537e030 100644 --- a/src/prerna/reactor/automation/run/AGENTS.md +++ b/src/prerna/reactor/automation/run/AGENTS.md @@ -19,9 +19,10 @@ contract. - Execute only the definition, inputs, and source captured for that run. - Keep node transitions, cancellation, waits, and terminal status durable. - A structured loop materializes independent child history rows for each - iteration. Keep the parent graph acyclic, use the ordinary node executors for - body work, enforce iteration and total-execution bounds before materializing - rows, and keep iteration scope isolated from the parent scope. + pass. Keep the parent graph acyclic, use the ordinary node executors for body + work, enforce pass and total-execution bounds before materializing rows, and + keep scope isolated from the parent. A `while` loop may carry only the prior + pass's declared body outputs into its next isolated scope. - Keep frame display data scoped to the execution Insight. A frame is not a durable-history contract. - Do not pass engine objects, arbitrary Java objects, or unbounded payloads diff --git a/src/prerna/reactor/automation/run/AutomationRunExecutionService.java b/src/prerna/reactor/automation/run/AutomationRunExecutionService.java index f90680e0c2..86d01a8333 100644 --- a/src/prerna/reactor/automation/run/AutomationRunExecutionService.java +++ b/src/prerna/reactor/automation/run/AutomationRunExecutionService.java @@ -290,9 +290,9 @@ private Map executeInControlOrder(Insight executionInsight, Stri } /** - * Executes a bounded, sequential foreach container. The parent graph remains - * acyclic; each iteration materializes independent child history rows and runs - * the loop body's own validated acyclic control path against an isolated scope. + * Executes a bounded, sequential loop container. The parent graph remains + * acyclic; each pass materializes independent child history rows and runs the + * loop body's own validated acyclic control path against an isolated scope. */ @SuppressWarnings("unchecked") private Map executeLoopNode(Insight executionInsight, String projectId, String runId, @@ -304,55 +304,92 @@ private Map executeLoopNode(Insight executionInsight, String pro streamNodeProgress(runId, loopNode, AutomationConstants.NODE_STATUS_RUNNING, null, null, null); try { Map config = (Map) loopNode.get(AutomationConstants.NODE_FIELD_CONFIG); - List items = loopItems(config.get(AutomationConstants.CONFIG_LOOP_ITEMS), parentScope, loopNodeId); - int batchSize = ((Number) config.get(AutomationConstants.CONFIG_LOOP_BATCH_SIZE)).intValue(); + String loopMode = String.valueOf(config.getOrDefault(AutomationConstants.CONFIG_LOOP_MODE, + AutomationConstants.LOOP_MODE_FOR_EACH)); int maximumIterations = ((Number) config.get(AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS)).intValue(); - int iterationCount = items.isEmpty() ? 0 : (items.size() + batchSize - 1) / batchSize; - if (iterationCount > maximumIterations) { - throw new IllegalArgumentException("Loop node '" + loopNodeId + "' requires " + iterationCount + List items = AutomationConstants.LOOP_MODE_FOR_EACH.equals(loopMode) + ? loopItems(config.get(AutomationConstants.CONFIG_LOOP_ITEMS), parentScope, loopNodeId) + : List.of(); + int batchSize = AutomationConstants.LOOP_MODE_FOR_EACH.equals(loopMode) + ? ((Number) config.get(AutomationConstants.CONFIG_LOOP_BATCH_SIZE)).intValue() + : 1; + int iterationLimit = switch (loopMode) { + case AutomationConstants.LOOP_MODE_FOR_EACH -> items.isEmpty() ? 0 + : (items.size() + batchSize - 1) / batchSize; + case AutomationConstants.LOOP_MODE_REPEAT -> ((Number) config + .get(AutomationConstants.CONFIG_LOOP_COUNT)).intValue(); + case AutomationConstants.LOOP_MODE_WHILE -> maximumIterations; + default -> throw new IllegalArgumentException("Unsupported loop mode: " + loopMode); + }; + if (iterationLimit > maximumIterations) { + throw new IllegalArgumentException("Loop node '" + loopNodeId + "' requires " + iterationLimit + " iterations, exceeding its maxIterations value of " + maximumIterations + "."); } List> bodyNodes = AutomationRuntime.nodesForGraph( AutomationRuntime.loopBodyNodes(loopNode), AutomationRuntime.loopBodyEdges(loopNode)); - long bodyNodeExecutions = (long) iterationCount * bodyNodes.size(); + long bodyNodeExecutions = (long) iterationLimit * bodyNodes.size(); if (bodyNodeExecutions > AutomationConstants.LOOP_MAX_NODE_EXECUTIONS) { throw new IllegalArgumentException("Loop node '" + loopNodeId + "' would execute " + bodyNodeExecutions + " body steps, exceeding the maximum of " - + AutomationConstants.LOOP_MAX_NODE_EXECUTIONS + ". Increase the batch size or reduce the loop body."); + + AutomationConstants.LOOP_MAX_NODE_EXECUTIONS + + ". Reduce the number of passes or the loop body."); } List> bodyEdges = AutomationRuntime.loopBodyEdges(loopNode); Map> bodyTargets = AutomationRuntime.controlTargets(bodyEdges); String bodyEntry = AutomationRuntime.graphEntryNodeId(bodyNodes, bodyEdges); String loopOutputVar = (String) loopNode.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); List> iterationResults = new ArrayList<>(); + Map> bodyNodesById = new LinkedHashMap<>(); + Map carriedOutputs = new LinkedHashMap<>(); + for (Map bodyNode : bodyNodes) { + bodyNodesById.put((String) bodyNode.get(AutomationConstants.NODE_FIELD_ID), bodyNode); + Object outputVar = bodyNode.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); + if (AutomationConstants.LOOP_MODE_WHILE.equals(loopMode) && outputVar instanceof String value) { + carriedOutputs.put(value, null); + } + } + String whileCondition = AutomationConstants.LOOP_MODE_WHILE.equals(loopMode) + ? (String) config.get(AutomationConstants.CONFIG_LOOP_CONDITION) + : null; - for (int iteration = 0; iteration < iterationCount; iteration++) { + for (int iteration = 0; iteration < iterationLimit; iteration++) { if (AutomationRunRegistry.isCancellationRequested(runId)) { throw new IllegalStateException("Run cancelled by user"); } - int from = iteration * batchSize; - int to = Math.min(items.size(), from + batchSize); - List batch = new ArrayList<>(items.subList(from, to)); Map loopContext = new LinkedHashMap<>(); loopContext.put("index", iteration); - loopContext.put("batch", batch); - loopContext.put("total", iterationCount); + loopContext.put("number", iteration + 1); loopContext.put("isFirst", iteration == 0); - loopContext.put("isLast", iteration == iterationCount - 1); - if (batch.size() == 1) { - loopContext.put("item", batch.get(0)); + if (AutomationConstants.LOOP_MODE_FOR_EACH.equals(loopMode)) { + int from = iteration * batchSize; + int to = Math.min(items.size(), from + batchSize); + List batch = new ArrayList<>(items.subList(from, to)); + loopContext.put("batch", batch); + loopContext.put("total", iterationLimit); + loopContext.put("isLast", iteration == iterationLimit - 1); + if (batch.size() == 1) { + loopContext.put("item", batch.get(0)); + } + } else if (AutomationConstants.LOOP_MODE_REPEAT.equals(loopMode)) { + loopContext.put("total", iterationLimit); + loopContext.put("isLast", iteration == iterationLimit - 1); + } else { + loopContext.put("previous", new LinkedHashMap<>(carriedOutputs)); + loopContext.put("maximum", maximumIterations); } Map iterationScope = new LinkedHashMap<>(parentScope); + if (AutomationConstants.LOOP_MODE_WHILE.equals(loopMode)) { + iterationScope.putAll(carriedOutputs); + } iterationScope.put(loopOutputVar, loopContext); + if (whileCondition != null && !AutomationConditionEvaluator.evaluate(whileCondition, iterationScope)) { + break; + } Map bodyTraceRoomIds = allocateTraceRoomIds(bodyNodes); Map executionNodeIds = AutomationRunStore.insertLoopIterationNodes(runId, loopNodeId, iteration, bodyNodes, bodyTraceRoomIds); - Map> bodyNodesById = new LinkedHashMap<>(); - for (Map bodyNode : bodyNodes) { - bodyNodesById.put((String) bodyNode.get(AutomationConstants.NODE_FIELD_ID), bodyNode); - } Set visited = new HashSet<>(); Map outputs = new LinkedHashMap<>(); @@ -405,14 +442,34 @@ private Map executeLoopNode(Insight executionInsight, String pro iterationResult.put("index", iteration); iterationResult.put("outputs", outputs); iterationResults.add(iterationResult); + if (AutomationConstants.LOOP_MODE_WHILE.equals(loopMode)) { + carriedOutputs.replaceAll((key, value) -> null); + carriedOutputs.putAll(outputs); + } AutomationRuntimeUtils.toBoundedRuntimeJson(iterationResults, AutomationConstants.NODE_OUTPUT_MAX_BYTES, "Automation loop '" + loopNodeId + "' collected results"); } + if (whileCondition != null && iterationResults.size() == maximumIterations) { + Map nextContext = new LinkedHashMap<>(); + nextContext.put("index", maximumIterations); + nextContext.put("number", maximumIterations + 1); + nextContext.put("isFirst", false); + nextContext.put("previous", new LinkedHashMap<>(carriedOutputs)); + nextContext.put("maximum", maximumIterations); + Map nextScope = new LinkedHashMap<>(parentScope); + nextScope.putAll(carriedOutputs); + nextScope.put(loopOutputVar, nextContext); + if (AutomationConditionEvaluator.evaluate(whileCondition, nextScope)) { + throw new IllegalArgumentException("Loop node '" + loopNodeId + "' reached its maximum of " + + maximumIterations + " iterations while its condition was still true."); + } + } Map value = new LinkedHashMap<>(); - value.put("processed", items.size()); - value.put("iterations", iterationCount); + value.put("processed", AutomationConstants.LOOP_MODE_FOR_EACH.equals(loopMode) ? items.size() + : iterationResults.size()); + value.put("iterations", iterationResults.size()); value.put("results", iterationResults); String output = AutomationRuntimeUtils.toBoundedRuntimeJson(value, AutomationConstants.NODE_OUTPUT_MAX_BYTES, "Automation loop '" + loopNodeId + "' output"); @@ -442,12 +499,7 @@ static List loopItems(Object configuredItems, Map scope, if (configuredItems instanceof String value) { Matcher reference = EXACT_SCOPE_REFERENCE.matcher(value.trim()); if (reference.matches()) { - String name = reference.group(1); - if (!scope.containsKey(name)) { - throw new IllegalArgumentException( - "Loop node '" + loopNodeId + "' references unavailable scope value: " + name); - } - resolved = scope.get(name); + resolved = loopScopeValue(reference.group(1), scope, loopNodeId); } } if (resolved instanceof Collection collection) { @@ -463,6 +515,33 @@ static List loopItems(Object configuredItems, Map scope, throw new IllegalArgumentException("Loop node '" + loopNodeId + "' items must resolve to an array or list."); } + /** Resolves a generated loop's exact dotted scope path without coercing its value. */ + private static Object loopScopeValue(String path, Map scope, String loopNodeId) { + String[] parts = path.split("\\."); + Object current = scope.get(parts[0]); + if (!scope.containsKey(parts[0])) { + throw new IllegalArgumentException( + "Loop node '" + loopNodeId + "' references unavailable scope value: " + path); + } + for (int index = 1; index < parts.length; index++) { + String part = parts[index]; + if (current instanceof Map map && map.containsKey(part)) { + current = map.get(part); + continue; + } + if (current instanceof List list && part.matches("[0-9]+")) { + int listIndex = Integer.parseInt(part); + if (listIndex < list.size()) { + current = list.get(listIndex); + continue; + } + } + throw new IllegalArgumentException( + "Loop node '" + loopNodeId + "' cannot resolve scope value: " + path); + } + return current; + } + /** * Evaluates one typed routing question through the configured TypeSafe/Jev * engine. The model may select only a configured route; responses below the diff --git a/src/prerna/reactor/automation/run/AutomationRunStore.java b/src/prerna/reactor/automation/run/AutomationRunStore.java index fc559a791f..06a930598a 100644 --- a/src/prerna/reactor/automation/run/AutomationRunStore.java +++ b/src/prerna/reactor/automation/run/AutomationRunStore.java @@ -752,45 +752,41 @@ private static void insertAllNodeOutputs(Connection conn, AbstractSqlQueryUtil q public static Map insertLoopIterationNodes(String runId, String loopNodeId, int iterationIndex, List> orderedNodes, Map traceRoomIds) { IRDBMSEngine schedulerDb = requireSchedulerDb("initialize an automation loop iteration"); - Map executionNodeIds = new LinkedHashMap<>(); - Connection conn = null; try { - conn = schedulerDb.getConnection(); - try (PreparedStatement total = conn.prepareStatement(INCREMENT_RUN_TOTAL_NODES); - PreparedStatement output = conn.prepareStatement(INSERT_LOOP_NODE_OUTPUT)) { - total.setInt(1, orderedNodes.size()); - total.setString(2, runId); - requireSingleRow(total.executeUpdate(), "extend the loop run node count", runId, loopNodeId); - for (int index = 0; index < orderedNodes.size(); index++) { - Map node = orderedNodes.get(index); - String sourceNodeId = (String) node.get(NODE_FIELD_ID); - String executionNodeId = UUID.randomUUID().toString(); - executionNodeIds.put(sourceNodeId, executionNodeId); - int parameter = 1; - output.setString(parameter++, runId); - output.setString(parameter++, executionNodeId); - output.setString(parameter++, (String) node.get(NODE_FIELD_LABEL)); - output.setInt(parameter++, LOOP_EXECUTION_ORDER_OFFSET - + iterationIndex * Math.max(1, orderedNodes.size()) + index); - output.setString(parameter++, NODE_STATUS_PENDING); - setNullableString(output, parameter++, traceRoomIds == null ? null : traceRoomIds.get(sourceNodeId)); - setNullableString(output, parameter++, AutomationRuntime.configuredAgentWorkspaceId(node)); - output.setString(parameter++, loopNodeId); - output.setInt(parameter++, iterationIndex); - output.setString(parameter++, sourceNodeId); - output.addBatch(); + return QueryExecutionUtility.write(schedulerDb, conn -> { + Map executionNodeIds = new LinkedHashMap<>(); + try (PreparedStatement total = conn.prepareStatement(INCREMENT_RUN_TOTAL_NODES); + PreparedStatement output = conn.prepareStatement(INSERT_LOOP_NODE_OUTPUT)) { + total.setInt(1, orderedNodes.size()); + total.setString(2, runId); + requireSingleRow(total.executeUpdate(), "extend the loop run node count", runId, loopNodeId); + for (int index = 0; index < orderedNodes.size(); index++) { + Map node = orderedNodes.get(index); + String sourceNodeId = (String) node.get(NODE_FIELD_ID); + String executionNodeId = UUID.randomUUID().toString(); + executionNodeIds.put(sourceNodeId, executionNodeId); + int parameter = 1; + output.setString(parameter++, runId); + output.setString(parameter++, executionNodeId); + output.setString(parameter++, (String) node.get(NODE_FIELD_LABEL)); + output.setInt(parameter++, LOOP_EXECUTION_ORDER_OFFSET + + iterationIndex * Math.max(1, orderedNodes.size()) + index); + output.setString(parameter++, NODE_STATUS_PENDING); + schedulerDb.getQueryUtil().setNullableString(output, parameter++, + traceRoomIds == null ? null : traceRoomIds.get(sourceNodeId)); + schedulerDb.getQueryUtil().setNullableString(output, parameter++, + AutomationRuntime.configuredAgentWorkspaceId(node)); + output.setString(parameter++, loopNodeId); + output.setInt(parameter++, iterationIndex); + output.setString(parameter++, sourceNodeId); + output.addBatch(); + } + output.executeBatch(); } - output.executeBatch(); - } - if (!conn.getAutoCommit()) { - conn.commit(); - } - return executionNodeIds; + return executionNodeIds; + }); } catch (Exception e) { - rollback(conn, e); throw new IllegalStateException("Unable to initialize the automation loop iteration.", e); - } finally { - closeConnection(schedulerDb, conn); } } @@ -1156,28 +1152,23 @@ public static void skipPendingNodes(String runId, List nodeIds, String r return; } IRDBMSEngine schedulerDb = requireSchedulerDb("persist skipped automation loop nodes"); - Connection conn = null; try { - conn = schedulerDb.getConnection(); - try (PreparedStatement ps = conn.prepareStatement(SKIP_PENDING_NODE_OUTPUT)) { - for (String nodeId : nodeIds) { - ps.setString(1, NODE_STATUS_SKIPPED); - setNullableString(ps, 2, reason); - ps.setString(3, runId); - ps.setString(4, nodeId); - ps.setString(5, NODE_STATUS_PENDING); - ps.addBatch(); + QueryExecutionUtility.write(schedulerDb, conn -> { + try (PreparedStatement ps = conn.prepareStatement(SKIP_PENDING_NODE_OUTPUT)) { + for (String nodeId : nodeIds) { + ps.setString(1, NODE_STATUS_SKIPPED); + schedulerDb.getQueryUtil().setNullableString(ps, 2, reason); + ps.setString(3, runId); + ps.setString(4, nodeId); + ps.setString(5, NODE_STATUS_PENDING); + ps.addBatch(); + } + ps.executeBatch(); } - ps.executeBatch(); - } - if (!conn.getAutoCommit()) { - conn.commit(); - } - } catch (SQLException e) { - rollback(conn, e); + return null; + }); + } catch (Exception e) { throw new IllegalStateException("Unable to persist skipped automation loop nodes.", e); - } finally { - closeConnection(schedulerDb, conn); } } @@ -1513,13 +1504,6 @@ private static void requireSingleRow(int updatedRows, String operation, String r + ": expected one row but updated " + updatedRows + "."); } - /** - * Binds a nullable VARCHAR column value, using {@code setNull(Types.VARCHAR)} - * instead of {@code setString(index, null)} when the value is absent - some - * JDBC drivers require an explicit SQL type for a null bind rather than - * inferring it from a null String argument. - */ - private static Timestamp toTimestamp(Instant instant) { return Utility.getSqlTimestampUTC(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); } diff --git a/src/prerna/reactor/automation/utils/AutomationRuntimeUtils.java b/src/prerna/reactor/automation/utils/AutomationRuntimeUtils.java index f8c8ea12e0..dc0335a979 100644 --- a/src/prerna/reactor/automation/utils/AutomationRuntimeUtils.java +++ b/src/prerna/reactor/automation/utils/AutomationRuntimeUtils.java @@ -37,7 +37,6 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayDeque; import java.util.Base64; import java.util.Collections; import java.util.HashMap; @@ -167,9 +166,8 @@ public static Map parseJsonObject(String value, String field) { /** * Validates container depth before serialization, then limits the encoded UTF-8 - * payload. Runtime values originate as JSON, so repeated container identity has - * no defined meaning and is rejected with cycles before Gson can recurse - * indefinitely. + * payload. Repeated references serialize as ordinary duplicated JSON values; + * only an object that appears again on its active ancestor path is cyclic. * * @param value runtime value to serialize * @param maxBytes maximum UTF-8 payload size @@ -187,37 +185,39 @@ public static String toBoundedRuntimeJson(Object value, int maxBytes, String val } private static void validateRuntimeDepth(Object value, String valueName) { - ArrayDeque pending = new ArrayDeque<>(); - Set visitedContainers = Collections.newSetFromMap(new IdentityHashMap<>()); - pending.push(new RuntimeValueDepth(value, 0)); - while (!pending.isEmpty()) { - RuntimeValueDepth current = pending.pop(); - Object currentValue = current.value(); - if (!(currentValue instanceof Map) && !(currentValue instanceof Iterable) - && (currentValue == null || !currentValue.getClass().isArray())) { - continue; - } - if (current.depth() >= AutomationConstants.RUNTIME_JSON_MAX_DEPTH) { - throw new IllegalArgumentException(valueName + " exceeds the maximum JSON depth of " - + AutomationConstants.RUNTIME_JSON_MAX_DEPTH + "."); - } - if (!visitedContainers.add(currentValue)) { - throw new IllegalArgumentException(valueName + " contains a repeated or cyclic JSON container."); - } - int childDepth = current.depth() + 1; - if (currentValue instanceof Map map) { + Set ancestors = Collections.newSetFromMap(new IdentityHashMap<>()); + validateRuntimeDepth(value, valueName, 0, ancestors); + } + + private static void validateRuntimeDepth(Object value, String valueName, int depth, Set ancestors) { + if (!(value instanceof Map) && !(value instanceof Iterable) + && (value == null || !value.getClass().isArray())) { + return; + } + if (depth >= AutomationConstants.RUNTIME_JSON_MAX_DEPTH) { + throw new IllegalArgumentException(valueName + " exceeds the maximum JSON depth of " + + AutomationConstants.RUNTIME_JSON_MAX_DEPTH + "."); + } + if (!ancestors.add(value)) { + throw new IllegalArgumentException(valueName + " contains a cyclic JSON container."); + } + try { + int childDepth = depth + 1; + if (value instanceof Map map) { for (Object child : map.values()) { - pending.push(new RuntimeValueDepth(child, childDepth)); + validateRuntimeDepth(child, valueName, childDepth, ancestors); } - } else if (currentValue instanceof Iterable iterable) { + } else if (value instanceof Iterable iterable) { for (Object child : iterable) { - pending.push(new RuntimeValueDepth(child, childDepth)); + validateRuntimeDepth(child, valueName, childDepth, ancestors); } } else { - for (int index = 0; index < Array.getLength(currentValue); index++) { - pending.push(new RuntimeValueDepth(Array.get(currentValue, index), childDepth)); + for (int index = 0; index < Array.getLength(value); index++) { + validateRuntimeDepth(Array.get(value, index), valueName, childDepth, ancestors); } } + } finally { + ancestors.remove(value); } } @@ -235,6 +235,4 @@ public static String generatePreview(String s) { : s.substring(0, AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH); } - private record RuntimeValueDepth(Object value, int depth) { - } } diff --git a/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java b/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java index b0566a5854..442b6147a5 100644 --- a/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java @@ -382,6 +382,45 @@ void acceptsABoundedLoopWithAnAcyclicBody() { AutomationDefinitionValidator.parseAndValidate(definition(Map.of(), loop)); } + @Test + void acceptsDottedForEachRepeatAndWhileLoopConfiguration() { + Map dottedForEach = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_FOR_EACH, + AutomationConstants.CONFIG_LOOP_ITEMS, "${download.files}", + AutomationConstants.CONFIG_LOOP_BATCH_SIZE, 1, + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), dottedForEach)); + + Map repeat = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_REPEAT, + AutomationConstants.CONFIG_LOOP_COUNT, 5, + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), repeat)); + + Map whileLoop = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_WHILE, + AutomationConstants.CONFIG_LOOP_CONDITION, "${status} != \"complete\"", + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), whileLoop)); + } + + @Test + void rejectsRepeatCountAboveItsSafetyLimitAndInvalidWhileCondition() { + Map repeat = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_REPEAT, + AutomationConstants.CONFIG_LOOP_COUNT, 11, + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + assertThrows(IllegalArgumentException.class, + () -> AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), repeat))); + + Map whileLoop = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_WHILE, + AutomationConstants.CONFIG_LOOP_CONDITION, "status is complete", + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + assertThrows(IllegalArgumentException.class, + () -> AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), whileLoop))); + } + @Test void permitsAnIncompleteLoopBodyOnlyWhileAuthoring() { Map loop = workNode(AutomationConstants.NODE_CONTROL_LOOP, diff --git a/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java b/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java index 72009b4045..654890213c 100644 --- a/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java @@ -106,6 +106,17 @@ void storageDownloadAdvertisesItsStructuredResult() { assertEquals(false, fieldsByKey.get("filePath").get("required")); } + @Test + void storageListAdvertisesOptionalFileTypes() { + AutomationNodeDefinition definition = find(AutomationConstants.NODE_STORAGE_LIST); + assertNotNull(definition); + Map> fieldsByKey = definition.configFields().stream() + .collect(java.util.stream.Collectors.toMap(AutomationNodeDefinition.ConfigField::key, + AutomationNodeDefinition.ConfigField::toMap)); + assertEquals("string[]", fieldsByKey.get("extensions").get("type")); + assertEquals(List.of(), definition.defaultConfig().get("extensions")); + } + @Test void optionalEngineParametersAreBackendOwned() { Map> uploadFields = find(AutomationConstants.NODE_STORAGE_UPLOAD).configFields() @@ -154,6 +165,13 @@ void loopAdvertisesBoundedSequentialConfiguration() { assertEquals(AutomationConstants.LOOP_MAX_ITERATIONS, fieldsByKey.get(AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS).get("maximum")); assertEquals(1, fieldsByKey.get(AutomationConstants.CONFIG_LOOP_BATCH_SIZE).get("minimum")); + assertEquals(1, fieldsByKey.get(AutomationConstants.CONFIG_LOOP_COUNT).get("minimum")); + assertEquals(false, fieldsByKey.get(AutomationConstants.CONFIG_LOOP_ITEMS).get("required")); + assertEquals(false, fieldsByKey.get(AutomationConstants.CONFIG_LOOP_COUNT).get("required")); + assertNotNull(fieldsByKey.get(AutomationConstants.CONFIG_LOOP_CONDITION)); + assertEquals(AutomationNodeDefinition.OutputFieldType.OBJECT_LIST, + definition.outputFields().stream().filter(field -> "results".equals(field.key())).findFirst().orElseThrow() + .type()); } /** Writes need edit rights on the engine; a read only needs view. */ diff --git a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java index 58875b9d65..5dd52696c0 100644 --- a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java @@ -124,6 +124,21 @@ void storageDownloadReturnsTransferDetailsWithoutReimplementingPathRules() { assertFalse(source.contains("return storage.copyToLocal(")); } + @Test + void storageListCanFilterFileTypesWithoutChangingTheStorageEngine() { + Map config = new LinkedHashMap<>(); + config.put("engineId", "storage-1"); + config.put("path", "incoming"); + config.put("extensions", java.util.List.of("pdf", ".png")); + + String source = AutomationSourceRenderer + .renderNode(node(AutomationConstants.NODE_STORAGE_LIST, config)); + + assertTrue(source.contains("paths = storage.list(scope.resolve(STORAGE_PATH))")); + assertTrue(source.contains("path.rstrip(\"/\").lower().endswith(suffixes)")); + assertTrue(source.contains("FILE_TYPES = [\"pdf\",\".png\"]")); + } + @Test void visionAcceptsOneOrManyMediaPathsWithoutNestingThem() { Map config = new LinkedHashMap<>(); diff --git a/test/prerna/reactor/automation/run/AutomationRunExecutionServiceUnitTests.java b/test/prerna/reactor/automation/run/AutomationRunExecutionServiceUnitTests.java index 52a6bae3e5..c0cf114247 100644 --- a/test/prerna/reactor/automation/run/AutomationRunExecutionServiceUnitTests.java +++ b/test/prerna/reactor/automation/run/AutomationRunExecutionServiceUnitTests.java @@ -121,6 +121,8 @@ void exposesRunInsightOnlyWhileItRemainsInTheInsightStore() { void resolvesLoopItemsWithoutCoercingNativeValues() { List items = List.of(Map.of("id", 1), Map.of("id", 2)); assertEquals(items, AutomationRunExecutionService.loopItems("${records}", Map.of("records", items), "loop")); + assertEquals(items, AutomationRunExecutionService.loopItems("${query.data.rows}", + Map.of("query", Map.of("data", Map.of("rows", items))), "loop")); assertEquals(List.of("a", "b"), AutomationRunExecutionService.loopItems(new String[] { "a", "b" }, Map.of(), "loop")); } diff --git a/test/prerna/reactor/automation/utils/AutomationRuntimeUtilsUnitTests.java b/test/prerna/reactor/automation/utils/AutomationRuntimeUtilsUnitTests.java index d6cd00ff13..3fcc2e6d36 100644 --- a/test/prerna/reactor/automation/utils/AutomationRuntimeUtilsUnitTests.java +++ b/test/prerna/reactor/automation/utils/AutomationRuntimeUtilsUnitTests.java @@ -126,6 +126,17 @@ void boundedJsonRejectsSelfReferencingValues() { () -> AutomationRuntimeUtils.toBoundedRuntimeJson(cyclic, 4096, "Automation run scope")); } + @Test + void boundedJsonAcceptsRepeatedAcyclicContainers() { + Map row = Map.of("id", 7); + Map value = new LinkedHashMap<>(); + value.put("rows", List.of(row)); + value.put("current", row); + + assertEquals("{\"rows\":[{\"id\":7}],\"current\":{\"id\":7}}", + AutomationRuntimeUtils.toBoundedRuntimeJson(value, 4096, "Automation run scope")); + } + @Test void boundedJsonRejectsValuesNestedTooDeeply() { List deepest = new ArrayList<>(); From a8bf5198e7fd23abcbd0e760ff3925911e798b34 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Mon, 5 Oct 2026 12:33:53 -0400 Subject: [PATCH 4/7] fix(automation): preserve storage paths for loop processing --- .../automation/definition/AutomationSourceRenderer.java | 9 ++++++++- .../definition/AutomationSourceRendererUnitTests.java | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java index 1a22f56888..2e496c7d30 100644 --- a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java +++ b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java @@ -352,7 +352,12 @@ def _normalized_suffixes(values): def run(scope): storage = StorageEngine(engine_id=scope.resolve(ENGINE_ID)) - paths = storage.list(scope.resolve(STORAGE_PATH)) + details = storage.listDetails(scope.resolve(STORAGE_PATH)) + paths = [ + item.get("Path") + for item in details + if isinstance(item, dict) and isinstance(item.get("Path"), str) + ] file_types = scope.resolve(FILE_TYPES) if not file_types: return paths @@ -482,6 +487,8 @@ def run(scope): localPath=destination, ) files = _destination_files(destination) + if not files: + raise RuntimeError("No file was downloaded from storage path: " + storage_path) return { "success": copied, "storagePath": storage_path, diff --git a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java index 5dd52696c0..cfa9728bb3 100644 --- a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java @@ -118,6 +118,8 @@ void storageDownloadReturnsTransferDetailsWithoutReimplementingPathRules() { assertTrue(source.contains("_pixel_value(\"options\", \"regex\")")); assertTrue(source.contains("\"files\": files")); assertTrue(source.contains("\"filePath\": files[0] if len(files) == 1 else None")); + assertTrue(source.contains("if not files:")); + assertTrue(source.contains("No file was downloaded from storage path")); assertTrue(source.contains("or \"/\"")); assertFalse(source.contains("pathlib")); assertFalse(source.contains(".replace(")); @@ -134,7 +136,8 @@ void storageListCanFilterFileTypesWithoutChangingTheStorageEngine() { String source = AutomationSourceRenderer .renderNode(node(AutomationConstants.NODE_STORAGE_LIST, config)); - assertTrue(source.contains("paths = storage.list(scope.resolve(STORAGE_PATH))")); + assertTrue(source.contains("details = storage.listDetails(scope.resolve(STORAGE_PATH))")); + assertTrue(source.contains("item.get(\"Path\")")); assertTrue(source.contains("path.rstrip(\"/\").lower().endswith(suffixes)")); assertTrue(source.contains("FILE_TYPES = [\"pdf\",\".png\"]")); } From 64a5829400d5cc25fe3019e56571052e33248cbd Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Mon, 5 Oct 2026 16:23:04 -0400 Subject: [PATCH 5/7] feat(automation): add data and browser workflow nodes --- py/semoss_automation_runtime.py | 252 ++++++++++++++++++ .../automation/AutomationConstants.java | 22 ++ .../AutomationDefinitionValidator.java | 68 +++++ .../definition/AutomationNodeCatalog.java | 58 ++++ .../definition/AutomationNodeType.java | 6 +- .../definition/AutomationSourceRenderer.java | 189 +++++++++++++ .../project/AutomationProjectService.java | 29 +- .../run/AutomationBrowserSessionReactor.java | 69 +++++ ...utomationDefinitionValidatorUnitTests.java | 50 ++++ .../AutomationNodeCatalogUnitTests.java | 34 +++ .../AutomationSourceRendererUnitTests.java | 59 +++- 11 files changed, 833 insertions(+), 3 deletions(-) create mode 100644 src/prerna/reactor/automation/run/AutomationBrowserSessionReactor.java diff --git a/py/semoss_automation_runtime.py b/py/semoss_automation_runtime.py index 4eda8ca24f..7022cd4666 100644 --- a/py/semoss_automation_runtime.py +++ b/py/semoss_automation_runtime.py @@ -10,10 +10,16 @@ import base64 import json import re +import xml.etree.ElementTree as ET +from collections.abc import Callable from typing import Any _PLACEHOLDER_PATTERN = re.compile(r"\$\{([^}]+)\}") +_DATA_PATH_TOKEN_PATTERN = re.compile( + r"([^.\[\]]+)|\[(\d+|\"(?:\\.|[^\"])*\"|'(?:\\.|[^'])*')\]" +) +_MISSING = object() class AutomationScope(dict[str, Any]): @@ -99,6 +105,252 @@ def resolve_config(self, value: Any) -> Any: return self.resolve(value) +def extract_data_element( + source: Any, + path: str, + source_format: str = "auto", + missing_value: Any = None, + null_value: Any = None, + asset_reader: Callable[[str], str] | None = None, +) -> Any: + """Return one nested JSON/XML value with distinct missing and null fallbacks.""" + data = _structured_data(source, source_format, asset_reader) + value = _nested_value(data, _data_path_tokens(path)) + if value is _MISSING: + return missing_value + if value is None: + return null_value + return value + + +def transform_records( + records: Any, + operation: str, + columns: Any = None, + mapping: Any = None, + column: Any = "", + operator: str = "equals", + value: Any = None, + descending: bool = False, +) -> list[dict[str, Any]]: + """Apply one business transformation to row-shaped Automation data.""" + if not isinstance(records, list) or not all( + isinstance(row, dict) for row in records + ): + raise ValueError("Transform data requires a list of row objects.") + + import pandas as pd + + frame = pd.DataFrame.from_records(records) + selected_columns = _column_names(columns) + mapping = mapping or {} + if not isinstance(mapping, dict): + raise ValueError("Transform data column values must be a JSON object.") + + if operation == "select": + _require_columns(frame, selected_columns) + frame = frame.loc[:, selected_columns] + elif operation == "remove": + _require_columns(frame, selected_columns) + frame = frame.drop(columns=selected_columns) + elif operation == "rename": + _require_columns(frame, list(mapping)) + frame = frame.rename(columns={str(key): str(item) for key, item in mapping.items()}) + elif operation == "filter": + column_name = str(column or "").strip() + _require_columns(frame, [column_name]) + frame = frame.loc[_comparison_mask(frame[column_name], operator, value)] + elif operation == "fillMissing": + _require_columns(frame, list(mapping)) + for name, replacement in mapping.items(): + frame[name] = frame[name].fillna(replacement) + elif operation == "sort": + _require_columns(frame, selected_columns) + frame = frame.sort_values(selected_columns, ascending=not descending) + elif operation == "deduplicate": + if selected_columns: + _require_columns(frame, selected_columns) + frame = frame.drop_duplicates(subset=selected_columns or None) + else: + raise ValueError(f"Unsupported data transformation: {operation}.") + + return json.loads(frame.to_json(orient="records", date_format="iso")) + + +def _column_names(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + raise ValueError("Transform data columns must be a list or comma-separated text.") + + +def _require_columns(frame: Any, columns: list[str]) -> None: + if not columns: + raise ValueError("Choose at least one column for this transformation.") + missing = [name for name in columns if name not in frame.columns] + if missing: + raise ValueError("Unknown data column(s): " + ", ".join(missing)) + + +def _comparison_mask(series: Any, operator: str, value: Any) -> Any: + if operator == "equals": + return series == value + if operator == "notEquals": + return series != value + if operator == "contains": + return series.astype("string").str.contains(str(value), case=False, na=False) + if operator == "greaterThan": + return series > value + if operator == "greaterThanOrEqual": + return series >= value + if operator == "lessThan": + return series < value + if operator == "lessThanOrEqual": + return series <= value + if operator == "isEmpty": + return series.isna() | series.astype("string").str.strip().eq("") + if operator == "isNotEmpty": + return ~(series.isna() | series.astype("string").str.strip().eq("")) + raise ValueError(f"Unsupported filter comparison: {operator}.") + + +def _structured_data( + source: Any, + source_format: str, + asset_reader: Callable[[str], str] | None, +) -> Any: + normalized_format = source_format.strip().lower() + if normalized_format not in {"auto", "json", "xml"}: + raise ValueError("Data format must be auto, json, or xml.") + if isinstance(source, (dict, list)): + if normalized_format == "xml": + raise ValueError("XML input must be text or a run-workspace file path.") + return source + if not isinstance(source, str) or not source.strip(): + raise ValueError("Data must be a JSON/XML value, encoded content, or file path.") + + candidates = [source] + decoded = _decoded_text(source) + if decoded is not None: + candidates.append(decoded) + + errors: list[Exception] = [] + for candidate in candidates: + try: + return _parse_structured_text(candidate, normalized_format) + except (json.JSONDecodeError, ET.ParseError, UnicodeError, ValueError) as error: + errors.append(error) + if asset_reader is not None: + try: + return _parse_structured_text(asset_reader(source), normalized_format) + except (json.JSONDecodeError, ET.ParseError, UnicodeError, ValueError) as error: + errors.append(error) + raise ValueError( + "Unable to read JSON or XML from the supplied value or run-workspace file." + ) from errors[-1] + + +def _parse_structured_text(value: str, source_format: str) -> Any: + text = value.lstrip("\ufeff\r\n\t ") + if source_format == "json" or source_format == "auto" and text[:1] in "[{": + return json.loads(text) + if source_format == "xml" or source_format == "auto" and text.startswith("<"): + root = ET.fromstring(text) + return {root.tag: _xml_value(root)} + if source_format == "auto": + try: + return json.loads(text) + except json.JSONDecodeError: + root = ET.fromstring(text) + return {root.tag: _xml_value(root)} + raise ValueError(f"Input is not valid {source_format.upper()}.") + + +def _decoded_text(value: str) -> str | None: + try: + return base64.b64decode(value, validate=True).decode("utf-8-sig") + except (ValueError, UnicodeError): + return None + + +def _xml_value(element: ET.Element) -> Any: + children = list(element) + attributes: dict[str, Any] = { + f"@{name}": value for name, value in element.attrib.items() + } + text = (element.text or "").strip() + if not children: + if not attributes: + return text or None + if text: + attributes["#text"] = text + return attributes + + result = attributes + for child in children: + value = _xml_value(child) + existing = result.get(child.tag, _MISSING) + if existing is _MISSING: + result[child.tag] = value + elif isinstance(existing, list): + existing.append(value) + else: + result[child.tag] = [existing, value] + if text: + result["#text"] = text + return result + + +def _data_path_tokens(path: str) -> list[str | int]: + value = path.strip() + if not value: + raise ValueError("Value path cannot be blank.") + if value.startswith("/"): + return [ + int(token) if token.isdigit() else token.replace("~1", "/").replace("~0", "~") + for token in value.split("/")[1:] + ] + if value == "$": + return [] + if value.startswith("$."): + value = value[2:] + + tokens: list[str | int] = [] + cursor = 0 + for match in _DATA_PATH_TOKEN_PATTERN.finditer(value): + separator = value[cursor : match.start()] + if separator not in {"", "."}: + raise ValueError(f"Invalid value path near '{separator}'.") + raw = match.group(1) or match.group(2) + if raw is None: + continue + if raw.isdigit(): + tokens.append(int(raw)) + elif raw[:1] in {"\"", "'"}: + tokens.append(json.loads(raw) if raw.startswith("\"") else raw[1:-1]) + else: + tokens.append(raw) + cursor = match.end() + if cursor != len(value) or not tokens: + raise ValueError("Value path is invalid.") + return tokens + + +def _nested_value(value: Any, tokens: list[str | int]) -> Any: + current = value + for token in tokens: + if isinstance(current, dict) and token in current: + current = current[token] + elif isinstance(current, list) and isinstance(token, int) and token < len(current): + current = current[token] + else: + return _MISSING + return current + + def execute_node( encoded_scope: str, encoded_source: str, diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 3a0eb2f154..c976801765 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -167,6 +167,9 @@ private AutomationConstants() { public static final String NODE_STORAGE_UPLOAD = "storage.upload"; public static final String NODE_STORAGE_DOWNLOAD = "storage.download"; public static final String NODE_STORAGE_DELETE = "storage.delete"; + public static final String NODE_DATA_EXTRACT = "data.extract"; + public static final String NODE_DATA_TRANSFORM = "data.transform"; + public static final String NODE_BROWSER_PLAYWRIGHT = "browser.playwright"; public static final String NODE_VECTOR_SEARCH = "vector.search"; public static final String NODE_VECTOR_ADD = "vector.add"; public static final String NODE_VECTOR_DELETE = "vector.delete"; @@ -196,6 +199,22 @@ private AutomationConstants() { public static final String CONFIG_PIXEL = "pixel"; public static final String CONFIG_GLOBALS = "globals"; public static final String CONFIG_DEFAULT_VALUE = "defaultValue"; + public static final String CONFIG_SOURCE = "source"; + public static final String CONFIG_PATH = "path"; + public static final String CONFIG_FORMAT = "format"; + public static final String CONFIG_MISSING_VALUE = "missingValue"; + public static final String CONFIG_NULL_VALUE = "nullValue"; + public static final String CONFIG_OPERATION = "operation"; + public static final String CONFIG_COLUMNS = "columns"; + public static final String CONFIG_MAPPING = "mapping"; + public static final String CONFIG_OPERATOR = "operator"; + public static final String CONFIG_VALUE = "value"; + public static final String CONFIG_DESCENDING = "descending"; + public static final String CONFIG_PROJECT_ID = "projectId"; + public static final String CONFIG_RECORDING_FILE = "recordingFile"; + public static final String CONFIG_INPUTS = "inputs"; + public static final String CONFIG_BROWSER_SUCCESS_URL_PREFIX = "successUrlPrefix"; + public static final String CONFIG_BROWSER_TIMEOUT_SECONDS = "timeoutSeconds"; public static final String CONFIG_DESCRIPTION = "description"; public static final String CONFIG_ANSWER = "answer"; public static final String JEV_QUESTION_TYPE_CHOICE = "choice"; @@ -231,6 +250,9 @@ private AutomationConstants() { public static final int DEFAULT_LIST_RUNS_LIMIT = 25; public static final int WAIT_MIN_SECONDS = 0; public static final int WAIT_MAX_SECONDS = 3600; + public static final int BROWSER_DEFAULT_TIMEOUT_SECONDS = 30; + public static final int BROWSER_MIN_TIMEOUT_SECONDS = 1; + public static final int BROWSER_MAX_TIMEOUT_SECONDS = 300; public static final int LOOP_MIN_BATCH_SIZE = 1; public static final int LOOP_MAX_BATCH_SIZE = 1_000; public static final int LOOP_DEFAULT_MAX_ITERATIONS = 1_000; diff --git a/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java b/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java index 7057b86799..aafe49bc9a 100644 --- a/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java +++ b/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java @@ -277,6 +277,17 @@ private static void validateNodeConfig(String nodeId, AutomationNodeType nodeTyp requireConfigString(nodeId, config, "path"); validateOptionalConfigString(nodeId, config, "version"); } + case DATA_EXTRACT -> validateDataExtractConfig(nodeId, config); + case DATA_TRANSFORM -> validateDataTransformConfig(nodeId, config); + case BROWSER_PLAYWRIGHT -> { + requireConfigString(nodeId, config, AutomationConstants.CONFIG_PROJECT_ID); + requireConfigString(nodeId, config, AutomationConstants.CONFIG_RECORDING_FILE); + validateOptionalConfigObject(nodeId, config, AutomationConstants.CONFIG_INPUTS); + requireConfigString(nodeId, config, AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX); + validateBoundedInteger(nodeId, config, AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, + AutomationConstants.BROWSER_MIN_TIMEOUT_SECONDS, + AutomationConstants.BROWSER_MAX_TIMEOUT_SECONDS); + } case VECTOR_SEARCH -> { requireConfigString(nodeId, config, "value"); validateOptionalConfigObject(nodeId, config, "filters"); @@ -311,6 +322,59 @@ private static void validateNodeConfig(String nodeId, AutomationNodeType nodeTyp } } + private static void validateDataExtractConfig(String nodeId, Map config) { + Object source = config.get(AutomationConstants.CONFIG_SOURCE); + if (source == null || source instanceof String string && string.isBlank()) { + throw new IllegalArgumentException( + "Node '" + nodeId + "' config.source must contain data, a scope reference, or a file path."); + } + requireConfigString(nodeId, config, AutomationConstants.CONFIG_PATH); + String format = config.get(AutomationConstants.CONFIG_FORMAT) instanceof String value ? value : "auto"; + if (!("auto".equals(format) || "json".equals(format) || "xml".equals(format))) { + throw new IllegalArgumentException( + "Node '" + nodeId + "' config.format must be auto, json, or xml."); + } + } + + private static void validateDataTransformConfig(String nodeId, Map config) { + Object source = config.get(AutomationConstants.CONFIG_SOURCE); + boolean scopeReference = source instanceof String value && isScopePlaceholder(value.trim()); + if (!(source instanceof List) && !scopeReference) { + throw new IllegalArgumentException("Node '" + nodeId + + "' config.source must be an array or an exact scope reference such as ${query_results}."); + } + String operation = requireNonblankString(config.get(AutomationConstants.CONFIG_OPERATION), + "Node '" + nodeId + "' config.operation"); + if (!Set.of("select", "remove", "rename", "filter", "fillMissing", "sort", "deduplicate") + .contains(operation)) { + throw new IllegalArgumentException("Node '" + nodeId + "' config.operation is unsupported."); + } + validateOptionalConfigObject(nodeId, config, AutomationConstants.CONFIG_MAPPING); + validateOptionalConfigBoolean(nodeId, config, AutomationConstants.CONFIG_DESCENDING); + validateOptionalStringListOrPlaceholder(nodeId, config, AutomationConstants.CONFIG_COLUMNS); + if (Set.of("select", "remove", "sort").contains(operation) + && !hasNonemptyConfigList(config, AutomationConstants.CONFIG_COLUMNS)) { + throw new IllegalArgumentException( + "Node '" + nodeId + "' config.columns must contain at least one column."); + } + if (Set.of("rename", "fillMissing").contains(operation)) { + Object mapping = config.get(AutomationConstants.CONFIG_MAPPING); + if (!(mapping instanceof Map values) || values.isEmpty()) { + throw new IllegalArgumentException( + "Node '" + nodeId + "' config.mapping must contain at least one column."); + } + } + if ("filter".equals(operation)) { + requireConfigString(nodeId, config, AutomationConstants.CONFIG_PATH); + String operator = requireNonblankString(config.get(AutomationConstants.CONFIG_OPERATOR), + "Node '" + nodeId + "' config.operator"); + if (!Set.of("equals", "notEquals", "contains", "greaterThan", "greaterThanOrEqual", "lessThan", + "lessThanOrEqual", "isEmpty", "isNotEmpty").contains(operator)) { + throw new IllegalArgumentException("Node '" + nodeId + "' config.operator is unsupported."); + } + } + } + private static boolean isRoutingNode(AutomationNodeType nodeType) { return nodeType == AutomationNodeType.CONTROL_IF || nodeType == AutomationNodeType.CONTROL_JEV; } @@ -915,6 +979,10 @@ private static void validateLoopBodies(List> nodes, Set bodyNodeTypes = validateNodes(bodyNodes, false); for (String bodyNodeId : bodyNodeTypes.keySet()) { diff --git a/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java b/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java index e9de3cf66f..953dbcb7af 100644 --- a/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java +++ b/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java @@ -158,6 +158,64 @@ private static List createDefinitions() { definitions.add(storageDefinition(AutomationNodeType.STORAGE_DELETE, "Delete file", "Delete a file from connected storage.", true, false)); + definitions.add(definition(AutomationNodeType.DATA_EXTRACT, "Extract value", + "Read one value from JSON, XML, or a file in this run's workspace.", + orderedConfig(AutomationConstants.CONFIG_SOURCE, "", AutomationConstants.CONFIG_PATH, "", + AutomationConstants.CONFIG_FORMAT, "auto", AutomationConstants.CONFIG_MISSING_VALUE, null, + AutomationConstants.CONFIG_NULL_VALUE, null), + List.of(field(AutomationConstants.CONFIG_SOURCE, ConfigFieldType.TEXT, "Data or file path", true, ""), + field(AutomationConstants.CONFIG_PATH, ConfigFieldType.STRING, "Value path", true, ""), + field(AutomationConstants.CONFIG_FORMAT, ConfigFieldType.STRING, "Data format", true, "auto"), + field(AutomationConstants.CONFIG_MISSING_VALUE, ConfigFieldType.JSON, + "If the path is missing", false, null), + field(AutomationConstants.CONFIG_NULL_VALUE, ConfigFieldType.JSON, "If the value is null", false, + null)), + controlInputs(), controlAndResultOutputs())); + definitions.add(definition(AutomationNodeType.DATA_TRANSFORM, "Transform data", + "Clean or reshape rows from an earlier step without writing Python.", + orderedConfig(AutomationConstants.CONFIG_SOURCE, List.of(), AutomationConstants.CONFIG_OPERATION, + "select", AutomationConstants.CONFIG_COLUMNS, List.of(), AutomationConstants.CONFIG_MAPPING, + Map.of(), AutomationConstants.CONFIG_PATH, "", AutomationConstants.CONFIG_OPERATOR, "equals", + AutomationConstants.CONFIG_VALUE, null, AutomationConstants.CONFIG_DESCENDING, false), + List.of(field(AutomationConstants.CONFIG_SOURCE, ConfigFieldType.JSON, "Rows to transform", true, + List.of()), + field(AutomationConstants.CONFIG_OPERATION, ConfigFieldType.STRING, "Transformation", true, + "select"), + field(AutomationConstants.CONFIG_COLUMNS, ConfigFieldType.STRING_LIST, "Columns", false, + List.of()), + field(AutomationConstants.CONFIG_MAPPING, ConfigFieldType.JSON, "Column values", false, Map.of()), + field(AutomationConstants.CONFIG_PATH, ConfigFieldType.STRING, "Column", false, ""), + field(AutomationConstants.CONFIG_OPERATOR, ConfigFieldType.STRING, "Comparison", false, "equals"), + field(AutomationConstants.CONFIG_VALUE, ConfigFieldType.JSON, "Comparison value", false, null), + field(AutomationConstants.CONFIG_DESCENDING, ConfigFieldType.BOOLEAN, "Descending", false, false)), + controlInputs(), controlAndResultOutputs())); + + definitions.add(definition(AutomationNodeType.BROWSER_PLAYWRIGHT, "Run browser recording", + "Replay a saved Playwright recording and verify its final page.", + orderedConfig(AutomationConstants.CONFIG_PROJECT_ID, "", AutomationConstants.CONFIG_RECORDING_FILE, "", + AutomationConstants.CONFIG_INPUTS, Map.of(), + AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "", + AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, + AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS), + List.of(field(AutomationConstants.CONFIG_PROJECT_ID, ConfigFieldType.STRING, "Recording app", true, ""), + field(AutomationConstants.CONFIG_RECORDING_FILE, ConfigFieldType.STRING, "Recording", true, ""), + field(AutomationConstants.CONFIG_INPUTS, ConfigFieldType.JSON, "Recording inputs", false, Map.of()), + field(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, ConfigFieldType.STRING, + "Successful page starts with", true, ""), + boundedIntegerField(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, + "Wait for success (seconds)", AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS, + AutomationConstants.BROWSER_MIN_TIMEOUT_SECONDS, + AutomationConstants.BROWSER_MAX_TIMEOUT_SECONDS)), + List.of(outputField("status", OutputFieldType.STRING, "Status", + "Completed after the expected page became idle.", true), + outputField("recordingFile", OutputFieldType.STRING, "Recording", "Saved recording that ran.", + true), + outputField("passes", OutputFieldType.NUMBER, "Replay passes", + "Number of replay passes used.", true), + outputField("finalUrl", OutputFieldType.STRING, "Final page", + "Final page address without query parameters or fragments.", true)), + controlInputs(), controlAndResultOutputs())); + definitions.add(vectorDefinition(AutomationNodeType.VECTOR_SEARCH, "Search vectors", "Search records in a vector database.", true)); definitions.add(vectorDefinition(AutomationNodeType.VECTOR_ADD, "Add vector documents", diff --git a/src/prerna/reactor/automation/definition/AutomationNodeType.java b/src/prerna/reactor/automation/definition/AutomationNodeType.java index c5c9dc41b0..260040ee4a 100644 --- a/src/prerna/reactor/automation/definition/AutomationNodeType.java +++ b/src/prerna/reactor/automation/definition/AutomationNodeType.java @@ -71,6 +71,10 @@ public enum AutomationNodeType { Permission.VIEW, true, true), STORAGE_DELETE(AutomationConstants.NODE_STORAGE_DELETE, Category.STORAGE, IEngine.CATALOG_TYPE.STORAGE, Permission.EDIT, true, true), + DATA_EXTRACT(AutomationConstants.NODE_DATA_EXTRACT, Category.DATA, null, Permission.NONE, true, false), + DATA_TRANSFORM(AutomationConstants.NODE_DATA_TRANSFORM, Category.DATA, null, Permission.NONE, true, false), + BROWSER_PLAYWRIGHT(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, Category.BROWSER, null, Permission.VIEW, true, + false), VECTOR_SEARCH(AutomationConstants.NODE_VECTOR_SEARCH, Category.VECTOR, IEngine.CATALOG_TYPE.VECTOR, Permission.VIEW, true, true), VECTOR_ADD(AutomationConstants.NODE_VECTOR_ADD, Category.VECTOR, IEngine.CATALOG_TYPE.VECTOR, Permission.EDIT, true, @@ -93,7 +97,7 @@ public enum AutomationNodeType { * Logical grouping used by clients without parsing the persisted type string. */ public enum Category { - TRIGGER, DATABASE, MODEL, STORAGE, VECTOR, FUNCTION, APP, AGENT, CONTROL, DEVELOPER + TRIGGER, DATABASE, MODEL, STORAGE, DATA, BROWSER, VECTOR, FUNCTION, APP, AGENT, CONTROL, DEVELOPER } /** Minimum resource permission required by the node when it references one. */ diff --git a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java index 2e496c7d30..80e6955896 100644 --- a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java +++ b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java @@ -79,6 +79,9 @@ public static String renderNode(Map node) { case STORAGE_UPLOAD -> storageUploadSource(config); case STORAGE_DOWNLOAD -> storageDownloadSource(config); case STORAGE_DELETE -> storageDeleteSource(config); + case DATA_EXTRACT -> dataExtractSource(config); + case DATA_TRANSFORM -> dataTransformSource(config); + case BROWSER_PLAYWRIGHT -> browserPlaywrightSource(config); case VECTOR_SEARCH -> vectorSearchSource(config); case VECTOR_ADD -> vectorAddSource(config); case VECTOR_DELETE -> vectorDeleteSource(config); @@ -501,6 +504,192 @@ raise RuntimeError("No file was downloaded from storage path: " + storage_path) valueOrDefault(config, "version", "")); } + private static String dataExtractSource(Map config) { + return """ + # Extract a nested value from inline JSON/XML or a file in this run's Insight workspace. + from semoss import Insight + from semoss_automation_runtime import extract_data_element + import base64 + import json + + SOURCE = %s + PATH = %s + FORMAT = %s + MISSING_VALUE = %s + NULL_VALUE = %s + + def _pixel_value(name, value): + return name + "=[" + json.dumps(value) + "]" + + def _read_insight_asset(file_path): + pixel = "GetInsightAssetsBase64(" + _pixel_value("filePath", file_path) + ");" + encoded = Insight().run_pixel(pixel, raw=False) + return base64.b64decode(encoded).decode("utf-8-sig") + + def run(scope): + return extract_data_element( + source=scope.resolve(SOURCE), + path=str(scope.resolve(PATH)), + source_format=str(scope.resolve(FORMAT) or "auto"), + missing_value=scope.resolve(MISSING_VALUE), + null_value=scope.resolve(NULL_VALUE), + asset_reader=_read_insight_asset, + ) + """.formatted(value(config, AutomationConstants.CONFIG_SOURCE), + value(config, AutomationConstants.CONFIG_PATH), + valueOrDefault(config, AutomationConstants.CONFIG_FORMAT, "auto"), + value(config, AutomationConstants.CONFIG_MISSING_VALUE), + value(config, AutomationConstants.CONFIG_NULL_VALUE)); + } + + private static String dataTransformSource(Map config) { + return """ + # Apply a bounded business transformation to row-shaped Automation data. + from semoss_automation_runtime import transform_records + + SOURCE = %s + OPERATION = %s + COLUMNS = %s + MAPPING = %s + COLUMN = %s + OPERATOR = %s + VALUE = %s + DESCENDING = %s + + def run(scope): + return transform_records( + records=scope.resolve(SOURCE), + operation=OPERATION, + columns=scope.resolve(COLUMNS), + mapping=scope.resolve_config(MAPPING), + column=scope.resolve(COLUMN), + operator=OPERATOR, + value=scope.resolve(VALUE), + descending=bool(DESCENDING), + ) + """.formatted(value(config, AutomationConstants.CONFIG_SOURCE), + value(config, AutomationConstants.CONFIG_OPERATION), + valueOrDefault(config, AutomationConstants.CONFIG_COLUMNS, List.of()), + valueOrDefault(config, AutomationConstants.CONFIG_MAPPING, Map.of()), + valueOrDefault(config, AutomationConstants.CONFIG_PATH, ""), + valueOrDefault(config, AutomationConstants.CONFIG_OPERATOR, "equals"), + value(config, AutomationConstants.CONFIG_VALUE), + valueOrDefault(config, AutomationConstants.CONFIG_DESCENDING, false)); + } + + private static String browserPlaywrightSource(Map config) { + return """ + # Replay a saved SEMOSS Playwright recording in this Automation run. + from semoss import Insight + import json + import time + from urllib.parse import urlsplit + + PROJECT_ID = %s + RECORDING_FILE = %s + INPUTS = %s + SUCCESS_URL_PREFIX = %s + TIMEOUT_SECONDS = %s + MAX_PASSES = 100 + POLL_SECONDS = 0.25 + + def _pixel_value(name, value): + return name + "=[" + json.dumps(value) + "]" + + def _safe_url(value): + parts = urlsplit(str(value or "")) + if parts.scheme and parts.netloc: + return parts.scheme + "://" + parts.netloc + parts.path + return str(value or "").split("?", 1)[0].split("#", 1)[0] + + def _browser_state(session_id, tab_id): + pixel = "CheckNetworkIdle(" + ", ".join([ + _pixel_value("sessionId", session_id), + _pixel_value("tabId", tab_id), + _pixel_value("quietMillis", 500), + ]) + ");" + response = Insight().run_pixel(pixel, raw=True) + result = response[0]["pixelReturn"][-1] + if "ERROR" in result.get("operationType", []): + raise RuntimeError(result.get("output") or "Unable to inspect the browser result") + state = result.get("output") + if not isinstance(state, dict): + raise RuntimeError("Browser result did not include page state") + return state + + def _wait_for_success(session_id, tab_id, expected_prefix, timeout_seconds): + deadline = time.monotonic() + timeout_seconds + final_url = "" + while True: + state = _browser_state(session_id, tab_id) + final_url = _safe_url(state.get("currentUrl")) + if state.get("isNetworkIdle") is True and final_url.startswith(expected_prefix): + return final_url + if time.monotonic() >= deadline: + raise RuntimeError( + "Browser recording did not reach the expected page. " + + "Expected a page starting with " + expected_prefix + + "; final page was " + (final_url or "unavailable") + ) + time.sleep(POLL_SECONDS) + + def run(scope): + session_response = Insight().run_pixel("AutomationBrowserSession();", raw=True) + session_result = session_response[0]["pixelReturn"][-1] + if "ERROR" in session_result.get("operationType", []): + raise RuntimeError(session_result.get("output") or "Unable to start browser session") + session_id = session_result.get("output") + if not isinstance(session_id, str) or not session_id: + raise RuntimeError("Browser session did not return an id") + project_id = scope.resolve(PROJECT_ID) + recording_file = scope.resolve(RECORDING_FILE) + inputs = scope.resolve_config(INPUTS) + success_url_prefix = _safe_url(scope.resolve(SUCCESS_URL_PREFIX)) + timeout_seconds = int(TIMEOUT_SECONDS) + latest = {} + tab_id = "tab-1" + tab_stack = [] + for pass_number in range(1, MAX_PASSES + 1): + pixel = "ReplayStep(" + ", ".join([ + _pixel_value("sessionId", session_id), + _pixel_value("fileName", recording_file), + "paramValues=[" + json.dumps(inputs) + "]", + "executeAll=[true]", + _pixel_value("tabId", tab_id), + _pixel_value("project", project_id), + ]) + ");" + response = Insight().run_pixel(pixel, raw=True) + result = response[0]["pixelReturn"][-1] + if "ERROR" in result.get("operationType", []): + raise RuntimeError(result.get("output") or "Browser recording failed") + latest = result.get("output") or {} + new_tab_id = latest.get("newTabId") + if isinstance(new_tab_id, str) and new_tab_id and new_tab_id != tab_id: + tab_stack.append(tab_id) + tab_id = new_tab_id + continue + if latest.get("isLastPage") is True: + if tab_stack: + tab_id = tab_stack.pop() + continue + final_url = _wait_for_success( + session_id, tab_id, success_url_prefix, timeout_seconds + ) + return { + "status": "completed", + "recordingFile": recording_file, + "passes": pass_number, + "finalUrl": final_url, + } + raise RuntimeError("Browser recording exceeded the maximum replay passes.") + """.formatted(value(config, AutomationConstants.CONFIG_PROJECT_ID), + value(config, AutomationConstants.CONFIG_RECORDING_FILE), + valueOrDefault(config, AutomationConstants.CONFIG_INPUTS, Map.of()), + value(config, AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX), + valueOrDefault(config, AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, + AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS)); + } + private static String vectorSearchSource(Map config) { return """ # Search a SEMOSS vector engine directly through the Python SDK. diff --git a/src/prerna/reactor/automation/project/AutomationProjectService.java b/src/prerna/reactor/automation/project/AutomationProjectService.java index 2ebc380e55..1d8f34ebe6 100644 --- a/src/prerna/reactor/automation/project/AutomationProjectService.java +++ b/src/prerna/reactor/automation/project/AutomationProjectService.java @@ -184,7 +184,9 @@ public static AutomationDefinitionService.DefinitionFiles saveDefinition(String definitionJson, nodeSources); AutomationMcpSync.sync(projectId, files.definition(), user); syncDefinitionAssets(projectId, user, "Update automation definition"); - return files; + // Return the same normalized aggregate a subsequent GetAutomation call will + // read. This keeps the optimistic-concurrency token stable after Save. + return AutomationDefinitionService.load(projectId); } finally { projectLock.unlock(); } @@ -238,6 +240,8 @@ public static void validateDefinitionReferences(AutomationDefinitionValidator.Va validateAgentWorkspaceReference(node, config, user); } else if (typedNode == AutomationNodeType.APP_PIXEL) { validateAppProjectReference(node, config, user); + } else if (typedNode == AutomationNodeType.BROWSER_PLAYWRIGHT) { + validateBrowserProjectReference(node, config, user); } IEngine.CATALOG_TYPE expectedType = typedNode.getEngineType(); @@ -330,12 +334,35 @@ private static void validateAppProjectReference(Map node, Map node, Map config, + User user) { + String nodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); + Object rawProjectId = config.get(AutomationConstants.CONFIG_PROJECT_ID); + if (!(rawProjectId instanceof String projectId) || projectId.isBlank()) { + throw invalidBrowserProjectReference(nodeId, String.valueOf(rawProjectId)); + } + projectId = projectId.trim(); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw invalidBrowserProjectReference(nodeId, projectId); + } + String projectType = SecurityProjectUtils.getProjectTypeForId(projectId); + if (!(IProject.PROJECT_TYPE.CODE.name().equals(projectType) + || IProject.PROJECT_TYPE.BLOCKS.name().equals(projectType))) { + throw invalidBrowserProjectReference(nodeId, projectId); + } + } + private static IllegalArgumentException invalidAppReference(String nodeId, String appId) { return new IllegalArgumentException("Automation node '" + nodeId + "' appId '" + appId + "' is not an accessible CODE or BLOCKS app. Call MyProjects with " + "projectType=['CODE','BLOCKS'] and use its project_id value."); } + private static IllegalArgumentException invalidBrowserProjectReference(String nodeId, String projectId) { + return new IllegalArgumentException("Automation browser node '" + nodeId + "' projectId '" + projectId + + "' is not an accessible CODE or BLOCKS app with a Playwright recording."); + } + private static IllegalArgumentException invalidEngineReference(String nodeId, String engineId, IEngine.CATALOG_TYPE expectedType) { return new IllegalArgumentException("Automation node '" + nodeId + "' engineId '" + engineId diff --git a/src/prerna/reactor/automation/run/AutomationBrowserSessionReactor.java b/src/prerna/reactor/automation/run/AutomationBrowserSessionReactor.java new file mode 100644 index 0000000000..98d3270699 --- /dev/null +++ b/src/prerna/reactor/automation/run/AutomationBrowserSessionReactor.java @@ -0,0 +1,69 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.run; + +import com.microsoft.playwright.Page; + +import prerna.reactor.AbstractReactor; +import prerna.reactor.playwright.PlaywrightSession; +import prerna.remoteviewer.service.RemoteBrowserSession; +import prerna.remoteviewer.service.RemoteBrowserSessionManager; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Prepares the authenticated user's existing remote browser infrastructure for + * one top-level Automation browser step. + */ +public class AutomationBrowserSessionReactor extends AbstractReactor { + + private static final int VIEWPORT_WIDTH = 1280; + private static final int VIEWPORT_HEIGHT = 800; + + @Override + public NounMetadata execute() { + RemoteBrowserSession session = RemoteBrowserSessionManager.getInstance() + .createSession(this.insight.getUser(), "", VIEWPORT_WIDTH, VIEWPORT_HEIGHT); + PlaywrightSession playwrightSession = session.getPlaywrightSession(); + playwrightSession.getOperationLock().lock(); + try { + Page page = playwrightSession.resetPagesForNewViewer(); + page.setViewportSize(VIEWPORT_WIDTH, VIEWPORT_HEIGHT); + session.setActiveTabId("tab-1"); + session.clearRecordingBuffer(); + } finally { + playwrightSession.getOperationLock().unlock(); + } + return new NounMetadata(session.getSessionId(), PixelDataType.CONST_STRING); + } + + @Override + public String getReactorDescription() { + return "Prepares a remote-viewable browser session for a top-level Automation browser step."; + } +} diff --git a/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java b/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java index 442b6147a5..8f8e4e0f76 100644 --- a/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java @@ -382,6 +382,56 @@ void acceptsABoundedLoopWithAnAcyclicBody() { AutomationDefinitionValidator.parseAndValidate(definition(Map.of(), loop)); } + @Test + void rejectsBrowserAutomationInsideALoop() { + Map browserNode = workNode(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, + Map.of(AutomationConstants.CONFIG_PROJECT_ID, "recording-project", + AutomationConstants.CONFIG_RECORDING_FILE, "search.json", + AutomationConstants.CONFIG_INPUTS, Map.of(), + AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "https://example.com/results", + AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, + AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS)); + browserNode.put(AutomationConstants.NODE_FIELD_ID, "body-browser"); + browserNode.put(AutomationConstants.NODE_FIELD_OUTPUT_VAR, "browser_result"); + Map loop = workNode(AutomationConstants.NODE_CONTROL_LOOP, + Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_FOR_EACH, + AutomationConstants.CONFIG_LOOP_ITEMS, List.of("SEMOSS"), + AutomationConstants.CONFIG_LOOP_BATCH_SIZE, 1, + AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); + loop.put(AutomationConstants.NODE_FIELD_BODY, + Map.of(AutomationConstants.DOC_NODES, List.of(browserNode), + AutomationConstants.DOC_EDGES, List.of())); + + assertThrows(IllegalArgumentException.class, + () -> AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), loop))); + } + + @Test + void browserAutomationRequiresABoundedSuccessCondition() { + Map config = new LinkedHashMap<>(); + config.put(AutomationConstants.CONFIG_PROJECT_ID, "recording-project"); + config.put(AutomationConstants.CONFIG_RECORDING_FILE, "search.json"); + config.put(AutomationConstants.CONFIG_INPUTS, Map.of()); + config.put(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "https://example.com/results"); + config.put(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, + AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS); + Map browser = workNode(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, config); + + AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), browser)); + + config.put(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, ""); + assertThrows(IllegalArgumentException.class, + () -> AutomationDefinitionValidator.parseAndValidateForAuthoring( + definition(Map.of(), workNode(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, config)))); + + config.put(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "https://example.com/results"); + config.put(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, + AutomationConstants.BROWSER_MAX_TIMEOUT_SECONDS + 1); + assertThrows(IllegalArgumentException.class, + () -> AutomationDefinitionValidator.parseAndValidateForAuthoring( + definition(Map.of(), workNode(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, config)))); + } + @Test void acceptsDottedForEachRepeatAndWhileLoopConfiguration() { Map dottedForEach = workNode(AutomationConstants.NODE_CONTROL_LOOP, diff --git a/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java b/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java index 654890213c..ca70d34c87 100644 --- a/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java @@ -117,6 +117,40 @@ void storageListAdvertisesOptionalFileTypes() { assertEquals(List.of(), definition.defaultConfig().get("extensions")); } + @Test + void dataExtractAdvertisesOneGenericJsonXmlContract() { + AutomationNodeDefinition definition = find(AutomationConstants.NODE_DATA_EXTRACT); + assertNotNull(definition); + assertEquals("data", definition.toMap().get("category")); + Map> fieldsByKey = definition.configFields().stream() + .collect(java.util.stream.Collectors.toMap(AutomationNodeDefinition.ConfigField::key, + AutomationNodeDefinition.ConfigField::toMap)); + assertEquals("textarea", fieldsByKey.get(AutomationConstants.CONFIG_SOURCE).get("type")); + assertEquals("string", fieldsByKey.get(AutomationConstants.CONFIG_PATH).get("type")); + assertEquals("auto", definition.defaultConfig().get(AutomationConstants.CONFIG_FORMAT)); + assertEquals(false, definition.nodeType().supportsCustomCode()); + } + + @Test + void dataTransformAndBrowserRecordingAreBackendOwned() { + AutomationNodeDefinition transform = find(AutomationConstants.NODE_DATA_TRANSFORM); + AutomationNodeDefinition browser = find(AutomationConstants.NODE_BROWSER_PLAYWRIGHT); + assertNotNull(transform); + assertNotNull(browser); + assertEquals("data", transform.toMap().get("category")); + assertEquals("browser", browser.toMap().get("category")); + assertEquals("VIEW", browser.toMap().get("requiredPermission")); + assertEquals(false, transform.nodeType().supportsCustomCode()); + assertEquals(false, browser.nodeType().supportsCustomCode()); + assertEquals(Map.of(), browser.defaultConfig().get(AutomationConstants.CONFIG_INPUTS)); + assertEquals("", browser.defaultConfig().get(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX)); + assertEquals(AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS, + browser.defaultConfig().get(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS)); + assertEquals(Set.of("status", "recordingFile", "passes", "finalUrl"), + browser.outputFields().stream().map(AutomationNodeDefinition.OutputField::key) + .collect(java.util.stream.Collectors.toSet())); + } + @Test void optionalEngineParametersAreBackendOwned() { Map> uploadFields = find(AutomationConstants.NODE_STORAGE_UPLOAD).configFields() diff --git a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java index cfa9728bb3..92f5ff7a5e 100644 --- a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java @@ -142,6 +142,62 @@ void storageListCanFilterFileTypesWithoutChangingTheStorageEngine() { assertTrue(source.contains("FILE_TYPES = [\"pdf\",\".png\"]")); } + @Test + void dataExtractReadsOnlyFromTheCurrentInsightWorkspace() { + Map config = new LinkedHashMap<>(); + config.put(AutomationConstants.CONFIG_SOURCE, "${download.filePath}"); + config.put(AutomationConstants.CONFIG_PATH, "orders[0].id"); + config.put(AutomationConstants.CONFIG_FORMAT, "json"); + config.put(AutomationConstants.CONFIG_MISSING_VALUE, "missing"); + config.put(AutomationConstants.CONFIG_NULL_VALUE, "null"); + + String source = AutomationSourceRenderer.renderNode(node(AutomationConstants.NODE_DATA_EXTRACT, config)); + + assertTrue(source.contains("GetInsightAssetsBase64(")); + assertTrue(source.contains("extract_data_element(")); + assertTrue(source.contains("source=scope.resolve(SOURCE)")); + assertFalse(source.contains("open(")); + } + + @Test + void dataTransformUsesTheSharedAutomationRuntime() { + Map config = new LinkedHashMap<>(); + config.put(AutomationConstants.CONFIG_SOURCE, "${query_rows}"); + config.put(AutomationConstants.CONFIG_OPERATION, "filter"); + config.put(AutomationConstants.CONFIG_COLUMNS, java.util.List.of()); + config.put(AutomationConstants.CONFIG_MAPPING, Map.of()); + config.put(AutomationConstants.CONFIG_PATH, "status"); + config.put(AutomationConstants.CONFIG_OPERATOR, "equals"); + config.put(AutomationConstants.CONFIG_VALUE, "active"); + config.put(AutomationConstants.CONFIG_DESCENDING, false); + + String source = AutomationSourceRenderer.renderNode(node(AutomationConstants.NODE_DATA_TRANSFORM, config)); + + assertTrue(source.contains("from semoss_automation_runtime import transform_records")); + assertTrue(source.contains("records=scope.resolve(SOURCE)")); + } + + @Test + void browserRecordingUsesAutomationOwnedBrowserSession() { + Map config = new LinkedHashMap<>(); + config.put(AutomationConstants.CONFIG_PROJECT_ID, "project-1"); + config.put(AutomationConstants.CONFIG_RECORDING_FILE, "intake.json"); + config.put(AutomationConstants.CONFIG_INPUTS, Map.of("Order number", "123")); + config.put(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "https://example.com/complete"); + config.put(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, 45); + + String source = AutomationSourceRenderer + .renderNode(node(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, config)); + + assertTrue(source.contains("run_pixel(\"AutomationBrowserSession();\", raw=True)")); + assertTrue(source.contains("ReplayStep(")); + assertTrue(source.contains("CheckNetworkIdle(")); + assertTrue(source.contains("final_url.startswith(expected_prefix)")); + assertTrue(source.contains("\"finalUrl\": final_url")); + assertTrue(source.contains("_pixel_value(\"project\", project_id)")); + assertFalse(source.contains("Playwright.create")); + } + @Test void visionAcceptsOneOrManyMediaPathsWithoutNestingThem() { Map config = new LinkedHashMap<>(); @@ -215,7 +271,8 @@ void customPythonStartsWithAnEditableRunScopeTemplate() { @Test void everyNodeSourceResolvesThroughScope() { for (AutomationNodeType type : AutomationNodeType.values()) { - if (type == AutomationNodeType.CONTROL_IF || type == AutomationNodeType.CONTROL_JEV) { + if (type == AutomationNodeType.CONTROL_IF || type == AutomationNodeType.CONTROL_JEV + || type == AutomationNodeType.CONTROL_LOOP) { continue; } String source = AutomationSourceRenderer.renderNode(node(type.getType(), databaseConfig())); From 976746a5f9cd24f0904ad02b4234970000686a8d Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Wed, 7 Oct 2026 10:52:04 -0400 Subject: [PATCH 6/7] refactor(automation): remove deferred browser and transform nodes --- .../automation-loop-phase-1-2026-09-25.md | 63 -------- py/semoss_automation_runtime.py | 94 ----------- .../automation/AutomationConstants.java | 12 -- .../AutomationDefinitionService.java | 2 +- .../AutomationDefinitionValidator.java | 55 +------ .../definition/AutomationNodeCatalog.java | 45 ------ .../definition/AutomationNodeType.java | 5 +- .../definition/AutomationSourceRenderer.java | 152 +----------------- .../project/AutomationProjectService.java | 25 --- .../run/AutomationBrowserSessionReactor.java | 69 -------- ...utomationDefinitionValidatorUnitTests.java | 50 ------ .../AutomationNodeCatalogUnitTests.java | 20 --- .../AutomationSourceRendererUnitTests.java | 39 ----- 13 files changed, 4 insertions(+), 627 deletions(-) delete mode 100644 outputs/review/automation-loop-phase-1-2026-09-25.md delete mode 100644 src/prerna/reactor/automation/run/AutomationBrowserSessionReactor.java diff --git a/outputs/review/automation-loop-phase-1-2026-09-25.md b/outputs/review/automation-loop-phase-1-2026-09-25.md deleted file mode 100644 index 6c81c68486..0000000000 --- a/outputs/review/automation-loop-phase-1-2026-09-25.md +++ /dev/null @@ -1,63 +0,0 @@ -# Automation loop phase 1 - -## Status - -This work is parked for later review on dedicated backend and frontend branches. It is not intended to ship until the execution contract and UX are validated with representative automations. - -## Current approach - -- A loop is a first-class `control.loop` container in the parent automation graph. -- The parent graph stays acyclic. Iteration is owned by the Java execution service rather than represented with a circular control edge. -- The loop owns a nested acyclic body graph with ordinary automation nodes and control edges. -- The body has one entry node and may contain decisions and multiple routes. -- Phase 1 executes iterations sequentially with explicit maximum-iteration and body-size limits. -- Each iteration receives an isolated copy of the parent scope plus a loop context containing its index, total, first/last flags, current item when singular, and current batch. -- Body executions receive durable child history rows tied to the loop node and iteration index. -- Cancellation is checked between iterations and between body nodes. -- Nested loops and durable agent input waits inside a loop are intentionally unsupported in phase 1. - -## Current authoring UX - -- The loop appears as a distinct container card on the main automation canvas. -- Its collapsed state summarizes the repeated steps. -- Its expanded state contains a nested mini-canvas for the loop body. -- Users can insert steps at a route, edit body nodes, and use decisions inside the body. -- Expanding the loop increases its visual width and updates the surrounding React Flow node internals. - -## Deliberately unresolved - -### Input consumption - -Batching must not become the only loop contract. The current implementation supports `batchSize`, but this should be treated as an optional execution strategy while the following are tested: - -1. one item per iteration; -2. fixed-size batches; -3. a server-owned cursor or paged data reference; -4. streaming/provider-backed iteration for large tabular results. - -The final contract should let the user express the intent to repeat work without requiring them to understand frames, cursors, or backend storage. - -### Additional loop types - -The container/body design is intended to support later modes without circular parent graphs: - -- `forEach` for a collection; -- counted `for` ranges; -- bounded `while` conditions; -- paged or cursor-driven data iteration. - -Every mode must remain bounded, cancellable, observable, and resumable before it is enabled. - -## Before resuming implementation - -- Decide the canonical loop-input contract after testing lists, database results, and large provider-backed data. -- Define retry, resume, and partial-failure behavior at the iteration level. -- Confirm whether collected results are inline, summarized, or stored as run-owned data references. -- Validate branch convergence and output availability inside the nested body. -- Improve expanded-container layout so neighboring parent nodes never overlap. -- Test history rendering for parent loop rows and per-iteration child rows. -- Add end-to-end coverage for cancellation, limits, branching, and empty inputs. - -## Scope boundaries - -The parked work does not introduce circular parent-graph edges, parallel iteration, nested loops, agent approval waits, or a requirement that all loops consume batches. diff --git a/py/semoss_automation_runtime.py b/py/semoss_automation_runtime.py index 7022cd4666..d4a737d4fb 100644 --- a/py/semoss_automation_runtime.py +++ b/py/semoss_automation_runtime.py @@ -123,100 +123,6 @@ def extract_data_element( return value -def transform_records( - records: Any, - operation: str, - columns: Any = None, - mapping: Any = None, - column: Any = "", - operator: str = "equals", - value: Any = None, - descending: bool = False, -) -> list[dict[str, Any]]: - """Apply one business transformation to row-shaped Automation data.""" - if not isinstance(records, list) or not all( - isinstance(row, dict) for row in records - ): - raise ValueError("Transform data requires a list of row objects.") - - import pandas as pd - - frame = pd.DataFrame.from_records(records) - selected_columns = _column_names(columns) - mapping = mapping or {} - if not isinstance(mapping, dict): - raise ValueError("Transform data column values must be a JSON object.") - - if operation == "select": - _require_columns(frame, selected_columns) - frame = frame.loc[:, selected_columns] - elif operation == "remove": - _require_columns(frame, selected_columns) - frame = frame.drop(columns=selected_columns) - elif operation == "rename": - _require_columns(frame, list(mapping)) - frame = frame.rename(columns={str(key): str(item) for key, item in mapping.items()}) - elif operation == "filter": - column_name = str(column or "").strip() - _require_columns(frame, [column_name]) - frame = frame.loc[_comparison_mask(frame[column_name], operator, value)] - elif operation == "fillMissing": - _require_columns(frame, list(mapping)) - for name, replacement in mapping.items(): - frame[name] = frame[name].fillna(replacement) - elif operation == "sort": - _require_columns(frame, selected_columns) - frame = frame.sort_values(selected_columns, ascending=not descending) - elif operation == "deduplicate": - if selected_columns: - _require_columns(frame, selected_columns) - frame = frame.drop_duplicates(subset=selected_columns or None) - else: - raise ValueError(f"Unsupported data transformation: {operation}.") - - return json.loads(frame.to_json(orient="records", date_format="iso")) - - -def _column_names(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(item).strip() for item in value if str(item).strip()] - if isinstance(value, str): - return [item.strip() for item in value.split(",") if item.strip()] - raise ValueError("Transform data columns must be a list or comma-separated text.") - - -def _require_columns(frame: Any, columns: list[str]) -> None: - if not columns: - raise ValueError("Choose at least one column for this transformation.") - missing = [name for name in columns if name not in frame.columns] - if missing: - raise ValueError("Unknown data column(s): " + ", ".join(missing)) - - -def _comparison_mask(series: Any, operator: str, value: Any) -> Any: - if operator == "equals": - return series == value - if operator == "notEquals": - return series != value - if operator == "contains": - return series.astype("string").str.contains(str(value), case=False, na=False) - if operator == "greaterThan": - return series > value - if operator == "greaterThanOrEqual": - return series >= value - if operator == "lessThan": - return series < value - if operator == "lessThanOrEqual": - return series <= value - if operator == "isEmpty": - return series.isna() | series.astype("string").str.strip().eq("") - if operator == "isNotEmpty": - return ~(series.isna() | series.astype("string").str.strip().eq("")) - raise ValueError(f"Unsupported filter comparison: {operator}.") - - def _structured_data( source: Any, source_format: str, diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index c976801765..1d38efdba2 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -168,8 +168,6 @@ private AutomationConstants() { public static final String NODE_STORAGE_DOWNLOAD = "storage.download"; public static final String NODE_STORAGE_DELETE = "storage.delete"; public static final String NODE_DATA_EXTRACT = "data.extract"; - public static final String NODE_DATA_TRANSFORM = "data.transform"; - public static final String NODE_BROWSER_PLAYWRIGHT = "browser.playwright"; public static final String NODE_VECTOR_SEARCH = "vector.search"; public static final String NODE_VECTOR_ADD = "vector.add"; public static final String NODE_VECTOR_DELETE = "vector.delete"; @@ -206,15 +204,8 @@ private AutomationConstants() { public static final String CONFIG_NULL_VALUE = "nullValue"; public static final String CONFIG_OPERATION = "operation"; public static final String CONFIG_COLUMNS = "columns"; - public static final String CONFIG_MAPPING = "mapping"; public static final String CONFIG_OPERATOR = "operator"; public static final String CONFIG_VALUE = "value"; - public static final String CONFIG_DESCENDING = "descending"; - public static final String CONFIG_PROJECT_ID = "projectId"; - public static final String CONFIG_RECORDING_FILE = "recordingFile"; - public static final String CONFIG_INPUTS = "inputs"; - public static final String CONFIG_BROWSER_SUCCESS_URL_PREFIX = "successUrlPrefix"; - public static final String CONFIG_BROWSER_TIMEOUT_SECONDS = "timeoutSeconds"; public static final String CONFIG_DESCRIPTION = "description"; public static final String CONFIG_ANSWER = "answer"; public static final String JEV_QUESTION_TYPE_CHOICE = "choice"; @@ -250,9 +241,6 @@ private AutomationConstants() { public static final int DEFAULT_LIST_RUNS_LIMIT = 25; public static final int WAIT_MIN_SECONDS = 0; public static final int WAIT_MAX_SECONDS = 3600; - public static final int BROWSER_DEFAULT_TIMEOUT_SECONDS = 30; - public static final int BROWSER_MIN_TIMEOUT_SECONDS = 1; - public static final int BROWSER_MAX_TIMEOUT_SECONDS = 300; public static final int LOOP_MIN_BATCH_SIZE = 1; public static final int LOOP_MAX_BATCH_SIZE = 1_000; public static final int LOOP_DEFAULT_MAX_ITERATIONS = 1_000; diff --git a/src/prerna/reactor/automation/definition/AutomationDefinitionService.java b/src/prerna/reactor/automation/definition/AutomationDefinitionService.java index ba60d13c20..01a3b41e82 100644 --- a/src/prerna/reactor/automation/definition/AutomationDefinitionService.java +++ b/src/prerna/reactor/automation/definition/AutomationDefinitionService.java @@ -588,7 +588,7 @@ private static boolean requiresPythonSource(Map node) { * runtime and therefore has no persisted Python source file. * * @param node automation node - * @return {@code true} for a Java-owned control node + * @return {@code true} for a node evaluated directly by Java */ private static boolean isJavaOwnedNode(Map node) { Object nodeType = node.get(AutomationConstants.NODE_FIELD_TYPE); diff --git a/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java b/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java index aafe49bc9a..0610a3497a 100644 --- a/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java +++ b/src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java @@ -200,7 +200,7 @@ private static Map validateNodes(List> nodes if (isJavaOwnedNode(typedNode) && !AutomationConstants.NODE_CODE_MODE_GENERATED.equals(codeMode)) { throw new IllegalArgumentException( - "Control node '" + nodeId + "' must use generated mode; it is evaluated by Java."); + "Java-owned node '" + nodeId + "' must use generated mode."); } if (AutomationConstants.NODE_CODE_MODE_CUSTOM.equals(codeMode) && !(config instanceof Map)) { throw new IllegalArgumentException("Custom node '" + nodeId + "' must declare a config object."); @@ -278,16 +278,6 @@ private static void validateNodeConfig(String nodeId, AutomationNodeType nodeTyp validateOptionalConfigString(nodeId, config, "version"); } case DATA_EXTRACT -> validateDataExtractConfig(nodeId, config); - case DATA_TRANSFORM -> validateDataTransformConfig(nodeId, config); - case BROWSER_PLAYWRIGHT -> { - requireConfigString(nodeId, config, AutomationConstants.CONFIG_PROJECT_ID); - requireConfigString(nodeId, config, AutomationConstants.CONFIG_RECORDING_FILE); - validateOptionalConfigObject(nodeId, config, AutomationConstants.CONFIG_INPUTS); - requireConfigString(nodeId, config, AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX); - validateBoundedInteger(nodeId, config, AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, - AutomationConstants.BROWSER_MIN_TIMEOUT_SECONDS, - AutomationConstants.BROWSER_MAX_TIMEOUT_SECONDS); - } case VECTOR_SEARCH -> { requireConfigString(nodeId, config, "value"); validateOptionalConfigObject(nodeId, config, "filters"); @@ -336,45 +326,6 @@ private static void validateDataExtractConfig(String nodeId, Map } } - private static void validateDataTransformConfig(String nodeId, Map config) { - Object source = config.get(AutomationConstants.CONFIG_SOURCE); - boolean scopeReference = source instanceof String value && isScopePlaceholder(value.trim()); - if (!(source instanceof List) && !scopeReference) { - throw new IllegalArgumentException("Node '" + nodeId - + "' config.source must be an array or an exact scope reference such as ${query_results}."); - } - String operation = requireNonblankString(config.get(AutomationConstants.CONFIG_OPERATION), - "Node '" + nodeId + "' config.operation"); - if (!Set.of("select", "remove", "rename", "filter", "fillMissing", "sort", "deduplicate") - .contains(operation)) { - throw new IllegalArgumentException("Node '" + nodeId + "' config.operation is unsupported."); - } - validateOptionalConfigObject(nodeId, config, AutomationConstants.CONFIG_MAPPING); - validateOptionalConfigBoolean(nodeId, config, AutomationConstants.CONFIG_DESCENDING); - validateOptionalStringListOrPlaceholder(nodeId, config, AutomationConstants.CONFIG_COLUMNS); - if (Set.of("select", "remove", "sort").contains(operation) - && !hasNonemptyConfigList(config, AutomationConstants.CONFIG_COLUMNS)) { - throw new IllegalArgumentException( - "Node '" + nodeId + "' config.columns must contain at least one column."); - } - if (Set.of("rename", "fillMissing").contains(operation)) { - Object mapping = config.get(AutomationConstants.CONFIG_MAPPING); - if (!(mapping instanceof Map values) || values.isEmpty()) { - throw new IllegalArgumentException( - "Node '" + nodeId + "' config.mapping must contain at least one column."); - } - } - if ("filter".equals(operation)) { - requireConfigString(nodeId, config, AutomationConstants.CONFIG_PATH); - String operator = requireNonblankString(config.get(AutomationConstants.CONFIG_OPERATOR), - "Node '" + nodeId + "' config.operator"); - if (!Set.of("equals", "notEquals", "contains", "greaterThan", "greaterThanOrEqual", "lessThan", - "lessThanOrEqual", "isEmpty", "isNotEmpty").contains(operator)) { - throw new IllegalArgumentException("Node '" + nodeId + "' config.operator is unsupported."); - } - } - } - private static boolean isRoutingNode(AutomationNodeType nodeType) { return nodeType == AutomationNodeType.CONTROL_IF || nodeType == AutomationNodeType.CONTROL_JEV; } @@ -979,10 +930,6 @@ private static void validateLoopBodies(List> nodes, Set bodyNodeTypes = validateNodes(bodyNodes, false); for (String bodyNodeId : bodyNodeTypes.keySet()) { diff --git a/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java b/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java index 953dbcb7af..da0b4c133a 100644 --- a/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java +++ b/src/prerna/reactor/automation/definition/AutomationNodeCatalog.java @@ -171,51 +171,6 @@ private static List createDefinitions() { field(AutomationConstants.CONFIG_NULL_VALUE, ConfigFieldType.JSON, "If the value is null", false, null)), controlInputs(), controlAndResultOutputs())); - definitions.add(definition(AutomationNodeType.DATA_TRANSFORM, "Transform data", - "Clean or reshape rows from an earlier step without writing Python.", - orderedConfig(AutomationConstants.CONFIG_SOURCE, List.of(), AutomationConstants.CONFIG_OPERATION, - "select", AutomationConstants.CONFIG_COLUMNS, List.of(), AutomationConstants.CONFIG_MAPPING, - Map.of(), AutomationConstants.CONFIG_PATH, "", AutomationConstants.CONFIG_OPERATOR, "equals", - AutomationConstants.CONFIG_VALUE, null, AutomationConstants.CONFIG_DESCENDING, false), - List.of(field(AutomationConstants.CONFIG_SOURCE, ConfigFieldType.JSON, "Rows to transform", true, - List.of()), - field(AutomationConstants.CONFIG_OPERATION, ConfigFieldType.STRING, "Transformation", true, - "select"), - field(AutomationConstants.CONFIG_COLUMNS, ConfigFieldType.STRING_LIST, "Columns", false, - List.of()), - field(AutomationConstants.CONFIG_MAPPING, ConfigFieldType.JSON, "Column values", false, Map.of()), - field(AutomationConstants.CONFIG_PATH, ConfigFieldType.STRING, "Column", false, ""), - field(AutomationConstants.CONFIG_OPERATOR, ConfigFieldType.STRING, "Comparison", false, "equals"), - field(AutomationConstants.CONFIG_VALUE, ConfigFieldType.JSON, "Comparison value", false, null), - field(AutomationConstants.CONFIG_DESCENDING, ConfigFieldType.BOOLEAN, "Descending", false, false)), - controlInputs(), controlAndResultOutputs())); - - definitions.add(definition(AutomationNodeType.BROWSER_PLAYWRIGHT, "Run browser recording", - "Replay a saved Playwright recording and verify its final page.", - orderedConfig(AutomationConstants.CONFIG_PROJECT_ID, "", AutomationConstants.CONFIG_RECORDING_FILE, "", - AutomationConstants.CONFIG_INPUTS, Map.of(), - AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "", - AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, - AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS), - List.of(field(AutomationConstants.CONFIG_PROJECT_ID, ConfigFieldType.STRING, "Recording app", true, ""), - field(AutomationConstants.CONFIG_RECORDING_FILE, ConfigFieldType.STRING, "Recording", true, ""), - field(AutomationConstants.CONFIG_INPUTS, ConfigFieldType.JSON, "Recording inputs", false, Map.of()), - field(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, ConfigFieldType.STRING, - "Successful page starts with", true, ""), - boundedIntegerField(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, - "Wait for success (seconds)", AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS, - AutomationConstants.BROWSER_MIN_TIMEOUT_SECONDS, - AutomationConstants.BROWSER_MAX_TIMEOUT_SECONDS)), - List.of(outputField("status", OutputFieldType.STRING, "Status", - "Completed after the expected page became idle.", true), - outputField("recordingFile", OutputFieldType.STRING, "Recording", "Saved recording that ran.", - true), - outputField("passes", OutputFieldType.NUMBER, "Replay passes", - "Number of replay passes used.", true), - outputField("finalUrl", OutputFieldType.STRING, "Final page", - "Final page address without query parameters or fragments.", true)), - controlInputs(), controlAndResultOutputs())); - definitions.add(vectorDefinition(AutomationNodeType.VECTOR_SEARCH, "Search vectors", "Search records in a vector database.", true)); definitions.add(vectorDefinition(AutomationNodeType.VECTOR_ADD, "Add vector documents", diff --git a/src/prerna/reactor/automation/definition/AutomationNodeType.java b/src/prerna/reactor/automation/definition/AutomationNodeType.java index 260040ee4a..28db9f2f3e 100644 --- a/src/prerna/reactor/automation/definition/AutomationNodeType.java +++ b/src/prerna/reactor/automation/definition/AutomationNodeType.java @@ -72,9 +72,6 @@ public enum AutomationNodeType { STORAGE_DELETE(AutomationConstants.NODE_STORAGE_DELETE, Category.STORAGE, IEngine.CATALOG_TYPE.STORAGE, Permission.EDIT, true, true), DATA_EXTRACT(AutomationConstants.NODE_DATA_EXTRACT, Category.DATA, null, Permission.NONE, true, false), - DATA_TRANSFORM(AutomationConstants.NODE_DATA_TRANSFORM, Category.DATA, null, Permission.NONE, true, false), - BROWSER_PLAYWRIGHT(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, Category.BROWSER, null, Permission.VIEW, true, - false), VECTOR_SEARCH(AutomationConstants.NODE_VECTOR_SEARCH, Category.VECTOR, IEngine.CATALOG_TYPE.VECTOR, Permission.VIEW, true, true), VECTOR_ADD(AutomationConstants.NODE_VECTOR_ADD, Category.VECTOR, IEngine.CATALOG_TYPE.VECTOR, Permission.EDIT, true, @@ -97,7 +94,7 @@ public enum AutomationNodeType { * Logical grouping used by clients without parsing the persisted type string. */ public enum Category { - TRIGGER, DATABASE, MODEL, STORAGE, DATA, BROWSER, VECTOR, FUNCTION, APP, AGENT, CONTROL, DEVELOPER + TRIGGER, DATABASE, MODEL, STORAGE, DATA, VECTOR, FUNCTION, APP, AGENT, CONTROL, DEVELOPER } /** Minimum resource permission required by the node when it references one. */ diff --git a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java index 80e6955896..6ca9f1dab6 100644 --- a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java +++ b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java @@ -65,7 +65,7 @@ public static String renderNode(Map node) { String source = switch (nodeType) { case TRIGGER_START -> triggerSource(); case CONTROL_IF, CONTROL_JEV, CONTROL_LOOP -> - throw new IllegalArgumentException("Java-owned control nodes do not have Python source."); + throw new IllegalArgumentException("Java-owned nodes do not have Python source."); case DATABASE_QUERY -> databaseQuerySource(config); case DATABASE_INSERT -> databaseWriteSource(config, "insertData"); case DATABASE_UPDATE -> databaseWriteSource(config, "updateData"); @@ -80,8 +80,6 @@ public static String renderNode(Map node) { case STORAGE_DOWNLOAD -> storageDownloadSource(config); case STORAGE_DELETE -> storageDeleteSource(config); case DATA_EXTRACT -> dataExtractSource(config); - case DATA_TRANSFORM -> dataTransformSource(config); - case BROWSER_PLAYWRIGHT -> browserPlaywrightSource(config); case VECTOR_SEARCH -> vectorSearchSource(config); case VECTOR_ADD -> vectorAddSource(config); case VECTOR_DELETE -> vectorDeleteSource(config); @@ -542,154 +540,6 @@ def run(scope): value(config, AutomationConstants.CONFIG_NULL_VALUE)); } - private static String dataTransformSource(Map config) { - return """ - # Apply a bounded business transformation to row-shaped Automation data. - from semoss_automation_runtime import transform_records - - SOURCE = %s - OPERATION = %s - COLUMNS = %s - MAPPING = %s - COLUMN = %s - OPERATOR = %s - VALUE = %s - DESCENDING = %s - - def run(scope): - return transform_records( - records=scope.resolve(SOURCE), - operation=OPERATION, - columns=scope.resolve(COLUMNS), - mapping=scope.resolve_config(MAPPING), - column=scope.resolve(COLUMN), - operator=OPERATOR, - value=scope.resolve(VALUE), - descending=bool(DESCENDING), - ) - """.formatted(value(config, AutomationConstants.CONFIG_SOURCE), - value(config, AutomationConstants.CONFIG_OPERATION), - valueOrDefault(config, AutomationConstants.CONFIG_COLUMNS, List.of()), - valueOrDefault(config, AutomationConstants.CONFIG_MAPPING, Map.of()), - valueOrDefault(config, AutomationConstants.CONFIG_PATH, ""), - valueOrDefault(config, AutomationConstants.CONFIG_OPERATOR, "equals"), - value(config, AutomationConstants.CONFIG_VALUE), - valueOrDefault(config, AutomationConstants.CONFIG_DESCENDING, false)); - } - - private static String browserPlaywrightSource(Map config) { - return """ - # Replay a saved SEMOSS Playwright recording in this Automation run. - from semoss import Insight - import json - import time - from urllib.parse import urlsplit - - PROJECT_ID = %s - RECORDING_FILE = %s - INPUTS = %s - SUCCESS_URL_PREFIX = %s - TIMEOUT_SECONDS = %s - MAX_PASSES = 100 - POLL_SECONDS = 0.25 - - def _pixel_value(name, value): - return name + "=[" + json.dumps(value) + "]" - - def _safe_url(value): - parts = urlsplit(str(value or "")) - if parts.scheme and parts.netloc: - return parts.scheme + "://" + parts.netloc + parts.path - return str(value or "").split("?", 1)[0].split("#", 1)[0] - - def _browser_state(session_id, tab_id): - pixel = "CheckNetworkIdle(" + ", ".join([ - _pixel_value("sessionId", session_id), - _pixel_value("tabId", tab_id), - _pixel_value("quietMillis", 500), - ]) + ");" - response = Insight().run_pixel(pixel, raw=True) - result = response[0]["pixelReturn"][-1] - if "ERROR" in result.get("operationType", []): - raise RuntimeError(result.get("output") or "Unable to inspect the browser result") - state = result.get("output") - if not isinstance(state, dict): - raise RuntimeError("Browser result did not include page state") - return state - - def _wait_for_success(session_id, tab_id, expected_prefix, timeout_seconds): - deadline = time.monotonic() + timeout_seconds - final_url = "" - while True: - state = _browser_state(session_id, tab_id) - final_url = _safe_url(state.get("currentUrl")) - if state.get("isNetworkIdle") is True and final_url.startswith(expected_prefix): - return final_url - if time.monotonic() >= deadline: - raise RuntimeError( - "Browser recording did not reach the expected page. " - + "Expected a page starting with " + expected_prefix - + "; final page was " + (final_url or "unavailable") - ) - time.sleep(POLL_SECONDS) - - def run(scope): - session_response = Insight().run_pixel("AutomationBrowserSession();", raw=True) - session_result = session_response[0]["pixelReturn"][-1] - if "ERROR" in session_result.get("operationType", []): - raise RuntimeError(session_result.get("output") or "Unable to start browser session") - session_id = session_result.get("output") - if not isinstance(session_id, str) or not session_id: - raise RuntimeError("Browser session did not return an id") - project_id = scope.resolve(PROJECT_ID) - recording_file = scope.resolve(RECORDING_FILE) - inputs = scope.resolve_config(INPUTS) - success_url_prefix = _safe_url(scope.resolve(SUCCESS_URL_PREFIX)) - timeout_seconds = int(TIMEOUT_SECONDS) - latest = {} - tab_id = "tab-1" - tab_stack = [] - for pass_number in range(1, MAX_PASSES + 1): - pixel = "ReplayStep(" + ", ".join([ - _pixel_value("sessionId", session_id), - _pixel_value("fileName", recording_file), - "paramValues=[" + json.dumps(inputs) + "]", - "executeAll=[true]", - _pixel_value("tabId", tab_id), - _pixel_value("project", project_id), - ]) + ");" - response = Insight().run_pixel(pixel, raw=True) - result = response[0]["pixelReturn"][-1] - if "ERROR" in result.get("operationType", []): - raise RuntimeError(result.get("output") or "Browser recording failed") - latest = result.get("output") or {} - new_tab_id = latest.get("newTabId") - if isinstance(new_tab_id, str) and new_tab_id and new_tab_id != tab_id: - tab_stack.append(tab_id) - tab_id = new_tab_id - continue - if latest.get("isLastPage") is True: - if tab_stack: - tab_id = tab_stack.pop() - continue - final_url = _wait_for_success( - session_id, tab_id, success_url_prefix, timeout_seconds - ) - return { - "status": "completed", - "recordingFile": recording_file, - "passes": pass_number, - "finalUrl": final_url, - } - raise RuntimeError("Browser recording exceeded the maximum replay passes.") - """.formatted(value(config, AutomationConstants.CONFIG_PROJECT_ID), - value(config, AutomationConstants.CONFIG_RECORDING_FILE), - valueOrDefault(config, AutomationConstants.CONFIG_INPUTS, Map.of()), - value(config, AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX), - valueOrDefault(config, AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, - AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS)); - } - private static String vectorSearchSource(Map config) { return """ # Search a SEMOSS vector engine directly through the Python SDK. diff --git a/src/prerna/reactor/automation/project/AutomationProjectService.java b/src/prerna/reactor/automation/project/AutomationProjectService.java index 1d8f34ebe6..0b065f7093 100644 --- a/src/prerna/reactor/automation/project/AutomationProjectService.java +++ b/src/prerna/reactor/automation/project/AutomationProjectService.java @@ -240,8 +240,6 @@ public static void validateDefinitionReferences(AutomationDefinitionValidator.Va validateAgentWorkspaceReference(node, config, user); } else if (typedNode == AutomationNodeType.APP_PIXEL) { validateAppProjectReference(node, config, user); - } else if (typedNode == AutomationNodeType.BROWSER_PLAYWRIGHT) { - validateBrowserProjectReference(node, config, user); } IEngine.CATALOG_TYPE expectedType = typedNode.getEngineType(); @@ -334,35 +332,12 @@ private static void validateAppProjectReference(Map node, Map node, Map config, - User user) { - String nodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); - Object rawProjectId = config.get(AutomationConstants.CONFIG_PROJECT_ID); - if (!(rawProjectId instanceof String projectId) || projectId.isBlank()) { - throw invalidBrowserProjectReference(nodeId, String.valueOf(rawProjectId)); - } - projectId = projectId.trim(); - if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { - throw invalidBrowserProjectReference(nodeId, projectId); - } - String projectType = SecurityProjectUtils.getProjectTypeForId(projectId); - if (!(IProject.PROJECT_TYPE.CODE.name().equals(projectType) - || IProject.PROJECT_TYPE.BLOCKS.name().equals(projectType))) { - throw invalidBrowserProjectReference(nodeId, projectId); - } - } - private static IllegalArgumentException invalidAppReference(String nodeId, String appId) { return new IllegalArgumentException("Automation node '" + nodeId + "' appId '" + appId + "' is not an accessible CODE or BLOCKS app. Call MyProjects with " + "projectType=['CODE','BLOCKS'] and use its project_id value."); } - private static IllegalArgumentException invalidBrowserProjectReference(String nodeId, String projectId) { - return new IllegalArgumentException("Automation browser node '" + nodeId + "' projectId '" + projectId - + "' is not an accessible CODE or BLOCKS app with a Playwright recording."); - } - private static IllegalArgumentException invalidEngineReference(String nodeId, String engineId, IEngine.CATALOG_TYPE expectedType) { return new IllegalArgumentException("Automation node '" + nodeId + "' engineId '" + engineId diff --git a/src/prerna/reactor/automation/run/AutomationBrowserSessionReactor.java b/src/prerna/reactor/automation/run/AutomationBrowserSessionReactor.java deleted file mode 100644 index 98d3270699..0000000000 --- a/src/prerna/reactor/automation/run/AutomationBrowserSessionReactor.java +++ /dev/null @@ -1,69 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation.run; - -import com.microsoft.playwright.Page; - -import prerna.reactor.AbstractReactor; -import prerna.reactor.playwright.PlaywrightSession; -import prerna.remoteviewer.service.RemoteBrowserSession; -import prerna.remoteviewer.service.RemoteBrowserSessionManager; -import prerna.sablecc2.om.PixelDataType; -import prerna.sablecc2.om.nounmeta.NounMetadata; - -/** - * Prepares the authenticated user's existing remote browser infrastructure for - * one top-level Automation browser step. - */ -public class AutomationBrowserSessionReactor extends AbstractReactor { - - private static final int VIEWPORT_WIDTH = 1280; - private static final int VIEWPORT_HEIGHT = 800; - - @Override - public NounMetadata execute() { - RemoteBrowserSession session = RemoteBrowserSessionManager.getInstance() - .createSession(this.insight.getUser(), "", VIEWPORT_WIDTH, VIEWPORT_HEIGHT); - PlaywrightSession playwrightSession = session.getPlaywrightSession(); - playwrightSession.getOperationLock().lock(); - try { - Page page = playwrightSession.resetPagesForNewViewer(); - page.setViewportSize(VIEWPORT_WIDTH, VIEWPORT_HEIGHT); - session.setActiveTabId("tab-1"); - session.clearRecordingBuffer(); - } finally { - playwrightSession.getOperationLock().unlock(); - } - return new NounMetadata(session.getSessionId(), PixelDataType.CONST_STRING); - } - - @Override - public String getReactorDescription() { - return "Prepares a remote-viewable browser session for a top-level Automation browser step."; - } -} diff --git a/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java b/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java index 8f8e4e0f76..442b6147a5 100644 --- a/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationDefinitionValidatorUnitTests.java @@ -382,56 +382,6 @@ void acceptsABoundedLoopWithAnAcyclicBody() { AutomationDefinitionValidator.parseAndValidate(definition(Map.of(), loop)); } - @Test - void rejectsBrowserAutomationInsideALoop() { - Map browserNode = workNode(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, - Map.of(AutomationConstants.CONFIG_PROJECT_ID, "recording-project", - AutomationConstants.CONFIG_RECORDING_FILE, "search.json", - AutomationConstants.CONFIG_INPUTS, Map.of(), - AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "https://example.com/results", - AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, - AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS)); - browserNode.put(AutomationConstants.NODE_FIELD_ID, "body-browser"); - browserNode.put(AutomationConstants.NODE_FIELD_OUTPUT_VAR, "browser_result"); - Map loop = workNode(AutomationConstants.NODE_CONTROL_LOOP, - Map.of(AutomationConstants.CONFIG_LOOP_MODE, AutomationConstants.LOOP_MODE_FOR_EACH, - AutomationConstants.CONFIG_LOOP_ITEMS, List.of("SEMOSS"), - AutomationConstants.CONFIG_LOOP_BATCH_SIZE, 1, - AutomationConstants.CONFIG_LOOP_MAX_ITERATIONS, 10)); - loop.put(AutomationConstants.NODE_FIELD_BODY, - Map.of(AutomationConstants.DOC_NODES, List.of(browserNode), - AutomationConstants.DOC_EDGES, List.of())); - - assertThrows(IllegalArgumentException.class, - () -> AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), loop))); - } - - @Test - void browserAutomationRequiresABoundedSuccessCondition() { - Map config = new LinkedHashMap<>(); - config.put(AutomationConstants.CONFIG_PROJECT_ID, "recording-project"); - config.put(AutomationConstants.CONFIG_RECORDING_FILE, "search.json"); - config.put(AutomationConstants.CONFIG_INPUTS, Map.of()); - config.put(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "https://example.com/results"); - config.put(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, - AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS); - Map browser = workNode(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, config); - - AutomationDefinitionValidator.parseAndValidateForAuthoring(definition(Map.of(), browser)); - - config.put(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, ""); - assertThrows(IllegalArgumentException.class, - () -> AutomationDefinitionValidator.parseAndValidateForAuthoring( - definition(Map.of(), workNode(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, config)))); - - config.put(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "https://example.com/results"); - config.put(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, - AutomationConstants.BROWSER_MAX_TIMEOUT_SECONDS + 1); - assertThrows(IllegalArgumentException.class, - () -> AutomationDefinitionValidator.parseAndValidateForAuthoring( - definition(Map.of(), workNode(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, config)))); - } - @Test void acceptsDottedForEachRepeatAndWhileLoopConfiguration() { Map dottedForEach = workNode(AutomationConstants.NODE_CONTROL_LOOP, diff --git a/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java b/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java index ca70d34c87..f551cc3913 100644 --- a/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationNodeCatalogUnitTests.java @@ -131,26 +131,6 @@ void dataExtractAdvertisesOneGenericJsonXmlContract() { assertEquals(false, definition.nodeType().supportsCustomCode()); } - @Test - void dataTransformAndBrowserRecordingAreBackendOwned() { - AutomationNodeDefinition transform = find(AutomationConstants.NODE_DATA_TRANSFORM); - AutomationNodeDefinition browser = find(AutomationConstants.NODE_BROWSER_PLAYWRIGHT); - assertNotNull(transform); - assertNotNull(browser); - assertEquals("data", transform.toMap().get("category")); - assertEquals("browser", browser.toMap().get("category")); - assertEquals("VIEW", browser.toMap().get("requiredPermission")); - assertEquals(false, transform.nodeType().supportsCustomCode()); - assertEquals(false, browser.nodeType().supportsCustomCode()); - assertEquals(Map.of(), browser.defaultConfig().get(AutomationConstants.CONFIG_INPUTS)); - assertEquals("", browser.defaultConfig().get(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX)); - assertEquals(AutomationConstants.BROWSER_DEFAULT_TIMEOUT_SECONDS, - browser.defaultConfig().get(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS)); - assertEquals(Set.of("status", "recordingFile", "passes", "finalUrl"), - browser.outputFields().stream().map(AutomationNodeDefinition.OutputField::key) - .collect(java.util.stream.Collectors.toSet())); - } - @Test void optionalEngineParametersAreBackendOwned() { Map> uploadFields = find(AutomationConstants.NODE_STORAGE_UPLOAD).configFields() diff --git a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java index 92f5ff7a5e..d3a6778adf 100644 --- a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java @@ -159,45 +159,6 @@ void dataExtractReadsOnlyFromTheCurrentInsightWorkspace() { assertFalse(source.contains("open(")); } - @Test - void dataTransformUsesTheSharedAutomationRuntime() { - Map config = new LinkedHashMap<>(); - config.put(AutomationConstants.CONFIG_SOURCE, "${query_rows}"); - config.put(AutomationConstants.CONFIG_OPERATION, "filter"); - config.put(AutomationConstants.CONFIG_COLUMNS, java.util.List.of()); - config.put(AutomationConstants.CONFIG_MAPPING, Map.of()); - config.put(AutomationConstants.CONFIG_PATH, "status"); - config.put(AutomationConstants.CONFIG_OPERATOR, "equals"); - config.put(AutomationConstants.CONFIG_VALUE, "active"); - config.put(AutomationConstants.CONFIG_DESCENDING, false); - - String source = AutomationSourceRenderer.renderNode(node(AutomationConstants.NODE_DATA_TRANSFORM, config)); - - assertTrue(source.contains("from semoss_automation_runtime import transform_records")); - assertTrue(source.contains("records=scope.resolve(SOURCE)")); - } - - @Test - void browserRecordingUsesAutomationOwnedBrowserSession() { - Map config = new LinkedHashMap<>(); - config.put(AutomationConstants.CONFIG_PROJECT_ID, "project-1"); - config.put(AutomationConstants.CONFIG_RECORDING_FILE, "intake.json"); - config.put(AutomationConstants.CONFIG_INPUTS, Map.of("Order number", "123")); - config.put(AutomationConstants.CONFIG_BROWSER_SUCCESS_URL_PREFIX, "https://example.com/complete"); - config.put(AutomationConstants.CONFIG_BROWSER_TIMEOUT_SECONDS, 45); - - String source = AutomationSourceRenderer - .renderNode(node(AutomationConstants.NODE_BROWSER_PLAYWRIGHT, config)); - - assertTrue(source.contains("run_pixel(\"AutomationBrowserSession();\", raw=True)")); - assertTrue(source.contains("ReplayStep(")); - assertTrue(source.contains("CheckNetworkIdle(")); - assertTrue(source.contains("final_url.startswith(expected_prefix)")); - assertTrue(source.contains("\"finalUrl\": final_url")); - assertTrue(source.contains("_pixel_value(\"project\", project_id)")); - assertFalse(source.contains("Playwright.create")); - } - @Test void visionAcceptsOneOrManyMediaPathsWithoutNestingThem() { Map config = new LinkedHashMap<>(); From b2e760171af9512aa1edb59051fc13fe67d1c3ae Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Wed, 7 Oct 2026 16:58:35 -0400 Subject: [PATCH 7/7] fix(automation): harden loop runtime data paths --- py/semoss_automation_runtime.py | 12 +++- py/testing/test_semoss_automation_runtime.py | 69 +++++++++++++++++++ .../definition/AutomationSourceRenderer.java | 1 - .../AutomationSourceRendererUnitTests.java | 1 + 4 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 py/testing/test_semoss_automation_runtime.py diff --git a/py/semoss_automation_runtime.py b/py/semoss_automation_runtime.py index d4a737d4fb..9c34f43d6c 100644 --- a/py/semoss_automation_runtime.py +++ b/py/semoss_automation_runtime.py @@ -223,6 +223,10 @@ def _data_path_tokens(path: str) -> list[str | int]: return [] if value.startswith("$."): value = value[2:] + elif value.startswith("$["): + value = value[1:] + elif value.startswith("$"): + raise ValueError("Value path must use '$.' or '$[' after the root marker.") tokens: list[str | int] = [] cursor = 0 @@ -267,7 +271,13 @@ def execute_node( """Execute one persisted node module with a fresh module namespace.""" scope = _decode_scope(encoded_scope) source = _decode(encoded_source) - module: dict[str, Any] = {"__name__": "__automation_node__"} + module: dict[str, Any] = { + "__name__": "__automation_node__", + # Generated nodes use runtime-owned helpers through this execution boundary. + # This avoids depending on the worker's ambient sys.path or importing a + # second copy of this module from inside persisted node source. + "extract_data_element": extract_data_element, + } # Java selects this persisted source only after authorizing the run. Keeping # exec here makes that trust boundary explicit and avoids hidden source edits. exec(source, module) diff --git a/py/testing/test_semoss_automation_runtime.py b/py/testing/test_semoss_automation_runtime.py new file mode 100644 index 0000000000..f3f3436c66 --- /dev/null +++ b/py/testing/test_semoss_automation_runtime.py @@ -0,0 +1,69 @@ +"""Focused regression tests for SEMOSS Automation Python runtime helpers.""" + +import base64 +import json +import unittest + +from semoss_automation_runtime import extract_data_element + + +class ExtractDataElementTests(unittest.TestCase): + def test_reads_dot_and_array_path(self) -> None: + source = {"orders": [{"id": 17}]} + + self.assertEqual(17, extract_data_element(source, "orders[0].id")) + self.assertEqual(17, extract_data_element(source, "$.orders[0].id")) + + def test_reads_quoted_key_from_root(self) -> None: + source = {"a.b": {"value": "ok"}} + + self.assertEqual("ok", extract_data_element(source, '$["a.b"].value')) + + def test_reads_json_pointer(self) -> None: + source = {"orders": [{"id": 17}]} + + self.assertEqual(17, extract_data_element(source, "/orders/0/id")) + + def test_distinguishes_missing_and_null(self) -> None: + source = {"present": None} + + self.assertEqual( + "missing", + extract_data_element(source, "absent", missing_value="missing"), + ) + self.assertEqual( + "null", + extract_data_element(source, "present", null_value="null"), + ) + + def test_reads_base64_json_and_xml(self) -> None: + encoded = base64.b64encode(json.dumps({"value": 2}).encode()).decode() + + self.assertEqual(2, extract_data_element(encoded, "value")) + self.assertEqual( + "ok", + extract_data_element( + 'ok', + "root.item.#text", + ), + ) + + def test_reads_run_workspace_asset(self) -> None: + self.assertEqual( + "ready", + extract_data_element( + "/orders/example.json", + "status", + asset_reader=lambda path: json.dumps( + {"path": path, "status": "ready"} + ), + ), + ) + + def test_rejects_invalid_root_marker(self) -> None: + with self.assertRaisesRegex(ValueError, "root marker"): + extract_data_element({"value": 1}, "$value") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java index 6ca9f1dab6..a562330945 100644 --- a/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java +++ b/src/prerna/reactor/automation/definition/AutomationSourceRenderer.java @@ -506,7 +506,6 @@ private static String dataExtractSource(Map config) { return """ # Extract a nested value from inline JSON/XML or a file in this run's Insight workspace. from semoss import Insight - from semoss_automation_runtime import extract_data_element import base64 import json diff --git a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java index d3a6778adf..e6eeabee47 100644 --- a/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java +++ b/test/prerna/reactor/automation/definition/AutomationSourceRendererUnitTests.java @@ -156,6 +156,7 @@ void dataExtractReadsOnlyFromTheCurrentInsightWorkspace() { assertTrue(source.contains("GetInsightAssetsBase64(")); assertTrue(source.contains("extract_data_element(")); assertTrue(source.contains("source=scope.resolve(SOURCE)")); + assertFalse(source.contains("from semoss_automation_runtime import")); assertFalse(source.contains("open(")); }