Add structured failureReason to operation error response

multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/FailureReason.java

@@ -0,0 +1,36 @@
+package org.cloudfoundry.multiapps.controller.api.model;
+
+import org.cloudfoundry.multiapps.common.Nullable;
+import org.immutables.value.Value;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+@Value.Immutable
+@JsonSerialize(as = ImmutableFailureReason.class)
+@JsonDeserialize(as = ImmutableFailureReason.class)
+public interface FailureReason {
+
+    enum Code {
+        CONTENT_VALIDATION_ERROR,
+        MODULE_STARTUP_FAILURE,
+        SERVICE_OPERATION_FAILURE,
+        CF_API_UNAVAILABLE,
+        QUOTA_EXCEEDED,
+        INTERNAL_ERROR
+    }
+
+    @Nullable
+    @JsonProperty("code")
+    Code getCode();
+
+    @Nullable
+    @JsonProperty("retryable")
+    Boolean getRetryable();
+
+    @Nullable
+    @JsonProperty("summary")
+    String getSummary();
+
+}

multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/Operation.java

@@ -100,6 +100,9 @@ public static List<State> getFinalStates() {
     @Nullable
     public abstract ErrorType getErrorType();
 
+    @Nullable
+    public abstract FailureReason getFailureReason();
+
     public abstract List<Message> getMessages();
 
     @AllowNulls

multiapps-controller-api/src/main/resources/mtarest.yaml

@@ -779,6 +779,9 @@ definitions:
         enum:
         - "CONTENT"
         - "INFRASTRUCTURE"
+      failureReason:
+        readOnly: true
+        $ref: "#/definitions/FailureReason"
       messages:
         type: "array"
         readOnly: true
@@ -789,6 +792,25 @@ definitions:
         readOnly: true
         additionalProperties:
           type: "object"
+  FailureReason:
+    type: "object"
+    properties:
+      code:
+        type: "string"
+        readOnly: true
+        enum:
+        - "CONTENT_VALIDATION_ERROR"
+        - "MODULE_STARTUP_FAILURE"
+        - "SERVICE_OPERATION_FAILURE"
+        - "CF_API_UNAVAILABLE"
+        - "QUOTA_EXCEEDED"
+        - "INTERNAL_ERROR"
+      retryable:
+        type: "boolean"
+        readOnly: true
+      summary:
+        type: "string"
+        readOnly: true
   ProcessType:
     type: "object"
     properties:

multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandler.java

@@ -11,9 +11,12 @@
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.cloudfoundry.multiapps.common.ContentException;
+import org.cloudfoundry.multiapps.controller.api.model.FailureReason;
 import org.cloudfoundry.multiapps.controller.api.model.ImmutableOperation;
 import org.cloudfoundry.multiapps.controller.api.model.Operation;
 import org.cloudfoundry.multiapps.controller.core.cloudlogging.CloudLoggingServiceClient;
+import org.cloudfoundry.multiapps.controller.client.facade.CloudControllerException;
+import org.cloudfoundry.multiapps.controller.client.facade.CloudServiceBrokerException;
 import org.cloudfoundry.multiapps.controller.persistence.model.HistoricOperationEvent;
 import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableHistoricOperationEvent;
 import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableProgressMessage;
@@ -24,15 +27,20 @@
 import org.cloudfoundry.multiapps.controller.persistence.services.ProgressMessageService;
 import org.cloudfoundry.multiapps.controller.process.Messages;
 import org.cloudfoundry.multiapps.controller.process.flowable.FlowableFacade;
+import org.cloudfoundry.multiapps.controller.process.variables.Variables;
 import org.flowable.common.engine.api.delegate.event.FlowableEngineEvent;
 import org.flowable.engine.HistoryService;
 import org.flowable.engine.ProcessEngineConfiguration;
 import org.flowable.engine.impl.context.Context;
 import org.flowable.engine.runtime.Execution;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Named
 public class OperationInErrorStateHandler {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(OperationInErrorStateHandler.class);
+
     protected final FlowableFacade flowableFacade;
     private final OperationService operationService;
     private final ProgressMessageService progressMessageService;
@@ -58,6 +66,7 @@ public void handle(FlowableEngineEvent event, String errorMessage) {
 
     public void handle(FlowableEngineEvent event, Throwable throwable) {
         HistoricOperationEvent.EventType eventType = toEventType(throwable);
+        storeFailureReasonCode(event, throwable);
         handle(event, eventType, throwable.getMessage());
     }
 
@@ -74,6 +83,31 @@ HistoricOperationEvent.EventType toEventType(Throwable throwable) {
             : HistoricOperationEvent.EventType.FAILED_BY_INFRASTRUCTURE_ERROR;
     }
 
+    FailureReason.Code toFailureReasonCode(Throwable throwable) {
+        if (hasCause(throwable, ContentException.class)) {
+            return FailureReason.Code.CONTENT_VALIDATION_ERROR;
+        }
+        if (hasCause(throwable, CloudServiceBrokerException.class)) {
+            return FailureReason.Code.SERVICE_OPERATION_FAILURE;
+        }
+        if (hasCause(throwable, CloudControllerException.class)) {
+            return FailureReason.Code.CF_API_UNAVAILABLE;
+        }
+        return FailureReason.Code.INTERNAL_ERROR;
+    }
+
+    private void storeFailureReasonCode(FlowableEngineEvent event, Throwable throwable) {
+        String processInstanceId = flowableFacade.getProcessInstanceId(event.getExecutionId());
+        FailureReason.Code code = toFailureReasonCode(throwable);
+        try {
+            getProcessEngineConfiguration().getRuntimeService()
+                                           .setVariable(processInstanceId, Variables.FAILURE_REASON_CODE.getName(),
+                                                        code.name());
+        } catch (Exception e) {
+            LOGGER.warn("Failed to store failure reason code for process {}: {}", processInstanceId, e.getMessage());
+        }
+    }
+
     private <T extends Throwable> boolean hasCause(Throwable throwable, Class<T> clazz) {
         return ExceptionUtils.getThrowableList(throwable)
                              .stream()

multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelper.java

@@ -1,32 +1,43 @@
 package org.cloudfoundry.multiapps.controller.process.util;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
 
 import org.cloudfoundry.multiapps.controller.api.model.ErrorType;
+import org.cloudfoundry.multiapps.controller.api.model.FailureReason;
+import org.cloudfoundry.multiapps.controller.api.model.ImmutableDryRunResult;
+import org.cloudfoundry.multiapps.controller.api.model.ImmutableFailureReason;
 import org.cloudfoundry.multiapps.controller.api.model.ImmutableOperation;
 import org.cloudfoundry.multiapps.controller.api.model.Operation;
+import org.cloudfoundry.multiapps.controller.process.client.DryRunChangeSet;
 import org.cloudfoundry.multiapps.controller.persistence.model.HistoricOperationEvent;
 import org.cloudfoundry.multiapps.controller.persistence.services.OperationService;
+import org.cloudfoundry.multiapps.controller.process.flowable.FlowableFacade;
 import org.cloudfoundry.multiapps.controller.process.metadata.ProcessTypeToOperationMetadataMapper;
+import org.cloudfoundry.multiapps.controller.process.variables.Serializer;
+import org.cloudfoundry.multiapps.controller.process.variables.Variables;
+import org.flowable.variable.api.history.HistoricVariableInstance;
 
 @Named
 public class OperationsHelper {
 
     private final OperationService operationService;
     private final ProcessTypeToOperationMetadataMapper metadataMapper;
     private final ProcessHelper processHelper;
+    private final FlowableFacade flowableFacade;
 
     @Inject
     public OperationsHelper(OperationService operationService, ProcessTypeToOperationMetadataMapper metadataMapper,
-                            ProcessHelper processHelper) {
+                            ProcessHelper processHelper, FlowableFacade flowableFacade) {
         this.operationService = operationService;
         this.metadataMapper = metadataMapper;
         this.processHelper = processHelper;
-
+        this.flowableFacade = flowableFacade;
     }
 
     public String getProcessDefinitionKey(Operation operation) {
@@ -36,11 +47,65 @@ public String getProcessDefinitionKey(Operation operation) {
     public Operation addErrorType(Operation operation) {
         if (operation.getState() == Operation.State.ERROR) {
             return ImmutableOperation.copyOf(operation)
-                                     .withErrorType(getErrorType(operation));
+                                     .withErrorType(getErrorType(operation))
+                                     .withFailureReason(getFailureReason(operation));
         }
         return operation;
     }
 
+    public Operation addDryRunResult(Operation operation) {
+        List<DryRunChangeSet> changeSets = findAllHistoricVariableValues(operation.getProcessId(),
+                                                                         Variables.DRY_RUN_CHANGE_SET.getName(),
+                                                                         Variables.DRY_RUN_CHANGE_SET.getSerializer());
+        if (changeSets.isEmpty()) {
+            return operation;
+        }
+        DryRunChangeSet merged = new DryRunChangeSet();
+        for (DryRunChangeSet cs : changeSets) {
+            addDistinct(merged.appsCreate, cs.appsCreate);
+            addDistinct(merged.appsUpdate, cs.appsUpdate);
+            addDistinct(merged.appsDelete, cs.appsDelete);
+            addDistinct(merged.servicesCreate, cs.servicesCreate);
+            addDistinct(merged.servicesUpdate, cs.servicesUpdate);
+            addDistinct(merged.servicesDelete, cs.servicesDelete);
+            addDistinct(merged.routesCreate, cs.routesCreate);
+            addDistinct(merged.routesDelete, cs.routesDelete);
+        }
+        return ImmutableOperation.copyOf(operation)
+                                 .withDryRunResult(ImmutableDryRunResult.builder()
+                                                                        .appsToCreate(merged.appsCreate)
+                                                                        .appsToUpdate(merged.appsUpdate)
+                                                                        .appsToDelete(merged.appsDelete)
+                                                                        .servicesToCreate(merged.servicesCreate)
+                                                                        .servicesToUpdate(merged.servicesUpdate)
+                                                                        .servicesToDelete(merged.servicesDelete)
+                                                                        .routesToCreate(merged.routesCreate)
+                                                                        .routesToDelete(merged.routesDelete)
+                                                                        .build());
+    }
+
+    private static <T> void addDistinct(List<T> target, List<T> source) {
+        for (T item : source) {
+            if (!target.contains(item)) {
+                target.add(item);
+            }
+        }
+    }
+
+    public Operation addOutputs(Operation operation) {
+        HistoricVariableInstance variable = findHistoricVariable(operation.getProcessId(), Variables.DEPLOYMENT_OUTPUTS.getName());
+        if (variable == null || variable.getValue() == null) {
+            return operation;
+        }
+        Map<String, String> outputs = Variables.DEPLOYMENT_OUTPUTS.getSerializer()
+                                                                  .deserialize(variable.getValue());
+        if (outputs == null || outputs.isEmpty()) {
+            return operation;
+        }
+        return ImmutableOperation.copyOf(operation)
+                                 .withOutputs(outputs);
+    }
+
     public Operation releaseLockIfNeeded(Operation operation) {
         // Fixes bug XSBUG-2035: Inconsistency in 'operation', 'act_hi_procinst' and 'act_ru_execution' tables
         if (operation.hasAcquiredLock() && operation.getState()
@@ -60,6 +125,39 @@ public List<Operation> releaseLocksIfNeeded(List<Operation> operations) {
                          .collect(Collectors.toList());
     }
 
+    private HistoricVariableInstance findHistoricVariable(String processId, String variableName) {
+        HistoricVariableInstance variable = flowableFacade.getHistoricVariableInstance(processId, variableName);
+        if (variable != null) {
+            return variable;
+        }
+        for (String subProcessId : flowableFacade.getHistoricSubProcessIds(processId)) {
+            variable = flowableFacade.getHistoricVariableInstance(subProcessId, variableName);
+            if (variable != null) {
+                return variable;
+            }
+        }
+        return null;
+    }
+
+    private <T> List<T> findAllHistoricVariableValues(String processId, String variableName,
+                                                      Serializer<T> serializer) {
+        List<T> results = new ArrayList<>();
+        collectHistoricVariableValues(processId, variableName, serializer, results);
+        return results;
+    }
+
+    private <T> void collectHistoricVariableValues(String processId, String variableName,
+                                                   Serializer<T> serializer,
+                                                   List<T> results) {
+        HistoricVariableInstance variable = flowableFacade.getHistoricVariableInstance(processId, variableName);
+        if (variable != null && variable.getValue() != null) {
+            results.add(serializer.deserialize(variable.getValue()));
+        }
+        for (String subProcessId : flowableFacade.getHistoricSubProcessIds(processId)) {
+            collectHistoricVariableValues(subProcessId, variableName, serializer, results);
+        }
+    }
+
     private ErrorType getErrorType(Operation operation) {
         List<HistoricOperationEvent> historicEvents = processHelper.getHistoricOperationEventByProcessId(operation.getProcessId());
         if (historicEvents.isEmpty()) {
@@ -79,4 +177,23 @@ private ErrorType toErrorType(HistoricOperationEvent.EventType historicType) {
         }
         return null;
     }
+
+    private FailureReason getFailureReason(Operation operation) {
+        HistoricVariableInstance variable = flowableFacade.getHistoricVariableInstance(operation.getProcessId(),
+                                                                                       Variables.FAILURE_REASON_CODE.getName());
+        if (variable == null || variable.getValue() == null) {
+            return null;
+        }
+        FailureReason.Code code = FailureReason.Code.valueOf(variable.getValue().toString());
+        return ImmutableFailureReason.builder()
+                                     .code(code)
+                                     .retryable(isRetryable(code))
+                                     .build();
+    }
+
+    private boolean isRetryable(FailureReason.Code code) {
+        return code == FailureReason.Code.SERVICE_OPERATION_FAILURE
+            || code == FailureReason.Code.CF_API_UNAVAILABLE
+            || code == FailureReason.Code.INTERNAL_ERROR;
+    }
 }

multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelperTest.java

@@ -11,6 +11,7 @@
 import org.cloudfoundry.multiapps.controller.persistence.model.HistoricOperationEvent;
 import org.cloudfoundry.multiapps.controller.persistence.services.OperationService;
 import org.cloudfoundry.multiapps.controller.process.Constants;
+import org.cloudfoundry.multiapps.controller.process.flowable.FlowableFacade;
 import org.cloudfoundry.multiapps.controller.process.metadata.ProcessTypeToOperationMetadataMapper;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -29,13 +30,15 @@ class OperationsHelperTest {
     private ProcessTypeToOperationMetadataMapper metadataMapper;
     @Mock
     private ProcessHelper processHelper;
+    @Mock
+    private FlowableFacade flowableFacade;
 
     private final OperationsHelper operationsHelper;
 
     OperationsHelperTest() throws Exception {
         MockitoAnnotations.openMocks(this)
                           .close();
-        operationsHelper = new OperationsHelper(operationService, metadataMapper, processHelper);
+        operationsHelper = new OperationsHelper(operationService, metadataMapper, processHelper, flowableFacade);
     }
 
     @Test

multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImpl.java

@@ -274,6 +274,8 @@ private Operation getOperationByOperationGuidAndSpaceGuid(String operationId, St
                                                   .spaceId(spaceGuid)
                                                   .singleResult();
             operation = operationsHelper.addErrorType(operation);
+            operation = operationsHelper.addDryRunResult(operation);
+            operation = operationsHelper.addOutputs(operation);
             return operationsHelper.releaseLockIfNeeded(operation);
         } catch (NoResultException e) {
             throw new NotFoundException(e, Messages.OPERATION_0_NOT_FOUND, operationId);