diff --git a/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/FailureReason.java b/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/FailureReason.java new file mode 100644 index 0000000000..d3ee507964 --- /dev/null +++ b/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/FailureReason.java @@ -0,0 +1,36 @@ +package org.cloudfoundry.multiapps.controller.api.model; + +import org.cloudfoundry.multiapps.common.Nullable; +import org.immutables.value.Value; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@Value.Immutable +@JsonSerialize(as = ImmutableFailureReason.class) +@JsonDeserialize(as = ImmutableFailureReason.class) +public interface FailureReason { + + enum Code { + CONTENT_VALIDATION_ERROR, + MODULE_STARTUP_FAILURE, + SERVICE_OPERATION_FAILURE, + CF_API_UNAVAILABLE, + QUOTA_EXCEEDED, + INTERNAL_ERROR + } + + @Nullable + @JsonProperty("code") + Code getCode(); + + @Nullable + @JsonProperty("retryable") + Boolean getRetryable(); + + @Nullable + @JsonProperty("summary") + String getSummary(); + +} diff --git a/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/Operation.java b/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/Operation.java index 9aa5af8ac6..25b9ef92fd 100644 --- a/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/Operation.java +++ b/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/Operation.java @@ -100,6 +100,9 @@ public static List getFinalStates() { @Nullable public abstract ErrorType getErrorType(); + @Nullable + public abstract FailureReason getFailureReason(); + public abstract List getMessages(); @AllowNulls diff --git a/multiapps-controller-api/src/main/resources/mtarest.yaml b/multiapps-controller-api/src/main/resources/mtarest.yaml index 851d4d6640..93a3260873 100644 --- a/multiapps-controller-api/src/main/resources/mtarest.yaml +++ b/multiapps-controller-api/src/main/resources/mtarest.yaml @@ -779,6 +779,9 @@ definitions: enum: - "CONTENT" - "INFRASTRUCTURE" + failureReason: + readOnly: true + $ref: "#/definitions/FailureReason" messages: type: "array" readOnly: true @@ -789,6 +792,25 @@ definitions: readOnly: true additionalProperties: type: "object" + FailureReason: + type: "object" + properties: + code: + type: "string" + readOnly: true + enum: + - "CONTENT_VALIDATION_ERROR" + - "MODULE_STARTUP_FAILURE" + - "SERVICE_OPERATION_FAILURE" + - "CF_API_UNAVAILABLE" + - "QUOTA_EXCEEDED" + - "INTERNAL_ERROR" + retryable: + type: "boolean" + readOnly: true + summary: + type: "string" + readOnly: true ProcessType: type: "object" properties: diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandler.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandler.java index 8ee94acfac..f5c6e28500 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandler.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandler.java @@ -11,9 +11,12 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.cloudfoundry.multiapps.common.ContentException; +import org.cloudfoundry.multiapps.controller.api.model.FailureReason; import org.cloudfoundry.multiapps.controller.api.model.ImmutableOperation; import org.cloudfoundry.multiapps.controller.api.model.Operation; import org.cloudfoundry.multiapps.controller.core.cloudlogging.CloudLoggingServiceClient; +import org.cloudfoundry.multiapps.controller.client.facade.CloudControllerException; +import org.cloudfoundry.multiapps.controller.client.facade.CloudServiceBrokerException; import org.cloudfoundry.multiapps.controller.persistence.model.HistoricOperationEvent; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableHistoricOperationEvent; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableProgressMessage; @@ -24,15 +27,20 @@ import org.cloudfoundry.multiapps.controller.persistence.services.ProgressMessageService; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.flowable.FlowableFacade; +import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.flowable.common.engine.api.delegate.event.FlowableEngineEvent; import org.flowable.engine.HistoryService; import org.flowable.engine.ProcessEngineConfiguration; import org.flowable.engine.impl.context.Context; import org.flowable.engine.runtime.Execution; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Named public class OperationInErrorStateHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(OperationInErrorStateHandler.class); + protected final FlowableFacade flowableFacade; private final OperationService operationService; private final ProgressMessageService progressMessageService; @@ -58,6 +66,7 @@ public void handle(FlowableEngineEvent event, String errorMessage) { public void handle(FlowableEngineEvent event, Throwable throwable) { HistoricOperationEvent.EventType eventType = toEventType(throwable); + storeFailureReasonCode(event, throwable); handle(event, eventType, throwable.getMessage()); } @@ -74,6 +83,31 @@ HistoricOperationEvent.EventType toEventType(Throwable throwable) { : HistoricOperationEvent.EventType.FAILED_BY_INFRASTRUCTURE_ERROR; } + FailureReason.Code toFailureReasonCode(Throwable throwable) { + if (hasCause(throwable, ContentException.class)) { + return FailureReason.Code.CONTENT_VALIDATION_ERROR; + } + if (hasCause(throwable, CloudServiceBrokerException.class)) { + return FailureReason.Code.SERVICE_OPERATION_FAILURE; + } + if (hasCause(throwable, CloudControllerException.class)) { + return FailureReason.Code.CF_API_UNAVAILABLE; + } + return FailureReason.Code.INTERNAL_ERROR; + } + + private void storeFailureReasonCode(FlowableEngineEvent event, Throwable throwable) { + String processInstanceId = flowableFacade.getProcessInstanceId(event.getExecutionId()); + FailureReason.Code code = toFailureReasonCode(throwable); + try { + getProcessEngineConfiguration().getRuntimeService() + .setVariable(processInstanceId, Variables.FAILURE_REASON_CODE.getName(), + code.name()); + } catch (Exception e) { + LOGGER.warn("Failed to store failure reason code for process {}: {}", processInstanceId, e.getMessage()); + } + } + private boolean hasCause(Throwable throwable, Class clazz) { return ExceptionUtils.getThrowableList(throwable) .stream() diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelper.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelper.java index e972d2e3e5..dfe26882e8 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelper.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelper.java @@ -1,17 +1,27 @@ package org.cloudfoundry.multiapps.controller.process.util; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import jakarta.inject.Inject; import jakarta.inject.Named; import org.cloudfoundry.multiapps.controller.api.model.ErrorType; +import org.cloudfoundry.multiapps.controller.api.model.FailureReason; +import org.cloudfoundry.multiapps.controller.api.model.ImmutableDryRunResult; +import org.cloudfoundry.multiapps.controller.api.model.ImmutableFailureReason; import org.cloudfoundry.multiapps.controller.api.model.ImmutableOperation; import org.cloudfoundry.multiapps.controller.api.model.Operation; +import org.cloudfoundry.multiapps.controller.process.client.DryRunChangeSet; import org.cloudfoundry.multiapps.controller.persistence.model.HistoricOperationEvent; import org.cloudfoundry.multiapps.controller.persistence.services.OperationService; +import org.cloudfoundry.multiapps.controller.process.flowable.FlowableFacade; import org.cloudfoundry.multiapps.controller.process.metadata.ProcessTypeToOperationMetadataMapper; +import org.cloudfoundry.multiapps.controller.process.variables.Serializer; +import org.cloudfoundry.multiapps.controller.process.variables.Variables; +import org.flowable.variable.api.history.HistoricVariableInstance; @Named public class OperationsHelper { @@ -19,14 +29,15 @@ public class OperationsHelper { private final OperationService operationService; private final ProcessTypeToOperationMetadataMapper metadataMapper; private final ProcessHelper processHelper; + private final FlowableFacade flowableFacade; @Inject public OperationsHelper(OperationService operationService, ProcessTypeToOperationMetadataMapper metadataMapper, - ProcessHelper processHelper) { + ProcessHelper processHelper, FlowableFacade flowableFacade) { this.operationService = operationService; this.metadataMapper = metadataMapper; this.processHelper = processHelper; - + this.flowableFacade = flowableFacade; } public String getProcessDefinitionKey(Operation operation) { @@ -36,11 +47,65 @@ public String getProcessDefinitionKey(Operation operation) { public Operation addErrorType(Operation operation) { if (operation.getState() == Operation.State.ERROR) { return ImmutableOperation.copyOf(operation) - .withErrorType(getErrorType(operation)); + .withErrorType(getErrorType(operation)) + .withFailureReason(getFailureReason(operation)); } return operation; } + public Operation addDryRunResult(Operation operation) { + List changeSets = findAllHistoricVariableValues(operation.getProcessId(), + Variables.DRY_RUN_CHANGE_SET.getName(), + Variables.DRY_RUN_CHANGE_SET.getSerializer()); + if (changeSets.isEmpty()) { + return operation; + } + DryRunChangeSet merged = new DryRunChangeSet(); + for (DryRunChangeSet cs : changeSets) { + addDistinct(merged.appsCreate, cs.appsCreate); + addDistinct(merged.appsUpdate, cs.appsUpdate); + addDistinct(merged.appsDelete, cs.appsDelete); + addDistinct(merged.servicesCreate, cs.servicesCreate); + addDistinct(merged.servicesUpdate, cs.servicesUpdate); + addDistinct(merged.servicesDelete, cs.servicesDelete); + addDistinct(merged.routesCreate, cs.routesCreate); + addDistinct(merged.routesDelete, cs.routesDelete); + } + return ImmutableOperation.copyOf(operation) + .withDryRunResult(ImmutableDryRunResult.builder() + .appsToCreate(merged.appsCreate) + .appsToUpdate(merged.appsUpdate) + .appsToDelete(merged.appsDelete) + .servicesToCreate(merged.servicesCreate) + .servicesToUpdate(merged.servicesUpdate) + .servicesToDelete(merged.servicesDelete) + .routesToCreate(merged.routesCreate) + .routesToDelete(merged.routesDelete) + .build()); + } + + private static void addDistinct(List target, List source) { + for (T item : source) { + if (!target.contains(item)) { + target.add(item); + } + } + } + + public Operation addOutputs(Operation operation) { + HistoricVariableInstance variable = findHistoricVariable(operation.getProcessId(), Variables.DEPLOYMENT_OUTPUTS.getName()); + if (variable == null || variable.getValue() == null) { + return operation; + } + Map outputs = Variables.DEPLOYMENT_OUTPUTS.getSerializer() + .deserialize(variable.getValue()); + if (outputs == null || outputs.isEmpty()) { + return operation; + } + return ImmutableOperation.copyOf(operation) + .withOutputs(outputs); + } + public Operation releaseLockIfNeeded(Operation operation) { // Fixes bug XSBUG-2035: Inconsistency in 'operation', 'act_hi_procinst' and 'act_ru_execution' tables if (operation.hasAcquiredLock() && operation.getState() @@ -60,6 +125,39 @@ public List releaseLocksIfNeeded(List operations) { .collect(Collectors.toList()); } + private HistoricVariableInstance findHistoricVariable(String processId, String variableName) { + HistoricVariableInstance variable = flowableFacade.getHistoricVariableInstance(processId, variableName); + if (variable != null) { + return variable; + } + for (String subProcessId : flowableFacade.getHistoricSubProcessIds(processId)) { + variable = flowableFacade.getHistoricVariableInstance(subProcessId, variableName); + if (variable != null) { + return variable; + } + } + return null; + } + + private List findAllHistoricVariableValues(String processId, String variableName, + Serializer serializer) { + List results = new ArrayList<>(); + collectHistoricVariableValues(processId, variableName, serializer, results); + return results; + } + + private void collectHistoricVariableValues(String processId, String variableName, + Serializer serializer, + List results) { + HistoricVariableInstance variable = flowableFacade.getHistoricVariableInstance(processId, variableName); + if (variable != null && variable.getValue() != null) { + results.add(serializer.deserialize(variable.getValue())); + } + for (String subProcessId : flowableFacade.getHistoricSubProcessIds(processId)) { + collectHistoricVariableValues(subProcessId, variableName, serializer, results); + } + } + private ErrorType getErrorType(Operation operation) { List historicEvents = processHelper.getHistoricOperationEventByProcessId(operation.getProcessId()); if (historicEvents.isEmpty()) { @@ -79,4 +177,23 @@ private ErrorType toErrorType(HistoricOperationEvent.EventType historicType) { } return null; } + + private FailureReason getFailureReason(Operation operation) { + HistoricVariableInstance variable = flowableFacade.getHistoricVariableInstance(operation.getProcessId(), + Variables.FAILURE_REASON_CODE.getName()); + if (variable == null || variable.getValue() == null) { + return null; + } + FailureReason.Code code = FailureReason.Code.valueOf(variable.getValue().toString()); + return ImmutableFailureReason.builder() + .code(code) + .retryable(isRetryable(code)) + .build(); + } + + private boolean isRetryable(FailureReason.Code code) { + return code == FailureReason.Code.SERVICE_OPERATION_FAILURE + || code == FailureReason.Code.CF_API_UNAVAILABLE + || code == FailureReason.Code.INTERNAL_ERROR; + } } diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelperTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelperTest.java index de7c6910da..fddd7a2b57 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelperTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationsHelperTest.java @@ -11,6 +11,7 @@ import org.cloudfoundry.multiapps.controller.persistence.model.HistoricOperationEvent; import org.cloudfoundry.multiapps.controller.persistence.services.OperationService; import org.cloudfoundry.multiapps.controller.process.Constants; +import org.cloudfoundry.multiapps.controller.process.flowable.FlowableFacade; import org.cloudfoundry.multiapps.controller.process.metadata.ProcessTypeToOperationMetadataMapper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -29,13 +30,15 @@ class OperationsHelperTest { private ProcessTypeToOperationMetadataMapper metadataMapper; @Mock private ProcessHelper processHelper; + @Mock + private FlowableFacade flowableFacade; private final OperationsHelper operationsHelper; OperationsHelperTest() throws Exception { MockitoAnnotations.openMocks(this) .close(); - operationsHelper = new OperationsHelper(operationService, metadataMapper, processHelper); + operationsHelper = new OperationsHelper(operationService, metadataMapper, processHelper, flowableFacade); } @Test diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImpl.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImpl.java index eb92cd01da..1a1bb26653 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImpl.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImpl.java @@ -274,6 +274,8 @@ private Operation getOperationByOperationGuidAndSpaceGuid(String operationId, St .spaceId(spaceGuid) .singleResult(); operation = operationsHelper.addErrorType(operation); + operation = operationsHelper.addDryRunResult(operation); + operation = operationsHelper.addOutputs(operation); return operationsHelper.releaseLockIfNeeded(operation); } catch (NoResultException e) { throw new NotFoundException(e, Messages.OPERATION_0_NOT_FOUND, operationId);