ResourceManagerImplTest.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package com.cloud.resource;

import com.cloud.agent.AgentManager;
import com.cloud.agent.api.StartupCommand;
import com.cloud.agent.api.GetVncPortAnswer;
import com.cloud.agent.api.GetVncPortCommand;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventTypes;
import com.cloud.event.EventVO;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.ha.HighAvailabilityManager;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.StorageManager;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.utils.Ternary;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.ssh.SSHCmdHelper;
import com.cloud.utils.ssh.SshException;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.dao.UserVmDetailsDao;
import com.cloud.vm.dao.VMInstanceDao;
import com.trilead.ssh2.Connection;
import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd;
import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.BDDMockito;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedConstruction;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;

import static com.cloud.resource.ResourceState.Event.ErrorsCorrected;
import static com.cloud.resource.ResourceState.Event.InternalEnterMaintenance;
import static com.cloud.resource.ResourceState.Event.UnableToMaintain;
import static com.cloud.resource.ResourceState.Event.UnableToMigrate;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@RunWith(MockitoJUnitRunner.class)
public class ResourceManagerImplTest {

    @Mock
    private CapacityDao capacityDao;
    @Mock
    private StorageManager storageManager;
    @Mock
    private HighAvailabilityManager haManager;
    @Mock
    private UserVmDetailsDao userVmDetailsDao;
    @Mock
    private AgentManager agentManager;
    @Mock
    private HostDao hostDao;
    @Mock
    private VMInstanceDao vmInstanceDao;
    @Mock
    private ConfigurationDao configurationDao;
    @Mock
    private VolumeDao volumeDao;

    @Spy
    @InjectMocks
    private ResourceManagerImpl resourceManager = new ResourceManagerImpl();

    @Mock
    private HostVO host;
    @Mock
    private VMInstanceVO vm1;
    @Mock
    private VMInstanceVO vm2;

    @Mock
    private GetVncPortAnswer getVncPortAnswerVm1;
    @Mock
    private GetVncPortAnswer getVncPortAnswerVm2;
    @Mock
    private VolumeVO rootDisk1;
    @Mock
    private VolumeVO rootDisk2;
    @Mock
    private VolumeVO dataDisk;

    @Mock
    private Connection sshConnection;

    private static long hostId = 1L;
    private static final String hostUsername = "user";
    private static final String hostPassword = "password";
    private static final String hostPrivateKey = "privatekey";
    private static final String hostPrivateIp = "192.168.1.10";

    private static long vm1Id = 1L;
    private static String vm1InstanceName = "i-1-VM";
    private static long vm2Id = 2L;
    private static String vm2InstanceName = "i-2-VM";

    private static String vm1VncAddress = "10.2.2.2";
    private static int vm1VncPort = 5900;
    private static String vm2VncAddress = "10.2.2.2";
    private static int vm2VncPort = 5901;

    private static long poolId = 1L;
    private List<VolumeVO> rootDisks;
    private List<VolumeVO> dataDisks;
    private MockedStatic<SSHCmdHelper> sshHelperMocked;
    private MockedStatic<ActionEventUtils> actionEventUtilsMocked;
    private MockedConstruction<GetVncPortCommand> getVncPortCommandMockedConstruction;
    private AutoCloseable closeable;

    private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException {
        Field f = ConfigKey.class.getDeclaredField(name);
        f.setAccessible(true);
        f.set(configKey, o);
    }

    @Before
    public void setup() throws Exception {
        closeable = MockitoAnnotations.openMocks(this);
        when(host.getType()).thenReturn(Host.Type.Routing);
        when(host.getId()).thenReturn(hostId);
        when(host.getResourceState()).thenReturn(ResourceState.Enabled);
        when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM);
        when(hostDao.findById(hostId)).thenReturn(host);
        when(host.getDetail("username")).thenReturn(hostUsername);
        when(host.getDetail("password")).thenReturn(hostPassword);
        when(configurationDao.getValue("ssh.privatekey")).thenReturn(hostPrivateKey);
        when(host.getStatus()).thenReturn(Status.Up);
        when(host.getPrivateIpAddress()).thenReturn(hostPrivateIp);
        when(vm1.getId()).thenReturn(vm1Id);
        when(vm2.getId()).thenReturn(vm2Id);
        when(vm1.getInstanceName()).thenReturn(vm1InstanceName);
        when(vm2.getInstanceName()).thenReturn(vm2InstanceName);
        when(vmInstanceDao.listByHostId(hostId)).thenReturn(new ArrayList<>());
        when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(new ArrayList<>());
        when(vmInstanceDao.listNonMigratingVmsByHostEqualsLastHost(hostId)).thenReturn(new ArrayList<>());
        actionEventUtilsMocked = Mockito.mockStatic(ActionEventUtils.class);
        BDDMockito.given(ActionEventUtils.onCompletedActionEvent(anyLong(), anyLong(), anyString(), anyString(), anyString(), anyLong(), anyString(), anyLong()))
                .willReturn(1L);
        when(getVncPortAnswerVm1.getAddress()).thenReturn(vm1VncAddress);
        when(getVncPortAnswerVm1.getPort()).thenReturn(vm1VncPort);
        when(getVncPortAnswerVm2.getAddress()).thenReturn(vm2VncAddress);
        when(getVncPortAnswerVm2.getPort()).thenReturn(vm2VncPort);
        getVncPortCommandMockedConstruction = Mockito.mockConstruction(GetVncPortCommand.class, (mock,context) -> {
            if (context.arguments().get(0).equals(vm1Id) && context.arguments().get(1) == vm1InstanceName) {
                when(agentManager.easySend(eq(hostId), eq(mock))).thenReturn(getVncPortAnswerVm1);
            } else if (context.arguments().get(0).equals(vm2Id) && context.arguments().get(1) == vm2InstanceName) {
                when(agentManager.easySend(eq(hostId), eq(mock))).thenReturn(getVncPortAnswerVm2);
            }
        });

        sshHelperMocked = Mockito.mockStatic(SSHCmdHelper.class);
        BDDMockito.given(SSHCmdHelper.acquireAuthorizedConnection(eq(hostPrivateIp), eq(22),
                eq(hostUsername), eq(hostPassword), eq(hostPrivateKey))).willReturn(sshConnection);
        BDDMockito.given(SSHCmdHelper.sshExecuteCmdOneShot(eq(sshConnection),
                eq("service cloudstack-agent restart"))).
                willReturn(new SSHCmdHelper.SSHCmdResult(0,"",""));

        overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "true");

        rootDisks = Arrays.asList(rootDisk1, rootDisk2);
        dataDisks = Collections.singletonList(dataDisk);
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(rootDisks);
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks);
    }

    @After
    public void tearDown() throws Exception {
        sshHelperMocked.close();
        actionEventUtilsMocked.close();
        getVncPortCommandMockedConstruction.close();
        closeable.close();
    }

    @Test(expected = InvalidParameterValueException.class)
    public void testCheckForDuplicateHostThrowsWhenIpAlreadyExists() {
        when(hostDao.findByIp("10.0.0.10")).thenReturn(host);
        resourceManager.checkForDuplicateHost("http://10.0.0.10");
    }

    @Test
    public void testCheckForDuplicateHostAllowsUniqueHost() {
        when(hostDao.findByIp("10.0.0.30")).thenReturn(null);
        resourceManager.checkForDuplicateHost("http://10.0.0.30");
        verify(hostDao, times(1)).findByIp("10.0.0.30");
    }

    @Test
    public void testCheckForDuplicateHostIgnoresNonRoutingHost() {
        when(host.getType()).thenReturn(Host.Type.SecondaryStorage);
        when(hostDao.findByIp("10.0.0.20")).thenReturn(host);
        resourceManager.checkForDuplicateHost("nfs://10.0.0.20/export/secondary");
        verify(hostDao, times(1)).findByIp("10.0.0.20");
    }

    @Test
    public void testCheckAndMaintainEnterMaintenanceModeNoVms() throws NoTransitionException {
        // Test entering into maintenance with no VMs running on host.
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager).setHostIntoMaintenance(host);
        verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
        verify(resourceManager).resourceStateTransitTo(eq(host), eq(InternalEnterMaintenance), anyLong());

        Assert.assertTrue(enterMaintenanceMode);
    }

    @Test
    public void testCheckAndMaintainProceedsWithPrepareForMaintenanceRunningVms() throws NoTransitionException {
        // Test proceeding through with no events if pending migrating works / retries left.
        setupRunningVMs();
        setupPendingMigrationRetries();
        verifyNoChangeInMaintenance();
    }

    @Test
    public void testCheckAndMaintainErrorInMaintenanceRunningVms() throws NoTransitionException {
        // Test entering into ErrorInMaintenance when no pending migrations etc, and due to - Running VMs
        setupRunningVMs();
        setupNoPendingMigrationRetries();
        verifyErrorInMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInMaintenanceWithErrorVms() throws NoTransitionException {
        // Test entering into ErrorInMaintenance when no pending migrations etc, and due to - no migrating but error VMs
        setupErrorVms();
        setupNoPendingMigrationRetries();
        verifyErrorInMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceFailedMigrationsPendingRetries() throws NoTransitionException {
        // Test entering into ErrorInPrepareForMaintenance when pending migrations retries and due to - Failed Migrations
        setupFailedMigrations();
        setupPendingMigrationRetries();
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Running)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceWithErrorVmsPendingRetries() throws NoTransitionException {
        // Test entering into ErrorInMaintenance when pending migrations retries due to - no migrating but error VMs
        setupErrorVms();
        setupPendingMigrationRetries();
        when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceFailedMigrationsAndMigratingVms() throws NoTransitionException {
        // Test entering into ErrorInPrepareForMaintenance when no pending migrations retries
        // but executing migration and due to - Failed Migrations
        setupFailedMigrations();
        setupNoPendingMigrationRetries();
        when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceWithErrorVmsAndMigratingVms() throws NoTransitionException {
        // Test entering into ErrorInPrepareForMaintenance when no pending migrations retries
        // but executing migration and due to - Error Vms
        setupErrorVms();
        setupNoPendingMigrationRetries();
        when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceFailedMigrationsAndStoppingVms() throws NoTransitionException {
        // Test entering into ErrorInPrepareForMaintenance when no pending migrations retries
        // but stopping VMs and due to - Failed Migrations
        setupFailedMigrations();
        setupNoPendingMigrationRetries();
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Stopping)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainReturnsToPrepareForMaintenanceRunningVms() throws NoTransitionException {
        // Test switching back to PrepareForMaintenance
        when(host.getResourceState()).thenReturn(ResourceState.ErrorInPrepareForMaintenance);
        setupRunningVMs();
        setupPendingMigrationRetries();
        verifyReturnToPrepareForMaintenanceCalls();
    }

    @Test
    public void testConfigureVncAccessForKVMHostFailedMigrations() {
        when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM);
        List<VMInstanceVO> vms = Arrays.asList(vm1, vm2);
        resourceManager.configureVncAccessForKVMHostFailedMigrations(host, vms);
        verify(agentManager).pullAgentOutMaintenance(hostId);
        verify(resourceManager).setKVMVncAccess(hostId, vms);
        verify(agentManager, times(vms.size())).easySend(eq(hostId), any(GetVncPortCommand.class));
        verify(agentManager).pullAgentToMaintenance(hostId);
        verify(userVmDetailsDao).addDetail(eq(vm1Id), eq("kvm.vnc.address"), eq(vm1VncAddress), anyBoolean());
        verify(userVmDetailsDao).addDetail(eq(vm1Id), eq("kvm.vnc.port"), eq(String.valueOf(vm1VncPort)), anyBoolean());
        verify(userVmDetailsDao).addDetail(eq(vm2Id), eq("kvm.vnc.address"), eq(vm2VncAddress), anyBoolean());
        verify(userVmDetailsDao).addDetail(eq(vm2Id), eq("kvm.vnc.port"), eq(String.valueOf(vm2VncPort)), anyBoolean());
    }

    @Test(expected = CloudRuntimeException.class)
    public void testGetHostCredentialsMissingParameter() {
        when(host.getDetail("password")).thenReturn(null);
        when(configurationDao.getValue("ssh.privatekey")).thenReturn(null);
        resourceManager.getHostCredentials(host);
    }

    private HostVO mockExistingRoutingHost(long dcId, Long podId, Long clusterId) {
        HostVO existing = Mockito.mock(HostVO.class);
        when(existing.getType()).thenReturn(Host.Type.Routing);
        when(existing.getDataCenterId()).thenReturn(dcId);
        when(existing.getPodId()).thenReturn(podId);
        when(existing.getClusterId()).thenReturn(clusterId);
        when(existing.getUuid()).thenReturn("host-uuid");
        return existing;
    }

    @Test(expected = InvalidParameterValueException.class)
    public void testValidateExistingHostLocationImmutableRejectsZoneChange() {
        HostVO existing = mockExistingRoutingHost(1L, 10L, 100L);
        StartupCommand startup = Mockito.mock(StartupCommand.class);
        when(startup.getPrivateIpAddress()).thenReturn("10.10.10.10");
        resourceManager.validateExistingHostLocationImmutable(existing, false, 2L, 10L, 100L, startup);
    }

    @Test(expected = InvalidParameterValueException.class)
    public void testValidateExistingHostLocationImmutableRejectsPodChange() {
        HostVO existing = mockExistingRoutingHost(1L, 10L, 100L);
        StartupCommand startup = Mockito.mock(StartupCommand.class);
        when(startup.getPrivateIpAddress()).thenReturn("10.10.10.10");
        resourceManager.validateExistingHostLocationImmutable(existing, false, 1L, 11L, 100L, startup);
    }

    @Test(expected = InvalidParameterValueException.class)
    public void testValidateExistingHostLocationImmutableRejectsClusterChange() {
        HostVO existing = mockExistingRoutingHost(1L, 10L, 100L);
        StartupCommand startup = Mockito.mock(StartupCommand.class);
        when(startup.getPrivateIpAddress()).thenReturn("10.10.10.10");
        resourceManager.validateExistingHostLocationImmutable(existing, false, 1L, 10L, 101L, startup);
    }

    @Test
    public void testValidateExistingHostLocationImmutableAllowsSameTupleReconnect() {
        HostVO existing = mockExistingRoutingHost(1L, 10L, 100L);
        resourceManager.validateExistingHostLocationImmutable(existing, false, 1L, 10L, 100L, null);
    }

    @Test
    public void testValidateExistingHostLocationImmutableAllowsNewHost() {
        HostVO existing = mockExistingRoutingHost(2L, 20L, 200L);
        resourceManager.validateExistingHostLocationImmutable(existing, true, 1L, 10L, 100L, null);
    }

    @Test
    public void testValidateExistingHostLocationImmutableSkipsNonRoutingHost() {
        HostVO existing = Mockito.mock(HostVO.class);
        when(existing.getType()).thenReturn(Host.Type.SecondaryStorageVM);
        resourceManager.validateExistingHostLocationImmutable(existing, false, 1L, 10L, 100L, null);
    }

    @Test
    public void testValidateExistingHostLocationImmutableSkipsPartialLocationRow() {
        HostVO existing = Mockito.mock(HostVO.class);
        when(existing.getType()).thenReturn(Host.Type.Routing);
        when(existing.getDataCenterId()).thenReturn(1L);
        when(existing.getPodId()).thenReturn(null);
        when(existing.getClusterId()).thenReturn(null);
        resourceManager.validateExistingHostLocationImmutable(existing, false, 2L, 10L, 100L, null);
    }

    @Test
    public void testValidateExistingHostLocationImmutableSkipsNullExistingHost() {
        resourceManager.validateExistingHostLocationImmutable(null, false, 2L, 10L, 100L, null);
    }

    @Test
    public void testGetHostCredentials() {
        Ternary<String, String, String> credentials = resourceManager.getHostCredentials(host);
        Assert.assertNotNull(credentials);
        Assert.assertEquals(hostUsername, credentials.first());
        Assert.assertEquals(hostPassword, credentials.second());
        Assert.assertEquals(hostPrivateKey, credentials.third());
    }

    @Test(expected = CloudRuntimeException.class)
    public void testConnectAndRestartAgentOnHostCannotConnect() {
        BDDMockito.given(SSHCmdHelper.acquireAuthorizedConnection(eq(hostPrivateIp), eq(22),
                eq(hostUsername), eq(hostPassword), eq(hostPrivateKey))).willReturn(null);
        resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword, hostPrivateKey);
    }

    @Test(expected = CloudRuntimeException.class)
    public void testConnectAndRestartAgentOnHostCannotRestart() throws Exception {
        BDDMockito.given(SSHCmdHelper.sshExecuteCmdOneShot(eq(sshConnection),
                eq("service cloudstack-agent restart"))).willThrow(new SshException("exception"));
        resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword, hostPrivateKey);
    }

    @Test
    public void testConnectAndRestartAgentOnHost() {
        when(agentManager.getHostSshPort(any())).thenReturn(22);
        resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword, hostPrivateKey);
    }

    @Test
    public void testHandleAgentSSHEnabledNotConnectedAgent() {
        when(host.getStatus()).thenReturn(Status.Disconnected);
        when(agentManager.getHostSshPort(any())).thenReturn(22);
        resourceManager.handleAgentIfNotConnected(host, false);
        verify(resourceManager).getHostCredentials(eq(host));
        verify(resourceManager).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword), eq(hostPrivateKey));
    }

    @Test
    public void testHandleAgentSSHEnabledConnectedAgent() {
        when(host.getStatus()).thenReturn(Status.Up);
        resourceManager.handleAgentIfNotConnected(host, false);
        verify(resourceManager, never()).getHostCredentials(eq(host));
        verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword), eq(hostPrivateKey));
    }

    @Test(expected = CloudRuntimeException.class)
    public void testHandleAgentSSHDisabledNotConnectedAgent() throws NoSuchFieldException, IllegalAccessException {
        when(host.getStatus()).thenReturn(Status.Disconnected);
        overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "false");
        resourceManager.handleAgentIfNotConnected(host, false);
    }

    @Test
    public void testHandleAgentSSHDisabledConnectedAgent() {
        when(host.getStatus()).thenReturn(Status.Up);
        resourceManager.handleAgentIfNotConnected(host, false);
        verify(resourceManager, never()).getHostCredentials(eq(host));
        verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword), eq(hostPrivateKey));
    }

    @Test
    public void testHandleAgentVMsMigrating() {
        resourceManager.handleAgentIfNotConnected(host, true);
        verify(resourceManager, never()).getHostCredentials(eq(host));
        verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword), eq(hostPrivateKey));
    }

    private void setupNoPendingMigrationRetries() {
        when(haManager.hasPendingMigrationsWork(vm1.getId())).thenReturn(false);
        when(haManager.hasPendingMigrationsWork(vm2.getId())).thenReturn(false);
    }

    private void setupRunningVMs() {
        when(vmInstanceDao.listByHostId(hostId)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Migrating, VirtualMachine.State.Running, VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Error, VirtualMachine.State.Unknown)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Running)).thenReturn(Arrays.asList(vm1, vm2));
    }

    private void setupPendingMigrationRetries() {
        when(haManager.hasPendingMigrationsWork(vm1.getId())).thenReturn(true);
    }

    private void setupFailedMigrations() {
        when(vmInstanceDao.listByHostId(hostId)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Migrating, VirtualMachine.State.Running, VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Error, VirtualMachine.State.Unknown)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.listNonMigratingVmsByHostEqualsLastHost(hostId)).thenReturn(Arrays.asList(vm1));
    }

    private void setupErrorVms() {
        when(vmInstanceDao.listByHostId(hostId)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Migrating, VirtualMachine.State.Running, VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Error, VirtualMachine.State.Unknown)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Unknown, VirtualMachine.State.Error)).thenReturn(Arrays.asList(vm1));
    }

    private void verifyErrorInMaintenanceCalls() throws NoTransitionException {
        when(host.getResourceState()).thenReturn(ResourceState.PrepareForMaintenance);
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager).setHostIntoErrorInMaintenance(eq(host), any());
        verify(resourceManager, never()).setHostIntoMaintenance(any());
        verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
        verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMaintain), anyLong());
        actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent(
                anyLong(), anyLong(), eq(EventVO.LEVEL_ERROR), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR),
                anyString(), eq(hostId), isNull(), eq(0L)));
        Assert.assertFalse(enterMaintenanceMode);
    }

    private void verifyErrorInPrepareForMaintenanceCalls() throws NoTransitionException {
        when(host.getResourceState()).thenReturn(ResourceState.PrepareForMaintenance);
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager).setHostIntoErrorInPrepareForMaintenance(eq(host), any());
        verify(resourceManager, never()).setHostIntoMaintenance(any());
        verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
        verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMigrate), anyLong());
        String expectedDescription = String.format("failed to prepare host %s for maintenance due to migration or VM state errors", host);
        actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent(
                anyLong(), anyLong(), eq(EventVO.LEVEL_ERROR), eq(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR),
                eq(expectedDescription), eq(hostId), isNull(), eq(0L)));
        Assert.assertFalse(enterMaintenanceMode);
    }

    @Test
    public void testMaintenancePrepareErrorEventMapsToHostEntity() {
        Assert.assertEquals(Host.class, EventTypes.getEntityClassForEvent(EventTypes.EVENT_MAINTENANCE_PREPARE_ERROR));
    }

    private void verifyReturnToPrepareForMaintenanceCalls() throws NoTransitionException {
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager).setHostIntoPrepareForMaintenanceAfterErrorsFixed(eq(host));
        verify(resourceManager).resourceStateTransitTo(eq(host), eq(ErrorsCorrected), anyLong());
        verify(resourceManager, never()).setHostIntoMaintenance(any());
        verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any());
        Assert.assertFalse(enterMaintenanceMode);
    }

    private void verifyNoChangeInMaintenance() throws NoTransitionException {
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager, never()).setHostIntoMaintenance(any());
        verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
        verify(resourceManager, never()).resourceStateTransitTo(any(), any(), anyLong());
        Assert.assertFalse(enterMaintenanceMode);
    }

    @Test
    public void declareHostAsDegradedTestDisconnected() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Disconnected, ResourceState.Enabled, ResourceState.Degraded);
    }

    @Test
    public void declareHostAsDegradedTestAlert() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Enabled, ResourceState.Degraded);
    }

    @Test(expected = InvalidParameterValueException.class)
    public void declareHostAsDegradedExpectNoTransitionException() throws NoTransitionException {
        Status[] statusArray = Status.values();
        for (int i = 0; i < statusArray.length - 1; i++) {
            if (statusArray[i] != Status.Alert && statusArray[i] != Status.Disconnected) {
                prepareAndTestDeclareHostAsDegraded(statusArray[i], ResourceState.Enabled, ResourceState.Enabled);
            }
        }
    }

    @Test(expected = NoTransitionException.class)
    public void declareHostAsDegradedTestAlreadyDegraded() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Degraded, ResourceState.Degraded);
    }

    @Test(expected = NoTransitionException.class)
    public void declareHostAsDegradedTestOnError() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Error, ResourceState.Degraded);
    }

    @Test(expected = NoTransitionException.class)
    public void declareHostAsDegradedTestOnCreating() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Creating, ResourceState.Degraded);
    }

    @Test(expected = NoTransitionException.class)
    public void declareHostAsDegradedTestOnErrorInMaintenance() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.ErrorInPrepareForMaintenance, ResourceState.Degraded);
    }

    @Test
    public void declareHostAsDegradedTestSupportedStates() throws NoTransitionException {
        ResourceState[] states = ResourceState.values();
        for (int i = 0; i < states.length - 1; i++) {
            if (states[i] == ResourceState.Enabled
                    || states[i] == ResourceState.Maintenance
                    || states[i] == ResourceState.Disabled) {
                prepareAndTestDeclareHostAsDegraded(Status.Alert, states[i], ResourceState.Degraded);
            }
        }
    }

    private void prepareAndTestDeclareHostAsDegraded(Status hostStatus, ResourceState originalState, ResourceState expectedResourceState) throws NoTransitionException {
        DeclareHostAsDegradedCmd declareHostAsDegradedCmd = Mockito.spy(new DeclareHostAsDegradedCmd());
        HostVO hostVo = createDummyHost(hostStatus);
        hostVo.setResourceState(originalState);
        when(declareHostAsDegradedCmd.getId()).thenReturn(0l);
        when(hostDao.findById(0l)).thenReturn(hostVo);

        Host result = resourceManager.declareHostAsDegraded(declareHostAsDegradedCmd);

        Assert.assertEquals(expectedResourceState, hostVo.getResourceState());
    }

    @Test
    public void cancelHostAsDegradedTest() throws NoTransitionException {
        prepareAndTestCancelHostAsDegraded(Status.Alert, ResourceState.Degraded, ResourceState.Enabled);
    }

    @Test(expected = NoTransitionException.class)
    public void cancelHostAsDegradedTestHostNotDegraded() throws NoTransitionException {
        prepareAndTestCancelHostAsDegraded(Status.Alert, ResourceState.Enabled, ResourceState.Enabled);
    }

    private void prepareAndTestCancelHostAsDegraded(Status hostStatus, ResourceState originalState, ResourceState expectedResourceState) throws NoTransitionException {
        CancelHostAsDegradedCmd cancelHostAsDegradedCmd = Mockito.spy(new CancelHostAsDegradedCmd());
        HostVO hostVo = createDummyHost(hostStatus);
        hostVo.setResourceState(originalState);
        when(cancelHostAsDegradedCmd.getId()).thenReturn(0l);
        when(hostDao.findById(0l)).thenReturn(hostVo);

        Host result = resourceManager.cancelHostAsDegraded(cancelHostAsDegradedCmd);

        Assert.assertEquals(expectedResourceState, hostVo.getResourceState());
    }

    private HostVO createDummyHost(Status hostStatus) {
        return new HostVO(1L, "host01", Host.Type.Routing, "192.168.1.1", "255.255.255.0", null, null, null, null, null, null, null, null, null, null, UUID.randomUUID().toString(),
                hostStatus, "1.0", null, null, 1L, null, 0, 0, null, 0, null);
    }

    @Test
    public void testDestroyLocalStoragePoolVolumesBothRootDisksAndDataDisks() {
        resourceManager.destroyLocalStoragePoolVolumes(poolId);
        verify(volumeDao, times(rootDisks.size() + dataDisks.size()))
                .updateAndRemoveVolume(any(VolumeVO.class));
    }

    @Test
    public void testDestroyLocalStoragePoolVolumesOnlyRootDisks() {
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null);
        resourceManager.destroyLocalStoragePoolVolumes(poolId);
        verify(volumeDao, times(rootDisks.size())).updateAndRemoveVolume(any(VolumeVO.class));
    }

    @Test
    public void testDestroyLocalStoragePoolVolumesOnlyDataDisks() {
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null);
        resourceManager.destroyLocalStoragePoolVolumes(poolId);
        verify(volumeDao, times(dataDisks.size())).updateAndRemoveVolume(any(VolumeVO.class));
    }

    @Test
    public void testDestroyLocalStoragePoolVolumesNoDisks() {
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null);
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null);
        resourceManager.destroyLocalStoragePoolVolumes(poolId);
        verify(volumeDao, never()).updateAndRemoveVolume(any(VolumeVO.class));
    }
}