diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index b3e672e2c17d..b0df7690ff99 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -209,8 +209,10 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.StorageAccessException; import com.cloud.exception.StorageUnavailableException; +import com.cloud.ha.HaWorkVO; import com.cloud.ha.HighAvailabilityManager; import com.cloud.ha.HighAvailabilityManager.WorkType; +import com.cloud.ha.dao.HighAvailabilityDao; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -361,6 +363,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @Inject private HighAvailabilityManager _haMgr; @Inject + private HighAvailabilityDao _haDao; + @Inject private HostPodDao _podDao; @Inject private DataCenterDao _dcDao; @@ -5395,9 +5399,43 @@ private void handlePowerOffReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { && HaVmRestartHostUp.value() && vm.getHypervisorType() != HypervisorType.VMware && vm.getHypervisorType() != HypervisorType.Hyperv) { - logger.info("Detected out-of-band stop of a HA enabled VM {}, will schedule restart.", vm); + logger.info("Detected out-of-band stop of HA enabled VM {}, transitioning to Stopped and scheduling HA restart (investigate=false).", vm); if (!_haMgr.hasPendingHaWork(vm.getId())) { - _haMgr.scheduleRestart(vm, true); + // FLXCLD-2814 FIX: The power state report already confirmed the VM is off + // (e.g. OOM-killed QEMU process). We cannot use _haMgr.scheduleRestart() because it + // calls advanceStop(), which submits a VM work job and blocks waiting for completion. + // This code runs on the AgentManager-Handler thread, and the VM job queue does not + // dispatch jobs from this context — causing an indefinite block that leaves the VM + // stuck in Running. Instead we release resources, transition to Stopped, and insert + // an HA work item directly into op_ha_work (Step.Scheduled) for the HA worker to pick up. + final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); + releaseVmResources(profile, true); + + // Save lastHostId before transition (stateTransitTo sets hostId=null) + final Long lastHostId = vm.getHostId(); + try { + stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOffReport, null); + } catch (final NoTransitionException e) { + logger.warn("Failed to transition VM {} to Stopped state: {}", vm, e.getMessage()); + return; + } + + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), + VM_SYNC_ALERT_SUBJECT, String.format("VM %s(%s) stopped out-of-band (OOM-killed or crashed). HA restart scheduled.", + vm.getHostName(), vm.getInstanceName())); + + // Insert HA work item directly — bypasses advanceStop/job queue entirely + final VMInstanceVO refreshedVm = _vmDao.findByUuid(vm.getUuid()); + final Long haHostId = lastHostId != null ? lastHostId : refreshedVm.getLastHostId(); + if (haHostId == null || haHostId == 0L) { + logger.warn("Cannot schedule HA for VM {} — no valid host_id available (would violate FK constraint).", vm); + } else { + final HaWorkVO work = new HaWorkVO(refreshedVm.getId(), refreshedVm.getType(), + WorkType.HA, HighAvailabilityManager.Step.Scheduled, haHostId, + refreshedVm.getState(), 0, refreshedVm.getUpdated(), null); + _haDao.persist(work); + logger.info("Scheduled VM for HA: {}", refreshedVm); + } } else { logger.info("VM {} already has a pending HA task working on it.", vm); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 0aa094e56d99..f8af6bb211bf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -533,6 +533,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_BLOCKED, PowerState.PowerOn); POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_NOSTATE, PowerState.PowerUnknown); POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_SHUTDOWN, PowerState.PowerOff); + // Report crashed domains as PowerOff immediately instead of omitting them + // from the report (which delays detection via the "missing VM" threshold). + POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_CRASHED, PowerState.PowerOff); } public VirtualRoutingResource virtRouterResource;