kiranchavala commented on issue #11956: URL: https://github.com/apache/cloudstack/issues/11956#issuecomment-3480715481
Same issue with local <img width="1628" height="480" alt="Image" src="https://github.com/user-attachments/assets/f4ef52ea-b726-4440-b3f0-4b1832785a79"; /> cloudstack logs ``` 2025-11-03 13:57:29,157 DEBUG [c.c.a.t.Request] (Work-Job-Executor-11:[ctx-56cb586b, job-173/job-174, ctx-dc8e2acc]) (logid:50c31d05) Seq 5-8185855272699297828: Sending { Cmd , MgmtId: 206863090450418, via: 5(raspberrypi5), Ver: v1, Flags: 100111, [{"com.cloud.agent.api.PrepareForMigrationCommand":{"vm":{"id":"30","name":"s-30-VM","state":"Migrating","type":"SecondaryStorageVm","cpus":"1","minSpeed":"500","maxSpeed":"500","minRam":"(512.00 MB) 536870912","maxRam":"(512.00 MB) 536870912","arch":"aarch64","os":"Debian GNU/Linux 12 (64-bit)","platformEmulator":"Debian GNU/Linux 12 (64-bit)","bootArgs":"","enableHA":"false","limitCpuUse":"false","enableDynamicallyScaleVm":"false","details":{},"uuid":"76f608cb-4ba5-4153-85d0-a2b160c9c3b9","enterHardwareSetup":"false","disks":[{"data":{"org.apache.cloudstack.storage.to.VolumeObjectTO":{"uuid":"16f3277c-ced8-4a0d-bae8-1f093de446f6","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"4f12a46e-6 fad-47fe-a98e-b2fdcb990206","name":"raspberrypi4-local-4f12a46e","id":"4","poolType":"Filesystem","host":"192.168.55.211","path":"/var/lib/libvirt/images","port":"0","url":"Filesystem://192.168.55.211/var/lib/libvirt/images/?ROLE=Primary&STOREUUID=4f12a46e-6fad-47fe-a98e-b2fdcb990206","isManaged":"false"}},"name":"ROOT-30","size":"(4.88 GB) 5242880000","path":"16f3277c-ced8-4a0d-bae8-1f093de446f6","volumeId":"36","vmName":"s-30-VM","accountId":"1","format":"QCOW2","provisioningType":"THIN","poolId":"4","id":"36","deviceId":"0","bytesReadRate":"(0 bytes) 0","bytesWriteRate":"(0 bytes) 0","iopsReadRate":"(0 bytes) 0","iopsWriteRate":"(0 bytes) 0","hypervisorType":"KVM","directDownload":"false","deployAsIs":"false","checkpointPaths":[],"checkpointImageStoreUrls":[],"followRedirects":"false"}},"diskSeq":"0","path":"16f3277c-ced8-4a0d-bae8-1f093de446f6","type":"ROOT","details":{"storageHost":"192.168.55.211","managed":"false","storagePort":"0","storageMigrateSecretConsumer":"afd2231e-2dc 6-4829-8ff4-c646750a9177","protocoltype":"Filesystem","storage.pool.disk.wait":"60","volumeSize":"(4.88 GB) 5242880000"}}],"nics":[{"deviceId":"2","networkRateMbps":"-1","defaultNic":"true","pxeDisable":"true","nicUuid":"6e3dcfa1-397a-4fe5-9f71-cd94ebcbb47e","details":{"MacAddressChanges":"true","ForgedTransmits":"true","PromiscuousMode":"false","MacLearning":"false"},"dpdkEnabled":"false","networkId":"200","networkSegmentName":"D1-A1-Z1-S200","uuid":"5830d80a-b2cf-4b69-a863-130a8f5a1890","ip":"192.168.55.50","netmask":"255.255.255.0","gateway":"192.168.55.1","mac":"1e:00:6b:00:00:16","dns1":"8.8.8.8","broadcastType":"Vlan","type":"Public","broadcastUri":"vlan://untagged","isolationUri":"vlan://untagged","securityGroupEnabled":"false"},{"deviceId":"0","networkRateMbps":"-1","defaultNic":"false","pxeDisable":"true","nicUuid":"432400b4-ba64-4d7d-9049-efb79b955466","details":{"MacAddressChanges":"true","ForgedTransmits":"true","PromiscuousMode":"false","MacLearning":"false"},"dpdkEnabl ed":"false","networkId":"202","networkSegmentName":"D1-A1-Z1-S202","uuid":"a0379ee9-d1a7-41e9-a111-90b116cd8439","ip":"169.254.24.249","netmask":"255.255.0.0","gateway":"169.254.0.1","mac":"0e:00:a9:fe:18:f9","broadcastType":"LinkLocal","type":"Control","securityGroupEnabled":"false"},{"deviceId":"1","networkRateMbps":"-1","defaultNic":"false","pxeDisable":"true","nicUuid":"2da02b0d-e4ca-4d09-a026-90dc5457ca4f","details":{"MacAddressChanges":"true","ForgedTransmits":"true","PromiscuousMode":"false","MacLearning":"false"},"dpdkEnabled":"false","networkId":"201","networkSegmentName":"D1-A1-Z1-S201","uuid":"c225423e-b1bc-45a7-89b4-d60ce848aaf7","ip":"192.168.55.80","netmask":"255.255.255.0","gateway":"192.168.55.1","mac":"1e:00:0e:00:00:0a","broadcastType":"Native","type":"Management","securityGroupEnabled":"false"}],"vcpuMaxLimit":"1","configDriveLocation":"SECONDARY","guestOsDetails":{},"extraConfig":{},"networkIdToNetworkNameMap":{}},"rollback":"true","wait":"0","bypassHostMaintenan ce":"false"}}] } 2025-11-03 13:57:29,165 DEBUG [c.c.a.t.Request] (AgentManager-Handler-13:[]) (logid:) Seq 5-8185855272699297828: Processing: { Ans: , MgmtId: 206863090450418, via: 5, Ver: v1, Flags: 110, [{"com.cloud.agent.api.Answer":{"result":"false","details":"com.cloud.utils.exception.CloudRuntimeException: Could not fetch storage pool 4f12a46e-6fad-47fe-a98e-b2fdcb990206 from libvirt due to org.libvirt.LibvirtException: Storage pool not found: no storage pool with matching uuid '4f12a46e-6fad-47fe-a98e-b2fdcb990206' at com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.getStoragePool(KVMStoragePoolManager.java:286) at com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.getStoragePool(KVMStoragePoolManager.java:272) at com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager.disconnectPhysicalDisksViaVmSpec(KVMStoragePoolManager.java:247) at com.cloud.hypervisor.kvm.resource.wrapper.LibvirtPrepareForMigrationCommandWrapper.handleRollback(LibvirtPrepareForMigrationCommandWrapper.java:160) at com.cloud.hypervisor.kvm.resource.wrapper.LibvirtPrepareForMigrationCommandWrapper.execute(LibvirtPrepareForMigrationCommandWrapper.java:59) at com.cloud.hypervisor.kvm.resource.wrapper.LibvirtPrepareForMigrationCommandWrapper.execute(LibvirtPrepareForMigrationCommandWrapper.java:50) at com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper.execute(LibvirtRequestWrapper.java:78) at com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:2280) at com.cloud.agent.Agent.processRequest(Agent.java:813) at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:1295) at com.cloud.utils.nio.Task.call(Task.java:83) ``` ``` root@raspberrypi4:/var/lib/libvirt/images# virsh pool-list Name State Autostart ------------------------------------------------------------ 4f12a46e-6fad-47fe-a98e-b2fdcb990206 active no c7d94500-acf4-3ee0-9410-56cd2aa3fda9 active no root@raspberrypi5:~# virsh pool-list Name State Autostart ------------------------------------------------------------ 5996bfe3-686a-4543-982c-83a968a042bf active no c7d94500-acf4-3ee0-9410-56cd2aa3fda9 active no ``` agent logs ``` 2025-11-03 13:56:19,367 INFO [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) VM [s-30-VM] will have CPU shares altered from [695] to [521] as part of migration because the cgroups version differs between hosts. 2025-11-03 13:56:19,426 INFO [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) Starting live migration of instance [s-30-VM] to destination host [qemu+tls://192.168.55.210/system] having the final XML configuration: [<?xml version="1.0" encoding="UTF-8"?><domain type="kvm"> <name>s-30-VM</name> <uuid>76f608cb-4ba5-4153-85d0-a2b160c9c3b9</uuid> <description>Debian GNU/Linux 12 (64-bit)</description> <memory unit="KiB">524288</memory> <currentMemory unit="KiB">524288</currentMemory> <vcpu placement="static">1</vcpu> <cputune> <shares>521</shares> </cputune> <resource> <partition>/machine</partition> </resource> <sysinfo type="smbios"> <system> <entry name="manufacturer">Apache Software Foundation</entry> <entry name="product">CloudStack KVM Hypervisor</entry> <entry name="serial">76f608cb-4ba5-4153-85d0-a2b160c9c3b9</entry> <entry name="uuid">76f608cb-4ba5-4153-85d0-a2b160c9c3b9</entry> </system> </sysinfo> <os> <type arch="aarch64" machine="virt-8.2">hvm</type> <loader readonly="yes" type="pflash">/usr/share/AAVMF/AAVMF_CODE.fd</loader> <nvram template="/usr/share/AAVMF/AAVMF_VARS.fd">/var/lib/libvirt/qemu/nvram/s-30-VM_VARS.fd</nvram> <boot dev="cdrom"/> <boot dev="hd"/> <smbios mode="sysinfo"/> </os> <features> <acpi/> <apic/> <pae/> <gic version="2"/> </features> <cpu check="none" mode="host-passthrough"> <topology cores="1" dies="1" sockets="1" threads="1"/> </cpu> <clock offset="utc"> <timer name="kvmclock"/> </clock> <on_poweroff>destroy</on_poweroff> <on_reboot>restart</on_reboot> <on_crash>destroy</on_crash> <devices> <emulator>/usr/bin/qemu-system-aarch64</emulator> <disk device="disk" type="file"> <driver cache="none" name="qemu" type="qcow2"/> <target bus="virtio" dev="vda"/> <serial>16f3277cced84a0dbae8</serial> <address bus="0x08" domain="0x0000" function="0x0" slot="0x00" type="pci"/> <source file="/var/lib/libvirt/images/afd2231e-2dc6-4829-8ff4-c646750a9177"/></disk> <disk device="cdrom" type="file"> <driver name="qemu" type="raw"/> <target bus="scsi" dev="hdc"/> <readonly/> <address bus="0" controller="0" target="0" type="drive" unit="2"/> </disk> <controller index="0" model="qemu-xhci" type="usb"> <address bus="0x05" domain="0x0000" function="0x0" slot="0x00" type="pci"/> </controller> <controller index="0" model="pcie-root" type="pci"/> <controller index="1" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="1" port="0x8"/> <address bus="0x00" domain="0x0000" function="0x0" multifunction="on" slot="0x01" type="pci"/> </controller> <controller index="2" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="2" port="0x9"/> <address bus="0x00" domain="0x0000" function="0x1" slot="0x01" type="pci"/> </controller> <controller index="3" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="3" port="0xa"/> <address bus="0x00" domain="0x0000" function="0x2" slot="0x01" type="pci"/> </controller> <controller index="4" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="4" port="0xb"/> <address bus="0x00" domain="0x0000" function="0x3" slot="0x01" type="pci"/> </controller> <controller index="5" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="5" port="0xc"/> <address bus="0x00" domain="0x0000" function="0x4" slot="0x01" type="pci"/> </controller> <controller index="6" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="6" port="0xd"/> <address bus="0x00" domain="0x0000" function="0x5" slot="0x01" type="pci"/> </controller> <controller index="7" model="pcie-to-pci-bridge" type="pci"> <model name="pcie-pci-bridge"/> <address bus="0x01" domain="0x0000" function="0x0" slot="0x00" type="pci"/> </controller> <controller index="8" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="8" port="0xe"/> <address bus="0x00" domain="0x0000" function="0x6" slot="0x01" type="pci"/> </controller> <controller index="9" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="9" port="0xf"/> <address bus="0x00" domain="0x0000" function="0x7" slot="0x01" type="pci"/> </controller> <controller index="10" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="10" port="0x10"/> <address bus="0x00" domain="0x0000" function="0x0" multifunction="on" slot="0x02" type="pci"/> </controller> <controller index="11" model="pcie-root-port" type="pci"> <model name="pcie-root-port"/> <target chassis="11" port="0x11"/> <address bus="0x00" domain="0x0000" function="0x1" slot="0x02" type="pci"/> </controller> <controller index="0" model="lsilogic" type="scsi"> <address bus="0x07" domain="0x0000" function="0x0" slot="0x01" type="pci"/> </controller> <controller index="0" type="virtio-serial"> <address bus="0x06" domain="0x0000" function="0x0" slot="0x00" type="pci"/> </controller> <interface type="bridge"> <mac address="0e:00:a9:fe:18:f9"/> <source bridge="cloud0"/> <model type="virtio"/> <link state="up"/> <rom bar="off" file=""/> <address bus="0x02" domain="0x0000" function="0x0" slot="0x00" type="pci"/> </interface> <interface type="bridge"> <mac address="1e:00:0e:00:00:0a"/> <source bridge="cloudbr0"/> <model type="virtio"/> <link state="up"/> <rom bar="off" file=""/> <address bus="0x03" domain="0x0000" function="0x0" slot="0x00" type="pci"/> </interface> <interface type="bridge"> <mac address="1e:00:6b:00:00:16"/> <source bridge="cloudbr0"/> <model type="virtio"/> <link state="up"/> <rom bar="off" file=""/> <address bus="0x04" domain="0x0000" function="0x0" slot="0x00" type="pci"/> </interface> <serial type="pty"> <target port="0"/> </serial> <console type="pty"> <target port="0" type="serial"/> </console> <channel type="unix"> <source mode="bind" path="/var/lib/libvirt/qemu/s-30-VM.org.qemu.guest_agent.0"/> <target name="org.qemu.guest_agent.0" type="virtio"/> <address bus="0" controller="0" port="1" type="virtio-serial"/> </channel> <input bus="usb" type="tablet"> <address bus="0" port="1" type="usb"/> </input> <input bus="usb" type="keyboard"> <address bus="0" port="2" type="usb"/> </input> <input bus="usb" type="mouse"> <address bus="0" port="3" type="usb"/> </input> <graphics autoport="yes" listen="192.168.55.210" passwd="TXui6rU0" port="-1" type="vnc"> <listen address="192.168.55.210" type="address"/> </graphics> <video> <model heads="1" primary="yes" type="virtio"/> <address bus="0x0a" domain="0x0000" function="0x0" slot="0x00" type="pci"/> </video> <watchdog action="none" model="i6300esb"> <address bus="0x07" domain="0x0000" function="0x0" slot="0x02" type="pci"/> </watchdog> <memballoon model="virtio"> <address bus="0x09" domain="0x0000" function="0x0" slot="0x00" type="pci"/> </memballoon> </devices> </domain>]. 2025-11-03 13:56:20,447 INFO [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) Waiting for migration of s-30-VM to complete, waited 1000ms 2025-11-03 13:56:21,449 INFO [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) Waiting for migration of s-30-VM to complete, waited 2000ms 2025-11-03 13:56:22,452 INFO [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) Waiting for migration of s-30-VM to complete, waited 3000ms 2025-11-03 13:56:23,455 INFO [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) Waiting for migration of s-30-VM to complete, waited 4000ms 2025-11-03 13:56:24,458 INFO [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) Waiting for migration of s-30-VM to complete, waited 5000ms 2025-11-03 13:56:25,538 INFO [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) Waiting for migration of s-30-VM to complete, waited 6000ms 2025-11-03 13:56:26,541 INFO [resource.wrapper.LibvirtMig 2025-11-03 13:57:29,093 INFO [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) Migration thread of VM [s-30-VM] finished. 2025-11-03 13:57:29,095 ERROR [resource.wrapper.LibvirtMigrateCommandWrapper] (AgentRequest-Handler-2:[]) (logid:50c31d05) Can't migrate domain [s-30-VM] due to: [org.libvirt.LibvirtException: operation failed: job 'migration out' failed: Unable to read from socket: Connection reset by peer]. java.util.concurrent.ExecutionException: org.libvirt.LibvirtException: operation failed: job 'migration out' failed: Unable to read from socket: Connection reset by peer at java.base/java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.base/java.util.concurrent.FutureTask.get(FutureTask.java:205) at com.cloud.hypervisor.kvm.resource.wrapper.LibvirtMigrateCommandWrapper.execute(LibvirtMigrateCommandWrapper.java:326) at com.cloud.hypervisor.kvm.resource.wrapper.LibvirtMigrateCommandWrapper.execute(LibvirtMigrateCommandWrapper.java:91) at com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper.execute(LibvirtRequestWrapper.java:78) at com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:2280) at com.cloud.agent.Agent.processRequest(Agent.java:813) at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:1295) at com.cloud.utils.nio.Task.call(Task.java:83) at com.cloud.utils.nio.Task.call(Task.java:29) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.libvirt.LibvirtException: operation failed: job 'migration out' failed: Unable to read from socket: Connection reset by peer at org.libvirt.ErrorHandler.processError(Unknown Source) at org.libvirt.ErrorHandler.processError(Unknown Source) at org.libvirt.Domain.migrate(Unknown Source) at com.cloud.hypervisor.kvm.resource.MigrateKVMAsync.call(MigrateKVMAsync.java:142) at com.cloud.hypervisor.kvm.resource.MigrateKVMAsync.call(MigrateKVMAsync.java:34) ... 4 more ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
