[ https://issues.apache.org/jira/browse/CLOUDSTACK-2330?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Rajesh Battala updated CLOUDSTACK-2330: --------------------------------------- Attachment: deployKVM.png > [Automation]QEMU domains crash and report to be tainted > ------------------------------------------------------- > > Key: CLOUDSTACK-2330 > URL: https://issues.apache.org/jira/browse/CLOUDSTACK-2330 > Project: CloudStack > Issue Type: Bug > Security Level: Public(Anyone can view this level - this is the > default.) > Components: Management Server > Affects Versions: 4.2.0 > Environment: KVM, > Master build > Reporter: Rayees Namathponnan > Assignee: Rajesh Battala > Priority: Blocker > Fix For: 4.2.0 > > Attachments: CLOUDSTACK-2330.rar, CLOUDSTACK-2330.zip, deployKVM.png > > > Automation running on KVM with advanced zone,VR deployment failing during > automation run > Observed below error in MS log, attaching MS and agent log > 2013-05-04 13:43:13,032 DEBUG > [network.router.VirtualNetworkApplianceManagerImpl] (Job-Executor-99:job-369) > Reapplying vm data (userData and metaData) entries as a part of domR > VM[DomainRouter|r-155-VM] start... > 2013-05-04 13:43:13,040 DEBUG [agent.transport.Request] > (Job-Executor-99:job-369) Seq 4-98894092: Sending { Cmd , MgmtId: > 29066118877352, via: 4, Ver: v1, Flags: 100111, [{"Sta > rtCommand":{"vm":{"id":155,"name":"r-155-VM","type":"DomainRouter","cpus":1,"minSpeed":500,"maxSpeed":500,"minRam":134217728,"maxRam":134217728,"arch":"x86_64","os":"Debian > GNU/ > Linux 5.0 (32-bit)","bootArgs":" template=domP name=r-155-VM > eth2ip=10.223.122.89 eth2mask=255.255.255.192 gateway=10.223.122.65 > eth0ip=10.1.1.1 eth0mask=255.255.255.0 domain=cs > 2auto.advanced dhcprange=10.1.1.1 eth1ip=169.254.3.220 eth1mask=255.255.0.0 > type=router disable_rp_filter=true > dns1=72.52.126.11","rebootOnCrash":false,"enableHA":true,"limitCpu > Use":false,"vncPassword":"a3b0c243e954e90e","params":{},"uuid":"f114bcc5-1f3f-4fcb-85f7-3ed9d40467dc","disks":[{"id":155,"name":"ROOT-155","size":741212160,"type":"ROOT","storag > ePoolType":"NetworkFilesystem","storagePoolUuid":"fff90cb5-06dd-33b3-8815-d78c08ca01d9","deviceId":0}],"nics":[{"deviceId":2,"networkRateMbps":200,"defaultNic":true,"uuid":"1dee > a081-1ca4-4af5-ba98-bdac0dac722f","ip":"10.223.122.89","netmask":"255.255.255.192","gateway":"10.223.122.65","mac":"06:31:6c:00:00:52","dns1":"72.52.126.11","broadcastType":"Vla > n","type":"Public","broadcastUri":"vlan://1221","isolationUri":"vlan://1221","isSecurityGroupEnabled":false},{"deviceId":0,"networkRateMbps":200,"defaultNic":false,"uuid":"abb44 > 284-bbc9-41e6-a1d5-53b2a9256ee6","ip":"10.1.1.1","netmask":"255.255.255.0","mac":"02:00:71:47:00:02","dns1":"72.52.126.11","broadcastType":"Vlan","type":"Guest","broadcastUri":" > vlan://2354","isolationUri":"vlan://2354","isSecurityGroupEnabled":false},{"deviceId":1,"networkRateMbps":-1,"defaultNic":false,"uuid":"8b060664-4c4f-4f0f-bffc-87871cbab662","ip > ":"169.254.3.220","netmask":"255.255.0.0","gateway":"169.254.0.1","mac":"0e:00:a9:fe:03:dc","broadcastType":"LinkLocal","type":"Control","isSecurityGroupEnabled":false}]},"hostI > p":"10.223.50.66","wait":0}},{"check.CheckSshCommand":{"ip":"169.254.3.220","port":3922,"interval":6,"retries":100,"name":"r-155-VM","wait":0}},{"GetDomRVersionCmd":{"accessDeta > ils":{"router.ip":"169.254.3.220","router.name":"r-155-VM"},"wait":0}},{},{"routing.IpAssocCommand":{"ipAddresses":[{"accountId":2,"publicIp":"10.223.122.89","sourceNat":true,"a > dd":true,"oneToOneNat":false,"firstIP":true,"vlanId":"1221","vlanGateway":"10.223.122.65","vlanNetmask":"255.255.255.192","vifMacAddress":"06:bb:60:00:00:52","networkRate":200," > trafficType":"Public"}],"accessDetails":{"router.guest.ip":"10.1.1.1","zone.network.type":"Advanced","router.ip":"169.254.3.220","router.name":"r-155-VM"},"wait":0}}] > } > 2013-05-04 13:43:13,306 DEBUG [agent.transport.Request] > (AgentManager-Handler-10:null) Seq 4-98894092: Processing: { Ans: , MgmtId: > 29066118877352, via: 4, Ver: v1, Flags: 110, > > [{"Answer":{"result":false,"details":"com.cloud.utils.exception.CloudRuntimeException: > org.libvirt.LibvirtException: invalid argument: > virStorageVolLookupByName\n\tat com.cloud > .hypervisor.kvm.storage.LibvirtStorageAdaptor.getVolume(LibvirtStorageAdaptor.java:95)\n\tat > > com.cloud.hypervisor.kvm.storage.LibvirtStorageAdaptor.getPhysicalDisk(LibvirtStorag > eAdaptor.java:390)\n\tat > com.cloud.hypervisor.kvm.storage.LibvirtStoragePool.getPhysicalDisk(LibvirtStoragePool.java:123)\n\tat > com.cloud.hypervisor.kvm.resource.LibvirtComputin > gResource.createVbd(LibvirtComputingResource.java:3323)\n\tat > com.cloud.hypervisor.kvm.resource.LibvirtComputingResource.execute(LibvirtComputingResource.java:3214)\n\tat > com.cl > oud.hypervisor.kvm.resource.LibvirtComputingResource.executeRequest(LibvirtComputingResource.java:1172)\n\tat > com.cloud.agent.Agent.processRequest(Agent.java:525)\n\tat com.clou > d.agent.Agent$AgentRequestHandler.doTask(Agent.java:852)\n\tat > com.cloud.utils.nio.Task.run(Task.java:83)\n\tat > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecu > tor.java:1110)\n\tat > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)\n\tat > java.lang.Thread.run(Thread.java:679)\n","wait":0}},{"Answer":{"result > ":false,"details":"Stopped by previous > failure","wait":0}},{"Answer":{"result":false,"details":"Stopped by previous > failure","wait":0}},{"Answer":{"result":false,"details":"Stop > ped by previous > failure","wait":0}},{"Answer":{"result":false,"details":"Stopped by previous > failure","wait":0}}] } > 2013-05-04 13:43:13,306 DEBUG [agent.manager.AgentAttache] > (AgentManager-Handler-10:null) Seq 4-98894092: No more commands found > 2013-05-04 13:43:13,306 DEBUG [agent.transport.Request] > (Job-Executor-99:job-369) Seq 4-98894092: Received: { Ans: , MgmtId: > 29066118877352, via: 4, Ver: v1, Flags: 110, { Answ > er, Answer, Answer, Answer, Answer } } > 2013-05-04 13:43:13,311 ERROR [cloud.vm.VirtualMachineManagerImpl] > (Job-Executor-99:job-369) Failed to start instance VM[DomainRouter|r-155-VM] > com.cloud.utils.exception.CloudRuntimeException: Unable to get answer that is > of class com.cloud.agent.api.StartAnswer > at com.cloud.agent.manager.Commands.getAnswer(Commands.java:80) > at > com.cloud.vm.VirtualMachineManagerImpl.advanceStart(VirtualMachineManagerImpl.java:841) > at > com.cloud.vm.VirtualMachineManagerImpl.start(VirtualMachineManagerImpl.java:529) > at > com.cloud.network.router.VirtualNetworkApplianceManagerImpl.start(VirtualNetworkApplianceManagerImpl.java:2641) > at > com.cloud.network.router.VirtualNetworkApplianceManagerImpl.startVirtualRouter(VirtualNetworkApplianceManagerImpl.java:1850) > at > com.cloud.network.router.VirtualNetworkApplianceManagerImpl.startRouters(VirtualNetworkApplianceManagerImpl.java:1950) > at > com.cloud.network.router.VirtualNetworkApplianceManagerImpl.deployVirtualRouterInGuestNetwork(VirtualNetworkApplianceManagerImpl.java:1928) > at > com.cloud.network.element.VirtualRouterElement.implement(VirtualRouterElement.java:176) > at > com.cloud.network.NetworkManagerImpl.implementNetworkElementsAndResources(NetworkManagerImpl.java:1576) > at > com.cloud.network.NetworkManagerImpl.implementNetwork(NetworkManagerImpl.java:1481) > at > com.cloud.utils.component.ComponentInstantiationPostProcessor$InterceptorDispatcher.intercept(ComponentInstantiationPostProcessor.java:125) > What could be reason to mark the VM as tainted? > Libvirt.log: > ===== > 2013-05-04 07:03:54.035+0000: 26209: warning : qemuDomainObjTaint:1297 : > Domain id=2 name='r-6-VM' uuid=5b34a307-7433-3ae8-b172-87a5668e193b is > tainted: high-privileges > 2013-05-04 07:13:36.851+0000: 26206: error : qemuMonitorIO:574 : internal > error End of file from monitor > r-6-vm.log: (guest log) > ================== > 2013-05-04 07:03:54.034+0000: starting up > LC_ALL=C PATH=/sbin:/usr/sbin:/bin:/usr/bin QEMU_AUDIO_DRV=none > /usr/libexec/qemu-kvm -S -M rhel6.3.0 -enable-kvm -m 128 -smp > 1,sockets=1,cores=1,threads=1 -name r-6-VM -uuid > 5b34a307-7433-3ae8-b172-87a5668e193b -nodefconfig -nodefaults > +-chardev > socket,id=charmonitor,path=/var/lib/libvirt/qemu/r-6-VM.monitor,server,nowait > -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown > -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device > +virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x6 -drive > file=/mnt/fff90cb5-06dd-33b3-8815-d78c08ca01d9/5b1702c1-ddb8-4ed3-b24b-ed1bea5ec300,if=none,id=drive-virtio-disk0,format=qcow2,cache=none > -device > +virtio-blk-pci,scsi=off,bus=pci.0,addr=0x7,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=2 > -drive > file=/usr/share/cloudstack-common/vms/systemvm.iso,if=none,media=cdrom,id=drive-ide0-1-0,readonly=on,format=raw,cache=none > -device > +ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0,bootindex=1 > -netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=29 -device > virtio-net-pci,netdev=hostnet0,id=net0,mac=02:00:4c:23:00:02,bus=pci.0,addr=0x3 > -netdev > +tap,fd=30,id=hostnet1,vhost=on,vhostfd=31 -device > virtio-net-pci,netdev=hostnet1,id=net1,mac=0e:00:a9:fe:01:04,bus=pci.0,addr=0x4 > -netdev tap,fd=32,id=hostnet2,vhost=on,vhostfd=33 -device > +virtio-net-pci,netdev=hostnet2,id=net2,mac=06:c5:2c:00:00:3f,bus=pci.0,addr=0x5 > -chardev pty,id=charserial0 -device > isa-serial,chardev=charserial0,id=serial0 -chardev > +socket,id=charchannel0,path=/var/lib/libvirt/qemu/r-6-VM.agent,server,nowait > -device > virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=r-6-VM.vport > -device usb-tablet,id=input0 -vnc 0.0.0.0:1 -vga cirrus > +-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x8 > Domain id=2 is tainted: high-privileges > char device redirected to /dev/pts/1 > qemu: terminating on signal 15 from pid 26206 > 2013-05-04 13:57:11.189+0000: shutting down > Cloudstackagent err: > libvir: QEMU error : Domain not found: no domain with matching uuid > '5b34a307-7433-3ae8-b172-87a5668e193b' -- This message is automatically generated by JIRA. If you think it was sent incorrectly, please contact your JIRA administrators For more information on JIRA, see: http://www.atlassian.com/software/jira