Pavan Kumar Bandarupally created CLOUDSTACK-5454: ----------------------------------------------------
Summary: Deleting a snapshot from Secondary Staging store after migration returns SUCCESS Key: CLOUDSTACK-5454 URL: https://issues.apache.org/jira/browse/CLOUDSTACK-5454 Project: CloudStack Issue Type: Bug Security Level: Public (Anyone can view this level - this is the default.) Components: Snapshot Affects Versions: 4.3.0 Environment: Management Server: 4.3 Host: kvm Reporter: Pavan Kumar Bandarupally Priority: Critical Fix For: 4.3.0 After an NFS store has been migrated to Object store, it will be a read only store. Any entities ,snapshots or templates , on this store prior to migration should also be read only, which means user should not be able to delete them. Deletion for template works as expected but the same for snapshot doesn't work. In the UI user will be shown a successful task completion for snapshot deletion. In the MS logs an exception is thrown but the job is rendered SUCCESS. Job Trace below: 2013-12-11 18:06:57,128 DEBUG [c.c.a.ApiServlet] (catalina-exec-25:ctx-08bb8b65) ===START=== 10.146.0.11 -- GET command=deleteSnapshot&id=d7a4b078-4fe7-4933-89b8-316c7c9be61c&response=json&sessionkey=u6C2YtXK2DU2l%2BSFpYbZ4wgNyww%3D&_=1386746147984 2013-12-11 18:06:56,591 DEBUG [c.c.a.m.AgentManagerImpl] (AgentManager-Handler-7:null) SeqA 2-20435: Processing Seq 2-20435: { Cmd , MgmtId: -1, via: 2, Ver: v1, Flags: 11, [{"com.cloud.agent.api.ConsoleProxyLoadReportCommand":{"_proxyVmId":1,"_loadInfo":"{\n \"connections\": []\n}","wait":0}}] } 2013-12-11 18:07:00,367 DEBUG [c.c.h.d.HostDaoImpl] (ClusteredAgentManager Timer:ctx-7a541b2c) Completed resetting hosts suitable for reconnect 2013-12-11 18:07:00,375 DEBUG [c.c.h.d.HostDaoImpl] (ClusteredAgentManager Timer:ctx-7a541b2c) Acquiring hosts for clusters already owned by this management server 2013-12-11 18:07:00,377 DEBUG [c.c.h.d.HostDaoImpl] (ClusteredAgentManager Timer:ctx-7a541b2c) Completed acquiring hosts for clusters already owned by this management server 2013-12-11 18:07:00,377 DEBUG [c.c.h.d.HostDaoImpl] (ClusteredAgentManager Timer:ctx-7a541b2c) Acquiring hosts for clusters not owned by any management server 2013-12-11 18:07:00,378 DEBUG [c.c.h.d.HostDaoImpl] (ClusteredAgentManager Timer:ctx-7a541b2c) Completed acquiring hosts for clusters not owned by any management server 2013-12-11 18:07:00,444 DEBUG [c.c.a.m.AgentManagerImpl] (AgentManager-Handler-7:null) SeqA 2-20435: Sending Seq 2-20435: { Ans: , MgmtId: 6915098673184, via: 2, Ver: v1, Flags: 100010, [{"com.cloud.agent.api.AgentControlAnswer":{"result":true,"wait":0}}] } 2013-12-11 18:07:00,704 DEBUG [c.c.s.s.SecondaryStorageManagerImpl] (secstorage-1:ctx-d8b5b2d1) Zone 1 is ready to launch secondary storage VM 2013-12-11 18:07:00,712 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (catalina-exec-25:ctx-08bb8b65 ctx-62ca804c) submit async job-43, details: AsyncJobVO {id:43, userId: 2, accountId: 2, instanceType: Snapshot, instanceId: 1, cmd: org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotCmd, cmdInfo: {"id":"d7a4b078-4fe7-4933-89b8-316c7c9be61c","response":"json","sessionkey":"u6C2YtXK2DU2l+SFpYbZ4wgNyww\u003d","cmdEventType":"SNAPSHOT.DELETE","ctxUserId":"2","httpmethod":"GET","_":"1386746147984","ctxAccountId":"2","ctxStartEventId":"121"}, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 6915098673184, completeMsid: null, lastUpdated: null, lastPolled: null, created: null} 2013-12-11 18:07:00,714 DEBUG [c.c.a.ApiServlet] (catalina-exec-25:ctx-08bb8b65 ctx-62ca804c) ===END=== 10.146.0.11 -- GET command=deleteSnapshot&id=d7a4b078-4fe7-4933-89b8-316c7c9be61c&response=json&sessionkey=u6C2YtXK2DU2l%2BSFpYbZ4wgNyww%3D&_=1386746147984 2013-12-11 18:07:00,716 INFO [o.a.c.f.j.i.AsyncJobMonitor] (Job-Executor-13:ctx-e02ea08a) Add job-43 into job monitoring 2013-12-11 18:07:00,716 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Job-Executor-13:ctx-e02ea08a) Executing AsyncJobVO {id:43, userId: 2, accountId: 2, instanceType: Snapshot, instanceId: 1, cmd: org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotCmd, cmdInfo: {"id":"d7a4b078-4fe7-4933-89b8-316c7c9be61c","response":"json","sessionkey":"u6C2YtXK2DU2l+SFpYbZ4wgNyww\u003d","cmdEventType":"SNAPSHOT.DELETE","ctxUserId":"2","httpmethod":"GET","_":"1386746147984","ctxAccountId":"2","ctxStartEventId":"121"}, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 6915098673184, completeMsid: null, lastUpdated: null, lastPolled: null, created: null} 2013-12-11 18:07:00,815 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] (Job-Executor-13:ctx-e02ea08a ctx-62ca804c) delete snapshot chain for snapshot: 1 2013-12-11 18:07:00,818 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] (Job-Executor-13:ctx-e02ea08a ctx-62ca804c) Snapshot: 1 doesn't have children, so it's ok to delete it and its parents 2013-12-11 18:07:00,897 DEBUG [c.c.a.t.Request] (Job-Executor-13:ctx-e02ea08a ctx-62ca804c) Seq 3-1217003746: Sending { Cmd , MgmtId: 6915098673184, via: 3(s-2-VM), Ver: v1, Flags: 100011, [{"org.apache.cloudstack.storage.command.DeleteCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"volume":{"uuid":"5ae95928-a717-48d3-8e33-f8b0c7ecc9de","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"9340974e-0bc0-39cb-a5b2-7303f47548aa","id":1,"poolType":"NetworkFilesystem","host":"10.147.28.7","path":"/export/home/pavan/primaryKVMS3","port":2049,"url":"NetworkFilesystem://10.147.28.7//export/home/pavan/primaryKVMS3/?ROLE=Primary&STOREUUID=9340974e-0bc0-39cb-a5b2-7303f47548aa"}},"name":"ROOT-3","size":8589934592,"path":"5ae95928-a717-48d3-8e33-f8b0c7ecc9de","volumeId":3,"vmName":"i-2-3-VM","accountId":2,"format":"QCOW2","id":3,"deviceId":0,"hypervisorType":"KVM"},"dataStore":{"com.cloud.agent.api.to.S3TO":{"id":2,"uuid":"4c9d7dc1-6e98-4792-8a59-1cd401c0582e","endPoint":"10.147.29.56:8080","bucketName":"pavankvmbucket","httpsFlag":false,"created":"Dec 10, 2013 10:31:15 PM","enableRRS":false,"maxSingleUploadSizeInBytes":5368709120}},"vmName":"i-2-3-VM","name":"CoSVMbeforeMigration_ROOT-3_20131210115423","hypervisorType":"KVM","id":1,"quiescevm":false}},"wait":0}}] } 2013-12-11 18:07:00,952 DEBUG [c.c.c.ConsoleProxyManagerImpl] (consoleproxy-1:ctx-7e8c193a) Zone 1 is ready to launch console proxy 2013-12-11 18:07:01,143 DEBUG [c.c.a.t.Request] (AgentManager-Handler-6:null) Seq 3-1217003746: Processing: { Ans: , MgmtId: 6915098673184, via: 3, Ver: v1, Flags: 10, [{"com.cloud.agent.api.Answer":{"result":false,"details":"Failed to delete snapshot null from bucket pavankvmbucket due to the following error: The key must be specified when deleting an object","wait":0}}] } 2013-12-11 18:07:01,143 DEBUG [c.c.a.t.Request] (Job-Executor-13:ctx-e02ea08a ctx-62ca804c) Seq 3-1217003746: Received: { Ans: , MgmtId: 6915098673184, via: 3, Ver: v1, Flags: 10, { Answer } } 2013-12-11 18:07:01,144 DEBUG [o.a.c.s.s.SnapshotServiceImpl] (Job-Executor-13:ctx-e02ea08a ctx-62ca804c) delete snapshot failedFailed to delete snapshot null from bucket pavankvmbucket due to the following error: The key must be specified when deleting an object 2013-12-11 18:07:01,175 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] (Job-Executor-13:ctx-e02ea08a ctx-62ca804c) delete snapshot failed: com.cloud.utils.exception.CloudRuntimeException: Failed to delete snapshot null from bucket pavankvmbucket due to the following error: The key must be specified when deleting an object at org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.deleteSnapshot(SnapshotServiceImpl.java:386) at org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshotChain(XenserverSnapshotStrategy.java:169) at org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshot(XenserverSnapshotStrategy.java:229) at com.cloud.storage.snapshot.SnapshotManagerImpl.deleteSnapshot(SnapshotManagerImpl.java:428) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) at com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:50) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) at $Proxy161.deleteSnapshot(Unknown Source) at org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotCmd.execute(DeleteSnapshotCmd.java:97) at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:161) at com.cloud.api.ApiAsyncJobDispatcher.runJobInContext(ApiAsyncJobDispatcher.java:109) at com.cloud.api.ApiAsyncJobDispatcher$1.run(ApiAsyncJobDispatcher.java:66) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53) at com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:63) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:520) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:679) 2013-12-11 18:07:01,246 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Job-Executor-13:ctx-e02ea08a ctx-62ca804c) Complete async job-43, jobStatus: SUCCEEDED, resultCode: 0, result: org.apache.cloudstack.api.response.SuccessResponse/null/{"success":true} 2013-12-11 18:07:01,259 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Job-Executor-13:ctx-e02ea08a) Done executing org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotCmd for job-43 2013-12-11 18:07:01,282 INFO [o.a.c.f.j.i.AsyncJobMonitor] (Job-Executor-13:ctx-e02ea08a) Remove job-43 from job monitoring -- This message was sent by Atlassian JIRA (v6.1.4#6159)