[ https://issues.apache.org/jira/browse/CLOUDSTACK-5828?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Rayees Namathponnan updated CLOUDSTACK-5828: -------------------------------------------- Description: Steps to reproduce Step 1 : Create an account Step 2 : Deploy VM Step 3 : create ROOT disks snapshot Step 4 : delete account Result Account got deleted, also snapshot got removed from secondary storage but below error trough in MS log 2014-01-07 12:40:07,204 DEBUG [c.c.s.s.SnapshotManagerImpl] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Deleted all snapshots for volume: 947 under account: 531 2014-01-07 12:40:07,210 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) delete snapshot chain for snapshot: 29 2014-01-07 12:40:07,211 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Snapshot: 29 doesn't have children, so it's ok to delete it and its parents 2014-01-07 12:40:07,277 DEBUG [c.c.a.t.Request] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Seq 5-1429084154: Sending { Cmd , MgmtId: 29066118877352, via: 5(s-1-VM), Ver: v1, Flags: 1000 11, [{"org.apache.cloudstack.storage.command.DeleteCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"snapshots/531/947/4fce7500-82c1-4f11-ba1d-18033717491a"," volume":{"uuid":"f3ebed2a-91ba-41d3-b215-398ae8534de3","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"fff90cb5-06dd-33b3-8815-d78c08ca01d9 ","id":1,"poolType":"NetworkFilesystem","host":"10.223.110.232","path":"/export/home/rayees/SC_QA_AUTO4/primary","port":2049,"url":"NetworkFilesystem://10.223.110.232//export/home/rayees/ SC_QA_AUTO4/primary/?ROLE=Primary&STOREUUID=fff90cb5-06dd-33b3-8815-d78c08ca01d9"}},"name":"ROOT-885","size":8589934592,"path":"f3ebed2a-91ba-41d3-b215-398ae8534de3","volumeId":947,"vmNam e":"i-531-885-QA","accountId":531,"format":"QCOW2","id":947,"deviceId":0,"hypervisorType":"KVM"},"dataStore":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.223.110.232:/export/home/ray ees/SC_QA_AUTO4/secondary","_role":"Image"}},"vmName":"i-531-885-QA","name":"r1vm1_ROOT-885_20140107202857","hypervisorType":"KVM","id":29,"quiescevm":false,"physicalSize":0}},"wait":0}}] } 2014-01-07 12:40:07,623 DEBUG [c.c.a.t.Request] (AgentManager-Handler-5:null) Seq 5-1429084154: Processing: { Ans: , MgmtId: 29066118877352, via: 5, Ver: v1, Flags: 10, [{"com.cloud.agen t.api.Answer":{"result":false,"details":"snapshot directory 947 doesn't exist","wait":0}}] } 2014-01-07 12:40:07,624 DEBUG [c.c.a.t.Request] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Seq 5-1429084154: Received: { Ans: , MgmtId: 29066118877352, via: 5, Ver: v1, Flags: 10, { Ans wer } } 2014-01-07 12:40:07,624 DEBUG [o.a.c.s.s.SnapshotServiceImpl] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) delete snapshot failedsnapshot directory 947 doesn't exist 2014-01-07 12:40:07,631 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) delete snapshot failed: com.cloud.utils.exception.CloudRuntimeException: snapshot directory 947 doesn't exist at org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.deleteSnapshot(SnapshotServiceImpl.java:391) at org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshotChain(XenserverSnapshotStrategy.java:176) at org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshot(XenserverSnapshotStrategy.java:236) at com.cloud.storage.snapshot.SnapshotManagerImpl.deleteSnapshotDirsForAccount(SnapshotManagerImpl.java:615) at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) at $Proxy160.deleteSnapshotDirsForAccount(Unknown Source) at com.cloud.user.AccountManagerImpl.cleanupAccount(AccountManagerImpl.java:597) at com.cloud.user.AccountManagerImpl.deleteAccount(AccountManagerImpl.java:561) at com.cloud.user.AccountManagerImpl.deleteUserAccount(AccountManagerImpl.java:1308) at sun.reflect.GeneratedMethodAccessor511.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) at com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:50) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) at $Proxy82.deleteUserAccount(Unknown Source) at org.apache.cloudstack.region.RegionManagerImpl.deleteUserAccount(RegionManagerImpl.java:193) at org.apache.cloudstack.region.RegionServiceImpl.deleteUserAccount(RegionServiceImpl.java:118) at org.apache.cloudstack.api.command.admin.account.DeleteAccountCmd.execute(DeleteAccountCmd.java:101) at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:161) at com.cloud.api.ApiAsyncJobDispatcher.runJobInContext(ApiAsyncJobDispatcher.java:109) at com.cloud.api.ApiAsyncJobDispatcher$1.run(ApiAsyncJobDispatcher.java:66) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) : 2014-01-07 12:40:38,604 DEBUG [c.c.a.t.Request] (AgentManager-Handler-11:null) Seq 5-1429084156: Processing: { Ans: , MgmtId: 29066118877352, via: 5, Ver: v1, Flags: 110, [{"com.cloud.ag ent.api.Answer":{"result":true,"details":"snapshot directory 14 doesn't exist","wait":0}}] } 2014-01-07 12:40:38,604 DEBUG [c.c.a.m.AgentAttache] (AgentManager-Handler-11:null) Seq 5-1429084156: No more commands found 2014-01-07 12:40:38,604 DEBUG [c.c.a.t.Request] (AccountChecker-1:ctx-1f2e307e) Seq 5-1429084156: Received: { Ans: , MgmtId: 29066118877352, via: 5, Ver: v1, Flags: 110, { Answer } } 2014-01-07 12:40:38,604 DEBUG [c.c.s.s.SnapshotManagerImpl] (AccountChecker-1:ctx-1f2e307e) Deleted all snapshots for volume: 14 under account: 10 2014-01-07 12:40:38,628 WARN [c.c.u.AccountManagerImpl] (AccountChecker-1:ctx-1f2e307e) Failed to cleanup account Acct[2d95c740-8593-4325-badf-6c1f9d034329-test-TestSnapshotRootDisk-RZEB AX] due to java.lang.NullPointerException at com.cloud.storage.snapshot.SnapshotManagerImpl.deleteSnapshotDirsForAccount(SnapshotManagerImpl.java:618) at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) at $Proxy160.deleteSnapshotDirsForAccount(Unknown Source) at com.cloud.user.AccountManagerImpl.cleanupAccount(AccountManagerImpl.java:597) at com.cloud.user.AccountManagerImpl$AccountCleanupTask.runInContext(AccountManagerImpl.java:1541) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:165) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:267) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:679) 2014-01-07 12:40:38,629 INFO [c.c.u.AccountManagerImpl] (AccountChecker-1:ctx-1f2e307e) Cleanup for account 10 is needed. 2014-01-07 12:40:38,630 DEBUG [c.c.u.AccountManagerImpl] (AccountChecker-1:ctx-1f2e307e) Cleaning up 67 was: Steps to reproduce Step 1 : Create an account Step 2 : Deploy VM Step 3 : create ROOT disks snapshot Step 4 : delete account Result Account got deleted, but below error trough in MS log 2014-01-07 12:40:07,204 DEBUG [c.c.s.s.SnapshotManagerImpl] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Deleted all snapshots for volume: 947 under account: 531 2014-01-07 12:40:07,210 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) delete snapshot chain for snapshot: 29 2014-01-07 12:40:07,211 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Snapshot: 29 doesn't have children, so it's ok to delete it and its parents 2014-01-07 12:40:07,277 DEBUG [c.c.a.t.Request] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Seq 5-1429084154: Sending { Cmd , MgmtId: 29066118877352, via: 5(s-1-VM), Ver: v1, Flags: 1000 11, [{"org.apache.cloudstack.storage.command.DeleteCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"snapshots/531/947/4fce7500-82c1-4f11-ba1d-18033717491a"," volume":{"uuid":"f3ebed2a-91ba-41d3-b215-398ae8534de3","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"fff90cb5-06dd-33b3-8815-d78c08ca01d9 ","id":1,"poolType":"NetworkFilesystem","host":"10.223.110.232","path":"/export/home/rayees/SC_QA_AUTO4/primary","port":2049,"url":"NetworkFilesystem://10.223.110.232//export/home/rayees/ SC_QA_AUTO4/primary/?ROLE=Primary&STOREUUID=fff90cb5-06dd-33b3-8815-d78c08ca01d9"}},"name":"ROOT-885","size":8589934592,"path":"f3ebed2a-91ba-41d3-b215-398ae8534de3","volumeId":947,"vmNam e":"i-531-885-QA","accountId":531,"format":"QCOW2","id":947,"deviceId":0,"hypervisorType":"KVM"},"dataStore":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.223.110.232:/export/home/ray ees/SC_QA_AUTO4/secondary","_role":"Image"}},"vmName":"i-531-885-QA","name":"r1vm1_ROOT-885_20140107202857","hypervisorType":"KVM","id":29,"quiescevm":false,"physicalSize":0}},"wait":0}}] } 2014-01-07 12:40:07,623 DEBUG [c.c.a.t.Request] (AgentManager-Handler-5:null) Seq 5-1429084154: Processing: { Ans: , MgmtId: 29066118877352, via: 5, Ver: v1, Flags: 10, [{"com.cloud.agen t.api.Answer":{"result":false,"details":"snapshot directory 947 doesn't exist","wait":0}}] } 2014-01-07 12:40:07,624 DEBUG [c.c.a.t.Request] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Seq 5-1429084154: Received: { Ans: , MgmtId: 29066118877352, via: 5, Ver: v1, Flags: 10, { Ans wer } } 2014-01-07 12:40:07,624 DEBUG [o.a.c.s.s.SnapshotServiceImpl] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) delete snapshot failedsnapshot directory 947 doesn't exist 2014-01-07 12:40:07,631 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) delete snapshot failed: com.cloud.utils.exception.CloudRuntimeException: snapshot directory 947 doesn't exist at org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.deleteSnapshot(SnapshotServiceImpl.java:391) at org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshotChain(XenserverSnapshotStrategy.java:176) at org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshot(XenserverSnapshotStrategy.java:236) at com.cloud.storage.snapshot.SnapshotManagerImpl.deleteSnapshotDirsForAccount(SnapshotManagerImpl.java:615) at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) at $Proxy160.deleteSnapshotDirsForAccount(Unknown Source) at com.cloud.user.AccountManagerImpl.cleanupAccount(AccountManagerImpl.java:597) at com.cloud.user.AccountManagerImpl.deleteAccount(AccountManagerImpl.java:561) at com.cloud.user.AccountManagerImpl.deleteUserAccount(AccountManagerImpl.java:1308) at sun.reflect.GeneratedMethodAccessor511.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) at com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:50) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) at $Proxy82.deleteUserAccount(Unknown Source) at org.apache.cloudstack.region.RegionManagerImpl.deleteUserAccount(RegionManagerImpl.java:193) at org.apache.cloudstack.region.RegionServiceImpl.deleteUserAccount(RegionServiceImpl.java:118) at org.apache.cloudstack.api.command.admin.account.DeleteAccountCmd.execute(DeleteAccountCmd.java:101) at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:161) at com.cloud.api.ApiAsyncJobDispatcher.runJobInContext(ApiAsyncJobDispatcher.java:109) at com.cloud.api.ApiAsyncJobDispatcher$1.run(ApiAsyncJobDispatcher.java:66) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) : 2014-01-07 12:40:38,604 DEBUG [c.c.a.t.Request] (AgentManager-Handler-11:null) Seq 5-1429084156: Processing: { Ans: , MgmtId: 29066118877352, via: 5, Ver: v1, Flags: 110, [{"com.cloud.ag ent.api.Answer":{"result":true,"details":"snapshot directory 14 doesn't exist","wait":0}}] } 2014-01-07 12:40:38,604 DEBUG [c.c.a.m.AgentAttache] (AgentManager-Handler-11:null) Seq 5-1429084156: No more commands found 2014-01-07 12:40:38,604 DEBUG [c.c.a.t.Request] (AccountChecker-1:ctx-1f2e307e) Seq 5-1429084156: Received: { Ans: , MgmtId: 29066118877352, via: 5, Ver: v1, Flags: 110, { Answer } } 2014-01-07 12:40:38,604 DEBUG [c.c.s.s.SnapshotManagerImpl] (AccountChecker-1:ctx-1f2e307e) Deleted all snapshots for volume: 14 under account: 10 2014-01-07 12:40:38,628 WARN [c.c.u.AccountManagerImpl] (AccountChecker-1:ctx-1f2e307e) Failed to cleanup account Acct[2d95c740-8593-4325-badf-6c1f9d034329-test-TestSnapshotRootDisk-RZEB AX] due to java.lang.NullPointerException at com.cloud.storage.snapshot.SnapshotManagerImpl.deleteSnapshotDirsForAccount(SnapshotManagerImpl.java:618) at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) at $Proxy160.deleteSnapshotDirsForAccount(Unknown Source) at com.cloud.user.AccountManagerImpl.cleanupAccount(AccountManagerImpl.java:597) at com.cloud.user.AccountManagerImpl$AccountCleanupTask.runInContext(AccountManagerImpl.java:1541) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:165) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:267) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:679) 2014-01-07 12:40:38,629 INFO [c.c.u.AccountManagerImpl] (AccountChecker-1:ctx-1f2e307e) Cleanup for account 10 is needed. 2014-01-07 12:40:38,630 DEBUG [c.c.u.AccountManagerImpl] (AccountChecker-1:ctx-1f2e307e) Cleaning up 67 > [Automation] Delete snapshot error observed, after deleting account > ------------------------------------------------------------------- > > Key: CLOUDSTACK-5828 > URL: https://issues.apache.org/jira/browse/CLOUDSTACK-5828 > Project: CloudStack > Issue Type: Bug > Security Level: Public(Anyone can view this level - this is the > default.) > Components: Snapshot > Affects Versions: 4.3.0 > Environment: kvm (RHEL 6.3) > Branch : 4.3 > Reporter: Rayees Namathponnan > Priority: Critical > Fix For: 4.3.0 > > Attachments: KVM_Jan_07_14.rar > > > Steps to reproduce > Step 1 : Create an account > Step 2 : Deploy VM > Step 3 : create ROOT disks snapshot > Step 4 : delete account > Result > Account got deleted, also snapshot got removed from secondary storage but > below error trough in MS log > 2014-01-07 12:40:07,204 DEBUG [c.c.s.s.SnapshotManagerImpl] > (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Deleted all snapshots for volume: > 947 under account: 531 > 2014-01-07 12:40:07,210 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] > (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) delete snapshot chain for > snapshot: 29 > 2014-01-07 12:40:07,211 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] > (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) Snapshot: 29 doesn't have > children, so it's ok to delete it and its parents > 2014-01-07 12:40:07,277 DEBUG [c.c.a.t.Request] (Job-Executor-29:ctx-5dc172d6 > ctx-d8044988) Seq 5-1429084154: Sending { Cmd , MgmtId: 29066118877352, via: > 5(s-1-VM), Ver: v1, Flags: 1000 > 11, > [{"org.apache.cloudstack.storage.command.DeleteCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"snapshots/531/947/4fce7500-82c1-4f11-ba1d-18033717491a"," > volume":{"uuid":"f3ebed2a-91ba-41d3-b215-398ae8534de3","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"fff90cb5-06dd-33b3-8815-d78c08ca01d9 > ","id":1,"poolType":"NetworkFilesystem","host":"10.223.110.232","path":"/export/home/rayees/SC_QA_AUTO4/primary","port":2049,"url":"NetworkFilesystem://10.223.110.232//export/home/rayees/ > SC_QA_AUTO4/primary/?ROLE=Primary&STOREUUID=fff90cb5-06dd-33b3-8815-d78c08ca01d9"}},"name":"ROOT-885","size":8589934592,"path":"f3ebed2a-91ba-41d3-b215-398ae8534de3","volumeId":947,"vmNam > e":"i-531-885-QA","accountId":531,"format":"QCOW2","id":947,"deviceId":0,"hypervisorType":"KVM"},"dataStore":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.223.110.232:/export/home/ray > ees/SC_QA_AUTO4/secondary","_role":"Image"}},"vmName":"i-531-885-QA","name":"r1vm1_ROOT-885_20140107202857","hypervisorType":"KVM","id":29,"quiescevm":false,"physicalSize":0}},"wait":0}}] > } > 2014-01-07 12:40:07,623 DEBUG [c.c.a.t.Request] (AgentManager-Handler-5:null) > Seq 5-1429084154: Processing: { Ans: , MgmtId: 29066118877352, via: 5, Ver: > v1, Flags: 10, [{"com.cloud.agen > t.api.Answer":{"result":false,"details":"snapshot directory 947 doesn't > exist","wait":0}}] } > 2014-01-07 12:40:07,624 DEBUG [c.c.a.t.Request] (Job-Executor-29:ctx-5dc172d6 > ctx-d8044988) Seq 5-1429084154: Received: { Ans: , MgmtId: 29066118877352, > via: 5, Ver: v1, Flags: 10, { Ans > wer } } > 2014-01-07 12:40:07,624 DEBUG [o.a.c.s.s.SnapshotServiceImpl] > (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) delete snapshot failedsnapshot > directory 947 doesn't exist > 2014-01-07 12:40:07,631 DEBUG [o.a.c.s.s.XenserverSnapshotStrategy] > (Job-Executor-29:ctx-5dc172d6 ctx-d8044988) delete snapshot failed: > com.cloud.utils.exception.CloudRuntimeException: snapshot directory 947 > doesn't exist > at > org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.deleteSnapshot(SnapshotServiceImpl.java:391) > at > org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshotChain(XenserverSnapshotStrategy.java:176) > at > org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.deleteSnapshot(XenserverSnapshotStrategy.java:236) > at > com.cloud.storage.snapshot.SnapshotManagerImpl.deleteSnapshotDirsForAccount(SnapshotManagerImpl.java:615) > at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:616) > at > org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) > at > org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) > at > org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) > at $Proxy160.deleteSnapshotDirsForAccount(Unknown Source) > at > com.cloud.user.AccountManagerImpl.cleanupAccount(AccountManagerImpl.java:597) > at > com.cloud.user.AccountManagerImpl.deleteAccount(AccountManagerImpl.java:561) > at > com.cloud.user.AccountManagerImpl.deleteUserAccount(AccountManagerImpl.java:1308) > at sun.reflect.GeneratedMethodAccessor511.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:616) > at > org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) > at > com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:50) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161) > at > org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) > at > org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) > at $Proxy82.deleteUserAccount(Unknown Source) > at > org.apache.cloudstack.region.RegionManagerImpl.deleteUserAccount(RegionManagerImpl.java:193) > at > org.apache.cloudstack.region.RegionServiceImpl.deleteUserAccount(RegionServiceImpl.java:118) > at > org.apache.cloudstack.api.command.admin.account.DeleteAccountCmd.execute(DeleteAccountCmd.java:101) > at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:161) > at > com.cloud.api.ApiAsyncJobDispatcher.runJobInContext(ApiAsyncJobDispatcher.java:109) > at > com.cloud.api.ApiAsyncJobDispatcher$1.run(ApiAsyncJobDispatcher.java:66) > at > org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) > : > 2014-01-07 12:40:38,604 DEBUG [c.c.a.t.Request] > (AgentManager-Handler-11:null) Seq 5-1429084156: Processing: { Ans: , > MgmtId: 29066118877352, via: 5, Ver: v1, Flags: 110, [{"com.cloud.ag > ent.api.Answer":{"result":true,"details":"snapshot directory 14 doesn't > exist","wait":0}}] } > 2014-01-07 12:40:38,604 DEBUG [c.c.a.m.AgentAttache] > (AgentManager-Handler-11:null) Seq 5-1429084156: No more commands found > 2014-01-07 12:40:38,604 DEBUG [c.c.a.t.Request] > (AccountChecker-1:ctx-1f2e307e) Seq 5-1429084156: Received: { Ans: , MgmtId: > 29066118877352, via: 5, Ver: v1, Flags: 110, { Answer } } > 2014-01-07 12:40:38,604 DEBUG [c.c.s.s.SnapshotManagerImpl] > (AccountChecker-1:ctx-1f2e307e) Deleted all snapshots for volume: 14 under > account: 10 > 2014-01-07 12:40:38,628 WARN [c.c.u.AccountManagerImpl] > (AccountChecker-1:ctx-1f2e307e) Failed to cleanup account > Acct[2d95c740-8593-4325-badf-6c1f9d034329-test-TestSnapshotRootDisk-RZEB > AX] due to > java.lang.NullPointerException > at > com.cloud.storage.snapshot.SnapshotManagerImpl.deleteSnapshotDirsForAccount(SnapshotManagerImpl.java:618) > at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:616) > at > org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) > at > org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) > at > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) > at > org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) > at $Proxy160.deleteSnapshotDirsForAccount(Unknown Source) > at > com.cloud.user.AccountManagerImpl.cleanupAccount(AccountManagerImpl.java:597) > at > com.cloud.user.AccountManagerImpl$AccountCleanupTask.runInContext(AccountManagerImpl.java:1541) > at > org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49) > at > org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) > at > org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103) > at > org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53) > at > org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46) > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) > at > java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351) > at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:165) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:267) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) > at java.lang.Thread.run(Thread.java:679) > 2014-01-07 12:40:38,629 INFO [c.c.u.AccountManagerImpl] > (AccountChecker-1:ctx-1f2e307e) Cleanup for account 10 is needed. > 2014-01-07 12:40:38,630 DEBUG [c.c.u.AccountManagerImpl] > (AccountChecker-1:ctx-1f2e307e) Cleaning up 67 -- This message was sent by Atlassian JIRA (v6.1.5#6160)