Pavan Kumar Bandarupally created CLOUDSTACK-5819: ----------------------------------------------------
Summary: extractTemplate fails with Vmware host on migration of NFS to S3 Key: CLOUDSTACK-5819 URL: https://issues.apache.org/jira/browse/CLOUDSTACK-5819 Project: CloudStack Issue Type: Bug Security Level: Public (Anyone can view this level - this is the default.) Components: Template Affects Versions: 4.3.0 Environment: Latest Management server 4.3 Two zones one with Xen host and other with VmWare host Reporter: Pavan Kumar Bandarupally Priority: Critical Fix For: 4.3.0 Trying to download a template created from snapshot of root volume or template of root volume after migration of NFS to S3 store throws an error. Job Trace: ================== 2014-01-07 22:07:44,221 DEBUG [c.c.a.ApiServlet] (catalina-exec-8:ctx-3c03896e) ===START=== 10.146.0.11 -- GET command=extractTemplate&mode=HTTP_DOWNLOAD&id=3e35e5f4-9067-4c6e-ba15-3eb80494a6e8&response=json&sessionkey=BGTkrKxSTvEyZG7Nx4tTVof99Tw%3D&_=1389093486904 2014-01-07 22:07:44,255 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (catalina-exec-8:ctx-3c03896e ctx-f274cea7) submit async job-89, details: AsyncJobVO {id:89, userId: 2, accountId: 2, instanceType: Template, instanceId: 203, cmd: org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd, cmdInfo: {"response":"json","id":"3e35e5f4-9067-4c6e-ba15-3eb80494a6e8","sessionkey":"BGTkrKxSTvEyZG7Nx4tTVof99Tw\u003d","cmdEventType":"TEMPLATE.EXTRACT","ctxUserId":"2","httpmethod":"GET","_":"1389093486904","ctxAccountId":"2","ctxStartEventId":"176","mode":"HTTP_DOWNLOAD"}, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 7337246982268, completeMsid: null, lastUpdated: null, lastPolled: null, created: null} 2014-01-07 22:07:44,258 DEBUG [c.c.a.ApiServlet] (catalina-exec-8:ctx-3c03896e ctx-f274cea7) ===END=== 10.146.0.11 -- GET command=extractTemplate&mode=HTTP_DOWNLOAD&id=3e35e5f4-9067-4c6e-ba15-3eb80494a6e8&response=json&sessionkey=BGTkrKxSTvEyZG7Nx4tTVof99Tw%3D&_=1389093486904 2014-01-07 22:07:44,262 INFO [o.a.c.f.j.i.AsyncJobMonitor] (Job-Executor-37:ctx-b68520ba) Add job-89 into job monitoring 2014-01-07 22:07:44,262 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Job-Executor-37:ctx-b68520ba) Executing AsyncJobVO {id:89, userId: 2, accountId: 2, instanceType: Template, instanceId: 203, cmd: org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd, cmdInfo: {"response":"json","id":"3e35e5f4-9067-4c6e-ba15-3eb80494a6e8","sessionkey":"BGTkrKxSTvEyZG7Nx4tTVof99Tw\u003d","cmdEventType":"TEMPLATE.EXTRACT","ctxUserId":"2","httpmethod":"GET","_":"1389093486904","ctxAccountId":"2","ctxStartEventId":"176","mode":"HTTP_DOWNLOAD"}, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 7337246982268, completeMsid: null, lastUpdated: null, lastPolled: null, created: null} 2014-01-07 22:07:44,290 DEBUG [o.a.c.s.i.TemplateDataFactoryImpl] (Job-Executor-37:ctx-b68520ba ctx-f274cea7) template 203 is already in store:3, type:Image 2014-01-07 22:07:44,300 DEBUG [o.a.c.s.i.TemplateDataFactoryImpl] (Job-Executor-37:ctx-b68520ba ctx-f274cea7) template 203 is already in store:1, type:ImageCache 2014-01-07 22:07:44,302 DEBUG [o.a.c.s.i.TemplateDataFactoryImpl] (Job-Executor-37:ctx-b68520ba ctx-f274cea7) template 203 is already in store:3, type:Image 2014-01-07 22:07:44,325 DEBUG [o.a.c.s.m.AncientDataMotionStrategy] (Job-Executor-37:ctx-b68520ba ctx-f274cea7) copyAsync inspecting src type TEMPLATE copyAsync inspecting dest type TEMPLATE 2014-01-07 22:07:44,371 DEBUG [c.c.a.t.Request] (Job-Executor-37:ctx-b68520ba ctx-f274cea7) Seq 3-242024639: Sending { Cmd , MgmtId: 7337246982268, via: 3(s-1-VM), Ver: v1, Flags: 100011, [{"org.apache.cloudstack.storage.command.CopyCommand":{"srcTO":{"org.apache.cloudstack.storage.to.TemplateObjectTO":{"path":"template/tmpl/2/203/288329c8b-bb60-325f-8da8-b7e0e9764f99.ova","uuid":"3e35e5f4-9067-4c6e-ba15-3eb80494a6e8","id":203,"format":"OVA","accountId":2,"hvm":true,"displayText":"tmplt from root vol on Vmware before","imageDataStore":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.147.28.7/export/home/pavan/secondaryVmWareMZ","_role":"ImageCache"}},"name":"288329c8b-bb60-325f-8da8-b7e0e9764f99","hypervisorType":"VMware"}},"destTO":{"org.apache.cloudstack.storage.to.TemplateObjectTO":{"path":"template/tmpl/2/203/288329c8b-bb60-325f-8da8-b7e0e9764f99","uuid":"3e35e5f4-9067-4c6e-ba15-3eb80494a6e8","id":203,"format":"OVA","accountId":2,"hvm":true,"displayText":"tmplt from root vol on Vmware before","imageDataStore":{"com.cloud.agent.api.to.S3TO":{"id":3,"uuid":"a2731bc7-33e4-42b2-9687-28626b315238","endPoint":"10.147.29.56:8080","bucketName":"pavanvmwmzbucket","httpsFlag":false,"created":"Jan 7, 2014 9:26:21 PM","enableRRS":false,"maxSingleUploadSizeInBytes":5368709120}},"name":"288329c8b-bb60-325f-8da8-b7e0e9764f99","hypervisorType":"VMware"}},"executeInSequence":false,"options":{},"wait":10800}}] } 2014-01-07 22:07:44,563 DEBUG [c.c.a.t.Request] (AgentManager-Handler-10:null) Seq 3-242024639: Processing: { Ans: , MgmtId: 7337246982268, via: 3, Ver: v1, Flags: 10, [{"org.apache.cloudstack.storage.command.CopyCmdAnswer":{"result":false,"details":"Can't find src file:/mnt/SecStorage/ebe00e50-7dbc-340d-8e05-b49629b8e37f/template/tmpl/2/203/288329c8b-bb60-325f-8da8-b7e0e9764f99.ova","wait":0}}] } 2014-01-07 22:07:44,564 DEBUG [c.c.a.t.Request] (Job-Executor-37:ctx-b68520ba ctx-f274cea7) Seq 3-242024639: Received: { Ans: , MgmtId: 7337246982268, via: 3, Ver: v1, Flags: 10, { CopyCmdAnswer } } 2014-01-07 22:07:44,583 ERROR [c.c.a.ApiAsyncJobDispatcher] (Job-Executor-37:ctx-b68520ba) Unexpected exception while executing org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd com.cloud.utils.exception.CloudRuntimeException: sync template from cache to region wide store failed for image store VmWS3Store at org.apache.cloudstack.storage.image.TemplateServiceImpl.syncTemplateToRegionStore(TemplateServiceImpl.java:728) at com.cloud.template.TemplateManagerImpl.extract(TemplateManagerImpl.java:488) at com.cloud.template.TemplateManagerImpl.extract(TemplateManagerImpl.java:406) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150) at com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:50) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204) at $Proxy158.extract(Unknown Source) at org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd.execute(ExtractTemplateCmd.java:129) at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:161) at com.cloud.api.ApiAsyncJobDispatcher.runJobInContext(ApiAsyncJobDispatcher.java:109) at com.cloud.api.ApiAsyncJobDispatcher$1.run(ApiAsyncJobDispatcher.java:66) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53) at com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:63) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:524) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:679) 2014-01-07 22:07:44,586 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Job-Executor-37:ctx-b68520ba) Complete async job-89, jobStatus: FAILED, resultCode: 530, result: org.apache.cloudstack.api.response.ExceptionResponse/null/{"uuidList":[],"errorcode":530,"errortext":"sync template from cache to region wide store failed for image store VmWS3Store"} 2014-01-07 22:07:44,597 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Job-Executor-37:ctx-b68520ba) Done executing org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd for job-89 2014-01-07 22:07:44,604 INFO [o.a.c.f.j.i.AsyncJobMonitor] (Job-Executor-37:ctx-b68520ba) Remove job-89 from job monitoring -- This message was sent by Atlassian JIRA (v6.1.5#6160)