[ https://issues.apache.org/jira/browse/HDDS-4339?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Li Cheng reassigned HDDS-4339: ------------------------------ Assignee: Li Cheng > Ozone S3 gateway throws NPE with goofys > --------------------------------------- > > Key: HDDS-4339 > URL: https://issues.apache.org/jira/browse/HDDS-4339 > Project: Hadoop Distributed Data Store > Issue Type: Bug > Affects Versions: 1.0.0 > Reporter: Li Cheng > Assignee: Li Cheng > Priority: Blocker > Labels: pull-request-available > Attachments: image-2020-10-13-15-23-49-864.png > > > Configured goofys and s3g on different hosts and Fiotest writes files on the > goofys mount point. Export AWS secrets on the s3g host. See a bunch of NPE in > s3g logs. > # Looks like missing AWS auth header could cause NPE. Looks like > AWSSignatureProcessor.init() doesn't handle header missing which causes NPE. > # Why it's missing AWS auth header is also unknown. > Note that there are files that have been successfully written into Ozone via > goofys, while not all of them are succeeded. > > 2020-10-13 11:18:43,425 [qtp1686100174-1238] ERROR > org.apache.hadoop.ozone.s3.OzoneClientProducer: Error: > org.jboss.weld.exceptions.WeldException: WELD-000049: Unable to invoke public > void org.apache.hadoop.ozone.s3.AWSSignatureProcessor.init() throws > java.lang.Exception on > org.apache.hadoop.ozone.s3.AWSSignatureProcessor@5535155b > at > org.jboss.weld.injection.producer.DefaultLifecycleCallbackInvoker.invokeMethods(DefaultLifecycleCallbackInvoker.java:99) > at > org.jboss.weld.injection.producer.DefaultLifecycleCallbackInvoker.postConstruct(DefaultLifecycleCallbackInvoker.java:80) > at > org.jboss.weld.injection.producer.BasicInjectionTarget.postConstruct(BasicInjectionTarget.java:122) > at > org.glassfish.jersey.ext.cdi1x.internal.CdiComponentProvider$InjectionManagerInjectedCdiTarget.postConstruct(CdiComponentProvider.java:887) > at org.jboss.weld.bean.ManagedBean.create(ManagedBean.java:162) > at org.jboss.weld.context.AbstractContext.get(AbstractContext.java:96) > at > org.jboss.weld.bean.ContextualInstanceStrategy$DefaultContextualInstanceStrategy.get(ContextualInstanceStrategy.java:100) > at > org.jboss.weld.bean.ContextualInstanceStrategy$CachingContextualInstanceStrategy.get(ContextualInstanceStrategy.java:177) > at org.jboss.weld.bean.ContextualInstance.get(ContextualInstance.java:50) > at > org.jboss.weld.bean.proxy.ContextBeanInstance.getInstance(ContextBeanInstance.java:99) > at > org.jboss.weld.bean.proxy.ProxyMethodHandler.getInstance(ProxyMethodHandler.java:125) > at > org.apache.hadoop.ozone.s3.AWSSignatureProcessor$Proxy$_$$_WeldClientProxy.getAwsAccessId(Unknown > Source) > at > org.apache.hadoop.ozone.s3.OzoneClientProducer.getClient(OzoneClientProducer.java:79) > at > org.apache.hadoop.ozone.s3.OzoneClientProducer.createClient(OzoneClientProducer.java:68) > at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.jboss.weld.injection.StaticMethodInjectionPoint.invoke(StaticMethodInjectionPoint.java:88) > at > org.jboss.weld.injection.StaticMethodInjectionPoint.invoke(StaticMethodInjectionPoint.java:78) > at > org.jboss.weld.injection.producer.ProducerMethodProducer.produce(ProducerMethodProducer.java:100) > at > org.jboss.weld.injection.producer.AbstractMemberProducer.produce(AbstractMemberProducer.java:161) > at > org.jboss.weld.bean.AbstractProducerBean.create(AbstractProducerBean.java:180) > at > org.jboss.weld.context.unbound.DependentContextImpl.get(DependentContextImpl.java:70) > at > org.jboss.weld.bean.ContextualInstanceStrategy$DefaultContextualInstanceStrategy.get(ContextualInstanceStrategy.java:100) > at org.jboss.weld.bean.ContextualInstance.get(ContextualInstance.java:50) > at > org.jboss.weld.manager.BeanManagerImpl.getReference(BeanManagerImpl.java:785) > at > org.jboss.weld.manager.BeanManagerImpl.getInjectableReference(BeanManagerImpl.java:885) > at > org.jboss.weld.injection.FieldInjectionPoint.inject(FieldInjectionPoint.java:92) > at org.jboss.weld.util.Beans.injectBoundFields(Beans.java:358) > at org.jboss.weld.util.Beans.injectFieldsAndInitializers(Beans.java:369) > at > org.jboss.weld.injection.producer.ResourceInjector$1.proceed(ResourceInjector.java:70) > at > org.jboss.weld.injection.InjectionContextImpl.run(InjectionContextImpl.java:48) > at > org.jboss.weld.injection.producer.ResourceInjector.inject(ResourceInjector.java:72) > at > org.jboss.weld.injection.producer.BasicInjectionTarget.inject(BasicInjectionTarget.java:117) > at > org.glassfish.jersey.ext.cdi1x.internal.CdiComponentProvider$InjectionManagerInjectedCdiTarget.inject(CdiComponentProvider.java:873) > at org.jboss.weld.bean.ManagedBean.create(ManagedBean.java:159) > at > org.jboss.weld.context.unbound.DependentContextImpl.get(DependentContextImpl.java:70) > at > org.jboss.weld.bean.ContextualInstanceStrategy$DefaultContextualInstanceStrategy.get(ContextualInstanceStrategy.java:100) > at org.jboss.weld.bean.ContextualInstance.get(ContextualInstance.java:50) > at > org.jboss.weld.manager.BeanManagerImpl.getReference(BeanManagerImpl.java:785) > at > org.jboss.weld.manager.BeanManagerImpl.getReference(BeanManagerImpl.java:808) > at > org.jboss.weld.util.ForwardingBeanManager.getReference(ForwardingBeanManager.java:61) > at > org.jboss.weld.bean.builtin.BeanManagerProxy.getReference(BeanManagerProxy.java:85) > at > org.glassfish.jersey.ext.cdi1x.internal.CdiUtil.getBeanReference(CdiUtil.java:151) > at > org.glassfish.jersey.ext.cdi1x.internal.AbstractCdiBeanSupplier$1.getInstance(AbstractCdiBeanSupplier.java:93) > at > org.glassfish.jersey.ext.cdi1x.internal.AbstractCdiBeanSupplier._provide(AbstractCdiBeanSupplier.java:127) > at > org.glassfish.jersey.ext.cdi1x.internal.RequestScopedCdiBeanSupplier.get(RequestScopedCdiBeanSupplier.java:70) > at > org.glassfish.jersey.inject.hk2.InstanceSupplierFactoryBridge.provide(InstanceSupplierFactoryBridge.java:77) > at org.jvnet.hk2.internal.FactoryCreator.create(FactoryCreator.java:153) > at org.jvnet.hk2.internal.SystemDescriptor.create(SystemDescriptor.java:487) > at > org.jvnet.hk2.internal.PerLookupContext.findOrCreate(PerLookupContext.java:70) > at org.jvnet.hk2.internal.Utilities.createService(Utilities.java:2126) > at > org.jvnet.hk2.internal.ServiceLocatorImpl.internalGetService(ServiceLocatorImpl.java:777) > at > org.jvnet.hk2.internal.ServiceLocatorImpl.internalGetService(ServiceLocatorImpl.java:740) > at > org.jvnet.hk2.internal.ServiceLocatorImpl.getService(ServiceLocatorImpl.java:710) > at > org.glassfish.jersey.inject.hk2.AbstractHk2InjectionManager.getInstance(AbstractHk2InjectionManager.java:184) > at > org.glassfish.jersey.inject.hk2.ImmediateHk2InjectionManager.getInstance(ImmediateHk2InjectionManager.java:54) > at > org.glassfish.jersey.internal.inject.Injections.getOrCreate(Injections.java:129) > at > org.glassfish.jersey.server.model.MethodHandler$ClassBasedMethodHandler.getInstance(MethodHandler.java:284) > at > org.glassfish.jersey.server.internal.routing.PushMethodHandlerRouter.apply(PushMethodHandlerRouter.java:75) > at > org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:110) > at > org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:113) > at > org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:113) > at > org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:113) > at > org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:93) > at > org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:62) > at org.glassfish.jersey.process.internal.Stages.process(Stages.java:197) > at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:269) > at org.glassfish.jersey.internal.Errors$1.call(Errors.java:272) > at org.glassfish.jersey.internal.Errors$1.call(Errors.java:268) > at org.glassfish.jersey.internal.Errors.process(Errors.java:316) > at org.glassfish.jersey.internal.Errors.process(Errors.java:298) > at org.glassfish.jersey.internal.Errors.process(Errors.java:268) > at > org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:289) > at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:256) > at > org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:703) > at > org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:416) > at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:370) > at > org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:389) > at > org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:342) > at > org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:229) > at > org.eclipse.jetty.servlet.ServletHolder$NotAsyncServlet.service(ServletHolder.java:1395) > at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:755) > at > org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1617) > at > org.apache.hadoop.ozone.s3.RootPageDisplayFilter.doFilter(RootPageDisplayFilter.java:53) > at > org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1596) > at > org.apache.hadoop.hdds.server.http.HttpServer2$QuotingInputFilter.doFilter(HttpServer2.java:1666) > at > org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) > at > org.apache.hadoop.hdds.server.http.NoCacheFilter.doFilter(NoCacheFilter.java:48) > at > org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) > at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) > at > org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:590) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) > at > org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1607) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) > at > org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1297) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) > at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) > at > org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1577) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) > at > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1212) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) > at > org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) > at org.eclipse.jetty.server.Server.handle(Server.java:500) > at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) > at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) > at > org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270) > at > org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) > at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129) > at > org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:388) > at > org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) > at > org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) > at java.lang.Thread.run(Thread.java:748) > Caused by: java.lang.reflect.InvocationTargetException > at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.jboss.weld.injection.producer.DefaultLifecycleCallbackInvoker.invokeMethods(DefaultLifecycleCallbackInvoker.java:97) > ... 121 more > Caused by: java.lang.NullPointerException > > !image-2020-10-13-15-23-49-864.png! -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-issues-h...@hadoop.apache.org