[
https://issues.apache.org/jira/browse/HDDS-4386?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Glen Geng updated HDDS-4386:
----------------------------
Description: (was: Configured goofys and s3g on different hosts and
Fiotest writes files on the goofys mount point. Export AWS secrets on the s3g
host. See a bunch of NPE in s3g logs.
# Looks like missing AWS auth header could cause NPE. Looks like
AWSSignatureProcessor.init() doesn't handle header missing which causes NPE.
# Why it's missing AWS auth header is also unknown.
Note that there are files that have been successfully written into Ozone via
goofys, while not all of them are succeeded.
2020-10-13 11:18:43,425 [qtp1686100174-1238] ERROR
org.apache.hadoop.ozone.s3.OzoneClientProducer: Error:
org.jboss.weld.exceptions.WeldException: WELD-000049: Unable to invoke public
void org.apache.hadoop.ozone.s3.AWSSignatureProcessor.init() throws
java.lang.Exception on org.apache.hadoop.ozone.s3.AWSSignatureProcessor@5535155b
at
org.jboss.weld.injection.producer.DefaultLifecycleCallbackInvoker.invokeMethods(DefaultLifecycleCallbackInvoker.java:99)
at
org.jboss.weld.injection.producer.DefaultLifecycleCallbackInvoker.postConstruct(DefaultLifecycleCallbackInvoker.java:80)
at
org.jboss.weld.injection.producer.BasicInjectionTarget.postConstruct(BasicInjectionTarget.java:122)
at
org.glassfish.jersey.ext.cdi1x.internal.CdiComponentProvider$InjectionManagerInjectedCdiTarget.postConstruct(CdiComponentProvider.java:887)
at org.jboss.weld.bean.ManagedBean.create(ManagedBean.java:162)
at org.jboss.weld.context.AbstractContext.get(AbstractContext.java:96)
at
org.jboss.weld.bean.ContextualInstanceStrategy$DefaultContextualInstanceStrategy.get(ContextualInstanceStrategy.java:100)
at
org.jboss.weld.bean.ContextualInstanceStrategy$CachingContextualInstanceStrategy.get(ContextualInstanceStrategy.java:177)
at org.jboss.weld.bean.ContextualInstance.get(ContextualInstance.java:50)
at
org.jboss.weld.bean.proxy.ContextBeanInstance.getInstance(ContextBeanInstance.java:99)
at
org.jboss.weld.bean.proxy.ProxyMethodHandler.getInstance(ProxyMethodHandler.java:125)
at
org.apache.hadoop.ozone.s3.AWSSignatureProcessor$Proxy$_$$_WeldClientProxy.getAwsAccessId(Unknown
Source)
at
org.apache.hadoop.ozone.s3.OzoneClientProducer.getClient(OzoneClientProducer.java:79)
at
org.apache.hadoop.ozone.s3.OzoneClientProducer.createClient(OzoneClientProducer.java:68)
at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.jboss.weld.injection.StaticMethodInjectionPoint.invoke(StaticMethodInjectionPoint.java:88)
at
org.jboss.weld.injection.StaticMethodInjectionPoint.invoke(StaticMethodInjectionPoint.java:78)
at
org.jboss.weld.injection.producer.ProducerMethodProducer.produce(ProducerMethodProducer.java:100)
at
org.jboss.weld.injection.producer.AbstractMemberProducer.produce(AbstractMemberProducer.java:161)
at
org.jboss.weld.bean.AbstractProducerBean.create(AbstractProducerBean.java:180)
at
org.jboss.weld.context.unbound.DependentContextImpl.get(DependentContextImpl.java:70)
at
org.jboss.weld.bean.ContextualInstanceStrategy$DefaultContextualInstanceStrategy.get(ContextualInstanceStrategy.java:100)
at org.jboss.weld.bean.ContextualInstance.get(ContextualInstance.java:50)
at
org.jboss.weld.manager.BeanManagerImpl.getReference(BeanManagerImpl.java:785)
at
org.jboss.weld.manager.BeanManagerImpl.getInjectableReference(BeanManagerImpl.java:885)
at
org.jboss.weld.injection.FieldInjectionPoint.inject(FieldInjectionPoint.java:92)
at org.jboss.weld.util.Beans.injectBoundFields(Beans.java:358)
at org.jboss.weld.util.Beans.injectFieldsAndInitializers(Beans.java:369)
at
org.jboss.weld.injection.producer.ResourceInjector$1.proceed(ResourceInjector.java:70)
at
org.jboss.weld.injection.InjectionContextImpl.run(InjectionContextImpl.java:48)
at
org.jboss.weld.injection.producer.ResourceInjector.inject(ResourceInjector.java:72)
at
org.jboss.weld.injection.producer.BasicInjectionTarget.inject(BasicInjectionTarget.java:117)
at
org.glassfish.jersey.ext.cdi1x.internal.CdiComponentProvider$InjectionManagerInjectedCdiTarget.inject(CdiComponentProvider.java:873)
at org.jboss.weld.bean.ManagedBean.create(ManagedBean.java:159)
at
org.jboss.weld.context.unbound.DependentContextImpl.get(DependentContextImpl.java:70)
at
org.jboss.weld.bean.ContextualInstanceStrategy$DefaultContextualInstanceStrategy.get(ContextualInstanceStrategy.java:100)
at org.jboss.weld.bean.ContextualInstance.get(ContextualInstance.java:50)
at
org.jboss.weld.manager.BeanManagerImpl.getReference(BeanManagerImpl.java:785)
at
org.jboss.weld.manager.BeanManagerImpl.getReference(BeanManagerImpl.java:808)
at
org.jboss.weld.util.ForwardingBeanManager.getReference(ForwardingBeanManager.java:61)
at
org.jboss.weld.bean.builtin.BeanManagerProxy.getReference(BeanManagerProxy.java:85)
at
org.glassfish.jersey.ext.cdi1x.internal.CdiUtil.getBeanReference(CdiUtil.java:151)
at
org.glassfish.jersey.ext.cdi1x.internal.AbstractCdiBeanSupplier$1.getInstance(AbstractCdiBeanSupplier.java:93)
at
org.glassfish.jersey.ext.cdi1x.internal.AbstractCdiBeanSupplier._provide(AbstractCdiBeanSupplier.java:127)
at
org.glassfish.jersey.ext.cdi1x.internal.RequestScopedCdiBeanSupplier.get(RequestScopedCdiBeanSupplier.java:70)
at
org.glassfish.jersey.inject.hk2.InstanceSupplierFactoryBridge.provide(InstanceSupplierFactoryBridge.java:77)
at org.jvnet.hk2.internal.FactoryCreator.create(FactoryCreator.java:153)
at org.jvnet.hk2.internal.SystemDescriptor.create(SystemDescriptor.java:487)
at
org.jvnet.hk2.internal.PerLookupContext.findOrCreate(PerLookupContext.java:70)
at org.jvnet.hk2.internal.Utilities.createService(Utilities.java:2126)
at
org.jvnet.hk2.internal.ServiceLocatorImpl.internalGetService(ServiceLocatorImpl.java:777)
at
org.jvnet.hk2.internal.ServiceLocatorImpl.internalGetService(ServiceLocatorImpl.java:740)
at
org.jvnet.hk2.internal.ServiceLocatorImpl.getService(ServiceLocatorImpl.java:710)
at
org.glassfish.jersey.inject.hk2.AbstractHk2InjectionManager.getInstance(AbstractHk2InjectionManager.java:184)
at
org.glassfish.jersey.inject.hk2.ImmediateHk2InjectionManager.getInstance(ImmediateHk2InjectionManager.java:54)
at
org.glassfish.jersey.internal.inject.Injections.getOrCreate(Injections.java:129)
at
org.glassfish.jersey.server.model.MethodHandler$ClassBasedMethodHandler.getInstance(MethodHandler.java:284)
at
org.glassfish.jersey.server.internal.routing.PushMethodHandlerRouter.apply(PushMethodHandlerRouter.java:75)
at
org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:110)
at
org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:113)
at
org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:113)
at
org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:113)
at
org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:93)
at
org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:62)
at org.glassfish.jersey.process.internal.Stages.process(Stages.java:197)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:269)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:272)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:268)
at org.glassfish.jersey.internal.Errors.process(Errors.java:316)
at org.glassfish.jersey.internal.Errors.process(Errors.java:298)
at org.glassfish.jersey.internal.Errors.process(Errors.java:268)
at
org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:289)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:256)
at
org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:703)
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:416)
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:370)
at
org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:389)
at
org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:342)
at
org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:229)
at
org.eclipse.jetty.servlet.ServletHolder$NotAsyncServlet.service(ServletHolder.java:1395)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:755)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1617)
at
org.apache.hadoop.ozone.s3.RootPageDisplayFilter.doFilter(RootPageDisplayFilter.java:53)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1596)
at
org.apache.hadoop.hdds.server.http.HttpServer2$QuotingInputFilter.doFilter(HttpServer2.java:1666)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at
org.apache.hadoop.hdds.server.http.NoCacheFilter.doFilter(NoCacheFilter.java:48)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:590)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1607)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1297)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1577)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1212)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)
at
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:388)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.jboss.weld.injection.producer.DefaultLifecycleCallbackInvoker.invokeMethods(DefaultLifecycleCallbackInvoker.java:97)
... 121 more
Caused by: java.lang.NullPointerException
!image-2020-10-13-15-23-49-864.png!)
> Each EndpointStateMachine uses its own thread pool to talk with SCM/Recon
> -------------------------------------------------------------------------
>
> Key: HDDS-4386
> URL: https://issues.apache.org/jira/browse/HDDS-4386
> Project: Hadoop Distributed Data Store
> Issue Type: Bug
> Affects Versions: 1.0.0
> Reporter: Glen Geng
> Assignee: Glen Geng
> Priority: Blocker
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: ozone-issues-h...@hadoop.apache.org
