[ https://issues.apache.org/jira/browse/HBASE-25971?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Michael Stack resolved HBASE-25971. ----------------------------------- Resolution: Not A Problem Resolving as 'Not a problem'. This seems to have been some issue around building artifacts for RC testing. Subsequent builds worked (after HBASE-25989) > FanOutOneBlockAsyncDFSOutputHelper stuck when run against hadoop-3.3.1-RC3 > -------------------------------------------------------------------------- > > Key: HBASE-25971 > URL: https://issues.apache.org/jira/browse/HBASE-25971 > Project: HBase > Issue Type: Bug > Reporter: Michael Stack > Priority: Major > > This is in the log: > {code} > 2021-06-04 21:29:39,138 DEBUG [master/oss-master-1:16000:becomeActiveMaster] > ipc.ProtobufRpcEngine: Call: addBlock took 6ms > 2021-06-04 21:29:39,169 WARN [RS-EventLoopGroup-1-1] > concurrent.DefaultPromise: An exception was thrown by > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete() > java.lang.IllegalArgumentException: object is not an instance of declaring > class > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > at > org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.<init>(ProtobufDecoder.java:69) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:343) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:112) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:425) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:552) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:419) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:112) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:605) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104) > at > org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) > at > org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) > at > org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) > at > org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) > at > org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:470) > at > org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:378) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) > at > org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) > at java.base/java.lang.Thread.run(Thread.java:834) > {code} > These are WARNs but Master startup is stuck here: > {code} > "master/oss-master-1:16000:becomeActiveMaster" #88 daemon prio=5 os_prio=0 > cpu=666.20ms elapsed=1834.30s tid=0x00005603036c6230 nid=0xd3 in > Object.wait() [0x00007f335f440000] > java.lang.Thread.State: WAITING (on object monitor) > at java.lang.Object.wait(java.base@11.0.9/Native Method) > - waiting on <no object reference available> > at java.lang.Object.wait(java.base@11.0.9/Object.java:328) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.awaitUninterruptibly(DefaultPromise.java:275) > - waiting to re-lock in wait() <0x000000008e0ff540> (a > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.syncUninterruptibly(DefaultPromise.java:411) > at > org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.syncUninterruptibly(DefaultPromise.java:35) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.createOutput(FanOutOneBlockAsyncDFSOutputHelper.java:549) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$400(FanOutOneBlockAsyncDFSOutputHelper.java:112) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$8.doCall(FanOutOneBlockAsyncDFSOutputHelper.java:616) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$8.doCall(FanOutOneBlockAsyncDFSOutputHelper.java:611) > at > org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) > at > org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.createOutput(FanOutOneBlockAsyncDFSOutputHelper.java:624) > at > org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutputHelper.createOutput(AsyncFSOutputHelper.java:53) > at > org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.initOutput(AsyncProtobufLogWriter.java:180) > at > org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:166) > at > org.apache.hadoop.hbase.wal.AsyncFSWALProvider.createAsyncWriter(AsyncFSWALProvider.java:113) > at > org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:663) > at > org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:131) > at > org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:852) > at > org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:555) > at > org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.init(AbstractFSWAL.java:496) > at > org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:161) > at > org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:63) > at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:296) > at > org.apache.hadoop.hbase.master.region.MasterRegion.createWAL(MasterRegion.java:187) > at > org.apache.hadoop.hbase.master.region.MasterRegion.open(MasterRegion.java:260) > at > org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:303) > at > org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:104) > at > org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:834) > at > org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2091) > at > org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:507) > at > org.apache.hadoop.hbase.master.HMaster$$Lambda$164/0x0000000100559440.run(Unknown > Source) > at java.lang.Thread.run(java.base@11.0.9/Thread.java:834) > {code} > I don't see these log messges which I was thinking I should.... > {code} > try { > protobufMessageLiteClass = > Class.forName("org.apache.hadoop.thirdparty.protobuf.MessageLite"); > protobufMessageLiteBuilderClass = Class.forName( > "org.apache.hadoop.thirdparty.protobuf.MessageLite.Builder"); > LOG.debug("Hadoop 3.3 and above shades protobuf."); > } catch (ClassNotFoundException e) { > LOG.debug("Hadoop 3.2 and below use unshaded protobuf.", e); > } > {code} -- This message was sent by Atlassian Jira (v8.3.4#803005)