This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 49755f71f2d HBASE-28714 Hadoop check for hadoop 3.4.0 is failing
(#6064)
49755f71f2d is described below
commit 49755f71f2d6f139ac56c2384c431337395470c3
Author: Duo Zhang <zhang...@apache.org>
AuthorDate: Wed Jul 10 10:18:01 2024 +0800
HBASE-28714 Hadoop check for hadoop 3.4.0 is failing (#6064)
Signed-off-by: Nick Dimiduk <ndimi...@apache.org>
(cherry picked from commit d2d341e6e1971c2696ad702180eb682375d85be5)
---
.../hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java | 4 ++--
.../io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java | 12 +++++++++---
.../org/apache/hadoop/hbase/io/asyncfs/ProtobufDecoder.java | 12 +++++++-----
.../org/apache/hadoop/hbase/client/TestFlushFromClient.java | 5 +++--
pom.xml | 8 ++++++++
5 files changed, 29 insertions(+), 12 deletions(-)
diff --git
a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
index 7c2ba4ac999..ccd169217f9 100644
---
a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
+++
b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
@@ -30,7 +30,6 @@ import static
org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStag
import static
org.apache.hbase.thirdparty.io.netty.channel.ChannelOption.CONNECT_TIMEOUT_MILLIS;
import static
org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleState.READER_IDLE;
-import com.google.protobuf.CodedOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.lang.reflect.InvocationTargetException;
@@ -92,6 +91,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream;
import org.apache.hbase.thirdparty.io.netty.bootstrap.Bootstrap;
import org.apache.hbase.thirdparty.io.netty.buffer.ByteBuf;
import org.apache.hbase.thirdparty.io.netty.buffer.ByteBufAllocator;
@@ -409,7 +409,7 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
writeBlockProtoBuilder.setStorageType(PBHelperClient.convertStorageType(storageType)).build();
int protoLen = proto.getSerializedSize();
ByteBuf buffer =
- channel.alloc().buffer(3 +
CodedOutputStream.computeRawVarint32Size(protoLen) + protoLen);
+ channel.alloc().buffer(3 +
CodedOutputStream.computeUInt32SizeNoTag(protoLen) + protoLen);
buffer.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION);
buffer.writeByte(Op.WRITE_BLOCK.code);
proto.writeDelimitedTo(new ByteBufOutputStream(buffer));
diff --git
a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
index 00b6631379b..4f5ae5b22a9 100644
---
a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
+++
b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
@@ -21,7 +21,6 @@ import static
org.apache.hadoop.hbase.util.NettyFutureUtils.safeWrite;
import static
org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY;
import static
org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleState.READER_IDLE;
-import com.google.protobuf.CodedOutputStream;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
@@ -81,6 +80,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
+import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream;
import org.apache.hbase.thirdparty.io.netty.buffer.ByteBuf;
import org.apache.hbase.thirdparty.io.netty.buffer.ByteBufOutputStream;
import org.apache.hbase.thirdparty.io.netty.buffer.CompositeByteBuf;
@@ -391,7 +391,7 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
Class<?> builderClass =
DataTransferEncryptorMessageProto.Builder.class;
// Try the unrelocated ByteString
- Class<?> byteStringClass = com.google.protobuf.ByteString.class;
+ Class<?> byteStringClass;
try {
// See if it can load the relocated ByteString, which comes from
hadoop-thirdparty.
byteStringClass =
Class.forName("org.apache.hadoop.thirdparty.protobuf.ByteString");
@@ -400,6 +400,12 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
} catch (ClassNotFoundException e) {
LOG.debug("Did not find relocated ByteString class from
hadoop-thirdparty."
+ " Assuming this is below Hadoop 3.3.0", e);
+ try {
+ byteStringClass = Class.forName("com.google.protobuf.ByteString");
+ LOG.debug("com.google.protobuf.ByteString found.");
+ } catch (ClassNotFoundException ex) {
+ throw new RuntimeException(ex);
+ }
}
// LiteralByteString is a package private class in protobuf. Make it
accessible.
@@ -446,7 +452,7 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
}
DataTransferEncryptorMessageProto proto = builder.build();
int size = proto.getSerializedSize();
- size += CodedOutputStream.computeRawVarint32Size(size);
+ size += CodedOutputStream.computeUInt32SizeNoTag(size);
ByteBuf buf = ctx.alloc().buffer(size);
proto.writeDelimitedTo(new ByteBufOutputStream(buf));
safeWrite(ctx, buf);
diff --git
a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/ProtobufDecoder.java
b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/ProtobufDecoder.java
index a0b5cc00841..35344708b36 100644
---
a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/ProtobufDecoder.java
+++
b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/ProtobufDecoder.java
@@ -109,11 +109,7 @@ public class ProtobufDecoder extends
MessageToMessageDecoder<ByteBuf> {
static {
boolean hasParser = false;
-
// These are the protobuf classes coming from Hadoop. Not the one from
hbase-shaded-protobuf
- protobufMessageLiteClass = com.google.protobuf.MessageLite.class;
- protobufMessageLiteBuilderClass =
com.google.protobuf.MessageLite.Builder.class;
-
try {
protobufMessageLiteClass =
Class.forName("org.apache.hadoop.thirdparty.protobuf.MessageLite");
protobufMessageLiteBuilderClass =
@@ -121,6 +117,12 @@ public class ProtobufDecoder extends
MessageToMessageDecoder<ByteBuf> {
LOG.debug("Hadoop 3.3 and above shades protobuf.");
} catch (ClassNotFoundException e) {
LOG.debug("Hadoop 3.2 and below use unshaded protobuf.", e);
+ try {
+ protobufMessageLiteClass =
Class.forName("com.google.protobuf.MessageLite");
+ protobufMessageLiteBuilderClass =
Class.forName("com.google.protobuf.MessageLite$Builder");
+ } catch (ClassNotFoundException ex) {
+ throw new RuntimeException("can not initialize protobuf related
classes for hadoop", ex);
+ }
}
try {
@@ -130,7 +132,7 @@ public class ProtobufDecoder extends
MessageToMessageDecoder<ByteBuf> {
hasParser = true;
} catch (NoSuchMethodException e) {
// If the method is not found, we are in trouble. Abort.
- throw new RuntimeException(e);
+ throw new RuntimeException("can not initialize protobuf related classes
for hadoop", e);
}
HAS_PARSER = hasParser;
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java
index dea3f3d53cb..15023def30f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -47,6 +46,8 @@ import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
+
@Category({ MediumTests.class, ClientTests.class })
public class TestFlushFromClient {
@@ -76,7 +77,7 @@ public class TestFlushFromClient {
@AfterClass
public static void tearDownAfterClass() throws Exception {
- IOUtils.cleanup(null, asyncConn);
+ Closeables.close(asyncConn, true);
TEST_UTIL.shutdownMiniCluster();
}
diff --git a/pom.xml b/pom.xml
index 4547493ca78..bf03bc69ed2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4284,6 +4284,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -4374,6 +4378,10 @@
<groupId>javax.ws.rs</groupId>
<artifactId>jsr311-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
