This is an automated email from the ASF dual-hosted git repository.
djwang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
The following commit(s) were added to refs/heads/main by this push:
new 50e908b4 Update HBASE to 2.3.7 in PXF (#62)
50e908b4 is described below
commit 50e908b42f6ec0404f438860a36c2ee20ce6d167
Author: Nikolay Antonov <[email protected]>
AuthorDate: Fri Feb 13 04:50:32 2026 +0500
Update HBASE to 2.3.7 in PXF (#62)
### hbase-client update
Update `hbase-client` from `1.3.2` to `2.3.7` (latest version with minimal
3rd-party library updates) with following decision made:
* Use hadoop 2 version - Despite of the fact that we are using hadoop 3 in
automation tests, we still using hadoop2 libs for all other connectors.
* Use shaded version of libraries when it relocates java-packages to new
namespaces
* Use non-shaded version of libraries when it just bundles several jars to
single fat-jar. Fat jars hides dependencies from gradle, however puts classes
on classpath. This will lead to unpredictable issues.
Fortunately, `hbase-client:2.3.7` depends on `hadoop-2.10.0` that we are
using! No changes here.
### update details
```
runtimeClasspath - Runtime classpath of source set 'main'.
\--- org.apache.hbase:hbase-client:2.3.7
+--- org.apache.hbase.thirdparty:hbase-shaded-protobuf:3.3.0
+--- org.apache.hbase:hbase-common:2.3.7
| +--- org.apache.hbase:hbase-logging:2.3.7
| +--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0
| | \--- com.google.errorprone:error_prone_annotations:2.3.4
| +--- org.apache.hbase.thirdparty:hbase-shaded-gson:3.3.0
| +--- org.apache.hbase.thirdparty:hbase-shaded-netty:3.3.0
| +--- commons-codec:commons-codec:1.13
| +--- org.apache.commons:commons-lang3:3.9
| +--- commons-io:commons-io:2.11.0
| +--- com.google.protobuf:protobuf-java:2.5.0
| +--- org.apache.htrace:htrace-core4:4.2.0-incubating
| +--- org.apache.commons:commons-crypto:1.0.0
| +--- org.apache.yetus:audience-annotations:0.5.0
| \--- org.apache.hadoop:hadoop-common:2.10.0 -> 2.10.2
+--- org.apache.hbase:hbase-hadoop-compat:2.3.7
| +--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0
(*)
| +--- org.apache.hbase:hbase-metrics-api:2.3.7
| | +--- org.apache.hbase:hbase-common:2.3.7 (*)
| | +--- org.apache.commons:commons-lang3:3.9
| | +---
org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0 (*)
+--- org.apache.hbase:hbase-hadoop2-compat:2.3.7
| +--- org.apache.hbase:hbase-hadoop-compat:2.3.7 (*)
| +--- org.apache.hbase:hbase-common:2.3.7 (*)
| +--- org.apache.hbase:hbase-metrics:2.3.7
| | +---
org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0 (*)
| | +--- org.apache.hbase:hbase-common:2.3.7 (*)
| | +--- org.apache.hbase:hbase-metrics-api:2.3.7 (*)
| | +--- io.dropwizard.metrics:metrics-core:3.2.6
| +--- org.apache.hbase:hbase-metrics-api:2.3.7 (*)
| +--- org.apache.hadoop:hadoop-common:2.10.0 -> 2.10.2 (*)
| +--- javax.activation:javax.activation-api:1.2.0
| +--- org.apache.commons:commons-lang3:3.9
| +--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0
(*)
+--- org.apache.hbase:hbase-protocol-shaded:2.3.7
| +--- org.apache.hbase.thirdparty:hbase-shaded-protobuf:3.3.0
+--- org.apache.hbase:hbase-protocol:2.3.7
| +--- com.google.protobuf:protobuf-java:2.5.0
+--- commons-codec:commons-codec:1.13
+--- commons-io:commons-io:2.11.0
+--- org.apache.commons:commons-lang3:3.9
+--- org.slf4j:slf4j-api:1.7.30 -> 1.7.36
+--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0 (*)
+--- com.google.protobuf:protobuf-java:2.5.0
+--- org.apache.hbase.thirdparty:hbase-shaded-netty:3.3.0
+--- org.apache.zookeeper:zookeeper:3.5.7 (*)
+--- org.apache.htrace:htrace-core4:4.2.0-incubating
+--- org.jruby.jcodings:jcodings:1.0.18
+--- org.jruby.joni:joni:2.1.11
| \--- org.jruby.jcodings:jcodings:1.0.13 -> 1.0.18
+--- io.dropwizard.metrics:metrics-core:3.2.6 (*)
+--- org.apache.commons:commons-crypto:1.0.0
+--- org.apache.yetus:audience-annotations:0.5.0
+--- org.apache.hadoop:hadoop-auth:2.10.0 -> 2.10.2 (*)
\--- org.apache.hadoop:hadoop-common:2.10.0 -> 2.10.2 (*)
(*) - dependencies omitted (listed previously)
```
### Other changes
* Automation tests: upgrade hbase to 2.3.7
---
ci/singlecluster/Dockerfile | 6 +--
ci/singlecluster/README.HDP3.md | 2 +-
server/build.gradle | 27 +++++++++++--
server/gradle.properties | 2 +-
server/pxf-hbase/build.gradle | 47 ++++++++++++++++------
.../pxf/plugins/hbase/HBaseDataFragmenter.java | 1 -
.../hbase/utilities/HBaseDoubleComparator.java | 6 +--
.../hbase/utilities/HBaseFloatComparator.java | 6 +--
.../hbase/utilities/HBaseIntegerComparator.java | 6 +--
9 files changed, 72 insertions(+), 31 deletions(-)
diff --git a/ci/singlecluster/Dockerfile b/ci/singlecluster/Dockerfile
index 8e8c4621..abb60e3c 100644
--- a/ci/singlecluster/Dockerfile
+++ b/ci/singlecluster/Dockerfile
@@ -12,14 +12,14 @@ RUN sudo apt-get update && \
ENV HADOOP_VERSION=3.1.2
ENV HIVE_VERSION=3.1.3
ENV ZOOKEEPER_VERSION=3.5.9
-ENV HBASE_VERSION=2.0.6
+ENV HBASE_VERSION=2.3.7
ENV TEZ_VERSION=0.9.2
# checksums from archive.apache.org
ENV
HADOOP_SHA512="0e0ee817c89b3c4eb761eca7f16640742a83b0e99b6fda26c1bee2baabedad93aab86e252bf5f1e2381c6d464bc4003d10c7cc0f61b2062f4c59732ca24d1bd9"
ENV
HIVE_SHA256="0c9b6a6359a7341b6029cc9347435ee7b379f93846f779d710b13f795b54bb16"
ENV
ZOOKEEPER_SHA512="0e5a64713abc6f36d961dd61a06f681868171a9d9228366e512a01324806d263e05508029c94d8e18307811867cdc39d848e736c252bf56c461273ef74c66a45"
-ENV
HBASE_SHA512="a0e10904ecf7f059b77bc0ce704254046a978126db720cc7e55dc53b87097715da64b8391fe3cc94348bc432871ad8f29891dc8df1ea052eb628da0fdca97c93"
+ENV
HBASE_SHA512="1032521025660daa70260cdc931f52a26c87596be444451fe1fa88b526ede55e9d6b4220e91ff6f7422bec11f30d64fa6745e95a9c36971fdb1a264a2c745693"
ENV
TEZ_SHA512="a2d94bd9fa778d42a8bac9d9da8e263e469ddfef93968b06434716554995f490231de5607541ac236e770aa0158b64250c38bc1cd57dbfa629fea705f2ffa2f5"
# faster mirror:
@@ -63,7 +63,7 @@ RUN mkdir -p $ZOOKEEPER_ROOT && \
RUN mkdir -p $HBASE_ROOT && \
curl -fSL "$HBASE_URL" -o hbase.tar.gz && \
echo "$HBASE_SHA512 hbase.tar.gz" | sha512sum -c && \
- tar xvf hbase.tar.gz -C $HBASE_ROOT --strip-components 1
--exclude="docs/*" && \
+ tar xvf hbase.tar.gz -C $HBASE_ROOT --strip-components 1
--exclude="docs/*" --exclude="lib/*-tests.jar" --exclude="lib/shaded-clients"
&& \
rm hbase.tar.gz
RUN mkdir -p $TEZ_ROOT && \
diff --git a/ci/singlecluster/README.HDP3.md b/ci/singlecluster/README.HDP3.md
index 16506de1..3a906e50 100644
--- a/ci/singlecluster/README.HDP3.md
+++ b/ci/singlecluster/README.HDP3.md
@@ -7,7 +7,7 @@ It contains the following versions:
- Hadoop 3.3.6
- Hive 3.1.3
- Zookeeper 3.5.9
-- HBase 2.0.6
+- HBase 2.3.7
- Tez 0.9.2
This version of Single cluster requires users to make some manual changes to
the configuration files once the tarball has been unpacked (see Initialization
steps below).
diff --git a/server/build.gradle b/server/build.gradle
index a1b6275c..410ec1f2 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -90,6 +90,7 @@ configure(javaProjects) {
dependency("commons-configuration:commons-configuration:1.10")
dependency("commons-io:commons-io:2.7")
dependency("commons-lang:commons-lang:2.6")
+ dependency("commons-lang:commons-lang3:3.9")
dependency("commons-logging:commons-logging:1.1.3")
dependency("io.airlift:aircompressor:0.27")
dependency("javax.jdo:jdo-api:3.0.1")
@@ -97,8 +98,9 @@ configure(javaProjects) {
dependency("net.sf.opencsv:opencsv:2.3")
dependency("org.antlr:antlr-runtime:3.5.2")
dependency("org.apache.commons:commons-compress:1.20")
+ dependency("org.apache.commons:commons-crypto:1.0.0")
dependency("org.apache.htrace:htrace-core:3.1.0-incubating")
- dependency("org.apache.htrace:htrace-core4:4.0.1-incubating")
+ dependency("org.apache.htrace:htrace-core4:4.2.0-incubating")
dependency("org.apache.zookeeper:zookeeper:3.4.6")
dependency("org.codehaus.woodstox:stax2-api:3.1.4")
@@ -139,11 +141,28 @@ configure(javaProjects) {
// HBase dependencies
dependencySet(group:"org.apache.hbase", version:"${hbaseVersion}")
{
- entry("hbase-annotations")
entry("hbase-client")
entry("hbase-common")
entry("hbase-protocol")
+ entry("hbase-protocol-shaded")
+ entry("hbase-logging")
+ entry("hbase-hadoop-compat")
+ entry("hbase-hadoop2-compat")
+ entry("hbase-metrics-api")
+ entry("hbase-metrics")
}
+ dependencySet(group:"org.apache.hbase.thirdparty",
version:"3.3.0") {
+ entry("hbase-shaded-protobuf")
+ entry("hbase-shaded-miscellaneous")
+ entry("hbase-shaded-gson")
+ entry("hbase-shaded-netty")
+ entry("hbase-unsafe")
+ }
+ dependency("org.apache.yetus:audience-annotations:0.5.0")
+ dependency("io.opentelemetry:opentelemetry-api:1.49.0")
+ dependency("io.opentelemetry:opentelemetry-context:1.49.0")
+
dependency("io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha")
+ dependency("io.dropwizard.metrics:metrics-core:3.2.6")
// Hive dependencies
dependency("org.apache.hive:hive-storage-api:${hiveStorageApiVersion}")
@@ -237,7 +256,7 @@ configure(javaProjects) {
options.compilerArgs += [
"-g", "-Xlint:varargs", "-Xlint:cast", "-Xlint:classfile",
"-Xlint:dep-ann", "-Xlint:divzero",
"-Xlint:empty", "-Xlint:finally", "-Xlint:overrides",
"-Xlint:path", "-Xlint:-processing", "-Xlint:static",
- "-Xlint:try", "-Xlint:fallthrough", "-Xlint:deprecation",
"-Xlint:unchecked", "-Xlint:-options", "-Werror"
+ "-Xlint:try", "-Xlint:fallthrough", "-Xlint:unchecked",
"-Xlint:-options", "-Werror"
]
}
@@ -245,7 +264,7 @@ configure(javaProjects) {
options.compilerArgs += [
"-g", "-Xlint:varargs", "-Xlint:cast", "-Xlint:classfile",
"-Xlint:dep-ann", "-Xlint:divzero",
"-Xlint:empty", "-Xlint:finally", "-Xlint:overrides",
"-Xlint:path", "-Xlint:-processing", "-Xlint:static",
- "-Xlint:try", "-Xlint:fallthrough", "-Xlint:deprecation",
"-Xlint:unchecked", "-Xlint:-options", "-Werror"
+ "-Xlint:try", "-Xlint:fallthrough", "-Xlint:unchecked",
"-Xlint:-options", "-Werror"
]
}
diff --git a/server/gradle.properties b/server/gradle.properties
index 42da880a..ea8b0c66 100644
--- a/server/gradle.properties
+++ b/server/gradle.properties
@@ -21,7 +21,7 @@ license=ASL 2.0
hadoopVersion=2.10.2
hiveVersion=2.3.8
hiveStorageApiVersion=2.7.3
-hbaseVersion=1.3.2
+hbaseVersion=2.3.7
junitVersion=4.11
parquetVersion=1.12.3
awsJavaSdk=1.12.261
diff --git a/server/pxf-hbase/build.gradle b/server/pxf-hbase/build.gradle
index 026d602b..5924b8f2 100644
--- a/server/pxf-hbase/build.gradle
+++ b/server/pxf-hbase/build.gradle
@@ -14,23 +14,47 @@ dependencies {
*******************************/
compileOnly("com.google.code.findbugs:annotations")
- compileOnly("org.apache.hbase:hbase-annotations")
/*******************************
- * Implementation Dependencies
+ * Project Dependencies
*******************************/
implementation(project(':pxf-api'))
- implementation("com.google.protobuf:protobuf-java")
implementation("commons-collections:commons-collections")
- implementation("org.apache.hbase:hbase-client") {
transitive = false }
- implementation("org.apache.hbase:hbase-common") {
transitive = false }
- implementation("org.apache.hbase:hbase-protocol") {
transitive = false }
- implementation("org.apache.htrace:htrace-core") {
transitive = false }
- implementation("org.apache.zookeeper:zookeeper") {
transitive = false }
- implementation("io.netty:netty-common") {
transitive = false }
- implementation("io.netty:netty-transport") {
transitive = false }
- implementation("com.yammer.metrics:metrics-core") {
transitive = false }
+
+ /*******************************
+ * Hbase
+ *******************************/
+
+ implementation("org.apache.hbase:hbase-client") {
transitive = false }
+ implementation("org.apache.hbase.thirdparty:hbase-shaded-protobuf") {
transitive = false }
+ implementation("org.apache.hbase:hbase-common") {
transitive = false }
+ implementation("org.apache.hbase:hbase-logging") {
transitive = false }
+ implementation("org.apache.hbase.thirdparty:hbase-shaded-miscellaneous") {
transitive = false }
+ implementation("org.apache.hbase.thirdparty:hbase-shaded-gson") {
transitive = false }
+ implementation("org.apache.hbase.thirdparty:hbase-shaded-netty") {
transitive = false }
+ implementation("org.apache.commons:commons-lang3") {
transitive = false }
+ implementation("org.apache.commons:commons-crypto") {
transitive = false }
+ implementation("org.apache.hadoop:hadoop-common") {
transitive = false }
+ implementation("org.apache.hadoop:hadoop-auth") {
transitive = false }
+ implementation("org.apache.hbase:hbase-hadoop-compat") {
transitive = false }
+ implementation("org.apache.hbase:hbase-metrics-api") {
transitive = false }
+ implementation("org.apache.hbase:hbase-metrics") {
transitive = false }
+ implementation("org.apache.hbase:hbase-hadoop2-compat") {
transitive = false }
+ implementation("org.apache.hbase:hbase-protocol-shaded") {
transitive = false }
+ implementation("org.apache.hbase:hbase-protocol") {
transitive = false }
+ implementation("com.google.protobuf:protobuf-java") {
transitive = false }
+ implementation("org.apache.zookeeper:zookeeper") {
transitive = false }
+ implementation("io.netty:netty-common") {
transitive = false }
+ implementation("io.netty:netty-transport") {
transitive = false }
+// skip JRuby - it is part of interactive shell
+// implementation("org.jruby.jcodings:jcodings:1.0.58")
{ transitive = false }
+// implementation("org.jruby.joni:joni:2.2.1")
{ transitive = false }
+ implementation("org.apache.yetus:audience-annotations") {
transitive = false }
+ implementation("io.opentelemetry:opentelemetry-api") {
transitive = false }
+ implementation("io.opentelemetry:opentelemetry-context") {
transitive = false }
+ implementation("io.opentelemetry.semconv:opentelemetry-semconv") {
transitive = false }
+ implementation("io.dropwizard.metrics:metrics-core:3.2.6") {
transitive = false }
implementation("org.springframework.boot:spring-boot-starter-log4j2")
@@ -39,7 +63,6 @@ dependencies {
*******************************/
testCompileOnly("com.google.code.findbugs:annotations")
- testCompileOnly("org.apache.hbase:hbase-annotations")
testImplementation("com.esotericsoftware:minlog")
testImplementation("com.esotericsoftware:reflectasm")
testImplementation('org.springframework.boot:spring-boot-starter-test')
diff --git
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/HBaseDataFragmenter.java
b/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/HBaseDataFragmenter.java
index 21d5d42d..774fb9c2 100644
---
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/HBaseDataFragmenter.java
+++
b/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/HBaseDataFragmenter.java
@@ -81,7 +81,6 @@ public class HBaseDataFragmenter extends BaseFragmenter {
public List<Fragment> getFragments() throws Exception {
// check that Zookeeper and HBase master are available
- HBaseAdmin.checkHBaseAvailable(configuration);
connection = ConnectionFactory.createConnection(configuration);
Admin hbaseAdmin = connection.getAdmin();
if (!HBaseUtilities.isTableAvailable(hbaseAdmin,
context.getDataSource())) {
diff --git
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java
b/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java
index 3627ca97..6dedaf85 100644
---
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java
+++
b/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java
@@ -19,11 +19,11 @@ package org.apache.cloudberry.pxf.plugins.hbase.utilities;
* under the License.
*/
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import
org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
-import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ComparatorProtos;
import org.apache.hadoop.hbase.util.Bytes;
public class HBaseDoubleComparator extends ByteArrayComparable {
diff --git
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseFloatComparator.java
b/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseFloatComparator.java
index bf26b003..a5291de6 100644
---
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseFloatComparator.java
+++
b/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseFloatComparator.java
@@ -19,11 +19,11 @@ package org.apache.cloudberry.pxf.plugins.hbase.utilities;
* under the License.
*/
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import
org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
-import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ComparatorProtos;
import org.apache.hadoop.hbase.util.Bytes;
public class HBaseFloatComparator extends ByteArrayComparable{
diff --git
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
b/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
index ffc0a10e..a1e589c5 100644
---
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
+++
b/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
@@ -23,11 +23,11 @@ package org.apache.cloudberry.pxf.plugins.hbase.utilities;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
import org.apache.hadoop.hbase.filter.SubstringComparator;
-import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ComparatorProtos;
import org.apache.hadoop.hbase.util.Bytes;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import
org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
/**
* This is a Filter comparator for HBase It is external to PXF HBase code.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
