This is an automated email from the ASF dual-hosted git repository.
CalvinKirs pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 9b094acb7f8 branch-4.1: [improvement](fe) Evaluate protobuf 4 with
Arrow 19 (#61943) (#63720)
9b094acb7f8 is described below
commit 9b094acb7f853a67de4227b3c092aefa679dbdb6
Author: Calvin Kirs <[email protected]>
AuthorDate: Tue Aug 4 17:40:52 2026 +0800
branch-4.1: [improvement](fe) Evaluate protobuf 4 with Arrow 19 (#61943)
(#63720)
#61943
---------
---
conf/be.conf | 2 +-
conf/fe.conf | 2 +-
.../rolemapping/UnifiedRoleMappingCelEngine.java | 2 +-
.../org/apache/doris/persist/gson/GsonUtils.java | 6 +++-
.../gson/GsonProtobufCompatibilityTest.java | 40 ++++++++++++++++++++++
fe/pom.xml | 10 +++---
regression-test/pipeline/p0/conf/be.conf | 2 +-
regression-test/pipeline/p0/conf/fe.conf | 2 +-
run-regression-test.sh | 7 +++-
9 files changed, 61 insertions(+), 12 deletions(-)
diff --git a/conf/be.conf b/conf/be.conf
index c1381918107..e9024580a65 100644
--- a/conf/be.conf
+++ b/conf/be.conf
@@ -21,7 +21,7 @@ CUR_DATE=`date +%Y%m%d-%H%M%S`
LOG_DIR="${DORIS_HOME}/log/"
# For jdk 17, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djol.skipHotspotSAAttach=true
-Xmx2048m -DlogPath=$LOG_DIR/jni.log
-Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true
-Dsun.java.command=DorisBE -XX:-CriticalJNINatives
-XX:+IgnoreUnrecognizedVMOptions -Darrow.enable_null_check_for_get=false
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add [...]
+JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djol.skipHotspotSAAttach=true
-Xmx2048m -DlogPath=$LOG_DIR/jni.log
-Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true
-Dsun.java.command=DorisBE -XX:-CriticalJNINatives
-XX:+IgnoreUnrecognizedVMOptions -Darrow.enable_null_check_for_get=false
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add [...]
# Set your own JAVA_HOME
# JAVA_HOME=/path/to/jdk/
diff --git a/conf/fe.conf b/conf/fe.conf
index 6a22a103a8b..b77b084411a 100644
--- a/conf/fe.conf
+++ b/conf/fe.conf
@@ -27,7 +27,7 @@ CUR_DATE=`date +%Y%m%d-%H%M%S`
LOG_DIR = ${DORIS_HOME}/log
# For jdk 17, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8
-Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR
-Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Darrow.enable_null_check_for_get=false
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java [...]
+JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8
-Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR
-Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Darrow.enable_null_check_for_get=false
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java [...]
# Set your own JAVA_HOME
# JAVA_HOME=/path/to/jdk/
diff --git
a/fe/fe-authentication/fe-authentication-role-mapping/src/main/java/org/apache/doris/authentication/rolemapping/UnifiedRoleMappingCelEngine.java
b/fe/fe-authentication/fe-authentication-role-mapping/src/main/java/org/apache/doris/authentication/rolemapping/UnifiedRoleMappingCelEngine.java
index 079e085cf87..73567c528ff 100644
---
a/fe/fe-authentication/fe-authentication-role-mapping/src/main/java/org/apache/doris/authentication/rolemapping/UnifiedRoleMappingCelEngine.java
+++
b/fe/fe-authentication/fe-authentication-role-mapping/src/main/java/org/apache/doris/authentication/rolemapping/UnifiedRoleMappingCelEngine.java
@@ -29,8 +29,8 @@ import dev.cel.common.types.SimpleType;
import dev.cel.compiler.CelCompiler;
import dev.cel.compiler.CelCompilerFactory;
import dev.cel.parser.CelStandardMacro;
+import dev.cel.runtime.CelFunctionBinding;
import dev.cel.runtime.CelRuntime;
-import dev.cel.runtime.CelRuntime.CelFunctionBinding;
import dev.cel.runtime.CelRuntimeFactory;
import java.util.ArrayList;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
index 1097a31232a..a95a315eb5e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
@@ -257,6 +257,7 @@ import com.google.gson.annotations.SerializedName;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
+import com.google.protobuf.MessageLite;
import org.apache.commons.lang3.reflect.TypeUtils;
import java.io.ByteArrayInputStream;
@@ -682,7 +683,10 @@ public class GsonUtils {
/* due to java.lang.IllegalArgumentException:
com.lmax.disruptor.RingBuffer
<org.apache.doris.scheduler.disruptor.TimerTaskEvent> declares multiple
JSON fields named p1 */
- return
clazz.getName().startsWith("com.lmax.disruptor.RingBuffer");
+ return
clazz.getName().startsWith("com.lmax.disruptor.RingBuffer")
+ // Protobuf 4 builders expose duplicate
internal fields such as
+ // "meAsParent". They are runtime-only and
must not enter FE metadata.
+ ||
MessageLite.Builder.class.isAssignableFrom(clazz);
}
});
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/persist/gson/GsonProtobufCompatibilityTest.java
b/fe/fe-core/src/test/java/org/apache/doris/persist/gson/GsonProtobufCompatibilityTest.java
new file mode 100644
index 00000000000..2239a23b3b9
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/persist/gson/GsonProtobufCompatibilityTest.java
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.persist.gson;
+
+import com.google.gson.annotations.SerializedName;
+import doris.segment_v2.SegmentV2;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class GsonProtobufCompatibilityTest {
+ private static class ProtobufBuilderHolder {
+ @SerializedName("name")
+ private final String name = "holder";
+
+ @SerializedName("builder")
+ private final Object builder =
SegmentV2.ColumnPathPartInfo.newBuilder().setKey("nested_field");
+ }
+
+ @Test
+ public void testSerializeGeneratedProtobufBuilder() {
+ String json = GsonUtils.GSON.toJson(new ProtobufBuilderHolder());
+ Assert.assertTrue(json, json.contains("\"name\":\"holder\""));
+ Assert.assertFalse(json, json.contains("\"builder\""));
+ }
+}
diff --git a/fe/pom.xml b/fe/pom.xml
index e8dc5655cec..067a682770d 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -298,8 +298,9 @@ under the License.
<!--Need to ensure that the version is the same as in
arrow/java/pom.xml or compatible with it.-->
<grpc.version>1.65.1</grpc.version>
<check.freamework.version>3.53.0</check.freamework.version>
- <cel.version>0.9.1-proto3</cel.version>
- <protobuf.version>3.25.8</protobuf.version>
+ <!-- FE-only Arrow 19 startup POC: align protobuf runtime with Arrow
Java 19's protobuf BOM. -->
+ <protobuf.version>4.33.4</protobuf.version>
+ <cel.version>0.12.0</cel.version>
<!-- we use protoc-jar-maven-plugin to generate protobuf generated
code -->
<!-- see
https://repo.maven.apache.org/maven2/com/google/protobuf/protoc/ to get correct
version -->
<protoc.artifact.version>3.24.3</protoc.artifact.version>
@@ -335,9 +336,8 @@ under the License.
<iceberg.version>1.10.1</iceberg.version>
<!-- 0.56.1 has bug that "SplitMode" in query response may not be
set-->
<maxcompute.version>0.53.2-public</maxcompute.version>
- <!-- Arrow 19.0.1 will MacOS compile error and decimal type error when
convert to Parquet
- https://github.com/apache/doris/pull/51217-->
- <arrow.version>17.0.0</arrow.version>
+ <!-- FE-only POC: validate whether Arrow Java 19.0.0 can compile
before touching BE/C++ thirdparty. -->
+ <arrow.version>19.0.0</arrow.version>
<presto.hadoop.version>2.7.4-11</presto.hadoop.version>
<presto.hive.version>3.0.0-8</presto.hive.version>
<!-- lakesoul -->
diff --git a/regression-test/pipeline/p0/conf/be.conf
b/regression-test/pipeline/p0/conf/be.conf
index eefcb428284..b732395e7e4 100644
--- a/regression-test/pipeline/p0/conf/be.conf
+++ b/regression-test/pipeline/p0/conf/be.conf
@@ -23,7 +23,7 @@ PPROF_TMPDIR="$DORIS_HOME/log/"
JAVA_OPTS="-Xcheck:jni -Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log
-Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation
-XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true
-Dsun.java.command=DorisBE -XX:-CriticalJNINatives
-Dcom.mysql.cj.disableAbandonedConnectionCleanup=true"
# For jdk 17, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djol.skipHotspotSAAttach=true
-Xmx2048m -DlogPath=$LOG_DIR/jni.log
-Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true
-Dsun.java.command=DorisBE -XX:-CriticalJNINatives
-XX:+IgnoreUnrecognizedVMOptions -Darrow.enable_null_check_for_get=false
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add [...]
+JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djol.skipHotspotSAAttach=true
-Xmx2048m -DlogPath=$LOG_DIR/jni.log
-Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true
-Dsun.java.command=DorisBE -XX:-CriticalJNINatives
-XX:+IgnoreUnrecognizedVMOptions -Darrow.enable_null_check_for_get=false
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add [...]
# Set your own JAVA_HOME
# JAVA_HOME=/path/to/jdk/
diff --git a/regression-test/pipeline/p0/conf/fe.conf
b/regression-test/pipeline/p0/conf/fe.conf
index 74e33fa122c..c79ec24eb5f 100644
--- a/regression-test/pipeline/p0/conf/fe.conf
+++ b/regression-test/pipeline/p0/conf/fe.conf
@@ -30,7 +30,7 @@ LOG_DIR = ${DORIS_HOME}/log
JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx4096m
-XX:+HeapDumpOnOutOfMemoryError -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC
-XX:MaxGCPauseMillis=200 -XX:+PrintGCDateStamps -XX:+PrintGCDetails
-XX:+PrintClassHistogramAfterFullGC -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE
-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M
-Dlog4j2.formatMsgNoLookups=true
-Dcom.mysql.cj.disableAbandonedConnectionCleanup=true"
# For jdk 17, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8
-Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m
-XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR
-Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Darrow.enable_null_check_for_get=false
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.
[...]
+JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8
-Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m
-XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR
-Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
-Darrow.enable_null_check_for_get=false
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.
[...]
sys_log_level = INFO
sys_log_mode = NORMAL
diff --git a/run-regression-test.sh b/run-regression-test.sh
index bf926627319..6b454698a14 100755
--- a/run-regression-test.sh
+++ b/run-regression-test.sh
@@ -256,6 +256,12 @@ if ! test -f ${RUN_JAR:+${RUN_JAR}}; then
cd "${DORIS_HOME}"
fi
+# Arrow Flight SQL JDBC needs java.nio opened when the regression framework
runs on JDK 17+.
+if [[ -n "${JAVA_MAJOR_VERSION}" ]] && [[ "${JAVA_MAJOR_VERSION}" -ge 17 ]] \
+ && [[ " ${JAVA_OPTS:-} " != *"--add-opens=java.base/java.nio="* ]]; then
+ JAVA_OPTS="${JAVA_OPTS:+${JAVA_OPTS}
}--add-opens=java.base/java.nio=ALL-UNNAMED"
+fi
+
REGRESSION_OPTIONS_PREFIX=''
# contains framework options and not start with -
@@ -267,7 +273,6 @@ fi
echo "===== Run Regression Test ====="
-# if use jdk17, add java option "--add-opens=java.base/java.nio=ALL-UNNAMED"
if [[ "${TEAMCITY}" -eq 1 ]]; then
JAVA_OPTS="${JAVA_OPTS} -DstdoutAppenderType=teamcity -Xmx2048m"
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
