This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new f0a6007cde1 branch-4.1: [chore](external) Upgrade Hive shade and align
BE exec version (#66907)
f0a6007cde1 is described below
commit f0a6007cde1e8fce71beb2fa75322d89ed348b9d
Author: Socrates <[email protected]>
AuthorDate: Wed Aug 19 15:38:01 2026 +0800
branch-4.1: [chore](external) Upgrade Hive shade and align BE exec version
(#66907)
### What problem does this PR solve?
Related PRs: #66809, #66685
`branch-4.1` uses Paimon 1.4.2, but `hive-catalog-shade` 3.1.1 embeds
Paimon 1.3 classes. The existing packaging workaround therefore removes
`org/apache/paimon/**` and related service descriptors from the shade
artifact and relies on separately managed Paimon dependencies.
`hive-catalog-shade` 3.1.2 now embeds Paimon 1.4.2, so the workaround is
no longer needed.
In addition, external table sink hash routing on `branch-4.1` currently
uses BE execution version 12. Master reserves version 12 for Iceberg
Variant compatibility and reserves version 13 for external sink hash
routing in #66809. Release branches must use the same compatibility
boundary.
### What changed?
- Upgrade `hive-catalog-shade` from 3.1.1 to 3.1.2.
- Remove the dependency-order workaround and the direct
`paimon-hive-connector-3.1` dependency.
- Remove the Maven Ant repackaging step that strips
`org/apache/paimon/**` and Paimon service descriptors.
- Move external table sink hash routing from BE execution version 12 to
13.
- Advance the FE and BE maximum execution version to 13 and document
master's version 12 reservation.
### Validation
- Verified XML syntax with `xmllint`.
- Verified Maven dependency resolution includes
`hive-catalog-shade:3.1.2` and Paimon 1.4.2 modules.
- Verified the Paimon core and Hive storage-handler classes embedded in
`hive-catalog-shade:3.1.2` match Paimon 1.4.2.
- Formatted the changed BE files with clang-format 16.0.6.
- Ran `git diff --check`.
- BE compilation was intentionally skipped because this change only
updates compatibility constants and packaging metadata.
### Release note
None
---
be/src/agent/be_exec_version_manager.cpp | 6 +-
be/src/agent/be_exec_version_manager.h | 2 +-
.../main/java/org/apache/doris/common/Config.java | 2 +-
fe/fe-core/pom.xml | 83 +++++++---------------
fe/pom.xml | 2 +-
5 files changed, 31 insertions(+), 64 deletions(-)
diff --git a/be/src/agent/be_exec_version_manager.cpp
b/be/src/agent/be_exec_version_manager.cpp
index de9dfb9745d..a2d3d05f384 100644
--- a/be/src/agent/be_exec_version_manager.cpp
+++ b/be/src/agent/be_exec_version_manager.cpp
@@ -126,11 +126,13 @@ void
BeExecVersionManager::check_function_compatibility(int current_be_exec_vers
// a. use new fixed object serialization way.
// 11: start from master
// a. enforce Iceberg SQL MERGE cardinality only when every executing BE
supports it.
-// 12: start from Doris 4.1
+// 12: start from master
+// a. support Variant columns and delete-only writer omission in Iceberg SQL
MERGE.
+// 13: start from Doris 4.1
// a. support strict ownership hash routing for external table sink writers.
// b. support Paimon default fixed-bucket routing in the external sink
exchange.
-const int BeExecVersionManager::max_be_exec_version = 12;
+const int BeExecVersionManager::max_be_exec_version = 13;
const int BeExecVersionManager::min_be_exec_version = 0;
std::map<std::string, std::set<int>>
BeExecVersionManager::_function_change_map {};
std::set<std::string> BeExecVersionManager::_function_restrict_map;
diff --git a/be/src/agent/be_exec_version_manager.h
b/be/src/agent/be_exec_version_manager.h
index b19d1aabac8..5d081efb258 100644
--- a/be/src/agent/be_exec_version_manager.h
+++ b/be/src/agent/be_exec_version_manager.h
@@ -34,7 +34,7 @@ constexpr inline int USE_CONST_SERDE =
8; // support const column in serialize/deserialize function: PR #41175
constexpr inline int USE_NEW_FIXED_OBJECT_SERIALIZATION_VERSION = 10;
constexpr inline int SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION = 11;
-constexpr inline int SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION = 12;
+constexpr inline int SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION = 13;
class BeExecVersionManager {
public:
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 6432720fb62..1c6cf2996d5 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2260,7 +2260,7 @@ public class Config extends ConfigBase {
* Max data version of backends serialize block.
*/
@ConfField(mutable = false)
- public static int max_be_exec_version = 12;
+ public static int max_be_exec_version = 13;
/**
* Min data version of backends serialize block.
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index a880c5ffad4..9b8240c9610 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -468,32 +468,6 @@ under the License.
<groupId>com.google.flatbuffers</groupId>
<artifactId>flatbuffers-java</artifactId>
</dependency>
- <!-- Must precede hive-catalog-shade: that published shade artifact
embeds
- Paimon 1.3 classes, while Doris compiles against paimon.version.
-->
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-hive-connector-3.1</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-format</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-s3</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-jindo</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>hive-catalog-shade</artifactId>
@@ -604,6 +578,30 @@ under the License.
<artifactId>iceberg-aws</artifactId>
<version>${iceberg.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-format</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-s3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-jindo</artifactId>
+ </dependency>
+
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>glue</artifactId>
@@ -1117,44 +1115,11 @@ under the License.
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
- <!-- hive-catalog-shade 3.1.1 embeds Paimon 1.3.
The FE supplies
- Paimon from the managed paimon.version
instead. Repackage this
- dependency below without its embedded Paimon
runtime. -->
-
<excludeArtifactIds>hive-catalog-shade</excludeArtifactIds>
<skip>${skip.plugin}</skip>
</configuration>
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>3.1.0</version>
- <executions>
- <execution>
- <id>package-hive-catalog-shade-without-paimon</id>
- <phase>package</phase>
- <configuration>
- <target>
- <delete
dir="${project.build.directory}/hive-catalog-shade-without-paimon"/>
- <mkdir
dir="${project.build.directory}/hive-catalog-shade-without-paimon"/>
- <unzip
src="${settings.localRepository}/org/apache/doris/hive-catalog-shade/${doris.hive.catalog.shade.version}/hive-catalog-shade-${doris.hive.catalog.shade.version}.jar"
-
dest="${project.build.directory}/hive-catalog-shade-without-paimon">
- <patternset>
- <exclude name="org/apache/paimon/**"/>
- <exclude
name="META-INF/services/org.apache.paimon*"/>
- </patternset>
- </unzip>
- <jar
destfile="${project.build.directory}/lib/hive-catalog-shade-${doris.hive.catalog.shade.version}.jar"
-
basedir="${project.build.directory}/hive-catalog-shade-without-paimon"/>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
diff --git a/fe/pom.xml b/fe/pom.xml
index 3783d2660e6..b1dfc212182 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -226,7 +226,7 @@ under the License.
<module>fe-authentication</module>
</modules>
<properties>
-
<doris.hive.catalog.shade.version>3.1.1</doris.hive.catalog.shade.version>
+
<doris.hive.catalog.shade.version>3.1.2</doris.hive.catalog.shade.version>
<!-- iceberg 1.9.1 depends avro on 1.12 -->
<avro.version>1.12.1</avro.version>
<parquet.version>1.17.0</parquet.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
