This is an automated email from the ASF dual-hosted git repository. danny0405 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new b14f9e48d3d [HUDI-7005] Fix hudi-aws-bundle relocation issue with avro (#9946) b14f9e48d3d is described below commit b14f9e48d3d81cb765e5b2fb355eb2c1e24ee582 Author: Prabhu Joseph <prabhujose.ga...@gmail.com> AuthorDate: Sat Nov 4 11:04:06 2023 +0530 [HUDI-7005] Fix hudi-aws-bundle relocation issue with avro (#9946) --- .../main/java/org/apache/hudi/sink/utils/HiveSyncContext.java | 9 +++++++-- packaging/hudi-aws-bundle/pom.xml | 2 -- packaging/hudi-flink-bundle/pom.xml | 1 - 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java index 0ea783feb5c..0d4cd10b8c7 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java @@ -18,9 +18,9 @@ package org.apache.hudi.sink.utils; -import org.apache.hudi.aws.sync.AwsGlueCatalogSyncTool; import org.apache.hudi.common.config.SerializableConfiguration; import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.configuration.HadoopConfigurations; import org.apache.hudi.hive.HiveSyncTool; @@ -67,6 +67,9 @@ public class HiveSyncContext { private final Properties props; private final HiveConf hiveConf; + public static final String AWS_GLUE_CATALOG_SYNC_TOOL_CLASS = + "org.apache.hudi.aws.sync.AwsGlueCatalogSyncTool"; + private HiveSyncContext(Properties props, HiveConf hiveConf) { this.props = props; this.hiveConf = hiveConf; @@ -75,7 +78,9 @@ public class HiveSyncContext { public HiveSyncTool hiveSyncTool() { HiveSyncMode syncMode = HiveSyncMode.of(props.getProperty(HIVE_SYNC_MODE.key())); if (syncMode == HiveSyncMode.GLUE) { - return new AwsGlueCatalogSyncTool(props, hiveConf); + return ((HiveSyncTool) ReflectionUtils.loadClass(AWS_GLUE_CATALOG_SYNC_TOOL_CLASS, + new Class<?>[] {Properties.class, org.apache.hadoop.conf.Configuration.class}, + props, hiveConf)); } return new HiveSyncTool(props, hiveConf); } diff --git a/packaging/hudi-aws-bundle/pom.xml b/packaging/hudi-aws-bundle/pom.xml index 6d2596f49a3..1250296cceb 100644 --- a/packaging/hudi-aws-bundle/pom.xml +++ b/packaging/hudi-aws-bundle/pom.xml @@ -72,12 +72,10 @@ </transformers> <artifactSet> <includes combine.children="append"> - <include>org.apache.hudi:hudi-common</include> <include>org.apache.hudi:hudi-hadoop-mr</include> <include>org.apache.hudi:hudi-sync-common</include> <include>org.apache.hudi:hudi-hive-sync</include> <include>org.apache.hudi:hudi-aws</include> - <include>org.apache.parquet:parquet-avro</include> <include>org.reactivestreams:reactive-streams</include> <include>com.amazonaws:dynamodb-lock-client</include> <include>org.apache.httpcomponents:httpclient</include> diff --git a/packaging/hudi-flink-bundle/pom.xml b/packaging/hudi-flink-bundle/pom.xml index 4a490c43a98..e141e975232 100644 --- a/packaging/hudi-flink-bundle/pom.xml +++ b/packaging/hudi-flink-bundle/pom.xml @@ -84,7 +84,6 @@ <include>org.apache.hudi:hudi-sync-common</include> <include>org.apache.hudi:hudi-hadoop-mr</include> <include>org.apache.hudi:hudi-timeline-service</include> - <include>org.apache.hudi:hudi-aws</include> <!-- Kryo --> <include>com.esotericsoftware:kryo-shaded</include>