This is an automated email from the ASF dual-hosted git repository. garyli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new 43a0776 [HUDI-1586] [Common Core] [Flink Integration] Reduce the coupling of hadoop. (#2540) 43a0776 is described below commit 43a0776c7c88a5f7beac6c8853db7e341810635a Author: ZhangChaoMing <72908278+zhangchaom...@users.noreply.github.com> AuthorDate: Sun Feb 21 11:54:04 2021 +0800 [HUDI-1586] [Common Core] [Flink Integration] Reduce the coupling of hadoop. (#2540) Co-authored-by: zhangchaoming <zhangchaom...@360.com> --- hudi-common/pom.xml | 2 + .../java/org/apache/hudi/common/fs/FSUtils.java | 7 +--- hudi-flink/pom.xml | 43 ++-------------------- 3 files changed, 8 insertions(+), 44 deletions(-) diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml index 2981155..41588ca 100644 --- a/hudi-common/pom.xml +++ b/hudi-common/pom.xml @@ -144,6 +144,7 @@ <artifactId>*</artifactId> </exclusion> </exclusions> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> @@ -154,6 +155,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java index a2b3889..341e3f5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java @@ -80,9 +80,6 @@ public class FSUtils { private static final PathFilter ALLOW_ALL_FILTER = file -> true; public static Configuration prepareHadoopConf(Configuration conf) { - conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); - conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); - // look for all properties, prefixed to be picked up for (Entry<String, String> prop : System.getenv().entrySet()) { if (prop.getKey().startsWith(HOODIE_ENV_PROPS_PREFIX)) { @@ -607,8 +604,8 @@ public class FSUtils { * Helper to filter out paths under metadata folder when running fs.globStatus. * @param fs File System * @param globPath Glob Path - * @return - * @throws IOException + * @return the file status list of globPath exclude the meta folder + * @throws IOException when having trouble listing the path */ public static List<FileStatus> getGlobStatusExcludingMetaFolder(FileSystem fs, Path globPath) throws IOException { FileStatus[] statuses = fs.globStatus(globPath); diff --git a/hudi-flink/pom.xml b/hudi-flink/pom.xml index 2a0f395..81a7849 100644 --- a/hudi-flink/pom.xml +++ b/hudi-flink/pom.xml @@ -153,39 +153,11 @@ <scope>provided</scope> </dependency> - <!-- Hadoop --> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> - </dependency> + <!-- Parquet --> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> + <groupId>org.apache.parquet</groupId> + <artifactId>parquet-avro</artifactId> + <scope>test</scope> </dependency> <!-- Avro --> @@ -197,13 +169,6 @@ <scope>compile</scope> </dependency> - <!-- Parquet --> - <dependency> - <groupId>org.apache.parquet</groupId> - <artifactId>parquet-avro</artifactId> - <scope>compile</scope> - </dependency> - <!-- Hadoop --> <dependency> <groupId>org.apache.hadoop</groupId>