This is an automated email from the ASF dual-hosted git repository. vinoth pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push: new 66c7fa2 Reword confusing message and reducing the severity level 66c7fa2 is described below commit 66c7fa2322d0dce8ebb29f9b15a711f5f7b72884 Author: Balaji Varadarajan <varad...@uber.com> AuthorDate: Wed Jun 19 14:15:02 2019 -0700 Reword confusing message and reducing the severity level --- hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java index 8e8ded9..11f545a 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java +++ b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java @@ -341,7 +341,10 @@ public abstract class HoodieTable<T extends HoodieRecordPayload> implements Seri .filter(p -> p.endsWith(".parquet")).collect(Collectors.toList()); // Contains list of partially created files. These needs to be cleaned up. invalidDataPaths.removeAll(validDataPaths); - logger.warn("InValid data paths=" + invalidDataPaths); + if (!invalidDataPaths.isEmpty()) { + logger.info("Removing duplicate data files created due to spark retries before committing. Paths=" + + invalidDataPaths); + } Map<String, List<Pair<String, String>>> groupByPartition = invalidDataPaths.stream() .map(dp -> Pair.of(new Path(dp).getParent().toString(), dp))