This is an automated email from the ASF dual-hosted git repository.

vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 2577014  1. Minor changes to fix compaction 2. Adding 2 compaction 
policies
2577014 is described below

commit 2577014617220abe3def503e931f811d238c1a10
Author: Nishith Agarwal <[email protected]>
AuthorDate: Wed Apr 3 12:25:35 2019 -0700

    1. Minor changes to fix compaction 2. Adding 2 compaction policies
---
 .../uber/hoodie/config/HoodieHBaseIndexConfig.java |  6 ++
 .../io/compact/HoodieRealtimeTableCompactor.java   |  4 +
 .../BoundedPartitionAwareCompactionStrategy.java   | 71 ++++++++++++++++
 .../strategy/DayBasedCompactionStrategy.java       |  4 +-
 .../UnBoundedPartitionAwareCompactionStrategy.java | 63 ++++++++++++++
 .../io/strategy/TestHoodieCompactionStrategy.java  | 97 +++++++++++++++++++++-
 .../hoodie/common/model/HoodieCommitMetadata.java  |  8 +-
 7 files changed, 246 insertions(+), 7 deletions(-)

diff --git 
a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieHBaseIndexConfig.java
 
b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieHBaseIndexConfig.java
index 48e8031..3c6f1cf 100644
--- 
a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieHBaseIndexConfig.java
+++ 
b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieHBaseIndexConfig.java
@@ -29,6 +29,7 @@ public class HoodieHBaseIndexConfig extends 
DefaultHoodieConfig {
   public static final String HBASE_ZKPORT_PROP = "hoodie.index.hbase.zkport";
   public static final String HBASE_TABLENAME_PROP = "hoodie.index.hbase.table";
   public static final String HBASE_GET_BATCH_SIZE_PROP = 
"hoodie.index.hbase.get.batch.size";
+  public static final String HBASE_ZK_ZNODEPARENT = 
"hoodie.index.hbase.zknode.path";
   /**
    * Note that if HBASE_PUT_BATCH_SIZE_AUTO_COMPUTE_PROP is set to true, this 
batch size will not
    * be honored for HBase Puts
@@ -133,6 +134,11 @@ public class HoodieHBaseIndexConfig extends 
DefaultHoodieConfig {
       return this;
     }
 
+    public Builder hbaseZkZnodeParent(String zkZnodeParent) {
+      props.setProperty(HBASE_ZK_ZNODEPARENT, zkZnodeParent);
+      return this;
+    }
+
     /**
      * <p>
      * Method to set maximum QPS allowed per Region Server. This should be 
same across various
diff --git 
a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java
 
b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java
index 9848631..b4abbca 100644
--- 
a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java
+++ 
b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java
@@ -185,6 +185,10 @@ public class HoodieRealtimeTableCompactor implements 
HoodieCompactor {
     // filter the partition paths if needed to reduce list status
     partitionPaths = 
config.getCompactionStrategy().filterPartitionPaths(config, partitionPaths);
 
+    if (partitionPaths.isEmpty()) {
+      // In case no partitions could be picked, return no compaction plan
+      return null;
+    }
     TableFileSystemView.RealtimeView fileSystemView = 
hoodieTable.getRTFileSystemView();
     log.info("Compaction looking for files to compact in " + partitionPaths + 
" partitions");
     List<HoodieCompactionOperation> operations =
diff --git 
a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
 
b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
new file mode 100644
index 0000000..5a87a4e
--- /dev/null
+++ 
b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
@@ -0,0 +1,71 @@
+/*
+ *  Copyright (c) 2018 Uber Technologies, Inc. ([email protected])
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *           http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.uber.hoodie.io.compact.strategy;
+
+import com.uber.hoodie.avro.model.HoodieCompactionOperation;
+import com.uber.hoodie.avro.model.HoodieCompactionPlan;
+import com.uber.hoodie.config.HoodieWriteConfig;
+import java.text.SimpleDateFormat;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.time.DateUtils;
+
+/**
+ * This strategy ensures that the last N partitions are picked up even if 
there are later partitions created for the
+ * dataset. lastNPartitions is defined as the N partitions  before the 
currentDate.
+ * currentDay = 2018/01/01
+ * The dataset has partitions for 2018/02/02 and 2018/03/03 beyond the 
currentDay
+ * This strategy will pick up the following partitions for compaction :
+ * (2018/01/01, allPartitionsInRange[(2018/01/01 - lastNPartitions) to 
2018/01/01), 2018/02/02, 2018/03/03)
+ */
+public class BoundedPartitionAwareCompactionStrategy extends 
DayBasedCompactionStrategy {
+
+  SimpleDateFormat dateFormat = new SimpleDateFormat(datePartitionFormat);
+
+  @Override
+  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig 
writeConfig,
+      List<HoodieCompactionOperation> operations, List<HoodieCompactionPlan> 
pendingCompactionPlans) {
+    // The earliest partition to compact - current day minus the target 
partitions limit
+    String earliestPartitionPathToCompact = 
dateFormat.format(DateUtils.addDays(new Date(), -1 * writeConfig
+            .getTargetPartitionsPerDayBasedCompaction()));
+    // Filter out all partitions greater than earliestPartitionPathToCompact
+    List<HoodieCompactionOperation> eligibleCompactionOperations = 
operations.stream()
+        
.collect(Collectors.groupingBy(HoodieCompactionOperation::getPartitionPath)).entrySet().stream()
+        .sorted(Map.Entry.comparingByKey(comparator))
+        .filter(e -> comparator.compare(earliestPartitionPathToCompact, 
e.getKey()) >= 0)
+        .flatMap(e -> e.getValue().stream())
+        .collect(Collectors.toList());
+
+    return eligibleCompactionOperations;
+  }
+
+  @Override
+  public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, 
List<String> partitionPaths) {
+    // The earliest partition to compact - current day minus the target 
partitions limit
+    String earliestPartitionPathToCompact = 
dateFormat.format(DateUtils.addDays(new Date(), -1 * writeConfig
+        .getTargetPartitionsPerDayBasedCompaction()));
+    // Get all partitions and sort them
+    List<String> filteredPartitionPaths = 
partitionPaths.stream().map(partition -> partition.replace("/", "-"))
+        .sorted(Comparator.reverseOrder()).map(partitionPath -> 
partitionPath.replace("-", "/"))
+        .filter(e -> comparator.compare(earliestPartitionPathToCompact, e) >= 
0)
+        .collect(Collectors.toList());
+    return filteredPartitionPaths;
+  }
+}
diff --git 
a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/DayBasedCompactionStrategy.java
 
b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/DayBasedCompactionStrategy.java
index ee4d9a5..574ad96 100644
--- 
a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/DayBasedCompactionStrategy.java
+++ 
b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/DayBasedCompactionStrategy.java
@@ -38,9 +38,9 @@ import java.util.stream.Collectors;
 public class DayBasedCompactionStrategy extends CompactionStrategy {
 
   // For now, use SimpleDateFormat as default partition format
-  private static String datePartitionFormat = "yyyy/MM/dd";
+  protected static String datePartitionFormat = "yyyy/MM/dd";
   // Sorts compaction in LastInFirstCompacted order
-  private static Comparator<String> comparator = (String leftPartition,
+  protected static Comparator<String> comparator = (String leftPartition,
       String rightPartition) -> {
     try {
       Date left = new SimpleDateFormat(datePartitionFormat, Locale.ENGLISH)
diff --git 
a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
 
b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
new file mode 100644
index 0000000..c4c81de
--- /dev/null
+++ 
b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
@@ -0,0 +1,63 @@
+/*
+ *  Copyright (c) 2018 Uber Technologies, Inc. ([email protected])
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *           http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.uber.hoodie.io.compact.strategy;
+
+import com.uber.hoodie.avro.model.HoodieCompactionOperation;
+import com.uber.hoodie.avro.model.HoodieCompactionPlan;
+import com.uber.hoodie.config.HoodieWriteConfig;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * UnBoundedPartitionAwareCompactionStrategy is a custom UnBounded Strategy.
+ * This will filter all the partitions that are eligible to be compacted by a
+ * {@link BoundedPartitionAwareCompactionStrategy} and return the result.
+ * This is done so that a long running 
UnBoundedPartitionAwareCompactionStrategy does not step over partitions
+ * in a shorter running BoundedPartitionAwareCompactionStrategy. Essentially, 
this is an inverse of the
+ * partitions chosen in BoundedPartitionAwareCompactionStrategy
+ *
+ * @see CompactionStrategy
+ */
+public class UnBoundedPartitionAwareCompactionStrategy extends 
CompactionStrategy {
+
+  @Override
+  public List<HoodieCompactionOperation> orderAndFilter(HoodieWriteConfig 
config,
+      final List<HoodieCompactionOperation> operations, final 
List<HoodieCompactionPlan> pendingCompactionWorkloads) {
+    BoundedPartitionAwareCompactionStrategy 
boundedPartitionAwareCompactionStrategy
+        = new BoundedPartitionAwareCompactionStrategy();
+    List<HoodieCompactionOperation> operationsToExclude = 
boundedPartitionAwareCompactionStrategy
+        .orderAndFilter(config, operations, pendingCompactionWorkloads);
+    List<HoodieCompactionOperation> allOperations = new 
ArrayList<>(operations);
+    allOperations.removeAll(operationsToExclude);
+    return allOperations;
+  }
+
+  @Override
+  public List<String> filterPartitionPaths(HoodieWriteConfig writeConfig, 
List<String> partitionPaths) {
+    List<String> allPartitionPaths = partitionPaths.stream().map(partition -> 
partition.replace("/", "-"))
+        .sorted(Comparator.reverseOrder()).map(partitionPath -> 
partitionPath.replace("-", "/"))
+        .collect(Collectors.toList());
+    BoundedPartitionAwareCompactionStrategy 
boundedPartitionAwareCompactionStrategy
+        = new BoundedPartitionAwareCompactionStrategy();
+    List<String> partitionsToExclude = 
boundedPartitionAwareCompactionStrategy.filterPartitionPaths(writeConfig,
+        partitionPaths);
+    allPartitionPaths.removeAll(partitionsToExclude);
+    return allPartitionPaths;
+  }
+}
diff --git 
a/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieCompactionStrategy.java
 
b/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieCompactionStrategy.java
index e3d96b2..9dfe5ba 100644
--- 
a/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieCompactionStrategy.java
+++ 
b/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieCompactionStrategy.java
@@ -29,15 +29,20 @@ import com.uber.hoodie.common.util.collection.Pair;
 import com.uber.hoodie.config.HoodieCompactionConfig;
 import com.uber.hoodie.config.HoodieWriteConfig;
 import com.uber.hoodie.io.compact.strategy.BoundedIOCompactionStrategy;
+import 
com.uber.hoodie.io.compact.strategy.BoundedPartitionAwareCompactionStrategy;
 import com.uber.hoodie.io.compact.strategy.DayBasedCompactionStrategy;
 import com.uber.hoodie.io.compact.strategy.LogFileSizeBasedCompactionStrategy;
 import com.uber.hoodie.io.compact.strategy.UnBoundedCompactionStrategy;
+import 
com.uber.hoodie.io.compact.strategy.UnBoundedPartitionAwareCompactionStrategy;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Random;
 import java.util.stream.Collectors;
+import org.apache.commons.lang3.time.DateUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -109,7 +114,7 @@ public class TestHoodieCompactionStrategy {
   }
 
   @Test
-  public void testPartitionAwareCompactionSimple() {
+  public void testDayBasedCompactionSimple() {
     Map<Long, List<Long>> sizesMap = Maps.newHashMap();
     sizesMap.put(120 * MB, Lists.newArrayList(60 * MB, 10 * MB, 80 * MB));
     sizesMap.put(110 * MB, Lists.newArrayList());
@@ -141,6 +146,96 @@ public class TestHoodieCompactionStrategy {
     assertTrue("DayBasedCompactionStrategy should sort partitions in 
descending order", comparision >= 0);
   }
 
+  @Test
+  public void testBoundedPartitionAwareCompactionSimple() {
+    Map<Long, List<Long>> sizesMap = Maps.newHashMap();
+    sizesMap.put(120 * MB, Lists.newArrayList(60 * MB, 10 * MB, 80 * MB));
+    sizesMap.put(110 * MB, Lists.newArrayList());
+    sizesMap.put(100 * MB, Lists.newArrayList(MB));
+    sizesMap.put(70 * MB, Lists.newArrayList(MB));
+    sizesMap.put(80 * MB, Lists.newArrayList(MB));
+    sizesMap.put(90 * MB, Lists.newArrayList(1024 * MB));
+
+    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
+    Date today = new Date();
+    String currentDay = format.format(today);
+    String currentDayMinus1 = format.format(DateUtils.addDays(today, -1));
+    String currentDayMinus2 = format.format(DateUtils.addDays(today, -2));
+    String currentDayMinus3 = format.format(DateUtils.addDays(today, -3));
+    String currentDayPlus1 = format.format(DateUtils.addDays(today, 1));
+    String currentDayPlus5 = format.format(DateUtils.addDays(today, 5));
+
+    Map<Long, String> keyToPartitionMap = new ImmutableMap.Builder()
+        .put(120 * MB, currentDay)
+        .put(110 * MB, currentDayMinus1)
+        .put(100 * MB, currentDayMinus2)
+        .put(80 * MB, currentDayMinus3)
+        .put(90 * MB, currentDayPlus1)
+        .put(70 * MB, currentDayPlus5)
+        .build();
+
+    BoundedPartitionAwareCompactionStrategy strategy = new 
BoundedPartitionAwareCompactionStrategy();
+    HoodieWriteConfig writeConfig = 
HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(
+        HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy)
+            .withTargetPartitionsPerDayBasedCompaction(2)
+            .build()).build();
+    List<HoodieCompactionOperation> operations = 
createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap);
+    List<HoodieCompactionOperation> returned = 
strategy.orderAndFilter(writeConfig, operations, new ArrayList<>());
+
+    assertTrue("BoundedPartitionAwareCompactionStrategy should have resulted 
in fewer compactions",
+        returned.size() < operations.size());
+    Assert.assertEquals("BoundedPartitionAwareCompactionStrategy should have 
resulted in fewer compactions",
+        returned.size(), 5);
+
+    int comparision = 
strategy.getComparator().compare(returned.get(returned.size() - 
1).getPartitionPath(), returned
+        .get(0).getPartitionPath());
+    // Either the partition paths are sorted in descending order or they are 
equal
+    assertTrue("BoundedPartitionAwareCompactionStrategy should sort partitions 
in descending order", comparision >= 0);
+  }
+
+  @Test
+  public void testUnboundedPartitionAwareCompactionSimple() {
+    Map<Long, List<Long>> sizesMap = Maps.newHashMap();
+    sizesMap.put(120 * MB, Lists.newArrayList(60 * MB, 10 * MB, 80 * MB));
+    sizesMap.put(110 * MB, Lists.newArrayList());
+    sizesMap.put(100 * MB, Lists.newArrayList(MB));
+    sizesMap.put(80 * MB, Lists.newArrayList(MB));
+    sizesMap.put(70 * MB, Lists.newArrayList(MB));
+    sizesMap.put(90 * MB, Lists.newArrayList(1024 * MB));
+
+    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
+    Date today = new Date();
+    String currentDay = format.format(today);
+    String currentDayMinus1 = format.format(DateUtils.addDays(today, -1));
+    String currentDayMinus2 = format.format(DateUtils.addDays(today, -2));
+    String currentDayMinus3 = format.format(DateUtils.addDays(today, -3));
+    String currentDayPlus1 = format.format(DateUtils.addDays(today, 1));
+    String currentDayPlus5 = format.format(DateUtils.addDays(today, 5));
+
+    Map<Long, String> keyToPartitionMap = new ImmutableMap.Builder()
+        .put(120 * MB, currentDay)
+        .put(110 * MB, currentDayMinus1)
+        .put(100 * MB, currentDayMinus2)
+        .put(80 * MB, currentDayMinus3)
+        .put(90 * MB, currentDayPlus1)
+        .put(70 * MB, currentDayPlus5)
+        .build();
+
+    UnBoundedPartitionAwareCompactionStrategy strategy = new 
UnBoundedPartitionAwareCompactionStrategy();
+    HoodieWriteConfig writeConfig = 
HoodieWriteConfig.newBuilder().withPath("/tmp").withCompactionConfig(
+        HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy)
+            .withTargetPartitionsPerDayBasedCompaction(2)
+            .build()).build();
+    List<HoodieCompactionOperation> operations = 
createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap);
+    List<HoodieCompactionOperation> returned = 
strategy.orderAndFilter(writeConfig, operations, new ArrayList<>());
+
+    assertTrue("UnBoundedPartitionAwareCompactionStrategy should not include 
last " + writeConfig
+            .getTargetPartitionsPerDayBasedCompaction() + " partitions or 
later partitions from today",
+        returned.size() < operations.size());
+    Assert.assertEquals("BoundedPartitionAwareCompactionStrategy should have 
resulted in 1 compaction",
+        returned.size(), 1);
+  }
+
   private List<HoodieCompactionOperation> 
createCompactionOperations(HoodieWriteConfig config,
       Map<Long, List<Long>> sizesMap) {
     Map<Long, String> keyToPartitionMap = sizesMap.entrySet().stream().map(e ->
diff --git 
a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java
 
b/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java
index 0aee2e8..969de46 100644
--- 
a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java
+++ 
b/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java
@@ -136,7 +136,7 @@ public class HoodieCommitMetadata implements Serializable {
     long totalFilesInsert = 0;
     for (List<HoodieWriteStat> stats : partitionToWriteStats.values()) {
       for (HoodieWriteStat stat : stats) {
-        if (stat.getPrevCommit() != null && 
stat.getPrevCommit().equals("null")) {
+        if (stat.getPrevCommit() != null && 
stat.getPrevCommit().equalsIgnoreCase("null")) {
           totalFilesInsert++;
         }
       }
@@ -148,7 +148,7 @@ public class HoodieCommitMetadata implements Serializable {
     long totalFilesUpdated = 0;
     for (List<HoodieWriteStat> stats : partitionToWriteStats.values()) {
       for (HoodieWriteStat stat : stats) {
-        if (stat.getPrevCommit() != null && 
!stat.getPrevCommit().equals("null")) {
+        if (stat.getPrevCommit() != null && 
!stat.getPrevCommit().equalsIgnoreCase("null")) {
           totalFilesUpdated++;
         }
       }
@@ -170,8 +170,8 @@ public class HoodieCommitMetadata implements Serializable {
     long totalInsertRecordsWritten = 0;
     for (List<HoodieWriteStat> stats : partitionToWriteStats.values()) {
       for (HoodieWriteStat stat : stats) {
-        if (stat.getPrevCommit() != null && 
stat.getPrevCommit().equals("null")) {
-          totalInsertRecordsWritten += stat.getNumWrites();
+        if (stat.getPrevCommit() != null && 
stat.getPrevCommit().equalsIgnoreCase("null")) {
+          totalInsertRecordsWritten += stat.getNumInserts();
         }
       }
     }

Reply via email to