Repository: carbondata Updated Branches: refs/heads/branch-1.3 1137c285f -> 6e435de5e
[CARBONDATA-2137] Delete query performance improved Following is the configuration used : SPARK_EXECUTOR_MEMORY : 200G SPARK_DRIVER_MEMORY : 20G SPARK_EXECUTOR_CORES : 32 SPARK_EXECUTOR_INSTANCEs : 3 Earlier it was taking 20 minute now it is taking approx 5 minute This closes #1937 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6e435de5 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6e435de5 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6e435de5 Branch: refs/heads/branch-1.3 Commit: 6e435de5e04ace63fe5b105e2f180ef0932d80d3 Parents: 1137c28 Author: rahulforallp <rahul.ku...@knoldus.in> Authored: Tue Feb 6 18:41:35 2018 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Fri Feb 9 16:48:31 2018 +0530 ---------------------------------------------------------------------- .../carbondata/core/locks/HdfsFileLock.java | 2 +- .../carbondata/core/locks/LocalFileLock.java | 2 +- .../core/mutate/DeleteDeltaBlockDetails.java | 27 +++++++++++++++----- 3 files changed, 22 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e435de5/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java index 1a46770..cc98b03 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java @@ -85,7 +85,7 @@ public class HdfsFileLock extends AbstractCarbonLock { return true; } catch (IOException e) { - LOGGER.error(e, e.getMessage()); + LOGGER.info(e.getMessage()); return false; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e435de5/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index c3dfb57..4fee4c4 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -118,7 +118,7 @@ public class LocalFileLock extends AbstractCarbonLock { return false; } } catch (IOException e) { - LOGGER.error(e, e.getMessage()); + LOGGER.info(e.getMessage()); return false; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e435de5/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java index 4275cca..7affe12 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java @@ -19,7 +19,10 @@ package org.apache.carbondata.core.mutate; import java.io.Serializable; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; @@ -31,7 +34,7 @@ public class DeleteDeltaBlockDetails implements Serializable { private static final long serialVersionUID = 1206104914918495724L; - private List<DeleteDeltaBlockletDetails> blockletDetails; + private Map<String, DeleteDeltaBlockletDetails> blockletDetailsMap; private String blockName; /** @@ -42,7 +45,7 @@ public class DeleteDeltaBlockDetails implements Serializable { public DeleteDeltaBlockDetails(String blockName) { this.blockName = blockName; - blockletDetails = new ArrayList<DeleteDeltaBlockletDetails>(); + blockletDetailsMap = new TreeMap<>(); } public String getBlockName() { @@ -68,15 +71,25 @@ public class DeleteDeltaBlockDetails implements Serializable { } public List<DeleteDeltaBlockletDetails> getBlockletDetails() { - return blockletDetails; + + List<DeleteDeltaBlockletDetails> deleteDeltaBlockletDetailsList = new ArrayList<>(); + Iterator<Map.Entry<String, DeleteDeltaBlockletDetails>> iterator = + blockletDetailsMap.entrySet().iterator(); + while (iterator.hasNext()) { + deleteDeltaBlockletDetailsList.add(iterator.next().getValue()); + } + return deleteDeltaBlockletDetailsList; } public boolean addBlockletDetails(DeleteDeltaBlockletDetails blocklet) { - int index = blockletDetails.indexOf(blocklet); - if (blockletDetails.isEmpty() || index == -1) { - return blockletDetails.add(blocklet); + DeleteDeltaBlockletDetails deleteDeltaBlockletDetails = + blockletDetailsMap.get(blocklet.getBlockletKey()); + if (null == deleteDeltaBlockletDetails) { + blockletDetailsMap.put(blocklet.getBlockletKey(), blocklet); + return true; } else { - return blockletDetails.get(index).addDeletedRows(blocklet.getDeletedRows()); + deleteDeltaBlockletDetails.addDeletedRows(blocklet.getDeletedRows()); + return true; } }