[CARBONDATA-2778]Fixed bug when select after delete and cleanup is showing empty records
Problem: In case if delete operation when it is found that the data being deleted is leading to a state where one complete block data is getting deleted. In that case the status if that block is marked for delete and during the next delete operation run the block is deleted along with its carbonIndex file. The problem arises due to deletion of carbonIndex file because for multiple blocks there can be one carbonIndex file as one carbonIndex file represents one task. Solution: Do not delete the carbondata and carbonIndex file. After compaction it will automatically take care of deleting the stale data and stale segments. This closes #2548 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d5870057 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d5870057 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d5870057 Branch: refs/heads/branch-1.4 Commit: d5870057a5ac76c042d24e7048f20bbd3f9b41eb Parents: 5f6116a Author: kunal642 <kunalkapoor...@gmail.com> Authored: Tue Jul 24 16:12:54 2018 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Tue Jul 31 00:11:26 2018 +0530 ---------------------------------------------------------------------- .../core/mutate/CarbonUpdateUtil.java | 16 ------------ .../SegmentUpdateStatusManager.java | 27 -------------------- 2 files changed, 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5870057/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java index 4a8d2e8..7df3937 100644 --- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java @@ -541,22 +541,6 @@ public class CarbonUpdateUtil { compareTimestampsAndDelete(invalidFile, forceDelete, false); } - CarbonFile[] blockRelatedFiles = updateStatusManager - .getAllBlockRelatedFiles(allSegmentFiles, - block.getActualBlockName()); - - // now for each invalid index file need to check the query execution time out - // and then delete. - - for (CarbonFile invalidFile : blockRelatedFiles) { - - if (compareTimestampsAndDelete(invalidFile, forceDelete, false)) { - if (invalidFile.getName().endsWith(CarbonCommonConstants.UPDATE_INDEX_FILE_EXT)) { - updateSegmentFile = true; - } - } - } - } else { invalidDeleteDeltaFiles = updateStatusManager .getDeleteDeltaInvalidFilesList(block, false, http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5870057/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java index 083325d..5d5e8b0 100644 --- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java +++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java @@ -826,31 +826,4 @@ public class SegmentUpdateStatusManager { return files.toArray(new CarbonFile[files.size()]); } - - /** - * - * @param allSegmentFiles - * @return - */ - public CarbonFile[] getAllBlockRelatedFiles(CarbonFile[] allSegmentFiles, - String actualBlockName) { - List<CarbonFile> files = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); - - for (CarbonFile eachFile : allSegmentFiles) { - - // for carbon data. - if (eachFile.getName().equalsIgnoreCase(actualBlockName)) { - files.add(eachFile); - } - - // get carbon index files of the block. - String indexFileName = CarbonTablePath.getCarbonIndexFileName(actualBlockName); - if (eachFile.getName().equalsIgnoreCase(indexFileName)) { - files.add(eachFile); - } - - } - - return files.toArray(new CarbonFile[files.size()]); - } }
