This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 765712a [CARBONDATA-3481] Multi-thread pruning fails when datamaps count is just near numOfThreadsForPruning 765712a is described below commit 765712aad649625fc3c8815cff82996b32d51422 Author: ajantha-bhat <ajanthab...@gmail.com> AuthorDate: Thu Jul 25 18:50:19 2019 +0530 [CARBONDATA-3481] Multi-thread pruning fails when datamaps count is just near numOfThreadsForPruning Cause : When the datamaps count is just near numOfThreadsForPruning, As code is checking '>= ', last thread may not get the datamaps for prune. Hence array out of index exception is thrown in this scenario. There is no issues with higher number of datamaps. Solution: In this scenario launch threads based on the distribution value, not on the hardcoded value This closes #3336 --- .../org/apache/carbondata/core/datamap/TableDataMap.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java index 33fc3b1..ecdd586 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java @@ -207,9 +207,6 @@ public final class TableDataMap extends OperationEventListener { */ int numOfThreadsForPruning = CarbonProperties.getNumOfThreadsForPruning(); - LOG.info( - "Number of threads selected for multi-thread block pruning is " + numOfThreadsForPruning - + ". total files: " + totalFiles + ". total segments: " + segments.size()); int filesPerEachThread = totalFiles / numOfThreadsForPruning; int prev; int filesCount = 0; @@ -254,6 +251,15 @@ public final class TableDataMap extends OperationEventListener { // this should not happen throw new RuntimeException(" not all the files processed "); } + if (datamapListForEachThread.size() < numOfThreadsForPruning) { + // If the total datamaps fitted in lesser number of threads than numOfThreadsForPruning. + // Launch only that many threads where datamaps are fitted while grouping. + LOG.info("Datamaps is distributed in " + datamapListForEachThread.size() + " threads"); + numOfThreadsForPruning = datamapListForEachThread.size(); + } + LOG.info( + "Number of threads selected for multi-thread block pruning is " + numOfThreadsForPruning + + ". total files: " + totalFiles + ". total segments: " + segments.size()); List<Future<Void>> results = new ArrayList<>(numOfThreadsForPruning); final Map<Segment, List<ExtendedBlocklet>> prunedBlockletMap = new ConcurrentHashMap<>(segments.size());