Repository: carbondata Updated Branches: refs/heads/master e6d03d112 -> c471386fa
[CARBONDATA-2364][HotFix] Remove useless code in dataloading Remove useless and performance related code in dataloading This closes #2193 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c471386f Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c471386f Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c471386f Branch: refs/heads/master Commit: c471386fa13ce9b36a34a2e9378835fc972a5a63 Parents: e6d03d1 Author: xuchuanyin <xuchuan...@hust.edu.cn> Authored: Fri Apr 20 09:21:03 2018 +0800 Committer: Jacky Li <jacky.li...@qq.com> Committed: Fri Apr 20 23:11:15 2018 +0800 ---------------------------------------------------------------------- ...arallelReadMergeSorterWithColumnRangeImpl.java | 1 - .../steps/DataWriterProcessorStepImpl.java | 18 ------------------ 2 files changed, 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/c471386f/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java index 7c37b88..3a378eb 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java @@ -227,7 +227,6 @@ public class UnsafeParallelReadMergeSorterWithColumnRangeImpl extends AbstractMe try { while (iterator.hasNext()) { CarbonRowBatch batch = iterator.next(); - int i = 0; while (batch.hasNext()) { CarbonRow row = batch.next(); if (row != null) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/c471386f/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java index 0467b11..b09fb7d 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java @@ -107,24 +107,6 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep { int i = 0; // do this concurrently for (Iterator<CarbonRowBatch> iterator : iterators) { - String[] storeLocation = getStoreLocation(tableIdentifier); - - CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel - .createCarbonFactDataHandlerModel(configuration, storeLocation, i, 0); - CarbonFactHandler dataHandler = null; - boolean rowsNotExist = true; - while (iterator.hasNext()) { - if (rowsNotExist) { - rowsNotExist = false; - dataHandler = CarbonFactHandlerFactory - .createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR); - dataHandler.initialise(); - } - processBatch(iterator.next(), dataHandler); - } - if (!rowsNotExist) { - finish(dataHandler); - } rangeExecutorServiceSubmitList.add( rangeExecutorService.submit(new WriterForwarder(iterator, tableIdentifier, i))); i++;
