[CARBONDATA-1653] Rename aggType to measureDataType This closes #1444
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4d70a211 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4d70a211 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4d70a211 Branch: refs/heads/pre-aggregate Commit: 4d70a2118fb26936e0b7eee9a0ce2147cb0501c9 Parents: 93a604d Author: Jacky Li <jacky.li...@qq.com> Authored: Sat Oct 28 16:26:38 2017 +0530 Committer: chenliang613 <chenliang...@apache.org> Committed: Sat Oct 28 23:39:26 2017 +0530 ---------------------------------------------------------------------- .../scan/executor/impl/AbstractQueryExecutor.java | 4 ++-- .../core/scan/executor/util/QueryUtil.java | 17 ----------------- .../core/scan/executor/util/QueryUtilTest.java | 9 --------- .../merger/CompactionResultSortProcessor.java | 8 +------- .../sort/sortdata/IntermediateFileMerger.java | 6 +++--- .../sort/sortdata/SortTempFileChunkHolder.java | 12 ++++++------ .../store/CarbonFactDataHandlerModel.java | 6 +++--- 7 files changed, 15 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java index b2c1a2b..56ed575 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java @@ -161,10 +161,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis()); queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic); // calculating the total number of aggeragted columns - int aggTypeCount = queryModel.getQueryMeasures().size(); + int measureCount = queryModel.getQueryMeasures().size(); int currentIndex = 0; - DataType[] dataTypes = new DataType[aggTypeCount]; + DataType[] dataTypes = new DataType[measureCount]; for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) { // adding the data type and aggregation type of all the measure this http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java index 59387b3..6aab8a0 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java @@ -678,23 +678,6 @@ public class QueryUtil { } /** - * below method will be used to get the actual type aggregator - * - * @param aggType - * @return index in aggrgetor - */ - public static int[] getActualTypeIndex(List<String> aggType) { - List<Integer> indexList = new ArrayList<Integer>(); - for (int i = 0; i < aggType.size(); i++) { - if (!CarbonCommonConstants.SUM.equals(aggType.get(i)) && !CarbonCommonConstants.AVERAGE - .equals(aggType.get(i))) { - indexList.add(i); - } - } - return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()])); - } - - /** * Below method will be used to get the key structure for the column group * * @param segmentProperties segment properties http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java index 23504d3..b764569 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java @@ -221,15 +221,6 @@ public class QueryUtilTest extends TestCase { assertArrayEquals(expectedValue, actualValue); } - @Test public void testGetActualTypeIndex() { - List<String> dummyList = new ArrayList<>(); - dummyList.add("test1"); - dummyList.add("test2"); - int[] actualValue = QueryUtil.getActualTypeIndex(dummyList); - int[] expectedValue = { 0, 1 }; - assertArrayEquals(expectedValue, actualValue); - } - @AfterClass public void tearDown() { segmentProperties = null; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java index 0e20ef5..a361f3a 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java @@ -155,7 +155,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor { try { initTempStoreLocation(); initSortDataRows(); - initAggType(); + dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount); processResult(resultIteratorList); // After delete command, if no records are fetched from one split, // below steps are not required to be initialized. @@ -406,10 +406,4 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor { true, false); } - /** - * initialise aggregation type for measures for their storage format - */ - private void initAggType() { - dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount); - } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java index fa50f1c..ebc811c 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java @@ -326,7 +326,7 @@ public class IntermediateFileMerger implements Callable<Void> { return; } try { - DataType[] aggType = mergerParameters.getMeasureDataType(); + DataType[] measureDataType = mergerParameters.getMeasureDataType(); int[] mdkArray = (int[]) row[0]; byte[][] nonDictArray = (byte[][]) row[1]; int mdkIndex = 0; @@ -346,7 +346,7 @@ public class IntermediateFileMerger implements Callable<Void> { for (int counter = 0; counter < mergerParameters.getMeasureColCount(); counter++) { if (null != NonDictionaryUtil.getMeasure(fieldIndex, row)) { stream.write((byte) 1); - DataType dataType = aggType[counter]; + DataType dataType = measureDataType[counter]; if (dataType == DataTypes.BOOLEAN) { stream.writeBoolean((boolean)NonDictionaryUtil.getMeasure(fieldIndex, row)); } else if (dataType == DataTypes.SHORT) { @@ -362,7 +362,7 @@ public class IntermediateFileMerger implements Callable<Void> { stream.writeInt(bigDecimalInBytes.length); stream.write(bigDecimalInBytes); } else { - throw new IllegalArgumentException("unsupported data type:" + aggType[counter]); + throw new IllegalArgumentException("unsupported data type:" + measureDataType[counter]); } } else { stream.write((byte) 0); http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java index afa4cd7..91bc83c 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java @@ -129,7 +129,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold private int noDictionaryCount; - private DataType[] aggType; + private DataType[] measureDataTypes; /** * to store whether dimension is of dictionary type or not @@ -150,11 +150,11 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold * @param measureCount * @param fileBufferSize * @param noDictionaryCount - * @param aggType + * @param measureDataTypes * @param isNoDictionaryDimensionColumn */ public SortTempFileChunkHolder(File tempFile, int dimensionCount, int complexDimensionCount, - int measureCount, int fileBufferSize, int noDictionaryCount, DataType[] aggType, + int measureCount, int fileBufferSize, int noDictionaryCount, DataType[] measureDataTypes, boolean[] isNoDictionaryDimensionColumn, boolean[] isNoDictionarySortColumn, String tableName) { // set temp file @@ -170,7 +170,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold this.fileBufferSize = fileBufferSize; this.executorService = Executors .newFixedThreadPool(1, new CarbonThreadFactory("SafeSortTempChunkHolderPool:" + tableName)); - this.aggType = aggType; + this.measureDataTypes = measureDataTypes; this.isNoDictionaryDimensionColumn = isNoDictionaryDimensionColumn; this.isNoDictionarySortColumn = isNoDictionarySortColumn; @@ -344,7 +344,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold // read measure values for (int i = 0; i < this.measureCount; i++) { if (stream.readByte() == 1) { - DataType dataType = aggType[i]; + DataType dataType = measureDataTypes[i]; if (dataType == DataTypes.BOOLEAN) { measures[index++] = stream.readBoolean(); } else if (dataType == DataTypes.SHORT) { @@ -361,7 +361,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold stream.readFully(buff); measures[index++] = DataTypeUtil.byteToBigDecimal(buff); } else { - throw new IllegalArgumentException("unsupported data type:" + aggType[i]); + throw new IllegalArgumentException("unsupported data type:" + measureDataTypes[i]); } } else { measures[index++] = null; http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java index 2c346b2..b6ac19d 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java @@ -299,12 +299,12 @@ public class CarbonFactDataHandlerModel { Map<Integer, GenericDataType> complexIndexMap = new HashMap<Integer, GenericDataType>(segmentProperties.getComplexDimensions().size()); carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap); - DataType[] aggType = new DataType[segmentProperties.getMeasures().size()]; + DataType[] measureDataTypes = new DataType[segmentProperties.getMeasures().size()]; int i = 0; for (CarbonMeasure msr : segmentProperties.getMeasures()) { - aggType[i++] = msr.getDataType(); + measureDataTypes[i++] = msr.getDataType(); } - carbonFactDataHandlerModel.setMeasureDataType(aggType); + carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes); String carbonDataDirectoryPath = CarbonDataProcessorUtil .checkAndCreateCarbonStoreLocation(loadModel.getStorePath(), loadModel.getDatabaseName(), tableName, loadModel.getPartitionId(), loadModel.getSegmentId());