http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java index 7f735c2..fd92fc1 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.block.SegmentProperties; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -35,11 +35,10 @@ import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.scan.expression.Expression; -import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; import org.apache.carbondata.core.util.ByteUtil; import org.apache.carbondata.core.util.CarbonUtil; @@ -72,7 +71,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } ifDefaultValueMatchesFilter(); - if (isDimensionPresentInCurrentBlock[0] == true) { + if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } @@ -119,11 +118,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut boolean isScanRequired = false; if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) { if (isMeasurePresentInCurrentBlock[0]) { - minValue = blockMinValue[measureBlocksIndex[0] + lastDimensionColOrdinal]; + minValue = blockMinValue[measureChunkIndex[0] + lastDimensionColOrdinal]; isScanRequired = isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType()); } else { - minValue = blockMinValue[dimensionBlocksIndex[0]]; + minValue = blockMinValue[dimensionChunkIndex[0]]; isScanRequired = isScanRequired(minValue, filterRangeValues); } } else { @@ -169,67 +168,69 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut } @Override - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws FilterUnsupportedException, IOException { + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws IOException { // select all rows if dimension does not exists in the current block if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) { - int numberOfRows = blockChunkHolder.getDataBlock().nodeSize(); + int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows(); return FilterUtil - .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(), + .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(), numberOfRows, true); } if (isDimensionPresentInCurrentBlock[0]) { - int blockIndex = - segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]); - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + int chunkIndex = + segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]); + if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readDimensionChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } DimensionRawColumnChunk rawColumnChunk = - blockChunkHolder.getDimensionRawDataChunk()[blockIndex]; + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMinValues() != null) { if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) { - BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i), + BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } } else { - BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i), + BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } } return bitSetGroup; - } else if (isMeasurePresentInCurrentBlock[0]) { - int blockIndex = - segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]); - if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) { - blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex); + } else { + int chunkIndex = + segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]); + if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readMeasureChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } - MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex]; + MeasureRawColumnChunk rawColumnChunk = + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]; BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMinValues() != null) { if (isScanRequired(rawColumnChunk.getMinValues()[i], this.msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType())) { BitSet bitSet = - getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i), + getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } } else { BitSet bitSet = - getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i), + getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]); bitSetGroup.setBitSet(bitSet, i); } } return bitSetGroup; } - return null; } private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage, @@ -262,7 +263,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut return bitSet; } - private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage, int numerOfRows) { byte[] defaultValue = null; if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { @@ -271,7 +272,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut dimColEvaluatorInfoList.get(0).getDimension().getDataType()); int key = directDictionaryGenerator.generateDirectSurrogateKey(null); CarbonDimension currentBlockDimension = - segmentProperties.getDimensions().get(dimensionBlocksIndex[0]); + segmentProperties.getDimensions().get(dimensionChunkIndex[0]); if (currentBlockDimension.isSortColumn()) { defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension, this.segmentProperties.getSortColumnsGenerator()); @@ -282,14 +283,14 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY; } BitSet bitSet = null; - if (dimensionColumnDataChunk.isExplicitSorted()) { - bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows, + if (dimensionColumnPage.isExplicitSorted()) { + bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows, defaultValue); } else { - bitSet = setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows, defaultValue); + bitSet = setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows, defaultValue); } - if (dimensionColumnDataChunk.isNoDicitionaryColumn()) { - FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet, + if (dimensionColumnPage.isNoDicitionaryColumn()) { + FilterUtil.removeNullValues(dimensionColumnPage, bitSet, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY); } return bitSet; @@ -301,12 +302,12 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut * column is not supported by default so column index mapping will be present for * accesing the members from the block. * - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param numerOfRows * @return BitSet. */ private BitSet setFilterdIndexToBitSetWithColumnIndex( - DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows, + DimensionColumnPage dimensionColumnPage, int numerOfRows, byte[] defaultValue) { BitSet bitSet = new BitSet(numerOfRows); int start = 0; @@ -318,7 +319,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut //find the number of default values to skip the null value in case of direct dictionary if (null != defaultValue) { start = CarbonUtil - .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, defaultValue, true); if (start < 0) { skip = -(start + 1); @@ -336,7 +337,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut for (int i = 0; i < filterValues.length; i++) { start = CarbonUtil - .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1, + .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[i], false); if (start >= 0) { // Logic will handle the case where the range filter member is not present in block @@ -344,7 +345,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut // set inorder to apply filters. this is Lesser than filter so the range will be taken // from the prev element which is Lesser than filter member. start = - CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[i]); + CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[i]); } if (start < 0) { start = -(start + 1); @@ -355,14 +356,14 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut // will be pointing to the next consecutive position. So compare it again and point to the // previous value returned from getFirstIndexUsingBinarySearch. if (ByteUtil.compare(filterValues[i], - dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start))) + dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start))) < 0) { start = start - 1; } } last = start; for (int j = start; j >= skip; j--) { - bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j)); + bitSet.set(dimensionColumnPage.getInvertedIndex(j)); last--; } startIndex = last; @@ -379,11 +380,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut * be called if the column is sorted default so column index * mapping will be present for accesing the members from the block. * - * @param dimensionColumnDataChunk + * @param dimensionColumnPage * @param numerOfRows * @return BitSet. */ - private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk, + private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage, int numerOfRows, byte[] defaultValue) { BitSet bitSet = new BitSet(numerOfRows); byte[][] filterValues = this.filterRangeValues; @@ -396,7 +397,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut //find the number of default values to skip the null value in case of direct dictionary if (null != defaultValue) { start = CarbonUtil - .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, + .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, defaultValue, true); if (start < 0) { skip = -(start + 1); @@ -413,11 +414,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut } for (int k = 0; k < filterValues.length; k++) { start = CarbonUtil - .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, + .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1, filterValues[k], false); if (start >= 0) { start = - CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[k]); + CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[k]); } if (start < 0) { start = -(start + 1); @@ -428,7 +429,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut // When negative value of start is returned from getFirstIndexUsingBinarySearch the Start // will be pointing to the next consecutive position. So compare it again and point to the // previous value returned from getFirstIndexUsingBinarySearch. - if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) < 0) { + if (ByteUtil.compare(filterValues[k], dimensionColumnPage.getChunkData(start)) < 0) { start = start - 1; } } @@ -445,7 +446,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut } else { for (int k = 0; k < filterValues.length; k++) { for (int i = 0; i < numerOfRows; i++) { - if (ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValues[k]) < 0) { + if (ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValues[k]) < 0) { bitSet.set(i); } } @@ -454,21 +455,24 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut return bitSet; } - @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { + @Override + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { if (isDimensionPresentInCurrentBlock[0]) { if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) { - super.readBlocks(blockChunkHolder); + super.readColumnChunks(rawBlockletColumnChunks); } - int blockIndex = dimensionBlocksIndex[0]; - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + int chunkIndex = dimensionChunkIndex[0]; + if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readDimensionChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } } else if (isMeasurePresentInCurrentBlock[0]) { - int blockIndex = measureBlocksIndex[0]; - if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) { - blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex); + int chunkIndex = measureChunkIndex[0]; + if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { + rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = + rawBlockletColumnChunks.getDataBlock().readMeasureChunk( + rawBlockletColumnChunks.getFileReader(), chunkIndex); } } }
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java index 4b3738a..2cbb234 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java @@ -22,7 +22,7 @@ import java.util.BitSet; import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.filter.intf.RowIntf; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; public class TrueFilterExecutor implements FilterExecuter { @@ -33,19 +33,20 @@ public class TrueFilterExecutor implements FilterExecuter { * @return * @throws FilterUnsupportedException */ - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws FilterUnsupportedException, IOException { - int numberOfPages = blockChunkHolder.getDataBlock().numberOfPages(); + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException { + int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages(); BitSetGroup group = new BitSetGroup(numberOfPages); for (int i = 0; i < numberOfPages; i++) { BitSet set = new BitSet(); - set.flip(0, blockChunkHolder.getDataBlock().getPageRowCount(i)); + set.flip(0, rawBlockletColumnChunks.getDataBlock().getPageRowCount(i)); group.setBitSet(set, i); } return group; } - @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) { + @Override + public boolean applyFilter(RowIntf value, int dimOrdinalMax) { return true; } @@ -66,9 +67,9 @@ public class TrueFilterExecutor implements FilterExecuter { /** * It just reads necessary block for filter executor, it does not uncompress the data. * - * @param blockChunkHolder + * @param rawBlockletColumnChunks */ - public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) { // do nothing } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java index 516447f..bc3b39c 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java @@ -21,27 +21,29 @@ import java.util.BitSet; import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.filter.intf.RowIntf; -import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; import org.apache.carbondata.core.util.BitSetGroup; public class ValueBasedFilterExecuterImpl implements FilterExecuter { @Override - public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine) - throws FilterUnsupportedException, IOException { - + public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, + boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException { return new BitSetGroup(0); } - @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) + @Override + public boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException, IOException { throw new FilterUnsupportedException("Unsupported ValueBasedFilterExecuterImpl on row"); } - @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { + @Override + public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { return new BitSet(1); } - @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { + @Override + public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java deleted file mode 100644 index b3ed34e..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.scan.filter.intf; - -import org.apache.carbondata.core.scan.expression.Expression; - -public class FilterOptimizerBasic implements FilterOptimizer { - @Override public Expression optimizeFilter() { - return null; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java index e5cb0ad..bbd33ec 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java @@ -19,14 +19,13 @@ package org.apache.carbondata.core.scan.filter.optimizer; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.RangeExpressionEvaluator; -import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer; import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerImpl; public class RangeFilterOptmizer extends FilterOptimizerImpl { RangeExpressionEvaluator rangeExpEvaluator; - public RangeFilterOptmizer(FilterOptimizer filterOptimizer, Expression filterExpression) { + public RangeFilterOptmizer(Expression filterExpression) { super(filterExpression); this.rangeExpEvaluator = new RangeExpressionEvaluator(filterExpression); http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java index bb66d18..996965d 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java @@ -46,17 +46,13 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { protected boolean isIncludeFilter; private DimColumnResolvedFilterInfo dimColResolvedFilterInfo; private MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo; - private AbsoluteTableIdentifier tableIdentifier; - private boolean isMeasure; public ConditionalFilterResolverImpl(Expression exp, boolean isExpressionResolve, - boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier, boolean isMeasure) { + boolean isIncludeFilter, boolean isMeasure) { this.exp = exp; this.isExpressionResolve = isExpressionResolve; this.isIncludeFilter = isIncludeFilter; - this.tableIdentifier = tableIdentifier; - this.isMeasure = isMeasure; - if (isMeasure == false) { + if (!isMeasure) { this.dimColResolvedFilterInfo = new DimColumnResolvedFilterInfo(); } else { this.msrColResolvedFilterInfo = new MeasureColumnResolvedFilterInfo(); @@ -230,10 +226,6 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { return msrColResolvedFilterInfo; } - public AbsoluteTableIdentifier getTableIdentifier() { - return tableIdentifier; - } - /** * method will calculates the start key based on the filter surrogates */ @@ -245,12 +237,6 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties, setOfStartKeyByteArray); } -// else { -// FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(), -// segmentProperties, startKey, startKeyList); -// FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties, -// setOfStartKeyByteArray); -// } } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java index 33bafb1..c846c02 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java @@ -40,7 +40,7 @@ public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl { public RowLevelFilterResolverImpl(Expression exp, boolean isExpressionResolve, boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) { - super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false); + super(exp, isExpressionResolve, isIncludeFilter, false); dimColEvaluatorInfoList = new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>( http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java index 0e7666c..e85e9d1 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java @@ -61,7 +61,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm public RowLevelRangeFilterResolverImpl(Expression exp, boolean isExpressionResolve, boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) { - super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false); + super(exp, isExpressionResolve, isIncludeFilter, false); dimColEvaluatorInfoList = new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>( http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java index f8586ed..4165424 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java @@ -77,10 +77,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo } } - public Map<CarbonMeasure, List<ColumnFilterInfo>> getMeasureResolvedFilterInstance() { - return measureResolvedFilter; - } - public ColumnFilterInfo getFilterValues() { return resolvedFilterValueObj; } @@ -109,10 +105,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo this.type = dataType; } - public boolean isMeasureExistsInCurrentSlice() { - return true; - } - public CarbonColumn getCarbonColumn() { return carbonColumn; } @@ -125,10 +117,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo return carbonMeasure; } - public boolean isMeasureExistsInCurrentSilce() { - return isMeasureExistsInCurrentSilce; - } - public void setMeasureExistsInCurrentSilce(boolean measureExistsInCurrentSilce) { isMeasureExistsInCurrentSilce = measureExistsInCurrentSilce; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java index d403ef4..104f69e 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java @@ -28,9 +28,9 @@ import org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolver public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl { public TrueConditionalResolverImpl(Expression exp, boolean isExpressionResolve, - boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) { + boolean isIncludeFilter) { - super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false); + super(exp, isExpressionResolve, isIncludeFilter, false); } @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier, @@ -47,16 +47,4 @@ public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl { return FilterExecuterType.TRUE; } - /** - * Method will the read filter expression corresponding to the resolver. - * This method is required in row level executer inorder to evaluate the filter - * expression against spark, as mentioned above row level is a special type - * filter resolver. - * - * @return Expression - */ - public Expression getFilterExpresion() { - return exp; - } - } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java index c90816a..b1e5a7d 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java @@ -19,7 +19,6 @@ package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor; import java.io.IOException; import java.util.Collections; -import java.util.List; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; @@ -47,7 +46,6 @@ public class RangeDictionaryColumnVisitor extends DictionaryColumnVisitor if (visitableObj instanceof DimColumnResolvedFilterInfo) { DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj; ColumnFilterInfo resolvedFilterObject = null; - List<String> evaluateResultListFinal; resolvedFilterObject = FilterUtil .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(), metadata.getColumnExpression(), metadata.isIncludeFilter(), http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java b/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java deleted file mode 100644 index 5a5b042..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.scan.model; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.scan.expression.Expression; - -/** - * This class contains all the logical information about the query like dimensions,measures, - * sort order, topN etc.. - */ -public class CarbonQueryPlan implements Serializable { - /** - * - */ - private static final long serialVersionUID = -9036044826928017164L; - - /** - * Database name - */ - private String databaseName; - - /** - * Table name - */ - private String tableName; - - /** - * List of dimensions. - * Ex : select employee_name,department_name,sum(salary) from employee, then employee_name - * and department_name are dimensions - * If there is no dimensions asked in query then it would be remained as empty. - */ - private List<QueryDimension> dimensions = - new ArrayList<QueryDimension>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - - /** - * List of measures. - * Ex : select employee_name,department_name,sum(salary) from employee, then sum(salary) - * would be measure. - * If there is no dimensions asked in query then it would be remained as empty. - */ - private List<QueryMeasure> measures = - new ArrayList<QueryMeasure>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - - /** - * expression - */ - private Expression expression; - - /** - * queryId - */ - private String queryId; - - /** - * If it is raw detail query, no need to aggregate in backend. And it returns with dictionary data - * with out decoding. - */ - private boolean rawDetailQuery; - - /** - * Constructor created with database name and table name. - * - * @param databaseName - * @param tableName - */ - public CarbonQueryPlan(String databaseName, String tableName) { - this.tableName = tableName; - this.databaseName = databaseName; - } - - /** - * @return the dimensions - */ - public List<QueryDimension> getDimensions() { - return dimensions; - } - - public void addDimension(QueryDimension dimension) { - this.dimensions.add(dimension); - } - - /** - * @return the measures - */ - public List<QueryMeasure> getMeasures() { - return measures; - } - - public void addMeasure(QueryMeasure measure) { - this.measures.add(measure); - } - - public Expression getFilterExpression() { - return expression; - } - - public void setFilterExpression(Expression expression) { - this.expression = expression; - } - - /** - * @return the databaseName - */ - public String getDatabaseName() { - return databaseName; - } - - /** - * @return the tableName - */ - public String getTableName() { - return tableName; - } - - public String getQueryId() { - return queryId; - } - - public void setQueryId(String queryId) { - this.queryId = queryId; - } - - public boolean isRawDetailQuery() { - return rawDetailQuery; - } - - public void setRawDetailQuery(boolean rawDetailQuery) { - this.rawDetailQuery = rawDetailQuery; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java new file mode 100644 index 0000000..0768221 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.scan.model; + +/** + * Contains information for a column for projection + */ +public class ProjectionColumn { + + /** + * name of the column + */ + protected String columnName; + + /** + * query order in which result of the query will be send + */ + private int projectionOrdinal; + + ProjectionColumn(String columnName) { + this.columnName = columnName; + } + + /** + * @return the columnName + */ + public String getColumnName() { + return columnName; + } + + /** + * @return the projectionOrdinal + */ + public int getOrdinal() { + return projectionOrdinal; + } + + /** + * @param projectionOrdinal the projectionOrdinal to set + */ + public void setOrdinal(int projectionOrdinal) { + this.projectionOrdinal = projectionOrdinal; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java new file mode 100644 index 0000000..806de85 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.scan.model; + +import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; + +/** + * dimension column which holds the information about the column + * this is done to avoid heavy object serialization + */ +public class ProjectionDimension extends ProjectionColumn { + + /** + * actual dimension column + */ + private CarbonDimension dimension; + + public ProjectionDimension(CarbonDimension dimension) { + super(dimension.getColName()); + this.dimension = dimension; + } + + /** + * @return the dimension + */ + public CarbonDimension getDimension() { + return dimension; + } + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java new file mode 100644 index 0000000..93d9c82 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.scan.model; + +import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; + +/** + * query plan measure, this class will holds the information + * about measure present in the query, this is done to avoid the serialization + * of the heavy object + */ +public class ProjectionMeasure extends ProjectionColumn { + + /** + * actual carbon measure object + */ + private CarbonMeasure measure; + + public ProjectionMeasure(CarbonMeasure measure) { + super(measure.getColName()); + this.measure = measure; + } + + /** + * @return the measure + */ + public CarbonMeasure getMeasure() { + return measure; + } + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java deleted file mode 100644 index 1336b87..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.scan.model; - -import java.io.Serializable; - -/** - * query column which will have information about column - */ -public class QueryColumn implements Serializable { - - /** - * serialVersionUID - */ - private static final long serialVersionUID = -4222306600480181084L; - - /** - * name of the column - */ - protected String columnName; - - /** - * query order in which result of the query will be send - */ - private int queryOrder; - - public QueryColumn(String columnName) { - this.columnName = columnName; - } - - /** - * @return the columnName - */ - public String getColumnName() { - return columnName; - } - - /** - * @return the queryOrder - */ - public int getQueryOrder() { - return queryOrder; - } - - /** - * @param queryOrder the queryOrder to set - */ - public void setQueryOrder(int queryOrder) { - this.queryOrder = queryOrder; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java deleted file mode 100644 index e8f07a7..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.scan.model; - -import java.io.Serializable; - -import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; - -/** - * query plan dimension which will holds the information about the query plan dimension - * this is done to avoid heavy object serialization - */ -public class QueryDimension extends QueryColumn implements Serializable { - - /** - * serialVersionUID - */ - private static final long serialVersionUID = -8492704093776645651L; - /** - * actual dimension column - */ - private transient CarbonDimension dimension; - - public QueryDimension(String columnName) { - super(columnName); - } - - /** - * @return the dimension - */ - public CarbonDimension getDimension() { - return dimension; - } - - /** - * @param dimension the dimension to set - */ - public void setDimension(CarbonDimension dimension) { - this.dimension = dimension; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java deleted file mode 100644 index 73a8ad5..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.scan.model; - -import java.io.Serializable; - -import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; - -/** - * query plan measure, this class will holds the information - * about measure present in the query, this is done to avoid the serialization - * of the heavy object - */ -public class QueryMeasure extends QueryColumn implements Serializable { - - /** - * serialVersionUID - */ - private static final long serialVersionUID = 1035512411375495414L; - - /** - * actual carbon measure object - */ - private transient CarbonMeasure measure; - - public QueryMeasure(String columnName) { - super(columnName); - } - - /** - * @return the measure - */ - public CarbonMeasure getMeasure() { - return measure; - } - - /** - * @param measure the measure to set - */ - public void setMeasure(CarbonMeasure measure) { - this.measure = measure; - } - -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java index 655ed5e..4e3665f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java @@ -17,7 +17,6 @@ package org.apache.carbondata.core.scan.model; -import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -46,25 +45,17 @@ import org.apache.carbondata.core.util.DataTypeConverter; * about the query, This will be sent from driver to executor ' * This will be refereed to executing the query. */ -public class QueryModel implements Serializable { +public class QueryModel { /** - * serialization version - */ - private static final long serialVersionUID = -4674677234007089052L; - /** * this will hold the information about the dictionary dimension * which to */ - public transient Map<String, Dictionary> columnToDictionaryMapping; - /** - * list of dimension selected for in query - */ - private List<QueryDimension> queryDimension; + private transient Map<String, Dictionary> columnToDictionaryMapping; /** - * list of measure selected in query + * list of projection columns in query */ - private List<QueryMeasure> queryMeasures; + private QueryProjection projection; /** * query id */ @@ -79,10 +70,6 @@ public class QueryModel implements Serializable { */ private List<TableBlockInfo> tableBlockInfos; /** - * absolute table identifier - */ - private AbsoluteTableIdentifier absoluteTableIdentifier; - /** * To handle most of the computation in query engines like spark and hive, carbon should give * raw detailed records to it. */ @@ -117,43 +104,15 @@ public class QueryModel implements Serializable { */ private boolean readPageByPage; - public QueryModel() { + private QueryModel(CarbonTable carbonTable) { tableBlockInfos = new ArrayList<TableBlockInfo>(); - queryDimension = new ArrayList<QueryDimension>(); - queryMeasures = new ArrayList<QueryMeasure>(); invalidSegmentIds = new ArrayList<>(); + this.table = carbonTable; + this.queryId = String.valueOf(System.nanoTime()); } - public static QueryModel createModel(AbsoluteTableIdentifier absoluteTableIdentifier, - CarbonQueryPlan queryPlan, CarbonTable carbonTable, DataTypeConverter converter) { - QueryModel queryModel = new QueryModel(); - queryModel.setAbsoluteTableIdentifier(absoluteTableIdentifier); - - fillQueryModel(queryPlan, carbonTable, queryModel); - - queryModel.setForcedDetailRawQuery(queryPlan.isRawDetailQuery()); - queryModel.setQueryId(queryPlan.getQueryId()); - queryModel.setConverter(converter); - return queryModel; - } - - private static void fillQueryModel(CarbonQueryPlan queryPlan, CarbonTable carbonTable, - QueryModel queryModel) { - queryModel.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier()); - queryModel.setQueryDimension(queryPlan.getDimensions()); - queryModel.setQueryMeasures(queryPlan.getMeasures()); - if (null != queryPlan.getFilterExpression()) { - boolean[] isFilterDimensions = new boolean[carbonTable.getDimensionOrdinalMax()]; - boolean[] isFilterMeasures = - new boolean[carbonTable.getNumberOfMeasures(carbonTable.getTableName())]; - processFilterExpression(carbonTable, queryPlan.getFilterExpression(), isFilterDimensions, - isFilterMeasures); - queryModel.setIsFilterDimensions(isFilterDimensions); - queryModel.setIsFilterMeasures(isFilterMeasures); - } - //TODO need to remove this code, and executor will load the table - // from file metadata - queryModel.setTable(carbonTable); + public static QueryModel newInstance(CarbonTable carbonTable) { + return new QueryModel(carbonTable); } public static void processFilterExpression(CarbonTable carbonTable, Expression filterExpression, @@ -239,42 +198,26 @@ public class QueryModel implements Serializable { */ public CarbonColumn[] getProjectionColumns() { CarbonColumn[] carbonColumns = - new CarbonColumn[getQueryDimension().size() + getQueryMeasures().size()]; - for (QueryDimension dimension : getQueryDimension()) { - carbonColumns[dimension.getQueryOrder()] = dimension.getDimension(); + new CarbonColumn[getProjectionDimensions().size() + getProjectionMeasures().size()]; + for (ProjectionDimension dimension : getProjectionDimensions()) { + carbonColumns[dimension.getOrdinal()] = dimension.getDimension(); } - for (QueryMeasure msr : getQueryMeasures()) { - carbonColumns[msr.getQueryOrder()] = msr.getMeasure(); + for (ProjectionMeasure msr : getProjectionMeasures()) { + carbonColumns[msr.getOrdinal()] = msr.getMeasure(); } return carbonColumns; } - /** - * @return the queryDimension - */ - public List<QueryDimension> getQueryDimension() { - return queryDimension; + public void setProjection(QueryProjection projection) { + this.projection = projection; } - /** - * @param queryDimension the queryDimension to set - */ - public void setQueryDimension(List<QueryDimension> queryDimension) { - this.queryDimension = queryDimension; + public List<ProjectionDimension> getProjectionDimensions() { + return projection.getDimensions(); } - /** - * @return the queryMeasures - */ - public List<QueryMeasure> getQueryMeasures() { - return queryMeasures; - } - - /** - * @param queryMeasures the queryMeasures to set - */ - public void setQueryMeasures(List<QueryMeasure> queryMeasures) { - this.queryMeasures = queryMeasures; + public List<ProjectionMeasure> getProjectionMeasures() { + return projection.getMeasures(); } /** @@ -285,13 +228,6 @@ public class QueryModel implements Serializable { } /** - * @param queryId the queryId to set - */ - public void setQueryId(String queryId) { - this.queryId = queryId; - } - - /** * @return the tableBlockInfos */ public List<TableBlockInfo> getTableBlockInfos() { @@ -320,14 +256,7 @@ public class QueryModel implements Serializable { * @return the absoluteTableIdentifier */ public AbsoluteTableIdentifier getAbsoluteTableIdentifier() { - return absoluteTableIdentifier; - } - - /** - * @param absoluteTableIdentifier the absoluteTableIdentifier to set - */ - public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) { - this.absoluteTableIdentifier = absoluteTableIdentifier; + return table.getAbsoluteTableIdentifier(); } /** @@ -337,13 +266,6 @@ public class QueryModel implements Serializable { return table; } - /** - * @param table the table to set - */ - public void setTable(CarbonTable table) { - this.table = table; - } - public boolean isForcedDetailRawQuery() { return forcedDetailRawQuery; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java new file mode 100644 index 0000000..d4496e0 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.scan.model; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; + +/** + * Contains projection columns in the query + */ +public class QueryProjection { + + /** + * List of dimensions. + * Ex : select employee_name,department_name,sum(salary) from employee, then employee_name + * and department_name are dimensions + * If there is no dimensions asked in query then it would be remained as empty. + */ + private List<ProjectionDimension> dimensions = + new ArrayList<ProjectionDimension>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + + /** + * List of measures. + * Ex : select employee_name,department_name,sum(salary) from employee, then sum(salary) + * would be measure. + * If there is no dimensions asked in query then it would be remained as empty. + */ + private List<ProjectionMeasure> measures = + new ArrayList<ProjectionMeasure>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + + /** + * Constructor created with database name and table name. + * + */ + public QueryProjection() { + } + + /** + * @return the dimensions + */ + public List<ProjectionDimension> getDimensions() { + return dimensions; + } + + public void addDimension(CarbonDimension dimension, int queryOrdinal) { + ProjectionDimension queryDimension = new ProjectionDimension(dimension); + queryDimension.setOrdinal(queryOrdinal); + this.dimensions.add(queryDimension); + } + + /** + * @return the measures + */ + public List<ProjectionMeasure> getMeasures() { + return measures; + } + + public void addMeasure(CarbonMeasure measure, int queryOrdinal) { + ProjectionMeasure queryMeasure = new ProjectionMeasure(measure); + queryMeasure.setOrdinal(queryOrdinal); + this.measures.add(queryMeasure); + } + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java deleted file mode 100644 index eb5e3f9..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.scan.processor; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.carbondata.common.CarbonIterator; -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.datastore.DataRefNode; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.scan.collector.ResultCollectorFactory; -import org.apache.carbondata.core.scan.collector.ScannedResultCollector; -import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; -import org.apache.carbondata.core.scan.result.AbstractScannedResult; -import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch; -import org.apache.carbondata.core.scan.scanner.BlockletScanner; -import org.apache.carbondata.core.scan.scanner.impl.FilterScanner; -import org.apache.carbondata.core.scan.scanner.impl.NonFilterScanner; -import org.apache.carbondata.core.stats.QueryStatisticsModel; -import org.apache.carbondata.core.util.TaskMetricsMap; - -/** - * This abstract class provides a skeletal implementation of the - * Block iterator. - */ -public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Object[]>> { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(AbstractDataBlockIterator.class.getName()); - - /** - * iterator which will be used to iterate over data blocks - */ - protected CarbonIterator<DataRefNode> dataBlockIterator; - - /** - * result collector which will be used to aggregate the scanned result - */ - protected ScannedResultCollector scannerResultAggregator; - - /** - * processor which will be used to process the block processing can be - * filter processing or non filter processing - */ - protected BlockletScanner blockletScanner; - - /** - * batch size of result - */ - protected int batchSize; - - protected ExecutorService executorService; - - private Future<AbstractScannedResult> future; - - private Future<BlocksChunkHolder> futureIo; - - protected AbstractScannedResult scannedResult; - - private BlockExecutionInfo blockExecutionInfo; - - private FileHolder fileReader; - - private AtomicBoolean nextBlock; - - private AtomicBoolean nextRead; - - public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader, - int batchSize, QueryStatisticsModel queryStatisticsModel, ExecutorService executorService) { - this.blockExecutionInfo = blockExecutionInfo; - this.fileReader = fileReader; - dataBlockIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(), - blockExecutionInfo.getNumberOfBlockToScan()); - if (blockExecutionInfo.getFilterExecuterTree() != null) { - blockletScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel); - } else { - blockletScanner = new NonFilterScanner(blockExecutionInfo, queryStatisticsModel); - } - this.scannerResultAggregator = - ResultCollectorFactory.getScannedResultCollector(blockExecutionInfo); - this.batchSize = batchSize; - this.executorService = executorService; - this.nextBlock = new AtomicBoolean(false); - this.nextRead = new AtomicBoolean(false); - } - - public boolean hasNext() { - if (scannedResult != null && scannedResult.hasNext()) { - return true; - } else { - if (null != scannedResult) { - scannedResult.freeMemory(); - } - return dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get(); - } - } - - protected boolean updateScanner() { - try { - if (scannedResult != null && scannedResult.hasNext()) { - return true; - } else { - scannedResult = getNextScannedResult(); - while (scannedResult != null) { - if (scannedResult.hasNext()) { - return true; - } - scannedResult = getNextScannedResult(); - } - nextBlock.set(false); - nextRead.set(false); - return false; - } - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - private AbstractScannedResult getNextScannedResult() throws Exception { - AbstractScannedResult result = null; - if (blockExecutionInfo.isPrefetchBlocklet()) { - if (dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get()) { - if (future == null) { - future = execute(); - } - result = future.get(); - nextBlock.set(false); - if (dataBlockIterator.hasNext() || nextRead.get()) { - nextBlock.set(true); - future = execute(); - } - } - } else { - if (dataBlockIterator.hasNext()) { - BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolder(); - if (blocksChunkHolder != null) { - result = blockletScanner.scanBlocklet(blocksChunkHolder); - } - } - } - return result; - } - - private BlocksChunkHolder getBlocksChunkHolder() throws IOException { - BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolderInternal(); - while (blocksChunkHolder == null && dataBlockIterator.hasNext()) { - blocksChunkHolder = getBlocksChunkHolderInternal(); - } - return blocksChunkHolder; - } - - private BlocksChunkHolder getBlocksChunkHolderInternal() throws IOException { - BlocksChunkHolder blocksChunkHolder = - new BlocksChunkHolder(blockExecutionInfo.getTotalNumberDimensionBlock(), - blockExecutionInfo.getTotalNumberOfMeasureBlock(), fileReader); - blocksChunkHolder.setDataBlock(dataBlockIterator.next()); - if (blocksChunkHolder.getDataBlock().getColumnsMaxValue() == null) { - return blocksChunkHolder; - } - if (blockletScanner.isScanRequired(blocksChunkHolder)) { - return blocksChunkHolder; - } - return null; - } - - private Future<AbstractScannedResult> execute() { - return executorService.submit(new Callable<AbstractScannedResult>() { - @Override public AbstractScannedResult call() throws Exception { - if (futureIo == null) { - futureIo = executeRead(); - } - BlocksChunkHolder blocksChunkHolder = futureIo.get(); - futureIo = null; - nextRead.set(false); - if (blocksChunkHolder != null) { - if (dataBlockIterator.hasNext()) { - nextRead.set(true); - futureIo = executeRead(); - } - return blockletScanner.scanBlocklet(blocksChunkHolder); - } - return null; - } - }); - } - - private Future<BlocksChunkHolder> executeRead() { - return executorService.submit(new Callable<BlocksChunkHolder>() { - @Override public BlocksChunkHolder call() throws Exception { - try { - TaskMetricsMap.getInstance().registerThreadCallback(); - if (dataBlockIterator.hasNext()) { - BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolder(); - if (blocksChunkHolder != null) { - blockletScanner.readBlocklet(blocksChunkHolder); - return blocksChunkHolder; - } - } - return null; - } finally { - // update read bytes metrics for this thread - TaskMetricsMap.getInstance().updateReadBytes(Thread.currentThread().getId()); - } - } - }); - } - - public abstract void processNextBatch(CarbonColumnarBatch columnarBatch); - - /** - * Close the resources - */ - public void close() { - // free the current scanned result - if (null != scannedResult && !scannedResult.hasNext()) { - scannedResult.freeMemory(); - } - // free any pre-fetched memory if present - if (null != future) { - try { - AbstractScannedResult abstractScannedResult = future.get(); - if (abstractScannedResult != null) { - abstractScannedResult.freeMemory(); - } - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java index 3ea4e1d..f0d81ff 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java @@ -23,7 +23,7 @@ import org.apache.carbondata.core.datastore.DataRefNode; /** * Below class will be used to iterate over data block */ -public class BlockletIterator extends CarbonIterator<DataRefNode> { +class BlockletIterator extends CarbonIterator<DataRefNode> { /** * data store block */ @@ -47,17 +47,18 @@ public class BlockletIterator extends CarbonIterator<DataRefNode> { * Constructor * * @param datablock first data block - * @param totalNumberOfBlocksToScan total number of blocks to be scanned + * @param totalNumberOfBlockletToScan total number of blocklets to be scanned */ - public BlockletIterator(DataRefNode datablock, long totalNumberOfBlocksToScan) { + BlockletIterator(DataRefNode datablock, long totalNumberOfBlockletToScan) { this.datablock = datablock; - this.totalNumberOfBlocksToScan = totalNumberOfBlocksToScan; + this.totalNumberOfBlocksToScan = totalNumberOfBlockletToScan; } /** * is all the blocks assigned to this iterator has been processed */ - @Override public boolean hasNext() { + @Override + public boolean hasNext() { return hasNext; } @@ -67,7 +68,8 @@ public class BlockletIterator extends CarbonIterator<DataRefNode> { * @return next data block * */ - @Override public DataRefNode next() { + @Override + public DataRefNode next() { // get the current blocks DataRefNode datablockTemp = datablock; // store the next data block http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java deleted file mode 100644 index 60090d0..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.scan.processor; - -import org.apache.carbondata.core.datastore.DataRefNode; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; -import org.apache.carbondata.core.util.BitSetGroup; - -/** - * Block chunk holder which will hold the dimension and - * measure chunk - */ -public class BlocksChunkHolder { - - /** - * dimension column data chunk - */ - private DimensionRawColumnChunk[] dimensionRawDataChunk; - - /** - * measure column data chunk - */ - private MeasureRawColumnChunk[] measureRawDataChunk; - - /** - * file reader which will use to read the block from file - */ - private FileHolder fileReader; - - /** - * data block - */ - private DataRefNode dataBlock; - - private BitSetGroup bitSetGroup; - - public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock) { - dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock]; - measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock]; - } - - public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock, - FileHolder fileReader) { - dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock]; - measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock]; - this.fileReader = fileReader; - } - - /** - * @return the dimensionRawDataChunk - */ - public DimensionRawColumnChunk[] getDimensionRawDataChunk() { - return dimensionRawDataChunk; - } - - /** - * @param dimensionRawDataChunk the dimensionRawDataChunk to set - */ - public void setDimensionRawDataChunk(DimensionRawColumnChunk[] dimensionRawDataChunk) { - this.dimensionRawDataChunk = dimensionRawDataChunk; - } - - /** - * @return the measureRawDataChunk - */ - public MeasureRawColumnChunk[] getMeasureRawDataChunk() { - return measureRawDataChunk; - } - - /** - * @param measureRawDataChunk the measureRawDataChunk to set - */ - public void setMeasureRawDataChunk(MeasureRawColumnChunk[] measureRawDataChunk) { - this.measureRawDataChunk = measureRawDataChunk; - } - - /** - * @return the fileReader - */ - public FileHolder getFileReader() { - return fileReader; - } - - /** - * @param fileReader the fileReader to set - */ - public void setFileReader(FileHolder fileReader) { - this.fileReader = fileReader; - } - - /** - * @return the dataBlock - */ - public DataRefNode getDataBlock() { - return dataBlock; - } - - /** - * @param dataBlock the dataBlock to set - */ - public void setDataBlock(DataRefNode dataBlock) { - this.dataBlock = dataBlock; - } - - /*** - * To reset the measure chunk and dimension chunk - * array - */ - public void reset() { - for (int i = 0; i < measureRawDataChunk.length; i++) { - this.measureRawDataChunk[i] = null; - } - for (int i = 0; i < dimensionRawDataChunk.length; i++) { - this.dimensionRawDataChunk[i] = null; - } - } - - public BitSetGroup getBitSetGroup() { - return bitSetGroup; - } - - public void setBitSetGroup(BitSetGroup bitSetGroup) { - this.bitSetGroup = bitSetGroup; - } -}