Repository: carbondata Updated Branches: refs/heads/branch-1.1 d4adc09d4 -> c013d42a1
Fixed binary comparisions and comments. Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f274bf4c Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f274bf4c Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f274bf4c Branch: refs/heads/branch-1.1 Commit: f274bf4c16f4823c4dcb13cf0e5222f693b946b7 Parents: 17db292 Author: ravipesala <ravi.pes...@gmail.com> Authored: Mon Jul 3 13:23:56 2017 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Mon Jul 3 15:18:20 2017 +0530 ---------------------------------------------------------------------- .../core/scan/filter/ColumnFilterInfo.java | 6 +- .../carbondata/core/scan/filter/FilterUtil.java | 48 ++++++++------ .../executer/ExcludeFilterExecuterImpl.java | 7 +- .../executer/IncludeFilterExecuterImpl.java | 57 +++++----------- .../MeasureColumnExecuterFilterInfo.java | 6 +- .../executer/RestructureEvaluatorImpl.java | 18 +++-- .../RowLevelRangeGrtThanFiterExecuterImpl.java | 68 +++++++------------ ...elRangeGrtrThanEquaToFilterExecuterImpl.java | 69 +++++++------------- ...velRangeLessThanEqualFilterExecuterImpl.java | 64 +++++++----------- .../RowLevelRangeLessThanFiterExecuterImpl.java | 63 +++++++----------- .../RowLevelRangeTypeExecuterFacory.java | 25 ++++--- .../resolver/ConditionalFilterResolverImpl.java | 2 +- .../RowLevelRangeFilterResolverImpl.java | 32 +++++---- .../carbondata/core/util/DataTypeUtil.java | 4 +- 14 files changed, 191 insertions(+), 278 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java index 008d908..ce137ee 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java @@ -38,7 +38,7 @@ public class ColumnFilterInfo implements Serializable { */ private List<byte[]> noDictionaryFilterValuesList; - private List<byte[]> measuresFilterValuesList; + private List<Object> measuresFilterValuesList; public List<byte[]> getNoDictionaryFilterValuesList() { return noDictionaryFilterValuesList; @@ -78,11 +78,11 @@ public class ColumnFilterInfo implements Serializable { this.implicitColumnFilterList = implicitColumnFilterList; } - public List<byte[]> getMeasuresFilterValuesList() { + public List<Object> getMeasuresFilterValuesList() { return measuresFilterValuesList; } - public void setMeasuresFilterValuesList(List<byte[]> measuresFilterValuesList) { + public void setMeasuresFilterValuesList(List<Object> measuresFilterValuesList) { this.measuresFilterValuesList = measuresFilterValuesList; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index 9bdf7f2..3bb78db 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -437,33 +437,26 @@ public final class FilterUtil { public static ColumnFilterInfo getMeasureValKeyMemberForFilter( List<String> evaluateResultListFinal, boolean isIncludeFilter, DataType dataType, CarbonMeasure carbonMeasure) throws FilterUnsupportedException { - List<byte[]> filterValuesList = new ArrayList<byte[]>(20); + List<Object> filterValuesList = new ArrayList<>(20); String result = null; try { int length = evaluateResultListFinal.size(); for (int i = 0; i < length; i++) { result = evaluateResultListFinal.get(i); if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) { - filterValuesList.add(new byte[0]); + filterValuesList.add(null); continue; } filterValuesList - .add(DataTypeUtil.getMeasureByteArrayBasedOnDataTypes(result, dataType, carbonMeasure)); + .add(DataTypeUtil.getMeasureValueBasedOnDataType(result, dataType, carbonMeasure)); } } catch (Throwable ex) { throw new FilterUnsupportedException("Unsupported Filter condition: " + result, ex); } - Comparator<byte[]> filterMeasureComaparator = new Comparator<byte[]>() { - - @Override public int compare(byte[] filterMember1, byte[] filterMember2) { - // TODO Auto-generated method stub - return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2); - } - - }; + Comparator filterMeasureComaparator = FilterUtil.getComparatorByDataTypeForMeasure(dataType); Collections.sort(filterValuesList, filterMeasureComaparator); ColumnFilterInfo columnFilterInfo = null; if (filterValuesList.size() > 0) { @@ -717,12 +710,7 @@ public final class FilterUtil { * @return */ public static byte[][] getKeyArray(ColumnFilterInfo columnFilterInfo, - CarbonDimension carbonDimension, CarbonMeasure carbonMeasure, - SegmentProperties segmentProperties) { - if (null != carbonMeasure) { - return columnFilterInfo.getMeasuresFilterValuesList() - .toArray((new byte[columnFilterInfo.getMeasuresFilterValuesList().size()][])); - } + CarbonDimension carbonDimension, SegmentProperties segmentProperties) { if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) { return columnFilterInfo.getNoDictionaryFilterValuesList() .toArray((new byte[columnFilterInfo.getNoDictionaryFilterValuesList().size()][])); @@ -1127,10 +1115,11 @@ public final class FilterUtil { DimColumnExecuterFilterInfo dimColumnExecuterInfo, CarbonMeasure measures, MeasureColumnExecuterFilterInfo msrColumnExecuterInfo) { if (null != measures) { - byte[][] keysBasedOnFilter = getKeyArray(filterValues, null, measures, segmentProperties); + Object[] keysBasedOnFilter = filterValues.getMeasuresFilterValuesList() + .toArray((new Object[filterValues.getMeasuresFilterValuesList().size()])); msrColumnExecuterInfo.setFilterKeys(keysBasedOnFilter); } else { - byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, null, segmentProperties); + byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, segmentProperties); dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter); } } @@ -1603,6 +1592,13 @@ public final class FilterUtil { static class DoubleComparator implements Comparator<Object> { @Override public int compare(Object key1, Object key2) { + if (key1 == null && key2 == null) { + return 0; + } else if (key1 == null) { + return -1; + } else if (key2 == null) { + return 1; + } double key1Double1 = (double)key1; double key1Double2 = (double)key2; if (key1Double1 < key1Double2) { @@ -1617,6 +1613,13 @@ public final class FilterUtil { static class LongComparator implements Comparator<Object> { @Override public int compare(Object key1, Object key2) { + if (key1 == null && key2 == null) { + return 0; + } else if (key1 == null) { + return -1; + } else if (key2 == null) { + return 1; + } long longKey1 = (long) key1; long longKey2 = (long) key2; if (longKey1 < longKey2) { @@ -1631,6 +1634,13 @@ public final class FilterUtil { static class BigDecimalComparator implements Comparator<Object> { @Override public int compare(Object key1, Object key2) { + if (key1 == null && key2 == null) { + return 0; + } else if (key1 == null) { + return -1; + } else if (key2 == null) { + return 1; + } return ((BigDecimal) key1).compareTo((BigDecimal) key2); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java index a716a8b..d073cbf 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java @@ -135,24 +135,23 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter { // the filter values. The one that matches sets it Bitset. BitSet bitSet = new BitSet(numerOfRows); bitSet.flip(0, numerOfRows); - byte[][] filterValues = msrColumnExecutorInfo.getFilterKeys(); + Object[] filterValues = msrColumnExecutorInfo.getFilterKeys(); Comparator comparator = FilterUtil.getComparatorByDataTypeForMeasure(msrType); for (int i = 0; i < filterValues.length; i++) { - if (filterValues[i].length == 0) { + if (filterValues[i] == null) { BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.flip(j); } continue; } - Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType); for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { // Check if filterValue[i] matches with measure Values. Object msrValue = DataTypeUtil .getMeasureObjectBasedOnDataType(measureColumnDataChunk, startIndex, msrColumnEvaluatorInfo.getMeasure()); - if (comparator.compare(msrValue, filter) == 0) { + if (comparator.compare(msrValue, filterValues[i]) == 0) { // This is a match. bitSet.flip(startIndex); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java index 394e561..653a490 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java @@ -17,8 +17,6 @@ package org.apache.carbondata.core.scan.filter.executer; import java.io.IOException; -import java.math.BigDecimal; -import java.nio.ByteBuffer; import java.util.BitSet; import java.util.Comparator; @@ -44,6 +42,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo; protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo; protected SegmentProperties segmentProperties; + protected Comparator comparator; /** * is dimension column data is natural sorted */ @@ -54,7 +53,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { boolean isMeasure) { this.segmentProperties = segmentProperties; - if (isMeasure == false) { + if (!isMeasure) { this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo; dimColumnExecuterInfo = new DimColumnExecuterFilterInfo(); FilterUtil @@ -67,6 +66,8 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { } else { this.msrColumnEvaluatorInfo = msrColumnEvaluatorInfo; msrColumnExecutorInfo = new MeasureColumnExecuterFilterInfo(); + comparator = + FilterUtil.getComparatorByDataTypeForMeasure(getMeasureDataType(msrColumnEvaluatorInfo)); FilterUtil .prepareKeysFromSurrogates(msrColumnEvaluatorInfo.getFilterValues(), segmentProperties, null, null, msrColumnEvaluatorInfo.getMeasure(), msrColumnExecutorInfo); @@ -155,25 +156,24 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { // Get the measure values from the chunk. compare sequentially with the // the filter values. The one that matches sets it Bitset. BitSet bitSet = new BitSet(rowsInPage); - byte[][] filterValues = msrColumnExecutorInfo.getFilterKeys(); + Object[] filterValues = msrColumnExecutorInfo.getFilterKeys(); Comparator comparator = FilterUtil.getComparatorByDataTypeForMeasure(msrType); for (int i = 0; i < filterValues.length; i++) { - if (filterValues[i].length == 0) { + if (filterValues[i] == null) { BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } - Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType); for (int startIndex = 0; startIndex < rowsInPage; startIndex++) { // Check if filterValue[i] matches with measure Values. Object msrValue = DataTypeUtil .getMeasureObjectBasedOnDataType(measureColumnDataChunk, startIndex, msrColumnEvaluatorInfo.getMeasure()); - if (comparator.compare(msrValue, filter) == 0) { + if (comparator.compare(msrValue, filterValues[i]) == 0) { // This is a match. bitSet.set(startIndex); } @@ -269,13 +269,12 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], filterValues); } else if (null != msrColumnExecutorInfo) { - filterValues = msrColumnExecutorInfo.getFilterKeys(); columnIndex = msrColumnEvaluatorInfo.getColumnIndex(); blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping().get(columnIndex) + segmentProperties .getLastDimensionColOrdinal(); - isScanRequired = isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], filterValues, - msrColumnEvaluatorInfo.getType()); + isScanRequired = isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], + msrColumnExecutorInfo.getFilterKeys(), msrColumnEvaluatorInfo.getType()); } if (isScanRequired) { @@ -306,40 +305,14 @@ public class IncludeFilterExecuterImpl implements FilterExecuter { return isScanRequired; } - private boolean isScanRequired(byte[] maxValue, byte[] minValue, byte[][] filterValue, + private boolean isScanRequired(byte[] maxValue, byte[] minValue, Object[] filterValue, DataType dataType) { + Object maxObject = DataTypeUtil.getMeasureObjectFromDataType(maxValue, dataType); + Object minObject = DataTypeUtil.getMeasureObjectFromDataType(minValue, dataType); for (int i = 0; i < filterValue.length; i++) { - if (filterValue[i].length == 0 || maxValue.length == 0 || minValue.length == 0) { - return isScanRequired(maxValue, minValue, filterValue); - } else { - switch (dataType) { - case DOUBLE: - double maxValueDouble = ByteBuffer.wrap(maxValue).getDouble(); - double minValueDouble = ByteBuffer.wrap(minValue).getDouble(); - double filterValueDouble = ByteBuffer.wrap(filterValue[i]).getDouble(); - if (filterValueDouble <= maxValueDouble && filterValueDouble >= minValueDouble) { - return true; - } - break; - case INT: - case SHORT: - case LONG: - long maxValueLong = ByteBuffer.wrap(maxValue).getLong(); - long minValueLong = ByteBuffer.wrap(minValue).getLong(); - long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong(); - if (filterValueLong <= maxValueLong && filterValueLong >= minValueLong) { - return true; - } - break; - case DECIMAL: - BigDecimal maxDecimal = DataTypeUtil.byteToBigDecimal(maxValue); - BigDecimal minDecimal = DataTypeUtil.byteToBigDecimal(minValue); - BigDecimal filterDecimal = DataTypeUtil.byteToBigDecimal(filterValue[i]); - if (filterDecimal.compareTo(maxDecimal) <= 0 - && filterDecimal.compareTo(minDecimal) >= 0) { - return true; - } - } + if (comparator.compare(filterValue[i], maxObject) <= 0 + && comparator.compare(filterValue[i], minObject) >= 0) { + return true; } } return false; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/MeasureColumnExecuterFilterInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/MeasureColumnExecuterFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/MeasureColumnExecuterFilterInfo.java index cc7e837..a19e617 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/MeasureColumnExecuterFilterInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/MeasureColumnExecuterFilterInfo.java @@ -18,13 +18,13 @@ package org.apache.carbondata.core.scan.filter.executer; public class MeasureColumnExecuterFilterInfo { - byte[][] filterKeys; + Object[] filterKeys; - public void setFilterKeys(byte[][] filterKeys) { + public void setFilterKeys(Object[] filterKeys) { this.filterKeys = filterKeys; } - public byte[][] getFilterKeys() { + public Object[] getFilterKeys() { return filterKeys; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java index 8f3d2b1..008bf22 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java @@ -18,6 +18,7 @@ package org.apache.carbondata.core.scan.filter.executer; import java.nio.charset.Charset; +import java.util.Comparator; import java.util.List; import org.apache.carbondata.core.constants.CarbonCommonConstants; @@ -25,9 +26,11 @@ import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.scan.filter.ColumnFilterInfo; +import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo; import org.apache.carbondata.core.util.ByteUtil; +import org.apache.carbondata.core.util.DataTypeUtil; /** * Abstract class for restructure @@ -93,14 +96,17 @@ public abstract class RestructureEvaluatorImpl implements FilterExecuter { boolean isDefaultValuePresentInFilterValues = false; ColumnFilterInfo filterValues = measureColumnResolvedFilterInfo.getFilterValues(); CarbonMeasure measure = measureColumnResolvedFilterInfo.getMeasure(); - byte[] defaultValue = measure.getDefaultValue(); - if (null == defaultValue) { + Comparator comparator = + FilterUtil.getComparatorByDataTypeForMeasure(measure.getDataType()); + Object defaultValue = null; + if (null != measure.getDefaultValue()) { // default value for case where user gives is Null condition - defaultValue = new byte[0]; + defaultValue = DataTypeUtil + .getMeasureObjectFromDataType(measure.getDefaultValue(), measure.getDataType()); } - List<byte[]> measureFilterValuesList = filterValues.getMeasuresFilterValuesList(); - for (byte[] filterValue : measureFilterValuesList) { - int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(defaultValue, filterValue); + List<Object> measureFilterValuesList = filterValues.getMeasuresFilterValuesList(); + for (Object filterValue : measureFilterValuesList) { + int compare = comparator.compare(defaultValue, filterValue); if (compare == 0) { isDefaultValuePresentInFilterValues = true; break; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java index cbe59a1..3f3a223 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java @@ -17,8 +17,6 @@ package org.apache.carbondata.core.scan.filter.executer; import java.io.IOException; -import java.math.BigDecimal; -import java.nio.ByteBuffer; import java.util.BitSet; import java.util.Comparator; import java.util.List; @@ -47,6 +45,8 @@ import org.apache.carbondata.core.util.DataTypeUtil; public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecuterImpl { private byte[][] filterRangeValues; + private Object[] msrFilterRangeValues; + private Comparator comparator; /** * flag to check whether default values is present in the filter value list @@ -57,12 +57,17 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, - SegmentProperties segmentProperties) { + Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; + this.msrFilterRangeValues = msrFilterRangeValues; lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal(); - if (isDimensionPresentInCurrentBlock[0] == true) { + if (isMeasurePresentInCurrentBlock[0]) { + CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); + comparator = FilterUtil.getComparatorByDataTypeForMeasure(measure.getDataType()); + } + if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } @@ -90,9 +95,9 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); byte[] defaultValue = measure.getDefaultValue(); if (null != defaultValue) { - for (int k = 0; k < filterRangeValues.length; k++) { - int maxCompare = - ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue); + for (int k = 0; k < msrFilterRangeValues.length; k++) { + int maxCompare = comparator.compare(msrFilterRangeValues[k], + DataTypeUtil.getMeasureObjectFromDataType(defaultValue, measure.getDataType())); if (maxCompare < 0) { isDefaultValuePresentInFilter = true; break; @@ -110,7 +115,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute if (isMeasurePresentInCurrentBlock[0]) { maxValue = blockMaxValue[measureBlocksIndex[0] + lastDimensionColOrdinal]; isScanRequired = - isScanRequired(maxValue, filterRangeValues, msrColEvalutorInfoList.get(0).getType()); + isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType()); } else { maxValue = blockMaxValue[dimensionBlocksIndex[0]]; isScanRequired = isScanRequired(maxValue, filterRangeValues); @@ -143,35 +148,12 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute return isScanRequired; } - private boolean isScanRequired(byte[] maxValue, byte[][] filterValue, + private boolean isScanRequired(byte[] maxValue, Object[] filterValue, DataType dataType) { + Object value = DataTypeUtil.getMeasureObjectFromDataType(maxValue, dataType); for (int i = 0; i < filterValue.length; i++) { - if (filterValue[i].length == 0 || maxValue.length == 0) { - return isScanRequired(maxValue, filterValue); - } - switch (dataType) { - case DOUBLE: - double maxValueDouble = ByteBuffer.wrap(maxValue).getDouble(); - double filterValueDouble = ByteBuffer.wrap(filterValue[i]).getDouble(); - if (filterValueDouble < maxValueDouble) { - return true; - } - break; - case INT: - case SHORT: - case LONG: - long maxValueLong = ByteBuffer.wrap(maxValue).getLong(); - long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong(); - if (filterValueLong < maxValueLong) { - return true; - } - break; - case DECIMAL: - BigDecimal maxDecimal = DataTypeUtil.byteToBigDecimal(maxValue); - BigDecimal filterDecimal = DataTypeUtil.byteToBigDecimal(filterValue[i]); - if (filterDecimal.compareTo(maxDecimal) < 0) { - return true; - } + if (comparator.compare(filterValue[i], value) < 0) { + return true; } } return false; @@ -229,10 +211,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMaxValues() != null) { - if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, + if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType())) { - int compare = ByteUtil.UnsafeComparer.INSTANCE - .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]); + int compare = comparator.compare(msrFilterRangeValues[0], DataTypeUtil + .getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i], + msrColEvalutorInfoList.get(0).getType())); if (compare < 0) { BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]); bitSet.flip(0, rawColumnChunk.getRowCount()[i]); @@ -259,24 +242,21 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute private BitSet getFilteredIndexesForMeasures(MeasureColumnDataChunk measureColumnDataChunk, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); - byte[][] filterValues = this.filterRangeValues; - DataType msrType = msrColEvalutorInfoList.get(0).getType(); - Comparator comparator = FilterUtil.getComparatorByDataTypeForMeasure(msrType); + Object[] filterValues = this.msrFilterRangeValues; for (int i = 0; i < filterValues.length; i++) { - if (filterValues[i].length == 0) { + if (filterValues[i] == null) { BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } - Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType); for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { Object msrValue = DataTypeUtil .getMeasureObjectBasedOnDataType(measureColumnDataChunk, startIndex, msrColEvalutorInfoList.get(0).getMeasure()); - if (comparator.compare(msrValue, filter) > 0) { + if (comparator.compare(msrValue, filterValues[i]) > 0) { // This is a match. bitSet.set(startIndex); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java index 5c7878d..85d1f06 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java @@ -17,8 +17,6 @@ package org.apache.carbondata.core.scan.filter.executer; import java.io.IOException; -import java.math.BigDecimal; -import java.nio.ByteBuffer; import java.util.BitSet; import java.util.Comparator; import java.util.List; @@ -48,6 +46,8 @@ import org.apache.carbondata.core.util.DataTypeUtil; public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilterExecuterImpl { protected byte[][] filterRangeValues; + private Object[] msrFilterRangeValues; + private Comparator comparator; /** * flag to check whether default values is present in the filter value list @@ -59,13 +59,17 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, - SegmentProperties segmentProperties) { + Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; + this.msrFilterRangeValues = msrFilterRangeValues; lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal(); - - if (isDimensionPresentInCurrentBlock[0] == true) { + if (isMeasurePresentInCurrentBlock[0]) { + CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); + comparator = FilterUtil.getComparatorByDataTypeForMeasure(measure.getDataType()); + } + if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } @@ -93,9 +97,9 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); byte[] defaultValue = measure.getDefaultValue(); if (null != defaultValue) { - for (int k = 0; k < filterRangeValues.length; k++) { - int maxCompare = - ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue); + for (int k = 0; k < msrFilterRangeValues.length; k++) { + int maxCompare = comparator.compare(msrFilterRangeValues[k], + DataTypeUtil.getMeasureObjectFromDataType(defaultValue, measure.getDataType())); if (maxCompare <= 0) { isDefaultValuePresentInFilter = true; break; @@ -113,7 +117,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte if (isMeasurePresentInCurrentBlock[0]) { maxValue = blockMaxValue[measureBlocksIndex[0] + lastDimensionColOrdinal]; isScanRequired = - isScanRequired(maxValue, filterRangeValues, msrColEvalutorInfoList.get(0).getType()); + isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType()); } else { maxValue = blockMaxValue[dimensionBlocksIndex[0]]; isScanRequired = isScanRequired(maxValue, filterRangeValues); @@ -145,35 +149,12 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte return isScanRequired; } - private boolean isScanRequired(byte[] maxValue, byte[][] filterValue, + private boolean isScanRequired(byte[] maxValue, Object[] filterValue, DataType dataType) { + Object value = DataTypeUtil.getMeasureObjectFromDataType(maxValue, dataType); for (int i = 0; i < filterValue.length; i++) { - if (filterValue[i].length == 0 || maxValue.length == 0) { - return isScanRequired(maxValue, filterValue); - } - switch (dataType) { - case DOUBLE: - double maxValueDouble = ByteBuffer.wrap(maxValue).getDouble(); - double filterValueDouble = ByteBuffer.wrap(filterValue[i]).getDouble(); - if (filterValueDouble <= maxValueDouble) { - return true; - } - break; - case INT: - case SHORT: - case LONG: - long maxValueLong = ByteBuffer.wrap(maxValue).getLong(); - long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong(); - if (filterValueLong <= maxValueLong) { - return true; - } - break; - case DECIMAL: - BigDecimal maxDecimal = DataTypeUtil.byteToBigDecimal(maxValue); - BigDecimal filterDecimal = DataTypeUtil.byteToBigDecimal(filterValue[i]); - if (filterDecimal.compareTo(maxDecimal) <= 0) { - return true; - } + if (comparator.compare(filterValue[i], value) <= 0) { + return true; } } return false; @@ -232,10 +213,11 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMaxValues() != null) { - if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, + if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType())) { - int compare = ByteUtil.UnsafeComparer.INSTANCE - .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]); + int compare = comparator.compare(msrFilterRangeValues[0], DataTypeUtil + .getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i], + msrColEvalutorInfoList.get(0).getType())); if (compare <= 0) { BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]); bitSet.flip(0, rawColumnChunk.getRowCount()[i]); @@ -262,24 +244,21 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte private BitSet getFilteredIndexesForMeasures(MeasureColumnDataChunk measureColumnDataChunk, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); - byte[][] filterValues = this.filterRangeValues; - DataType msrType = msrColEvalutorInfoList.get(0).getType(); - Comparator comparator = FilterUtil.getComparatorByDataTypeForMeasure(msrType); + Object[] filterValues = this.msrFilterRangeValues; for (int i = 0; i < filterValues.length; i++) { - if (filterValues[i].length == 0) { + if (filterValues[i] == null) { BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } - Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType); for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { Object msrValue = DataTypeUtil .getMeasureObjectBasedOnDataType(measureColumnDataChunk, startIndex, msrColEvalutorInfoList.get(0).getMeasure()); - if (comparator.compare(msrValue, filter) >= 0) { + if (comparator.compare(msrValue, filterValues[i]) >= 0) { // This is a match. bitSet.set(startIndex); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java index ed14aa1..4c7ee30 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java @@ -17,8 +17,6 @@ package org.apache.carbondata.core.scan.filter.executer; import java.io.IOException; -import java.math.BigDecimal; -import java.nio.ByteBuffer; import java.util.BitSet; import java.util.Comparator; import java.util.List; @@ -49,6 +47,8 @@ import org.apache.carbondata.core.util.DataTypeUtil; public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilterExecuterImpl { protected byte[][] filterRangeValues; + protected Object[] msrFilterRangeValues; + protected Comparator comparator; /** * flag to check whether default values is present in the filter value list @@ -59,13 +59,18 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, - SegmentProperties segmentProperties) { + Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal(); this.filterRangeValues = filterRangeValues; + this.msrFilterRangeValues = msrFilterRangeValues; + if (isMeasurePresentInCurrentBlock[0]) { + CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); + comparator = FilterUtil.getComparatorByDataTypeForMeasure(measure.getDataType()); + } ifDefaultValueMatchesFilter(); - if (isDimensionPresentInCurrentBlock[0] == true) { + if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } @@ -92,9 +97,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); byte[] defaultValue = measure.getDefaultValue(); if (null != defaultValue) { - for (int k = 0; k < filterRangeValues.length; k++) { - int maxCompare = - ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue); + for (int k = 0; k < msrFilterRangeValues.length; k++) { + int maxCompare = comparator.compare(msrFilterRangeValues[k], + DataTypeUtil.getMeasureObjectFromDataType(defaultValue, measure.getDataType())); if (maxCompare >= 0) { isDefaultValuePresentInFilter = true; break; @@ -112,7 +117,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter if (isMeasurePresentInCurrentBlock[0]) { minValue = blockMinValue[measureBlocksIndex[0] + lastDimensionColOrdinal]; isScanRequired = - isScanRequired(minValue, filterRangeValues, msrColEvalutorInfoList.get(0).getType()); + isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType()); } else { minValue = blockMinValue[dimensionBlocksIndex[0]]; isScanRequired = isScanRequired(minValue, filterRangeValues); @@ -143,35 +148,13 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter return isScanRequired; } - private boolean isScanRequired(byte[] minValue, byte[][] filterValue, + private boolean isScanRequired(byte[] minValue, Object[] filterValue, DataType dataType) { + Object value = + DataTypeUtil.getMeasureObjectFromDataType(minValue, dataType); for (int i = 0; i < filterValue.length; i++) { - if (filterValue[i].length == 0 || minValue.length == 0) { - return isScanRequired(minValue, filterValue); - } - switch (dataType) { - case DOUBLE: - double minValueDouble = ByteBuffer.wrap(minValue).getDouble(); - double filterValueDouble = ByteBuffer.wrap(filterValue[i]).getDouble(); - if (filterValueDouble >= minValueDouble) { - return true; - } - break; - case INT: - case SHORT: - case LONG: - long minValueLong = ByteBuffer.wrap(minValue).getLong(); - long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong(); - if (filterValueLong >= minValueLong) { - return true; - } - break; - case DECIMAL: - BigDecimal minDecimal = DataTypeUtil.byteToBigDecimal(minValue); - BigDecimal filterDecimal = DataTypeUtil.byteToBigDecimal(filterValue[i]); - if (filterDecimal.compareTo(minDecimal) >= 0) { - return true; - } + if (comparator.compare(filterValue[i], value) >= 0) { + return true; } } return false; @@ -221,7 +204,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMinValues() != null) { - if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, + if (isScanRequired(rawColumnChunk.getMinValues()[i], this.msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType())) { BitSet bitSet = getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i), @@ -243,24 +226,21 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter private BitSet getFilteredIndexesForMeasures(MeasureColumnDataChunk measureColumnDataChunk, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); - byte[][] filterValues = this.filterRangeValues; - DataType msrType = msrColEvalutorInfoList.get(0).getType(); - Comparator comparator = FilterUtil.getComparatorByDataTypeForMeasure(msrType); + Object[] filterValues = this.msrFilterRangeValues; for (int i = 0; i < filterValues.length; i++) { - if (filterValues[i].length == 0) { + if (filterValues[i] == null) { BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } - Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType); for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { Object msrValue = DataTypeUtil .getMeasureObjectBasedOnDataType(measureColumnDataChunk, startIndex, msrColEvalutorInfoList.get(0).getMeasure()); - if (comparator.compare(msrValue, filter) <= 0) { + if (comparator.compare(msrValue, filterValues[i]) <= 0) { // This is a match. bitSet.set(startIndex); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java index a600118..a73b3be 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java @@ -17,8 +17,6 @@ package org.apache.carbondata.core.scan.filter.executer; import java.io.IOException; -import java.math.BigDecimal; -import java.nio.ByteBuffer; import java.util.BitSet; import java.util.Comparator; import java.util.List; @@ -49,6 +47,8 @@ import org.apache.carbondata.core.util.DataTypeUtil; public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecuterImpl { private byte[][] filterRangeValues; + private Object[] msrFilterRangeValues; + private Comparator comparator; /** * flag to check whether default values is present in the filter value list @@ -59,11 +59,16 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, - SegmentProperties segmentProperties) { + Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; + this.msrFilterRangeValues = msrFilterRangeValues; lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal(); + if (isMeasurePresentInCurrentBlock[0]) { + CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); + comparator = FilterUtil.getComparatorByDataTypeForMeasure(measure.getDataType()); + } ifDefaultValueMatchesFilter(); if (isDimensionPresentInCurrentBlock[0] == true) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() @@ -92,9 +97,12 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); byte[] defaultValue = measure.getDefaultValue(); if (null != defaultValue) { - for (int k = 0; k < filterRangeValues.length; k++) { + + for (int k = 0; k < msrFilterRangeValues.length; k++) { + Object convertedValue = + DataTypeUtil.getMeasureObjectFromDataType(defaultValue, measure.getDataType()); int maxCompare = - ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue); + comparator.compare(msrFilterRangeValues[k], convertedValue); if (maxCompare > 0) { isDefaultValuePresentInFilter = true; break; @@ -112,7 +120,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut if (isMeasurePresentInCurrentBlock[0]) { minValue = blockMinValue[measureBlocksIndex[0] + lastDimensionColOrdinal]; isScanRequired = - isScanRequired(minValue, filterRangeValues, msrColEvalutorInfoList.get(0).getType()); + isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType()); } else { minValue = blockMinValue[dimensionBlocksIndex[0]]; isScanRequired = isScanRequired(minValue, filterRangeValues); @@ -144,35 +152,12 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut return isScanRequired; } - private boolean isScanRequired(byte[] minValue, byte[][] filterValue, + private boolean isScanRequired(byte[] minValue, Object[] filterValue, DataType dataType) { + Object value = DataTypeUtil.getMeasureObjectFromDataType(minValue, dataType); for (int i = 0; i < filterValue.length; i++) { - if (filterValue[i].length == 0 || minValue.length == 0) { - return isScanRequired(minValue, filterValue); - } - switch (dataType) { - case DOUBLE: - double minValueDouble = ByteBuffer.wrap(minValue).getDouble(); - double filterValueDouble = ByteBuffer.wrap(filterValue[i]).getDouble(); - if (filterValueDouble > minValueDouble) { - return true; - } - break; - case INT: - case SHORT: - case LONG: - long minValueLong = ByteBuffer.wrap(minValue).getLong(); - long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong(); - if (filterValueLong > minValueLong) { - return true; - } - break; - case DECIMAL: - BigDecimal minDecimal = DataTypeUtil.byteToBigDecimal(minValue); - BigDecimal filterDecimal = DataTypeUtil.byteToBigDecimal(filterValue[i]); - if (filterDecimal.compareTo(minDecimal) > 0) { - return true; - } + if (comparator.compare(filterValue[i], value) > 0) { + return true; } } return false; @@ -222,7 +207,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount()); for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) { if (rawColumnChunk.getMinValues() != null) { - if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, + if (isScanRequired(rawColumnChunk.getMinValues()[i], this.msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType())) { BitSet bitSet = getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i), @@ -244,24 +229,20 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut private BitSet getFilteredIndexesForMeasures(MeasureColumnDataChunk measureColumnDataChunk, int numerOfRows) { BitSet bitSet = new BitSet(numerOfRows); - byte[][] filterValues = this.filterRangeValues; - DataType msrType = msrColEvalutorInfoList.get(0).getType(); - Comparator comparator = FilterUtil.getComparatorByDataTypeForMeasure(msrType); + Object[] filterValues = this.msrFilterRangeValues; for (int i = 0; i < filterValues.length; i++) { - if (filterValues[i].length == 0) { + if (filterValues[i] == null) { BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet(); for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) { bitSet.set(j); } continue; } - Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType); for (int startIndex = 0; startIndex < numerOfRows; startIndex++) { Object msrValue = DataTypeUtil .getMeasureObjectBasedOnDataType(measureColumnDataChunk, startIndex, msrColEvalutorInfoList.get(0).getMeasure()); - - if (comparator.compare(msrValue, filter) < 0) { + if (comparator.compare(msrValue, filterValues[i]) < 0) { // This is a match. bitSet.set(startIndex); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java index 53cbc66..2bedf01 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java @@ -49,37 +49,42 @@ public class RowLevelRangeTypeExecuterFacory { filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getFilterRangeValues(segmentProperties), segmentProperties); + .getFilterRangeValues(segmentProperties), + ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) + .getMeasureFilterRangeValues(), segmentProperties); case ROWLEVEL_LESSTHAN_EQUALTO: return new RowLevelRangeLessThanEqualFilterExecuterImpl( ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getDimColEvaluatorInfoList(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getMsrColEvalutorInfoList(), - filterExpressionResolverTree.getFilterExpression(), + .getMsrColEvalutorInfoList(), filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getFilterRangeValues(segmentProperties), segmentProperties); + .getFilterRangeValues(segmentProperties), + ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) + .getMeasureFilterRangeValues(), segmentProperties); case ROWLEVEL_GREATERTHAN_EQUALTO: return new RowLevelRangeGrtrThanEquaToFilterExecuterImpl( ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getDimColEvaluatorInfoList(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getMsrColEvalutorInfoList(), - filterExpressionResolverTree.getFilterExpression(), + .getMsrColEvalutorInfoList(), filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getFilterRangeValues(segmentProperties), segmentProperties); + .getFilterRangeValues(segmentProperties), + ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) + .getMeasureFilterRangeValues(), segmentProperties); case ROWLEVEL_GREATERTHAN: return new RowLevelRangeGrtThanFiterExecuterImpl( ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getDimColEvaluatorInfoList(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getMsrColEvalutorInfoList(), - filterExpressionResolverTree.getFilterExpression(), + .getMsrColEvalutorInfoList(), filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) - .getFilterRangeValues(segmentProperties), segmentProperties); + .getFilterRangeValues(segmentProperties), + ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) + .getMeasureFilterRangeValues(), segmentProperties); default: // Scenario wont come logic must break return null; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java index 705cafb..3be13b4 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java @@ -309,7 +309,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf { } else if (null != dimColResolvedFilterInfo.getFilterValues() && dimColResolvedFilterInfo .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { return FilterUtil.getKeyArray(this.dimColResolvedFilterInfo.getFilterValues(), - this.dimColResolvedFilterInfo.getDimension(), null, segmentProperties); + this.dimColResolvedFilterInfo.getDimension(), segmentProperties); } return null; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java index b8c7e09..d7e1b30 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java @@ -87,16 +87,20 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm .getDimensionFromCurrentBlock(this.dimColEvaluatorInfoList.get(0).getDimension()); if (null != dimensionFromCurrentBlock) { return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(), - dimensionFromCurrentBlock, null, segmentProperties); + dimensionFromCurrentBlock, segmentProperties); } - } else if (msrColEvalutorInfoList.size() > 0 && null != msrColEvalutorInfoList.get(0) + } + return null; + } + + public Object[] getMeasureFilterRangeValues() { + if (msrColEvalutorInfoList.size() > 0 && null != msrColEvalutorInfoList.get(0) .getFilterValues()) { - List<byte[]> measureFilterValuesList = + List<Object> measureFilterValuesList = msrColEvalutorInfoList.get(0).getFilterValues().getMeasuresFilterValuesList(); - return measureFilterValuesList.toArray((new byte[measureFilterValuesList.size()][])); + return measureFilterValuesList.toArray((new Object[measureFilterValuesList.size()])); } return null; - } /** @@ -177,22 +181,21 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm return filterValuesList; } - private List<byte[]> getMeasureRangeValues(CarbonMeasure carbonMeasure) { + private List<Object> getMeasureRangeValues(CarbonMeasure carbonMeasure) { List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20); if (this.getFilterExpression() instanceof BinaryConditionalExpression) { listOfExpressionResults = ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals(); } - List<byte[]> filterValuesList = new ArrayList<byte[]>(20); + List<Object> filterValuesList = new ArrayList<>(20); boolean invalidRowsPresent = false; for (ExpressionResult result : listOfExpressionResults) { try { if (result.getString() == null) { - filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes()); + filterValuesList.add(null); continue; } - filterValuesList.add(DataTypeUtil - .getMeasureByteArrayBasedOnDataTypes(result.getString(), + filterValuesList.add(DataTypeUtil.getMeasureValueBasedOnDataType(result.getString(), result.getDataType(), carbonMeasure)); } catch (FilterIllegalMemberException e) { // Any invalid member while evaluation shall be ignored, system will log the @@ -201,13 +204,8 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm FilterUtil.logError(e, invalidRowsPresent); } } - Comparator<byte[]> filterMeasureComaparator = new Comparator<byte[]>() { - @Override public int compare(byte[] filterMember1, byte[] filterMember2) { - return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2); - } - - }; - Collections.sort(filterValuesList, filterMeasureComaparator); + Collections.sort(filterValuesList, + FilterUtil.getComparatorByDataTypeForMeasure(carbonMeasure.getDataType())); return filterValuesList; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f274bf4c/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 9c03024..000c3ea 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -118,7 +118,9 @@ public final class DataTypeUtil { } public static Object getMeasureObjectFromDataType(byte[] data, DataType dataType) { - + if (data == null || data.length == 0) { + return null; + } switch (dataType) { case SHORT: case INT:
