sort columns
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fcf74017 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fcf74017 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fcf74017 Branch: refs/heads/12-dev Commit: fcf740175a71f04a72e8c493b10a13250fce7523 Parents: 914d61e Author: QiangCai <qiang...@qq.com> Authored: Thu Mar 2 17:48:54 2017 +0800 Committer: QiangCai <qiang...@qq.com> Committed: Fri Apr 7 16:01:16 2017 +0530 ---------------------------------------------------------------------- .../core/constants/CarbonCommonConstants.java | 1 + .../core/datastore/block/SegmentProperties.java | 42 +++ .../impl/VariableLengthDimensionDataChunk.java | 58 +++- .../impl/btree/BTreeDataRefNodeFinder.java | 24 +- .../core/keygenerator/mdkey/Bits.java | 6 +- .../ThriftWrapperSchemaConverterImpl.java | 18 + .../core/metadata/schema/table/CarbonTable.java | 24 ++ .../schema/table/column/ColumnSchema.java | 10 + .../impl/DictionaryBasedResultCollector.java | 12 +- ...structureBasedDictionaryResultCollector.java | 4 +- .../scan/filter/FilterExpressionProcessor.java | 4 +- .../carbondata/core/scan/filter/FilterUtil.java | 98 ++++-- ...velRangeLessThanEqualFilterExecuterImpl.java | 2 +- .../RowLevelRangeLessThanFiterExecuterImpl.java | 2 +- .../visitor/NoDictionaryTypeVisitor.java | 3 +- .../visitor/RangeNoDictionaryTypeVisitor.java | 3 +- .../AbstractDetailQueryResultIterator.java | 4 +- .../scan/result/vector/CarbonColumnVector.java | 7 + .../util/AbstractDataFileFooterConverter.java | 7 + .../apache/carbondata/core/util/ByteUtil.java | 336 ++++++++++++++++++- .../apache/carbondata/core/util/CarbonUtil.java | 18 + .../carbondata/core/util/DataTypeUtil.java | 70 ++++ .../impl/btree/BTreeBlockFinderTest.java | 10 +- .../core/scan/filter/FilterUtilTest.java | 7 +- .../carbondata/hadoop/CarbonInputFormat.java | 4 +- .../internal/index/impl/InMemoryBTreeIndex.java | 4 +- .../impl/DictionaryDecodeReadSupport.java | 3 + .../hadoop/util/BlockLevelTraverser.java | 4 +- .../hadoop/test/util/StoreCreator.java | 5 + .../testsuite/sortcolumns/TestSortColumns.scala | 267 +++++++++++++++ .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 125 +++---- .../execution/command/carbonTableSchema.scala | 66 ++-- .../apache/spark/sql/TestCarbonSqlParser.scala | 56 +--- .../readsupport/SparkRowReadSupportImpl.java | 13 +- .../vectorreader/ColumnarVectorWrapper.java | 13 + .../org/apache/spark/sql/TableCreator.scala | 145 ++++---- .../merger/CompactionResultSortProcessor.java | 10 +- .../merger/RowResultMergerProcessor.java | 14 +- .../newflow/CarbonDataLoadConfiguration.java | 24 ++ .../newflow/DataLoadProcessBuilder.java | 24 +- .../impl/NonDictionaryFieldConverterImpl.java | 21 +- .../sort/impl/ParallelReadMergeSorterImpl.java | 3 +- ...arallelReadMergeSorterWithBucketingImpl.java | 3 +- .../sort/unsafe/UnsafeCarbonRowPage.java | 11 +- .../newflow/sort/unsafe/UnsafeSortDataRows.java | 11 +- .../unsafe/comparator/UnsafeRowComparator.java | 10 +- .../UnsafeRowComparatorForNormalDIms.java | 8 +- .../holder/UnsafeFinalMergePageHolder.java | 4 +- .../unsafe/holder/UnsafeInmemoryHolder.java | 5 +- .../holder/UnsafeSortTempFileChunkHolder.java | 2 +- .../UnsafeSingleThreadFinalSortFilesMerger.java | 4 +- .../newflow/steps/NoSortProcessorStepImpl.java | 153 +++++++++ .../sortdata/IntermediateFileMerger.java | 3 +- .../sortdata/NewRowComparator.java | 12 +- .../sortdata/NewRowComparatorForNormalDims.java | 10 +- .../sortandgroupby/sortdata/RowComparator.java | 10 +- .../sortdata/RowComparatorForNormalDims.java | 10 +- .../sortandgroupby/sortdata/SortDataRows.java | 13 +- .../sortandgroupby/sortdata/SortParameters.java | 43 +++ .../sortdata/SortTempFileChunkHolder.java | 11 +- .../store/CarbonFactDataHandlerColumnar.java | 45 ++- .../store/CarbonFactDataHandlerModel.java | 7 + .../store/SingleThreadFinalSortFilesMerger.java | 8 +- 63 files changed, 1601 insertions(+), 353 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index b82d53c..6cee7f6 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -805,6 +805,7 @@ public final class CarbonCommonConstants { public static final String COLUMN_GROUPS = "column_groups"; public static final String DICTIONARY_EXCLUDE = "dictionary_exclude"; public static final String DICTIONARY_INCLUDE = "dictionary_include"; + public static final String SORT_COLUMNS = "sort_columns"; public static final String PARTITIONCLASS = "partitionclass"; public static final String PARTITIONCOUNT = "partitioncount"; public static final String COLUMN_PROPERTIES = "columnproperties"; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java index 6c7d014..80785e0 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java @@ -55,6 +55,14 @@ public class SegmentProperties { private KeyGenerator dimensionKeyGenerator; /** + * key generator which was used to generate the mdkey for dimensions in SORT_COLUMNS + * if SORT_COLUMNS contains all dimensions, it is same with dimensionKeyGenerator + * otherwise, it is different with dimensionKeyGenerator, the number of its dimensions is less + * than dimensionKeyGenerator. + */ + private KeyGenerator sortColumnsGenerator; + + /** * list of dimension present in the block */ private List<CarbonDimension> dimensions; @@ -151,6 +159,10 @@ public class SegmentProperties { */ private ColumnGroupModel colGroupModel; + private int numberOfSortColumns = 0; + + private int numberOfNoDictSortColumns = 0; + public SegmentProperties(List<ColumnSchema> columnsInTable, int[] columnCardinality) { dimensions = new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); complexDimensions = @@ -336,6 +348,9 @@ public class SegmentProperties { if (CarbonUtil.hasEncoding(columnSchema.getEncodingList(), Encoding.DICTIONARY) && !isComplexDimensionStarted && columnSchema.getNumberOfChild() == 0) { cardinalityIndexForNormalDimensionColumn.add(tableOrdinal); + if (columnSchema.isSortColumn()) { + this.numberOfSortColumns++; + } if (columnSchema.isColumnar()) { // if it is a columnar dimension participated in mdkey then added // key ordinal and dimension ordinal @@ -385,6 +400,10 @@ public class SegmentProperties { // for no dictionary dimension carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, -1); numberOfNoDictionaryDimension++; + if (columnSchema.isSortColumn()) { + this.numberOfSortColumns++; + this.numberOfNoDictSortColumns++; + } } dimensions.add(carbonDimension); } else { @@ -511,6 +530,14 @@ public class SegmentProperties { int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions); // create a key generator this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength); + if (this.getNumberOfDictSortColumns() == bitLength.length) { + this.sortColumnsGenerator = this.dimensionKeyGenerator; + } else { + int numberOfDictSortColumns = this.getNumberOfDictSortColumns(); + int [] sortColumnBitLength = new int[numberOfDictSortColumns]; + System.arraycopy(bitLength, 0, sortColumnBitLength, 0, numberOfDictSortColumns); + this.sortColumnsGenerator = new MultiDimKeyVarLengthGenerator(sortColumnBitLength); + } this.fixedLengthKeySplitter = new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions); // get the size of each value in file block @@ -636,6 +663,10 @@ public class SegmentProperties { return dimensionKeyGenerator; } + public KeyGenerator getSortColumnsGenerator() { + return sortColumnsGenerator; + } + /** * @return the dimensions */ @@ -797,4 +828,15 @@ public class SegmentProperties { return CarbonUtil.getMeasureFromCurrentBlock(this.measures, columnId); } + public int getNumberOfSortColumns() { + return numberOfSortColumns; + } + + public int getNumberOfNoDictSortColumns() { + return numberOfNoDictSortColumns; + } + + public int getNumberOfDictSortColumns() { + return this.numberOfSortColumns - this.numberOfNoDictSortColumns; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java index 92c1e2f..4897985 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java @@ -24,6 +24,18 @@ import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFacto import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; +import org.apache.carbondata.core.util.ByteUtil; + +import org.apache.spark.sql.types.BooleanType; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.Decimal; +import org.apache.spark.sql.types.DecimalType; +import org.apache.spark.sql.types.DoubleType; +import org.apache.spark.sql.types.FloatType; +import org.apache.spark.sql.types.IntegerType; +import org.apache.spark.sql.types.LongType; +import org.apache.spark.sql.types.ShortType; +import org.apache.spark.sql.types.StringType; /** * This class is gives access to variable length dimension data chunk store @@ -33,8 +45,8 @@ public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk /** * Constructor for this class * - * @param dataChunkStore data chunk - * @param chunkAttributes chunk attributes + * @param dataChunks data chunk + * @param invertedIndex chunk attributes */ public VariableLengthDimensionDataChunk(byte[] dataChunks, int[] invertedIndex, int[] invertedIndexReverse, int numberOfRows) { @@ -114,7 +126,26 @@ public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk if (value == null || Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, value)) { vector.putNull(vectorOffset++); } else { - vector.putBytes(vectorOffset++, value); + DataType dt = vector.getType(); + if (dt instanceof StringType) { + vector.putBytes(vectorOffset++, value); + } else if (dt instanceof BooleanType) { + vector.putBoolean(vectorOffset++, ByteUtil.toBoolean(value)); + } else if (dt instanceof ShortType) { + vector.putShort(vectorOffset++, ByteUtil.toShort(value, 0, value.length)); + } else if (dt instanceof IntegerType) { + vector.putInt(vectorOffset++, ByteUtil.toInt(value, 0, value.length)); + } else if (dt instanceof FloatType) { + vector.putFloat(vectorOffset++, ByteUtil.toFloat(value, 0)); + } else if (dt instanceof DoubleType) { + vector.putDouble(vectorOffset++, ByteUtil.toDouble(value, 0)); + } else if (dt instanceof LongType) { + vector.putLong(vectorOffset++, ByteUtil.toLong(value, 0, value.length)); + } else if (dt instanceof DecimalType) { + vector.putDecimal(vectorOffset++, + Decimal.apply(ByteUtil.toBigDecimal(value, 0, value.length)), + DecimalType.MAX_PRECISION()); + } } } return column + 1; @@ -143,7 +174,26 @@ public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk if (value == null || Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, value)) { vector.putNull(vectorOffset++); } else { - vector.putBytes(vectorOffset++, value); + DataType dt = vector.getType(); + if (dt instanceof StringType) { + vector.putBytes(vectorOffset++, value); + } else if (dt instanceof BooleanType) { + vector.putBoolean(vectorOffset++, ByteUtil.toBoolean(value)); + } else if (dt instanceof ShortType) { + vector.putShort(vectorOffset++, ByteUtil.toShort(value, 0, value.length)); + } else if (dt instanceof IntegerType) { + vector.putInt(vectorOffset++, ByteUtil.toInt(value, 0, value.length)); + } else if (dt instanceof FloatType) { + vector.putFloat(vectorOffset++, ByteUtil.toFloat(value, 0)); + } else if (dt instanceof DoubleType) { + vector.putDouble(vectorOffset++, ByteUtil.toDouble(value, 0)); + } else if (dt instanceof LongType) { + vector.putLong(vectorOffset++, ByteUtil.toLong(value, 0, value.length)); + } else if (dt instanceof DecimalType) { + vector.putDecimal(vectorOffset++, + Decimal.apply(ByteUtil.toBigDecimal(value, 0, value.length)), + DecimalType.MAX_PRECISION()); + } } } return column + 1; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java index 2e1709b..2f8aadf 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java @@ -49,18 +49,20 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder { private int[] eachColumnValueSize; /** - * this will be used during search for no dictionary column + * the number of no dictionary columns in SORT_COLUMNS */ - private int numberOfNoDictionaryColumns; + private int numberOfNoDictSortColumns; - public BTreeDataRefNodeFinder(int[] eachColumnValueSize) { - this.eachColumnValueSize = eachColumnValueSize; + /** + * the number of columns in SORT_COLUMNS + */ + private int numberOfSortColumns; - for (int i = 0; i < eachColumnValueSize.length; i++) { - if (eachColumnValueSize[i] == -1) { - numberOfNoDictionaryColumns++; - } - } + public BTreeDataRefNodeFinder(int[] eachColumnValueSize, int numberOfSortColumns, + int numberOfNoDictSortColumns) { + this.eachColumnValueSize = eachColumnValueSize; + this.numberOfNoDictSortColumns = numberOfNoDictSortColumns; + this.numberOfSortColumns = numberOfSortColumns; } /** @@ -213,7 +215,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder { int dictionaryKeyOffset = 0; int nonDictionaryKeyOffset = 0; int compareResult = 0; - int processedNoDictionaryColumn = numberOfNoDictionaryColumns; + int processedNoDictionaryColumn = numberOfNoDictSortColumns; ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys()); ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys()); int actualOffset = 0; @@ -221,7 +223,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder { int firstNoDcitionaryLength = 0; int secondNodeDictionaryLength = 0; - for (int i = 0; i < eachColumnValueSize.length; i++) { + for (int i = 0; i < numberOfSortColumns; i++) { if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) { compareResult = ByteUtil.UnsafeComparer.INSTANCE http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java index 7bd338d..044a77b 100644 --- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java +++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java @@ -119,7 +119,8 @@ public class Bits implements Serializable { protected long[] get(long[] keys) { long[] words = new long[wsize]; int ll = 0; - for (int i = lens.length - 1; i >= 0; i--) { + int minLength = Math.min(lens.length, keys.length); + for (int i = minLength - 1; i >= 0; i--) { long val = keys[i]; @@ -155,7 +156,8 @@ public class Bits implements Serializable { protected long[] get(int[] keys) { long[] words = new long[wsize]; int ll = 0; - for (int i = lens.length - 1; i >= 0; i--) { + int minLength = Math.min(lens.length, keys.length); + for (int i = minLength - 1; i >= 0; i--) { long val = keys[i]; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java index 974cc81..3d114ec 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java @@ -17,8 +17,11 @@ package org.apache.carbondata.core.metadata.converter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.BucketingInfo; @@ -172,6 +175,13 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { thriftColumnSchema.setInvisible(wrapperColumnSchema.isInvisible()); thriftColumnSchema.setColumnReferenceId(wrapperColumnSchema.getColumnReferenceId()); thriftColumnSchema.setSchemaOrdinal(wrapperColumnSchema.getSchemaOrdinal()); + + if (wrapperColumnSchema.isSortColumn()) { + Map<String, String> properties = new HashMap<String, String>(); + properties.put(CarbonCommonConstants.SORT_COLUMNS, "true"); + thriftColumnSchema.setColumnProperties(properties); + } + return thriftColumnSchema; } @@ -360,6 +370,14 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible()); wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId()); wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal()); + wrapperColumnSchema.setSortColumn(false); + Map<String, String> properties = externalColumnSchema.getColumnProperties(); + if (properties != null) { + String sortColumns = properties.get(CarbonCommonConstants.SORT_COLUMNS); + if (sortColumns != null) { + wrapperColumnSchema.setSortColumn(true); + } + } return wrapperColumnSchema; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java index dbc8836..fae9960 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java @@ -116,6 +116,16 @@ public class CarbonTable implements Serializable { */ private int blockSize; + /** + * the number of columns in SORT_COLUMNS + */ + private int numberOfSortColumns; + + /** + * the number of no dictionary columns in SORT_COLUMNS + */ + private int numberOfNoDictSortColumns; + public CarbonTable() { this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>(); this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>(); @@ -238,10 +248,16 @@ public class CarbonTable implements Serializable { i = dimensionOrdinal - 1; complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal); } else { + if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) { + this.numberOfSortColumns++; + } if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) { CarbonDimension dimension = new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), -1, -1, -1); + if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) { + this.numberOfNoDictSortColumns++; + } allDimensions.add(dimension); primitiveDimensions.add(dimension); } else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY) @@ -639,4 +655,12 @@ public class CarbonTable implements Serializable { } tableMeasuresMap.put(tableName, visibleMeasures); } + + public int getNumberOfSortColumns() { + return numberOfSortColumns; + } + + public int getNumberOfNoDictSortColumns() { + return numberOfNoDictSortColumns; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java index 7aa0900..08969fb 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java @@ -109,6 +109,8 @@ public class ColumnSchema implements Serializable { */ private boolean invisible = false; + private boolean isSortColumn = false; + /** * @return the columnName */ @@ -403,4 +405,12 @@ public class ColumnSchema implements Serializable { public void setSchemaOrdinal(int schemaOrdinal) { this.schemaOrdinal = schemaOrdinal; } + + public boolean isSortColumn() { + return isSortColumn; + } + + public void setSortColumn(boolean sortColumn) { + isSortColumn = sortColumn; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java index e5e4b3c..a7c93bb 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java @@ -88,7 +88,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect List<Object[]> listBasedResult = new ArrayList<>(batchSize); int rowCounter = 0; int[] surrogateResult; - String[] noDictionaryKeys; + byte[][] noDictionaryKeys; byte[][] complexTypeKeyArray; BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache = scannedResult.getDeleteDeltaDataCache(); @@ -96,7 +96,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect Object[] row = new Object[queryDimensions.length + queryMeasures.length]; if (isDimensionExists) { surrogateResult = scannedResult.getDictionaryKeyIntegerArray(); - noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray(); + noDictionaryKeys = scannedResult.getNoDictionaryKeyArray(); complexTypeKeyArray = scannedResult.getComplexTypeKeyArray(); dictionaryColumnIndex = 0; noDictionaryColumnIndex = 0; @@ -120,7 +120,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect } protected void fillDimensionData(AbstractScannedResult scannedResult, int[] surrogateResult, - String[] noDictionaryKeys, byte[][] complexTypeKeyArray, + byte[][] noDictionaryKeys, byte[][] complexTypeKeyArray, Map<Integer, GenericQueryType> comlexDimensionInfoMap, Object[] row, int i) { if (!dictionaryEncodingArray[i]) { if (implictColumnArray[i]) { @@ -134,9 +134,9 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect DataTypeUtil.getDataBasedOnDataType(scannedResult.getBlockletId(), DataType.STRING); } } else { - row[order[i]] = DataTypeUtil - .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++], - queryDimensions[i].getDimension().getDataType()); + row[order[i]] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn( + noDictionaryKeys[noDictionaryColumnIndex++], + queryDimensions[i].getDimension().getDataType()); } } else if (directDictionaryEncodingArray[i]) { if (directDictionaryGenerators[i] != null) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java index 71045ff..7c07923 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java @@ -48,7 +48,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe List<Object[]> listBasedResult = new ArrayList<>(batchSize); int rowCounter = 0; int[] surrogateResult; - String[] noDictionaryKeys; + byte[][] noDictionaryKeys; byte[][] complexTypeKeyArray; BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache = scannedResult.getDeleteDeltaDataCache(); @@ -58,7 +58,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe Object[] row = new Object[queryDimensions.length + queryMeasures.length]; if (isDimensionExists) { surrogateResult = scannedResult.getDictionaryKeyIntegerArray(); - noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray(); + noDictionaryKeys = scannedResult.getNoDictionaryKeyArray(); complexTypeKeyArray = scannedResult.getComplexTypeKeyArray(); dictionaryColumnIndex = 0; noDictionaryColumnIndex = 0; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java index 264c0ae..b5341a0 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java @@ -116,7 +116,9 @@ public class FilterExpressionProcessor implements FilterProcessor { + searchEndKey.getNoDictionaryKeys()); long startTimeInMillis = System.currentTimeMillis(); DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder( - tableSegment.getSegmentProperties().getEachDimColumnValueSize()); + tableSegment.getSegmentProperties().getEachDimColumnValueSize(), + tableSegment.getSegmentProperties().getNumberOfSortColumns(), + tableSegment.getSegmentProperties().getNumberOfNoDictSortColumns()); DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey); DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey); FilterExecuter filterExecuter = http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index 770aa7e..b116d9a 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -355,10 +355,23 @@ public final class FilterUtil { * @return DimColumnFilterInfo */ public static DimColumnFilterInfo getNoDictionaryValKeyMemberForFilter( - List<String> evaluateResultListFinal, boolean isIncludeFilter) { + List<String> evaluateResultListFinal, boolean isIncludeFilter, DataType dataType) + throws FilterUnsupportedException { List<byte[]> filterValuesList = new ArrayList<byte[]>(20); - for (String result : evaluateResultListFinal) { - filterValuesList.add(result.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET))); + String result = null; + try { + int length = evaluateResultListFinal.size(); + for (int i = 0; i < length; i++) { + result = evaluateResultListFinal.get(i); + if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) { + filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY); + continue; + } + filterValuesList.add( + DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, dataType)); + } + } catch (Throwable ex) { + throw new FilterUnsupportedException("Unsupported Filter condition: " + result, ex); } Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() { @@ -879,8 +892,10 @@ public final class FilterUtil { } int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal(); for (DimColumnFilterInfo info : values) { - if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) { - startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0); + if (keyOrdinalOfDimensionFromCurrentBlock < startKey.length) { + if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) { + startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0); + } } } long[] newStartKey = new long[startKey.length]; @@ -944,10 +959,12 @@ public final class FilterUtil { } int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal(); for (DimColumnFilterInfo info : values) { - if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList() - .get(info.getFilterList().size() - 1)) { - endKey[keyOrdinalOfDimensionFromCurrentBlock] = - info.getFilterList().get(info.getFilterList().size() - 1); + if (keyOrdinalOfDimensionFromCurrentBlock < endKey.length) { + if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList() + .get(info.getFilterList().size() - 1)) { + endKey[keyOrdinalOfDimensionFromCurrentBlock] = + info.getFilterList().get(info.getFilterList().size() - 1); + } } } long[] newEndKey = new long[endKey.length]; @@ -1040,44 +1057,43 @@ public final class FilterUtil { */ public static IndexKey prepareDefaultEndIndexKey(SegmentProperties segmentProperties) throws KeyGenException { - long[] dictionarySurrogateKey = - new long[segmentProperties.getDimensions().size() - segmentProperties - .getNumberOfNoDictionaryDimension()]; + long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()]; int index = 0; int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality(); - for (int i = 0; i < dimColumnsCardinality.length; i++) { + for (int i = 0; i < dictionarySurrogateKey.length; i++) { dictionarySurrogateKey[index++] = dimColumnsCardinality[i]; } IndexKey endIndexKey; byte[] dictionaryendMdkey = - segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey); + segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey); byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties); endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer); return endIndexKey; } public static byte[] getNoDictionaryDefaultEndKey(SegmentProperties segmentProperties) { + + int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns(); // in case of non filter query when no dictionary columns are present we // need to set the default end key, as for non filter query // we need to get the last // block of the btree so we are setting the max byte value in the end key ByteBuffer noDictionaryEndKeyBuffer = ByteBuffer.allocate( - (segmentProperties.getNumberOfNoDictionaryDimension() - * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties - .getNumberOfNoDictionaryDimension()); + (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + + numberOfNoDictionaryDimension); // end key structure will be //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes> //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value> // <second no dictionary column value> <N no dictionary column value> //example if we have 2 no dictionary column //<[0,4,0,5,127,127]> - short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension() - * CarbonCommonConstants.SHORT_SIZE_IN_BYTE); - for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) { + short startPoint = + (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE); + for (int i = 0; i < numberOfNoDictionaryDimension; i++) { noDictionaryEndKeyBuffer.putShort((startPoint)); startPoint++; } - for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) { + for (int i = 0; i < numberOfNoDictionaryDimension; i++) { noDictionaryEndKeyBuffer.put((byte) 127); } return noDictionaryEndKeyBuffer.array(); @@ -1094,11 +1110,9 @@ public final class FilterUtil { public static IndexKey prepareDefaultStartIndexKey(SegmentProperties segmentProperties) throws KeyGenException { IndexKey startIndexKey; - long[] dictionarySurrogateKey = - new long[segmentProperties.getDimensions().size() - segmentProperties - .getNumberOfNoDictionaryDimension()]; + long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()]; byte[] dictionaryStartMdkey = - segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey); + segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey); byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties); startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray); @@ -1106,26 +1120,27 @@ public final class FilterUtil { } public static byte[] getNoDictionaryDefaultStartKey(SegmentProperties segmentProperties) { + + int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns(); // in case of non filter query when no dictionary columns are present we // need to set the default start key, as for non filter query we need to get the first // block of the btree so we are setting the least byte value in the start key ByteBuffer noDictionaryStartKeyBuffer = ByteBuffer.allocate( - (segmentProperties.getNumberOfNoDictionaryDimension() - * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties - .getNumberOfNoDictionaryDimension()); + (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + + numberOfNoDictionaryDimension); // end key structure will be //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes> //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value> // <second no dictionary column value> <N no dictionary column value> //example if we have 2 no dictionary column //<[0,4,0,5,0,0]> - short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension() - * CarbonCommonConstants.SHORT_SIZE_IN_BYTE); - for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) { + short startPoint = + (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE); + for (int i = 0; i < numberOfNoDictionaryDimension; i++) { noDictionaryStartKeyBuffer.putShort((startPoint)); startPoint++; } - for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) { + for (int i = 0; i < numberOfNoDictionaryDimension; i++) { noDictionaryStartKeyBuffer.put((byte) 0); } return noDictionaryStartKeyBuffer.array(); @@ -1181,8 +1196,8 @@ public final class FilterUtil { FilterResolverIntf filterResolver, List<IndexKey> listOfStartEndKeys) { IndexKey searchStartKey = null; IndexKey searchEndKey = null; - long[] startKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()]; - long[] endKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()]; + long[] startKey = new long[segmentProperties.getNumberOfDictSortColumns()]; + long[] endKey = new long[segmentProperties.getNumberOfDictSortColumns()]; List<byte[]> listOfStartKeyByteArray = new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension()); List<byte[]> listOfEndKeyByteArray = @@ -1234,12 +1249,23 @@ public final class FilterUtil { pruneStartAndEndKeys(setOfStartKeyByteArray, listOfStartKeyByteArray); pruneStartAndEndKeys(setOfEndKeyByteArray, listOfEndKeyByteArray); + if (segmentProperties.getNumberOfNoDictSortColumns() == 0) { + listOfStartKeyByteArray = new ArrayList<byte[]>(); + listOfEndKeyByteArray = new ArrayList<byte[]>(); + } else if (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray + .size()) { + while (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray.size()) { + listOfStartKeyByteArray.remove(listOfStartKeyByteArray.size() - 1); + listOfEndKeyByteArray.remove(listOfEndKeyByteArray.size() - 1); + } + } + searchStartKey = FilterUtil - .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(), + .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getSortColumnsGenerator(), FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray)); searchEndKey = FilterUtil - .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(), + .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getSortColumnsGenerator(), FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray)); listOfStartEndKeys.add(searchStartKey); listOfStartEndKeys.add(searchEndKey); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java index 306becf..bd83eec 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java @@ -157,7 +157,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dimensionBlocksIndex[0]); defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension, - this.segmentProperties.getDimensionKeyGenerator()); + this.segmentProperties.getSortColumnsGenerator()); } if (dimensionColumnDataChunk.isExplicitSorted() && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java index f2a49d9..a439228 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java @@ -157,7 +157,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dimensionBlocksIndex[0]); defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension, - this.segmentProperties.getDimensionKeyGenerator()); + this.segmentProperties.getSortColumnsGenerator()); } if (dimensionColumnDataChunk.isExplicitSorted() && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java index 351d2c0..efedd93 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java @@ -67,7 +67,8 @@ public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf { throw new FilterUnsupportedException(e); } resolvedFilterObject = FilterUtil - .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter()); + .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), + metadata.getColumnExpression().getDataType()); visitableObj.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java index 8d0a8b4..d703ed0 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java @@ -72,7 +72,8 @@ public class RangeNoDictionaryTypeVisitor extends NoDictionaryTypeVisitor throw new FilterUnsupportedException(e); } resolvedFilterObject = FilterUtil - .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter()); + .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), + metadata.getColumnExpression().getDataType()); visitableObj.setFilterValues(resolvedFilterObject); } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java index 4e5681c..cdab4e4 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java @@ -104,7 +104,9 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato private void intialiseInfos() { for (BlockExecutionInfo blockInfo : blockExecutionInfos) { - DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize()); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize(), + blockInfo.getDataBlock().getSegmentProperties().getNumberOfSortColumns(), + blockInfo.getDataBlock().getSegmentProperties().getNumberOfNoDictSortColumns()); DataRefNode startDataBlock = finder .findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey()); while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java index 4952e07..82a0b45 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java @@ -17,10 +17,15 @@ package org.apache.carbondata.core.scan.result.vector; +import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.Decimal; public interface CarbonColumnVector { + void putBoolean(int rowId, boolean value); + + void putFloat(int rowId, float value); + void putShort(int rowId, short value); void putShorts(int rowId, int count, short value); @@ -59,4 +64,6 @@ public interface CarbonColumnVector { void reset(); + DataType getType(); + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java index 48a081d..52f2f31 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.BitSet; import java.util.List; +import java.util.Map; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.block.BlockInfo; @@ -195,6 +196,12 @@ public abstract class AbstractDataFileFooterConverter { wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id()); wrapperColumnSchema.setScale(externalColumnSchema.getScale()); wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value()); + Map<String, String> properties = externalColumnSchema.getColumnProperties(); + if (properties != null) { + if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) { + wrapperColumnSchema.setSortColumn(true); + } + } return wrapperColumnSchema; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java index 9b2c2ed..84a8634 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java @@ -17,7 +17,11 @@ package org.apache.carbondata.core.util; +import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.memory.CarbonUnsafe; @@ -27,7 +31,13 @@ import org.apache.carbondata.core.memory.CarbonUnsafe; */ public final class ByteUtil { - private static final int SIZEOF_LONG = 8; + public static final int SIZEOF_LONG = 8; + + public static final int SIZEOF_INT = 4; + + public static final int SIZEOF_SHORT = 2; + + public static final String UTF8_CSN = StandardCharsets.UTF_8.name(); private ByteUtil() { @@ -363,4 +373,328 @@ public final class ByteUtil { } + /** + * Stirng => byte[] + * + * @param s + * @return + */ + public static byte[] toBytes(String s) { + try { + return s.getBytes(UTF8_CSN); + } catch (UnsupportedEncodingException e) { + // should never happen! + throw new IllegalArgumentException("UTF8 decoding is not supported", e); + } + } + + /** + * byte[] => String + * + * @param b + * @param off + * @param len + * @return + */ + public static String toString(final byte[] b, int off, int len) { + if (b == null) { + return null; + } + if (len == 0) { + return ""; + } + try { + return new String(b, off, len, UTF8_CSN); + } catch (UnsupportedEncodingException e) { + // should never happen! + throw new IllegalArgumentException("UTF8 encoding is not supported", e); + } + } + + /** + * boolean => byte[] + * + * @param b + * @return + */ + public static byte[] toBytes(final boolean b) { + return new byte[] { b ? (byte) -1 : (byte) 0 }; + } + + /** + * byte[] => boolean + * + * @param b + * @return + */ + public static boolean toBoolean(final byte[] b) { + if (b.length != 1) { + throw new IllegalArgumentException("Array has wrong size: " + b.length); + } + return b[0] != (byte) 0; + } + + /** + * short => byte[] + * + * @param val + * @return + */ + public static byte[] toBytes(short val) { + byte[] b = new byte[SIZEOF_SHORT]; + b[1] = (byte) val; + val >>= 8; + b[0] = (byte) val; + return b; + } + + /** + * byte[] => short + * + * @param bytes + * @param offset + * @param length + * @return + */ + public static short toShort(byte[] bytes, int offset, final int length) { + if (length != SIZEOF_SHORT || offset + length > bytes.length) { + throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); + } + if (CarbonUnsafe.unsafe != null) { + if (CarbonUnsafe.ISLITTLEENDIAN) { + return Short.reverseBytes( + CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); + } else { + return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); + } + } else { + short n = 0; + n ^= bytes[offset] & 0xFF; + n <<= 8; + n ^= bytes[offset + 1] & 0xFF; + return n; + } + } + + /** + * int => byte[] + * + * @param val + * @return + */ + public static byte[] toBytes(int val) { + byte[] b = new byte[4]; + for (int i = 3; i > 0; i--) { + b[i] = (byte) val; + val >>>= 8; + } + b[0] = (byte) val; + return b; + } + + /** + * byte[] => int + * + * @param bytes + * @param offset + * @param length + * @return + */ + public static int toInt(byte[] bytes, int offset, final int length) { + if (length != SIZEOF_INT || offset + length > bytes.length) { + throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); + } + if (CarbonUnsafe.unsafe != null) { + if (CarbonUnsafe.ISLITTLEENDIAN) { + return Integer.reverseBytes( + CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); + } else { + return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); + } + } else { + int n = 0; + for (int i = offset; i < (offset + length); i++) { + n <<= 8; + n ^= bytes[i] & 0xFF; + } + return n; + } + } + + /** + * float => byte[] + * + * @param f + * @return + */ + public static byte[] toBytes(final float f) { + // Encode it as int + return toBytes(Float.floatToRawIntBits(f)); + } + + /** + * byte[] => float + * + * @param bytes + * @param offset + * @return + */ + public static float toFloat(byte[] bytes, int offset) { + return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT)); + } + + /** + * long => byte[] + * + * @param val + * @return + */ + public static byte[] toBytes(long val) { + byte[] b = new byte[8]; + for (int i = 7; i > 0; i--) { + b[i] = (byte) val; + val >>>= 8; + } + b[0] = (byte) val; + return b; + } + + /** + * byte[] => long + */ + public static long toLong(byte[] bytes, int offset, final int length) { + if (length != SIZEOF_LONG || offset + length > bytes.length) { + throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); + } + if (CarbonUnsafe.unsafe != null) { + if (CarbonUnsafe.ISLITTLEENDIAN) { + return Long.reverseBytes( + CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET)); + } else { + return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET); + } + } else { + long l = 0; + for (int i = offset; i < offset + length; i++) { + l <<= 8; + l ^= bytes[i] & 0xFF; + } + return l; + } + } + + /** + * doube => byte[] + * + * @param d + * @return + */ + public static byte[] toBytes(final double d) { + // Encode it as a long + return toBytes(Double.doubleToRawLongBits(d)); + } + + /** + * byte[] => double + * + * @param bytes + * @param offset + * @return + */ + public static double toDouble(final byte[] bytes, final int offset) { + return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG)); + } + + /** + * BigDecimal => byte[] + * + * @param val + * @return + */ + public static byte[] toBytes(BigDecimal val) { + byte[] valueBytes = val.unscaledValue().toByteArray(); + byte[] result = new byte[valueBytes.length + SIZEOF_INT]; + int offset = putInt(result, 0, val.scale()); + putBytes(result, offset, valueBytes, 0, valueBytes.length); + return result; + } + + /** + * byte[] => BigDecimal + * + * @param bytes + * @param offset + * @param length + * @return + */ + public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) { + if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) { + return null; + } + + int scale = toInt(bytes, offset, bytes.length); + byte[] tcBytes = new byte[length - SIZEOF_INT]; + System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT); + return new BigDecimal(new BigInteger(tcBytes), scale); + } + + private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes, + final int offset, final int length, final int expectedLength) { + String reason; + if (length != expectedLength) { + reason = "Wrong length: " + length + ", expected " + expectedLength; + } else { + reason = "offset (" + offset + ") + length (" + length + ") exceed the" + + " capacity of the array: " + bytes.length; + } + return new IllegalArgumentException(reason); + } + + /** + * Put an int value out to the specified byte array position. + * + * @param bytes the byte array + * @param offset position in the array + * @param val int to write out + * @return incremented offset + * @throws IllegalArgumentException if the byte array given doesn't have + * enough room at the offset specified. + */ + public static int putInt(byte[] bytes, int offset, int val) { + if (bytes.length - offset < SIZEOF_INT) { + throw new IllegalArgumentException( + "Not enough room to put an int at" + " offset " + offset + " in a " + bytes.length + + " byte array"); + } + if (CarbonUnsafe.unsafe != null) { + if (CarbonUnsafe.ISLITTLEENDIAN) { + val = Integer.reverseBytes(val); + } + CarbonUnsafe.unsafe.putInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET, val); + return offset + ByteUtil.SIZEOF_INT; + } else { + for (int i = offset + 3; i > offset; i--) { + bytes[i] = (byte) val; + val >>>= 8; + } + bytes[offset] = (byte) val; + return offset + SIZEOF_INT; + } + } + + /** + * Put bytes at the specified byte array position. + * + * @param tgtBytes the byte array + * @param tgtOffset position in the array + * @param srcBytes array to write out + * @param srcOffset source offset + * @param srcLength source length + * @return incremented offset + */ + public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, int srcOffset, + int srcLength) { + System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength); + return tgtOffset + srcLength; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index a442087..7c63f28 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -641,6 +641,24 @@ public final class CarbonUtil { } /** + * remove the quote char for a string, e.g. "abc" => abc, 'abc' => abc + * @param parseStr + * @return + */ + public static String unquoteChar(String parseStr) { + if (parseStr == null) { + return null; + } + if (parseStr.startsWith("'") && parseStr.endsWith("'")) { + return parseStr.substring(1, parseStr.length() - 1); + } else if (parseStr.startsWith("\"") && parseStr.endsWith("\"")) { + return parseStr.substring(1, parseStr.length() - 1); + } else { + return parseStr; + } + } + + /** * special char delimiter Converter * * @param delimiter http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index 76df425..9edd121 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -323,6 +323,76 @@ public final class DataTypeUtil { } + public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue, + DataType actualDataType) throws Throwable { + switch (actualDataType) { + case STRING: + return ByteUtil.toBytes(dimensionValue); + case BOOLEAN: + return ByteUtil.toBytes(Boolean.parseBoolean(dimensionValue)); + case SHORT: + return ByteUtil.toBytes(Short.parseShort(dimensionValue)); + case INT: + return ByteUtil.toBytes(Integer.parseInt(dimensionValue)); + case FLOAT: + return ByteUtil.toBytes(Float.parseFloat(dimensionValue)); + case LONG: + return ByteUtil.toBytes(Long.parseLong(dimensionValue)); + case DOUBLE: + return ByteUtil.toBytes(Double.parseDouble(dimensionValue)); + case DECIMAL: + return ByteUtil.toBytes(new BigDecimal(dimensionValue)); + default: + return ByteUtil.toBytes(dimensionValue); + } + } + + + /** + * Below method will be used to convert the data passed to its actual data + * type + * + * @param dataInBytes data + * @param actualDataType actual data type + * @return actual data after conversion + */ + public static Object getDataBasedOnDataTypeForNoDictionaryColumn(byte[] dataInBytes, + DataType actualDataType) { + if (null == dataInBytes || Arrays + .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, dataInBytes)) { + return null; + } + try { + switch (actualDataType) { + case STRING: + return UTF8String.fromBytes(dataInBytes); + case BOOLEAN: + return ByteUtil.toBoolean(dataInBytes); + case SHORT: + return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length); + case INT: + return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length); + case FLOAT: + return ByteUtil.toFloat(dataInBytes, 0); + case LONG: + return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length); + case DOUBLE: + return ByteUtil.toDouble(dataInBytes, 0); + case DECIMAL: + return ByteUtil.toBigDecimal(dataInBytes, 0, dataInBytes.length); + default: + return ByteUtil.toString(dataInBytes, 0, dataInBytes.length); + } + } catch (Throwable ex) { + String data = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS); + LOGGER.error("Cannot convert" + data + " to " + actualDataType.getName() + " type value" + ex + .getMessage()); + LOGGER.error("Problem while converting data type" + data); + return null; + } + } + + /** * Below method will be used to convert the data passed to its actual data * type http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java index a4adf92..245ff48 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java @@ -68,7 +68,7 @@ public class BTreeBlockFinderTest extends TestCase { builder.build(infos); DataRefNode dataBlock = builder.get(); assertTrue(dataBlock != null); - DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1); ByteBuffer buffer = ByteBuffer.allocate(4 + 2); buffer.rewind(); buffer.putShort((short) 1); @@ -88,7 +88,7 @@ public class BTreeBlockFinderTest extends TestCase { builder.build(infos); DataRefNode dataBlock = builder.get(); assertTrue(dataBlock != null); - DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1); ByteBuffer buffer = ByteBuffer.allocate(4 + 1); buffer.rewind(); buffer.put((byte) 1); @@ -109,7 +109,7 @@ public class BTreeBlockFinderTest extends TestCase { builder.build(infos); DataRefNode dataBlock = builder.get(); assertTrue(dataBlock != null); - DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0); int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 }); KeyGenerator multiDimKeyVarLengthGenerator = @@ -131,7 +131,7 @@ public class BTreeBlockFinderTest extends TestCase { builder.build(infos); DataRefNode dataBlock = builder.get(); assertTrue(dataBlock != null); - DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0); int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 }); KeyGenerator multiDimKeyVarLengthGenerator = @@ -160,7 +160,7 @@ public class BTreeBlockFinderTest extends TestCase { builder.build(infos); DataRefNode dataBlock = builder.get(); assertTrue(dataBlock != null); - DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }); + DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0); int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 }); KeyGenerator multiDimKeyVarLengthGenerator = http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java index 06063a4..406fbb7 100644 --- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java @@ -37,6 +37,7 @@ import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.LiteralExpression; import org.apache.carbondata.core.scan.expression.conditional.ListExpression; +import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.filter.intf.RowImpl; import mockit.Mock; @@ -362,7 +363,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest { assertFalse(result); } - @Test public void testGetNoDictionaryValKeyMemberForFilter() { + @Test public void testGetNoDictionaryValKeyMemberForFilter() throws FilterUnsupportedException { boolean isIncludeFilter = true; AbsoluteTableIdentifier absoluteTableIdentifier = new AbsoluteTableIdentifier(this.carbonStorePath, carbonTableIdentifier); @@ -370,7 +371,9 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest { List<String> evaluateResultListFinal = new ArrayList<>(); evaluateResultListFinal.add("test1"); evaluateResultListFinal.add("test2"); - assertTrue(FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter) instanceof DimColumnFilterInfo); + assertTrue(FilterUtil + .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter, + DataType.STRING) instanceof DimColumnFilterInfo); } @Test public void testPrepareDefaultStartIndexKey() throws KeyGenException { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java index b330f12..540a69c 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java @@ -578,7 +578,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> { // Add all blocks of btree into result DataRefNodeFinder blockFinder = - new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize()); + new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(), + segmentProperties.getNumberOfSortColumns(), + segmentProperties.getNumberOfNoDictSortColumns()); DataRefNode startBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey); DataRefNode endBlock = http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java index 7ba6133..d476f74 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java @@ -199,7 +199,9 @@ class InMemoryBTreeIndex implements Index { // Add all blocks of btree into result DataRefNodeFinder blockFinder = - new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize()); + new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(), + segmentProperties.getNumberOfSortColumns(), + segmentProperties.getNumberOfNoDictSortColumns()); DataRefNode startBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey); DataRefNode endBlock = http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java index 43953d0..cfdbd37 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java @@ -87,6 +87,9 @@ public class DictionaryDecodeReadSupport<T> implements CarbonReadSupport<T> { * threshold is reached */ @Override public void close() { + if (dictionaries == null) { + return; + } for (int i = 0; i < dictionaries.length; i++) { CarbonUtil.clearDictionaryCache(dictionaries[i]); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java index e39d214..b814468 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java @@ -53,7 +53,9 @@ public class BlockLevelTraverser { FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties()); DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder( - abstractIndex.getSegmentProperties().getEachDimColumnValueSize()); + abstractIndex.getSegmentProperties().getEachDimColumnValueSize(), + abstractIndex.getSegmentProperties().getNumberOfSortColumns(), + abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns()); DataRefNode currentBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fcf74017/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java index 51ce2c5..aa6a2b4 100644 --- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java +++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java @@ -202,6 +202,7 @@ public class StoreCreator { date.setColumnUniqueId(UUID.randomUUID().toString()); date.setDimensionColumn(true); date.setColumnGroup(2); + date.setSortColumn(true); columnSchemas.add(date); ColumnSchema country = new ColumnSchema(); @@ -212,6 +213,7 @@ public class StoreCreator { country.setColumnUniqueId(UUID.randomUUID().toString()); country.setDimensionColumn(true); country.setColumnGroup(3); + country.setSortColumn(true); columnSchemas.add(country); ColumnSchema name = new ColumnSchema(); @@ -222,6 +224,7 @@ public class StoreCreator { name.setColumnUniqueId(UUID.randomUUID().toString()); name.setDimensionColumn(true); name.setColumnGroup(4); + name.setSortColumn(true); columnSchemas.add(name); ColumnSchema phonetype = new ColumnSchema(); @@ -232,6 +235,7 @@ public class StoreCreator { phonetype.setColumnUniqueId(UUID.randomUUID().toString()); phonetype.setDimensionColumn(true); phonetype.setColumnGroup(5); + phonetype.setSortColumn(true); columnSchemas.add(phonetype); ColumnSchema serialname = new ColumnSchema(); @@ -242,6 +246,7 @@ public class StoreCreator { serialname.setColumnUniqueId(UUID.randomUUID().toString()); serialname.setDimensionColumn(true); serialname.setColumnGroup(6); + serialname.setSortColumn(true); columnSchemas.add(serialname); ColumnSchema salary = new ColumnSchema();