http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java new file mode 100644 index 0000000..276e5db --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java @@ -0,0 +1,882 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.scan.executor.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.carbondata.core.cache.Cache; +import org.apache.carbondata.core.cache.CacheProvider; +import org.apache.carbondata.core.cache.CacheType; +import org.apache.carbondata.core.cache.dictionary.Dictionary; +import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.block.SegmentProperties; +import org.apache.carbondata.core.keygenerator.KeyGenException; +import org.apache.carbondata.core.keygenerator.KeyGenerator; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.CarbonMetadata; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; +import org.apache.carbondata.core.scan.complextypes.ArrayQueryType; +import org.apache.carbondata.core.scan.complextypes.PrimitiveQueryType; +import org.apache.carbondata.core.scan.complextypes.StructQueryType; +import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; +import org.apache.carbondata.core.scan.expression.ColumnExpression; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.filter.GenericQueryType; +import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; +import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; +import org.apache.carbondata.core.scan.model.QueryDimension; +import org.apache.carbondata.core.scan.model.QueryMeasure; +import org.apache.carbondata.core.scan.model.QueryModel; +import org.apache.carbondata.core.util.CarbonUtil; + +import org.apache.commons.lang3.ArrayUtils; + +/** + * Utility class for query execution + */ +public class QueryUtil { + + /** + * Below method will be used to get the masked byte range based on the query + * dimension. It will give the range in the mdkey. This will be used to get + * the actual key array from masked mdkey + * + * @param queryDimensions query dimension selected in query + * @param keyGenerator key generator + * @return masked key + */ + public static int[] getMaskedByteRange(List<QueryDimension> queryDimensions, + KeyGenerator keyGenerator) { + Set<Integer> byteRangeSet = new TreeSet<Integer>(); + int[] byteRange = null; + for (int i = 0; i < queryDimensions.size(); i++) { + + // as no dictionary column and complex type columns + // are not selected in the mdkey + // so we will not select the those dimension for calculating the + // range + if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) { + continue; + } + // get the offset of the dimension in the mdkey + byteRange = + keyGenerator.getKeyByteOffsets(queryDimensions.get(i).getDimension().getKeyOrdinal()); + for (int j = byteRange[0]; j <= byteRange[1]; j++) { + byteRangeSet.add(j); + } + } + int[] maksedByteRange = new int[byteRangeSet.size()]; + int index = 0; + Iterator<Integer> iterator = byteRangeSet.iterator(); + // add the masked byte range + while (iterator.hasNext()) { + maksedByteRange[index++] = iterator.next(); + } + return maksedByteRange; + } + + public static int[] getMaskedByteRangeBasedOrdinal(List<Integer> ordinals, + KeyGenerator keyGenerator) { + Set<Integer> byteRangeSet = new TreeSet<Integer>(); + int[] byteRange = null; + for (int i = 0; i < ordinals.size(); i++) { + + // get the offset of the dimension in the mdkey + byteRange = keyGenerator.getKeyByteOffsets(ordinals.get(i)); + for (int j = byteRange[0]; j <= byteRange[1]; j++) { + byteRangeSet.add(j); + } + } + int[] maksedByteRange = new int[byteRangeSet.size()]; + int index = 0; + Iterator<Integer> iterator = byteRangeSet.iterator(); + // add the masked byte range + while (iterator.hasNext()) { + maksedByteRange[index++] = iterator.next(); + } + return maksedByteRange; + } + + /** + * Below method will return the max key based on the dimension ordinal + * + * @param keyOrdinalList + * @param generator + * @return + * @throws KeyGenException + */ + public static byte[] getMaxKeyBasedOnOrinal(List<Integer> keyOrdinalList, KeyGenerator generator) + throws KeyGenException { + long[] max = new long[generator.getDimCount()]; + Arrays.fill(max, 0L); + + for (int i = 0; i < keyOrdinalList.size(); i++) { + // adding for dimension which is selected in query + max[keyOrdinalList.get(i)] = Long.MAX_VALUE; + } + return generator.generateKey(max); + } + + /** + * To get the max key based on dimensions. i.e. all other dimensions will be + * set to 0 bits and the required query dimension will be masked with all + * LONG.MAX so that we can mask key and then compare while aggregating This + * can be useful during filter query when only few dimensions were selected + * out of row group + * + * @param queryDimensions dimension selected in query + * @param generator key generator + * @return max key for dimension + * @throws KeyGenException if any problem while generating the key + */ + public static byte[] getMaxKeyBasedOnDimensions(List<QueryDimension> queryDimensions, + KeyGenerator generator) throws KeyGenException { + long[] max = new long[generator.getDimCount()]; + Arrays.fill(max, 0L); + + for (int i = 0; i < queryDimensions.size(); i++) { + // as no dictionary column and complex type columns + // are not selected in the mdkey + // so we will not select the those dimension for calculating the + // range + if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) { + continue; + } + // adding for dimension which is selected in query + max[queryDimensions.get(i).getDimension().getKeyOrdinal()] = Long.MAX_VALUE; + } + + return generator.generateKey(max); + } + + /** + * Below method will be used to get the masked key for query + * + * @param keySize size of the masked key + * @param maskedKeyRanges masked byte range + * @return masked bytes + */ + public static int[] getMaskedByte(int keySize, int[] maskedKeyRanges) { + int[] maskedKey = new int[keySize]; + // all the non selected dimension will be filled with -1 + Arrays.fill(maskedKey, -1); + for (int i = 0; i < maskedKeyRanges.length; i++) { + maskedKey[maskedKeyRanges[i]] = i; + } + return maskedKey; + } + + /** + * Below method will be used to get the dimension block index in file based + * on query dimension + * + * @param queryDimensions query dimension + * @param dimensionOrdinalToBlockMapping mapping of dimension block in file to query dimension + * @return block index of file + */ + public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions, + Map<Integer, Integer> dimensionOrdinalToBlockMapping, + List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions, + List<Integer> allProjectionListDimensionIndexes) { + // using set as in row group columns will point to same block + Set<Integer> dimensionBlockIndex = new HashSet<Integer>(); + Set<Integer> filterDimensionOrdinal = getFilterDimensionOrdinal(filterDimensions); + int blockIndex = 0; + for (int i = 0; i < queryDimensions.size(); i++) { + if (queryDimensions.get(i).getDimension().hasEncoding(Encoding.IMPLICIT)) { + continue; + } + allProjectionListDimensionIndexes.add(queryDimensions.get(i).getDimension().getOrdinal()); + + if (!filterDimensionOrdinal.contains(queryDimensions.get(i).getDimension().getOrdinal())) { + blockIndex = + dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal()); + dimensionBlockIndex.add(blockIndex); + if (queryDimensions.get(i).getDimension().numberOfChild() > 0) { + addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension()); + } + } + } + for (int i = 0; i < customAggregationDimension.size(); i++) { + blockIndex = + dimensionOrdinalToBlockMapping.get(customAggregationDimension.get(i).getOrdinal()); + // not adding the children dimension as dimension aggregation + // is not push down in case of complex dimension + dimensionBlockIndex.add(blockIndex); + } + int[] dimensionIndex = ArrayUtils + .toPrimitive(dimensionBlockIndex.toArray(new Integer[dimensionBlockIndex.size()])); + Arrays.sort(dimensionIndex); + return dimensionIndex; + } + + /** + * Below method will be used to add the children block index + * this will be basically for complex dimension which will have children + * + * @param blockIndexes block indexes + * @param dimension parent dimension + */ + private static void addChildrenBlockIndex(Set<Integer> blockIndexes, CarbonDimension dimension) { + for (int i = 0; i < dimension.numberOfChild(); i++) { + addChildrenBlockIndex(blockIndexes, dimension.getListOfChildDimensions().get(i)); + blockIndexes.add(dimension.getListOfChildDimensions().get(i).getOrdinal()); + } + } + + /** + * Below method will be used to get the dictionary mapping for all the + * dictionary encoded dimension present in the query + * + * @param queryDimensions query dimension present in the query this will be used to + * convert the result from surrogate key to actual data + * @param absoluteTableIdentifier absolute table identifier + * @return dimension unique id to its dictionary map + * @throws IOException + */ + public static Map<String, Dictionary> getDimensionDictionaryDetail( + List<QueryDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions, + AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException { + // to store dimension unique column id list, this is required as + // dimension can be present in + // query dimension, as well as some aggregation function will be applied + // in the same dimension + // so we need to get only one instance of dictionary + // direct dictionary skip is done only for the dictionary lookup + Set<String> dictionaryDimensionFromQuery = new HashSet<String>(); + for (int i = 0; i < queryDimensions.size(); i++) { + List<Encoding> encodingList = queryDimensions.get(i).getDimension().getEncoder(); + // TODO need to remove the data type check for parent column in complex type no need to + // write encoding dictionary + if (CarbonUtil.hasEncoding(encodingList, Encoding.DICTIONARY) && !CarbonUtil + .hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY) && !CarbonUtil + .hasEncoding(encodingList, Encoding.IMPLICIT)) { + + if (queryDimensions.get(i).getDimension().numberOfChild() == 0) { + dictionaryDimensionFromQuery.add(queryDimensions.get(i).getDimension().getColumnId()); + } + if (queryDimensions.get(i).getDimension().numberOfChild() > 0) { + getChildDimensionDictionaryDetail(queryDimensions.get(i).getDimension(), + dictionaryDimensionFromQuery); + } + } + } + Iterator<CarbonDimension> iterator = filterComplexDimensions.iterator(); + while (iterator.hasNext()) { + getChildDimensionDictionaryDetail(iterator.next(), dictionaryDimensionFromQuery); + } + // converting to list as api exposed needed list which i think + // is not correct + List<String> dictionaryColumnIdList = + new ArrayList<String>(dictionaryDimensionFromQuery.size()); + dictionaryColumnIdList.addAll(dictionaryDimensionFromQuery); + return getDictionaryMap(dictionaryColumnIdList, absoluteTableIdentifier); + } + + /** + * Below method will be used to fill the children dimension column id + * + * @param queryDimensions query dimension + * @param dictionaryDimensionFromQuery dictionary dimension for query + */ + private static void getChildDimensionDictionaryDetail(CarbonDimension queryDimensions, + Set<String> dictionaryDimensionFromQuery) { + for (int j = 0; j < queryDimensions.numberOfChild(); j++) { + List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder(); + if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) { + getChildDimensionDictionaryDetail(queryDimensions.getListOfChildDimensions().get(j), + dictionaryDimensionFromQuery); + } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) { + dictionaryDimensionFromQuery + .add(queryDimensions.getListOfChildDimensions().get(j).getColumnId()); + } + } + } + + /** + * Below method will be used to get the column id to its dictionary mapping + * + * @param dictionaryColumnIdList dictionary column list + * @param absoluteTableIdentifier absolute table identifier + * @return dictionary mapping + * @throws IOException + */ + private static Map<String, Dictionary> getDictionaryMap(List<String> dictionaryColumnIdList, + AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException { + // this for dictionary unique identifier + List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers = + getDictionaryColumnUniqueIdentifierList(dictionaryColumnIdList, + absoluteTableIdentifier.getCarbonTableIdentifier()); + CacheProvider cacheProvider = CacheProvider.getInstance(); + Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider + .createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath()); + + List<Dictionary> columnDictionaryList = + forwardDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers); + Map<String, Dictionary> columnDictionaryMap = new HashMap<>(columnDictionaryList.size()); + for (int i = 0; i < dictionaryColumnUniqueIdentifiers.size(); i++) { + // TODO: null check for column dictionary, if cache size is less it + // might return null here, in that case throw exception + columnDictionaryMap.put(dictionaryColumnIdList.get(i), columnDictionaryList.get(i)); + } + return columnDictionaryMap; + } + + /** + * Below method will be used to get the dictionary column unique identifier + * + * @param dictionaryColumnIdList dictionary + * @param carbonTableIdentifier + * @return + */ + private static List<DictionaryColumnUniqueIdentifier> getDictionaryColumnUniqueIdentifierList( + List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier) { + CarbonTable carbonTable = + CarbonMetadata.getInstance().getCarbonTable(carbonTableIdentifier.getTableUniqueName()); + List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers = + new ArrayList<>(dictionaryColumnIdList.size()); + for (String columnId : dictionaryColumnIdList) { + CarbonDimension dimension = CarbonMetadata.getInstance() + .getCarbonDimensionBasedOnColIdentifier(carbonTable, columnId); + if (dimension != null) { + dictionaryColumnUniqueIdentifiers.add( + new DictionaryColumnUniqueIdentifier( + carbonTableIdentifier, + dimension.getColumnIdentifier(), + dimension.getDataType() + ) + ); + } + } + return dictionaryColumnUniqueIdentifiers; + } + + /** + * Below method will used to get the method will be used to get the measure + * block indexes to be read from the file + * + * @param queryMeasures query measure + * @param expressionMeasure measure present in the expression + * @param ordinalToBlockIndexMapping measure ordinal to block mapping + * @return block indexes + */ + public static int[] getMeasureBlockIndexes(List<QueryMeasure> queryMeasures, + List<CarbonMeasure> expressionMeasure, Map<Integer, Integer> ordinalToBlockIndexMapping, + Set<CarbonMeasure> filterMeasures) { + Set<Integer> measureBlockIndex = new HashSet<Integer>(); + Set<Integer> filterMeasureOrdinal = getFilterMeasureOrdinal(filterMeasures); + for (int i = 0; i < queryMeasures.size(); i++) { + if (!filterMeasureOrdinal.contains(queryMeasures.get(i).getMeasure().getOrdinal())) { + measureBlockIndex + .add(ordinalToBlockIndexMapping.get(queryMeasures.get(i).getMeasure().getOrdinal())); + } + } + for (int i = 0; i < expressionMeasure.size(); i++) { + measureBlockIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal())); + } + int[] measureIndexes = + ArrayUtils.toPrimitive(measureBlockIndex.toArray(new Integer[measureBlockIndex.size()])); + Arrays.sort(measureIndexes); + return measureIndexes; + } + + /** + * Below method will be used to get mapping whether dimension is present in + * order by or not + * + * @param sortedDimensions sort dimension present in order by query + * @param queryDimensions query dimension + * @return sort dimension indexes + */ + public static byte[] getSortDimensionIndexes(List<QueryDimension> sortedDimensions, + List<QueryDimension> queryDimensions) { + byte[] sortedDims = new byte[queryDimensions.size()]; + int indexOf = 0; + for (int i = 0; i < sortedDims.length; i++) { + indexOf = sortedDimensions.indexOf(queryDimensions.get(i)); + if (indexOf > -1) { + sortedDims[i] = 1; + } + } + return sortedDims; + } + + /** + * Below method will be used to get the mapping of block index and its + * restructuring info + * + * @param queryDimensions query dimension from query model + * @param segmentProperties segment properties + * @return map of block index to its restructuring info + * @throws KeyGenException if problem while key generation + */ + public static Map<Integer, KeyStructureInfo> getColumnGroupKeyStructureInfo( + List<QueryDimension> queryDimensions, SegmentProperties segmentProperties) + throws KeyGenException { + Map<Integer, KeyStructureInfo> rowGroupToItsRSInfo = new HashMap<Integer, KeyStructureInfo>(); + // get column group id and its ordinal mapping of column group + Map<Integer, List<Integer>> columnGroupAndItsOrdinalMappingForQuery = + getColumnGroupAndItsOrdinalMapping(queryDimensions); + Map<Integer, KeyGenerator> columnGroupAndItsKeygenartor = + segmentProperties.getColumnGroupAndItsKeygenartor(); + + Iterator<Entry<Integer, List<Integer>>> iterator = + columnGroupAndItsOrdinalMappingForQuery.entrySet().iterator(); + KeyStructureInfo restructureInfos = null; + while (iterator.hasNext()) { + Entry<Integer, List<Integer>> next = iterator.next(); + KeyGenerator keyGenerator = columnGroupAndItsKeygenartor.get(next.getKey()); + restructureInfos = new KeyStructureInfo(); + // sort the ordinal + List<Integer> ordinal = next.getValue(); + List<Integer> mdKeyOrdinal = new ArrayList<Integer>(); + //Un sorted + List<Integer> mdKeyOrdinalForQuery = new ArrayList<Integer>(); + for (Integer ord : ordinal) { + mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord)); + mdKeyOrdinalForQuery.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord)); + } + Collections.sort(mdKeyOrdinal); + // get the masked byte range for column group + int[] maskByteRanges = getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator); + // max key for column group + byte[] maxKey = getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator); + restructureInfos.setKeyGenerator(keyGenerator); + restructureInfos.setMaskByteRanges(maskByteRanges); + restructureInfos.setMaxKey(maxKey); + restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils + .toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()]))); + rowGroupToItsRSInfo + .put(segmentProperties.getDimensionOrdinalToBlockMapping().get(ordinal.get(0)), + restructureInfos); + } + return rowGroupToItsRSInfo; + } + + /** + * return true if given key is found in array + * + * @param data + * @param key + * @return + */ + public static boolean searchInArray(int[] data, int key) { + for (int i = 0; i < data.length; i++) { + if (key == data[i]) { + return true; + } + } + return false; + } + + /** + * Below method will be used to create a mapping of column group columns + * this mapping will have column group id to all the dimension ordinal + * present in the column group This mapping will be used during query + * execution, to create a mask key for the column group dimension which will + * be used in aggregation and filter query as column group dimension will be + * stored in bit level + */ + private static Map<Integer, List<Integer>> getColumnGroupAndItsOrdinalMapping( + List<QueryDimension> origdimensions) { + + List<QueryDimension> dimensions = new ArrayList<QueryDimension>(origdimensions.size()); + dimensions.addAll(origdimensions); + /** + * sort based on column group id + */ + Collections.sort(dimensions, new Comparator<QueryDimension>() { + + @Override public int compare(QueryDimension o1, QueryDimension o2) { + return Integer + .compare(o1.getDimension().columnGroupId(), o2.getDimension().columnGroupId()); + } + }); + // list of row groups this will store all the row group column + Map<Integer, List<Integer>> columnGroupAndItsOrdinalsMapping = + new HashMap<Integer, List<Integer>>(); + // to store a column group + List<Integer> currentColumnGroup = null; + // current index + int index = 0; + // previous column group to check all the column of row id has bee + // selected + int prvColumnGroupId = -1; + while (index < dimensions.size()) { + // if dimension group id is not zero and it is same as the previous + // column group id + // then we need to add ordinal of that column as it belongs to same + // column group + if (dimensions.get(index).getDimension().hasEncoding(Encoding.IMPLICIT)) { + index++; + continue; + } else if (!dimensions.get(index).getDimension().isColumnar() + && dimensions.get(index).getDimension().columnGroupId() == prvColumnGroupId + && null != currentColumnGroup) { + currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal()); + } + + // if dimension is not a columnar then it is column group column + else if (!dimensions.get(index).getDimension().isColumnar()) { + currentColumnGroup = new ArrayList<Integer>(); + columnGroupAndItsOrdinalsMapping + .put(dimensions.get(index).getDimension().columnGroupId(), currentColumnGroup); + currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal()); + } + // update the row id every time,this is required to group the + // columns + // of the same row group + prvColumnGroupId = dimensions.get(index).getDimension().columnGroupId(); + index++; + } + return columnGroupAndItsOrdinalsMapping; + } + + /** + * Below method will be used to get masked byte + * + * @param data actual data + * @param maxKey max key + * @param maskByteRanges mask byte range + * @param byteCount + * @return masked byte + */ + public static byte[] getMaskedKey(byte[] data, byte[] maxKey, int[] maskByteRanges, + int byteCount) { + byte[] maskedKey = new byte[byteCount]; + int counter = 0; + int byteRange = 0; + for (int i = 0; i < byteCount; i++) { + byteRange = maskByteRanges[i]; + if (byteRange != -1) { + maskedKey[counter++] = (byte) (data[byteRange] & maxKey[byteRange]); + } + } + return maskedKey; + } + + /** + * Below method will be used to fill block indexes of the query dimension + * which will be used in creating a output row Here is method we are passing + * two list which store the indexes one for dictionary column other for not + * dictionary column. This is done for specific purpose so that in one + * iteration we will be able to fill both type dimension block indexes + * + * @param queryDimensions dimension present in the query + * @param columnOrdinalToBlockIndexMapping column ordinal to block index mapping + * @param dictionaryDimensionBlockIndex list to store dictionary column block indexes + * @param noDictionaryDimensionBlockIndex list to store no dictionary block indexes + */ + public static void fillQueryDimensionsBlockIndexes(List<QueryDimension> queryDimensions, + Map<Integer, Integer> columnOrdinalToBlockIndexMapping, + Set<Integer> dictionaryDimensionBlockIndex, List<Integer> noDictionaryDimensionBlockIndex) { + for (QueryDimension queryDimension : queryDimensions) { + if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY) + && queryDimension.getDimension().numberOfChild() == 0) { + dictionaryDimensionBlockIndex + .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal())); + } else if ( + !CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.IMPLICIT) + && queryDimension.getDimension().numberOfChild() == 0) { + noDictionaryDimensionBlockIndex + .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal())); + } + } + } + + /** + * Below method will be used to resolve the query model + * resolve will be setting the actual dimension and measure object + * as from driver only column name will be passes to avoid the heavy object + * serialization + * + * @param queryModel query model + */ + public static void resolveQueryModel(QueryModel queryModel) { + CarbonMetadata.getInstance().addCarbonTable(queryModel.getTable()); + // TODO need to load the table from table identifier + CarbonTable carbonTable = queryModel.getTable(); + String tableName = + queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName(); + // resolve query dimension + for (QueryDimension queryDimension : queryModel.getQueryDimension()) { + queryDimension + .setDimension(carbonTable.getDimensionByName(tableName, queryDimension.getColumnName())); + } + // resolve query measure + for (QueryMeasure queryMeasure : queryModel.getQueryMeasures()) { + // in case of count start column name will be count * so + // first need to check any measure is present or not and as if measure + // if measure is present and if first measure is not a default + // measure than add measure otherwise + // than add first dimension as a measure + //as currently if measure is not present then + //we are adding default measure so first condition will + //never come false but if in future we can remove so not removing first if check + if (queryMeasure.getColumnName().equals("count(*)")) { + if (carbonTable.getMeasureByTableName(tableName).size() > 0 && !carbonTable + .getMeasureByTableName(tableName).get(0).getColName() + .equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)) { + queryMeasure.setMeasure(carbonTable.getMeasureByTableName(tableName).get(0)); + } else { + CarbonMeasure dummyMeasure = new CarbonMeasure( + carbonTable.getDimensionByTableName(tableName).get(0).getColumnSchema(), 0); + queryMeasure.setMeasure(dummyMeasure); + } + } else { + queryMeasure + .setMeasure(carbonTable.getMeasureByName(tableName, queryMeasure.getColumnName())); + } + } + } + + /** + * below method will be used to get the actual type aggregator + * + * @param aggType + * @return index in aggrgetor + */ + public static int[] getActualTypeIndex(List<String> aggType) { + List<Integer> indexList = new ArrayList<Integer>(); + for (int i = 0; i < aggType.size(); i++) { + if (!CarbonCommonConstants.SUM.equals(aggType.get(i)) && !CarbonCommonConstants.AVERAGE + .equals(aggType.get(i))) { + indexList.add(i); + } + } + return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()])); + } + + /** + * Below method will be used to get the key structure for the column group + * + * @param segmentProperties segment properties + * @param dimColumnEvaluatorInfo dimension evaluator info + * @return key structure info for column group dimension + * @throws KeyGenException + */ + public static KeyStructureInfo getKeyStructureInfo(SegmentProperties segmentProperties, + DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) throws KeyGenException { + int colGrpId = getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex()); + KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId); + List<Integer> mdKeyOrdinal = new ArrayList<Integer>(); + + mdKeyOrdinal.add(segmentProperties + .getColumnGroupMdKeyOrdinal(colGrpId, dimColumnEvaluatorInfo.getColumnIndex())); + int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator); + byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator); + KeyStructureInfo restructureInfos = new KeyStructureInfo(); + restructureInfos.setKeyGenerator(keyGenerator); + restructureInfos.setMaskByteRanges(maskByteRanges); + restructureInfos.setMaxKey(maxKey); + return restructureInfos; + } + + /** + * Below method will be used to get the column group id based on the ordinal + * + * @param segmentProperties segment properties + * @param ordinal ordinal to be searched + * @return column group id + */ + public static int getColumnGroupId(SegmentProperties segmentProperties, int ordinal) { + int[][] columnGroups = segmentProperties.getColumnGroups(); + int colGrpId = -1; + for (int i = 0; i < columnGroups.length; i++) { + if (columnGroups[i].length > 1) { + colGrpId++; + if (QueryUtil.searchInArray(columnGroups[i], ordinal)) { + break; + } + } + } + return colGrpId; + } + + /** + * Below method will be used to get the map of for complex dimension and its type + * which will be used to during query execution to + * + * @param queryDimensions complex dimension in query + * @param dimensionToBlockIndexMap dimension to block index in file map + * @return complex dimension and query type + */ + public static Map<Integer, GenericQueryType> getComplexDimensionsMap( + List<QueryDimension> queryDimensions, Map<Integer, Integer> dimensionToBlockIndexMap, + int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap, + Set<CarbonDimension> filterDimensions) { + Map<Integer, GenericQueryType> complexTypeMap = new HashMap<Integer, GenericQueryType>(); + for (QueryDimension dimension : queryDimensions) { + CarbonDimension actualDimension = dimension.getDimension(); + if (actualDimension.getNumberOfChild() == 0) { + continue; + } + fillParentDetails(dimensionToBlockIndexMap, actualDimension, complexTypeMap, + eachComplexColumnValueSize, columnIdToDictionaryMap); + } + if (null != filterDimensions) { + for (CarbonDimension filterDimension : filterDimensions) { + // do not fill nay details for implicit dimension type + if (filterDimension.hasEncoding(Encoding.IMPLICIT)) { + continue; + } + fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap, + eachComplexColumnValueSize, columnIdToDictionaryMap); + } + } + return complexTypeMap; + } + + private static void fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap, + CarbonDimension dimension, Map<Integer, GenericQueryType> complexTypeMap, + int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap) { + int parentBlockIndex = dimensionToBlockIndexMap.get(dimension.getOrdinal()); + GenericQueryType parentQueryType = dimension.getDataType().equals(DataType.ARRAY) ? + new ArrayQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex) : + new StructQueryType(dimension.getColName(), dimension.getColName(), + dimensionToBlockIndexMap.get(dimension.getOrdinal())); + complexTypeMap.put(dimension.getOrdinal(), parentQueryType); + parentBlockIndex = + fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex, + dimension, parentQueryType); + } + + private static int fillChildrenDetails(int[] eachComplexColumnValueSize, + Map<String, Dictionary> columnIdToDictionaryMap, int parentBlockIndex, + CarbonDimension dimension, GenericQueryType parentQueryType) { + for (int i = 0; i < dimension.getNumberOfChild(); i++) { + switch (dimension.getListOfChildDimensions().get(i).getDataType()) { + case ARRAY: + parentQueryType.addChildren( + new ArrayQueryType(dimension.getListOfChildDimensions().get(i).getColName(), + dimension.getColName(), ++parentBlockIndex)); + break; + case STRUCT: + parentQueryType.addChildren( + new StructQueryType(dimension.getListOfChildDimensions().get(i).getColName(), + dimension.getColName(), ++parentBlockIndex)); + break; + default: + boolean isDirectDictionary = CarbonUtil + .hasEncoding(dimension.getListOfChildDimensions().get(i).getEncoder(), + Encoding.DIRECT_DICTIONARY); + parentQueryType.addChildren( + new PrimitiveQueryType(dimension.getListOfChildDimensions().get(i).getColName(), + dimension.getColName(), ++parentBlockIndex, + dimension.getListOfChildDimensions().get(i).getDataType(), + eachComplexColumnValueSize[dimension.getListOfChildDimensions().get(i) + .getComplexTypeOrdinal()], columnIdToDictionaryMap + .get(dimension.getListOfChildDimensions().get(i).getColumnId()), + isDirectDictionary)); + } + if (dimension.getListOfChildDimensions().get(i).getNumberOfChild() > 0) { + parentBlockIndex = fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, + parentBlockIndex, dimension.getListOfChildDimensions().get(i), parentQueryType); + } + } + return parentBlockIndex; + } + + public static void getAllFilterDimensions(FilterResolverIntf filterResolverTree, + Set<CarbonDimension> filterDimensions, Set<CarbonMeasure> filterMeasure) { + if (null == filterResolverTree) { + return; + } + List<ColumnExpression> dimensionResolvedInfos = new ArrayList<ColumnExpression>(); + Expression filterExpression = filterResolverTree.getFilterExpression(); + addColumnDimensions(filterExpression, filterDimensions, filterMeasure); + for (ColumnExpression info : dimensionResolvedInfos) { + if (info.isDimension() && info.getDimension().getNumberOfChild() > 0) { + filterDimensions.add(info.getDimension()); + } + } + } + + /** + * This method will check if a given expression contains a column expression + * recursively and add the dimension instance to the set which holds the dimension + * instances of the complex filter expressions. + */ + private static void addColumnDimensions(Expression expression, + Set<CarbonDimension> filterDimensions, Set<CarbonMeasure> filterMeasure) { + if (null != expression && expression instanceof ColumnExpression) { + if (((ColumnExpression) expression).isDimension()) { + filterDimensions.add(((ColumnExpression) expression).getDimension()); + } else { + filterMeasure.add((CarbonMeasure) ((ColumnExpression) expression).getCarbonColumn()); + } + return; + } else if (null != expression) { + for (Expression child : expression.getChildren()) { + addColumnDimensions(child, filterDimensions, filterMeasure); + } + } + } + + private static Set<Integer> getFilterMeasureOrdinal(Set<CarbonMeasure> filterMeasures) { + Set<Integer> filterMeasuresOrdinal = new HashSet<>(); + for (CarbonMeasure filterMeasure : filterMeasures) { + filterMeasuresOrdinal.add(filterMeasure.getOrdinal()); + } + return filterMeasuresOrdinal; + } + + private static Set<Integer> getFilterDimensionOrdinal(Set<CarbonDimension> filterDimensions) { + Set<Integer> filterDimensionsOrdinal = new HashSet<>(); + for (CarbonDimension filterDimension : filterDimensions) { + filterDimensionsOrdinal.add(filterDimension.getOrdinal()); + getChildDimensionOrdinal(filterDimension, filterDimensionsOrdinal); + } + return filterDimensionsOrdinal; + } + + /** + * Below method will be used to fill the children dimension column id + */ + private static void getChildDimensionOrdinal(CarbonDimension queryDimensions, + Set<Integer> filterDimensionsOrdinal) { + for (int j = 0; j < queryDimensions.numberOfChild(); j++) { + List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder(); + if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) { + getChildDimensionOrdinal(queryDimensions.getListOfChildDimensions().get(j), + filterDimensionsOrdinal); + } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) { + filterDimensionsOrdinal.add(queryDimensions.getListOfChildDimensions().get(j).getOrdinal()); + } + } + } +}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java new file mode 100644 index 0000000..fe999d7 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.scan.executor.util; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; +import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo; +import org.apache.carbondata.core.scan.model.QueryDimension; +import org.apache.carbondata.core.scan.model.QueryMeasure; + +/** + * Utility class for restructuring + */ +public class RestructureUtil { + + /** + * Below method will be used to get the updated query dimension updation + * means, after restructuring some dimension will be not present in older + * table blocks in that case we need to select only those dimension out of + * query dimension which is present in the current table block + * + * @param queryDimensions + * @param tableBlockDimensions + * @return list of query dimension which is present in the table block + */ + public static List<QueryDimension> getUpdatedQueryDimension(List<QueryDimension> queryDimensions, + List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension) { + List<QueryDimension> presentDimension = + new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + // selecting only those dimension which is present in the query + for (QueryDimension queryDimension : queryDimensions) { + if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) { + presentDimension.add(queryDimension); + } else { + for (CarbonDimension tableDimension : tableBlockDimensions) { + if (tableDimension.equals(queryDimension.getDimension())) { + presentDimension.add(queryDimension); + } + } + } + } + for (QueryDimension queryDimimension : queryDimensions) { + for (CarbonDimension tableDimension : tableComplexDimension) { + if (tableDimension.equals(queryDimimension.getDimension())) { + presentDimension.add(queryDimimension); + } + } + } + return presentDimension; + } + + /** + * Below method is to add dimension children for complex type dimension as + * internally we are creating dimension column for each each complex + * dimension so when complex query dimension request will come in the query, + * we need to add its children as it is hidden from the user For example if + * complex dimension is of Array of String[2] so we are storing 3 dimension + * and when user will query for complex type i.e. array type we need to add + * its children and then we will read respective block and create a tuple + * based on all three dimension + * + * @param queryDimensions current query dimensions + * @param tableBlockDimensions dimensions which is present in the table block + * @return updated dimension(after adding complex type children) + */ + public static List<CarbonDimension> addChildrenForComplexTypeDimension( + List<CarbonDimension> queryDimensions, List<CarbonDimension> tableBlockDimensions) { + List<CarbonDimension> updatedQueryDimension = new ArrayList<CarbonDimension>(); + int numberOfChildren = 0; + for (CarbonDimension queryDimension : queryDimensions) { + // if number of child is zero, then it is not a complex dimension + // so directly add it query dimension + if (queryDimension.numberOfChild() == 0) { + updatedQueryDimension.add(queryDimension); + } + // if number of child is more than 1 then add all its children + numberOfChildren = queryDimension.getOrdinal() + queryDimension.numberOfChild(); + for (int j = queryDimension.getOrdinal(); j < numberOfChildren; j++) { + updatedQueryDimension.add(tableBlockDimensions.get(j)); + } + } + return updatedQueryDimension; + } + + /** + * Below method will be used to get the aggregator info object + * in this method some of the properties which will be extracted + * from query measure and current block measures will be set + * + * @param queryMeasures measures present in query + * @param currentBlockMeasures current block measures + * @return aggregator info + */ + public static AggregatorInfo getAggregatorInfos(List<QueryMeasure> queryMeasures, + List<CarbonMeasure> currentBlockMeasures) { + AggregatorInfo aggregatorInfos = new AggregatorInfo(); + int numberOfMeasureInQuery = queryMeasures.size(); + int[] measureOrdinals = new int[numberOfMeasureInQuery]; + Object[] defaultValues = new Object[numberOfMeasureInQuery]; + boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery]; + int index = 0; + for (QueryMeasure queryMeasure : queryMeasures) { + measureOrdinals[index] = queryMeasure.getMeasure().getOrdinal(); + // if query measure exists in current dimension measures + // then setting measure exists is true + // otherwise adding a default value of a measure + if (currentBlockMeasures.contains(queryMeasure.getMeasure())) { + measureExistsInCurrentBlock[index] = true; + } else { + defaultValues[index] = queryMeasure.getMeasure().getDefaultValue(); + } + index++; + } + aggregatorInfos.setDefaultValues(defaultValues); + aggregatorInfos.setMeasureOrdinals(measureOrdinals); + aggregatorInfos.setMeasureExists(measureExistsInCurrentBlock); + return aggregatorInfos; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java new file mode 100644 index 0000000..8885841 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression; + +public abstract class BinaryExpression extends Expression { + + private static final long serialVersionUID = 1L; + protected Expression left; + protected Expression right; + + public BinaryExpression(Expression left, Expression right) { + this.left = left; + this.right = right; + children.add(left); + children.add(right); + } + + public Expression getLeft() { + return left; + } + + public Expression getRight() { + return right; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java new file mode 100644 index 0000000..291b961 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression; + +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; +import org.apache.carbondata.core.scan.filter.intf.ExpressionType; +import org.apache.carbondata.core.scan.filter.intf.RowIntf; + +public class ColumnExpression extends LeafExpression { + + private static final long serialVersionUID = 1L; + + private String columnName; + + private boolean isDimension; + + private int colIndex = -1; + + private DataType dataType; + + private CarbonDimension dimension; + + private CarbonColumn carbonColumn; + + public ColumnExpression(String columnName, DataType dataType) { + this.columnName = columnName; + this.dataType = dataType; + + } + + public CarbonDimension getDimension() { + return dimension; + } + + public void setDimension(CarbonDimension dimension) { + this.dimension = dimension; + } + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + + public boolean isDimension() { + return isDimension; + } + + public void setDimension(boolean isDimension) { + this.isDimension = isDimension; + } + + public int getColIndex() { + return colIndex; + } + + public void setColIndex(int colIndex) { + this.colIndex = colIndex; + } + + public DataType getDataType() { + return dataType; + } + + public void setDataType(DataType dataType) { + this.dataType = dataType; + } + + @Override public ExpressionResult evaluate(RowIntf value) { + ExpressionResult expressionResult = + new ExpressionResult(dataType, (null == value ? null : value.getVal(colIndex))); + return expressionResult; + } + + @Override public ExpressionType getFilterExpressionType() { + // TODO Auto-generated method stub + return null; + } + + @Override public String getString() { + // TODO Auto-generated method stub + return "ColumnExpression(" + columnName + ')'; + } + + public CarbonColumn getCarbonColumn() { + return carbonColumn; + } + + public void setCarbonColumn(CarbonColumn carbonColumn) { + this.carbonColumn = carbonColumn; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java new file mode 100644 index 0000000..a816643 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; +import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; +import org.apache.carbondata.core.scan.filter.intf.ExpressionType; +import org.apache.carbondata.core.scan.filter.intf.RowIntf; + +public abstract class Expression implements Serializable { + + private static final long serialVersionUID = -7568676723039530713L; + protected List<Expression> children = + new ArrayList<Expression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + + public abstract ExpressionResult evaluate(RowIntf value) + throws FilterUnsupportedException, FilterIllegalMemberException; + + public abstract ExpressionType getFilterExpressionType(); + + public List<Expression> getChildren() { + return children; + } + + public abstract String getString(); + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java new file mode 100644 index 0000000..8622937 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java @@ -0,0 +1,545 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additiona l information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; +import org.apache.carbondata.core.util.CarbonProperties; + +public class ExpressionResult implements Comparable<ExpressionResult> { + + protected DataType dataType; + + protected Object value; + + private List<ExpressionResult> expressionResults; + + private boolean isLiteral = false; + + public ExpressionResult(DataType dataType, Object value) { + this.dataType = dataType; + this.value = value; + } + + public ExpressionResult(DataType dataType, Object value, boolean isLiteral) { + this(dataType, value); + this.isLiteral = isLiteral; + } + + public ExpressionResult(List<ExpressionResult> expressionResults) { + this.expressionResults = expressionResults; + } + + public void set(DataType dataType, Object value) { + this.dataType = dataType; + this.value = value; + this.expressionResults = null; + } + + public DataType getDataType() { + return dataType; + } + + public Integer getInt() throws FilterIllegalMemberException { + if (value == null) { + return null; + } + try { + switch (this.getDataType()) { + case STRING: + try { + return Integer.parseInt(value.toString()); + } catch (NumberFormatException e) { + throw new FilterIllegalMemberException(e); + } + case SHORT: + return ((Short) value).intValue(); + case INT: + case DOUBLE: + if (value instanceof Double) { + return ((Double) value).intValue(); + } + if (value instanceof Long) { + return ((Long) value).intValue(); + } + return (Integer) value; + case DATE: + if (value instanceof java.sql.Date) { + return (int) (((java.sql.Date) value).getTime()); + } else { + return (Integer) value; + } + case TIMESTAMP: + if (value instanceof Timestamp) { + return (int) (((Timestamp) value).getTime()); + } else { + if (isLiteral) { + Long l = (Long) value /1000; + return l.intValue(); + } + return (Integer) value; + } + default: + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to integer type value"); + } + + } catch (ClassCastException e) { + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Integer type value"); + } + } + + public Short getShort() throws FilterIllegalMemberException { + if (value == null) { + return null; + } + try { + switch (this.getDataType()) { + case STRING: + try { + return Short.parseShort(value.toString()); + } catch (NumberFormatException e) { + throw new FilterIllegalMemberException(e); + } + case SHORT: + case INT: + case DOUBLE: + + if (value instanceof Double) { + return ((Double) value).shortValue(); + } else if (value instanceof Integer) { + return ((Integer) value).shortValue(); + } + return (Short) value; + + case DATE: + + if (value instanceof java.sql.Date) { + return (short) (((java.sql.Date) value).getTime()); + } else { + return (Short) value; + } + case TIMESTAMP: + + if (value instanceof Timestamp) { + return (short) (((Timestamp) value).getTime()); + } else { + if (isLiteral) { + Long l = ((long)value/1000); + return l.shortValue(); + } + return (Short) value; + } + + default: + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to integer type value"); + } + + } catch (ClassCastException e) { + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Integer type value"); + } + } + + public String getString() throws FilterIllegalMemberException { + if (value == null) { + return null; + } + try { + switch (this.getDataType()) { + case DATE: + case TIMESTAMP: + SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)); + if (value instanceof Timestamp) { + return parser.format((Timestamp) value); + } else if (value instanceof java.sql.Date) { + return parser.format((java.sql.Date) value); + } else if (value instanceof Long) { + if (isLiteral) { + return parser.format(new Timestamp((long) value/1000)); + } + return parser.format(new Timestamp((long) value)); + } else if (value instanceof Integer) { + return parser.format(new java.sql.Date((int)value)); + } + return value.toString(); + default: + return value.toString(); + } + } catch (Exception e) { + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to String type value"); + } + } + + public Double getDouble() throws FilterIllegalMemberException { + if (value == null) { + return null; + } + try { + switch (this.getDataType()) { + case STRING: + try { + return Double.parseDouble(value.toString()); + } catch (NumberFormatException e) { + throw new FilterIllegalMemberException(e); + } + case SHORT: + return ((Short) value).doubleValue(); + case INT: + return ((Integer) value).doubleValue(); + case LONG: + return ((Long) value).doubleValue(); + case DOUBLE: + return (Double) value; + case DATE: + if (value instanceof java.sql.Date) { + return (double) ((java.sql.Date) value).getTime(); + } else { + return (Double) (value); + } + case TIMESTAMP: + if (value instanceof Timestamp) { + return (double) ((Timestamp) value).getTime(); + } else { + if (isLiteral) { + Long l = (Long) value/1000; + return l.doubleValue(); + } + return (Double) (value); + } + default: + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to double type value"); + } + } catch (ClassCastException e) { + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Double type value"); + } + } + + public Long getLong() throws FilterIllegalMemberException { + if (value == null) { + return null; + } + try { + switch (this.getDataType()) { + case STRING: + try { + return Long.parseLong(value.toString()); + } catch (NumberFormatException e) { + throw new FilterIllegalMemberException(e); + } + case SHORT: + return ((Short) value).longValue(); + case INT: + return (Long) value; + case LONG: + return (Long) value; + case DOUBLE: + return (Long) value; + case DATE: + if (value instanceof java.sql.Date) { + return ((java.sql.Date) value).getTime(); + } else { + return (Long) value; + } + case TIMESTAMP: + if (value instanceof Timestamp) { + return ((Timestamp) value).getTime(); + } else { + return (Long) value; + } + default: + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Long type value"); + } + } catch (ClassCastException e) { + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Long type value"); + } + + } + + //Add to judge for BigDecimal + public BigDecimal getDecimal() throws FilterIllegalMemberException { + if (value == null) { + return null; + } + try { + switch (this.getDataType()) { + case STRING: + try { + return new BigDecimal(value.toString()); + } catch (NumberFormatException e) { + throw new FilterIllegalMemberException(e); + } + case SHORT: + return new BigDecimal((short) value); + case INT: + return new BigDecimal((int) value); + case LONG: + return new BigDecimal((long) value); + case DOUBLE: + return new BigDecimal(value.toString()); + case DECIMAL: + return new BigDecimal(value.toString()); + case DATE: + if (value instanceof java.sql.Date) { + return new BigDecimal(((java.sql.Date) value).getTime()); + } else { + return new BigDecimal((long) value); + } + case TIMESTAMP: + if (value instanceof Timestamp) { + return new BigDecimal(((Timestamp) value).getTime()); + } else { + if (isLiteral) { + return new BigDecimal((long)value/1000); + } + return new BigDecimal((long) value); + } + default: + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Long type value"); + } + } catch (ClassCastException e) { + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Long type value"); + } + + } + + public Long getTime() throws FilterIllegalMemberException { + if (value == null) { + return null; + } + try { + switch (this.getDataType()) { + case STRING: + // Currently the query engine layer only supports yyyy-MM-dd HH:mm:ss date format + // no matter in which format the data is been stored, so while retrieving the direct + // surrogate value for filter member first it should be converted in date form as per + // above format and needs to retrieve time stamp. + SimpleDateFormat parser = + new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT); + Date dateToStr; + try { + dateToStr = parser.parse(value.toString()); + return dateToStr.getTime(); + } catch (ParseException e) { + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Time/Long type value"); + } + case SHORT: + return ((Short) value).longValue(); + case INT: + case LONG: + return (Long) value; + case DOUBLE: + return (Long) value; + case DATE: + if (value instanceof java.sql.Date) { + return ((Date) value).getTime(); + } else { + return (Long) value; + } + case TIMESTAMP: + if (value instanceof Timestamp) { + return ((Timestamp) value).getTime(); + } else { + if (isLiteral) { + return (Long) value/1000; + } + return (Long) value; + } + default: + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Time/Long type value"); + } + } catch (ClassCastException e) { + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Time/Long type value"); + } + + } + + public Boolean getBoolean() throws FilterIllegalMemberException { + if (value == null) { + return null; + } + try { + switch (this.getDataType()) { + case STRING: + try { + return Boolean.parseBoolean(value.toString()); + } catch (NumberFormatException e) { + throw new FilterIllegalMemberException(e); + } + + case BOOLEAN: + return Boolean.parseBoolean(value.toString()); + + default: + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to boolean type value"); + } + } catch (ClassCastException e) { + throw new FilterIllegalMemberException( + "Cannot convert" + this.getDataType().name() + " to Boolean type value"); + } + } + + public List<ExpressionResult> getList() { + if (null == expressionResults) { + List<ExpressionResult> a = new ArrayList<ExpressionResult>(20); + a.add(new ExpressionResult(dataType, value, isLiteral)); + return a; + } else { + return expressionResults; + } + } + + public List<String> getListAsString() throws FilterIllegalMemberException { + List<String> evaluateResultListFinal = new ArrayList<String>(20); + List<ExpressionResult> evaluateResultList = getList(); + for (ExpressionResult result : evaluateResultList) { + String resultString = result.getString(); + if (resultString == null) { + evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL); + continue; + } + evaluateResultListFinal.add(resultString); + } + return evaluateResultListFinal; + } + + @Override public int hashCode() { + final int prime = 31; + int result = 1; + if (null != expressionResults) { + result = prime * result + expressionResults.hashCode(); + } else if (null != value) { + result = prime * result + value.toString().hashCode(); + } else { + result = prime * result + "".hashCode(); + } + + return result; + } + + @Override public boolean equals(Object obj) { + if (!(obj instanceof ExpressionResult)) { + return false; + } + if (this == obj) { + return true; + } + if (getClass() != obj.getClass()) { + return false; + } + ExpressionResult objToCompare = (ExpressionResult) obj; + boolean result = false; + if (this.value == objToCompare.value) { + return true; + } + try { + switch (this.getDataType()) { + case STRING: + result = this.getString().equals(objToCompare.getString()); + break; + case SHORT: + result = this.getShort().equals(objToCompare.getShort()); + break; + case INT: + result = this.getInt().equals(objToCompare.getInt()); + break; + case LONG: + case DATE: + case TIMESTAMP: + result = this.getLong().equals(objToCompare.getLong()); + break; + case DOUBLE: + result = this.getDouble().equals(objToCompare.getDouble()); + break; + case DECIMAL: + result = this.getDecimal().equals(objToCompare.getDecimal()); + break; + default: + break; + } + } catch (FilterIllegalMemberException ex) { + return false; + } + + return result; + } + + public boolean isNull() { + return value == null; + } + + @Override public int compareTo(ExpressionResult o) { + try { + switch (o.dataType) { + case SHORT: + case INT: + case LONG: + case DOUBLE: + Double d1 = this.getDouble(); + Double d2 = o.getDouble(); + return d1.compareTo(d2); + case DECIMAL: + java.math.BigDecimal val1 = this.getDecimal(); + java.math.BigDecimal val2 = o.getDecimal(); + return val1.compareTo(val2); + case DATE: + case TIMESTAMP: + SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)); + Date date1 = null; + Date date2 = null; + date1 = parser.parse(this.getString()); + date2 = parser.parse(o.getString()); + return date1.compareTo(date2); + case STRING: + default: + return this.getString().compareTo(o.getString()); + } + } catch (Exception e) { + return -1; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/LeafExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/LeafExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/LeafExpression.java new file mode 100644 index 0000000..6c79b08 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/LeafExpression.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression; + +public abstract class LeafExpression extends Expression { + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java new file mode 100644 index 0000000..b546ee5 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression; + +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.scan.filter.intf.ExpressionType; +import org.apache.carbondata.core.scan.filter.intf.RowIntf; + +public class LiteralExpression extends LeafExpression { + + /** + * + */ + private static final long serialVersionUID = 1L; + private Object value; + private DataType dataType; + + public LiteralExpression(Object value, DataType dataType) { + this.value = value; + this.dataType = dataType; + } + + @Override public ExpressionResult evaluate(RowIntf value) { + ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true); + return expressionResult; + } + + public ExpressionResult getExpressionResult() { + ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true); + return expressionResult; + } + + @Override public ExpressionType getFilterExpressionType() { + // TODO Auto-generated method stub + return ExpressionType.LITERAL; + } + + @Override public String getString() { + // TODO Auto-generated method stub + return "LiteralExpression(" + value + ')'; + } + + /** + * getLiteralExpDataType. + * + * @return + */ + public DataType getLiteralExpDataType() { + return dataType; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/UnknownExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/UnknownExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/UnknownExpression.java new file mode 100644 index 0000000..818720a --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/UnknownExpression.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression; + +import java.util.List; + +public abstract class UnknownExpression extends Expression { + + public abstract List<ColumnExpression> getAllColumnList(); + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/BinaryConditionalExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/BinaryConditionalExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/BinaryConditionalExpression.java new file mode 100644 index 0000000..f6725de --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/BinaryConditionalExpression.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression.conditional; + +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.expression.logical.BinaryLogicalExpression; + +public abstract class BinaryConditionalExpression extends BinaryLogicalExpression + implements ConditionalExpression { + + /** + * + */ + private static final long serialVersionUID = 1L; + public boolean isNull; + public BinaryConditionalExpression(Expression left, Expression right) { + super(left, right); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ConditionalExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ConditionalExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ConditionalExpression.java new file mode 100644 index 0000000..6f6e7fa --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ConditionalExpression.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression.conditional; + +import java.util.List; + +import org.apache.carbondata.core.scan.expression.ColumnExpression; +import org.apache.carbondata.core.scan.expression.ExpressionResult; + +public interface ConditionalExpression { + + // Will get the column informations involved in the expressions by + // traversing the tree + List<ColumnExpression> getColumnList(); + + boolean isSingleDimension(); + + List<ExpressionResult> getLiterals(); + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java new file mode 100644 index 0000000..d699801 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.scan.expression.conditional; + +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.scan.expression.ExpressionResult; +import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; +import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; +import org.apache.carbondata.core.scan.filter.FilterUtil; +import org.apache.carbondata.core.scan.filter.intf.ExpressionType; +import org.apache.carbondata.core.scan.filter.intf.RowIntf; + +public class EqualToExpression extends BinaryConditionalExpression { + + private static final long serialVersionUID = 1L; + + public EqualToExpression(Expression left, Expression right) { + super(left, right); + } + + public EqualToExpression(Expression left, Expression right, boolean isNull) { + super(left, right); + this.isNull = isNull; + } + + @Override public ExpressionResult evaluate(RowIntf value) + throws FilterUnsupportedException, FilterIllegalMemberException { + ExpressionResult elRes = left.evaluate(value); + ExpressionResult erRes = right.evaluate(value); + + boolean result = false; + + ExpressionResult val1 = elRes; + ExpressionResult val2 = erRes; + + if (elRes.isNull() || erRes.isNull()) { + if (isNull) { + elRes.set(DataType.BOOLEAN, elRes.isNull() == erRes.isNull()); + } else { + elRes.set(DataType.BOOLEAN, false); + } + return elRes; + } + //default implementation if the data types are different for the resultsets + if (elRes.getDataType() != erRes.getDataType()) { + if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) { + val2 = elRes; + val1 = erRes; + } + } + + switch (val1.getDataType()) { + case STRING: + result = val1.getString().equals(val2.getString()); + break; + case SHORT: + result = val1.getShort().equals(val2.getShort()); + break; + case INT: + result = val1.getInt().equals(val2.getInt()); + break; + case DOUBLE: + result = FilterUtil.nanSafeEqualsDoubles(val1.getDouble(), val2.getDouble()); + break; + case DATE: + case TIMESTAMP: + result = val1.getTime().equals(val2.getTime()); + break; + case LONG: + result = val1.getLong().equals(val2.getLong()); + break; + case DECIMAL: + result = val1.getDecimal().compareTo(val2.getDecimal()) == 0; + break; + default: + throw new FilterUnsupportedException( + "DataType: " + val1.getDataType() + " not supported for the filter expression"); + } + val1.set(DataType.BOOLEAN, result); + return val1; + } + + @Override public ExpressionType getFilterExpressionType() { + return ExpressionType.EQUALS; + } + + @Override public String getString() { + return "EqualTo(" + left.getString() + ',' + right.getString() + ')'; + } + +}