http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java deleted file mode 100644 index ac717a9..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.scan.complextypes; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.apache.carbondata.core.cache.dictionary.Dictionary; -import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk; -import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator; -import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; -import org.apache.carbondata.core.keygenerator.mdkey.Bits; -import org.apache.carbondata.core.util.DataTypeUtil; -import org.apache.carbondata.scan.filter.GenericQueryType; -import org.apache.carbondata.scan.processor.BlocksChunkHolder; - -import org.apache.spark.sql.types.*; - -public class PrimitiveQueryType extends ComplexQueryType implements GenericQueryType { - - private String name; - private String parentname; - - private int keySize; - - private int blockIndex; - - private Dictionary dictionary; - - private org.apache.carbondata.core.carbon.metadata.datatype.DataType dataType; - - private boolean isDirectDictionary; - - public PrimitiveQueryType(String name, String parentname, int blockIndex, - org.apache.carbondata.core.carbon.metadata.datatype.DataType dataType, int keySize, - Dictionary dictionary, boolean isDirectDictionary) { - super(name, parentname, blockIndex); - this.dataType = dataType; - this.keySize = keySize; - this.dictionary = dictionary; - this.name = name; - this.parentname = parentname; - this.blockIndex = blockIndex; - this.isDirectDictionary = isDirectDictionary; - } - - @Override public void addChildren(GenericQueryType children) { - - } - - @Override public String getName() { - return name; - } - - @Override public void setName(String name) { - this.name = name; - } - - @Override public String getParentname() { - return parentname; - } - - @Override public void setParentname(String parentname) { - this.parentname = parentname; - - } - - @Override public int getColsCount() { - return 1; - } - - @Override public void parseBlocksAndReturnComplexColumnByteArray( - DimensionColumnDataChunk[] dimensionDataChunks, int rowNumber, - DataOutputStream dataOutputStream) throws IOException { - byte[] currentVal = - new byte[dimensionDataChunks[blockIndex].getColumnValueSize()]; - copyBlockDataChunk(dimensionDataChunks, rowNumber, currentVal); - dataOutputStream.write(currentVal); - } - - @Override public DataType getSchemaType() { - switch (dataType) { - case INT: - return IntegerType$.MODULE$; - case DOUBLE: - return DoubleType$.MODULE$; - case LONG: - return LongType$.MODULE$; - case BOOLEAN: - return BooleanType$.MODULE$; - case TIMESTAMP: - return TimestampType$.MODULE$; - case DATE: - return DateType$.MODULE$; - default: - return IntegerType$.MODULE$; - } - } - - @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) - throws IOException { - readBlockDataChunk(blockChunkHolder); - } - - @Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) { - byte[] data = new byte[keySize]; - surrogateData.get(data); - Bits bit = new Bits(new int[]{keySize * 8}); - int surrgateValue = (int)bit.getKeyArray(data, 0)[0]; - Object actualData = null; - if (isDirectDictionary) { - DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory - .getDirectDictionaryGenerator(dataType); - actualData = directDictionaryGenerator.getValueFromSurrogate(surrgateValue); - } else { - String dictionaryValueForKey = dictionary.getDictionaryValueForKey(surrgateValue); - actualData = DataTypeUtil.getDataBasedOnDataType(dictionaryValueForKey, this.dataType); - } - return actualData; - } -}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java deleted file mode 100644 index 36d483b..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.scan.complextypes; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk; -import org.apache.carbondata.scan.filter.GenericQueryType; -import org.apache.carbondata.scan.processor.BlocksChunkHolder; - -import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; -import org.apache.spark.sql.types.DataType; -import org.apache.spark.sql.types.Metadata; -import org.apache.spark.sql.types.StructField; -import org.apache.spark.sql.types.StructType; - -public class StructQueryType extends ComplexQueryType implements GenericQueryType { - - private List<GenericQueryType> children = new ArrayList<GenericQueryType>(); - private String name; - private String parentname; - - public StructQueryType(String name, String parentname, int blockIndex) { - super(name, parentname, blockIndex); - this.name = name; - this.parentname = parentname; - } - - @Override public void addChildren(GenericQueryType newChild) { - if (this.getName().equals(newChild.getParentname())) { - this.children.add(newChild); - } else { - for (GenericQueryType child : this.children) { - child.addChildren(newChild); - } - } - - } - - @Override public String getName() { - return name; - } - - @Override public void setName(String name) { - this.name = name; - } - - @Override public String getParentname() { - return parentname; - } - - @Override public void setParentname(String parentname) { - this.parentname = parentname; - - } - - @Override public int getColsCount() { - int colsCount = 1; - for (int i = 0; i < children.size(); i++) { - colsCount += children.get(i).getColsCount(); - } - return colsCount; - } - - @Override public void parseBlocksAndReturnComplexColumnByteArray( - DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber, - DataOutputStream dataOutputStream) throws IOException { - byte[] input = new byte[8]; - copyBlockDataChunk(dimensionColumnDataChunks, rowNumber, input); - ByteBuffer byteArray = ByteBuffer.wrap(input); - int childElement = byteArray.getInt(); - dataOutputStream.writeInt(childElement); - if (childElement > 0){ - for (int i = 0; i < childElement; i++) { - children.get(i) - .parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks, rowNumber, - dataOutputStream); - } - } - } - - @Override public DataType getSchemaType() { - StructField[] fields = new StructField[children.size()]; - for (int i = 0; i < children.size(); i++) { - fields[i] = new StructField(children.get(i).getName(), null, true, - Metadata.empty()); - } - return new StructType(fields); - } - - @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) - throws IOException { - readBlockDataChunk(blockChunkHolder); - - for (int i = 0; i < children.size(); i++) { - children.get(i).fillRequiredBlockData(blockChunkHolder); - } - } - - @Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) { - int childLength = surrogateData.getInt(); - Object[] fields = new Object[childLength]; - for (int i = 0; i < childLength; i++) { - fields[i] = children.get(i).getDataBasedOnDataTypeFromSurrogates(surrogateData); - } - - return new GenericInternalRow(fields); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java deleted file mode 100644 index 949771c..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor; - -import java.io.IOException; - -import org.apache.carbondata.common.CarbonIterator; -import org.apache.carbondata.scan.executor.exception.QueryExecutionException; -import org.apache.carbondata.scan.model.QueryModel; - -/** - * Interface for carbon query executor. - * Will be used to execute the query based on the query model - * and will return the iterator over query result - */ -public interface QueryExecutor<E> { - - /** - * Below method will be used to execute the query based on query model passed from driver - * - * @param queryModel query details - * @return query result iterator - * @throws QueryExecutionException if any failure while executing the query - * @throws IOException if fail to read files - */ - CarbonIterator<E> execute(QueryModel queryModel) - throws QueryExecutionException, IOException; - - /** - * Below method will be used for cleanup - * - * @throws QueryExecutionException - */ - void finish() throws QueryExecutionException; -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutorFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutorFactory.java b/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutorFactory.java deleted file mode 100644 index 947b992..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutorFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor; - -import org.apache.carbondata.scan.executor.impl.DetailQueryExecutor; -import org.apache.carbondata.scan.executor.impl.VectorDetailQueryExecutor; -import org.apache.carbondata.scan.model.QueryModel; - -/** - * Factory class to get the query executor from RDD - * This will return the executor based on query type - */ -public class QueryExecutorFactory { - - public static QueryExecutor getQueryExecutor(QueryModel queryModel) { - if (queryModel.isVectorReader()) { - return new VectorDetailQueryExecutor(); - } else { - return new DetailQueryExecutor(); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java b/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java deleted file mode 100644 index 5634e0f..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor.exception; - -import java.util.Locale; - -/** - * Exception class for query execution - * - * @author Administrator - */ -public class QueryExecutionException extends Exception { - - /** - * default serial version ID. - */ - private static final long serialVersionUID = 1L; - - /** - * The Error message. - */ - private String msg = ""; - - /** - * Constructor - * - * @param msg The error message for this exception. - */ - public QueryExecutionException(String msg) { - super(msg); - this.msg = msg; - } - - /** - * Constructor - * - * @param msg The error message for this exception. - */ - public QueryExecutionException(String msg, Throwable t) { - super(msg, t); - this.msg = msg; - } - - /** - * Constructor - * - * @param t - */ - public QueryExecutionException(Throwable t) { - super(t); - } - - /** - * This method is used to get the localized message. - * - * @param locale - A Locale object represents a specific geographical, - * political, or cultural region. - * @return - Localized error message. - */ - public String getLocalizedMessage(Locale locale) { - return ""; - } - - /** - * getLocalizedMessage - */ - @Override public String getLocalizedMessage() { - return super.getLocalizedMessage(); - } - - /** - * getMessage - */ - public String getMessage() { - return this.msg; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java deleted file mode 100644 index 8c9daaf..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java +++ /dev/null @@ -1,493 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import org.apache.carbondata.common.CarbonIterator; -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.common.logging.impl.StandardLogService; -import org.apache.carbondata.core.cache.CacheProvider; -import org.apache.carbondata.core.cache.CacheType; -import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier; -import org.apache.carbondata.core.carbon.datastore.BlockIndexStore; -import org.apache.carbondata.core.carbon.datastore.IndexKey; -import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex; -import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties; -import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo; -import org.apache.carbondata.core.carbon.datastore.block.TableBlockUniqueIdentifier; -import org.apache.carbondata.core.carbon.metadata.datatype.DataType; -import org.apache.carbondata.core.carbon.metadata.encoder.Encoding; -import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension; -import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure; -import org.apache.carbondata.core.carbon.path.CarbonStorePath; -import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic; -import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsConstants; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.keygenerator.KeyGenException; -import org.apache.carbondata.core.keygenerator.KeyGenerator; -import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory; -import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.carbondata.scan.executor.QueryExecutor; -import org.apache.carbondata.scan.executor.exception.QueryExecutionException; -import org.apache.carbondata.scan.executor.infos.AggregatorInfo; -import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo; -import org.apache.carbondata.scan.executor.infos.KeyStructureInfo; -import org.apache.carbondata.scan.executor.util.QueryUtil; -import org.apache.carbondata.scan.executor.util.RestructureUtil; -import org.apache.carbondata.scan.filter.FilterUtil; -import org.apache.carbondata.scan.model.QueryDimension; -import org.apache.carbondata.scan.model.QueryMeasure; -import org.apache.carbondata.scan.model.QueryModel; - -import org.apache.commons.lang3.ArrayUtils; - -/** - * This class provides a skeletal implementation of the {@link QueryExecutor} - * interface to minimize the effort required to implement this interface. This - * will be used to prepare all the properties required for query execution - */ -public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(AbstractQueryExecutor.class.getName()); - /** - * holder for query properties which will be used to execute the query - */ - protected QueryExecutorProperties queryProperties; - - /** - * query result iterator which will execute the query - * and give the result - */ - protected CarbonIterator queryIterator; - - public AbstractQueryExecutor() { - queryProperties = new QueryExecutorProperties(); - } - - /** - * Below method will be used to fill the executor properties based on query - * model it will parse the query model and get the detail and fill it in - * query properties - * - * @param queryModel - */ - protected void initQuery(QueryModel queryModel) throws IOException { - StandardLogService.setThreadName(StandardLogService.getPartitionID( - queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName()), - queryModel.getQueryId()); - LOGGER.info("Query will be executed on table: " + queryModel.getAbsoluteTableIdentifier() - .getCarbonTableIdentifier().getTableName()); - // add executor service for query execution - queryProperties.executorService = Executors.newFixedThreadPool(1); - // Initializing statistics list to record the query statistics - // creating copy on write to handle concurrent scenario - queryProperties.queryStatisticsRecorder = - CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId()); - queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder); - QueryUtil.resolveQueryModel(queryModel); - QueryStatistic queryStatistic = new QueryStatistic(); - // sort the block info - // so block will be loaded in sorted order this will be required for - // query execution - Collections.sort(queryModel.getTableBlockInfos()); - // get the table blocks - CacheProvider cacheProvider = CacheProvider.getInstance(); - BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex> cache = - (BlockIndexStore) cacheProvider - .createCache(CacheType.EXECUTOR_BTREE, queryModel.getTable().getStorePath()); - // remove the invalid table blocks, block which is deleted or compacted - cache.removeTableBlocks(queryModel.getInvalidSegmentIds(), - queryModel.getAbsoluteTableIdentifier()); - List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers = - prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(), - queryModel.getAbsoluteTableIdentifier()); - cache.removeTableBlocksIfHorizontalCompactionDone(queryModel); - queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers); - queryStatistic - .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis()); - queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic); - // - // // updating the restructuring infos for the query - queryProperties.keyStructureInfo = getKeyStructureInfo(queryModel, - queryProperties.dataBlocks.get(queryProperties.dataBlocks.size() - 1).getSegmentProperties() - .getDimensionKeyGenerator()); - - // calculating the total number of aggeragted columns - int aggTypeCount = queryModel.getQueryMeasures().size(); - - int currentIndex = 0; - DataType[] dataTypes = new DataType[aggTypeCount]; - - for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) { - // adding the data type and aggregation type of all the measure this - // can be used - // to select the aggregator - dataTypes[currentIndex] = carbonMeasure.getMeasure().getDataType(); - currentIndex++; - } - queryProperties.measureDataTypes = dataTypes; - // as aggregation will be executed in following order - // 1.aggregate dimension expression - // 2. expression - // 3. query measure - // so calculating the index of the expression start index - // and measure column start index - queryProperties.filterMeasures = new HashSet<>(); - queryProperties.complexFilterDimension = new HashSet<>(); - QueryUtil.getAllFilterDimensions(queryModel.getFilterExpressionResolverTree(), - queryProperties.complexFilterDimension, queryProperties.filterMeasures); - - queryStatistic = new QueryStatistic(); - // dictionary column unique column id to dictionary mapping - // which will be used to get column actual data - queryProperties.columnToDictionayMapping = QueryUtil - .getDimensionDictionaryDetail(queryModel.getQueryDimension(), - queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier()); - queryStatistic - .addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis()); - queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic); - queryModel.setColumnToDictionaryMapping(queryProperties.columnToDictionayMapping); - } - - private List<TableBlockUniqueIdentifier> prepareTableBlockUniqueIdentifier( - List<TableBlockInfo> tableBlockInfos, AbsoluteTableIdentifier absoluteTableIdentifier) { - List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers = - new ArrayList<>(tableBlockInfos.size()); - for (TableBlockInfo blockInfo : tableBlockInfos) { - tableBlockUniqueIdentifiers - .add(new TableBlockUniqueIdentifier(absoluteTableIdentifier, blockInfo)); - } - return tableBlockUniqueIdentifiers; - } - - /** - * Below method will be used to get the key structure info for the query - * - * @param queryModel query model - * @param keyGenerator - * @return key structure info - */ - private KeyStructureInfo getKeyStructureInfo(QueryModel queryModel, KeyGenerator keyGenerator) { - // getting the masked byte range for dictionary column - int[] maskByteRanges = - QueryUtil.getMaskedByteRange(queryModel.getQueryDimension(), keyGenerator); - - // max key for the dictionary dimension present in the query - byte[] maxKey = null; - try { - // getting the max key which will be used to masked and get the - // masked key - maxKey = QueryUtil.getMaxKeyBasedOnDimensions(queryModel.getQueryDimension(), keyGenerator); - } catch (KeyGenException e) { - LOGGER.error(e, "problem while getting the max key"); - } - - KeyStructureInfo restructureInfos = new KeyStructureInfo(); - restructureInfos.setKeyGenerator(keyGenerator); - restructureInfos.setMaskByteRanges(maskByteRanges); - restructureInfos.setMaxKey(maxKey); - return restructureInfos; - } - - protected List<BlockExecutionInfo> getBlockExecutionInfos(QueryModel queryModel) - throws IOException, QueryExecutionException { - initQuery(queryModel); - List<BlockExecutionInfo> blockExecutionInfoList = new ArrayList<BlockExecutionInfo>(); - // fill all the block execution infos for all the blocks selected in - // query - // and query will be executed based on that infos - for (int i = 0; i < queryProperties.dataBlocks.size(); i++) { - blockExecutionInfoList.add( - getBlockExecutionInfoForBlock(queryModel, queryProperties.dataBlocks.get(i), - queryModel.getTableBlockInfos().get(i).getBlockletInfos().getStartBlockletNumber(), - queryModel.getTableBlockInfos().get(i).getBlockletInfos().getNumberOfBlockletToScan(), - queryModel.getTableBlockInfos().get(i).getFilePath())); - } - if (null != queryModel.getStatisticsRecorder()) { - QueryStatistic queryStatistic = new QueryStatistic(); - queryStatistic.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_NUM, - blockExecutionInfoList.size()); - queryModel.getStatisticsRecorder().recordStatistics(queryStatistic); - } - return blockExecutionInfoList; - } - - /** - * Below method will be used to get the block execution info which is - * required to execute any block based on query model - * - * @param queryModel query model from user query - * @param blockIndex block index - * @return block execution info - * @throws QueryExecutionException any failure during block info creation - */ - protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel, - AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath) - throws QueryExecutionException { - BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo(); - SegmentProperties segmentProperties = blockIndex.getSegmentProperties(); - List<CarbonDimension> tableBlockDimensions = segmentProperties.getDimensions(); - KeyGenerator blockKeyGenerator = segmentProperties.getDimensionKeyGenerator(); - - // below is to get only those dimension in query which is present in the - // table block - List<QueryDimension> updatedQueryDimension = RestructureUtil - .getUpdatedQueryDimension(queryModel.getQueryDimension(), tableBlockDimensions, - segmentProperties.getComplexDimensions()); - // TODO add complex dimension children - int[] maskByteRangesForBlock = - QueryUtil.getMaskedByteRange(updatedQueryDimension, blockKeyGenerator); - int[] maksedByte = - QueryUtil.getMaskedByte(blockKeyGenerator.getKeySizeInBytes(), maskByteRangesForBlock); - int tableFactPathLength = CarbonStorePath - .getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getStorePath(), - queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier()).getFactDir() - .length() + 1; - blockExecutionInfo.setBlockId(filePath.substring(tableFactPathLength)); - blockExecutionInfo.setStartBlockletIndex(startBlockletIndex); - blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan); - blockExecutionInfo.setQueryDimensions( - updatedQueryDimension.toArray(new QueryDimension[updatedQueryDimension.size()])); - blockExecutionInfo.setQueryMeasures(queryModel.getQueryMeasures() - .toArray(new QueryMeasure[queryModel.getQueryMeasures().size()])); - blockExecutionInfo.setDataBlock(blockIndex); - blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator); - // adding aggregation info for query - blockExecutionInfo.setAggregatorInfo(getAggregatorInfoForBlock(queryModel, blockIndex)); - // setting whether raw record query or not - blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery()); - // setting the masked byte of the block which will be - // used to update the unpack the older block keys - blockExecutionInfo.setMaskedByteForBlock(maksedByte); - // total number dimension - blockExecutionInfo - .setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size()); - blockExecutionInfo - .setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size()); - blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier()); - blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil - .getComplexDimensionsMap(updatedQueryDimension, - segmentProperties.getDimensionOrdinalToBlockMapping(), - segmentProperties.getEachComplexDimColumnValueSize(), - queryProperties.columnToDictionayMapping, queryProperties.complexFilterDimension)); - // to check whether older block key update is required or not - blockExecutionInfo.setFixedKeyUpdateRequired( - !blockKeyGenerator.equals(queryProperties.keyStructureInfo.getKeyGenerator())); - IndexKey startIndexKey = null; - IndexKey endIndexKey = null; - if (null != queryModel.getFilterExpressionResolverTree()) { - // loading the filter executer tree for filter evaluation - blockExecutionInfo.setFilterExecuterTree(FilterUtil - .getFilterExecuterTree(queryModel.getFilterExpressionResolverTree(), segmentProperties, - blockExecutionInfo.getComlexDimensionInfoMap())); - List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2); - FilterUtil.traverseResolverTreeAndGetStartAndEndKey(segmentProperties, - queryModel.getFilterExpressionResolverTree(), listOfStartEndKeys); - startIndexKey = listOfStartEndKeys.get(0); - endIndexKey = listOfStartEndKeys.get(1); - } else { - try { - startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties); - endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties); - } catch (KeyGenException e) { - throw new QueryExecutionException(e); - } - } - //setting the start index key of the block node - blockExecutionInfo.setStartKey(startIndexKey); - //setting the end index key of the block node - blockExecutionInfo.setEndKey(endIndexKey); - // expression dimensions - List<CarbonDimension> expressionDimensions = - new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); - // expression measure - List<CarbonMeasure> expressionMeasures = - new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); - // setting all the dimension chunk indexes to be read from file - int numberOfElementToConsider = 0; - // list of dimensions to be projected - List<Integer> allProjectionListDimensionIdexes = new ArrayList<>(); - int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(updatedQueryDimension, - segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions, - queryProperties.complexFilterDimension, allProjectionListDimensionIdexes); - if (dimensionsBlockIndexes.length > 0) { - numberOfElementToConsider = dimensionsBlockIndexes[dimensionsBlockIndexes.length - 1] - == segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ? - dimensionsBlockIndexes.length - 1 : - dimensionsBlockIndexes.length; - blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(CarbonUtil - .getRangeIndex(dimensionsBlockIndexes, numberOfElementToConsider, - CarbonCommonConstants.NUMBER_OF_COLUMN_READ_IN_IO)); - } else { - blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[0][0]); - } - - int[] measureBlockIndexes = QueryUtil - .getMeasureBlockIndexes(queryModel.getQueryMeasures(), expressionMeasures, - segmentProperties.getMeasuresOrdinalToBlockMapping(), queryProperties.filterMeasures); - if (measureBlockIndexes.length > 0) { - - numberOfElementToConsider = measureBlockIndexes[measureBlockIndexes.length - 1] - == segmentProperties.getMeasures().size() - 1 ? - measureBlockIndexes.length - 1 : - measureBlockIndexes.length; - // setting all the measure chunk indexes to be read from file - blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(CarbonUtil - .getRangeIndex(measureBlockIndexes, numberOfElementToConsider, - CarbonCommonConstants.NUMBER_OF_COLUMN_READ_IN_IO)); - } else { - blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[0][0]); - } - // setting the key structure info which will be required - // to update the older block key with new key generator - blockExecutionInfo.setKeyStructureInfo(queryProperties.keyStructureInfo); - // setting the size of fixed key column (dictionary column) - blockExecutionInfo.setFixedLengthKeySize(getKeySize(updatedQueryDimension, segmentProperties)); - Set<Integer> dictionaryColumnBlockIndex = new HashSet<Integer>(); - List<Integer> noDictionaryColumnBlockIndex = new ArrayList<Integer>(); - // get the block index to be read from file for query dimension - // for both dictionary columns and no dictionary columns - QueryUtil.fillQueryDimensionsBlockIndexes(updatedQueryDimension, - segmentProperties.getDimensionOrdinalToBlockMapping(), dictionaryColumnBlockIndex, - noDictionaryColumnBlockIndex); - int[] queryDictionaryColumnBlockIndexes = ArrayUtils.toPrimitive( - dictionaryColumnBlockIndex.toArray(new Integer[dictionaryColumnBlockIndex.size()])); - // need to sort the dictionary column as for all dimension - // column key will be filled based on key order - Arrays.sort(queryDictionaryColumnBlockIndexes); - blockExecutionInfo.setDictionaryColumnBlockIndex(queryDictionaryColumnBlockIndexes); - // setting the no dictionary column block indexes - blockExecutionInfo.setNoDictionaryBlockIndexes(ArrayUtils.toPrimitive( - noDictionaryColumnBlockIndex.toArray(new Integer[noDictionaryColumnBlockIndex.size()]))); - // setting column id to dictionary mapping - blockExecutionInfo.setColumnIdToDcitionaryMapping(queryProperties.columnToDictionayMapping); - // setting each column value size - blockExecutionInfo.setEachColumnValueSize(segmentProperties.getEachDimColumnValueSize()); - blockExecutionInfo.setComplexColumnParentBlockIndexes( - getComplexDimensionParentBlockIndexes(updatedQueryDimension)); - blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader()); - try { - // to set column group and its key structure info which will be used - // to - // for getting the column group column data in case of final row - // and in case of dimension aggregation - blockExecutionInfo.setColumnGroupToKeyStructureInfo( - QueryUtil.getColumnGroupKeyStructureInfo(updatedQueryDimension, segmentProperties)); - } catch (KeyGenException e) { - throw new QueryExecutionException(e); - } - return blockExecutionInfo; - } - - /** - * This method will be used to get fixed key length size this will be used - * to create a row from column chunk - * - * @param queryDimension query dimension - * @param blockMetadataInfo block metadata info - * @return key size - */ - private int getKeySize(List<QueryDimension> queryDimension, SegmentProperties blockMetadataInfo) { - List<Integer> fixedLengthDimensionOrdinal = - new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); - int counter = 0; - while (counter < queryDimension.size()) { - if (queryDimension.get(counter).getDimension().numberOfChild() > 0) { - counter += queryDimension.get(counter).getDimension().numberOfChild(); - continue; - } else if (!CarbonUtil.hasEncoding(queryDimension.get(counter).getDimension().getEncoder(), - Encoding.DICTIONARY)) { - counter++; - } else { - fixedLengthDimensionOrdinal.add(queryDimension.get(counter).getDimension().getKeyOrdinal()); - counter++; - } - } - int[] dictioanryColumnOrdinal = ArrayUtils.toPrimitive( - fixedLengthDimensionOrdinal.toArray(new Integer[fixedLengthDimensionOrdinal.size()])); - if (dictioanryColumnOrdinal.length > 0) { - return blockMetadataInfo.getFixedLengthKeySplitter() - .getKeySizeByBlock(dictioanryColumnOrdinal); - } - return 0; - } - - /** - * Below method will be used to get the aggrgator info for the query - * - * @param queryModel query model - * @param tableBlock table block - * @return aggregator info - */ - private AggregatorInfo getAggregatorInfoForBlock(QueryModel queryModel, - AbstractIndex tableBlock) { - // getting the aggregate infos which will be used during aggregation - AggregatorInfo aggregatorInfos = RestructureUtil - .getAggregatorInfos(queryModel.getQueryMeasures(), - tableBlock.getSegmentProperties().getMeasures()); - // setting the measure aggregator for all aggregation function selected - // in query - aggregatorInfos.setMeasureDataTypes(queryProperties.measureDataTypes); - return aggregatorInfos; - } - - private int[] getComplexDimensionParentBlockIndexes(List<QueryDimension> queryDimensions) { - List<Integer> parentBlockIndexList = new ArrayList<Integer>(); - for (QueryDimension queryDimension : queryDimensions) { - if (CarbonUtil.hasDataType(queryDimension.getDimension().getDataType(), - new DataType[] { DataType.ARRAY, DataType.STRUCT, DataType.MAP })) { - parentBlockIndexList.add(queryDimension.getDimension().getOrdinal()); - } - } - return ArrayUtils - .toPrimitive(parentBlockIndexList.toArray(new Integer[parentBlockIndexList.size()])); - } - - /** - * Below method will be used to finish the execution - * - * @throws QueryExecutionException - */ - @Override public void finish() throws QueryExecutionException { - CarbonUtil.clearBlockCache(queryProperties.dataBlocks); - if(null != queryIterator) { - queryIterator.close(); - } - if (null != queryProperties.executorService) { - queryProperties.executorService.shutdown(); - try { - queryProperties.executorService.awaitTermination(1, TimeUnit.HOURS); - } catch (InterruptedException e) { - throw new QueryExecutionException(e); - } - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java deleted file mode 100644 index 6a34e68..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor.impl; - -import java.io.IOException; -import java.util.List; - -import org.apache.carbondata.common.CarbonIterator; -import org.apache.carbondata.scan.executor.exception.QueryExecutionException; -import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo; -import org.apache.carbondata.scan.model.QueryModel; -import org.apache.carbondata.scan.result.BatchResult; -import org.apache.carbondata.scan.result.iterator.DetailQueryResultIterator; - -/** - * Below class will be used to execute the detail query - * For executing the detail query it will pass all the block execution - * info to detail query result iterator and iterator will be returned - */ -public class DetailQueryExecutor extends AbstractQueryExecutor<BatchResult> { - - @Override - public CarbonIterator<BatchResult> execute(QueryModel queryModel) - throws QueryExecutionException, IOException { - List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel); - this.queryIterator = new DetailQueryResultIterator( - blockExecutionInfoList, - queryModel, - queryProperties.executorService - ); - return queryIterator; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java deleted file mode 100644 index c18e8c9..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor.impl; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutorService; - -import org.apache.carbondata.core.cache.dictionary.Dictionary; -import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex; -import org.apache.carbondata.core.carbon.metadata.datatype.DataType; -import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension; -import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure; -import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorder; -import org.apache.carbondata.scan.executor.infos.KeyStructureInfo; - -/** - * Holds all the properties required for query execution - */ -public class QueryExecutorProperties { - - /** - * holds the information required for updating the order block - * dictionary key - */ - public KeyStructureInfo keyStructureInfo; - - /** - * this will hold the information about the dictionary dimension - * which to - */ - public Map<String, Dictionary> columnToDictionayMapping; - - /** - * Measure datatypes - */ - public DataType[] measureDataTypes; - /** - * all the complex dimension which is on filter - */ - public Set<CarbonDimension> complexFilterDimension; - - public Set<CarbonMeasure> filterMeasures; - /** - * to record the query execution details phase wise - */ - public QueryStatisticsRecorder queryStatisticsRecorder; - /** - * executor service to execute the query - */ - public ExecutorService executorService; - /** - * list of blocks in which query will be executed - */ - protected List<AbstractIndex> dataBlocks; -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java deleted file mode 100644 index 04e60fb..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor.impl; - -import java.io.IOException; -import java.util.List; - -import org.apache.carbondata.common.CarbonIterator; -import org.apache.carbondata.scan.executor.exception.QueryExecutionException; -import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo; -import org.apache.carbondata.scan.model.QueryModel; -import org.apache.carbondata.scan.result.iterator.VectorDetailQueryResultIterator; - -/** - * Below class will be used to execute the detail query and returns columnar vectors. - */ -public class VectorDetailQueryExecutor extends AbstractQueryExecutor<Object> { - - @Override - public CarbonIterator<Object> execute(QueryModel queryModel) - throws QueryExecutionException, IOException { - List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel); - this.queryIterator = new VectorDetailQueryResultIterator( - blockExecutionInfoList, - queryModel, - queryProperties.executorService - ); - return this.queryIterator; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java deleted file mode 100644 index f4f0fe5..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor.infos; - -import org.apache.carbondata.core.carbon.metadata.datatype.DataType; - -/** - * Info class which store all the details - * which is required during aggregation - */ -public class AggregatorInfo { - - /** - * selected query measure ordinal - * which will be used to read the measures chunk data - * this will be storing the index of the measure in measures chunk - */ - private int[] measureOrdinals; - - /** - * This parameter will be used to - * check whether particular measure is present - * in the table block, if not then its default value will be used - */ - private boolean[] measureExists; - - /** - * this default value will be used to when some measure is not present - * in the table block, in case of restructuring of the table if user is adding any - * measure then in older block that measure wont be present so for measure default value - * will be used to aggregate in the older table block query execution - */ - private Object[] defaultValues; - - /** - * Datatype of each measure; - */ - private DataType[] measureDataTypes; - - /** - * @return the measureOrdinal - */ - public int[] getMeasureOrdinals() { - return measureOrdinals; - } - - /** - * @param measureOrdinal the measureOrdinal to set - */ - public void setMeasureOrdinals(int[] measureOrdinal) { - this.measureOrdinals = measureOrdinal; - } - - /** - * @return the measureExists - */ - public boolean[] getMeasureExists() { - return measureExists; - } - - /** - * @param measureExists the measureExists to set - */ - public void setMeasureExists(boolean[] measureExists) { - this.measureExists = measureExists; - } - - /** - * @return the defaultValues - */ - public Object[] getDefaultValues() { - return defaultValues; - } - - /** - * @param defaultValues the defaultValues to set - */ - public void setDefaultValues(Object[] defaultValues) { - this.defaultValues = defaultValues; - } - - public DataType[] getMeasureDataTypes() { - return measureDataTypes; - } - - public void setMeasureDataTypes(DataType[] measureDataTypes) { - this.measureDataTypes = measureDataTypes; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java deleted file mode 100644 index 88b5ca3..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java +++ /dev/null @@ -1,604 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor.infos; - -import java.util.Map; - -import org.apache.carbondata.core.cache.dictionary.Dictionary; -import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier; -import org.apache.carbondata.core.carbon.datastore.DataRefNode; -import org.apache.carbondata.core.carbon.datastore.IndexKey; -import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex; -import org.apache.carbondata.core.keygenerator.KeyGenerator; -import org.apache.carbondata.scan.filter.GenericQueryType; -import org.apache.carbondata.scan.filter.executer.FilterExecuter; -import org.apache.carbondata.scan.model.QueryDimension; -import org.apache.carbondata.scan.model.QueryMeasure; - -/** - * Below class will have all the properties which needed during query execution - * for one block - */ -public class BlockExecutionInfo { - - /** - * block on which query will be executed - */ - private AbstractIndex blockIndex; - - /** - * each segment key size can be different and in that case we need to update - * the fixed key with latest segment key generator. so this property will - * tell whether this is required or not if key size is same then it is not - * required - */ - private boolean isFixedKeyUpdateRequired; - - /** - * below to store all the information required for aggregation during query - * execution - */ - private AggregatorInfo aggregatorInfo; - - /** - * this will be used to get the first tentative block from which query - * execution start, this will be useful in case of filter query to get the - * start block based on filter values - */ - private IndexKey startKey; - - /** - * this will be used to get the last tentative block till which scanning - * will be done, this will be useful in case of filter query to get the last - * block based on filter values - */ - private IndexKey endKey; - - private String blockId; - - /** - * masked byte for block which will be used to unpack the fixed length key, - * this will be used for updating the older block key with new block key - * generator - */ - private int[] maskedByteForBlock; - - /** - * total number of dimension in block - */ - private int totalNumberDimensionBlock; - - /** - * total number of measure in block - */ - private int totalNumberOfMeasureBlock; - - /** - * will be used to read the dimension block from file - */ - private int[][] allSelectedDimensionBlocksIndexes; - - /** - * will be used to read the measure block from file - */ - private int[][] allSelectedMeasureBlocksIndexes; - - /** - * this will be used to update the older block fixed length keys with the - * new block fixed length key - */ - private KeyStructureInfo keyStructureInfo; - - /** - * first block from which query execution will start - */ - private DataRefNode firstDataBlock; - - /** - * number of block to be scanned in the query - */ - private long numberOfBlockToScan; - - /** - * key size of the fixed length dimension column - */ - private int fixedLengthKeySize; - - /** - * dictionary column block indexes based on query - */ - private int[] dictionaryColumnBlockIndex; - /** - * no dictionary column block indexes in based on the query order - */ - private int[] noDictionaryBlockIndexes; - - /** - * key generator used for generating the table block fixed length key - */ - private KeyGenerator blockKeyGenerator; - - /** - * each column value size - */ - private int[] eachColumnValueSize; - - /** - * column group block index in file to key structure info mapping - */ - private Map<Integer, KeyStructureInfo> columnGroupToKeyStructureInfo; - - /** - * mapping of dictionary dimension to its dictionary mapping which will be - * used to get the actual data from dictionary for aggregation, sorting - */ - private Map<String, Dictionary> columnIdToDcitionaryMapping; - - /** - * filter tree to execute the filter - */ - private FilterExecuter filterExecuterTree; - - /** - * whether it needs only raw byte records with out aggregation. - */ - private boolean isRawRecordDetailQuery; - - /** - * start index of blocklets - */ - private int startBlockletIndex; - - /** - * number of blocklet to be scanned - */ - private int numberOfBlockletToScan; - - /** - * complexParentIndexToQueryMap - */ - private Map<Integer, GenericQueryType> complexParentIndexToQueryMap; - - /** - * complex dimension parent block indexes; - */ - private int[] complexColumnParentBlockIndexes; - - /** - * @return the tableBlock - */ - public AbstractIndex getDataBlock() { - return blockIndex; - } - - /** - * list of dimension selected for in query - */ - private QueryDimension[] queryDimensions; - - /** - * list of measure selected in query - */ - private QueryMeasure[] queryMeasures; - - /** - * whether it needs to read data in vector/columnar format. - */ - private boolean vectorBatchCollector; - - /** - * absolute table identifier - */ - private AbsoluteTableIdentifier absoluteTableIdentifier; - - public AbsoluteTableIdentifier getAbsoluteTableIdentifier() { - return absoluteTableIdentifier; - } - - public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) { - this.absoluteTableIdentifier = absoluteTableIdentifier; - } - - /** - * @param blockIndex the tableBlock to set - */ - public void setDataBlock(AbstractIndex blockIndex) { - this.blockIndex = blockIndex; - } - - /** - * @return the isFixedKeyUpdateRequired - */ - public boolean isFixedKeyUpdateRequired() { - return isFixedKeyUpdateRequired; - } - - /** - * @param isFixedKeyUpdateRequired the isFixedKeyUpdateRequired to set - */ - public void setFixedKeyUpdateRequired(boolean isFixedKeyUpdateRequired) { - this.isFixedKeyUpdateRequired = isFixedKeyUpdateRequired; - } - - /** - * @return the aggregatorInfos - */ - public AggregatorInfo getAggregatorInfo() { - return aggregatorInfo; - } - - /** - * @param aggregatorInfo the aggregatorInfos to set - */ - public void setAggregatorInfo(AggregatorInfo aggregatorInfo) { - this.aggregatorInfo = aggregatorInfo; - } - - /** - * @return the startKey - */ - public IndexKey getStartKey() { - return startKey; - } - - /** - * @param startKey the startKey to set - */ - public void setStartKey(IndexKey startKey) { - this.startKey = startKey; - } - - /** - * @return the endKey - */ - public IndexKey getEndKey() { - return endKey; - } - - /** - * @param endKey the endKey to set - */ - public void setEndKey(IndexKey endKey) { - this.endKey = endKey; - } - - /** - * @return the maskedByteForBlock - */ - public int[] getMaskedByteForBlock() { - return maskedByteForBlock; - } - - - - /** - * @param maskedByteForBlock the maskedByteForBlock to set - */ - public void setMaskedByteForBlock(int[] maskedByteForBlock) { - this.maskedByteForBlock = maskedByteForBlock; - } - - /** - * @return the totalNumberDimensionBlock - */ - public int getTotalNumberDimensionBlock() { - return totalNumberDimensionBlock; - } - - /** - * @param totalNumberDimensionBlock the totalNumberDimensionBlock to set - */ - public void setTotalNumberDimensionBlock(int totalNumberDimensionBlock) { - this.totalNumberDimensionBlock = totalNumberDimensionBlock; - } - - /** - * @return the totalNumberOfMeasureBlock - */ - public int getTotalNumberOfMeasureBlock() { - return totalNumberOfMeasureBlock; - } - - /** - * @param totalNumberOfMeasureBlock the totalNumberOfMeasureBlock to set - */ - public void setTotalNumberOfMeasureBlock(int totalNumberOfMeasureBlock) { - this.totalNumberOfMeasureBlock = totalNumberOfMeasureBlock; - } - - /** - * @return the allSelectedDimensionBlocksIndexes - */ - public int[][] getAllSelectedDimensionBlocksIndexes() { - return allSelectedDimensionBlocksIndexes; - } - - /** - * @param allSelectedDimensionBlocksIndexes the allSelectedDimensionBlocksIndexes to set - */ - public void setAllSelectedDimensionBlocksIndexes(int[][] allSelectedDimensionBlocksIndexes) { - this.allSelectedDimensionBlocksIndexes = allSelectedDimensionBlocksIndexes; - } - - /** - * @return the allSelectedMeasureBlocksIndexes - */ - public int[][] getAllSelectedMeasureBlocksIndexes() { - return allSelectedMeasureBlocksIndexes; - } - - /** - * @param allSelectedMeasureBlocksIndexes the allSelectedMeasureBlocksIndexes to set - */ - public void setAllSelectedMeasureBlocksIndexes(int[][] allSelectedMeasureBlocksIndexes) { - this.allSelectedMeasureBlocksIndexes = allSelectedMeasureBlocksIndexes; - } - - /** - * @return the restructureInfos - */ - public KeyStructureInfo getKeyStructureInfo() { - return keyStructureInfo; - } - - /** - * @param keyStructureInfo the restructureInfos to set - */ - public void setKeyStructureInfo(KeyStructureInfo keyStructureInfo) { - this.keyStructureInfo = keyStructureInfo; - } - - /** - * @return the firstDataBlock - */ - public DataRefNode getFirstDataBlock() { - return firstDataBlock; - } - - /** - * @param firstDataBlock the firstDataBlock to set - */ - public void setFirstDataBlock(DataRefNode firstDataBlock) { - this.firstDataBlock = firstDataBlock; - } - - /** - * @return the numberOfBlockToScan - */ - public long getNumberOfBlockToScan() { - return numberOfBlockToScan; - } - - /** - * @param numberOfBlockToScan the numberOfBlockToScan to set - */ - public void setNumberOfBlockToScan(long numberOfBlockToScan) { - this.numberOfBlockToScan = numberOfBlockToScan; - } - - /** - * @return the fixedLengthKeySize - */ - public int getFixedLengthKeySize() { - return fixedLengthKeySize; - } - - /** - * @param fixedLengthKeySize the fixedLengthKeySize to set - */ - public void setFixedLengthKeySize(int fixedLengthKeySize) { - this.fixedLengthKeySize = fixedLengthKeySize; - } - - /** - * @return the filterEvaluatorTree - */ - public FilterExecuter getFilterExecuterTree() { - return filterExecuterTree; - } - - /** - * @param filterExecuterTree the filterEvaluatorTree to set - */ - public void setFilterExecuterTree(FilterExecuter filterExecuterTree) { - this.filterExecuterTree = filterExecuterTree; - } - - /** - * @return the tableBlockKeyGenerator - */ - public KeyGenerator getBlockKeyGenerator() { - return blockKeyGenerator; - } - - /** - * @param tableBlockKeyGenerator the tableBlockKeyGenerator to set - */ - public void setBlockKeyGenerator(KeyGenerator tableBlockKeyGenerator) { - this.blockKeyGenerator = tableBlockKeyGenerator; - } - - /** - * @return the eachColumnValueSize - */ - public int[] getEachColumnValueSize() { - return eachColumnValueSize; - } - - /** - * @param eachColumnValueSize the eachColumnValueSize to set - */ - public void setEachColumnValueSize(int[] eachColumnValueSize) { - this.eachColumnValueSize = eachColumnValueSize; - } - - /** - * @return the dictionaryColumnBlockIndex - */ - public int[] getDictionaryColumnBlockIndex() { - return dictionaryColumnBlockIndex; - } - - /** - * @param dictionaryColumnBlockIndex the dictionaryColumnBlockIndex to set - */ - public void setDictionaryColumnBlockIndex(int[] dictionaryColumnBlockIndex) { - this.dictionaryColumnBlockIndex = dictionaryColumnBlockIndex; - } - - /** - * @return the noDictionaryBlockIndexes - */ - public int[] getNoDictionaryBlockIndexes() { - return noDictionaryBlockIndexes; - } - - /** - * @param noDictionaryBlockIndexes the noDictionaryBlockIndexes to set - */ - public void setNoDictionaryBlockIndexes(int[] noDictionaryBlockIndexes) { - this.noDictionaryBlockIndexes = noDictionaryBlockIndexes; - } - - /** - * @return the columnGroupToKeyStructureInfo - */ - public Map<Integer, KeyStructureInfo> getColumnGroupToKeyStructureInfo() { - return columnGroupToKeyStructureInfo; - } - - /** - * @param columnGroupToKeyStructureInfo the columnGroupToKeyStructureInfo to set - */ - public void setColumnGroupToKeyStructureInfo( - Map<Integer, KeyStructureInfo> columnGroupToKeyStructureInfo) { - this.columnGroupToKeyStructureInfo = columnGroupToKeyStructureInfo; - } - - /** - * @param columnIdToDcitionaryMapping the columnIdToDcitionaryMapping to set - */ - public void setColumnIdToDcitionaryMapping(Map<String, Dictionary> columnIdToDcitionaryMapping) { - this.columnIdToDcitionaryMapping = columnIdToDcitionaryMapping; - } - - public boolean isRawRecordDetailQuery() { - return isRawRecordDetailQuery; - } - - public void setRawRecordDetailQuery(boolean rawRecordDetailQuery) { - isRawRecordDetailQuery = rawRecordDetailQuery; - } - - /** - * @return the complexParentIndexToQueryMap - */ - public Map<Integer, GenericQueryType> getComlexDimensionInfoMap() { - return complexParentIndexToQueryMap; - } - - /** - * @param complexDimensionInfoMap the complexParentIndexToQueryMap to set - */ - public void setComplexDimensionInfoMap(Map<Integer, GenericQueryType> complexDimensionInfoMap) { - this.complexParentIndexToQueryMap = complexDimensionInfoMap; - } - - /** - * @return the complexColumnParentBlockIndexes - */ - public int[] getComplexColumnParentBlockIndexes() { - return complexColumnParentBlockIndexes; - } - - /** - * @param complexColumnParentBlockIndexes the complexColumnParentBlockIndexes to set - */ - public void setComplexColumnParentBlockIndexes(int[] complexColumnParentBlockIndexes) { - this.complexColumnParentBlockIndexes = complexColumnParentBlockIndexes; - } - - public QueryDimension[] getQueryDimensions() { - return queryDimensions; - } - - public void setQueryDimensions(QueryDimension[] queryDimensions) { - this.queryDimensions = queryDimensions; - } - - public QueryMeasure[] getQueryMeasures() { - return queryMeasures; - } - - public void setQueryMeasures(QueryMeasure[] queryMeasures) { - this.queryMeasures = queryMeasures; - } - - /** - * The method to set the number of blocklets to be scanned - * - * @param numberOfBlockletToScan - */ - public void setNumberOfBlockletToScan(int numberOfBlockletToScan) { - this.numberOfBlockletToScan = numberOfBlockletToScan; - } - - /** - * get the no of blocklet to be scanned - * - * @return - */ - public int getNumberOfBlockletToScan() { - return numberOfBlockletToScan; - } - - /** - * returns the blocklet index to be scanned - * - * @return - */ - public int getStartBlockletIndex() { - return startBlockletIndex; - } - - /** - * set the blocklet index to be scanned - * - * @param startBlockletIndex - */ - public void setStartBlockletIndex(int startBlockletIndex) { - this.startBlockletIndex = startBlockletIndex; - } - - public boolean isVectorBatchCollector() { - return vectorBatchCollector; - } - - public void setVectorBatchCollector(boolean vectorBatchCollector) { - this.vectorBatchCollector = vectorBatchCollector; - } - - public String getBlockId() { - return blockId; - } - - public void setBlockId(String blockId) { - this.blockId = blockId; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java deleted file mode 100644 index d742e02..0000000 --- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.scan.executor.infos; - -import org.apache.carbondata.core.keygenerator.KeyGenerator; - -/** - * Below class will store the structure of the key - * used during query execution - */ -public class KeyStructureInfo { - - /** - * it's actually a latest key generator - * last table block as this key generator will be used to - * to update the mdkey of the older slice with the new slice - */ - private KeyGenerator keyGenerator; - - /** - * mask bytes ranges for the query - */ - private int[] maskByteRanges; - - /** - * max key for query execution - */ - private byte[] maxKey; - - /** - * dimension ordinals inside the column group - */ - private int[] mdkeyQueryDimensionOrdinal; - - /** - * @return the keyGenerator - */ - public KeyGenerator getKeyGenerator() { - return keyGenerator; - } - - /** - * @param keyGenerator the keyGenerator to set - */ - public void setKeyGenerator(KeyGenerator keyGenerator) { - this.keyGenerator = keyGenerator; - } - - /** - * @return the maskByteRanges - */ - public int[] getMaskByteRanges() { - return maskByteRanges; - } - - /** - * @param maskByteRanges the maskByteRanges to set - */ - public void setMaskByteRanges(int[] maskByteRanges) { - this.maskByteRanges = maskByteRanges; - } - - /** - * @return the maxKey - */ - public byte[] getMaxKey() { - return maxKey; - } - - /** - * @param maxKey the maxKey to set - */ - public void setMaxKey(byte[] maxKey) { - this.maxKey = maxKey; - } - - public int[] getMdkeyQueryDimensionOrdinal() { - return mdkeyQueryDimensionOrdinal; - } - - public void setMdkeyQueryDimensionOrdinal(int[] mdkeyQueryDimensionOrdinal) { - this.mdkeyQueryDimensionOrdinal = mdkeyQueryDimensionOrdinal; - } -}
