http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java index 2e87051..2e98f68 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java @@ -20,14 +20,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; -import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; @@ -37,7 +34,6 @@ import org.apache.carbondata.core.cache.CacheType; import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.compression.Compressor; import org.apache.carbondata.core.datastore.compression.CompressorFactory; import org.apache.carbondata.core.keygenerator.KeyGenException; @@ -55,12 +51,10 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.scan.complextypes.ArrayQueryType; import org.apache.carbondata.core.scan.complextypes.PrimitiveQueryType; import org.apache.carbondata.core.scan.complextypes.StructQueryType; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.filter.GenericQueryType; import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf; -import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; import org.apache.carbondata.core.scan.model.ProjectionDimension; import org.apache.carbondata.core.scan.model.ProjectionMeasure; import org.apache.carbondata.core.util.CarbonUtil; @@ -212,7 +206,7 @@ public class QueryUtil { */ public static int[] getDimensionChunkIndexes(List<ProjectionDimension> queryDimensions, Map<Integer, Integer> dimensionOrdinalToChunkMapping, - List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions, + Set<CarbonDimension> filterDimensions, Set<Integer> allProjectionListDimensionIndexes) { // using set as in row group columns will point to same block Set<Integer> dimensionChunkIndex = new HashSet<Integer>(); @@ -238,13 +232,6 @@ public class QueryUtil { } } } - for (int i = 0; i < customAggregationDimension.size(); i++) { - chunkIndex = - dimensionOrdinalToChunkMapping.get(customAggregationDimension.get(i).getOrdinal()); - // not adding the children dimension as dimension aggregation - // is not push down in case of complex dimension - dimensionChunkIndex.add(chunkIndex); - } int[] dimensionIndex = ArrayUtils .toPrimitive(dimensionChunkIndex.toArray(new Integer[dimensionChunkIndex.size()])); Arrays.sort(dimensionIndex); @@ -470,58 +457,6 @@ public class QueryUtil { } /** - * Below method will be used to get the mapping of block index and its - * restructuring info - * - * @param queryDimensions query dimension from query model - * @param segmentProperties segment properties - * @return map of block index to its restructuring info - * @throws KeyGenException if problem while key generation - */ - public static Map<Integer, KeyStructureInfo> getColumnGroupKeyStructureInfo( - List<ProjectionDimension> queryDimensions, SegmentProperties segmentProperties) - throws KeyGenException { - Map<Integer, KeyStructureInfo> rowGroupToItsRSInfo = new HashMap<Integer, KeyStructureInfo>(); - // get column group id and its ordinal mapping of column group - Map<Integer, List<Integer>> columnGroupAndItsOrdinalMappingForQuery = - getColumnGroupAndItsOrdinalMapping(queryDimensions); - Map<Integer, KeyGenerator> columnGroupAndItsKeygenartor = - segmentProperties.getColumnGroupAndItsKeygenartor(); - - Iterator<Entry<Integer, List<Integer>>> iterator = - columnGroupAndItsOrdinalMappingForQuery.entrySet().iterator(); - KeyStructureInfo restructureInfos = null; - while (iterator.hasNext()) { - Entry<Integer, List<Integer>> next = iterator.next(); - KeyGenerator keyGenerator = columnGroupAndItsKeygenartor.get(next.getKey()); - restructureInfos = new KeyStructureInfo(); - // sort the ordinal - List<Integer> ordinal = next.getValue(); - List<Integer> mdKeyOrdinal = new ArrayList<Integer>(); - //Un sorted - List<Integer> mdKeyOrdinalForQuery = new ArrayList<Integer>(); - for (Integer ord : ordinal) { - mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord)); - mdKeyOrdinalForQuery.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord)); - } - Collections.sort(mdKeyOrdinal); - // get the masked byte range for column group - int[] maskByteRanges = getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator); - // max key for column group - byte[] maxKey = getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator); - restructureInfos.setKeyGenerator(keyGenerator); - restructureInfos.setMaskByteRanges(maskByteRanges); - restructureInfos.setMaxKey(maxKey); - restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils - .toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()]))); - rowGroupToItsRSInfo - .put(segmentProperties.getDimensionOrdinalToChunkMapping().get(ordinal.get(0)), - restructureInfos); - } - return rowGroupToItsRSInfo; - } - - /** * return true if given key is found in array * * @param data @@ -538,69 +473,6 @@ public class QueryUtil { } /** - * Below method will be used to create a mapping of column group columns - * this mapping will have column group id to all the dimension ordinal - * present in the column group This mapping will be used during query - * execution, to create a mask key for the column group dimension which will - * be used in aggregation and filter query as column group dimension will be - * stored in bit level - */ - private static Map<Integer, List<Integer>> getColumnGroupAndItsOrdinalMapping( - List<ProjectionDimension> origDimensions) { - - List<ProjectionDimension> dimensions = new ArrayList<>(origDimensions.size()); - dimensions.addAll(origDimensions); - /* - * sort based on column group id - */ - Collections.sort(dimensions, new Comparator<ProjectionDimension>() { - - @Override public int compare(ProjectionDimension o1, ProjectionDimension o2) { - return Integer - .compare(o1.getDimension().columnGroupId(), o2.getDimension().columnGroupId()); - } - }); - // list of row groups this will store all the row group column - Map<Integer, List<Integer>> columnGroupAndItsOrdinalsMapping = - new HashMap<Integer, List<Integer>>(); - // to store a column group - List<Integer> currentColumnGroup = null; - // current index - int index = 0; - // previous column group to check all the column of row id has bee - // selected - int prvColumnGroupId = -1; - while (index < dimensions.size()) { - // if dimension group id is not zero and it is same as the previous - // column group id - // then we need to add ordinal of that column as it belongs to same - // column group - if (dimensions.get(index).getDimension().hasEncoding(Encoding.IMPLICIT)) { - index++; - continue; - } else if (!dimensions.get(index).getDimension().isColumnar() - && dimensions.get(index).getDimension().columnGroupId() == prvColumnGroupId - && null != currentColumnGroup) { - currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal()); - } - - // if dimension is not a columnar then it is column group column - else if (!dimensions.get(index).getDimension().isColumnar()) { - currentColumnGroup = new ArrayList<Integer>(); - columnGroupAndItsOrdinalsMapping - .put(dimensions.get(index).getDimension().columnGroupId(), currentColumnGroup); - currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal()); - } - // update the row id every time,this is required to group the - // columns - // of the same row group - prvColumnGroupId = dimensions.get(index).getDimension().columnGroupId(); - index++; - } - return columnGroupAndItsOrdinalsMapping; - } - - /** * Below method will be used to get masked byte * * @param data actual data @@ -655,52 +527,6 @@ public class QueryUtil { } /** - * Below method will be used to get the key structure for the column group - * - * @param segmentProperties segment properties - * @param dimColumnEvaluatorInfo dimension evaluator info - * @return key structure info for column group dimension - * @throws KeyGenException - */ - public static KeyStructureInfo getKeyStructureInfo(SegmentProperties segmentProperties, - DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) throws KeyGenException { - int colGrpId = getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex()); - KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId); - List<Integer> mdKeyOrdinal = new ArrayList<Integer>(); - - mdKeyOrdinal.add(segmentProperties - .getColumnGroupMdKeyOrdinal(colGrpId, dimColumnEvaluatorInfo.getColumnIndex())); - int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator); - byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator); - KeyStructureInfo restructureInfos = new KeyStructureInfo(); - restructureInfos.setKeyGenerator(keyGenerator); - restructureInfos.setMaskByteRanges(maskByteRanges); - restructureInfos.setMaxKey(maxKey); - return restructureInfos; - } - - /** - * Below method will be used to get the column group id based on the ordinal - * - * @param segmentProperties segment properties - * @param ordinal ordinal to be searched - * @return column group id - */ - public static int getColumnGroupId(SegmentProperties segmentProperties, int ordinal) { - int[][] columnGroups = segmentProperties.getColumnGroups(); - int colGrpId = -1; - for (int i = 0; i < columnGroups.length; i++) { - if (columnGroups[i].length > 1) { - colGrpId++; - if (QueryUtil.searchInArray(columnGroups[i], ordinal)) { - break; - } - } - } - return colGrpId; - } - - /** * Below method will be used to get the map of for complex dimension and its type * which will be used to during query execution to *
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java index 91f2e4e..29b0f13 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java @@ -17,7 +17,6 @@ package org.apache.carbondata.core.scan.expression.exception; -import java.util.Locale; /** * FilterIllegalMemberException class representing exception which can cause while evaluating @@ -49,40 +48,12 @@ public class FilterIllegalMemberException extends Exception { /** * Constructor * - * @param msg The error message for this exception. - */ - public FilterIllegalMemberException(String msg, Throwable t) { - super(msg, t); - this.msg = msg; - } - - /** - * Constructor - * */ public FilterIllegalMemberException(Throwable t) { super(t); } /** - * This method is used to get the localized message. - * - * @param locale - A Locale object represents a specific geographical, - * political, or cultural region. - * @return - Localized error message. - */ - public String getLocalizedMessage(Locale locale) { - return ""; - } - - /** - * getLocalizedMessage - */ - @Override public String getLocalizedMessage() { - return super.getLocalizedMessage(); - } - - /** * getMessage */ public String getMessage() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java index a85c52e..969c4fd 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java @@ -17,8 +17,6 @@ package org.apache.carbondata.core.scan.expression.exception; -import java.util.Locale; - public class FilterUnsupportedException extends Exception { /** @@ -60,24 +58,6 @@ public class FilterUnsupportedException extends Exception { } /** - * This method is used to get the localized message. - * - * @param locale - A Locale object represents a specific geographical, - * political, or cultural region. - * @return - Localized error message. - */ - public String getLocalizedMessage(Locale locale) { - return ""; - } - - /** - * getLocalizedMessage - */ - @Override public String getLocalizedMessage() { - return super.getLocalizedMessage(); - } - - /** * getMessage */ public String getMessage() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java index 5e8010c..b01dc08 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java @@ -64,9 +64,8 @@ public class FalseExpression extends BinaryConditionalExpression { public ExpressionType getFilterExpressionType() { return ExpressionType.FALSE; } - @Override - public String getString() { - return null; + @Override public String getString() { + return "False(" + (null == left ? null : left.getString()); } @Override http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java index 5d09fb1..b49b6f9 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java @@ -59,9 +59,8 @@ public class RangeExpression extends BinaryConditionalExpression { return ExpressionType.RANGE; } - @Override - public String getString() { - return null; + @Override public String getString() { + return "Range(" + left.getString() + ',' + right.getString() + ')'; } @Override http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java index 3948589..43a05f4 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java @@ -62,9 +62,8 @@ public class TrueExpression extends BinaryConditionalExpression { public ExpressionType getFilterExpressionType() { return ExpressionType.TRUE; } - @Override - public String getString() { - return null; + @Override public String getString() { + return "True(" + (null == left ? null : left.getString()); } @Override http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index a99bdee..0587b33 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -239,7 +239,7 @@ public final class FilterUtil { DimColumnResolvedFilterInfo dimColResolvedFilterInfo, MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo, SegmentProperties segmentProperties) { - if (null != msrColResolvedFilterInfo && msrColResolvedFilterInfo.getMeasure().isColumnar()) { + if (null != msrColResolvedFilterInfo && msrColResolvedFilterInfo.getMeasure().isMeasure()) { CarbonMeasure measuresFromCurrentBlock = segmentProperties .getMeasureFromCurrentBlock(msrColResolvedFilterInfo.getMeasure().getColumnId()); if (null != measuresFromCurrentBlock) { @@ -420,7 +420,7 @@ public final class FilterUtil { MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo, SegmentProperties segmentProperties) { - if (null != msrColResolvedFilterInfo && msrColResolvedFilterInfo.getMeasure().isColumnar()) { + if (null != msrColResolvedFilterInfo && msrColResolvedFilterInfo.getMeasure().isMeasure()) { CarbonMeasure measuresFromCurrentBlock = segmentProperties .getMeasureFromCurrentBlock(msrColResolvedFilterInfo.getMeasure().getColumnId()); if (null != measuresFromCurrentBlock) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java index 24461a5..78055fe 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java @@ -48,7 +48,7 @@ import org.apache.carbondata.core.util.CarbonUtil; * for this Range. Also search the data block and set the required bitsets which falls within * the Range of the RANGE Expression. */ -public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl { +public class RangeValueFilterExecuterImpl implements FilterExecuter { private DimColumnResolvedFilterInfo dimColEvaluatorInfo; private Expression exp; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java index 1fce7a3..7e5826c 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java @@ -37,7 +37,6 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage; import org.apache.carbondata.core.datastore.page.ColumnPage; -import org.apache.carbondata.core.keygenerator.KeyGenException; import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator; import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; @@ -46,8 +45,6 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; -import org.apache.carbondata.core.scan.executor.util.QueryUtil; import org.apache.carbondata.core.scan.executor.util.RestructureUtil; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.MatchExpression; @@ -427,7 +424,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } } else { int dictionaryValue = readSurrogatesFromColumnChunk(blockChunkHolder, index, pageIndex, - dimColumnEvaluatorInfo, dimensionChunkIndex[i]); + dimensionChunkIndex[i]); if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { memberString = @@ -588,46 +585,17 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { * * @param blockChunkHolder * @param index - * @param dimColumnEvaluatorInfo * @return */ private int readSurrogatesFromColumnChunk(RawBlockletColumnChunks blockChunkHolder, int index, - int page, DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int chunkIndex) { + int page, int chunkIndex) { DimensionColumnPage dataChunk = blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex].decodeColumnPage(page); - if (dimColumnEvaluatorInfo.getDimension().isColumnar()) { - byte[] rawData = dataChunk.getChunkData(index); - ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE); - return CarbonUtil.getSurrogateKey(rawData, byteBuffer); - } else { - return readSurrogatesFromColumnGroupBlock(dataChunk, index, dimColumnEvaluatorInfo); - } - + byte[] rawData = dataChunk.getChunkData(index); + ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE); + return CarbonUtil.getSurrogateKey(rawData, byteBuffer); } - /** - * @param index - * @param dimColumnEvaluatorInfo - * @return read surrogate of given row of given column group dimension - */ - private int readSurrogatesFromColumnGroupBlock(DimensionColumnPage chunk, int index, - DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) { - try { - KeyStructureInfo keyStructureInfo = - QueryUtil.getKeyStructureInfo(segmentProperties, dimColumnEvaluatorInfo); - byte[] colData = chunk.getChunkData(index); - long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData); - int colGroupId = - QueryUtil.getColumnGroupId(segmentProperties, dimensionChunkIndex[0]); - return (int) result[segmentProperties - .getColumnGroupMdKeyOrdinal(colGroupId, dimensionChunkIndex[0])]; - } catch (KeyGenException e) { - LOGGER.error(e); - } - return 0; - } - - @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { BitSet bitSet = new BitSet(1); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java deleted file mode 100644 index bc3b39c..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.scan.filter.executer; - -import java.io.IOException; -import java.util.BitSet; - -import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; -import org.apache.carbondata.core.scan.filter.intf.RowIntf; -import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks; -import org.apache.carbondata.core.util.BitSetGroup; - -public class ValueBasedFilterExecuterImpl implements FilterExecuter { - - @Override - public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, - boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException { - return new BitSetGroup(0); - } - - @Override - public boolean applyFilter(RowIntf value, int dimOrdinalMax) - throws FilterUnsupportedException, IOException { - throw new FilterUnsupportedException("Unsupported ValueBasedFilterExecuterImpl on row"); - } - - @Override - public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { - return new BitSet(1); - } - - @Override - public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java deleted file mode 100644 index 32b1ae1..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.scan.filter.intf; - -import org.apache.carbondata.core.scan.expression.Expression; - -public class FilterOptimizerImpl implements FilterOptimizer { - - protected Expression filterExpression; - - public FilterOptimizerImpl(Expression filterExpression) { - this.filterExpression = filterExpression; - } - - @Override public Expression optimizeFilter() { - return filterExpression; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java index bbd33ec..3032bb5 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java @@ -19,14 +19,13 @@ package org.apache.carbondata.core.scan.filter.optimizer; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.RangeExpressionEvaluator; -import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerImpl; +import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer; -public class RangeFilterOptmizer extends FilterOptimizerImpl { +public class RangeFilterOptmizer implements FilterOptimizer { RangeExpressionEvaluator rangeExpEvaluator; public RangeFilterOptmizer(Expression filterExpression) { - super(filterExpression); this.rangeExpEvaluator = new RangeExpressionEvaluator(filterExpression); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java index c316b5c..3e58538 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java @@ -44,5 +44,4 @@ public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl { @Override public FilterExecuterType getFilterExecuterType() { return FilterExecuterType.TRUE; } - } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java index eadd502..aac76e8 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java @@ -36,7 +36,6 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil; import org.apache.carbondata.core.mutate.DeleteDeltaVo; import org.apache.carbondata.core.mutate.TupleIdEnum; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.core.scan.filter.GenericQueryType; import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch; @@ -105,16 +104,6 @@ public abstract class BlockletScannedResult { protected int[] noDictionaryColumnChunkIndexes; /** - * column group to is key structure info - * which will be used to get the key from the complete - * column group key - * For example if only one dimension of the column group is selected - * then from complete column group key it will be used to mask the key and - * get the particular column key - */ - protected Map<Integer, KeyStructureInfo> columnGroupKeyStructureInfo; - - /** * */ private Map<Integer, GenericQueryType> complexParentIndexToQueryMap; @@ -155,7 +144,6 @@ public abstract class BlockletScannedResult { this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize(); this.noDictionaryColumnChunkIndexes = blockExecutionInfo.getNoDictionaryColumnChunkIndexes(); this.dictionaryColumnChunkIndexes = blockExecutionInfo.getDictionaryColumnChunkIndex(); - this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo(); this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap(); this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes(); this.totalDimensionsSize = blockExecutionInfo.getProjectionDimensions().length; @@ -213,8 +201,7 @@ public abstract class BlockletScannedResult { int offset = 0; for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) { offset += dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter].fillRawData( - rowId, offset, completeKey, - columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i])); + rowId, offset, completeKey); } rowCounter++; return completeKey; @@ -232,8 +219,7 @@ public abstract class BlockletScannedResult { int column = 0; for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) { column = dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter] - .fillSurrogateKey(rowId, column, completeKey, - columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i])); + .fillSurrogateKey(rowId, column, completeKey); } rowCounter++; return completeKey; @@ -246,8 +232,7 @@ public abstract class BlockletScannedResult { int column = 0; for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) { column = dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter] - .fillVector(vectorInfo, column, - columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i])); + .fillVector(vectorInfo, column); } } @@ -258,8 +243,7 @@ public abstract class BlockletScannedResult { int column = 0; for (int i = 0; i < this.noDictionaryColumnChunkIndexes.length; i++) { column = dimensionColumnPages[noDictionaryColumnChunkIndexes[i]][pageCounter] - .fillVector(vectorInfo, column, - columnGroupKeyStructureInfo.get(noDictionaryColumnChunkIndexes[i])); + .fillVector(vectorInfo, column); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java index 7de3e71..26b1135 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java @@ -102,8 +102,7 @@ public class FilterQueryScannedResult extends BlockletScannedResult { column = dimensionColumnPages[chunkIndex][pageCounter].fillVector( pageFilteredRowId[pageCounter], vectorInfo, - column, - columnGroupKeyStructureInfo.get(chunkIndex)); + column); } } @@ -116,8 +115,7 @@ public class FilterQueryScannedResult extends BlockletScannedResult { column = dimensionColumnPages[chunkIndex][pageCounter].fillVector( pageFilteredRowId[pageCounter], vectorInfo, - column, - columnGroupKeyStructureInfo.get(chunkIndex)); + column); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java index 5956595..b5f9d66 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java @@ -72,8 +72,7 @@ public class NonFilterQueryScannedResult extends BlockletScannedResult { for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) { for (int j = 0; j < validRowIds.size(); j++) { columnDataOffsets[j] += dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter] - .fillRawData(validRowIds.get(j), columnDataOffsets[j], dictionaryKeyArrayList.get(j), - columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i])); + .fillRawData(validRowIds.get(j), columnDataOffsets[j], dictionaryKeyArrayList.get(j)); } } return dictionaryKeyArrayList; http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java index 1665ce6..c3d4df8 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java @@ -128,7 +128,6 @@ public class BlockletFullScanner implements BlockletScanner { } int lastPageSize = rawBlockletColumnChunks.getDataBlock().numRows() % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; - ; if (lastPageSize > 0) { numberOfRows[numberOfRows.length - 1] = lastPageSize; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java index 3d62d9e..4cce8af 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java @@ -198,13 +198,6 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper>, Serializa } /** - * @return - */ - public byte[] getImplicitColumnByteArray() { - return implicitColumnByteArray; - } - - /** * @param implicitColumnByteArray */ public void setImplicitColumnByteArray(byte[] implicitColumnByteArray) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java index 20536fb..1e30fe6 100644 --- a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java @@ -81,14 +81,16 @@ public class DriverQueryStatisticsRecorderImpl implements QueryStatisticsRecorde * @param statistic */ public void recordStatisticsForDriver(QueryStatistic statistic, String queryId) { - synchronized (lock) { - // refresh query Statistics Map - if (queryStatisticsMap.get(queryId) != null) { - queryStatisticsMap.get(queryId).add(statistic); - } else { - List<QueryStatistic> newQueryStatistics = new ArrayList<QueryStatistic>(); - newQueryStatistics.add(statistic); - queryStatisticsMap.put(queryId, newQueryStatistics); + if (null != queryId) { + synchronized (lock) { + // refresh query Statistics Map + if (queryStatisticsMap.get(queryId) != null) { + queryStatisticsMap.get(queryId).add(statistic); + } else { + List<QueryStatistic> newQueryStatistics = new ArrayList<QueryStatistic>(); + newQueryStatistics.add(statistic); + queryStatisticsMap.put(queryId, newQueryStatistics); + } } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java index ed60d37..9635896 100644 --- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java +++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java @@ -36,5 +36,4 @@ public class QueryStatisticsModel { public Map<String, QueryStatistic> getStatisticsTypeAndObjMap() { return statisticsTypeAndObjMap; } - } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java index 1b43b65..55381fb 100644 --- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java +++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java @@ -65,7 +65,7 @@ public class SegmentUpdateStatusManager { private static final LogService LOG = LogServiceFactory.getLogService(SegmentUpdateStatusManager.class.getName()); - private AbsoluteTableIdentifier identifier; + private final AbsoluteTableIdentifier identifier; private LoadMetadataDetails[] segmentDetails; private SegmentUpdateDetails[] updateDetails; private Map<String, SegmentUpdateDetails> blockAndDetailsMap; @@ -124,7 +124,7 @@ public class SegmentUpdateStatusManager { * @param actualBlockName * @return null if block is not present in segment update status. */ - public SegmentUpdateDetails getDetailsForABlock(String segID, String actualBlockName) { + private SegmentUpdateDetails getDetailsForABlock(String segID, String actualBlockName) { String blockIdentifier = CarbonUpdateUtil .getSegmentBlockNameKey(segID, actualBlockName); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java index 7cd0c18..f69c1ff 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java @@ -289,7 +289,6 @@ public abstract class AbstractDataFileFooterConverter { ColumnSchema wrapperColumnSchema = new ColumnSchema(); wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id()); wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name()); - wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar()); DataType dataType = thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type); if (DataTypes.isDecimal(dataType)) { DecimalType decimalType = (DecimalType) dataType; @@ -305,7 +304,6 @@ public abstract class AbstractDataFileFooterConverter { wrapperColumnSchema.setEncodingList(encoders); wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child()); wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision()); - wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id()); wrapperColumnSchema.setScale(externalColumnSchema.getScale()); wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value()); Map<String, String> properties = externalColumnSchema.getColumnProperties(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java deleted file mode 100644 index c1f3f5e..0000000 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.util; - -import java.io.IOException; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.constants.CarbonCommonConstants; - -/** - * Util class for merge activities of 2 loads. - */ -public class CarbonMergerUtil { - - /** - * Attribute for Carbon LOGGER - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(CarbonMergerUtil.class.getName()); - - public static int[] getCardinalityFromLevelMetadata(String path, String tableName) { - int[] localCardinality = null; - try { - localCardinality = CarbonUtil.getCardinalityFromLevelMetadataFile( - path + '/' + CarbonCommonConstants.LEVEL_METADATA_FILE + tableName + ".metadata"); - } catch (IOException e) { - LOGGER.error("Error occurred :: " + e.getMessage()); - } - - return localCardinality; - } - - /** - * read from the first non-empty level metadata - * @param paths paths - * @param tableName table name - * @return cardinality - */ - public static int[] getCardinalityFromLevelMetadata(String[] paths, String tableName) { - int[] localCardinality = null; - for (String path : paths) { - localCardinality = getCardinalityFromLevelMetadata(path, tableName); - if (null != localCardinality) { - break; - } - } - return localCardinality; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java index 58de030..8fc648b 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java @@ -20,9 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import java.util.Set; -import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.blocklet.BlockletEncodedColumnPage; import org.apache.carbondata.core.datastore.blocklet.EncodedBlocklet; import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage; @@ -42,7 +40,6 @@ import org.apache.carbondata.format.ColumnSchema; import org.apache.carbondata.format.CompressionCodec; import org.apache.carbondata.format.DataChunk2; import org.apache.carbondata.format.DataChunk3; -import org.apache.carbondata.format.Encoding; import org.apache.carbondata.format.FileFooter3; import org.apache.carbondata.format.FileHeader; import org.apache.carbondata.format.IndexHeader; @@ -54,6 +51,9 @@ import org.apache.carbondata.format.SegmentInfo; */ public class CarbonMetadataUtil { + private CarbonMetadataUtil() { + } + /** * Below method prepares the file footer object for carbon data file version 3 * @@ -222,28 +222,6 @@ public class CarbonMetadataUtil { } /** - * @param blockIndex - * @param encoding - * @param columnSchemas - * @param segmentProperties - * @return return true if given encoding is present in column - */ - private static boolean containsEncoding(int blockIndex, Encoding encoding, - List<ColumnSchema> columnSchemas, SegmentProperties segmentProperties) { - Set<Integer> dimOrdinals = segmentProperties.getDimensionOrdinalForBlock(blockIndex); - // column groups will always have dictionary encoding - if (dimOrdinals.size() > 1 && Encoding.DICTIONARY == encoding) { - return true; - } - for (Integer dimOrdinal : dimOrdinals) { - if (columnSchemas.get(dimOrdinal).encoders.contains(encoding)) { - return true; - } - } - return false; - } - - /** * Right now it is set to default values. We may use this in future */ public static ChunkCompressionMeta getSnappyChunkCompressionMeta() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java index 4df10ee..f7ace5e 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java @@ -821,10 +821,6 @@ public final class CarbonProperties { if (null != sessionPropertyValue) { return sessionPropertyValue; } - //TODO temporary fix - if ("carbon.leaf.node.size".equals(key)) { - return "120000"; - } return carbonProperties.getProperty(key); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 8afba76..e87e52c 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -36,7 +36,6 @@ import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; -import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel; import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException; import org.apache.carbondata.core.datastore.filesystem.CarbonFile; @@ -246,26 +245,6 @@ public final class CarbonUtil { } /** - * return ColumnGroupModel. check ColumnGroupModel for detail - * - * @param columnGroups : column groups - * @return ColumnGroupModel model - */ - public static ColumnGroupModel getColGroupModel(int[][] columnGroups) { - int[] columnSplit = new int[columnGroups.length]; - int noOfColumnStore = columnSplit.length; - - for (int i = 0; i < columnGroups.length; i++) { - columnSplit[i] = columnGroups[i].length; - } - ColumnGroupModel colGroupModel = new ColumnGroupModel(); - colGroupModel.setNoOfColumnStore(noOfColumnStore); - colGroupModel.setColumnSplit(columnSplit); - colGroupModel.setColumnGroup(columnGroups); - return colGroupModel; - } - - /** * This method will be used to update the dimension cardinality * * @param dimCardinality @@ -617,35 +596,6 @@ public final class CarbonUtil { } /** - * Read level metadata file and return cardinality - * - * @param levelPath - * @return - * @throws IOException - */ - public static int[] getCardinalityFromLevelMetadataFile(String levelPath) throws IOException { - DataInputStream dataInputStream = null; - int[] cardinality = null; - - try { - if (FileFactory.isFileExist(levelPath, FileFactory.getFileType(levelPath))) { - dataInputStream = - FileFactory.getDataInputStream(levelPath, FileFactory.getFileType(levelPath)); - - cardinality = new int[dataInputStream.readInt()]; - - for (int i = 0; i < cardinality.length; i++) { - cardinality[i] = dataInputStream.readInt(); - } - } - } finally { - closeStreams(dataInputStream); - } - - return cardinality; - } - - /** * From beeline if a delimeter is passed as \001, in code we get it as * escaped string as \\001. So this method will unescape the slash again and * convert it back t0 \001 @@ -1030,17 +980,11 @@ public final class CarbonUtil { */ public static int getNumberOfDimensionColumns(List<ColumnSchema> columnSchemaList) { int numberOfDimensionColumns = 0; - int previousColumnGroupId = -1; ColumnSchema columnSchema = null; for (int i = 0; i < columnSchemaList.size(); i++) { columnSchema = columnSchemaList.get(i); - if (columnSchema.isDimensionColumn() && columnSchema.isColumnar()) { + if (columnSchema.isDimensionColumn()) { numberOfDimensionColumns++; - } else if (columnSchema.isDimensionColumn()) { - if (previousColumnGroupId != columnSchema.getColumnGroupId()) { - previousColumnGroupId = columnSchema.getColumnGroupId(); - numberOfDimensionColumns++; - } } else { break; } @@ -1142,20 +1086,14 @@ public final class CarbonUtil { */ public static boolean[] identifyDimensionType(List<CarbonDimension> tableDimensionList) { List<Boolean> isDictionaryDimensions = new ArrayList<Boolean>(); - Set<Integer> processedColumnGroup = new HashSet<Integer>(); for (CarbonDimension carbonDimension : tableDimensionList) { List<CarbonDimension> childs = carbonDimension.getListOfChildDimensions(); //assuming complex dimensions will always be atlast if (null != childs && childs.size() > 0) { break; } - if (carbonDimension.isColumnar() && - hasEncoding(carbonDimension.getEncoder(), Encoding.DICTIONARY)) { + if (hasEncoding(carbonDimension.getEncoder(), Encoding.DICTIONARY)) { isDictionaryDimensions.add(true); - } else if (!carbonDimension.isColumnar()) { - if (processedColumnGroup.add(carbonDimension.columnGroupId())) { - isDictionaryDimensions.add(true); - } } else { isDictionaryDimensions.add(false); } @@ -1697,7 +1635,6 @@ public final class CarbonUtil { return valueEncoderMeta; } - /** * Below method will be used to convert indexes in range * Indexes=[0,1,2,3,4,5,6,7,8,9] @@ -2195,7 +2132,6 @@ public final class CarbonUtil { wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id()); wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId()); wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name()); - wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar()); DataType dataType = thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type); if (DataTypes.isDecimal(dataType)) { DecimalType decimalType = (DecimalType) dataType; @@ -2211,7 +2147,6 @@ public final class CarbonUtil { wrapperColumnSchema.setEncodingList(encoders); wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child()); wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision()); - wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id()); wrapperColumnSchema.setScale(externalColumnSchema.getScale()); wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value()); wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal()); http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java index 07391dc..b20a336 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java @@ -126,13 +126,8 @@ public class DataFileFooterConverter2 extends AbstractDataFileFooterConverter { ColumnSchema columnSchema = null; for (int i = 0; i < columnSchemaList.size(); i++) { columnSchema = columnSchemaList.get(i); - if (columnSchema.isDimensionColumn() && columnSchema.isColumnar()) { + if (columnSchema.isDimensionColumn()) { numberOfDimensionColumns++; - } else if (columnSchema.isDimensionColumn()) { - if (previousColumnGroupId != columnSchema.getColumnGroupId()) { - previousColumnGroupId = columnSchema.getColumnGroupId(); - numberOfDimensionColumns++; - } } else { break; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java b/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java deleted file mode 100644 index 2adfadc..0000000 --- a/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java +++ /dev/null @@ -1,457 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.util; - -import java.util.BitSet; - -import org.apache.carbondata.core.datastore.page.EncodedTablePage; -import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult; - -public class NodeHolder { - private EncodedTablePage encodedData; - - /** - * keyArray - */ - private byte[][] keyArray; - - /** - * dataArray - */ - private byte[][] dataArray; - - /** - * measureLenght - */ - private int[] measureLenght; - - /** - * startKey - */ - private byte[] startKey; - - /** - * endKey - */ - private byte[] endKey; - - /** - * entryCount - */ - private int entryCount; - /** - * keyLenghts - */ - private int[] keyLengths; - - /** - * keyIndexBlockLenght - */ - private int[] keyBlockIndexLength; - - /** - * isSortedKeyBlock - */ - private boolean[] isSortedKeyBlock; - - private byte[][] compressedIndex; - - private byte[][] compressedIndexMap; - - /** - * dataIndexMap - */ - private int[] dataIndexMapLength; - - /** - * compressedDataIndex - */ - private byte[][] compressedDataIndex; - - /** - * column max data - */ - private byte[][] dimensionColumnMaxData; - - /** - * column min data - */ - private byte[][] dimensionColumnMinData; - - private byte[][] measureColumnMaxData; - - private byte[][] measureColumnMinData; - - private SimpleStatsResult stats; - - /** - * array of rleEncodingForDictDim flag to identify the rleEncodingForDictDim - */ - private boolean[] rleEncodingForDictDim; - - /** - * true if given index is colgroup block - */ - private boolean[] colGrpBlocks; - - /** - * bit set which will holds the measure - * indexes which are null - */ - private BitSet[] measureNullValueIndex; - - /** - * total length of dimension values - */ - private int totalDimensionArrayLength; - - /** - * total length of all measure values - */ - private int totalMeasureArrayLength; - - /** - * data size this node holder is holding - */ - private int holderSize; - - /** - * to check all the pages to be - * written, this will be used for v3 format - */ - private boolean writeAll; - - /** - * @return the keyArray - */ - public byte[][] getKeyArray() { - return keyArray; - } - - /** - * @param keyArray the keyArray to set - */ - public void setKeyArray(byte[][] keyArray) { - this.keyArray = keyArray; - } - - /** - * @return the dataArray - */ - public byte[][] getDataArray() { - return dataArray; - } - - /** - * @param dataArray the dataArray to set - */ - public void setDataArray(byte[][] dataArray) { - this.dataArray = dataArray; - } - - /** - * @return the measureLenght - */ - public int[] getMeasureLenght() { - return measureLenght; - } - - /** - * @param measureLenght the measureLenght to set - */ - public void setMeasureLenght(int[] measureLenght) { - this.measureLenght = measureLenght; - } - - /** - * @return the startKey - */ - public byte[] getStartKey() { - return startKey; - } - - /** - * @param startKey the startKey to set - */ - public void setStartKey(byte[] startKey) { - this.startKey = startKey; - } - - /** - * @return the endKey - */ - public byte[] getEndKey() { - return endKey; - } - - /** - * @param endKey the endKey to set - */ - public void setEndKey(byte[] endKey) { - this.endKey = endKey; - } - - /** - * @return the entryCount - */ - public int getEntryCount() { - return entryCount; - } - - /** - * @param entryCount the entryCount to set - */ - public void setEntryCount(int entryCount) { - this.entryCount = entryCount; - } - - /** - * @return the keyLenghts - */ - public int[] getKeyLengths() { - return keyLengths; - } - - public void setKeyLengths(int[] keyLengths) { - this.keyLengths = keyLengths; - } - - /** - * @return the keyBlockIndexLength - */ - public int[] getKeyBlockIndexLength() { - return keyBlockIndexLength; - } - - /** - * @param keyBlockIndexLength the keyBlockIndexLength to set - */ - public void setKeyBlockIndexLength(int[] keyBlockIndexLength) { - this.keyBlockIndexLength = keyBlockIndexLength; - } - - /** - * @return the isSortedKeyBlock - */ - public boolean[] getIsSortedKeyBlock() { - return isSortedKeyBlock; - } - - /** - * @param isSortedKeyBlock the isSortedKeyBlock to set - */ - public void setIsSortedKeyBlock(boolean[] isSortedKeyBlock) { - this.isSortedKeyBlock = isSortedKeyBlock; - } - - /** - * @return the compressedIndexex - */ - public byte[][] getCompressedIndex() { - return compressedIndex; - } - - public void setCompressedIndex(byte[][] compressedIndex) { - this.compressedIndex = compressedIndex; - } - - /** - * @return the compressedIndexMap - */ - public byte[][] getCompressedIndexMap() { - return compressedIndexMap; - } - - /** - * @param compressedIndexMap the compressedIndexMap to set - */ - public void setCompressedIndexMap(byte[][] compressedIndexMap) { - this.compressedIndexMap = compressedIndexMap; - } - - /** - * @return the compressedDataIndex - */ - public byte[][] getCompressedDataIndex() { - return compressedDataIndex; - } - - /** - * @param compressedDataIndex the compressedDataIndex to set - */ - public void setCompressedDataIndex(byte[][] compressedDataIndex) { - this.compressedDataIndex = compressedDataIndex; - } - - /** - * @return the dataIndexMapLength - */ - public int[] getDataIndexMapLength() { - return dataIndexMapLength; - } - - /** - * @param dataIndexMapLength the dataIndexMapLength to set - */ - public void setDataIndexMapLength(int[] dataIndexMapLength) { - this.dataIndexMapLength = dataIndexMapLength; - } - - public byte[][] getDimensionColumnMaxData() { - return this.dimensionColumnMaxData; - } - - public void setDimensionColumnMaxData(byte[][] columnMaxData) { - this.dimensionColumnMaxData = columnMaxData; - } - - public byte[][] getDimensionColumnMinData() { - return this.dimensionColumnMinData; - } - - public void setDimensionColumnMinData(byte[][] columnMinData) { - this.dimensionColumnMinData = columnMinData; - } - - /** - * returns array of rleEncodingForDictDim flag to identify the aag blocks - * - * @return - */ - public boolean[] getRleEncodingForDictDim() { - return rleEncodingForDictDim; - } - - /** - * set array of rleEncodingForDictDim flag to identify the rleEncodingForDictDim - * - * @param rleEncodingForDictDim - */ - public void setRleEncodingForDictDim(boolean[] rleEncodingForDictDim) { - this.rleEncodingForDictDim = rleEncodingForDictDim; - } - - /** - * @return - */ - public boolean[] getColGrpBlocks() { - return this.colGrpBlocks; - } - - /** - * @param colGrpBlock true if block is column group - */ - public void setColGrpBlocks(boolean[] colGrpBlock) { - this.colGrpBlocks = colGrpBlock; - } - - /** - * @return the measureNullValueIndex - */ - public BitSet[] getMeasureNullValueIndex() { - return measureNullValueIndex; - } - - /** - * @param measureNullValueIndex the measureNullValueIndex to set - */ - public void setMeasureNullValueIndex(BitSet[] measureNullValueIndex) { - this.measureNullValueIndex = measureNullValueIndex; - } - - public int getTotalDimensionArrayLength() { - return totalDimensionArrayLength; - } - - public void setTotalDimensionArrayLength(int totalDimensionArrayLength) { - this.totalDimensionArrayLength = totalDimensionArrayLength; - } - - public int getTotalMeasureArrayLength() { - return totalMeasureArrayLength; - } - - public void setTotalMeasureArrayLength(int totalMeasureArrayLength) { - this.totalMeasureArrayLength = totalMeasureArrayLength; - } - - public byte[][] getMeasureColumnMaxData() { - return measureColumnMaxData; - } - - public void setMeasureColumnMaxData(byte[][] measureColumnMaxData) { - this.measureColumnMaxData = measureColumnMaxData; - } - - public byte[][] getMeasureColumnMinData() { - return measureColumnMinData; - } - - public void setMeasureColumnMinData(byte[][] measureColumnMinData) { - this.measureColumnMinData = measureColumnMinData; - } - - public int getHolderSize() { - return holderSize; - } - - public void setHolderSize(int holderSize) { - this.holderSize = holderSize; - } - - public void setWriteAll(boolean writeAll) { - this.writeAll = writeAll; - } - public boolean isWriteAll() { - return this.writeAll; - } - - public SimpleStatsResult getStats() { - return stats; - } - - public void setMeasureStats(SimpleStatsResult stats) { - this.stats = stats; - } - - public static byte[][] getKeyArray(EncodedTablePage encodedTablePage) { - int numDimensions = encodedTablePage.getNumDimensions(); - byte[][] keyArray = new byte[numDimensions][]; - for (int i = 0; i < numDimensions; i++) { - keyArray[i] = encodedTablePage.getDimension(i).getEncodedData().array(); - } - return keyArray; - } - - public static byte[][] getDataArray(EncodedTablePage encodedTablePage) { - int numMeasures = encodedTablePage.getNumMeasures(); - byte[][] dataArray = new byte[numMeasures][]; - for (int i = 0; i < numMeasures; i++) { - dataArray[i] = encodedTablePage.getMeasure(i).getEncodedData().array(); - } - return dataArray; - } - - public void setEncodedData(EncodedTablePage encodedData) { - this.encodedData = encodedData; - } - - public EncodedTablePage getEncodedData() { - return encodedData; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java b/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java index bd2d4c2..16dacb2 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java +++ b/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java @@ -130,10 +130,6 @@ public class TaskMetricsMap { return sum; } - public void clear() { - metricMap.clear(); - } - /** * adds spawaned thread callback entry in metricmap using parentThreadId * http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java index 14117f6..c88e248 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java +++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java @@ -155,13 +155,6 @@ public class CarbonTablePath { } /** - * Return absolute path of sort index file - */ - public static String getSortIndexFilePath(String tablePath, String columnId) { - return getMetadataPath(tablePath) + File.separator + columnId + SORT_INDEX_EXT; - } - - /** * Return sortindex file path based on specified dictionary path */ public static String getExternalSortIndexFilePath(String dictionaryPath, String columnId) { @@ -216,20 +209,6 @@ public class CarbonTablePath { } /** - * Gets absolute path of data file - * - * @param segmentId unique partition identifier - * @param filePartNo data file part number - * @param factUpdateTimeStamp unique identifier to identify an update - * @return absolute path of data file stored in carbon data format - */ - public static String getCarbonDataFilePath(String tablePath, String segmentId, Integer filePartNo, - Long taskNo, int batchNo, int bucketNumber, String factUpdateTimeStamp) { - return getSegmentPath(tablePath, segmentId) + File.separator + getCarbonDataFileName( - filePartNo, taskNo, bucketNumber, batchNo, factUpdateTimeStamp, segmentId); - } - - /** * Below method will be used to get the index file present in the segment folder * based on task id * @@ -285,20 +264,6 @@ public class CarbonTablePath { } } - public static String getCarbonIndexFilePath(String tablePath, String taskId, String segmentId, - int batchNo, String bucketNumber, String timeStamp, - ColumnarFormatVersion columnarFormatVersion) { - switch (columnarFormatVersion) { - case V1: - case V2: - return getCarbonIndexFilePath(tablePath, taskId, segmentId, bucketNumber); - default: - String segmentDir = getSegmentPath(tablePath, segmentId); - return segmentDir + File.separator + getCarbonIndexFileName(Long.parseLong(taskId), - Integer.parseInt(bucketNumber), batchNo, timeStamp, segmentId); - } - } - private static String getCarbonIndexFileName(String taskNo, int bucketNumber, String factUpdatedtimeStamp, String segmentNo) { if (bucketNumber == -1) { @@ -359,13 +324,6 @@ public class CarbonTablePath { // This partition is not used in any code logic, just keep backward compatibility public static final String DEPRECATED_PATITION_ID = "0"; - /** - * Return true if tablePath exists - */ - public static boolean exists(String tablePath) { - return FileFactory.getCarbonFile(tablePath, FileFactory.getFileType(tablePath)).exists(); - } - public static String getPartitionDir(String tablePath) { return getFactDir(tablePath) + File.separator + PARTITION_PREFIX + CarbonTablePath.DEPRECATED_PATITION_ID; @@ -734,14 +692,6 @@ public class CarbonTablePath { } /** - * Get the segment lock file according to table path and segment load name - */ - public static String getSegmentLockFilePath(String tablePath, String loadName) { - return getLockFilesDirPath(tablePath) + CarbonCommonConstants.FILE_SEPARATOR + - addSegmentPrefix(loadName) + LockUsage.LOCK; - } - - /** * return true if this lock file is a segment lock file otherwise false. */ public static boolean isSegmentLockFilePath(String lockFileName) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/writer/ByteArrayHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/ByteArrayHolder.java b/core/src/main/java/org/apache/carbondata/core/writer/ByteArrayHolder.java deleted file mode 100644 index 35c1f52..0000000 --- a/core/src/main/java/org/apache/carbondata/core/writer/ByteArrayHolder.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.core.writer; - -import java.util.Arrays; - -import org.apache.carbondata.core.util.ByteUtil; - -public class ByteArrayHolder implements Comparable<ByteArrayHolder> { - - /** - * mdkey - */ - private byte[] mdKey; - - /** - * primary key - */ - private int primaryKey; - - /** - * @param mdKey - * @param primaryKey - */ - public ByteArrayHolder(byte[] mdKey, int primaryKey) { - this.mdKey = mdKey; - this.primaryKey = primaryKey; - } - - @Override public int compareTo(ByteArrayHolder o) { - return ByteUtil.compare(mdKey, o.mdKey); - } - - @Override public boolean equals(Object obj) { - // TODO Auto-generated method stub - if (obj instanceof ByteArrayHolder) { - if (0 == ByteUtil.compare(mdKey, ((ByteArrayHolder) obj).mdKey)) { - return true; - } - } - return false; - } - - @Override public int hashCode() { - int prime = 31; - int result = prime * Arrays.hashCode(mdKey); - result = result + prime * primaryKey; - return result; - } - - public byte[] getMdKey() { - return mdKey; - } - - public int getPrimaryKey() { - return primaryKey; - } - -} - http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java index 3db1c4f..f0cc409 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java @@ -25,12 +25,5 @@ import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails; */ public interface CarbonDeleteDeltaWriter { - /** - * write method that accepts comma seperated deleted records - * - * @param value delete record details - * @throws IOException if an I/O error occurs - */ - void write(String value) throws IOException; void write(DeleteDeltaBlockDetails deleteBlockDetails) throws IOException; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java index 506d290..4cf3827 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java @@ -58,38 +58,6 @@ public class CarbonDeleteDeltaWriterImpl implements CarbonDeleteDeltaWriter { } /** - * This method will write the deleted records data in to disk. - * - * @param value deleted records - * @throws IOException if an I/O error occurs - */ - @Override public void write(String value) throws IOException { - BufferedWriter brWriter = null; - try { - FileFactory.createNewFile(filePath, fileType); - dataOutStream = FileFactory.getDataOutputStream(filePath, fileType); - brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream, - CarbonCommonConstants.DEFAULT_CHARSET)); - brWriter.write(value); - } catch (IOException ioe) { - LOGGER.error("Error message: " + ioe.getLocalizedMessage()); - throw ioe; - } finally { - if (null != brWriter) { - brWriter.flush(); - } - if (null != dataOutStream) { - dataOutStream.flush(); - } - if (null != brWriter) { - brWriter.close(); - } - - } - - } - - /** * This method will write the deleted records data in the json format. * @param deleteDeltaBlockDetails * @throws IOException http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java index d7b1a0f..28f81f3 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java @@ -26,7 +26,6 @@ import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl; import org.apache.carbondata.core.fileoperations.FileWriteOperation; import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.thrift.TBase; import org.apache.thrift.TException; import org.apache.thrift.protocol.TCompactProtocol; @@ -122,16 +121,6 @@ public class ThriftWriter { } /** - * Write the offset to the file - * - * @param offset - * @throws IOException - */ - public void writeOffset(long offset) throws IOException { - dataOutputStream.writeLong(offset); - } - - /** * Close the file stream. */ public void close() throws IOException { @@ -151,13 +140,4 @@ public class ThriftWriter { dataOutputStream = null; } } - - /** - * Flush data to HDFS file - */ - public void sync() throws IOException { - if (dataOutputStream instanceof FSDataOutputStream) { - ((FSDataOutputStream) dataOutputStream).hsync(); - } - } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java b/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java index 53349b8..5f9a05c 100644 --- a/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java +++ b/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java @@ -68,40 +68,6 @@ public class OperationListenerBus { } /** - * Removes a given map from the event. This method is thread-safe and can be called - * in any thread. - * - * @param eventType - */ - public void removeEvent(String eventType) { - eventMap.remove(eventType); - } - - /** - * Removes a given listener for an event. This method is thread-safe and can be called - * in any thread. - * - * @param eventType - * @param operationEventListener - */ - public void removeListener(String eventType, OperationEventListener operationEventListener) { - List<OperationEventListener> operationEventListeners = eventMap.get(eventType); - if (null != operationEventListeners) { - operationEventListeners.remove(operationEventListener); - } - } - - /** - * Notify all registered listeners on occurrence of an event - * Should be used for stateless events which cannot be mapped to a operation - * - * @param event - */ - public void fireEvent(Event event) throws Exception { - fireEvent(event, new OperationContext()); - } - - /** * Notify all registered listeners on occurrence of an event * * @param event http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java index 7c48f37..deb1a29 100644 --- a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java +++ b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java @@ -17,22 +17,16 @@ package org.apache.carbondata.core.cache; -import java.io.IOException; -import java.lang.reflect.Field; - import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache; import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastore.SegmentTaskIndexStore; -import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier; import org.apache.carbondata.core.util.CarbonProperties; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -72,32 +66,4 @@ public class CacheProviderTest { assertFalse(reverseDictionaryCache instanceof ForwardDictionaryCache); cacheProvider.dropAllCache(); } - - /** - * to test the driver and executor lru memory configuration - * - * @throws IOException - * @throws NoSuchFieldException - * @throws IllegalAccessException - */ - @Test public void driverExecutorCacheConfTest() - throws IOException, NoSuchFieldException, IllegalAccessException { - // get cache provider instance - CacheProvider cacheProvider = CacheProvider.getInstance(); - cacheProvider.dropAllCache(); - CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true"); - Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexStore> driverCache = - cacheProvider.createCache(CacheType.DRIVER_BTREE); - Field carbonLRUCacheField = SegmentTaskIndexStore.class.getDeclaredField("lruCache"); - carbonLRUCacheField.setAccessible(true); - CarbonLRUCache carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(driverCache); - Field lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize"); - lruCacheMemorySizeField.setAccessible(true); - long lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache); - String driverCacheSize = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE); - assertEquals(1024 * 1024 * Integer.parseInt(driverCacheSize), lruCacheMemorySize); - // drop cache - cacheProvider.dropAllCache(); - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java index 7d829b9..b572a36 100644 --- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java +++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java @@ -87,9 +87,9 @@ public class AbstractDictionaryCacheTest { * prepare the dataset required for running test cases */ protected void prepareDataSet() { - dataSet1 = Arrays.asList(new String[] { "a", "b", "c" }); - dataSet2 = Arrays.asList(new String[] { "d", "e", "f" }); - dataSet3 = Arrays.asList(new String[] { "b", "c", "a", "d" }); + dataSet1 = Arrays.asList("a", "b", "c"); + dataSet2 = Arrays.asList("d", "e", "f"); + dataSet3 = Arrays.asList("b", "c", "a", "d"); } /**