http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java index 4907dd9..b527f37 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java @@ -20,8 +20,8 @@ package org.apache.carbondata.core.datastore.impl.btree; import java.io.IOException; -import org.apache.carbondata.core.datastorage.FileHolder; import org.apache.carbondata.core.datastore.DataRefNode; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.IndexKey; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java index 3176b9b..7035fa1 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java @@ -22,8 +22,8 @@ import java.util.ArrayList; import java.util.List; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.FileHolder; import org.apache.carbondata.core.datastore.DataRefNode; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.IndexKey; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java index 4951c95..cfa0d2c 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java @@ -20,8 +20,8 @@ package org.apache.carbondata.core.datastore.impl.btree; import java.io.IOException; -import org.apache.carbondata.core.datastorage.FileHolder; import org.apache.carbondata.core.datastore.BTreeBuilderInfo; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java new file mode 100644 index 0000000..d74cf92 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.impl.data.compressed; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.NodeMeasureDataStore; +import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder; +import org.apache.carbondata.core.datastore.compression.WriterCompressModel; +import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.util.ValueCompressionUtil; + +public abstract class AbstractHeavyCompressedDoubleArrayDataStore + implements NodeMeasureDataStore //NodeMeasureDataStore<double[]> +{ + + private LogService LOGGER = + LogServiceFactory.getLogService(AbstractHeavyCompressedDoubleArrayDataStore.class.getName()); + + /** + * values. + */ + protected ValueCompressionHolder[] values; + + /** + * compressionModel. + */ + protected WriterCompressModel compressionModel; + + /** + * type + */ + private char[] type; + + /** + * AbstractHeavyCompressedDoubleArrayDataStore constructor. + * + * @param compressionModel + */ + public AbstractHeavyCompressedDoubleArrayDataStore(WriterCompressModel compressionModel) { + this.compressionModel = compressionModel; + if (null != compressionModel) { + this.type = compressionModel.getType(); + values = + new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length]; + } + } + + // this method first invokes encoding routine to encode the data chunk, + // followed by invoking compression routine for preparing the data chunk for writing. + @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) { + byte[][] returnValue = new byte[values.length][]; + for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) { + values[i] = compressionModel.getValueCompressionHolder()[i]; + if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) { + // first perform encoding of the data chunk + values[i].setValue( + ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i]) + .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i], + compressionModel.getMaxValue()[i], + compressionModel.getMantissa()[i])); + } else { + values[i].setValue(dataHolder[i].getWritableByteArrayValues()); + } + values[i].compress(); + returnValue[i] = values[i].getCompressedData(); + } + + return returnValue; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java new file mode 100644 index 0000000..793fc92 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.impl.data.compressed; + +import org.apache.carbondata.core.datastore.compression.WriterCompressModel; + +public class HeavyCompressedDoubleArrayDataInMemoryStore + extends AbstractHeavyCompressedDoubleArrayDataStore { + + public HeavyCompressedDoubleArrayDataInMemoryStore(WriterCompressModel compressionModel) { + super(compressionModel); + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java index 7db989a..44941c9 100644 --- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java @@ -19,10 +19,8 @@ package org.apache.carbondata.core.keygenerator.directdictionary; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.keygenerator.directdictionary.timestamp - .DateDirectDictionaryGenerator; -import org.apache.carbondata.core.keygenerator.directdictionary.timestamp - .TimeStampDirectDictionaryGenerator; +import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.DateDirectDictionaryGenerator; +import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator; import org.apache.carbondata.core.metadata.DataType; import org.apache.carbondata.core.util.CarbonProperties; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java index 59aaf53..d1df15d 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java @@ -22,7 +22,7 @@ import java.io.File; import java.io.Serializable; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.util.CarbonUtil; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java index 1f3090d..46ef5ec 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java @@ -21,7 +21,7 @@ package org.apache.carbondata.core.metadata; import java.util.BitSet; -import org.apache.carbondata.core.datastorage.compression.WriterCompressModel; +import org.apache.carbondata.core.datastore.compression.WriterCompressModel; public class BlockletInfoColumnar { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/metadata/ColumnIdentifier.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnIdentifier.java new file mode 100644 index 0000000..711339b --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnIdentifier.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.metadata; + +import java.io.Serializable; +import java.util.Map; + +/** + * Column unique identifier + */ +public class ColumnIdentifier implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 1L; + + /** + * column id + */ + private String columnId; + + /** + * column properties + */ + private Map<String, String> columnProperties; + + private DataType dataType; + + /** + * @param columnId + * @param columnProperties + */ + public ColumnIdentifier(String columnId, Map<String, String> columnProperties, + DataType dataType) { + this.columnId = columnId; + this.columnProperties = columnProperties; + this.dataType = dataType; + } + + /** + * @return columnId + */ + public String getColumnId() { + return columnId; + } + + /** + * @param columnProperty + * @return + */ + public String getColumnProperty(String columnProperty) { + String property = null; + if (null != columnProperties) { + property = columnProperties.get(columnProperty); + } + return property; + } + + public DataType getDataType() { + return this.dataType; + } + + @Override public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((columnId == null) ? 0 : columnId.hashCode()); + return result; + } + + @Override public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ColumnIdentifier other = (ColumnIdentifier) obj; + if (columnId == null) { + if (other.columnId != null) { + return false; + } + } else if (!columnId.equals(other.columnId)) { + return false; + } + return true; + } + + @Override public String toString() { + return "ColumnIdentifier [columnId=" + columnId + "]"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java index 7fb77c1..01000e8 100644 --- a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java +++ b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java @@ -21,14 +21,13 @@ package org.apache.carbondata.core.path; import java.io.File; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.hadoop.fs.Path; -import static org.apache.carbondata.core.constants.CarbonCommonConstants.INVALID_SEGMENT_ID; /** * Helps to get Table content paths. @@ -540,9 +539,9 @@ public class CarbonTablePath extends Path { return segmentDirSplits[1]; } } catch (Exception e) { - return INVALID_SEGMENT_ID; + return CarbonCommonConstants.INVALID_SEGMENT_ID; } - return INVALID_SEGMENT_ID; + return CarbonCommonConstants.INVALID_SEGMENT_ID; } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java index c313080..295cdc1 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java @@ -30,7 +30,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.atomic.AtomicFileOperations; import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.update.DeleteDeltaBlockDetails; import org.apache.carbondata.core.util.CarbonUtil; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java index 4d3c9cc..df92a3b 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java @@ -33,7 +33,7 @@ import java.util.concurrent.TimeUnit; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.update.DeleteDeltaBlockDetails; import org.apache.carbondata.core.update.DeleteDeltaBlockletDetails; import org.apache.carbondata.core.util.CarbonProperties; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java b/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java index 8a7abd6..0e3a3e0 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java @@ -22,7 +22,7 @@ package org.apache.carbondata.core.reader; import java.io.DataInputStream; import java.io.IOException; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.thrift.TBase; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java index 4472f4d..539cb6c 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java @@ -23,7 +23,7 @@ import java.util.List; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.path.CarbonTablePath; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java index 967ee5c..401c148 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java @@ -36,8 +36,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil; import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType; import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; -import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor - .FilterInfoTypeVisitorFactory; +import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor.FilterInfoTypeVisitorFactory; public class ConditionalFilterResolverImpl implements FilterResolverIntf { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java index fcfd924..af73b69 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java @@ -24,8 +24,8 @@ import java.util.List; import org.apache.carbondata.common.CarbonIterator; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.datastorage.FileHolder; import org.apache.carbondata.core.datastore.DataRefNode; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.scan.collector.ScannedResultCollector; import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedResultCollector; import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedVectorResultCollector; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java index 788488a..075ad6f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java @@ -18,8 +18,8 @@ */ package org.apache.carbondata.core.scan.processor; -import org.apache.carbondata.core.datastorage.FileHolder; import org.apache.carbondata.core.datastore.DataRefNode; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java index 6df1b98..48e3a0e 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java @@ -21,7 +21,7 @@ package org.apache.carbondata.core.scan.processor.impl; import java.util.ArrayList; import java.util.List; -import org.apache.carbondata.core.datastorage.FileHolder; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator; import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java index 69bd273..0ae02e1 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java @@ -25,10 +25,10 @@ import org.apache.carbondata.common.CarbonIterator; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.FileHolder; -import org.apache.carbondata.core.datastorage.impl.FileFactory; import org.apache.carbondata.core.datastore.DataRefNode; import org.apache.carbondata.core.datastore.DataRefNodeFinder; +import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; import org.apache.carbondata.core.scan.model.QueryModel; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java index 3d17985..6787851 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java @@ -23,7 +23,7 @@ import java.io.IOException; import java.util.BitSet; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.FileHolder; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk; import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java index d41c57d..1070767 100644 --- a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java @@ -26,10 +26,10 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; +import static org.apache.carbondata.core.util.CarbonUtil.printLine; import org.apache.commons.lang3.StringUtils; -import static org.apache.carbondata.core.util.CarbonUtil.printLine; /** * Class will be used to record and log the query statistics http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java index 3af7e65..d9d5651 100644 --- a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java @@ -18,11 +18,8 @@ package org.apache.carbondata.core.update; import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -31,9 +28,9 @@ import java.util.Set; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.path.CarbonStorePath; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java index 9ebbbfb..c43369e 100644 --- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java @@ -37,7 +37,7 @@ import org.apache.carbondata.core.atomic.AtomicFileOperations; import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl; import org.apache.carbondata.core.atomic.FileWriteOperation; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.path.CarbonStorePath; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java index 425b039..371e6f9 100644 --- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java @@ -36,9 +36,9 @@ import org.apache.carbondata.core.atomic.AtomicFileOperations; import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl; import org.apache.carbondata.core.atomic.FileWriteOperation; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.path.CarbonStorePath; import org.apache.carbondata.core.path.CarbonTablePath; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java index ca741c1..ede5883 100644 --- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java @@ -24,8 +24,8 @@ import java.io.IOException; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.util.CarbonProperties; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java index eeca76c..1691b37 100644 --- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java @@ -27,7 +27,7 @@ import java.nio.channels.OverlappingFileLockException; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java index 35cca0f..c85e12d 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java @@ -31,9 +31,9 @@ import java.util.Set; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.compression.CompressorFactory; -import org.apache.carbondata.core.datastorage.compression.WriterCompressModel; import org.apache.carbondata.core.datastore.block.SegmentProperties; +import org.apache.carbondata.core.datastore.compression.CompressorFactory; +import org.apache.carbondata.core.datastore.compression.WriterCompressModel; import org.apache.carbondata.core.metadata.BlockletInfoColumnar; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.metadata.ValueEncoderMeta; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 7e59345..19e6835 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -45,18 +45,18 @@ import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.FileHolder; -import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel; -import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer; -import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel; -import org.apache.carbondata.core.datastorage.compression.WriterCompressModel; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.block.AbstractIndex; import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; +import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel; +import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; +import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel; +import org.apache.carbondata.core.datastore.compression.WriterCompressModel; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.DataType; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java index e590a5c..2615e6d 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java @@ -23,9 +23,9 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import org.apache.carbondata.core.datastorage.FileHolder; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.block.TableBlockInfo; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.core.metadata.blocklet.DataFileFooter; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java index 40f13f1..76c415d 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java @@ -27,31 +27,31 @@ import org.apache.carbondata.core.compression.BigDecimalCompressor; import org.apache.carbondata.core.compression.BigIntCompressor; import org.apache.carbondata.core.compression.DoubleCompressor; import org.apache.carbondata.core.compression.ValueCompressor; -import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel; -import org.apache.carbondata.core.datastorage.compression.ReaderCompressModel; -import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder; -import org.apache.carbondata.core.datastorage.compression.WriterCompressModel; -import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinByte; -import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinDefault; -import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinInt; -import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinLong; -import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinShort; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalByte; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalDefault; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalInt; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalLong; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinByte; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinDefault; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinInt; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinLong; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinShort; -import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalShort; -import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneByte; -import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneDefault; -import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneInt; -import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneLong; -import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneShort; -import org.apache.carbondata.core.datastorage.compression.type.CompressionBigDecimal; +import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel; +import org.apache.carbondata.core.datastore.compression.ReaderCompressModel; +import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder; +import org.apache.carbondata.core.datastore.compression.WriterCompressModel; +import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinByte; +import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinDefault; +import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinInt; +import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinLong; +import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinShort; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalByte; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalDefault; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalInt; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalLong; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinByte; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinDefault; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinInt; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinLong; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinShort; +import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalShort; +import org.apache.carbondata.core.datastore.compression.none.CompressionNoneByte; +import org.apache.carbondata.core.datastore.compression.none.CompressionNoneDefault; +import org.apache.carbondata.core.datastore.compression.none.CompressionNoneInt; +import org.apache.carbondata.core.datastore.compression.none.CompressionNoneLong; +import org.apache.carbondata.core.datastore.compression.none.CompressionNoneShort; +import org.apache.carbondata.core.datastore.compression.type.CompressionBigDecimal; import org.apache.carbondata.core.metadata.ValueEncoderMeta; public final class ValueCompressionUtil { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java index 77fb571..51c52db 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java @@ -27,7 +27,7 @@ import java.io.OutputStreamWriter; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.update.DeleteDeltaBlockDetails; import org.apache.carbondata.core.util.CarbonUtil; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java index e368788..896f334 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java @@ -28,8 +28,8 @@ import java.util.List; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.path.CarbonTablePath; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java index 6a1025a..3f446f5 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java @@ -22,7 +22,7 @@ package org.apache.carbondata.core.writer; import java.io.DataOutputStream; import java.io.IOException; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.hadoop.fs.FSDataOutputStream; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java index 89a1c52..5acaa49 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java @@ -26,8 +26,8 @@ import java.util.List; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.path.CarbonTablePath; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java index 30bfdd5..3438766 100644 --- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java +++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java @@ -37,8 +37,8 @@ import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.metadata.DataType; import org.apache.carbondata.core.path.CarbonStorePath; import org.apache.carbondata.core.path.CarbonTablePath; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.writer.CarbonDictionaryWriter; import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java index 8bc6af4..b4850d0 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java @@ -19,8 +19,8 @@ package org.apache.carbondata.core.datastorage; -import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder; -import org.apache.carbondata.core.datastorage.impl.CompressedDataMeasureDataWrapper; +import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder; +import org.apache.carbondata.core.datastore.impl.CompressedDataMeasureDataWrapper; import org.junit.BeforeClass; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java index 62700db..0b2c3bf 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java @@ -24,7 +24,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; -import org.apache.carbondata.core.datastorage.impl.DFSFileHolderImpl; +import org.apache.carbondata.core.datastore.impl.DFSFileHolderImpl; import mockit.Mock; import mockit.MockUp; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java index 51874c0..f73994b 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java @@ -22,7 +22,7 @@ package org.apache.carbondata.core.datastorage; import java.io.File; import java.io.IOException; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import mockit.Mock; import mockit.MockUp; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java index 854b14d..c74c04e 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java @@ -24,7 +24,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; -import org.apache.carbondata.core.datastorage.impl.FileHolderImpl; +import org.apache.carbondata.core.datastore.impl.FileHolderImpl; import mockit.Mock; import mockit.MockUp; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java deleted file mode 100644 index 29d2eae..0000000 --- a/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.datastorage.columnar; - -import java.util.List; - -import org.junit.BeforeClass; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class ColumnarKeyStoreDataHolderUnitTest { - - private static ColumnarKeyStoreDataHolder columnarKeyStoreDataHolder; - private static ColumnarKeyStoreMetadata columnarKeyStoreMetadata; - - @BeforeClass public static void setup() { - byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 }; - int eachRowSize = 2; - int[] reverseIndex = new int[] { 1, 5, 6, 3, 8 }; - columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize); - columnarKeyStoreMetadata.setColumnReverseIndex(reverseIndex); - columnarKeyStoreDataHolder = - new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata); - } - - @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndex() { - byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 }; - int eachRowSize = 1; - ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize); - ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew = - new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata); - int columnIndex = 5; - int expected_result = 8; - int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex); - assertEquals(expected_result, result); - } - - @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndexAndRowSizeTwo() { - byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 }; - int eachRowSize = 2; - ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize); - ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew = - new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata); - int columnIndex = 0; - int expected_result = 4104; - int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex); - assertEquals(expected_result, result); - } - - @Test public void testGetSurrogateKeyWithNotNullINGetColumnReverseIndex() { - int columnIndex = 0; - int expected_result = 8232; - int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex); - assertEquals(expected_result, result); - } - - @Test(expected = ArrayIndexOutOfBoundsException.class) - public void testExceptionInGetSurrogateKey() { - int columnIndex = 10; - int expected_result = 8232; - int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex); - assertEquals(expected_result, result); - } - - @Test public void testGetSurrogateKeyWithListOfByteWhileCreatingObject() { - byte[] keyBlockData = new byte[] { 32, 64, 32, 40, 64, 8, 8 }; - List<byte[]> noDictionaryValBasedKeyBlockData = new java.util.ArrayList<>(); - noDictionaryValBasedKeyBlockData.add(keyBlockData); - new ColumnarKeyStoreDataHolder(columnarKeyStoreMetadata); - int columnIndex = 0; - int expected_result = 8232; - int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex); - assertEquals(expected_result, result); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java deleted file mode 100644 index 49323d8..0000000 --- a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.filesystem; - -import mockit.Mock; -import mockit.MockUp; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Options; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class AlluxioCarbonFileTest { - - private static AlluxioCarbonFile alluxioCarbonFile; - private static FileStatus fileStatus; - private static FileStatus fileStatusWithOutDirectoryPermission; - private static String fileName; - private static File file; - - - @BeforeClass - static public void setUp() { - file = new File("Test.carbondata"); - if (!file.exists()) - try { - file.createNewFile(); - } catch (IOException e) { - e.printStackTrace(); - } - try { - FileOutputStream oFile = new FileOutputStream(file, true); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - - fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath())); - fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath())); - fileName = file.getAbsolutePath(); - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - } - - @AfterClass - static public void cleanUp() { - file.delete(); - } - - @Test - public void testRenameForceForException() throws IOException { - - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - throw new IOException(); - } - - }; - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - alluxioCarbonFile.renameForce(fileName); - } - - @Test - public void testListFilesWithOutDirectoryPermission() { - alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission); - assertTrue(alluxioCarbonFile.listFiles() == null); - } - - @Test - public void testConstructorWithFilePath() { - alluxioCarbonFile = new AlluxioCarbonFile(file.getAbsolutePath()); - assertTrue(alluxioCarbonFile instanceof AlluxioCarbonFile); - } - - @Test - public void testListFilesForNullListStatus() { - alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission); - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus[] listStatus(Path var1) throws IOException { - - return null; - } - - }; - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - assertTrue(alluxioCarbonFile.listFiles().length == 0); - } - - @Test - public void testListDirectory() { - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus[] listStatus(Path var1) throws IOException { - - FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))}; - return fileStatus; - } - - }; - - assertTrue(alluxioCarbonFile.listFiles().length == 1); - } - - @Test - public void testListFilesForException() throws IOException { - alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission); - - new MockUp<FileStatus>() { - @Mock - public Path getPath() { - return new Path(file.getAbsolutePath()); - } - - }; - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - throw new IOException(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus[] listStatus(Path var1) throws IOException { - - throw new IOException(); - } - - }; - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - alluxioCarbonFile.listFiles(); - } - - @Test - public void testListFilesWithCarbonFilter() { - CarbonFileFilter carbonFileFilter = new CarbonFileFilter() { - - @Override - public boolean accept(CarbonFile file) { - return true; - } - }; - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 1); - } - - @Test - public void testlistFilesWithoutFilter() { - CarbonFileFilter carbonFileFilter = new CarbonFileFilter() { - - @Override - public boolean accept(CarbonFile file) { - return false; - } - }; - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus[] listStatus(Path var1) throws IOException { - - FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))}; - return fileStatus; - } - - }; - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 0); - } - - @Test - public void testGetParentFile() { - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<Path>() { - @Mock - public Path getParent() { - return new Path(file.getAbsolutePath() - ); - } - - }; - new MockUp<FileStatus>() { - @Mock - public Path getPath() { - return new Path(file.getAbsolutePath()); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus getFileStatus(Path path) throws IOException { - - return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath())); - } - - }; - - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - assertFalse(alluxioCarbonFile.getParentFile().equals(null)); - } - - @Test - public void testForNonDisributedSystem() { - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new WebHdfsFileSystem(); - } - - }; - assertFalse(alluxioCarbonFile.renameForce(fileName)); - } - - @Test - public void testrenameForceForDisributedSystem() { - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public void rename(Path src, Path dst, final Options.Rename... options) throws IOException { - - } - - }; - - alluxioCarbonFile = new AlluxioCarbonFile(fileStatus); - assertTrue(alluxioCarbonFile.renameForce(fileName)); - - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java deleted file mode 100644 index b09e14e..0000000 --- a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java +++ /dev/null @@ -1,400 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.filesystem; - -import mockit.Mock; -import mockit.MockUp; -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Options; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; -import org.apache.hadoop.util.Progressable; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - - -public class HDFSCarbonFileTest { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(HDFSCarbonFile.class.getName()); - private static HDFSCarbonFile hdfsCarbonFile; - private static FileStatus fileStatus = null; - private static FileStatus fileStatusWithOutDirectoryPermission; - private static String fileName = null; - private static FileSystem fs = null; - private static Path pt; - - @BeforeClass - static public void setUp() throws IOException { - Configuration config = new Configuration(); -//adding local hadoop configuration - config.addResource(new Path("core-site.xml")); - config.addResource(new Path("hdfs-site.xml")); - fileName = "Test.carbondata"; //this path is HDFS path - pt = new Path(fileName); - fs = FileSystem.get(new Configuration(config)); - fs.create(pt); - if (fs.exists(pt)) { - OutputStream os = fs.create(pt, - new Progressable() { - public void progress() { - LOGGER.info("Started Writing to File==="); - } - }); - BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); - br.write("Hello World"); - br.close(); - fs.close(); - - fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName)); - fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(fileName)); - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - - } - } - - @AfterClass - static public void cleanUp() { - try { - fs.delete(pt, true); - } catch (IOException e) { - LOGGER.error("Exception Occured" + e.getMessage()); - } - } - - @Test - public void testRenameForceForException() throws IOException { - - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - throw new IOException(); - } - - }; - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - hdfsCarbonFile.renameForce(fileName); - } - - @Test - public void testListFilesWithOutDirectoryPermission() { - hdfsCarbonFile = new HDFSCarbonFile(fileStatusWithOutDirectoryPermission); - new MockUp<FileStatus>() { - @Mock - public boolean isDirectory() { - return false; - } - - }; - - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - - new MockUp<FileStatus>() { - @Mock - public Path getPath() { - return new Path(fileName); - } - - }; - assertEquals(hdfsCarbonFile.listFiles(), null); - } - - @Test - public void testConstructorWithFilePath() { - hdfsCarbonFile = new HDFSCarbonFile(fileName); - assertTrue(hdfsCarbonFile instanceof HDFSCarbonFile); - } - - @Test - public void testListFilesForNullListStatus() { - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus[] listStatus(Path var1) throws IOException { - - return null; - } - - }; - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - assertEquals(hdfsCarbonFile.listFiles().length, 0); - } - - @Test - public void testListDirectory() { - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus[] listStatus(Path var1) throws IOException { - - FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))}; - return fileStatus; - } - - }; - - assertEquals(hdfsCarbonFile.listFiles().length, 1); - } - - @Test - public void testListFilesForException() throws IOException { - new HDFSCarbonFile(fileStatusWithOutDirectoryPermission); - - new MockUp<FileStatus>() { - @Mock - public Path getPath() { - return new Path(fileName); - } - - }; - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - throw new IOException(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus[] listStatus(Path var1) throws IOException { - - throw new IOException(); - } - - }; - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - hdfsCarbonFile.listFiles(); - } - - @Test - public void testListFilesWithCarbonFilter() { - CarbonFileFilter carbonFileFilter = new CarbonFileFilter() { - - @Override - public boolean accept(CarbonFile file) { - return true; - } - }; - new MockUp<FileStatus>() { - @Mock - public boolean isDirectory() { - return true; - } - - }; - - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - - new MockUp<FileStatus>() { - @Mock - public Path getPath() { - return new Path(fileName); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus[] listStatus(Path var1) throws IOException { - - FileStatus fileStatus[] = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))}; - return fileStatus; - } - - }; - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 1); - } - - @Test - public void testlistFilesWithoutFilter() { - CarbonFileFilter carbonFileFilter = new CarbonFileFilter() { - - @Override - public boolean accept(CarbonFile file) { - return false; - } - }; - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus[] listStatus(Path var1) throws IOException { - - FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))}; - return fileStatus; - } - - }; - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 0); - } - - @Test - public void testGetParentFileForNull() { - - new MockUp<Path>() { - @Mock - public Path getParent() { - return null; - } - - }; - new MockUp<FileStatus>() { - @Mock - public Path getPath() { - return new Path(fileName); - } - - }; - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - - new MockUp<FileStatus>() { - @Mock - public Path getPath() { - return new Path(fileName); - } - - }; - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - assertEquals(hdfsCarbonFile.getParentFile(), null); - } - - @Test - public void testGetParentFile() { - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<Path>() { - @Mock - public Path getParent() { - return new Path(fileName); - } - - }; - new MockUp<FileStatus>() { - @Mock - public Path getPath() { - return new Path(fileName); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public FileStatus getFileStatus(Path file) throws IOException { - - return new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName)); - } - - }; - - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - assertTrue(hdfsCarbonFile.getParentFile() instanceof CarbonFile); - } - - @Test - public void testForNonDisributedSystem() { - new HDFSCarbonFile(fileStatus); - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new WebHdfsFileSystem(); - } - - }; - assertEquals(hdfsCarbonFile.renameForce(fileName), false); - } - - @Test - public void testrenameForceForDisributedSystem() { - new MockUp<Path>() { - @Mock - public FileSystem getFileSystem(Configuration conf) throws IOException { - return new DistributedFileSystem(); - } - - }; - new MockUp<DistributedFileSystem>() { - @Mock - public void rename(Path src, Path dst, final Options.Rename... options) throws IOException { - - } - - }; - hdfsCarbonFile = new HDFSCarbonFile(fileStatus); - assertEquals(hdfsCarbonFile.renameForce(fileName), true); - - } -} -
