fix style
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/48316190 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/48316190 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/48316190 Branch: refs/heads/movecorepackage Commit: 4831619042155b9988f103f3bca0519229c3d8f3 Parents: ae17158 Author: jackylk <jacky.li...@huawei.com> Authored: Fri Jan 13 16:13:46 2017 +0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Fri Jan 13 16:13:46 2017 +0800 ---------------------------------------------------------------------- .../core/atomic/AtomicFileOperationsImpl.java | 6 +- .../dictionary/AbstractDictionaryCache.java | 4 +- .../core/compression/BigDecimalCompressor.java | 2 +- .../core/compression/BigIntCompressor.java | 2 +- .../core/compression/DoubleCompressor.java | 2 +- .../core/compression/ValueCompressor.java | 2 +- .../carbondata/core/datastorage/FileHolder.java | 88 ---- .../core/datastorage/MeasureDataWrapper.java | 30 -- .../core/datastorage/NodeMeasureDataStore.java | 34 -- .../columnar/BlockIndexerStorageForInt.java | 226 --------- .../BlockIndexerStorageForNoInvertedIndex.java | 159 ------ .../datastorage/columnar/ColumnGroupModel.java | 116 ----- .../columnar/ColumnWithIntIndex.java | 82 ---- .../columnar/ColumnWithIntIndexForHighCard.java | 49 -- .../columnar/ColumnarKeyStoreDataHolder.java | 54 --- .../columnar/ColumnarKeyStoreMetadata.java | 53 -- .../core/datastorage/columnar/IndexStorage.java | 44 -- .../datastorage/columnar/UnBlockIndexer.java | 78 --- .../datastorage/compression/Compressor.java | 60 --- .../compression/CompressorFactory.java | 52 -- .../compression/MeasureMetaDataModel.java | 128 ----- .../compression/ReaderCompressModel.java | 61 --- .../compression/SnappyCompressor.java | 222 --------- .../compression/ValueCompressionHolder.java | 115 ----- .../compression/WriterCompressModel.java | 220 --------- .../decimal/CompressionMaxMinByte.java | 115 ----- .../decimal/CompressionMaxMinDefault.java | 117 ----- .../decimal/CompressionMaxMinInt.java | 113 ----- .../decimal/CompressionMaxMinLong.java | 113 ----- .../decimal/CompressionMaxMinShort.java | 115 ----- .../nondecimal/CompressionNonDecimalByte.java | 99 ---- .../CompressionNonDecimalDefault.java | 101 ---- .../nondecimal/CompressionNonDecimalInt.java | 100 ---- .../nondecimal/CompressionNonDecimalLong.java | 101 ---- .../CompressionNonDecimalMaxMinByte.java | 105 ---- .../CompressionNonDecimalMaxMinDefault.java | 107 ---- .../CompressionNonDecimalMaxMinInt.java | 105 ---- .../CompressionNonDecimalMaxMinLong.java | 106 ---- .../CompressionNonDecimalMaxMinShort.java | 104 ---- .../nondecimal/CompressionNonDecimalShort.java | 100 ---- .../compression/none/CompressionNoneByte.java | 104 ---- .../none/CompressionNoneDefault.java | 102 ---- .../compression/none/CompressionNoneInt.java | 102 ---- .../compression/none/CompressionNoneLong.java | 99 ---- .../compression/none/CompressionNoneShort.java | 104 ---- .../compression/type/CompressionBigDecimal.java | 146 ------ .../dataholder/CarbonReadDataHolder.java | 50 -- .../dataholder/CarbonWriteDataHolder.java | 280 ----------- .../filesystem/AbstractDFSCarbonFile.java | 217 --------- .../filesystem/AlluxioCarbonFile.java | 130 ----- .../core/datastorage/filesystem/CarbonFile.java | 66 --- .../filesystem/CarbonFileFilter.java | 24 - .../datastorage/filesystem/HDFSCarbonFile.java | 128 ----- .../datastorage/filesystem/LocalCarbonFile.java | 229 --------- .../filesystem/ViewFSCarbonFile.java | 126 ----- .../impl/CompressedDataMeasureDataWrapper.java | 37 -- .../datastorage/impl/DFSFileHolderImpl.java | 134 ----- .../core/datastorage/impl/FileFactory.java | 485 ------------------- .../core/datastorage/impl/FileHolderImpl.java | 198 -------- ...ractHeavyCompressedDoubleArrayDataStore.java | 90 ---- ...yCompressedDoubleArrayDataInMemoryStore.java | 30 -- .../carbondata/core/datastore/DataRefNode.java | 1 - .../carbondata/core/datastore/FileHolder.java | 88 ++++ .../core/datastore/MeasureDataWrapper.java | 30 ++ .../core/datastore/NodeMeasureDataStore.java | 34 ++ .../core/datastore/block/SegmentProperties.java | 5 +- .../core/datastore/block/TableBlockInfo.java | 2 +- .../datastore/chunk/MeasureColumnDataChunk.java | 2 +- .../impl/FixedLengthDimensionDataChunk.java | 3 +- .../impl/VariableLengthDimensionDataChunk.java | 3 +- .../chunk/reader/CarbonDataReaderFactory.java | 12 +- .../reader/DimensionColumnChunkReader.java | 2 +- .../chunk/reader/MeasureColumnChunkReader.java | 2 +- .../reader/dimension/AbstractChunkReader.java | 4 +- ...mpressedDimensionChunkFileBasedReaderV1.java | 4 +- ...mpressedDimensionChunkFileBasedReaderV2.java | 4 +- ...CompressedMeasureChunkFileBasedReaderV1.java | 8 +- ...CompressedMeasureChunkFileBasedReaderV2.java | 10 +- .../chunk/store/DimensionChunkStoreFactory.java | 12 +- .../columnar/BlockIndexerStorageForInt.java | 226 +++++++++ .../BlockIndexerStorageForNoInvertedIndex.java | 159 ++++++ .../datastore/columnar/ColumnGroupModel.java | 116 +++++ .../datastore/columnar/ColumnWithIntIndex.java | 82 ++++ .../columnar/ColumnWithIntIndexForHighCard.java | 49 ++ .../columnar/ColumnarKeyStoreDataHolder.java | 54 +++ .../columnar/ColumnarKeyStoreMetadata.java | 53 ++ .../core/datastore/columnar/IndexStorage.java | 44 ++ .../core/datastore/columnar/UnBlockIndexer.java | 78 +++ .../core/datastore/compression/Compressor.java | 60 +++ .../compression/CompressorFactory.java | 52 ++ .../compression/MeasureMetaDataModel.java | 128 +++++ .../compression/ReaderCompressModel.java | 61 +++ .../datastore/compression/SnappyCompressor.java | 222 +++++++++ .../compression/ValueCompressionHolder.java | 115 +++++ .../compression/WriterCompressModel.java | 220 +++++++++ .../decimal/CompressionMaxMinByte.java | 115 +++++ .../decimal/CompressionMaxMinDefault.java | 117 +++++ .../decimal/CompressionMaxMinInt.java | 113 +++++ .../decimal/CompressionMaxMinLong.java | 113 +++++ .../decimal/CompressionMaxMinShort.java | 115 +++++ .../nondecimal/CompressionNonDecimalByte.java | 99 ++++ .../CompressionNonDecimalDefault.java | 101 ++++ .../nondecimal/CompressionNonDecimalInt.java | 100 ++++ .../nondecimal/CompressionNonDecimalLong.java | 101 ++++ .../CompressionNonDecimalMaxMinByte.java | 105 ++++ .../CompressionNonDecimalMaxMinDefault.java | 107 ++++ .../CompressionNonDecimalMaxMinInt.java | 105 ++++ .../CompressionNonDecimalMaxMinLong.java | 106 ++++ .../CompressionNonDecimalMaxMinShort.java | 104 ++++ .../nondecimal/CompressionNonDecimalShort.java | 100 ++++ .../compression/none/CompressionNoneByte.java | 104 ++++ .../none/CompressionNoneDefault.java | 102 ++++ .../compression/none/CompressionNoneInt.java | 102 ++++ .../compression/none/CompressionNoneLong.java | 99 ++++ .../compression/none/CompressionNoneShort.java | 104 ++++ .../compression/type/CompressionBigDecimal.java | 146 ++++++ .../dataholder/CarbonReadDataHolder.java | 50 ++ .../dataholder/CarbonWriteDataHolder.java | 280 +++++++++++ .../filesystem/AbstractDFSCarbonFile.java | 217 +++++++++ .../datastore/filesystem/AlluxioCarbonFile.java | 130 +++++ .../core/datastore/filesystem/CarbonFile.java | 66 +++ .../datastore/filesystem/CarbonFileFilter.java | 24 + .../datastore/filesystem/HDFSCarbonFile.java | 128 +++++ .../datastore/filesystem/LocalCarbonFile.java | 229 +++++++++ .../datastore/filesystem/ViewFSCarbonFile.java | 126 +++++ .../impl/CompressedDataMeasureDataWrapper.java | 37 ++ .../core/datastore/impl/DFSFileHolderImpl.java | 134 +++++ .../core/datastore/impl/FileFactory.java | 485 +++++++++++++++++++ .../core/datastore/impl/FileHolderImpl.java | 198 ++++++++ .../impl/btree/AbstractBTreeLeafNode.java | 2 +- .../datastore/impl/btree/BTreeNonLeafNode.java | 2 +- .../impl/btree/BlockletBTreeLeafNode.java | 2 +- ...ractHeavyCompressedDoubleArrayDataStore.java | 90 ++++ ...yCompressedDoubleArrayDataInMemoryStore.java | 30 ++ .../DirectDictionaryKeyGeneratorFactory.java | 6 +- .../core/metadata/AbsoluteTableIdentifier.java | 2 +- .../core/metadata/BlockletInfoColumnar.java | 2 +- .../core/metadata/ColumnIdentifier.java | 112 +++++ .../carbondata/core/path/CarbonTablePath.java | 11 +- .../reader/CarbonDeleteDeltaFileReaderImpl.java | 2 +- .../reader/CarbonDeleteFilesDataReader.java | 2 +- .../carbondata/core/reader/ThriftReader.java | 2 +- .../CarbonDictionarySortIndexReaderImpl.java | 2 +- .../resolver/ConditionalFilterResolverImpl.java | 3 +- .../processor/AbstractDataBlockIterator.java | 2 +- .../core/scan/processor/BlocksChunkHolder.java | 2 +- .../processor/impl/DataBlockIteratorImpl.java | 2 +- .../AbstractDetailQueryResultIterator.java | 4 +- .../core/scan/scanner/impl/FilterScanner.java | 2 +- .../DriverQueryStatisticsRecorderImpl.java | 2 +- .../core/update/CarbonUpdateUtil.java | 9 +- .../core/updatestatus/SegmentStatusManager.java | 2 +- .../SegmentUpdateStatusManager.java | 6 +- .../core/updatestatus/locks/HdfsFileLock.java | 4 +- .../core/updatestatus/locks/LocalFileLock.java | 2 +- .../core/util/CarbonMetadataUtil.java | 4 +- .../apache/carbondata/core/util/CarbonUtil.java | 14 +- .../core/util/DataFileFooterConverter.java | 4 +- .../core/util/ValueCompressionUtil.java | 50 +- .../writer/CarbonDeleteDeltaWriterImpl.java | 2 +- .../core/writer/CarbonDictionaryWriterImpl.java | 4 +- .../carbondata/core/writer/ThriftWriter.java | 2 +- .../CarbonDictionarySortIndexWriterImpl.java | 4 +- .../dictionary/AbstractDictionaryCacheTest.java | 4 +- .../CompressedDataMeasureWrapperTest.java | 4 +- .../datastorage/DFSFileHolderImplUnitTest.java | 2 +- .../datastorage/FileFactoryImplUnitTest.java | 2 +- .../datastorage/FileHolderImplUnitTest.java | 2 +- .../ColumnarKeyStoreDataHolderUnitTest.java | 93 ---- .../filesystem/AlluxioCarbonFileTest.java | 292 ----------- .../filesystem/HDFSCarbonFileTest.java | 400 --------------- .../filesystem/LocalCarbonFileTest.java | 471 ------------------ .../filesystem/ViewFsCarbonFileTest.java | 311 ------------ ...mpressedMeasureChunkFileBasedReaderTest.java | 10 +- .../ColumnarKeyStoreDataHolderUnitTest.java | 93 ++++ .../filesystem/AlluxioCarbonFileTest.java | 292 +++++++++++ .../filesystem/HDFSCarbonFileTest.java | 400 +++++++++++++++ .../filesystem/LocalCarbonFileTest.java | 471 ++++++++++++++++++ .../filesystem/ViewFsCarbonFileTest.java | 311 ++++++++++++ ...CarbonDictionarySortIndexReaderImplTest.java | 4 +- .../core/util/CarbonMetadataUtilTest.java | 2 +- .../carbondata/core/util/CarbonUtilTest.java | 8 +- .../core/util/DataFileFooterConverterTest.java | 6 +- .../core/util/ValueCompressionUtilTest.java | 46 +- .../writer/CarbonDictionaryWriterImplTest.java | 4 +- .../core/writer/CarbonFooterWriterTest.java | 6 +- .../carbondata/examples/AlluxioExample.scala | 2 +- .../examples/GenerateDictionaryExample.scala | 4 +- .../examples/util/AllDictionaryUtil.scala | 2 +- .../carbondata/hadoop/CarbonInputFormat.java | 35 +- .../carbondata/hadoop/CarbonInputSplit.java | 4 +- .../carbondata/hadoop/CarbonRecordReader.java | 4 +- .../hadoop/api/CarbonTableInputFormat.java | 4 +- .../internal/index/impl/InMemoryBTreeIndex.java | 10 +- .../internal/segment/impl/IndexedSegment.java | 2 +- .../hadoop/util/BlockLevelTraverser.java | 4 +- .../hadoop/util/CarbonInputFormatUtil.java | 2 +- .../carbondata/hadoop/util/SchemaReader.java | 2 +- .../hadoop/test/util/StoreCreator.java | 2 +- .../dataload/TestLoadDataGeneral.scala | 2 +- .../blockprune/BlockPruneQueryTestCase.scala | 2 +- .../SubqueryWithFilterAndSortTestCase.scala | 2 +- .../ValueCompressionDataTypeTestCase.scala | 2 +- .../carbondata/spark/load/CarbonLoaderUtil.java | 42 +- .../spark/load/DeleteLoadFolders.java | 10 +- .../spark/merger/CarbonCompactionExecutor.java | 4 +- .../spark/merger/CarbonCompactionUtil.java | 6 +- .../spark/merger/CarbonDataMergerUtil.java | 24 +- .../spark/merger/RowResultMerger.java | 19 +- .../spark/merger/TupleConversionAdapter.java | 2 +- .../carbondata/spark/util/CarbonQueryUtil.java | 2 +- .../carbondata/spark/util/LoadMetadataUtil.java | 2 +- .../org/apache/carbondata/api/CarbonStore.scala | 2 +- .../carbondata/spark/CarbonSparkFactory.scala | 2 +- .../spark/DictionaryDetailHelper.scala | 6 +- .../spark/rdd/CarbonDataLoadRDD.scala | 3 +- .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 9 +- .../spark/rdd/CarbonIUDMergerRDD.scala | 1 - .../carbondata/spark/rdd/CarbonMergerRDD.scala | 4 +- .../carbondata/spark/rdd/CarbonScanRDD.scala | 4 +- .../apache/carbondata/spark/rdd/Compactor.scala | 1 - .../spark/rdd/DataManagementFunc.scala | 2 +- .../spark/rdd/NewCarbonDataLoadRDD.scala | 5 +- .../spark/rdd/UpdateCoalescedRDD.scala | 1 - .../carbondata/spark/util/CarbonScalaUtil.scala | 4 +- .../spark/util/GlobalDictionaryUtil.scala | 9 +- .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +- .../execution/command/carbonTableSchema.scala | 5 +- .../spark/sql/hive/DistributionUtil.scala | 2 +- .../scala/org/apache/spark/util/FileUtils.scala | 4 +- .../spark/CarbonDataFrameWriter.scala | 2 +- .../spark/rdd/CarbonDataRDDFactory.scala | 6 +- .../sql/CarbonDatasourceHadoopRelation.scala | 2 +- .../spark/sql/CarbonDatasourceRelation.scala | 2 +- .../spark/sql/CarbonDictionaryDecoder.scala | 2 +- .../scala/org/apache/spark/sql/CarbonScan.scala | 2 +- .../sql/execution/command/IUDCommands.scala | 15 +- .../execution/command/carbonTableSchema.scala | 13 +- .../apache/spark/sql/hive/CarbonMetastore.scala | 6 +- .../CompactionSystemLockFeatureTest.scala | 2 +- ...GlobalDictionaryUtilConcurrentTestCase.scala | 2 +- .../VectorizedCarbonRecordReader.java | 10 +- .../spark/DictionaryDetailHelper.scala | 4 +- .../execution/command/carbonTableSchema.scala | 2 +- .../apache/spark/sql/hive/CarbonMetastore.scala | 6 +- .../processing/csvload/DataGraphExecuter.java | 10 +- .../processing/csvload/GraphExecutionUtil.java | 12 +- .../processing/csvreaderstep/BlockDetails.java | 2 +- .../csvreaderstep/UnivocityCsvParser.java | 4 +- .../processing/datatypes/PrimitiveDataType.java | 6 +- .../graphgenerator/GraphGenerator.java | 6 +- .../processing/mdkeygen/MDKeyGenStep.java | 9 +- .../processing/model/CarbonLoadModel.java | 4 +- .../newflow/DataLoadProcessBuilder.java | 2 +- .../impl/DictionaryFieldConverterImpl.java | 4 +- .../DirectDictionaryFieldConverterImpl.java | 2 +- .../converter/impl/FieldEncoderFactory.java | 2 +- .../impl/MeasureFieldConverterImpl.java | 2 +- .../impl/NonDictionaryFieldConverterImpl.java | 2 +- ...arallelReadMergeSorterWithBucketingImpl.java | 2 +- .../steps/DataConverterProcessorStepImpl.java | 2 +- ...ConverterProcessorWithBucketingStepImpl.java | 6 +- .../steps/DataWriterProcessorStepImpl.java | 4 +- .../sortdata/CompressedTempSortFileWriter.java | 2 +- .../sortandgroupby/sortdata/SortParameters.java | 2 +- .../store/CarbonFactDataHandlerColumnar.java | 22 +- .../store/CarbonFactDataHandlerModel.java | 6 +- .../processing/store/StoreFactory.java | 6 +- .../store/colgroup/ColGroupBlockStorage.java | 2 +- .../store/writer/AbstractFactDataWriter.java | 14 +- .../store/writer/CarbonFactDataWriter.java | 4 +- .../processing/store/writer/NodeHolder.java | 2 +- .../writer/v1/CarbonFactDataWriterImplV1.java | 4 +- .../writer/v2/CarbonFactDataWriterImplV2.java | 2 +- .../csvbased/BadRecordsLogger.java | 4 +- .../CarbonCSVBasedDimSurrogateKeyGen.java | 2 +- .../csvbased/CarbonCSVBasedSeqGenMeta.java | 2 +- .../csvbased/CarbonCSVBasedSeqGenStep.java | 12 +- .../FileStoreSurrogateKeyGenForCSV.java | 8 +- .../util/CarbonDataProcessorUtil.java | 12 +- .../processing/util/CarbonSchemaParser.java | 8 +- .../carbondata/test/util/StoreCreator.java | 2 +- 282 files changed, 8480 insertions(+), 8358 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java index 33eeae0..db58b17 100644 --- a/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java @@ -24,9 +24,9 @@ import java.io.DataOutputStream; import java.io.IOException; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.impl.FileFactory; -import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory.FileType; public class AtomicFileOperationsImpl implements AtomicFileOperations { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java index 5fd06b4..e5fc83e 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java @@ -26,8 +26,8 @@ import org.apache.carbondata.core.cache.Cache; import org.apache.carbondata.core.cache.CacheType; import org.apache.carbondata.core.cache.CarbonLRUCache; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.path.CarbonTablePath; import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java index 4da8795..fe8192e 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.compression; -import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder; import org.apache.carbondata.core.util.BigDecimalCompressionFinder; import org.apache.carbondata.core.util.CompressionFinder; import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java index e509c5c..6a5250a 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.compression; -import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java index a716921..345865c 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java @@ -20,7 +20,7 @@ package org.apache.carbondata.core.compression; import java.math.BigDecimal; -import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java index a2d7b51..a74b886 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.compression; -import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder; import org.apache.carbondata.core.util.CompressionFinder; import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE; import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java deleted file mode 100644 index 96aa069..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage; - -import java.io.IOException; - -public interface FileHolder { - /** - * This method will be used to read the byte array from file based on offset - * and length(number of bytes) need to read - * - * @param filePath fully qualified file path - * @param offset reading start position, - * @param length number of bytes to be read - * @return read byte array - */ - byte[] readByteArray(String filePath, long offset, int length) throws IOException; - - /** - * This method will be used to read the byte array from file based on length(number of bytes) - * - * @param filePath fully qualified file path - * @param length number of bytes to be read - * @return read byte array - */ - byte[] readByteArray(String filePath, int length) throws IOException; - - /** - * This method will be used to read int from file from postion(offset), here - * length will be always 4 bacause int byte size if 4 - * - * @param filePath fully qualified file path - * @param offset reading start position, - * @return read int - */ - int readInt(String filePath, long offset) throws IOException; - - /** - * This method will be used to read long from file from postion(offset), here - * length will be always 8 bacause int byte size is 8 - * - * @param filePath fully qualified file path - * @param offset reading start position, - * @return read long - */ - long readLong(String filePath, long offset) throws IOException; - - /** - * This method will be used to read int from file from postion(offset), here - * length will be always 4 bacause int byte size if 4 - * - * @param filePath fully qualified file path - * @return read int - */ - int readInt(String filePath) throws IOException; - - /** - * This method will be used to read long value from file from postion(offset), here - * length will be always 8 because long byte size if 4 - * - * @param filePath fully qualified file path - * @param offset reading start position, - * @return read long - */ - long readDouble(String filePath, long offset) throws IOException; - - /** - * This method will be used to close all the streams currently present in the cache - */ - void finish() throws IOException; -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java deleted file mode 100644 index 6fc6c1d..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage; - -import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder; - -/** - * MeasureDataWrapper, interface. - */ -public interface MeasureDataWrapper { - CarbonReadDataHolder[] getValues(); - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java deleted file mode 100644 index fede06c..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage; - -import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; - -public interface NodeMeasureDataStore { - /** - * This method will be used to get the writable key array. - * writable measure data array will hold below information: - * <size of measure data array><measure data array> - * total length will be 4 bytes for size + measure data array length - * - * @return writable array (compressed or normal) - */ - byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray); -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java deleted file mode 100644 index 0a2a8bd..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.columnar; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.ByteUtil; - -public class BlockIndexerStorageForInt implements IndexStorage<int[]> { - private boolean alreadySorted; - - private int[] dataAfterComp; - - private int[] indexMap; - - private byte[][] keyBlock; - - private int[] dataIndexMap; - - private int totalSize; - - public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary, - boolean isSortRequired) { - ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary); - if (isSortRequired) { - Arrays.sort(columnWithIndexs); - } - compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock)); - if (compressData) { - compressDataMyOwnWay(columnWithIndexs); - } - } - - /** - * Create an object with each column array and respective index - * - * @return - */ - private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock, - boolean isNoDictionary) { - ColumnWithIntIndex[] columnWithIndexs; - if (isNoDictionary) { - columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length]; - for (int i = 0; i < columnWithIndexs.length; i++) { - columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i); - } - - } else { - columnWithIndexs = new ColumnWithIntIndex[keyBlock.length]; - for (int i = 0; i < columnWithIndexs.length; i++) { - columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i); - } - } - - return columnWithIndexs; - } - - private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs, - byte[][] keyBlock) { - int[] indexes = new int[columnWithIndexs.length]; - for (int i = 0; i < indexes.length; i++) { - indexes[i] = columnWithIndexs[i].getIndex(); - keyBlock[i] = columnWithIndexs[i].getColumn(); - } - this.keyBlock = keyBlock; - return indexes; - } - - /** - * It compresses depends up on the sequence numbers. - * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In - * first array the start and end of sequential numbers and second array - * keeps the indexes of where sequential numbers starts. If there is no - * sequential numbers then the same array it returns with empty second - * array. - * - * @param indexes - */ - public void compressMyOwnWay(int[] indexes) { - List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - int k = 0; - int i = 1; - for (; i < indexes.length; i++) { - if (indexes[i] - indexes[i - 1] == 1) { - k++; - } else { - if (k > 0) { - map.add((list.size())); - list.add(indexes[i - k - 1]); - list.add(indexes[i - 1]); - } else { - list.add(indexes[i - 1]); - } - k = 0; - } - } - if (k > 0) { - map.add((list.size())); - list.add(indexes[i - k - 1]); - list.add(indexes[i - 1]); - } else { - list.add(indexes[i - 1]); - } - dataAfterComp = convertToArray(list); - if (indexes.length == dataAfterComp.length) { - indexMap = new int[0]; - } else { - indexMap = convertToArray(map); - } - if (dataAfterComp.length == 2 && indexMap.length == 1) { - alreadySorted = true; - } - } - - private int[] convertToArray(List<Integer> list) { - int[] shortArray = new int[list.size()]; - for (int i = 0; i < shortArray.length; i++) { - shortArray[i] = list.get(i); - } - return shortArray; - } - - /** - * @return the alreadySorted - */ - public boolean isAlreadySorted() { - return alreadySorted; - } - - /** - * @return the dataAfterComp - */ - public int[] getDataAfterComp() { - return dataAfterComp; - } - - /** - * @return the indexMap - */ - public int[] getIndexMap() { - return indexMap; - } - - /** - * @return the keyBlock - */ - public byte[][] getKeyBlock() { - return keyBlock; - } - - private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) { - byte[] prvKey = indexes[0].getColumn(); - List<ColumnWithIntIndex> list = - new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - list.add(indexes[0]); - int counter = 1; - int start = 0; - List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - for (int i = 1; i < indexes.length; i++) { - if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) { - prvKey = indexes[i].getColumn(); - list.add(indexes[i]); - map.add(start); - map.add(counter); - start += counter; - counter = 1; - continue; - } - counter++; - } - map.add(start); - map.add(counter); - this.keyBlock = convertToKeyArray(list); - if (indexes.length == keyBlock.length) { - dataIndexMap = new int[0]; - } else { - dataIndexMap = convertToArray(map); - } - } - - private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) { - byte[][] shortArray = new byte[list.size()][]; - for (int i = 0; i < shortArray.length; i++) { - shortArray[i] = list.get(i).getColumn(); - totalSize += shortArray[i].length; - } - return shortArray; - } - - @Override public int[] getDataIndexMap() { - return dataIndexMap; - } - - @Override public int getTotalSize() { - return totalSize; - } - - @Override public byte[] getMin() { - return keyBlock[0]; - } - - @Override public byte[] getMax() { - return keyBlock[keyBlock.length - 1]; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java deleted file mode 100644 index 939d759..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.columnar; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.ByteUtil; - -public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> { - private byte[][] keyBlock; - private byte[][] sortedBlock; - private int totalSize; - private int[] dataIndexMap; - - public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData, - boolean isNoDictionary) { - // without invertedindex but can be RLE - if (compressData) { - // with RLE - byte[] prvKey = keyBlockInput[0]; - List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - list.add(keyBlockInput[0]); - int counter = 1; - int start = 0; - List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - int length = keyBlockInput.length; - for(int i = 1; i < length; i++) { - if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) { - prvKey = keyBlockInput[i]; - list.add(keyBlockInput[i]); - map.add(start); - map.add(counter); - start += counter; - counter = 1; - continue; - } - counter++; - } - map.add(start); - map.add(counter); - this.keyBlock = convertToKeyArray(list); - if (keyBlockInput.length == this.keyBlock.length) { - dataIndexMap = new int[0]; - } else { - dataIndexMap = convertToArray(map); - } - } else { - this.keyBlock = keyBlockInput; - dataIndexMap = new int[0]; - } - - this.sortedBlock = new byte[keyBlock.length][]; - System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length); - if (isNoDictionary) { - Arrays.sort(sortedBlock, new Comparator<byte[]>() { - @Override - public int compare(byte[] col1, byte[] col2) { - return ByteUtil.UnsafeComparer.INSTANCE - .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2); - } - }); - } else { - Arrays.sort(sortedBlock, new Comparator<byte[]>() { - @Override - public int compare(byte[] col1, byte[] col2) { - return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2); - } - }); - } - - } - - private int[] convertToArray(List<Integer> list) { - int[] shortArray = new int[list.size()]; - for(int i = 0; i < shortArray.length; i++) { - shortArray[i] = list.get(i); - } - return shortArray; - } - - private byte[][] convertToKeyArray(List<byte[]> list) { - byte[][] shortArray = new byte[list.size()][]; - for (int i = 0; i < shortArray.length; i++) { - shortArray[i] = list.get(i); - totalSize += shortArray[i].length; - } - return shortArray; - } - - @Override - public int[] getDataIndexMap() { - return dataIndexMap; - } - - @Override - public int getTotalSize() { - return totalSize; - } - - @Override - public boolean isAlreadySorted() { - return true; - } - - /** - * no use - * @return - */ - @Override - public int[] getDataAfterComp() { - return new int[0]; - } - - /** - * no use - * @return - */ - @Override - public int[] getIndexMap() { - return new int[0]; - } - - /** - * @return the keyBlock - */ - public byte[][] getKeyBlock() { - return keyBlock; - } - - @Override public byte[] getMin() { - return sortedBlock[0]; - } - - @Override public byte[] getMax() { - return sortedBlock[sortedBlock.length - 1]; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java deleted file mode 100644 index 7402842..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.datastorage.columnar; - -public class ColumnGroupModel { - - /** - * number of columns in columnar block - */ - private int[] columnSplit; - - /** - * total number of columns - */ - private int noOfColumnsStore; - - /** - * whether given index is columnar or not - * true: columnar - * false: row block - */ - private boolean[] columnarStore; - - /** - * column groups - * e.g - * {{0,1,2},3,4,{5,6}} - */ - private int[][] columnGroups; - - /** - * return columnSplit - * - * @return - */ - public int[] getColumnSplit() { - return columnSplit; - } - - /** - * set columnSplit - * - * @param split - */ - public void setColumnSplit(int[] split) { - this.columnSplit = split; - } - - /** - * @return no of columnar block - */ - public int getNoOfColumnStore() { - return this.noOfColumnsStore; - } - - /** - * set no of columnar block - * - * @param noOfColumnsStore - */ - public void setNoOfColumnStore(int noOfColumnsStore) { - this.noOfColumnsStore = noOfColumnsStore; - } - - /** - * it's an identifier for row block or single column block - * - * @param columnarStore - */ - public void setColumnarStore(boolean[] columnarStore) { - this.columnarStore = columnarStore; - } - - /** - * set column groups - * - * @param columnGroups - */ - public void setColumnGroup(int[][] columnGroups) { - this.columnGroups = columnGroups; - } - - /** - * check if given column group is columnar - * - * @param colGroup - * @return true if given block is columnar - */ - public boolean isColumnar(int colGroup) { - return columnarStore[colGroup]; - } - - /** - * @return columngroups - */ - public int[][] getColumnGroup() { - return this.columnGroups; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java deleted file mode 100644 index d569572..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.columnar; - -import java.util.Arrays; - -import org.apache.carbondata.core.util.ByteUtil; - -public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> { - protected byte[] column; - - private int index; - - public ColumnWithIntIndex(byte[] column, int index) { - this.column = column; - this.index = index; - } - - public ColumnWithIntIndex() { - } - - /** - * @return the column - */ - public byte[] getColumn() { - return column; - } - - /** - * @param column the column to set - */ - public void setColumn(byte[] column) { - this.column = column; - } - - /** - * @return the index - */ - public int getIndex() { - return index; - } - - /** - * @param index the index to set - */ - public void setIndex(int index) { - this.index = index; - } - - @Override public int compareTo(ColumnWithIntIndex o) { - return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column); - } - - @Override public boolean equals(Object obj) { - if(obj == null || getClass() != obj.getClass()) { - return false; - } - ColumnWithIntIndex o = (ColumnWithIntIndex)obj; - return Arrays.equals(column, o.column) && index == o.index; - } - - @Override public int hashCode() { - return Arrays.hashCode(column) + index; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java deleted file mode 100644 index 61145f7..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.columnar; - -import java.util.Arrays; - -import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer; - -public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex - implements Comparable<ColumnWithIntIndex> { - - public ColumnWithIntIndexForHighCard(byte[] column, int index) { - super(column, index); - } - - @Override public int compareTo(ColumnWithIntIndex o) { - return UnsafeComparer.INSTANCE - .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2); - } - - @Override public boolean equals(Object obj) { - if(obj == null || getClass() != obj.getClass()) { - return false; - } - ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj; - return Arrays.equals(column, o.column) && getIndex() == o.getIndex(); - } - - @Override public int hashCode() { - return Arrays.hashCode(column) + getIndex(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java deleted file mode 100644 index 31bfd00..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.columnar; - -import java.nio.ByteBuffer; - -public class ColumnarKeyStoreDataHolder { - private byte[] keyblockData; - private ColumnarKeyStoreMetadata columnarKeyStoreMetadata; - - public ColumnarKeyStoreDataHolder(final byte[] keyblockData, - final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) { - this.keyblockData = keyblockData; - this.columnarKeyStoreMetadata = columnarKeyStoreMetadata; - } - - public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) { - this.columnarKeyStoreMetadata = columnarKeyStoreMetadata; - } - - public int getSurrogateKey(int columnIndex) { - byte[] actual = new byte[4]; - int startIndex; - if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) { - startIndex = - columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata - .getEachRowSize(); - } else { - startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize(); - } - int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize(); - System.arraycopy(keyblockData, startIndex, actual, destPos, - columnarKeyStoreMetadata.getEachRowSize()); - return ByteBuffer.wrap(actual).getInt(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java deleted file mode 100644 index 6badce5..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.columnar; - -class ColumnarKeyStoreMetadata { - - private int[] columnReverseIndex; - - private int eachRowSize; - - ColumnarKeyStoreMetadata(int eachRowSize) { - this.eachRowSize = eachRowSize; - } - - /** - * @return the eachRowSize - */ - int getEachRowSize() { - return eachRowSize; - } - - /** - * @return the columnReverseIndex - */ - int[] getColumnReverseIndex() { - return columnReverseIndex; - } - - /** - * @param columnReverseIndex the columnReverseIndex to set - */ - void setColumnReverseIndex(int[] columnReverseIndex) { - this.columnReverseIndex = columnReverseIndex; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java deleted file mode 100644 index bc8b60e..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.columnar; - -public interface IndexStorage<T> { - boolean isAlreadySorted(); - - T getDataAfterComp(); - - T getIndexMap(); - - byte[][] getKeyBlock(); - - T getDataIndexMap(); - - int getTotalSize(); - - /** - * @return min value of block - */ - byte[] getMin(); - - /** - * @return max value of block - */ - byte[] getMax(); -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java deleted file mode 100644 index 4f2a80e..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.columnar; - -import java.util.Arrays; - -public final class UnBlockIndexer { - - private UnBlockIndexer() { - - } - - public static int[] uncompressIndex(int[] indexData, int[] indexMap) { - int actualSize = indexData.length; - for (int i = 0; i < indexMap.length; i++) { - actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1; - } - int[] indexes = new int[actualSize]; - int k = 0; - for (int i = 0; i < indexData.length; i++) { - int index = Arrays.binarySearch(indexMap, i); - if (index > -1) { - for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) { - indexes[k] = j; - k++; - } - i++; - } else { - indexes[k] = indexData[i]; - k++; - } - } - return indexes; - } - - public static byte[] uncompressData(byte[] data, int[] index, int keyLen) { - if (index.length < 1) { - return data; - } - int numberOfCopy = 0; - int actualSize = 0; - int srcPos = 0; - int destPos = 0; - for (int i = 1; i < index.length; i += 2) { - actualSize += index[i]; - } - byte[] uncompressedData = new byte[actualSize * keyLen]; - int picIndex = 0; - for (int i = 0; i < data.length; i += keyLen) { - numberOfCopy = index[picIndex * 2 + 1]; - picIndex++; - for (int j = 0; j < numberOfCopy; j++) { - System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen); - destPos += keyLen; - } - srcPos += keyLen; - } - return uncompressedData; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java deleted file mode 100644 index 7982d54..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.compression; - -public interface Compressor { - - byte[] compressByte(byte[] unCompInput); - - byte[] unCompressByte(byte[] compInput); - - byte[] unCompressByte(byte[] compInput, int offset, int length); - - byte[] compressShort(short[] unCompInput); - - short[] unCompressShort(byte[] compInput); - - short[] unCompressShort(byte[] compInput, int offset, int lenght); - - byte[] compressInt(int[] unCompInput); - - int[] unCompressInt(byte[] compInput); - - int[] unCompressInt(byte[] compInput, int offset, int length); - - byte[] compressLong(long[] unCompInput); - - long[] unCompressLong(byte[] compInput); - - long[] unCompressLong(byte[] compInput, int offset, int length); - - byte[] compressFloat(float[] unCompInput); - - float[] unCompressFloat(byte[] compInput); - - float[] unCompressFloat(byte[] compInput, int offset, int length); - - byte[] compressDouble(double[] unCompInput); - - double[] unCompressDouble(byte[] compInput); - - double[] unCompressDouble(byte[] compInput, int offset, int length); - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java deleted file mode 100644 index 67c24d1..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.compression; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.CarbonProperties; - -public class CompressorFactory { - - private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory(); - - private final Compressor compressor; - - private CompressorFactory() { - String compressorType = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR); - switch (compressorType) { - case "snappy": - compressor = new SnappyCompressor(); - break; - default: - throw new RuntimeException( - "Invalid compressor type provided! Please provide valid compressor type"); - } - } - - public static CompressorFactory getInstance() { - return COMPRESSOR_FACTORY; - } - - public Compressor getCompressor() { - return compressor; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java deleted file mode 100644 index 41a1ef0..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.compression; - -public class MeasureMetaDataModel { - /** - * maxValue - */ - private Object[] maxValue; - - /** - * minValue - */ - private Object[] minValue; - - /** - * mantissa - */ - private int[] mantissa; - - /** - * measureCount - */ - private int measureCount; - - /** - * uniqueValue - */ - private Object[] uniqueValue; - - /** - * type - */ - private char[] type; - - /** - * dataTypeSelected - */ - private byte[] dataTypeSelected; - - public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa, - int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) { - this.minValue = minValue; - this.maxValue = maxValue; - this.mantissa = mantissa; - this.measureCount = measureCount; - this.uniqueValue = uniqueValue; - this.type = type; - this.dataTypeSelected = dataTypeSelected; - } - - /** - * get Max value - * - * @return - */ - public Object[] getMaxValue() { - return maxValue; - } - - /** - * getMinValue - * - * @return - */ - public Object[] getMinValue() { - return minValue; - } - - /** - * getMantissa - * - * @return - */ - public int[] getMantissa() { - return mantissa; - } - - /** - * getMeasureCount - * - * @return - */ - public int getMeasureCount() { - return measureCount; - } - - /** - * getUniqueValue - * - * @return - */ - public Object[] getUniqueValue() { - return uniqueValue; - } - - /** - * @return the type - */ - public char[] getType() { - return type; - } - - /** - * @return the dataTypeSelected - */ - public byte[] getDataTypeSelected() { - return dataTypeSelected; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java deleted file mode 100644 index 02da265..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.compression; - -import org.apache.carbondata.core.metadata.ValueEncoderMeta; -import org.apache.carbondata.core.util.ValueCompressionUtil; - -// Used in read path for decompression preparation -public class ReaderCompressModel { - private ValueEncoderMeta valueEncoderMeta; - - private ValueCompressionUtil.DataType convertedDataType; - - private ValueCompressionHolder valueHolder; - - public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) { - this.valueEncoderMeta = valueEncoderMeta; - } - - public ValueCompressionUtil.DataType getConvertedDataType() { - return convertedDataType; - } - - public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) { - this.convertedDataType = convertedDataType; - } - - public Object getMaxValue() { - return valueEncoderMeta.getMaxValue(); - } - - public int getMantissa() { - return valueEncoderMeta.getMantissa(); - } - - public ValueCompressionHolder getValueCompressionHolder() { - return valueHolder; - } - - public void setValueCompressionHolder(ValueCompressionHolder valueHolder) { - this.valueHolder = valueHolder; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java deleted file mode 100644 index 0ca6512..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.compression; - -import java.io.IOException; -import java.lang.reflect.Field; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; - -import org.xerial.snappy.Snappy; -import org.xerial.snappy.SnappyNative; - -public class SnappyCompressor implements Compressor { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(SnappyCompressor.class.getName()); - - private final SnappyNative snappyNative; - - public SnappyCompressor() { - Snappy snappy = new Snappy(); - Field privateField = null; - try { - privateField = snappy.getClass().getDeclaredField("impl"); - } catch (NoSuchFieldException | SecurityException e) { - throw new RuntimeException(e); - } - privateField.setAccessible(true); - try { - snappyNative = (SnappyNative) privateField.get(snappy); - } catch (IllegalArgumentException | IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - @Override public byte[] compressByte(byte[] unCompInput) { - try { - return Snappy.rawCompress(unCompInput, unCompInput.length); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public byte[] unCompressByte(byte[] compInput) { - try { - return Snappy.uncompress(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return compInput; - } - - @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) { - int uncompressedLength = 0; - byte[] data = null; - try { - uncompressedLength = Snappy.uncompressedLength(compInput, offset, length); - data = new byte[uncompressedLength]; - Snappy.uncompress(compInput, offset, length, data, 0); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return data; - } - - @Override public byte[] compressShort(short[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public short[] unCompressShort(byte[] compInput) { - try { - return Snappy.uncompressShortArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) { - try { - return Snappy.uncompressShortArray(compInput, offset, lenght); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public byte[] compressInt(int[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public int[] unCompressInt(byte[] compInput) { - try { - return Snappy.uncompressIntArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public int[] unCompressInt(byte[] compInput, int offset, int length) { - try { - return Snappy.uncompressIntArray(compInput, offset, length); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public byte[] compressLong(long[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public long[] unCompressLong(byte[] compInput) { - try { - return Snappy.uncompressLongArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public long[] unCompressLong(byte[] compInput, int offset, int length) { - try { - return Snappy.uncompressLongArray(compInput, offset, length); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public byte[] compressFloat(float[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public float[] unCompressFloat(byte[] compInput) { - try { - return Snappy.uncompressFloatArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) { - try { - return Snappy.uncompressFloatArray(compInput, offset, length); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public byte[] compressDouble(double[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public double[] unCompressDouble(byte[] compInput) { - try { - return Snappy.uncompressDoubleArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) { - try { - int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length); - double[] result = new double[uncompressedLength / 8]; - snappyNative.rawUncompress(compInput, offset, length, result, 0); - return result; - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java deleted file mode 100644 index 425c266..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.compression; - -import java.math.BigDecimal; - -import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; - - - -/** - * ValueCompressionHolder is the base class for handling - * compression / decompression of the measure data chunk - */ -public abstract class ValueCompressionHolder<T> { - - /** - * compressedValue - */ - protected byte[] compressedValue; - - /** - * @param compressor the compressor used to decompress the data - * @param dataType data type of the data - * @param data compressed data - */ - public void unCompress(Compressor compressor, DataType dataType, byte[] data, - int offset, int length) { - switch (dataType) { - case DATA_BYTE: - setValue((T)compressor.unCompressByte(data, offset, length)); - break; - case DATA_SHORT: - setValue((T)compressor.unCompressShort(data, offset, length)); - break; - case DATA_INT: - setValue((T)compressor.unCompressInt(data, offset, length)); - break; - case DATA_LONG: - case DATA_BIGINT: - setValue((T)compressor.unCompressLong(data, offset, length)); - break; - case DATA_FLOAT: - setValue((T)compressor.unCompressFloat(data, offset, length)); - break; - default: - setValue((T)compressor.unCompressDouble(data, offset, length)); - break; - } - } - - /** - * @param compressor the compressor used to compress the data - * @param dataType data type of the data - * @param data original data - */ - public byte[] compress(Compressor compressor, DataType dataType, Object data) { - switch (dataType) { - case DATA_BYTE: - return compressor.compressByte((byte[])data); - case DATA_SHORT: - return compressor.compressShort((short[])data); - case DATA_INT: - return compressor.compressInt((int[])data); - case DATA_LONG: - case DATA_BIGINT: - return compressor.compressLong((long[])data); - case DATA_FLOAT: - return compressor.compressFloat((float[])data); - case DATA_DOUBLE: - default: - return compressor.compressDouble((double[])data); - } - } - - public abstract void setValue(T value); - - public abstract T getValue(); - - public abstract void setValueInBytes(byte[] value); - - public abstract void compress(); - - public abstract void uncompress(DataType dataType, byte[] compressData, int offset, - int length, int decimal, Object maxValueObject); - - public byte[] getCompressedData() { return compressedValue; } - - public abstract long getLongValue(int index); - - public abstract double getDoubleValue(int index); - - public abstract BigDecimal getBigDecimalValue(int index); - - public abstract void freeMemory(); - - -}
