move org.apache.carbon.common in core
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/952cf517 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/952cf517 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/952cf517 Branch: refs/heads/movecorepackage Commit: 952cf51745ffeb69a6a5dcc93405364c359933c8 Parents: d2ae472 Author: jackylk <jacky.li...@huawei.com> Authored: Fri Jan 13 14:43:33 2017 +0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Fri Jan 13 14:43:33 2017 +0800 ---------------------------------------------------------------------- .../common/ext/ColumnUniqueIdGenerator.java | 41 - .../common/ext/DictionaryFactory.java | 119 -- .../carbondata/common/ext/PathFactory.java | 46 - .../common/factory/CarbonCommonFactory.java | 54 - .../BlockletLevelDeleteDeltaDataCache.java | 46 - .../iudprocessor/iuddata/BlockMappingVO.java | 54 - .../iuddata/BlockletDeleteDeltaCacheLoader.java | 80 -- .../iuddata/DeleteDeltaCacheLoaderIntf.java | 27 - .../iudprocessor/iuddata/RowCountDetailsVO.java | 68 - .../core/AbsoluteTableIdentifier.java | 2 +- .../dictionary/AbstractDictionaryCache.java | 6 +- .../dictionary/DictionaryCacheLoaderImpl.java | 2 +- .../core/compression/BigDecimalCompressor.java | 2 +- .../core/compression/BigIntCompressor.java | 2 +- .../core/compression/DoubleCompressor.java | 2 +- .../core/compression/ValueCompressor.java | 2 +- .../carbondata/core/datastorage/FileHolder.java | 88 ++ .../core/datastorage/MeasureDataWrapper.java | 30 + .../core/datastorage/NodeMeasureDataStore.java | 34 + .../columnar/BlockIndexerStorageForInt.java | 226 +++ .../BlockIndexerStorageForNoInvertedIndex.java | 159 ++ .../datastorage/columnar/ColumnGroupModel.java | 116 ++ .../columnar/ColumnWithIntIndex.java | 82 ++ .../columnar/ColumnWithIntIndexForHighCard.java | 49 + .../columnar/ColumnarKeyStoreDataHolder.java | 54 + .../columnar/ColumnarKeyStoreMetadata.java | 53 + .../core/datastorage/columnar/IndexStorage.java | 44 + .../datastorage/columnar/UnBlockIndexer.java | 78 + .../datastorage/compression/Compressor.java | 60 + .../compression/CompressorFactory.java | 52 + .../compression/MeasureMetaDataModel.java | 128 ++ .../compression/ReaderCompressModel.java | 61 + .../compression/SnappyCompressor.java | 222 +++ .../compression/ValueCompressionHolder.java | 115 ++ .../compression/WriterCompressModel.java | 220 +++ .../decimal/CompressionMaxMinByte.java | 115 ++ .../decimal/CompressionMaxMinDefault.java | 117 ++ .../decimal/CompressionMaxMinInt.java | 113 ++ .../decimal/CompressionMaxMinLong.java | 113 ++ .../decimal/CompressionMaxMinShort.java | 115 ++ .../nondecimal/CompressionNonDecimalByte.java | 99 ++ .../CompressionNonDecimalDefault.java | 101 ++ .../nondecimal/CompressionNonDecimalInt.java | 100 ++ .../nondecimal/CompressionNonDecimalLong.java | 101 ++ .../CompressionNonDecimalMaxMinByte.java | 105 ++ .../CompressionNonDecimalMaxMinDefault.java | 107 ++ .../CompressionNonDecimalMaxMinInt.java | 105 ++ .../CompressionNonDecimalMaxMinLong.java | 106 ++ .../CompressionNonDecimalMaxMinShort.java | 104 ++ .../nondecimal/CompressionNonDecimalShort.java | 100 ++ .../compression/none/CompressionNoneByte.java | 104 ++ .../none/CompressionNoneDefault.java | 102 ++ .../compression/none/CompressionNoneInt.java | 102 ++ .../compression/none/CompressionNoneLong.java | 99 ++ .../compression/none/CompressionNoneShort.java | 104 ++ .../compression/type/CompressionBigDecimal.java | 146 ++ .../dataholder/CarbonReadDataHolder.java | 50 + .../dataholder/CarbonWriteDataHolder.java | 280 ++++ .../filesystem/AbstractDFSCarbonFile.java | 217 +++ .../filesystem/AlluxioCarbonFile.java | 130 ++ .../core/datastorage/filesystem/CarbonFile.java | 66 + .../filesystem/CarbonFileFilter.java | 24 + .../datastorage/filesystem/HDFSCarbonFile.java | 128 ++ .../datastorage/filesystem/LocalCarbonFile.java | 229 +++ .../filesystem/ViewFSCarbonFile.java | 126 ++ .../impl/CompressedDataMeasureDataWrapper.java | 37 + .../datastorage/impl/DFSFileHolderImpl.java | 134 ++ .../core/datastorage/impl/FileFactory.java | 485 +++++++ .../core/datastorage/impl/FileHolderImpl.java | 198 +++ ...ractHeavyCompressedDoubleArrayDataStore.java | 90 ++ ...yCompressedDoubleArrayDataInMemoryStore.java | 30 + .../core/datastorage/store/FileHolder.java | 88 -- .../datastorage/store/MeasureDataWrapper.java | 30 - .../datastorage/store/NodeMeasureDataStore.java | 34 - .../columnar/BlockIndexerStorageForInt.java | 226 --- .../BlockIndexerStorageForNoInvertedIndex.java | 159 -- .../store/columnar/ColumnGroupModel.java | 116 -- .../store/columnar/ColumnWithIntIndex.java | 82 -- .../columnar/ColumnWithIntIndexForHighCard.java | 49 - .../columnar/ColumnarKeyStoreDataHolder.java | 54 - .../columnar/ColumnarKeyStoreMetadata.java | 53 - .../store/columnar/IndexStorage.java | 44 - .../store/columnar/UnBlockIndexer.java | 78 - .../store/compression/Compressor.java | 60 - .../store/compression/CompressorFactory.java | 52 - .../store/compression/MeasureMetaDataModel.java | 128 -- .../store/compression/ReaderCompressModel.java | 61 - .../store/compression/SnappyCompressor.java | 222 --- .../compression/ValueCompressionHolder.java | 115 -- .../store/compression/WriterCompressModel.java | 220 --- .../decimal/CompressionMaxMinByte.java | 115 -- .../decimal/CompressionMaxMinDefault.java | 117 -- .../decimal/CompressionMaxMinInt.java | 113 -- .../decimal/CompressionMaxMinLong.java | 113 -- .../decimal/CompressionMaxMinShort.java | 115 -- .../nondecimal/CompressionNonDecimalByte.java | 99 -- .../CompressionNonDecimalDefault.java | 101 -- .../nondecimal/CompressionNonDecimalInt.java | 100 -- .../nondecimal/CompressionNonDecimalLong.java | 101 -- .../CompressionNonDecimalMaxMinByte.java | 105 -- .../CompressionNonDecimalMaxMinDefault.java | 107 -- .../CompressionNonDecimalMaxMinInt.java | 105 -- .../CompressionNonDecimalMaxMinLong.java | 106 -- .../CompressionNonDecimalMaxMinShort.java | 104 -- .../nondecimal/CompressionNonDecimalShort.java | 100 -- .../compression/none/CompressionNoneByte.java | 104 -- .../none/CompressionNoneDefault.java | 102 -- .../compression/none/CompressionNoneInt.java | 102 -- .../compression/none/CompressionNoneLong.java | 99 -- .../compression/none/CompressionNoneShort.java | 104 -- .../compression/type/CompressionBigDecimal.java | 146 -- .../store/dataholder/CarbonReadDataHolder.java | 50 - .../store/dataholder/CarbonWriteDataHolder.java | 280 ---- .../store/filesystem/AbstractDFSCarbonFile.java | 217 --- .../store/filesystem/AlluxioCarbonFile.java | 130 -- .../store/filesystem/CarbonFile.java | 66 - .../store/filesystem/CarbonFileFilter.java | 24 - .../store/filesystem/HDFSCarbonFile.java | 128 -- .../store/filesystem/LocalCarbonFile.java | 229 --- .../store/filesystem/ViewFSCarbonFile.java | 126 -- .../impl/CompressedDataMeasureDataWrapper.java | 37 - .../store/impl/DFSFileHolderImpl.java | 134 -- .../datastorage/store/impl/FileFactory.java | 485 ------- .../datastorage/store/impl/FileHolderImpl.java | 198 --- ...ractHeavyCompressedDoubleArrayDataStore.java | 90 -- ...yCompressedDoubleArrayDataInMemoryStore.java | 30 - .../core/datastorage/util/StoreFactory.java | 35 - .../core/datastore/BlockIndexStore.java | 4 +- .../carbondata/core/datastore/DataRefNode.java | 4 +- .../core/datastore/SegmentTaskIndexStore.java | 2 +- .../core/datastore/block/SegmentProperties.java | 2 +- .../block/SegmentTaskIndexWrapper.java | 2 +- .../core/datastore/block/TableBlockInfo.java | 2 +- .../chunk/DimensionColumnDataChunk.java | 4 +- .../datastore/chunk/MeasureColumnDataChunk.java | 2 +- .../impl/ColumnGroupDimensionDataChunk.java | 4 +- .../impl/FixedLengthDimensionDataChunk.java | 6 +- .../impl/VariableLengthDimensionDataChunk.java | 6 +- .../reader/DimensionColumnChunkReader.java | 2 +- .../chunk/reader/MeasureColumnChunkReader.java | 2 +- .../reader/dimension/AbstractChunkReader.java | 4 +- ...mpressedDimensionChunkFileBasedReaderV1.java | 4 +- ...mpressedDimensionChunkFileBasedReaderV2.java | 4 +- ...CompressedMeasureChunkFileBasedReaderV1.java | 8 +- ...CompressedMeasureChunkFileBasedReaderV2.java | 10 +- .../impl/btree/AbstractBTreeLeafNode.java | 4 +- .../datastore/impl/btree/BTreeNonLeafNode.java | 4 +- .../impl/btree/BlockletBTreeLeafNode.java | 2 +- .../IncrementalColumnDictionaryGenerator.java | 2 +- .../carbondata/core/load/BlockDetails.java | 2 +- .../core/metadata/BlockletInfoColumnar.java | 2 +- .../carbondata/core/partition/Partitioner.java | 26 - .../partition/impl/HashPartitionerImpl.java | 105 -- .../carbondata/core/path/CarbonTablePath.java | 6 +- .../reader/CarbonDeleteDeltaFileReaderImpl.java | 2 +- .../reader/CarbonDeleteFilesDataReader.java | 2 +- .../CarbonDictionaryMetadataReaderImpl.java | 2 +- .../core/reader/CarbonDictionaryReaderImpl.java | 2 +- .../carbondata/core/reader/ThriftReader.java | 2 +- .../CarbonDictionarySortIndexReaderImpl.java | 4 +- .../scan/collector/ScannedResultCollector.java | 45 + .../impl/AbstractScannedResultCollector.java | 159 ++ .../impl/DictionaryBasedResultCollector.java | 172 +++ .../DictionaryBasedVectorResultCollector.java | 136 ++ .../collector/impl/RawBasedResultCollector.java | 74 + .../core/scan/complextypes/ArrayQueryType.java | 103 ++ .../scan/complextypes/ComplexQueryType.java | 64 + .../scan/complextypes/PrimitiveQueryType.java | 118 ++ .../core/scan/complextypes/StructQueryType.java | 114 ++ .../core/scan/executor/QueryExecutor.java | 51 + .../scan/executor/QueryExecutorFactory.java | 38 + .../exception/QueryExecutionException.java | 94 ++ .../executor/impl/AbstractQueryExecutor.java | 483 +++++++ .../scan/executor/impl/DetailQueryExecutor.java | 49 + .../executor/impl/QueryExecutorProperties.java | 73 + .../impl/VectorDetailQueryExecutor.java | 46 + .../scan/executor/infos/AggregatorInfo.java | 105 ++ .../scan/executor/infos/BlockExecutionInfo.java | 604 ++++++++ .../scan/executor/infos/KeyStructureInfo.java | 100 ++ .../core/scan/executor/util/QueryUtil.java | 882 ++++++++++++ .../scan/executor/util/RestructureUtil.java | 140 ++ .../core/scan/expression/BinaryExpression.java | 43 + .../core/scan/expression/ColumnExpression.java | 114 ++ .../core/scan/expression/Expression.java | 49 + .../core/scan/expression/ExpressionResult.java | 545 +++++++ .../core/scan/expression/LeafExpression.java | 24 + .../core/scan/expression/LiteralExpression.java | 69 + .../core/scan/expression/UnknownExpression.java | 28 + .../BinaryConditionalExpression.java | 37 + .../conditional/ConditionalExpression.java | 37 + .../conditional/EqualToExpression.java | 109 ++ .../GreaterThanEqualToExpression.java | 91 ++ .../conditional/GreaterThanExpression.java | 94 ++ .../expression/conditional/InExpression.java | 99 ++ .../conditional/LessThanEqualToExpression.java | 93 ++ .../conditional/LessThanExpression.java | 96 ++ .../expression/conditional/ListExpression.java | 62 + .../conditional/NotEqualsExpression.java | 105 ++ .../expression/conditional/NotInExpression.java | 98 ++ .../exception/FilterIllegalMemberException.java | 94 ++ .../exception/FilterUnsupportedException.java | 88 ++ .../scan/expression/logical/AndExpression.java | 63 + .../logical/BinaryLogicalExpression.java | 109 ++ .../expression/logical/FalseExpression.java | 70 + .../scan/expression/logical/OrExpression.java | 62 + .../core/scan/filter/DimColumnFilterInfo.java | 80 ++ .../scan/filter/FilterExpressionProcessor.java | 338 +++++ .../core/scan/filter/FilterProcessor.java | 57 + .../carbondata/core/scan/filter/FilterUtil.java | 1353 ++++++++++++++++++ .../core/scan/filter/GenericQueryType.java | 49 + .../filter/executer/AndFilterExecuterImpl.java | 63 + .../executer/DimColumnExecuterFilterInfo.java | 32 + .../ExcludeColGroupFilterExecuterImpl.java | 135 ++ .../executer/ExcludeFilterExecuterImpl.java | 153 ++ .../scan/filter/executer/FilterExecuter.java | 47 + .../IncludeColGroupFilterExecuterImpl.java | 207 +++ .../executer/IncludeFilterExecuterImpl.java | 178 +++ .../filter/executer/OrFilterExecuterImpl.java | 53 + .../executer/RestructureFilterExecuterImpl.java | 54 + .../executer/RowLevelFilterExecuterImpl.java | 368 +++++ .../RowLevelRangeGrtThanFiterExecuterImpl.java | 212 +++ ...elRangeGrtrThanEquaToFilterExecuterImpl.java | 199 +++ ...velRangeLessThanEqualFilterExecuterImpl.java | 248 ++++ .../RowLevelRangeLessThanFiterExecuterImpl.java | 255 ++++ .../RowLevelRangeTypeExecuterFacory.java | 92 ++ .../core/scan/filter/intf/ExpressionType.java | 45 + .../scan/filter/intf/FilterExecuterType.java | 28 + .../core/scan/filter/intf/RowImpl.java | 44 + .../core/scan/filter/intf/RowIntf.java | 31 + .../resolver/ConditionalFilterResolverImpl.java | 229 +++ .../filter/resolver/FilterResolverIntf.java | 100 ++ .../resolver/LogicalFilterResolverImpl.java | 115 ++ .../resolver/RowLevelFilterResolverImpl.java | 136 ++ .../RowLevelRangeFilterResolverImpl.java | 266 ++++ .../metadata/FilterResolverMetadata.java | 62 + .../DimColumnResolvedFilterInfo.java | 143 ++ .../MeasureColumnResolvedFilterInfo.java | 60 + .../visitor/CustomTypeDictionaryVisitor.java | 101 ++ .../visitor/DictionaryColumnVisitor.java | 67 + .../visitor/FilterInfoTypeVisitorFactory.java | 45 + .../visitor/NoDictionaryTypeVisitor.java | 64 + .../visitor/ResolvedFilterInfoVisitorIntf.java | 42 + .../core/scan/model/CarbonQueryPlan.java | 156 ++ .../carbondata/core/scan/model/QueryColumn.java | 68 + .../core/scan/model/QueryDimension.java | 58 + .../core/scan/model/QueryMeasure.java | 61 + .../carbondata/core/scan/model/QueryModel.java | 365 +++++ .../processor/AbstractDataBlockIterator.java | 146 ++ .../core/scan/processor/BlockletIterator.java | 88 ++ .../core/scan/processor/BlocksChunkHolder.java | 125 ++ .../processor/impl/DataBlockIteratorImpl.java | 73 + .../core/scan/result/AbstractScannedResult.java | 570 ++++++++ .../core/scan/result/BatchResult.java | 105 ++ .../result/impl/FilterQueryScannedResult.java | 182 +++ .../impl/NonFilterQueryScannedResult.java | 146 ++ .../AbstractDetailQueryResultIterator.java | 185 +++ .../scan/result/iterator/ChunkRowIterator.java | 79 + .../iterator/DetailQueryResultIterator.java | 85 ++ .../scan/result/iterator/RawResultIterator.java | 169 +++ .../result/iterator/VectorChunkRowIterator.java | 93 ++ .../VectorDetailQueryResultIterator.java | 52 + .../scan/result/vector/CarbonColumnVector.java | 47 + .../scan/result/vector/CarbonColumnarBatch.java | 62 + .../scan/result/vector/ColumnVectorInfo.java | 41 + .../vector/MeasureDataVectorProcessor.java | 268 ++++ .../vector/impl/CarbonColumnVectorImpl.java | 154 ++ .../scan/scanner/AbstractBlockletScanner.java | 93 ++ .../core/scan/scanner/BlockletScanner.java | 42 + .../core/scan/scanner/impl/FilterScanner.java | 200 +++ .../scan/scanner/impl/NonFilterScanner.java | 40 + .../core/scan/wrappers/ByteArrayWrapper.java | 221 +++ .../core/service/CarbonCommonFactory.java | 54 + .../service/impl/ColumnUniqueIdGenerator.java | 41 + .../core/service/impl/DictionaryFactory.java | 119 ++ .../core/service/impl/PathFactory.java | 44 + .../core/update/CarbonUpdateUtil.java | 10 +- .../apache/carbondata/core/update/UpdateVO.java | 108 -- .../core/update/data/BlockMappingVO.java | 54 + .../data/BlockletDeleteDeltaCacheLoader.java | 79 + .../data/BlockletLevelDeleteDeltaDataCache.java | 46 + .../update/data/DeleteDeltaCacheLoaderIntf.java | 27 + .../core/update/data/RowCountDetailsVO.java | 68 + .../carbondata/core/update/data/UpdateVO.java | 108 ++ .../core/updatestatus/SegmentStatusManager.java | 2 +- .../SegmentUpdateStatusManager.java | 8 +- .../core/util/CarbonMetadataUtil.java | 4 +- .../apache/carbondata/core/util/CarbonUtil.java | 20 +- .../core/util/DataFileFooterConverter.java | 4 +- .../core/util/ValueCompressionUtil.java | 33 +- .../writer/CarbonDeleteDeltaWriterImpl.java | 2 +- .../core/writer/CarbonDictionaryWriterImpl.java | 6 +- .../carbondata/core/writer/ThriftWriter.java | 2 +- .../CarbonDictionarySortIndexWriterImpl.java | 6 +- .../AtomicFileOperationsImpl.java | 6 +- .../apache/carbondata/locks/HdfsFileLock.java | 4 +- .../apache/carbondata/locks/LocalFileLock.java | 2 +- .../scan/collector/ScannedResultCollector.java | 45 - .../impl/AbstractScannedResultCollector.java | 159 -- .../impl/DictionaryBasedResultCollector.java | 172 --- .../DictionaryBasedVectorResultCollector.java | 136 -- .../collector/impl/RawBasedResultCollector.java | 74 - .../scan/complextypes/ArrayQueryType.java | 103 -- .../scan/complextypes/ComplexQueryType.java | 64 - .../scan/complextypes/PrimitiveQueryType.java | 118 -- .../scan/complextypes/StructQueryType.java | 114 -- .../carbondata/scan/executor/QueryExecutor.java | 51 - .../scan/executor/QueryExecutorFactory.java | 38 - .../exception/QueryExecutionException.java | 94 -- .../executor/impl/AbstractQueryExecutor.java | 483 ------- .../scan/executor/impl/DetailQueryExecutor.java | 49 - .../executor/impl/QueryExecutorProperties.java | 73 - .../impl/VectorDetailQueryExecutor.java | 46 - .../scan/executor/infos/AggregatorInfo.java | 105 -- .../scan/executor/infos/BlockExecutionInfo.java | 604 -------- .../scan/executor/infos/KeyStructureInfo.java | 100 -- .../scan/executor/util/QueryUtil.java | 882 ------------ .../scan/executor/util/RestructureUtil.java | 140 -- .../scan/expression/BinaryExpression.java | 43 - .../scan/expression/ColumnExpression.java | 114 -- .../carbondata/scan/expression/Expression.java | 49 - .../scan/expression/ExpressionResult.java | 545 ------- .../scan/expression/LeafExpression.java | 24 - .../scan/expression/LiteralExpression.java | 69 - .../scan/expression/UnknownExpression.java | 28 - .../BinaryConditionalExpression.java | 37 - .../conditional/ConditionalExpression.java | 37 - .../conditional/EqualToExpression.java | 109 -- .../GreaterThanEqualToExpression.java | 91 -- .../conditional/GreaterThanExpression.java | 94 -- .../expression/conditional/InExpression.java | 99 -- .../conditional/LessThanEqualToExpression.java | 93 -- .../conditional/LessThanExpression.java | 96 -- .../expression/conditional/ListExpression.java | 62 - .../conditional/NotEqualsExpression.java | 105 -- .../expression/conditional/NotInExpression.java | 98 -- .../exception/FilterIllegalMemberException.java | 94 -- .../exception/FilterUnsupportedException.java | 88 -- .../scan/expression/logical/AndExpression.java | 63 - .../logical/BinaryLogicalExpression.java | 109 -- .../expression/logical/FalseExpression.java | 70 - .../scan/expression/logical/OrExpression.java | 62 - .../scan/filter/DimColumnFilterInfo.java | 80 -- .../scan/filter/FilterExpressionProcessor.java | 338 ----- .../carbondata/scan/filter/FilterProcessor.java | 57 - .../carbondata/scan/filter/FilterUtil.java | 1353 ------------------ .../scan/filter/GenericQueryType.java | 49 - .../filter/executer/AndFilterExecuterImpl.java | 63 - .../executer/DimColumnExecuterFilterInfo.java | 32 - .../ExcludeColGroupFilterExecuterImpl.java | 135 -- .../executer/ExcludeFilterExecuterImpl.java | 153 -- .../scan/filter/executer/FilterExecuter.java | 47 - .../IncludeColGroupFilterExecuterImpl.java | 207 --- .../executer/IncludeFilterExecuterImpl.java | 178 --- .../filter/executer/OrFilterExecuterImpl.java | 53 - .../executer/RestructureFilterExecuterImpl.java | 54 - .../executer/RowLevelFilterExecuterImpl.java | 368 ----- .../RowLevelRangeGrtThanFiterExecuterImpl.java | 212 --- ...elRangeGrtrThanEquaToFilterExecuterImpl.java | 199 --- ...velRangeLessThanEqualFilterExecuterImpl.java | 248 ---- .../RowLevelRangeLessThanFiterExecuterImpl.java | 255 ---- .../RowLevelRangeTypeExecuterFacory.java | 92 -- .../scan/filter/intf/ExpressionType.java | 45 - .../scan/filter/intf/FilterExecuterType.java | 28 - .../carbondata/scan/filter/intf/RowImpl.java | 44 - .../carbondata/scan/filter/intf/RowIntf.java | 31 - .../resolver/ConditionalFilterResolverImpl.java | 229 --- .../filter/resolver/FilterResolverIntf.java | 100 -- .../resolver/LogicalFilterResolverImpl.java | 115 -- .../resolver/RowLevelFilterResolverImpl.java | 136 -- .../RowLevelRangeFilterResolverImpl.java | 266 ---- .../metadata/FilterResolverMetadata.java | 62 - .../DimColumnResolvedFilterInfo.java | 143 -- .../MeasureColumnResolvedFilterInfo.java | 60 - .../visitor/CustomTypeDictionaryVisitor.java | 101 -- .../visitor/DictionaryColumnVisitor.java | 67 - .../visitor/FilterInfoTypeVisitorFactory.java | 45 - .../visitor/NoDictionaryTypeVisitor.java | 64 - .../visitor/ResolvedFilterInfoVisitorIntf.java | 42 - .../carbondata/scan/model/CarbonQueryPlan.java | 156 -- .../carbondata/scan/model/QueryColumn.java | 68 - .../carbondata/scan/model/QueryDimension.java | 58 - .../carbondata/scan/model/QueryMeasure.java | 61 - .../carbondata/scan/model/QueryModel.java | 365 ----- .../processor/AbstractDataBlockIterator.java | 146 -- .../scan/processor/BlockletIterator.java | 88 -- .../scan/processor/BlocksChunkHolder.java | 125 -- .../processor/impl/DataBlockIteratorImpl.java | 73 - .../scan/result/AbstractScannedResult.java | 570 -------- .../carbondata/scan/result/BatchResult.java | 105 -- .../result/impl/FilterQueryScannedResult.java | 182 --- .../impl/NonFilterQueryScannedResult.java | 146 -- .../AbstractDetailQueryResultIterator.java | 185 --- .../scan/result/iterator/ChunkRowIterator.java | 79 - .../iterator/DetailQueryResultIterator.java | 85 -- .../scan/result/iterator/RawResultIterator.java | 169 --- .../result/iterator/VectorChunkRowIterator.java | 93 -- .../VectorDetailQueryResultIterator.java | 52 - .../scan/result/vector/CarbonColumnVector.java | 47 - .../scan/result/vector/CarbonColumnarBatch.java | 62 - .../scan/result/vector/ColumnVectorInfo.java | 41 - .../vector/MeasureDataVectorProcessor.java | 268 ---- .../vector/impl/CarbonColumnVectorImpl.java | 154 -- .../scan/scanner/AbstractBlockletScanner.java | 93 -- .../scan/scanner/BlockletScanner.java | 42 - .../scan/scanner/impl/FilterScanner.java | 200 --- .../scan/scanner/impl/NonFilterScanner.java | 40 - .../scan/wrappers/ByteArrayWrapper.java | 221 --- .../core/AbsoluteTableIdentifierTest.java | 105 ++ .../core/CarbonTableIdentifierTest.java | 98 ++ .../carbondata/core/ColumnIdentifierTest.java | 100 ++ .../dictionary/AbstractDictionaryCacheTest.java | 4 +- .../carbon/AbsoluteTableIdentifierTest.java | 105 -- .../core/carbon/CarbonTableIdentifierTest.java | 98 -- .../core/carbon/ColumnIdentifierTest.java | 100 -- .../impl/CompressedDataMeasureWrapperTest.java | 47 - .../store/impl/DFSFileHolderImplUnitTest.java | 153 -- .../store/impl/FileFactoryImplUnitTest.java | 153 -- .../store/impl/FileHolderImplUnitTest.java | 157 -- .../DriverQueryStatisticsRecorderImplTest.java | 124 -- .../QueryStasticsRecorderImplTest.java | 114 -- .../CompressedDataMeasureWrapperTest.java | 47 + .../datastorage/DFSFileHolderImplUnitTest.java | 153 ++ .../datastorage/FileFactoryImplUnitTest.java | 153 ++ .../datastorage/FileHolderImplUnitTest.java | 157 ++ .../ColumnarKeyStoreDataHolderUnitTest.java | 93 ++ .../filesystem/AlluxioCarbonFileTest.java | 292 ++++ .../filesystem/HDFSCarbonFileTest.java | 400 ++++++ .../filesystem/LocalCarbonFileTest.java | 471 ++++++ .../filesystem/ViewFsCarbonFileTest.java | 311 ++++ .../ColumnarKeyStoreDataHolderUnitTest.java | 93 -- .../store/filesystem/AlluxioCarbonFileTest.java | 295 ---- .../store/filesystem/HDFSCarbonFileTest.java | 402 ------ .../store/filesystem/LocalCarbonFileTest.java | 472 ------ .../store/filesystem/ViewFsCarbonFileTest.java | 314 ---- .../impl/ColumnGroupDimensionDataChunkTest.java | 4 +- .../impl/FixedLengthDimensionDataChunkTest.java | 2 +- ...mpressedMeasureChunkFileBasedReaderTest.java | 30 +- .../reader/CarbonDictionaryReaderImplTest.java | 4 +- ...CarbonDictionarySortIndexReaderImplTest.java | 4 +- .../DictionaryBasedResultCollectorTest.java | 154 ++ .../impl/RawBasedResultCollectorTest.java | 132 ++ .../scan/complextypes/ArrayQueryTypeTest.java | 57 + .../complextypes/PrimitiveQueryTypeTest.java | 127 ++ .../scan/complextypes/StructQueryTypeTest.java | 69 + .../core/scan/executor/util/QueryUtilTest.java | 241 ++++ .../scan/executor/util/RestructureUtilTest.java | 115 ++ .../scan/expression/ColumnExpressionTest.java | 71 + .../scan/expression/ExpressionResultTest.java | 498 +++++++ .../scan/expression/LiteralExpressionTest.java | 59 + .../conditional/EqualToExpressionUnitTest.java | 315 ++++ .../GreaterThanEqualToExpressionUnitTest.java | 293 ++++ .../GreaterThanExpressionUnitTest.java | 364 +++++ .../conditional/InExpressionUnitTest.java | 276 ++++ .../LessThanEqualToExpressionUnitTest.java | 368 +++++ .../conditional/LessThanExpressionUnitTest.java | 364 +++++ .../conditional/ListExpressionUnitTest.java | 64 + .../NotEqualsExpressionUnitTest.java | 406 ++++++ .../conditional/NotInExpressionUnitTest.java | 275 ++++ .../expression/logical/AndExpressionTest.java | 81 ++ .../expression/logical/FalseExpressionTest.java | 48 + .../expression/logical/OrExpressionTest.java | 79 + .../core/scan/filter/FilterUtilTest.java | 392 +++++ .../core/scan/result/BatchResultTest.java | 95 ++ .../impl/FilterQueryScannedResultTest.java | 210 +++ .../impl/NonFilterQueryScannedResultTest.java | 209 +++ .../scan/wrappers/ByteArrayWrapperTest.java | 107 ++ .../DriverQueryStatisticsRecorderImplTest.java | 124 ++ .../stats/QueryStasticsRecorderImplTest.java | 114 ++ .../core/util/CarbonMetadataUtilTest.java | 2 +- .../carbondata/core/util/CarbonUtilTest.java | 10 +- .../core/util/DataFileFooterConverterTest.java | 6 +- .../core/util/ValueCompressionUtilTest.java | 29 +- .../writer/CarbonDictionaryWriterImplTest.java | 4 +- .../core/writer/CarbonFooterWriterTest.java | 6 +- .../DictionaryBasedResultCollectorTest.java | 154 -- .../impl/RawBasedResultCollectorTest.java | 132 -- .../scan/complextypes/ArrayQueryTypeTest.java | 57 - .../complextypes/PrimitiveQueryTypeTest.java | 132 -- .../scan/complextypes/StructQueryTypeTest.java | 69 - .../scan/executor/util/QueryUtilTest.java | 241 ---- .../scan/executor/util/RestructureUtilTest.java | 115 -- .../scan/expression/ColumnExpressionTest.java | 71 - .../scan/expression/ExpressionResultTest.java | 498 ------- .../scan/expression/LiteralExpressionTest.java | 59 - .../conditional/EqualToExpressionUnitTest.java | 315 ---- .../GreaterThanEqualToExpressionUnitTest.java | 293 ---- .../GreaterThanExpressionUnitTest.java | 364 ----- .../conditional/InExpressionUnitTest.java | 276 ---- .../LessThanEqualToExpressionUnitTest.java | 368 ----- .../conditional/LessThanExpressionUnitTest.java | 364 ----- .../conditional/ListExpressionUnitTest.java | 64 - .../NotEqualsExpressionUnitTest.java | 406 ------ .../conditional/NotInExpressionUnitTest.java | 275 ---- .../expression/logical/AndExpressionTest.java | 81 -- .../expression/logical/FalseExpressionTest.java | 48 - .../expression/logical/OrExpressionTest.java | 79 - .../carbondata/scan/filter/FilterUtilTest.java | 392 ----- .../carbondata/scan/result/BatchResultTest.java | 95 -- .../impl/FilterQueryScannedResultTest.java | 210 --- .../impl/NonFilterQueryScannedResultTest.java | 209 --- .../scan/wrappers/ByteArrayWrapperTest.java | 107 -- .../scanner/impl/FilterScannerTest.java | 162 --- dev/findbugs-exclude.xml | 4 +- .../carbondata/examples/AlluxioExample.scala | 2 +- .../examples/util/AllDictionaryUtil.scala | 2 +- .../carbondata/hadoop/CarbonInputFormat.java | 16 +- .../carbondata/hadoop/CarbonInputSplit.java | 2 +- .../carbondata/hadoop/CarbonRecordReader.java | 10 +- .../hadoop/api/CarbonTableInputFormat.java | 4 +- .../carbondata/hadoop/internal/index/Index.java | 2 +- .../internal/index/impl/InMemoryBTreeIndex.java | 6 +- .../hadoop/internal/segment/Segment.java | 2 +- .../internal/segment/impl/IndexedSegment.java | 2 +- .../hadoop/util/BlockLevelTraverser.java | 2 +- .../hadoop/util/CarbonInputFormatUtil.java | 14 +- .../carbondata/hadoop/util/SchemaReader.java | 2 +- .../hadoop/ft/CarbonInputFormat_FT.java | 8 +- .../hadoop/ft/CarbonInputMapperTest.java | 8 +- .../test/util/ObjectSerializationUtilTest.java | 8 +- .../hadoop/test/util/StoreCreator.java | 2 +- .../dataload/TestLoadDataGeneral.scala | 2 +- .../blockprune/BlockPruneQueryTestCase.scala | 2 +- .../SubqueryWithFilterAndSortTestCase.scala | 2 +- .../ValueCompressionDataTypeTestCase.scala | 2 +- .../carbondata/spark/load/CarbonLoaderUtil.java | 8 +- .../spark/load/DeleteLoadFolders.java | 6 +- .../spark/merger/CarbonCompactionExecutor.java | 16 +- .../spark/merger/CarbonCompactionUtil.java | 2 +- .../spark/merger/CarbonDataMergerUtil.java | 6 +- .../spark/merger/RowResultMerger.java | 4 +- .../spark/merger/TupleConversionAdapter.java | 2 +- .../api/impl/QueryPartitionHelper.java | 2 +- .../carbondata/spark/util/CarbonQueryUtil.java | 2 +- .../apache/carbondata/spark/CarbonFilters.scala | 6 +- .../spark/DictionaryDetailHelper.scala | 6 +- .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 5 +- .../carbondata/spark/rdd/CarbonMergerRDD.scala | 4 +- .../carbondata/spark/rdd/CarbonScanRDD.scala | 4 +- .../spark/tasks/DictionaryWriterTask.scala | 2 +- .../spark/tasks/SortIndexWriterTask.scala | 2 +- .../carbondata/spark/util/CarbonScalaUtil.scala | 2 +- .../spark/util/GlobalDictionaryUtil.scala | 4 +- .../execution/command/carbonTableSchema.scala | 2 +- .../scala/org/apache/spark/util/FileUtils.scala | 4 +- .../apache/spark/sql/CarbonBoundReference.scala | 2 +- .../sql/CarbonDatasourceHadoopRelation.scala | 2 +- .../spark/sql/CarbonDatasourceRelation.scala | 2 +- .../scala/org/apache/spark/sql/CarbonScan.scala | 2 +- .../spark/sql/SparkUnknownExpression.scala | 8 +- .../sql/execution/command/IUDCommands.scala | 6 +- .../execution/command/carbonTableSchema.scala | 2 +- .../apache/spark/sql/hive/CarbonMetastore.scala | 4 +- .../CompactionSystemLockFeatureTest.scala | 2 +- ...GlobalDictionaryUtilConcurrentTestCase.scala | 4 +- .../readsupport/SparkRowReadSupportImpl.java | 4 +- .../vectorreader/ColumnarVectorWrapper.java | 2 +- .../VectorizedCarbonRecordReader.java | 22 +- .../apache/carbondata/spark/CarbonFilters.scala | 8 +- .../carbondata/spark/CarbonSparkFactory.scala | 2 +- .../spark/DictionaryDetailHelper.scala | 6 +- .../spark/rdd/CarbonDataRDDFactory.scala | 2 +- .../carbondata/spark/util/CarbonSparkUtil.scala | 2 +- .../carbondata/spark/util/QueryPlanUtil.scala | 2 +- .../spark/sql/CarbonDataFrameWriter.scala | 2 +- .../sql/CarbonDatasourceHadoopRelation.scala | 6 +- .../spark/sql/CarbonDictionaryDecoder.scala | 6 +- .../scala/org/apache/spark/sql/CarbonScan.scala | 2 +- .../spark/sql/SparkUnknownExpression.scala | 10 +- .../org/apache/spark/sql/TableCreator.scala | 2 +- .../execution/command/carbonTableSchema.scala | 9 +- .../apache/spark/sql/hive/CarbonMetastore.scala | 34 +- .../sql/optimizer/CarbonLateDecodeRule.scala | 2 +- .../processing/csvload/DataGraphExecuter.java | 8 +- .../processing/csvload/GraphExecutionUtil.java | 8 +- .../csvreaderstep/UnivocityCsvParser.java | 4 +- ...ConverterProcessorWithBucketingStepImpl.java | 4 +- .../newflow/steps/partition/Partitioner.java | 26 + .../partition/impl/HashPartitionerImpl.java | 105 ++ .../sortdata/CompressedTempSortFileWriter.java | 2 +- .../store/CarbonFactDataHandlerColumnar.java | 13 +- .../processing/store/StoreFactory.java | 35 + .../store/colgroup/ColGroupBlockStorage.java | 2 +- .../store/writer/AbstractFactDataWriter.java | 8 +- .../store/writer/CarbonFactDataWriter.java | 4 +- .../processing/store/writer/NodeHolder.java | 2 +- .../writer/v1/CarbonFactDataWriterImplV1.java | 4 +- .../csvbased/BadRecordsLogger.java | 4 +- .../util/CarbonDataProcessorUtil.java | 8 +- .../carbondata/test/util/StoreCreator.java | 2 +- 589 files changed, 31295 insertions(+), 31447 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java b/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java deleted file mode 100644 index 1f02e9a..0000000 --- a/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.common.ext; - -import java.util.UUID; - -import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; -import org.apache.carbondata.core.service.ColumnUniqueIdService; - -/** - * It returns unique id given column - */ -public class ColumnUniqueIdGenerator implements ColumnUniqueIdService { - - private static ColumnUniqueIdService columnUniqueIdService = new ColumnUniqueIdGenerator(); - - @Override public String generateUniqueId(String databaseName, ColumnSchema columnSchema) { - return UUID.randomUUID().toString(); - } - - public static ColumnUniqueIdService getInstance() { - return columnUniqueIdService; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java b/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java deleted file mode 100644 index a3b7639..0000000 --- a/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.common.ext; - -import org.apache.carbondata.core.CarbonTableIdentifier; -import org.apache.carbondata.core.ColumnIdentifier; -import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; -import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl; -import org.apache.carbondata.core.reader.CarbonDictionaryReader; -import org.apache.carbondata.core.reader.CarbonDictionaryReaderImpl; -import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader; -import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReaderImpl; -import org.apache.carbondata.core.service.DictionaryService; -import org.apache.carbondata.core.writer.CarbonDictionaryWriter; -import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl; -import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter; -import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriterImpl; - -/** - * service to get dictionary reader and writer - */ -public class DictionaryFactory implements DictionaryService { - - private static DictionaryService dictService = new DictionaryFactory(); - - /** - * get dictionary writer - * - * @param carbonTableIdentifier - * @param columnIdentifier - * @param carbonStorePath - * @return - */ - @Override public CarbonDictionaryWriter getDictionaryWriter( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { - return new CarbonDictionaryWriterImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier); - } - - /** - * get dictionary sort index writer - * - * @param carbonTableIdentifier - * @param columnIdentifier - * @param carbonStorePath - * @return - */ - @Override public CarbonDictionarySortIndexWriter getDictionarySortIndexWriter( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { - return new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier, - carbonStorePath); - } - - /** - * get dictionary metadata reader - * - * @param carbonTableIdentifier - * @param columnIdentifier - * @param carbonStorePath - * @return - */ - @Override public CarbonDictionaryMetadataReader getDictionaryMetadataReader( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { - return new CarbonDictionaryMetadataReaderImpl(carbonStorePath, carbonTableIdentifier, - columnIdentifier); - } - - /** - * get dictionary reader - * - * @param carbonTableIdentifier - * @param columnIdentifier - * @param carbonStorePath - * @return - */ - @Override public CarbonDictionaryReader getDictionaryReader( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { - return new CarbonDictionaryReaderImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier); - } - - /** - * get dictionary sort index reader - * - * @param carbonTableIdentifier - * @param columnIdentifier - * @param carbonStorePath - * @return - */ - @Override public CarbonDictionarySortIndexReader getDictionarySortIndexReader( - CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier, - String carbonStorePath) { - return new CarbonDictionarySortIndexReaderImpl(carbonTableIdentifier, columnIdentifier, - carbonStorePath); - } - - public static DictionaryService getInstance() { - return dictService; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java b/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java deleted file mode 100644 index b175334..0000000 --- a/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.common.ext; - -import org.apache.carbondata.core.CarbonTableIdentifier; -import org.apache.carbondata.core.path.CarbonStorePath; -import org.apache.carbondata.core.path.CarbonTablePath; -import org.apache.carbondata.core.service.PathService; - -/** - * Create helper to get path details - */ -public class PathFactory implements PathService { - - private static PathService pathService = new PathFactory(); - - /** - * @param storeLocation - * @param tableIdentifier - * @return store path related to tables - */ - @Override public CarbonTablePath getCarbonTablePath( - String storeLocation, CarbonTableIdentifier tableIdentifier) { - return CarbonStorePath.getCarbonTablePath(storeLocation, tableIdentifier); - } - - public static PathService getInstance() { - return pathService; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java b/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java deleted file mode 100644 index 09b4465..0000000 --- a/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.common.factory; - -import org.apache.carbondata.common.ext.ColumnUniqueIdGenerator; -import org.apache.carbondata.common.ext.DictionaryFactory; -import org.apache.carbondata.common.ext.PathFactory; -import org.apache.carbondata.core.service.ColumnUniqueIdService; -import org.apache.carbondata.core.service.DictionaryService; -import org.apache.carbondata.core.service.PathService; - -/** - * Interface to get services - */ -public class CarbonCommonFactory { - - /** - * @return dictionary service - */ - public static DictionaryService getDictionaryService() { - return DictionaryFactory.getInstance(); - } - - /** - * @return path service - */ - public static PathService getPathService() { - return PathFactory.getInstance(); - } - - /** - * @return unique id generator - */ - public static ColumnUniqueIdService getColumnUniqueIdGenerator() { - return ColumnUniqueIdGenerator.getInstance(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java deleted file mode 100644 index acc00f0..0000000 --- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.common.iudprocessor.cache; - -import org.roaringbitmap.RoaringBitmap; - -/** - * This class maintains delete delta data cache of each blocklet along with the block timestamp - */ -public class BlockletLevelDeleteDeltaDataCache { - private RoaringBitmap deleteDelataDataCache; - private String timeStamp; - - public BlockletLevelDeleteDeltaDataCache(int[] deleteDeltaFileData, String timeStamp) { - deleteDelataDataCache = RoaringBitmap.bitmapOf(deleteDeltaFileData); - this.timeStamp=timeStamp; - } - - public boolean contains(int key) { - return deleteDelataDataCache.contains(key); - } - - public int getSize() { - return deleteDelataDataCache.getCardinality(); - } - - public String getCacheTimeStamp() { - return timeStamp; - } -} - http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java deleted file mode 100644 index ab43b6b..0000000 --- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.common.iudprocessor.iuddata; - -import java.util.Map; - -/** - * VO class to store the details of segment and block count , block and its row count. - */ -public class BlockMappingVO { - - private Map<String, Long> blockRowCountMapping ; - - private Map<String, Long> segmentNumberOfBlockMapping ; - - private Map<String, RowCountDetailsVO> completeBlockRowDetailVO; - - public void setCompleteBlockRowDetailVO(Map<String, RowCountDetailsVO> completeBlockRowDetailVO) { - this.completeBlockRowDetailVO = completeBlockRowDetailVO; - } - - public Map<String, RowCountDetailsVO> getCompleteBlockRowDetailVO() { - return completeBlockRowDetailVO; - } - - public Map<String, Long> getBlockRowCountMapping() { - return blockRowCountMapping; - } - - public Map<String, Long> getSegmentNumberOfBlockMapping() { - return segmentNumberOfBlockMapping; - } - - public BlockMappingVO(Map<String, Long> blockRowCountMapping, - Map<String, Long> segmentNumberOfBlockMapping) { - this.blockRowCountMapping = blockRowCountMapping; - this.segmentNumberOfBlockMapping = segmentNumberOfBlockMapping; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java deleted file mode 100644 index 1af983b..0000000 --- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.common.iudprocessor.iuddata; - -import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache; -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.AbsoluteTableIdentifier; -import org.apache.carbondata.core.datastore.DataRefNode; -import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager; - -/** - * This class is responsible for loading delete delta file cache based on - * blocklet id of a particular block - */ -public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderIntf { - private String blockletID; - private DataRefNode blockletNode; - private AbsoluteTableIdentifier absoluteIdentifier; - private static final LogService LOGGER = - LogServiceFactory.getLogService(BlockletDeleteDeltaCacheLoader.class.getName()); - - public BlockletDeleteDeltaCacheLoader(String blockletID, - DataRefNode blockletNode, AbsoluteTableIdentifier absoluteIdentifier) { - this.blockletID = blockletID; - this.blockletNode = blockletNode; - this.absoluteIdentifier= absoluteIdentifier; - } - - /** - * This method will load the delete delta cache based on blocklet id of particular block with - * the help of SegmentUpdateStatusManager. - */ - public void loadDeleteDeltaFileDataToCache() { - SegmentUpdateStatusManager segmentUpdateStatusManager = - new SegmentUpdateStatusManager(absoluteIdentifier); - int[] deleteDeltaFileData = null; - BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache = null; - if (null == blockletNode.getDeleteDeltaDataCache()) { - try { - deleteDeltaFileData = segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID); - deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData, - segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, null)); - } catch (Exception e) { - LOGGER.debug("Unable to retrieve delete delta files"); - } - } else { - deleteDeltaDataCache = blockletNode.getDeleteDeltaDataCache(); - // if already cache is present then validate the cache using timestamp - String cacheTimeStamp = segmentUpdateStatusManager - .getTimestampForRefreshCache(blockletID, deleteDeltaDataCache.getCacheTimeStamp()); - if (null != cacheTimeStamp) { - try { - deleteDeltaFileData = - segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID); - deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData, - segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, cacheTimeStamp)); - } catch (Exception e) { - LOGGER.debug("Unable to retrieve delete delta files"); - } - } - } - blockletNode.setDeleteDeltaDataCache(deleteDeltaDataCache); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java deleted file mode 100644 index 2455910..0000000 --- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.common.iudprocessor.iuddata; - -/** - * This interface holds all methods required to load delete delta file data to cache - */ -public interface DeleteDeltaCacheLoaderIntf { - - void loadDeleteDeltaFileDataToCache(); - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java deleted file mode 100644 index 7b5cc31..0000000 --- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.common.iudprocessor.iuddata; - -import java.io.Serializable; - -/** - * VO class Details for block. - */ -public class RowCountDetailsVO implements Serializable { - - private static final long serialVersionUID = 1206104914918491749L; - - private long totalNumberOfRows; - - private long deletedRowsInBlock; - - public RowCountDetailsVO(long totalNumberOfRows, long deletedRowsInBlock) { - this.totalNumberOfRows = totalNumberOfRows; - this.deletedRowsInBlock = deletedRowsInBlock; - } - - public long getTotalNumberOfRows() { - return totalNumberOfRows; - } - - public long getDeletedRowsInBlock() { - return deletedRowsInBlock; - } - - @Override public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - RowCountDetailsVO that = (RowCountDetailsVO) obj; - - if (totalNumberOfRows != that.totalNumberOfRows) { - return false; - } - return deletedRowsInBlock == that.deletedRowsInBlock; - - } - - @Override public int hashCode() { - int result = (int) (totalNumberOfRows ^ (totalNumberOfRows >>> 32)); - result = 31 * result + (int) (deletedRowsInBlock ^ (deletedRowsInBlock >>> 32)); - return result; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java index c07b8bf..e5823b9 100644 --- a/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java +++ b/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java @@ -22,7 +22,7 @@ import java.io.File; import java.io.Serializable; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.store.impl.FileFactory; +import org.apache.carbondata.core.datastorage.impl.FileFactory; import org.apache.carbondata.core.util.CarbonUtil; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java index 9143eb0..b1bf459 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java @@ -22,14 +22,14 @@ package org.apache.carbondata.core.cache.dictionary; import java.io.IOException; import java.util.List; -import org.apache.carbondata.common.factory.CarbonCommonFactory; +import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.core.cache.Cache; import org.apache.carbondata.core.cache.CacheType; import org.apache.carbondata.core.cache.CarbonLRUCache; import org.apache.carbondata.core.path.CarbonTablePath; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.store.impl.FileFactory; +import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; +import org.apache.carbondata.core.datastorage.impl.FileFactory; import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; import org.apache.carbondata.core.service.DictionaryService; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java index 6a6a86d..7d57676 100644 --- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java @@ -24,7 +24,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import org.apache.carbondata.common.factory.CarbonCommonFactory; +import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.core.CarbonTableIdentifier; import org.apache.carbondata.core.ColumnIdentifier; import org.apache.carbondata.core.constants.CarbonCommonConstants; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java index 96ca2e1..4da8795 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.compression; -import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; import org.apache.carbondata.core.util.BigDecimalCompressionFinder; import org.apache.carbondata.core.util.CompressionFinder; import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java index c27887d..e509c5c 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.compression; -import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java index 383a515..a716921 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java @@ -20,7 +20,7 @@ package org.apache.carbondata.core.compression; import java.math.BigDecimal; -import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java index 31f86c0..a2d7b51 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.compression; -import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; import org.apache.carbondata.core.util.CompressionFinder; import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE; import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java new file mode 100644 index 0000000..96aa069 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastorage; + +import java.io.IOException; + +public interface FileHolder { + /** + * This method will be used to read the byte array from file based on offset + * and length(number of bytes) need to read + * + * @param filePath fully qualified file path + * @param offset reading start position, + * @param length number of bytes to be read + * @return read byte array + */ + byte[] readByteArray(String filePath, long offset, int length) throws IOException; + + /** + * This method will be used to read the byte array from file based on length(number of bytes) + * + * @param filePath fully qualified file path + * @param length number of bytes to be read + * @return read byte array + */ + byte[] readByteArray(String filePath, int length) throws IOException; + + /** + * This method will be used to read int from file from postion(offset), here + * length will be always 4 bacause int byte size if 4 + * + * @param filePath fully qualified file path + * @param offset reading start position, + * @return read int + */ + int readInt(String filePath, long offset) throws IOException; + + /** + * This method will be used to read long from file from postion(offset), here + * length will be always 8 bacause int byte size is 8 + * + * @param filePath fully qualified file path + * @param offset reading start position, + * @return read long + */ + long readLong(String filePath, long offset) throws IOException; + + /** + * This method will be used to read int from file from postion(offset), here + * length will be always 4 bacause int byte size if 4 + * + * @param filePath fully qualified file path + * @return read int + */ + int readInt(String filePath) throws IOException; + + /** + * This method will be used to read long value from file from postion(offset), here + * length will be always 8 because long byte size if 4 + * + * @param filePath fully qualified file path + * @param offset reading start position, + * @return read long + */ + long readDouble(String filePath, long offset) throws IOException; + + /** + * This method will be used to close all the streams currently present in the cache + */ + void finish() throws IOException; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java new file mode 100644 index 0000000..6fc6c1d --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastorage; + +import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder; + +/** + * MeasureDataWrapper, interface. + */ +public interface MeasureDataWrapper { + CarbonReadDataHolder[] getValues(); + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java new file mode 100644 index 0000000..fede06c --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastorage; + +import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; + +public interface NodeMeasureDataStore { + /** + * This method will be used to get the writable key array. + * writable measure data array will hold below information: + * <size of measure data array><measure data array> + * total length will be 4 bytes for size + measure data array length + * + * @return writable array (compressed or normal) + */ + byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java new file mode 100644 index 0000000..0a2a8bd --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastorage.columnar; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.ByteUtil; + +public class BlockIndexerStorageForInt implements IndexStorage<int[]> { + private boolean alreadySorted; + + private int[] dataAfterComp; + + private int[] indexMap; + + private byte[][] keyBlock; + + private int[] dataIndexMap; + + private int totalSize; + + public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary, + boolean isSortRequired) { + ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary); + if (isSortRequired) { + Arrays.sort(columnWithIndexs); + } + compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock)); + if (compressData) { + compressDataMyOwnWay(columnWithIndexs); + } + } + + /** + * Create an object with each column array and respective index + * + * @return + */ + private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock, + boolean isNoDictionary) { + ColumnWithIntIndex[] columnWithIndexs; + if (isNoDictionary) { + columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length]; + for (int i = 0; i < columnWithIndexs.length; i++) { + columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i); + } + + } else { + columnWithIndexs = new ColumnWithIntIndex[keyBlock.length]; + for (int i = 0; i < columnWithIndexs.length; i++) { + columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i); + } + } + + return columnWithIndexs; + } + + private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs, + byte[][] keyBlock) { + int[] indexes = new int[columnWithIndexs.length]; + for (int i = 0; i < indexes.length; i++) { + indexes[i] = columnWithIndexs[i].getIndex(); + keyBlock[i] = columnWithIndexs[i].getColumn(); + } + this.keyBlock = keyBlock; + return indexes; + } + + /** + * It compresses depends up on the sequence numbers. + * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In + * first array the start and end of sequential numbers and second array + * keeps the indexes of where sequential numbers starts. If there is no + * sequential numbers then the same array it returns with empty second + * array. + * + * @param indexes + */ + public void compressMyOwnWay(int[] indexes) { + List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + int k = 0; + int i = 1; + for (; i < indexes.length; i++) { + if (indexes[i] - indexes[i - 1] == 1) { + k++; + } else { + if (k > 0) { + map.add((list.size())); + list.add(indexes[i - k - 1]); + list.add(indexes[i - 1]); + } else { + list.add(indexes[i - 1]); + } + k = 0; + } + } + if (k > 0) { + map.add((list.size())); + list.add(indexes[i - k - 1]); + list.add(indexes[i - 1]); + } else { + list.add(indexes[i - 1]); + } + dataAfterComp = convertToArray(list); + if (indexes.length == dataAfterComp.length) { + indexMap = new int[0]; + } else { + indexMap = convertToArray(map); + } + if (dataAfterComp.length == 2 && indexMap.length == 1) { + alreadySorted = true; + } + } + + private int[] convertToArray(List<Integer> list) { + int[] shortArray = new int[list.size()]; + for (int i = 0; i < shortArray.length; i++) { + shortArray[i] = list.get(i); + } + return shortArray; + } + + /** + * @return the alreadySorted + */ + public boolean isAlreadySorted() { + return alreadySorted; + } + + /** + * @return the dataAfterComp + */ + public int[] getDataAfterComp() { + return dataAfterComp; + } + + /** + * @return the indexMap + */ + public int[] getIndexMap() { + return indexMap; + } + + /** + * @return the keyBlock + */ + public byte[][] getKeyBlock() { + return keyBlock; + } + + private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) { + byte[] prvKey = indexes[0].getColumn(); + List<ColumnWithIntIndex> list = + new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + list.add(indexes[0]); + int counter = 1; + int start = 0; + List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + for (int i = 1; i < indexes.length; i++) { + if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) { + prvKey = indexes[i].getColumn(); + list.add(indexes[i]); + map.add(start); + map.add(counter); + start += counter; + counter = 1; + continue; + } + counter++; + } + map.add(start); + map.add(counter); + this.keyBlock = convertToKeyArray(list); + if (indexes.length == keyBlock.length) { + dataIndexMap = new int[0]; + } else { + dataIndexMap = convertToArray(map); + } + } + + private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) { + byte[][] shortArray = new byte[list.size()][]; + for (int i = 0; i < shortArray.length; i++) { + shortArray[i] = list.get(i).getColumn(); + totalSize += shortArray[i].length; + } + return shortArray; + } + + @Override public int[] getDataIndexMap() { + return dataIndexMap; + } + + @Override public int getTotalSize() { + return totalSize; + } + + @Override public byte[] getMin() { + return keyBlock[0]; + } + + @Override public byte[] getMax() { + return keyBlock[keyBlock.length - 1]; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java new file mode 100644 index 0000000..939d759 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastorage.columnar; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.ByteUtil; + +public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> { + private byte[][] keyBlock; + private byte[][] sortedBlock; + private int totalSize; + private int[] dataIndexMap; + + public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData, + boolean isNoDictionary) { + // without invertedindex but can be RLE + if (compressData) { + // with RLE + byte[] prvKey = keyBlockInput[0]; + List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + list.add(keyBlockInput[0]); + int counter = 1; + int start = 0; + List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + int length = keyBlockInput.length; + for(int i = 1; i < length; i++) { + if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) { + prvKey = keyBlockInput[i]; + list.add(keyBlockInput[i]); + map.add(start); + map.add(counter); + start += counter; + counter = 1; + continue; + } + counter++; + } + map.add(start); + map.add(counter); + this.keyBlock = convertToKeyArray(list); + if (keyBlockInput.length == this.keyBlock.length) { + dataIndexMap = new int[0]; + } else { + dataIndexMap = convertToArray(map); + } + } else { + this.keyBlock = keyBlockInput; + dataIndexMap = new int[0]; + } + + this.sortedBlock = new byte[keyBlock.length][]; + System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length); + if (isNoDictionary) { + Arrays.sort(sortedBlock, new Comparator<byte[]>() { + @Override + public int compare(byte[] col1, byte[] col2) { + return ByteUtil.UnsafeComparer.INSTANCE + .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2); + } + }); + } else { + Arrays.sort(sortedBlock, new Comparator<byte[]>() { + @Override + public int compare(byte[] col1, byte[] col2) { + return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2); + } + }); + } + + } + + private int[] convertToArray(List<Integer> list) { + int[] shortArray = new int[list.size()]; + for(int i = 0; i < shortArray.length; i++) { + shortArray[i] = list.get(i); + } + return shortArray; + } + + private byte[][] convertToKeyArray(List<byte[]> list) { + byte[][] shortArray = new byte[list.size()][]; + for (int i = 0; i < shortArray.length; i++) { + shortArray[i] = list.get(i); + totalSize += shortArray[i].length; + } + return shortArray; + } + + @Override + public int[] getDataIndexMap() { + return dataIndexMap; + } + + @Override + public int getTotalSize() { + return totalSize; + } + + @Override + public boolean isAlreadySorted() { + return true; + } + + /** + * no use + * @return + */ + @Override + public int[] getDataAfterComp() { + return new int[0]; + } + + /** + * no use + * @return + */ + @Override + public int[] getIndexMap() { + return new int[0]; + } + + /** + * @return the keyBlock + */ + public byte[][] getKeyBlock() { + return keyBlock; + } + + @Override public byte[] getMin() { + return sortedBlock[0]; + } + + @Override public byte[] getMax() { + return sortedBlock[sortedBlock.length - 1]; + } + +}
