http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java deleted file mode 100644 index e47cf42..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.impl.data.compressed; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.NodeMeasureDataStore; -import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder; -import org.apache.carbondata.core.datastorage.compression.WriterCompressModel; -import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder; -import org.apache.carbondata.core.util.ValueCompressionUtil; - -public abstract class AbstractHeavyCompressedDoubleArrayDataStore - implements NodeMeasureDataStore //NodeMeasureDataStore<double[]> -{ - - private LogService LOGGER = - LogServiceFactory.getLogService(AbstractHeavyCompressedDoubleArrayDataStore.class.getName()); - - /** - * values. - */ - protected ValueCompressionHolder[] values; - - /** - * compressionModel. - */ - protected WriterCompressModel compressionModel; - - /** - * type - */ - private char[] type; - - /** - * AbstractHeavyCompressedDoubleArrayDataStore constructor. - * - * @param compressionModel - */ - public AbstractHeavyCompressedDoubleArrayDataStore(WriterCompressModel compressionModel) { - this.compressionModel = compressionModel; - if (null != compressionModel) { - this.type = compressionModel.getType(); - values = - new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length]; - } - } - - // this method first invokes encoding routine to encode the data chunk, - // followed by invoking compression routine for preparing the data chunk for writing. - @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) { - byte[][] returnValue = new byte[values.length][]; - for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) { - values[i] = compressionModel.getValueCompressionHolder()[i]; - if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) { - // first perform encoding of the data chunk - values[i].setValue( - ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i]) - .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i], - compressionModel.getMaxValue()[i], - compressionModel.getMantissa()[i])); - } else { - values[i].setValue(dataHolder[i].getWritableByteArrayValues()); - } - values[i].compress(); - returnValue[i] = values[i].getCompressedData(); - } - - return returnValue; - } - -}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java deleted file mode 100644 index ea42beb..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.impl.data.compressed; - -import org.apache.carbondata.core.datastorage.compression.WriterCompressModel; - -public class HeavyCompressedDoubleArrayDataInMemoryStore - extends AbstractHeavyCompressedDoubleArrayDataStore { - - public HeavyCompressedDoubleArrayDataInMemoryStore(WriterCompressModel compressionModel) { - super(compressionModel); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java index 9d4287a..7acfb39 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java @@ -20,7 +20,6 @@ package org.apache.carbondata.core.datastore; import java.io.IOException; -import org.apache.carbondata.core.datastorage.FileHolder; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk; import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java new file mode 100644 index 0000000..95ec433 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore; + +import java.io.IOException; + +public interface FileHolder { + /** + * This method will be used to read the byte array from file based on offset + * and length(number of bytes) need to read + * + * @param filePath fully qualified file path + * @param offset reading start position, + * @param length number of bytes to be read + * @return read byte array + */ + byte[] readByteArray(String filePath, long offset, int length) throws IOException; + + /** + * This method will be used to read the byte array from file based on length(number of bytes) + * + * @param filePath fully qualified file path + * @param length number of bytes to be read + * @return read byte array + */ + byte[] readByteArray(String filePath, int length) throws IOException; + + /** + * This method will be used to read int from file from postion(offset), here + * length will be always 4 bacause int byte size if 4 + * + * @param filePath fully qualified file path + * @param offset reading start position, + * @return read int + */ + int readInt(String filePath, long offset) throws IOException; + + /** + * This method will be used to read long from file from postion(offset), here + * length will be always 8 bacause int byte size is 8 + * + * @param filePath fully qualified file path + * @param offset reading start position, + * @return read long + */ + long readLong(String filePath, long offset) throws IOException; + + /** + * This method will be used to read int from file from postion(offset), here + * length will be always 4 bacause int byte size if 4 + * + * @param filePath fully qualified file path + * @return read int + */ + int readInt(String filePath) throws IOException; + + /** + * This method will be used to read long value from file from postion(offset), here + * length will be always 8 because long byte size if 4 + * + * @param filePath fully qualified file path + * @param offset reading start position, + * @return read long + */ + long readDouble(String filePath, long offset) throws IOException; + + /** + * This method will be used to close all the streams currently present in the cache + */ + void finish() throws IOException; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java new file mode 100644 index 0000000..ff6ed34 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore; + +import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder; + +/** + * MeasureDataWrapper, interface. + */ +public interface MeasureDataWrapper { + CarbonReadDataHolder[] getValues(); + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java new file mode 100644 index 0000000..c4e5d15 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore; + +import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder; + +public interface NodeMeasureDataStore { + /** + * This method will be used to get the writable key array. + * writable measure data array will hold below information: + * <size of measure data array><measure data array> + * total length will be 4 bytes for size + measure data array length + * + * @return writable array (compressed or normal) + */ + byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java index dde8e76..f8c4374 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java @@ -29,11 +29,10 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel; +import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel; import org.apache.carbondata.core.keygenerator.KeyGenerator; import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter; -import org.apache.carbondata.core.keygenerator.columnar.impl - .MultiDimKeyVarLengthVariableSplitGenerator; +import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator; import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator; import org.apache.carbondata.core.metadata.DataType; import org.apache.carbondata.core.metadata.Encoding; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java index c7ce3db..8774579 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java @@ -23,7 +23,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.path.CarbonTablePath; import org.apache.carbondata.core.path.CarbonTablePath.DataFileUtil; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java index ce09305..3c08f45 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.datastore.chunk; -import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder; +import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder; import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java index 6edc977..0730607 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java @@ -20,8 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.impl; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory - .DimensionStoreType; +import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType; import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java index 549d6d8..342c392 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java @@ -22,8 +22,7 @@ import java.util.Arrays; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory - .DimensionStoreType; +import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType; import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java index c83f2d7..3d4e3bf 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java @@ -18,14 +18,10 @@ */ package org.apache.carbondata.core.datastore.chunk.reader; -import org.apache.carbondata.core.datastore.chunk.reader.dimension - .v1.CompressedDimensionChunkFileBasedReaderV1; -import org.apache.carbondata.core.datastore.chunk.reader.dimension - .v2.CompressedDimensionChunkFileBasedReaderV2; -import org.apache.carbondata.core.datastore.chunk.reader.measure - .v1.CompressedMeasureChunkFileBasedReaderV1; -import org.apache.carbondata.core.datastore.chunk.reader.measure - .v2.CompressedMeasureChunkFileBasedReaderV2; +import org.apache.carbondata.core.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1; +import org.apache.carbondata.core.datastore.chunk.reader.dimension.v2.CompressedDimensionChunkFileBasedReaderV2; +import org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1; +import org.apache.carbondata.core.datastore.chunk.reader.measure.v2.CompressedMeasureChunkFileBasedReaderV2; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java index 92dcb60..4981241 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java @@ -20,7 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.reader; import java.io.IOException; -import org.apache.carbondata.core.datastorage.FileHolder; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java index 94004d9..a706d71 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java @@ -20,7 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.reader; import java.io.IOException; -import org.apache.carbondata.core.datastorage.FileHolder; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java index 4cc24d9..2de673a 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java @@ -19,9 +19,9 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.compression.Compressor; -import org.apache.carbondata.core.datastorage.compression.CompressorFactory; import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader; +import org.apache.carbondata.core.datastore.compression.Compressor; +import org.apache.carbondata.core.datastore.compression.CompressorFactory; import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor; import org.apache.carbondata.core.util.CarbonProperties; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java index dd487d9..a8d15fe 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java @@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v1; import java.io.IOException; import java.util.List; -import org.apache.carbondata.core.datastorage.FileHolder; -import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk; import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk; import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader; +import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; import org.apache.carbondata.core.metadata.Encoding; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java index 3bddf4e..ae5c9b4 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java @@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2; import java.io.IOException; import java.util.List; -import org.apache.carbondata.core.datastorage.FileHolder; -import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk; import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk; import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader; +import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.format.DataChunk2; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java index 48cb244..1f3fec0 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java @@ -21,12 +21,12 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v1; import java.io.IOException; import java.util.List; -import org.apache.carbondata.core.datastorage.FileHolder; -import org.apache.carbondata.core.datastorage.compression.ReaderCompressModel; -import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder; -import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader; +import org.apache.carbondata.core.datastore.compression.ReaderCompressModel; +import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder; +import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder; import org.apache.carbondata.core.metadata.ValueEncoderMeta; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java index 3325299..2731163 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java @@ -23,13 +23,13 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.List; -import org.apache.carbondata.core.datastorage.FileHolder; -import org.apache.carbondata.core.datastorage.compression.CompressorFactory; -import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder; -import org.apache.carbondata.core.datastorage.compression.WriterCompressModel; -import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder; +import org.apache.carbondata.core.datastore.FileHolder; import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk; import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader; +import org.apache.carbondata.core.datastore.compression.CompressorFactory; +import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder; +import org.apache.carbondata.core.datastore.compression.WriterCompressModel; +import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder; import org.apache.carbondata.core.metadata.ValueEncoderMeta; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java index df054c5..e1ae9b3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java @@ -17,14 +17,10 @@ package org.apache.carbondata.core.datastore.chunk.store; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastore.chunk.store.impl.safe - .SafeFixedLengthDimensionDataChunkStore; -import org.apache.carbondata.core.datastore.chunk.store.impl.safe - .SafeVariableLengthDimensionDataChunkStore; -import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe - .UnsafeFixedLengthDimensionDataChunkStore; -import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe - .UnsafeVariableLengthDimesionDataChunkStore; +import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeFixedLengthDimensionDataChunkStore; +import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeVariableLengthDimensionDataChunkStore; +import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeFixedLengthDimensionDataChunkStore; +import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeVariableLengthDimesionDataChunkStore; import org.apache.carbondata.core.util.CarbonProperties; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java new file mode 100644 index 0000000..b6155fb --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.columnar; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.ByteUtil; + +public class BlockIndexerStorageForInt implements IndexStorage<int[]> { + private boolean alreadySorted; + + private int[] dataAfterComp; + + private int[] indexMap; + + private byte[][] keyBlock; + + private int[] dataIndexMap; + + private int totalSize; + + public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary, + boolean isSortRequired) { + ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary); + if (isSortRequired) { + Arrays.sort(columnWithIndexs); + } + compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock)); + if (compressData) { + compressDataMyOwnWay(columnWithIndexs); + } + } + + /** + * Create an object with each column array and respective index + * + * @return + */ + private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock, + boolean isNoDictionary) { + ColumnWithIntIndex[] columnWithIndexs; + if (isNoDictionary) { + columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length]; + for (int i = 0; i < columnWithIndexs.length; i++) { + columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i); + } + + } else { + columnWithIndexs = new ColumnWithIntIndex[keyBlock.length]; + for (int i = 0; i < columnWithIndexs.length; i++) { + columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i); + } + } + + return columnWithIndexs; + } + + private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs, + byte[][] keyBlock) { + int[] indexes = new int[columnWithIndexs.length]; + for (int i = 0; i < indexes.length; i++) { + indexes[i] = columnWithIndexs[i].getIndex(); + keyBlock[i] = columnWithIndexs[i].getColumn(); + } + this.keyBlock = keyBlock; + return indexes; + } + + /** + * It compresses depends up on the sequence numbers. + * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In + * first array the start and end of sequential numbers and second array + * keeps the indexes of where sequential numbers starts. If there is no + * sequential numbers then the same array it returns with empty second + * array. + * + * @param indexes + */ + public void compressMyOwnWay(int[] indexes) { + List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + int k = 0; + int i = 1; + for (; i < indexes.length; i++) { + if (indexes[i] - indexes[i - 1] == 1) { + k++; + } else { + if (k > 0) { + map.add((list.size())); + list.add(indexes[i - k - 1]); + list.add(indexes[i - 1]); + } else { + list.add(indexes[i - 1]); + } + k = 0; + } + } + if (k > 0) { + map.add((list.size())); + list.add(indexes[i - k - 1]); + list.add(indexes[i - 1]); + } else { + list.add(indexes[i - 1]); + } + dataAfterComp = convertToArray(list); + if (indexes.length == dataAfterComp.length) { + indexMap = new int[0]; + } else { + indexMap = convertToArray(map); + } + if (dataAfterComp.length == 2 && indexMap.length == 1) { + alreadySorted = true; + } + } + + private int[] convertToArray(List<Integer> list) { + int[] shortArray = new int[list.size()]; + for (int i = 0; i < shortArray.length; i++) { + shortArray[i] = list.get(i); + } + return shortArray; + } + + /** + * @return the alreadySorted + */ + public boolean isAlreadySorted() { + return alreadySorted; + } + + /** + * @return the dataAfterComp + */ + public int[] getDataAfterComp() { + return dataAfterComp; + } + + /** + * @return the indexMap + */ + public int[] getIndexMap() { + return indexMap; + } + + /** + * @return the keyBlock + */ + public byte[][] getKeyBlock() { + return keyBlock; + } + + private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) { + byte[] prvKey = indexes[0].getColumn(); + List<ColumnWithIntIndex> list = + new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + list.add(indexes[0]); + int counter = 1; + int start = 0; + List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + for (int i = 1; i < indexes.length; i++) { + if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) { + prvKey = indexes[i].getColumn(); + list.add(indexes[i]); + map.add(start); + map.add(counter); + start += counter; + counter = 1; + continue; + } + counter++; + } + map.add(start); + map.add(counter); + this.keyBlock = convertToKeyArray(list); + if (indexes.length == keyBlock.length) { + dataIndexMap = new int[0]; + } else { + dataIndexMap = convertToArray(map); + } + } + + private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) { + byte[][] shortArray = new byte[list.size()][]; + for (int i = 0; i < shortArray.length; i++) { + shortArray[i] = list.get(i).getColumn(); + totalSize += shortArray[i].length; + } + return shortArray; + } + + @Override public int[] getDataIndexMap() { + return dataIndexMap; + } + + @Override public int getTotalSize() { + return totalSize; + } + + @Override public byte[] getMin() { + return keyBlock[0]; + } + + @Override public byte[] getMax() { + return keyBlock[keyBlock.length - 1]; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java new file mode 100644 index 0000000..d7b980e --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.columnar; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.ByteUtil; + +public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> { + private byte[][] keyBlock; + private byte[][] sortedBlock; + private int totalSize; + private int[] dataIndexMap; + + public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData, + boolean isNoDictionary) { + // without invertedindex but can be RLE + if (compressData) { + // with RLE + byte[] prvKey = keyBlockInput[0]; + List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + list.add(keyBlockInput[0]); + int counter = 1; + int start = 0; + List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); + int length = keyBlockInput.length; + for(int i = 1; i < length; i++) { + if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) { + prvKey = keyBlockInput[i]; + list.add(keyBlockInput[i]); + map.add(start); + map.add(counter); + start += counter; + counter = 1; + continue; + } + counter++; + } + map.add(start); + map.add(counter); + this.keyBlock = convertToKeyArray(list); + if (keyBlockInput.length == this.keyBlock.length) { + dataIndexMap = new int[0]; + } else { + dataIndexMap = convertToArray(map); + } + } else { + this.keyBlock = keyBlockInput; + dataIndexMap = new int[0]; + } + + this.sortedBlock = new byte[keyBlock.length][]; + System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length); + if (isNoDictionary) { + Arrays.sort(sortedBlock, new Comparator<byte[]>() { + @Override + public int compare(byte[] col1, byte[] col2) { + return ByteUtil.UnsafeComparer.INSTANCE + .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2); + } + }); + } else { + Arrays.sort(sortedBlock, new Comparator<byte[]>() { + @Override + public int compare(byte[] col1, byte[] col2) { + return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2); + } + }); + } + + } + + private int[] convertToArray(List<Integer> list) { + int[] shortArray = new int[list.size()]; + for(int i = 0; i < shortArray.length; i++) { + shortArray[i] = list.get(i); + } + return shortArray; + } + + private byte[][] convertToKeyArray(List<byte[]> list) { + byte[][] shortArray = new byte[list.size()][]; + for (int i = 0; i < shortArray.length; i++) { + shortArray[i] = list.get(i); + totalSize += shortArray[i].length; + } + return shortArray; + } + + @Override + public int[] getDataIndexMap() { + return dataIndexMap; + } + + @Override + public int getTotalSize() { + return totalSize; + } + + @Override + public boolean isAlreadySorted() { + return true; + } + + /** + * no use + * @return + */ + @Override + public int[] getDataAfterComp() { + return new int[0]; + } + + /** + * no use + * @return + */ + @Override + public int[] getIndexMap() { + return new int[0]; + } + + /** + * @return the keyBlock + */ + public byte[][] getKeyBlock() { + return keyBlock; + } + + @Override public byte[] getMin() { + return sortedBlock[0]; + } + + @Override public byte[] getMax() { + return sortedBlock[sortedBlock.length - 1]; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java new file mode 100644 index 0000000..4d4c19d --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.datastore.columnar; + +public class ColumnGroupModel { + + /** + * number of columns in columnar block + */ + private int[] columnSplit; + + /** + * total number of columns + */ + private int noOfColumnsStore; + + /** + * whether given index is columnar or not + * true: columnar + * false: row block + */ + private boolean[] columnarStore; + + /** + * column groups + * e.g + * {{0,1,2},3,4,{5,6}} + */ + private int[][] columnGroups; + + /** + * return columnSplit + * + * @return + */ + public int[] getColumnSplit() { + return columnSplit; + } + + /** + * set columnSplit + * + * @param split + */ + public void setColumnSplit(int[] split) { + this.columnSplit = split; + } + + /** + * @return no of columnar block + */ + public int getNoOfColumnStore() { + return this.noOfColumnsStore; + } + + /** + * set no of columnar block + * + * @param noOfColumnsStore + */ + public void setNoOfColumnStore(int noOfColumnsStore) { + this.noOfColumnsStore = noOfColumnsStore; + } + + /** + * it's an identifier for row block or single column block + * + * @param columnarStore + */ + public void setColumnarStore(boolean[] columnarStore) { + this.columnarStore = columnarStore; + } + + /** + * set column groups + * + * @param columnGroups + */ + public void setColumnGroup(int[][] columnGroups) { + this.columnGroups = columnGroups; + } + + /** + * check if given column group is columnar + * + * @param colGroup + * @return true if given block is columnar + */ + public boolean isColumnar(int colGroup) { + return columnarStore[colGroup]; + } + + /** + * @return columngroups + */ + public int[][] getColumnGroup() { + return this.columnGroups; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java new file mode 100644 index 0000000..4b35a7f --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.columnar; + +import java.util.Arrays; + +import org.apache.carbondata.core.util.ByteUtil; + +public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> { + protected byte[] column; + + private int index; + + public ColumnWithIntIndex(byte[] column, int index) { + this.column = column; + this.index = index; + } + + public ColumnWithIntIndex() { + } + + /** + * @return the column + */ + public byte[] getColumn() { + return column; + } + + /** + * @param column the column to set + */ + public void setColumn(byte[] column) { + this.column = column; + } + + /** + * @return the index + */ + public int getIndex() { + return index; + } + + /** + * @param index the index to set + */ + public void setIndex(int index) { + this.index = index; + } + + @Override public int compareTo(ColumnWithIntIndex o) { + return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column); + } + + @Override public boolean equals(Object obj) { + if(obj == null || getClass() != obj.getClass()) { + return false; + } + ColumnWithIntIndex o = (ColumnWithIntIndex)obj; + return Arrays.equals(column, o.column) && index == o.index; + } + + @Override public int hashCode() { + return Arrays.hashCode(column) + index; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java new file mode 100644 index 0000000..a6eb6c4 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.columnar; + +import java.util.Arrays; + +import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer; + +public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex + implements Comparable<ColumnWithIntIndex> { + + public ColumnWithIntIndexForHighCard(byte[] column, int index) { + super(column, index); + } + + @Override public int compareTo(ColumnWithIntIndex o) { + return UnsafeComparer.INSTANCE + .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2); + } + + @Override public boolean equals(Object obj) { + if(obj == null || getClass() != obj.getClass()) { + return false; + } + ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj; + return Arrays.equals(column, o.column) && getIndex() == o.getIndex(); + } + + @Override public int hashCode() { + return Arrays.hashCode(column) + getIndex(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java new file mode 100644 index 0000000..6c3acf9 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.columnar; + +import java.nio.ByteBuffer; + +public class ColumnarKeyStoreDataHolder { + private byte[] keyblockData; + private ColumnarKeyStoreMetadata columnarKeyStoreMetadata; + + public ColumnarKeyStoreDataHolder(final byte[] keyblockData, + final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) { + this.keyblockData = keyblockData; + this.columnarKeyStoreMetadata = columnarKeyStoreMetadata; + } + + public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) { + this.columnarKeyStoreMetadata = columnarKeyStoreMetadata; + } + + public int getSurrogateKey(int columnIndex) { + byte[] actual = new byte[4]; + int startIndex; + if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) { + startIndex = + columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata + .getEachRowSize(); + } else { + startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize(); + } + int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize(); + System.arraycopy(keyblockData, startIndex, actual, destPos, + columnarKeyStoreMetadata.getEachRowSize()); + return ByteBuffer.wrap(actual).getInt(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java new file mode 100644 index 0000000..706708f --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.columnar; + +class ColumnarKeyStoreMetadata { + + private int[] columnReverseIndex; + + private int eachRowSize; + + ColumnarKeyStoreMetadata(int eachRowSize) { + this.eachRowSize = eachRowSize; + } + + /** + * @return the eachRowSize + */ + int getEachRowSize() { + return eachRowSize; + } + + /** + * @return the columnReverseIndex + */ + int[] getColumnReverseIndex() { + return columnReverseIndex; + } + + /** + * @param columnReverseIndex the columnReverseIndex to set + */ + void setColumnReverseIndex(int[] columnReverseIndex) { + this.columnReverseIndex = columnReverseIndex; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java new file mode 100644 index 0000000..ac8f6b7 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.columnar; + +public interface IndexStorage<T> { + boolean isAlreadySorted(); + + T getDataAfterComp(); + + T getIndexMap(); + + byte[][] getKeyBlock(); + + T getDataIndexMap(); + + int getTotalSize(); + + /** + * @return min value of block + */ + byte[] getMin(); + + /** + * @return max value of block + */ + byte[] getMax(); +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java new file mode 100644 index 0000000..4d4eac9 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.columnar; + +import java.util.Arrays; + +public final class UnBlockIndexer { + + private UnBlockIndexer() { + + } + + public static int[] uncompressIndex(int[] indexData, int[] indexMap) { + int actualSize = indexData.length; + for (int i = 0; i < indexMap.length; i++) { + actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1; + } + int[] indexes = new int[actualSize]; + int k = 0; + for (int i = 0; i < indexData.length; i++) { + int index = Arrays.binarySearch(indexMap, i); + if (index > -1) { + for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) { + indexes[k] = j; + k++; + } + i++; + } else { + indexes[k] = indexData[i]; + k++; + } + } + return indexes; + } + + public static byte[] uncompressData(byte[] data, int[] index, int keyLen) { + if (index.length < 1) { + return data; + } + int numberOfCopy = 0; + int actualSize = 0; + int srcPos = 0; + int destPos = 0; + for (int i = 1; i < index.length; i += 2) { + actualSize += index[i]; + } + byte[] uncompressedData = new byte[actualSize * keyLen]; + int picIndex = 0; + for (int i = 0; i < data.length; i += keyLen) { + numberOfCopy = index[picIndex * 2 + 1]; + picIndex++; + for (int j = 0; j < numberOfCopy; j++) { + System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen); + destPos += keyLen; + } + srcPos += keyLen; + } + return uncompressedData; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java new file mode 100644 index 0000000..d9803a0 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.compression; + +public interface Compressor { + + byte[] compressByte(byte[] unCompInput); + + byte[] unCompressByte(byte[] compInput); + + byte[] unCompressByte(byte[] compInput, int offset, int length); + + byte[] compressShort(short[] unCompInput); + + short[] unCompressShort(byte[] compInput); + + short[] unCompressShort(byte[] compInput, int offset, int lenght); + + byte[] compressInt(int[] unCompInput); + + int[] unCompressInt(byte[] compInput); + + int[] unCompressInt(byte[] compInput, int offset, int length); + + byte[] compressLong(long[] unCompInput); + + long[] unCompressLong(byte[] compInput); + + long[] unCompressLong(byte[] compInput, int offset, int length); + + byte[] compressFloat(float[] unCompInput); + + float[] unCompressFloat(byte[] compInput); + + float[] unCompressFloat(byte[] compInput, int offset, int length); + + byte[] compressDouble(double[] unCompInput); + + double[] unCompressDouble(byte[] compInput); + + double[] unCompressDouble(byte[] compInput, int offset, int length); + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java new file mode 100644 index 0000000..022ac97 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.compression; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.CarbonProperties; + +public class CompressorFactory { + + private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory(); + + private final Compressor compressor; + + private CompressorFactory() { + String compressorType = CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR); + switch (compressorType) { + case "snappy": + compressor = new SnappyCompressor(); + break; + default: + throw new RuntimeException( + "Invalid compressor type provided! Please provide valid compressor type"); + } + } + + public static CompressorFactory getInstance() { + return COMPRESSOR_FACTORY; + } + + public Compressor getCompressor() { + return compressor; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java new file mode 100644 index 0000000..1ded77d --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.compression; + +public class MeasureMetaDataModel { + /** + * maxValue + */ + private Object[] maxValue; + + /** + * minValue + */ + private Object[] minValue; + + /** + * mantissa + */ + private int[] mantissa; + + /** + * measureCount + */ + private int measureCount; + + /** + * uniqueValue + */ + private Object[] uniqueValue; + + /** + * type + */ + private char[] type; + + /** + * dataTypeSelected + */ + private byte[] dataTypeSelected; + + public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa, + int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) { + this.minValue = minValue; + this.maxValue = maxValue; + this.mantissa = mantissa; + this.measureCount = measureCount; + this.uniqueValue = uniqueValue; + this.type = type; + this.dataTypeSelected = dataTypeSelected; + } + + /** + * get Max value + * + * @return + */ + public Object[] getMaxValue() { + return maxValue; + } + + /** + * getMinValue + * + * @return + */ + public Object[] getMinValue() { + return minValue; + } + + /** + * getMantissa + * + * @return + */ + public int[] getMantissa() { + return mantissa; + } + + /** + * getMeasureCount + * + * @return + */ + public int getMeasureCount() { + return measureCount; + } + + /** + * getUniqueValue + * + * @return + */ + public Object[] getUniqueValue() { + return uniqueValue; + } + + /** + * @return the type + */ + public char[] getType() { + return type; + } + + /** + * @return the dataTypeSelected + */ + public byte[] getDataTypeSelected() { + return dataTypeSelected; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java new file mode 100644 index 0000000..7a5c670 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.compression; + +import org.apache.carbondata.core.metadata.ValueEncoderMeta; +import org.apache.carbondata.core.util.ValueCompressionUtil; + +// Used in read path for decompression preparation +public class ReaderCompressModel { + private ValueEncoderMeta valueEncoderMeta; + + private ValueCompressionUtil.DataType convertedDataType; + + private ValueCompressionHolder valueHolder; + + public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) { + this.valueEncoderMeta = valueEncoderMeta; + } + + public ValueCompressionUtil.DataType getConvertedDataType() { + return convertedDataType; + } + + public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) { + this.convertedDataType = convertedDataType; + } + + public Object getMaxValue() { + return valueEncoderMeta.getMaxValue(); + } + + public int getMantissa() { + return valueEncoderMeta.getMantissa(); + } + + public ValueCompressionHolder getValueCompressionHolder() { + return valueHolder; + } + + public void setValueCompressionHolder(ValueCompressionHolder valueHolder) { + this.valueHolder = valueHolder; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java new file mode 100644 index 0000000..dd78bb1 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.compression; + +import java.io.IOException; +import java.lang.reflect.Field; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; + +import org.xerial.snappy.Snappy; +import org.xerial.snappy.SnappyNative; + +public class SnappyCompressor implements Compressor { + + private static final LogService LOGGER = + LogServiceFactory.getLogService(SnappyCompressor.class.getName()); + + private final SnappyNative snappyNative; + + public SnappyCompressor() { + Snappy snappy = new Snappy(); + Field privateField = null; + try { + privateField = snappy.getClass().getDeclaredField("impl"); + } catch (NoSuchFieldException | SecurityException e) { + throw new RuntimeException(e); + } + privateField.setAccessible(true); + try { + snappyNative = (SnappyNative) privateField.get(snappy); + } catch (IllegalArgumentException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + @Override public byte[] compressByte(byte[] unCompInput) { + try { + return Snappy.rawCompress(unCompInput, unCompInput.length); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + return null; + } + } + + @Override public byte[] unCompressByte(byte[] compInput) { + try { + return Snappy.uncompress(compInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return compInput; + } + + @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) { + int uncompressedLength = 0; + byte[] data = null; + try { + uncompressedLength = Snappy.uncompressedLength(compInput, offset, length); + data = new byte[uncompressedLength]; + Snappy.uncompress(compInput, offset, length, data, 0); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return data; + } + + @Override public byte[] compressShort(short[] unCompInput) { + try { + return Snappy.compress(unCompInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + return null; + } + } + + @Override public short[] unCompressShort(byte[] compInput) { + try { + return Snappy.uncompressShortArray(compInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } + + @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) { + try { + return Snappy.uncompressShortArray(compInput, offset, lenght); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } + + @Override public byte[] compressInt(int[] unCompInput) { + try { + return Snappy.compress(unCompInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + return null; + } + } + + @Override public int[] unCompressInt(byte[] compInput) { + try { + return Snappy.uncompressIntArray(compInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } + + @Override public int[] unCompressInt(byte[] compInput, int offset, int length) { + try { + return Snappy.uncompressIntArray(compInput, offset, length); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } + + @Override public byte[] compressLong(long[] unCompInput) { + try { + return Snappy.compress(unCompInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + return null; + } + } + + @Override public long[] unCompressLong(byte[] compInput) { + try { + return Snappy.uncompressLongArray(compInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } + + @Override public long[] unCompressLong(byte[] compInput, int offset, int length) { + try { + return Snappy.uncompressLongArray(compInput, offset, length); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } + + @Override public byte[] compressFloat(float[] unCompInput) { + try { + return Snappy.compress(unCompInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + return null; + } + } + + @Override public float[] unCompressFloat(byte[] compInput) { + try { + return Snappy.uncompressFloatArray(compInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } + + @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) { + try { + return Snappy.uncompressFloatArray(compInput, offset, length); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } + + @Override public byte[] compressDouble(double[] unCompInput) { + try { + return Snappy.compress(unCompInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + return null; + } + } + + @Override public double[] unCompressDouble(byte[] compInput) { + try { + return Snappy.uncompressDoubleArray(compInput); + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } + + @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) { + try { + int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length); + double[] result = new double[uncompressedLength / 8]; + snappyNative.rawUncompress(compInput, offset, length, result, 0); + return result; + } catch (IOException e) { + LOGGER.error(e, e.getMessage()); + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java new file mode 100644 index 0000000..8fed2ad --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.datastore.compression; + +import java.math.BigDecimal; + +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + + + +/** + * ValueCompressionHolder is the base class for handling + * compression / decompression of the measure data chunk + */ +public abstract class ValueCompressionHolder<T> { + + /** + * compressedValue + */ + protected byte[] compressedValue; + + /** + * @param compressor the compressor used to decompress the data + * @param dataType data type of the data + * @param data compressed data + */ + public void unCompress(Compressor compressor, DataType dataType, byte[] data, + int offset, int length) { + switch (dataType) { + case DATA_BYTE: + setValue((T)compressor.unCompressByte(data, offset, length)); + break; + case DATA_SHORT: + setValue((T)compressor.unCompressShort(data, offset, length)); + break; + case DATA_INT: + setValue((T)compressor.unCompressInt(data, offset, length)); + break; + case DATA_LONG: + case DATA_BIGINT: + setValue((T)compressor.unCompressLong(data, offset, length)); + break; + case DATA_FLOAT: + setValue((T)compressor.unCompressFloat(data, offset, length)); + break; + default: + setValue((T)compressor.unCompressDouble(data, offset, length)); + break; + } + } + + /** + * @param compressor the compressor used to compress the data + * @param dataType data type of the data + * @param data original data + */ + public byte[] compress(Compressor compressor, DataType dataType, Object data) { + switch (dataType) { + case DATA_BYTE: + return compressor.compressByte((byte[])data); + case DATA_SHORT: + return compressor.compressShort((short[])data); + case DATA_INT: + return compressor.compressInt((int[])data); + case DATA_LONG: + case DATA_BIGINT: + return compressor.compressLong((long[])data); + case DATA_FLOAT: + return compressor.compressFloat((float[])data); + case DATA_DOUBLE: + default: + return compressor.compressDouble((double[])data); + } + } + + public abstract void setValue(T value); + + public abstract T getValue(); + + public abstract void setValueInBytes(byte[] value); + + public abstract void compress(); + + public abstract void uncompress(DataType dataType, byte[] compressData, int offset, + int length, int decimal, Object maxValueObject); + + public byte[] getCompressedData() { return compressedValue; } + + public abstract long getLongValue(int index); + + public abstract double getDoubleValue(int index); + + public abstract BigDecimal getBigDecimalValue(int index); + + public abstract void freeMemory(); + + +}
