http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java deleted file mode 100644 index 6629d31..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datastore.chunk.impl; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType; -import org.apache.carbondata.core.metadata.datatype.DataType; -import org.apache.carbondata.core.metadata.datatype.DataTypes; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; -import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; -import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; - -/** - * This class is gives access to fixed length dimension data chunk store - */ -public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk { - - /** - * Constructor - * - * @param dataChunk data chunk - * @param invertedIndex inverted index - * @param invertedIndexReverse reverse inverted index - * @param numberOfRows number of rows - * @param columnValueSize size of each column value - */ - public FixedLengthDimensionDataChunk(byte[] dataChunk, int[] invertedIndex, - int[] invertedIndexReverse, int numberOfRows, int columnValueSize) { - long totalSize = null != invertedIndex ? - dataChunk.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) : - dataChunk.length; - dataChunkStore = DimensionChunkStoreFactory.INSTANCE - .getDimensionChunkStore(columnValueSize, null != invertedIndex, numberOfRows, totalSize, - DimensionStoreType.FIXEDLENGTH); - dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunk); - } - - /** - * Below method will be used to fill the data based on offset and row id - * - * @param data data to filed - * @param offset offset from which data need to be filed - * @param index row id of the chunk - * @param keyStructureInfo define the structure of the key - * @return how many bytes was copied - */ - @Override public int fillChunkData(byte[] data, int offset, int index, - KeyStructureInfo keyStructureInfo) { - dataChunkStore.fillRow(index, data, offset); - return dataChunkStore.getColumnValueSize(); - } - - /** - * Converts to column dictionary integer value - * - * @param rowId - * @param columnIndex - * @param row - * @param restructuringInfo - * @return - */ - @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row, - KeyStructureInfo restructuringInfo) { - row[columnIndex] = dataChunkStore.getSurrogate(rowId); - return columnIndex + 1; - } - - /** - * Fill the data to vector - * - * @param vectorInfo - * @param column - * @param restructuringInfo - * @return next column index - */ - @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column, - KeyStructureInfo restructuringInfo) { - ColumnVectorInfo columnVectorInfo = vectorInfo[column]; - int offset = columnVectorInfo.offset; - int vectorOffset = columnVectorInfo.vectorOffset; - int len = columnVectorInfo.size + offset; - CarbonColumnVector vector = columnVectorInfo.vector; - for (int j = offset; j < len; j++) { - int dict = dataChunkStore.getSurrogate(j); - if (columnVectorInfo.directDictionaryGenerator == null) { - vector.putInt(vectorOffset++, dict); - } else { - Object valueFromSurrogate = - columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict); - if (valueFromSurrogate == null) { - vector.putNull(vectorOffset++); - } else { - DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType(); - if (dataType == DataTypes.INT) { - vector.putInt(vectorOffset++, (int) valueFromSurrogate); - } else if (dataType == DataTypes.LONG) { - vector.putLong(vectorOffset++, (long) valueFromSurrogate); - } else { - throw new IllegalArgumentException("unsupported data type: " + - columnVectorInfo.directDictionaryGenerator.getReturnType()); - } - } - } - } - return column + 1; - } - - /** - * Fill the data to vector - * - * @param rowMapping - * @param vectorInfo - * @param column - * @param restructuringInfo - * @return next column index - */ - @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, - int column, KeyStructureInfo restructuringInfo) { - ColumnVectorInfo columnVectorInfo = vectorInfo[column]; - int offset = columnVectorInfo.offset; - int vectorOffset = columnVectorInfo.vectorOffset; - int len = columnVectorInfo.size + offset; - CarbonColumnVector vector = columnVectorInfo.vector; - for (int j = offset; j < len; j++) { - int dict = dataChunkStore.getSurrogate(rowMapping[j]); - if (columnVectorInfo.directDictionaryGenerator == null) { - vector.putInt(vectorOffset++, dict); - } else { - Object valueFromSurrogate = - columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict); - if (valueFromSurrogate == null) { - vector.putNull(vectorOffset++); - } else { - DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType(); - if (dataType == DataTypes.INT) { - vector.putInt(vectorOffset++, (int) valueFromSurrogate); - } else if (dataType == DataTypes.LONG) { - vector.putLong(vectorOffset++, (long) valueFromSurrogate); - } else { - throw new IllegalArgumentException("unsupported data type: " + - columnVectorInfo.directDictionaryGenerator.getReturnType()); - } - } - } - } - return column + 1; - } -}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java index fa0777b..5e8618b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java @@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.chunk.impl; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -28,7 +28,7 @@ import org.apache.carbondata.core.memory.MemoryException; /** * Contains raw measure data * 1. The read uncompressed raw data of column chunk with all pages is stored in this instance. - * 2. The raw data can be converted to processed chunk using convertToColumnPage method + * 2. The raw data can be converted to processed chunk using decodeColumnPage method * by specifying page number. */ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { @@ -37,7 +37,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { private MeasureColumnChunkReader chunkReader; - private FileHolder fileReader; + private FileReader fileReader; public MeasureRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length, MeasureColumnChunkReader chunkReader) { @@ -48,14 +48,14 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { /** * Convert all raw data with all pages to processed ColumnPage */ - public ColumnPage[] convertToColumnPage() { + public ColumnPage[] decodeAllColumnPages() { if (columnPages == null) { columnPages = new ColumnPage[pagesCount]; } for (int i = 0; i < pagesCount; i++) { try { if (columnPages[i] == null) { - columnPages[i] = chunkReader.convertToColumnPage(this, i); + columnPages[i] = chunkReader.decodeColumnPage(this, i); } } catch (Exception e) { throw new RuntimeException(e); @@ -68,21 +68,21 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { /** * Convert raw data with specified `columnIndex` processed to ColumnPage */ - public ColumnPage convertToColumnPage(int columnIndex) { - assert columnIndex < pagesCount; + public ColumnPage decodeColumnPage(int pageNumber) { + assert pageNumber < pagesCount; if (columnPages == null) { columnPages = new ColumnPage[pagesCount]; } try { - if (columnPages[columnIndex] == null) { - columnPages[columnIndex] = chunkReader.convertToColumnPage(this, columnIndex); + if (columnPages[pageNumber] == null) { + columnPages[pageNumber] = chunkReader.decodeColumnPage(this, pageNumber); } } catch (IOException | MemoryException e) { throw new RuntimeException(e); } - return columnPages[columnIndex]; + return columnPages[pageNumber]; } /** @@ -95,7 +95,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { assert index < pagesCount; try { - return chunkReader.convertToColumnPage(this, index); + return chunkReader.decodeColumnPage(this, index); } catch (IOException | MemoryException e) { throw new RuntimeException(e); } @@ -111,11 +111,11 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk { } } - public void setFileReader(FileHolder fileReader) { + public void setFileReader(FileReader fileReader) { this.fileReader = fileReader; } - public FileHolder getFileReader() { + public FileReader getFileReader() { return fileReader; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java new file mode 100644 index 0000000..d03b2de --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.datastore.chunk.impl; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory; +import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType; +import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; +import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; +import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; + +/** + * This class is gives access to variable length dimension data chunk store + */ +public class VariableLengthDimensionColumnPage extends AbstractDimensionColumnPage { + + /** + * Constructor for this class + * @param dataChunks + * @param invertedIndex + * @param invertedIndexReverse + * @param numberOfRows + */ + public VariableLengthDimensionColumnPage(byte[] dataChunks, int[] invertedIndex, + int[] invertedIndexReverse, int numberOfRows) { + long totalSize = null != invertedIndex ? + (dataChunks.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) + ( + numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)) : + (dataChunks.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + dataChunkStore = DimensionChunkStoreFactory.INSTANCE + .getDimensionChunkStore(0, null != invertedIndex, numberOfRows, totalSize, + DimensionStoreType.VARIABLELENGTH); + dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunks); + } + + /** + * Below method will be used to fill the data based on offset and row id + * + * @param rowId row id of the chunk + * @param offset offset from which data need to be filed + * @param data data to filed + * @param restructuringInfo define the structure of the key + * @return how many bytes was copied + */ + @Override public int fillRawData(int rowId, int offset, byte[] data, + KeyStructureInfo restructuringInfo) { + // no required in this case because this column chunk is not the part if + // mdkey + return 0; + } + + /** + * Converts to column dictionary integer value + * + * @param rowId + * @param chunkIndex + * @param outputSurrogateKey + * @param restructuringInfo + * @return + */ + @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey, + KeyStructureInfo restructuringInfo) { + return chunkIndex + 1; + } + + /** + * @return whether column is dictionary column or not + */ + @Override public boolean isNoDicitionaryColumn() { + return true; + } + + /** + * Fill the data to vector + * + * @param vectorInfo + * @param chunkIndex + * @param restructuringInfo + * @return next column index + */ + @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex, + KeyStructureInfo restructuringInfo) { + ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex]; + CarbonColumnVector vector = columnVectorInfo.vector; + int offset = columnVectorInfo.offset; + int vectorOffset = columnVectorInfo.vectorOffset; + int len = offset + columnVectorInfo.size; + for (int i = offset; i < len; i++) { + // Considering only String case now as we support only + // string in no dictionary case at present. + dataChunkStore.fillRow(i, vector, vectorOffset++); + } + return chunkIndex + 1; + } + + /** + * Fill the data to vector + * + * @param filteredRowId + * @param vectorInfo + * @param chunkIndex + * @param restructuringInfo + * @return next column index + */ + @Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, + int chunkIndex, KeyStructureInfo restructuringInfo) { + ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex]; + CarbonColumnVector vector = columnVectorInfo.vector; + int offset = columnVectorInfo.offset; + int vectorOffset = columnVectorInfo.vectorOffset; + int len = offset + columnVectorInfo.size; + for (int i = offset; i < len; i++) { + // Considering only String case now as we support only + // string in no dictionary case at present. + dataChunkStore.fillRow(filteredRowId[i], vector, vectorOffset++); + } + return chunkIndex + 1; + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java deleted file mode 100644 index 6c47bf5..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.datastore.chunk.impl; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory; -import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType; -import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; -import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector; -import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; - -/** - * This class is gives access to variable length dimension data chunk store - */ -public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk { - - /** - * Constructor for this class - * @param dataChunks - * @param invertedIndex - * @param invertedIndexReverse - * @param numberOfRows - */ - public VariableLengthDimensionDataChunk(byte[] dataChunks, int[] invertedIndex, - int[] invertedIndexReverse, int numberOfRows) { - long totalSize = null != invertedIndex ? - (dataChunks.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) + ( - numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)) : - (dataChunks.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - dataChunkStore = DimensionChunkStoreFactory.INSTANCE - .getDimensionChunkStore(0, null != invertedIndex, numberOfRows, totalSize, - DimensionStoreType.VARIABLELENGTH); - dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunks); - } - - /** - * Below method will be used to fill the data based on offset and row id - * - * @param data data to filed - * @param offset offset from which data need to be filed - * @param index row id of the chunk - * @param restructuringInfo define the structure of the key - * @return how many bytes was copied - */ - @Override public int fillChunkData(byte[] data, int offset, int index, - KeyStructureInfo restructuringInfo) { - // no required in this case because this column chunk is not the part if - // mdkey - return 0; - } - - /** - * Converts to column dictionary integer value - * - * @param rowId - * @param columnIndex - * @param row - * @param restructuringInfo - * @return - */ - @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row, - KeyStructureInfo restructuringInfo) { - return columnIndex + 1; - } - - /** - * @return whether column is dictionary column or not - */ - @Override public boolean isNoDicitionaryColumn() { - return true; - } - - /** - * @return length of each column - */ - @Override public int getColumnValueSize() { - return -1; - } - - /** - * Fill the data to vector - * - * @param vectorInfo - * @param column - * @param restructuringInfo - * @return next column index - */ - @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column, - KeyStructureInfo restructuringInfo) { - ColumnVectorInfo columnVectorInfo = vectorInfo[column]; - CarbonColumnVector vector = columnVectorInfo.vector; - int offset = columnVectorInfo.offset; - int vectorOffset = columnVectorInfo.vectorOffset; - int len = offset + columnVectorInfo.size; - for (int i = offset; i < len; i++) { - // Considering only String case now as we support only - // string in no dictionary case at present. - dataChunkStore.fillRow(i, vector, vectorOffset++); - } - return column + 1; - } - - /** - * Fill the data to vector - * - * @param rowMapping - * @param vectorInfo - * @param column - * @param restructuringInfo - * @return next column index - */ - @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, - int column, KeyStructureInfo restructuringInfo) { - ColumnVectorInfo columnVectorInfo = vectorInfo[column]; - CarbonColumnVector vector = columnVectorInfo.vector; - int offset = columnVectorInfo.offset; - int vectorOffset = columnVectorInfo.vectorOffset; - int len = offset + columnVectorInfo.size; - for (int i = offset; i < len; i++) { - // Considering only String case now as we support only - // string in no dictionary case at present. - dataChunkStore.fillRow(rowMapping[i], vector, vectorOffset++); - } - return column + 1; - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java index 7b5b9c8..fd81973 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java @@ -18,8 +18,8 @@ package org.apache.carbondata.core.datastore.chunk.reader; import java.io.IOException; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.memory.MemoryException; @@ -34,20 +34,20 @@ public interface DimensionColumnChunkReader { * Below method will be used to read the chunk based on block indexes * * @param fileReader file reader to read the blocks from file - * @param blockletIndexes blocklets to be read + * @param columnIndexRange blocklets to be read * @return dimension column chunks */ - DimensionRawColumnChunk[] readRawDimensionChunks(FileHolder fileReader, int[][] blockletIndexes) + DimensionRawColumnChunk[] readRawDimensionChunks(FileReader fileReader, int[][] columnIndexRange) throws IOException; /** * Below method will be used to read the chunk based on block index * * @param fileReader file reader to read the blocks from file - * @param blockletIndex block to be read + * @param columnIndex column to be read * @return dimension column chunk */ - DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, int blockletIndex) + DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int columnIndex) throws IOException; /** @@ -58,6 +58,6 @@ public interface DimensionColumnChunkReader { * @return * @throws IOException */ - DimensionColumnDataChunk convertToDimensionChunk(DimensionRawColumnChunk dimensionRawColumnChunk, + DimensionColumnPage decodeColumnPage(DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException, MemoryException; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java index 02dc6a2..bf76025 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java @@ -18,7 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.reader; import java.io.IOException; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.page.ColumnPage; import org.apache.carbondata.core.memory.MemoryException; @@ -32,20 +32,20 @@ public interface MeasureColumnChunkReader { * Method to read the blocks data based on block indexes * * @param fileReader file reader to read the blocks - * @param blockIndexes blocks to be read + * @param columnIndexRange blocks to be read * @return measure data chunks */ - MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, int[][] blockIndexes) + MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader, int[][] columnIndexRange) throws IOException; /** * Method to read the blocks data based on block index * * @param fileReader file reader to read the blocks - * @param blockIndex block to be read + * @param columnIndex block to be read * @return measure data chunk */ - MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int blockIndex) + MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex) throws IOException; /** @@ -55,7 +55,7 @@ public interface MeasureColumnChunkReader { * @return * @throws IOException */ - ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk, + ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException, MemoryException; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java index f083612..60950c9 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java @@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension; import java.io.IOException; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.format.Encoding; @@ -55,43 +55,43 @@ public abstract class AbstractChunkReaderV2V3Format extends AbstractChunkReader * For last column read is separately and process * * @param fileReader file reader to read the blocks from file - * @param blockletIndexes blocks range to be read + * @param columnIndexRange column index range to be read * @return dimension column chunks */ - @Override public DimensionRawColumnChunk[] readRawDimensionChunks(final FileHolder fileReader, - final int[][] blockletIndexes) throws IOException { + @Override public DimensionRawColumnChunk[] readRawDimensionChunks(final FileReader fileReader, + final int[][] columnIndexRange) throws IOException { // read the column chunk based on block index and add DimensionRawColumnChunk[] dataChunks = new DimensionRawColumnChunk[dimensionChunksOffset.size()]; // if blocklet index is empty then return empry data chunk - if (blockletIndexes.length == 0) { + if (columnIndexRange.length == 0) { return dataChunks; } DimensionRawColumnChunk[] groupChunk = null; int index = 0; // iterate till block indexes -1 as block index will be in sorted order, so to avoid // the last column reading in group - for (int i = 0; i < blockletIndexes.length - 1; i++) { + for (int i = 0; i < columnIndexRange.length - 1; i++) { index = 0; groupChunk = - readRawDimensionChunksInGroup(fileReader, blockletIndexes[i][0], blockletIndexes[i][1]); - for (int j = blockletIndexes[i][0]; j <= blockletIndexes[i][1]; j++) { + readRawDimensionChunksInGroup(fileReader, columnIndexRange[i][0], columnIndexRange[i][1]); + for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) { dataChunks[j] = groupChunk[index++]; } } // check last index is present in block index, if it is present then read separately - if (blockletIndexes[blockletIndexes.length - 1][0] == dimensionChunksOffset.size() - 1) { - dataChunks[blockletIndexes[blockletIndexes.length - 1][0]] = - readRawDimensionChunk(fileReader, blockletIndexes[blockletIndexes.length - 1][0]); + if (columnIndexRange[columnIndexRange.length - 1][0] == dimensionChunksOffset.size() - 1) { + dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] = + readRawDimensionChunk(fileReader, columnIndexRange[columnIndexRange.length - 1][0]); } // otherwise read the data in group else { - groupChunk = - readRawDimensionChunksInGroup(fileReader, blockletIndexes[blockletIndexes.length - 1][0], - blockletIndexes[blockletIndexes.length - 1][1]); + groupChunk = readRawDimensionChunksInGroup( + fileReader, columnIndexRange[columnIndexRange.length - 1][0], + columnIndexRange[columnIndexRange.length - 1][1]); index = 0; - for (int j = blockletIndexes[blockletIndexes.length - 1][0]; - j <= blockletIndexes[blockletIndexes.length - 1][1]; j++) { + for (int j = columnIndexRange[columnIndexRange.length - 1][0]; + j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) { dataChunks[j] = groupChunk[index++]; } } @@ -109,7 +109,7 @@ public abstract class AbstractChunkReaderV2V3Format extends AbstractChunkReader * @return measure raw chunkArray * @throws IOException */ - protected abstract DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader, + protected abstract DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader, int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException; /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java index 27a4d89..0dc1c1b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java @@ -20,12 +20,12 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk; +import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader; import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; @@ -61,14 +61,14 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead * Below method will be used to read the raw chunk based on block indexes * * @param fileReader file reader to read the blocks from file - * @param blockletIndexes blocks to be read + * @param columnIndexRange blocks to be read * @return dimension column chunks */ - @Override public DimensionRawColumnChunk[] readRawDimensionChunks(FileHolder fileReader, - int[][] blockletIndexes) throws IOException { + @Override public DimensionRawColumnChunk[] readRawDimensionChunks(FileReader fileReader, + int[][] columnIndexRange) throws IOException { DimensionRawColumnChunk[] dataChunks = new DimensionRawColumnChunk[dimensionColumnChunk.size()]; - for (int i = 0; i < blockletIndexes.length; i++) { - for (int j = blockletIndexes[i][0]; j <= blockletIndexes[i][1]; j++) { + for (int i = 0; i < columnIndexRange.length; i++) { + for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) { dataChunks[j] = readRawDimensionChunk(fileReader, j); } } @@ -82,7 +82,7 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead * @param columnIndex column to be read * @return dimension column chunk */ - @Override public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, + @Override public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int columnIndex) throws IOException { DataChunk dataChunk = dimensionColumnChunk.get(columnIndex); ByteBuffer buffer = null; @@ -92,20 +92,20 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead } DimensionRawColumnChunk rawColumnChunk = new DimensionRawColumnChunk(columnIndex, buffer, 0, dataChunk.getDataPageLength(), this); - rawColumnChunk.setFileHolder(fileReader); + rawColumnChunk.setFileReader(fileReader); rawColumnChunk.setPagesCount(1); rawColumnChunk.setRowCount(new int[] { numberOfRows }); return rawColumnChunk; } - @Override public DimensionColumnDataChunk convertToDimensionChunk( + @Override public DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException { int blockIndex = dimensionRawColumnChunk.getColumnIndex(); byte[] dataPage = null; int[] invertedIndexes = null; int[] invertedIndexesReverse = null; int[] rlePage = null; - FileHolder fileReader = dimensionRawColumnChunk.getFileReader(); + FileReader fileReader = dimensionRawColumnChunk.getFileReader(); ByteBuffer rawData = dimensionRawColumnChunk.getRawData(); dataPage = COMPRESSOR.unCompressByte(rawData.array(), (int) dimensionRawColumnChunk.getOffSet(), @@ -145,23 +145,23 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead rlePage = null; } // fill chunk attributes - DimensionColumnDataChunk columnDataChunk = null; + DimensionColumnPage columnDataChunk = null; if (dataChunk.isRowMajor()) { // to store fixed length column chunk values - columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, eachColumnValueSize[blockIndex], - numberOfRows); + columnDataChunk = new ColumnGroupDimensionColumnPage( + dataPage, eachColumnValueSize[blockIndex], numberOfRows); } // if no dictionary column then first create a no dictionary column chunk // and set to data chunk instance else if (!CarbonUtil .hasEncoding(dataChunk.getEncodingList(), Encoding.DICTIONARY)) { columnDataChunk = - new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows); } else { // to store fixed length column chunk values columnDataChunk = - new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows, eachColumnValueSize[blockIndex]); } return columnDataChunk; http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java index b43f89c..31fa819 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java @@ -19,12 +19,12 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk; +import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format; import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; @@ -56,7 +56,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead * @param columnIndex column to be read * @return dimension column chunk */ - public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, int columnIndex) + public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int columnIndex) throws IOException { int length = 0; if (dimensionChunksOffset.size() - 1 == columnIndex) { @@ -73,7 +73,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead } DimensionRawColumnChunk rawColumnChunk = new DimensionRawColumnChunk(columnIndex, buffer, 0, length, this); - rawColumnChunk.setFileHolder(fileReader); + rawColumnChunk.setFileReader(fileReader); rawColumnChunk.setPagesCount(1); rawColumnChunk.setRowCount(new int[] { numberOfRows }); return rawColumnChunk; @@ -90,7 +90,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead * @return measure raw chunkArray * @throws IOException */ - protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader, + protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader, int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException { long currentDimensionOffset = dimensionChunksOffset.get(startColumnBlockletIndex); ByteBuffer buffer = null; @@ -106,7 +106,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead int currentLength = (int) (dimensionChunksOffset.get(i + 1) - dimensionChunksOffset.get(i)); dataChunks[index] = new DimensionRawColumnChunk(i, buffer, runningLength, currentLength, this); - dataChunks[index].setFileHolder(fileReader); + dataChunks[index].setFileReader(fileReader); dataChunks[index].setPagesCount(1); dataChunks[index].setRowCount(new int[] { numberOfRows }); runningLength += currentLength; @@ -115,7 +115,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead return dataChunks; } - public DimensionColumnDataChunk convertToDimensionChunk( + public DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException { byte[] dataPage = null; int[] invertedIndexes = null; @@ -169,23 +169,23 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead dataPage = UnBlockIndexer.uncompressData(dataPage, rlePage, eachColumnValueSize[blockIndex]); } // fill chunk attributes - DimensionColumnDataChunk columnDataChunk = null; + DimensionColumnPage columnDataChunk = null; if (dimensionColumnChunk.isRowMajor()) { // to store fixed length column chunk values - columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, eachColumnValueSize[blockIndex], - numberOfRows); + columnDataChunk = new ColumnGroupDimensionColumnPage( + dataPage, eachColumnValueSize[blockIndex], numberOfRows); } // if no dictionary column then first create a no dictionary column chunk // and set to data chunk instance else if (!hasEncoding(dimensionColumnChunk.encoders, Encoding.DICTIONARY)) { columnDataChunk = - new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows); } else { // to store fixed length column chunk values columnDataChunk = - new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows, eachColumnValueSize[blockIndex]); } return columnDataChunk; http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java index 1edfd09..60f0b67 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java @@ -20,8 +20,8 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; @@ -70,7 +70,8 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3 * @param blockletColumnIndex blocklet index of the column in carbon data file * @return dimension raw chunk */ - public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, + @Override + public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int blockletColumnIndex) throws IOException { // get the current dimension offset long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex); @@ -116,7 +117,7 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3 * @param endBlockletColumnIndex blocklet index of the last dimension column * @ DimensionRawColumnChunk array */ - protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader, + protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader, int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException { // create raw chunk for each dimension column DimensionRawColumnChunk[] dimensionDataChunks = @@ -136,7 +137,7 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3 * @param pageNumber number * @return DimensionColumnDataChunk */ - @Override public DimensionColumnDataChunk convertToDimensionChunk( + @Override public DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException, MemoryException { // data chunk of page http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java index 566e9b7..0fdc515 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java @@ -20,11 +20,11 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.FileReader; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; -import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; -import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk; +import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage; +import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage; import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format; import org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper; import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer; @@ -78,23 +78,23 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead * 5. Create the raw chunk object and fill the details * * @param fileReader reader for reading the column from carbon data file - * @param blockletColumnIndex blocklet index of the column in carbon data file + * @param columnIndex blocklet index of the column in carbon data file * @return dimension raw chunk */ - public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, - int blockletColumnIndex) throws IOException { + public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, + int columnIndex) throws IOException { // get the current dimension offset - long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex); + long currentDimensionOffset = dimensionChunksOffset.get(columnIndex); int length = 0; // to calculate the length of the data to be read // column other than last column we can subtract the offset of current column with // next column and get the total length. // but for last column we need to use lastDimensionOffset which is the end position // of the last dimension, we can subtract current dimension offset from lastDimesionOffset - if (dimensionChunksOffset.size() - 1 == blockletColumnIndex) { + if (dimensionChunksOffset.size() - 1 == columnIndex) { length = (int) (lastDimensionOffsets - currentDimensionOffset); } else { - length = (int) (dimensionChunksOffset.get(blockletColumnIndex + 1) - currentDimensionOffset); + length = (int) (dimensionChunksOffset.get(columnIndex + 1) - currentDimensionOffset); } ByteBuffer buffer = null; // read the data from carbon data file @@ -103,15 +103,15 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead } // get the data chunk which will have all the details about the data pages DataChunk3 dataChunk = CarbonUtil.readDataChunk3(buffer, 0, length); - return getDimensionRawColumnChunk(fileReader, blockletColumnIndex, 0, length, buffer, + return getDimensionRawColumnChunk(fileReader, columnIndex, 0, length, buffer, dataChunk); } - protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileHolder fileReader, - int blockletColumnIndex, long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) { + protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileReader fileReader, + int columnIndex, long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) { // creating a raw chunks instance and filling all the details DimensionRawColumnChunk rawColumnChunk = - new DimensionRawColumnChunk(blockletColumnIndex, buffer, offset, length, this); + new DimensionRawColumnChunk(columnIndex, buffer, offset, length, this); int numberOfPages = dataChunk.getPage_length().size(); byte[][] maxValueOfEachPage = new byte[numberOfPages][]; byte[][] minValueOfEachPage = new byte[numberOfPages][]; @@ -124,7 +124,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead eachPageLength[i] = dataChunk.getData_chunk_list().get(i).getNumberOfRowsInpage(); } rawColumnChunk.setDataChunkV3(dataChunk); - rawColumnChunk.setFileHolder(fileReader); + rawColumnChunk.setFileReader(fileReader); rawColumnChunk.setPagesCount(dataChunk.getPage_length().size()); rawColumnChunk.setMaxValues(maxValueOfEachPage); rawColumnChunk.setMinValues(minValueOfEachPage); @@ -153,7 +153,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead * blocklet index of the last dimension column * @ DimensionRawColumnChunk array */ - protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader, + protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader, int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException { // to calculate the length of the data to be read // column we can subtract the offset of start column offset with @@ -188,9 +188,9 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead * * @param rawColumnPage dimension raw chunk * @param pageNumber number - * @return DimensionColumnDataChunk + * @return DimensionColumnPage */ - @Override public DimensionColumnDataChunk convertToDimensionChunk( + @Override public DimensionColumnPage decodeColumnPage( DimensionRawColumnChunk rawColumnPage, int pageNumber) throws IOException, MemoryException { // data chunk of blocklet column DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3(); @@ -228,20 +228,19 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead return false; } - protected DimensionColumnDataChunk decodeDimension(DimensionRawColumnChunk rawColumnPage, + protected DimensionColumnPage decodeDimension(DimensionRawColumnChunk rawColumnPage, ByteBuffer pageData, DataChunk2 pageMetadata, int offset) throws IOException, MemoryException { if (isEncodedWithMeta(pageMetadata)) { ColumnPage decodedPage = decodeDimensionByMeta(pageMetadata, pageData, offset); - return new ColumnPageWrapper(decodedPage, - eachColumnValueSize[rawColumnPage.getColumnIndex()]); + return new ColumnPageWrapper(decodedPage); } else { // following code is for backward compatibility return decodeDimensionLegacy(rawColumnPage, pageData, pageMetadata, offset); } } - private DimensionColumnDataChunk decodeDimensionLegacy(DimensionRawColumnChunk rawColumnPage, + private DimensionColumnPage decodeDimensionLegacy(DimensionRawColumnChunk rawColumnPage, ByteBuffer pageData, DataChunk2 pageMetadata, int offset) { byte[] dataPage; int[] rlePage; @@ -267,18 +266,18 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead eachColumnValueSize[rawColumnPage.getColumnIndex()]); } - DimensionColumnDataChunk columnDataChunk = null; + DimensionColumnPage columnDataChunk = null; // if no dictionary column then first create a no dictionary column chunk // and set to data chunk instance if (!hasEncoding(pageMetadata.encoders, Encoding.DICTIONARY)) { columnDataChunk = - new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, pageMetadata.getNumberOfRowsInpage()); } else { // to store fixed length column chunk values columnDataChunk = - new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse, + new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, pageMetadata.getNumberOfRowsInpage(), eachColumnValueSize[rawColumnPage.getColumnIndex()]); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java index 2239a2b..a3ed339 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.BitSet; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.compression.Compressor; import org.apache.carbondata.core.datastore.compression.CompressorFactory; @@ -56,36 +56,40 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu * separately and process * * @param fileReader file reader to read the blocks from file - * @param blockIndexes blocks range to be read + * @param columnIndexRange blocks range to be read, columnIndexGroup[i] is one group, inside the + * group, columnIndexGroup[i][0] is start column index, + * and columnIndexGroup[i][1] is end column index * @return measure column chunks * @throws IOException */ - public MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, int[][] blockIndexes) - throws IOException { + public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader, + int[][] columnIndexRange) throws IOException { // read the column chunk based on block index and add MeasureRawColumnChunk[] dataChunks = new MeasureRawColumnChunk[measureColumnChunkOffsets.size()]; - if (blockIndexes.length == 0) { + if (columnIndexRange.length == 0) { return dataChunks; } MeasureRawColumnChunk[] groupChunk = null; int index = 0; - for (int i = 0; i < blockIndexes.length - 1; i++) { + for (int i = 0; i < columnIndexRange.length - 1; i++) { index = 0; - groupChunk = readRawMeasureChunksInGroup(fileReader, blockIndexes[i][0], blockIndexes[i][1]); - for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) { + groupChunk = readRawMeasureChunksInGroup( + fileReader, columnIndexRange[i][0], columnIndexRange[i][1]); + for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) { dataChunks[j] = groupChunk[index++]; } } - if (blockIndexes[blockIndexes.length - 1][0] == measureColumnChunkOffsets.size() - 1) { - dataChunks[blockIndexes[blockIndexes.length - 1][0]] = - readRawMeasureChunk(fileReader, blockIndexes[blockIndexes.length - 1][0]); + if (columnIndexRange[columnIndexRange.length - 1][0] == measureColumnChunkOffsets.size() - 1) { + dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] = + readRawMeasureChunk(fileReader, columnIndexRange[columnIndexRange.length - 1][0]); } else { - groupChunk = readRawMeasureChunksInGroup(fileReader, blockIndexes[blockIndexes.length - 1][0], - blockIndexes[blockIndexes.length - 1][1]); + groupChunk = readRawMeasureChunksInGroup( + fileReader, columnIndexRange[columnIndexRange.length - 1][0], + columnIndexRange[columnIndexRange.length - 1][1]); index = 0; - for (int j = blockIndexes[blockIndexes.length - 1][0]; - j <= blockIndexes[blockIndexes.length - 1][1]; j++) { + for (int j = columnIndexRange[columnIndexRange.length - 1][0]; + j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) { dataChunks[j] = groupChunk[index++]; } } @@ -112,12 +116,12 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu * data from * * @param fileReader file reader to read the data - * @param startColumnBlockletIndex first column blocklet index to be read - * @param endColumnBlockletIndex end column blocklet index to be read + * @param startColumnIndex first column index to be read + * @param endColumnIndex end column index to be read * @return measure raw chunkArray * @throws IOException */ - protected abstract MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader, - int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException; + protected abstract MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader, + int startColumnIndex, int endColumnIndex) throws IOException; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java index ae55375..f0c1b75 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -56,14 +56,14 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun * Method to read the blocks data based on block indexes * * @param fileReader file reader to read the blocks - * @param blockIndexes blocks to be read + * @param columnIndexRange blocks to be read * @return measure data chunks */ - @Override public MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, - int[][] blockIndexes) throws IOException { + @Override public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader, + int[][] columnIndexRange) throws IOException { MeasureRawColumnChunk[] datChunk = new MeasureRawColumnChunk[measureColumnChunks.size()]; - for (int i = 0; i < blockIndexes.length; i++) { - for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) { + for (int i = 0; i < columnIndexRange.length; i++) { + for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) { datChunk[j] = readRawMeasureChunk(fileReader, j); } } @@ -77,7 +77,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun * @param columnIndex column to be read * @return measure data chunk */ - @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int columnIndex) + @Override public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex) throws IOException { DataChunk dataChunk = measureColumnChunks.get(columnIndex); ByteBuffer buffer = fileReader @@ -91,7 +91,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun } @Override - public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk, + public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException, MemoryException { int blockIndex = measureRawColumnChunk.getColumnIndex(); DataChunk dataChunk = measureColumnChunks.get(blockIndex); http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java index d61f98a..04d6e2e 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -48,7 +48,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun } @Override - public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int columnIndex) + public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex) throws IOException { int dataLength = 0; if (measureColumnChunkOffsets.size() - 1 == columnIndex) { @@ -83,30 +83,30 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun * data from * * @param fileReader file reader to read the data - * @param startColumnBlockletIndex first column blocklet index to be read - * @param endColumnBlockletIndex end column blocklet index to be read + * @param startColumnIndex first column blocklet index to be read + * @param endColumnIndex end column blocklet index to be read * @return measure raw chunkArray * @throws IOException */ - protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader, - int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException { - long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnBlockletIndex); + protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader, + int startColumnIndex, int endColumnIndex) throws IOException { + long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex); ByteBuffer buffer = null; synchronized (fileReader) { buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset, - (int) (measureColumnChunkOffsets.get(endColumnBlockletIndex + 1) - currentMeasureOffset)); + (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset)); } MeasureRawColumnChunk[] dataChunks = - new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1]; + new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1]; int runningLength = 0; int index = 0; - for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) { + for (int i = startColumnIndex; i <= endColumnIndex; i++) { int currentLength = (int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i)); MeasureRawColumnChunk measureRawColumnChunk = new MeasureRawColumnChunk(i, buffer, runningLength, currentLength, this); - measureRawColumnChunk.setFileReader(fileReader); measureRawColumnChunk.setRowCount(new int[] { numberOfRows }); + measureRawColumnChunk.setFileReader(fileReader); measureRawColumnChunk.setPagesCount(1); dataChunks[index] = measureRawColumnChunk; runningLength += currentLength; @@ -115,7 +115,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun return dataChunks; } - public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk, + public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException, MemoryException { int copyPoint = (int) measureRawColumnChunk.getOffSet(); int blockIndex = measureRawColumnChunk.getColumnIndex(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java index 4f6987b..6dc02a3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format; import org.apache.carbondata.core.datastore.page.ColumnPage; @@ -70,7 +70,7 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun * @param columnIndex column to be read * @return measure raw chunk */ - @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, + @Override public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex) throws IOException { int dataLength = 0; // to calculate the length of the data to be read @@ -99,9 +99,8 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun dataChunk); } - protected MeasureRawColumnChunk getMeasureRawColumnChunk(FileHolder fileReader, - int columnIndex, long offset, int dataLength, ByteBuffer buffer, - DataChunk3 dataChunk) { + MeasureRawColumnChunk getMeasureRawColumnChunk(FileReader fileReader, int columnIndex, + long offset, int dataLength, ByteBuffer buffer, DataChunk3 dataChunk) { // creating a raw chunks instance and filling all the details MeasureRawColumnChunk rawColumnChunk = new MeasureRawColumnChunk(columnIndex, buffer, offset, dataLength, this); @@ -140,30 +139,30 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun * * @param fileReader * reader which will be used to read the measure columns data from file - * @param startColumnBlockletIndex - * blocklet index of the first measure column - * @param endColumnBlockletIndex - * blocklet index of the last measure column + * @param startColumnIndex + * column index of the first measure column + * @param endColumnIndex + * column index of the last measure column * @return MeasureRawColumnChunk array */ - protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader, - int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException { + protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader, + int startColumnIndex, int endColumnIndex) throws IOException { // to calculate the length of the data to be read // column we can subtract the offset of start column offset with // end column+1 offset and get the total length. - long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnBlockletIndex); + long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex); ByteBuffer buffer = null; // read the data from carbon data file synchronized (fileReader) { buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset, - (int) (measureColumnChunkOffsets.get(endColumnBlockletIndex + 1) - currentMeasureOffset)); + (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset)); } // create raw chunk for each measure column MeasureRawColumnChunk[] measureDataChunk = - new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1]; + new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1]; int runningLength = 0; int index = 0; - for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) { + for (int i = startColumnIndex; i <= endColumnIndex; i++) { int currentLength = (int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i)); DataChunk3 dataChunk = @@ -180,25 +179,25 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun /** * Below method will be used to convert the compressed measure chunk raw data to actual data * - * @param rawColumnPage measure raw chunk + * @param rawColumnChunk measure raw chunk * @param pageNumber number - * @return DimensionColumnDataChunk + * @return DimensionColumnPage */ @Override - public ColumnPage convertToColumnPage( - MeasureRawColumnChunk rawColumnPage, int pageNumber) + public ColumnPage decodeColumnPage( + MeasureRawColumnChunk rawColumnChunk, int pageNumber) throws IOException, MemoryException { // data chunk of blocklet column - DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3(); + DataChunk3 dataChunk3 = rawColumnChunk.getDataChunkV3(); // data chunk of page DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber); // calculating the start point of data // as buffer can contain multiple column data, start point will be datachunkoffset + // data chunk length + page offset - int offset = (int) rawColumnPage.getOffSet() + - measureColumnChunkLength.get(rawColumnPage.getColumnIndex()) + + int offset = (int) rawColumnChunk.getOffSet() + + measureColumnChunkLength.get(rawColumnChunk.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber); - ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnPage.getRawData(), offset); + ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnChunk.getRawData(), offset); decodedPage.setNullBits(getNullBitSet(pageMetadata.presence)); return decodedPage; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java index 31ff4c0..6b37575 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java @@ -20,7 +20,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.carbondata.core.datastore.FileHolder; +import org.apache.carbondata.core.datastore.FileReader; import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk; import org.apache.carbondata.core.datastore.page.ColumnPage; import org.apache.carbondata.core.memory.MemoryException; @@ -66,7 +66,7 @@ public class CompressedMsrChunkFileBasedPageLevelReaderV3 * @param blockletColumnIndex blocklet index of the column in carbon data file * @return measure raw chunk */ - @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, + @Override public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int blockletColumnIndex) throws IOException { int dataLength = 0; // to calculate the length of the data to be read @@ -110,7 +110,7 @@ public class CompressedMsrChunkFileBasedPageLevelReaderV3 * @param endColumnBlockletIndex blocklet index of the last measure column * @return MeasureRawColumnChunk array */ - protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader, + protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader, int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException { // create raw chunk for each measure column MeasureRawColumnChunk[] measureDataChunk = @@ -130,7 +130,7 @@ public class CompressedMsrChunkFileBasedPageLevelReaderV3 * @param pageNumber number * @return DimensionColumnDataChunk */ - @Override public ColumnPage convertToColumnPage( + @Override public ColumnPage decodeColumnPage( MeasureRawColumnChunk rawColumnPage, int pageNumber) throws IOException, MemoryException { // data chunk of blocklet column http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java index fbdb499..c89ecc3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java @@ -17,48 +17,45 @@ package org.apache.carbondata.core.datastore.chunk.store; -import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage; import org.apache.carbondata.core.datastore.page.ColumnPage; import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; -public class ColumnPageWrapper implements DimensionColumnDataChunk { +public class ColumnPageWrapper implements DimensionColumnPage { private ColumnPage columnPage; - private int columnValueSize; - public ColumnPageWrapper(ColumnPage columnPage, int columnValueSize) { + public ColumnPageWrapper(ColumnPage columnPage) { this.columnPage = columnPage; - this.columnValueSize = columnValueSize; } @Override - public int fillChunkData(byte[] data, int offset, int columnIndex, - KeyStructureInfo restructuringInfo) { + public int fillRawData(int rowId, int offset, byte[] data, KeyStructureInfo restructuringInfo) { throw new UnsupportedOperationException("internal error"); } @Override - public int fillConvertedChunkData(int rowId, int columnIndex, int[] row, + public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey, KeyStructureInfo restructuringInfo) { throw new UnsupportedOperationException("internal error"); } @Override - public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column, + public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex, KeyStructureInfo restructuringInfo) { throw new UnsupportedOperationException("internal error"); } @Override - public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, int column, + public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int chunkIndex, KeyStructureInfo restructuringInfo) { throw new UnsupportedOperationException("internal error"); } @Override - public byte[] getChunkData(int columnIndex) { - return columnPage.getBytes(columnIndex); + public byte[] getChunkData(int rowId) { + return columnPage.getBytes(rowId); } @Override @@ -66,7 +63,7 @@ public class ColumnPageWrapper implements DimensionColumnDataChunk { throw new UnsupportedOperationException("internal error"); } - @Override public int getInvertedReverseIndex(int invertedIndex) { + @Override public int getInvertedReverseIndex(int rowId) { throw new UnsupportedOperationException("internal error"); } @@ -76,17 +73,12 @@ public class ColumnPageWrapper implements DimensionColumnDataChunk { } @Override - public int getColumnValueSize() { - return columnValueSize; - } - - @Override public boolean isExplicitSorted() { return false; } @Override - public int compareTo(int index, byte[] compareValue) { + public int compareTo(int rowId, byte[] compareValue) { throw new UnsupportedOperationException("internal error"); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java index 5072c75..28aed5b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java @@ -69,10 +69,10 @@ public interface DimensionDataChunkStore { /** * Below method will be used to get the reverse Inverted Index - * @param invertedIndex + * @param rowId * @return reverse Inverted Index */ - int getInvertedReverseIndex(int invertedIndex); + int getInvertedReverseIndex(int rowId); /** * Below method will be used to get the surrogate key of the @@ -102,9 +102,9 @@ public interface DimensionDataChunkStore { /** * to compare the two byte array * - * @param index index of first byte array + * @param rowId index of first byte array * @param compareValue value of to be compared * @return compare result */ - int compareTo(int index, byte[] compareValue); + int compareTo(int rowId, byte[] compareValue); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java index 8656878..41218d0 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java @@ -97,13 +97,13 @@ public class SafeFixedLengthDimensionDataChunkStore extends SafeAbsractDimension /** * to compare the two byte array * - * @param index index of first byte array + * @param rowId index of first byte array * @param compareValue value of to be compared * @return compare result */ - @Override public int compareTo(int index, byte[] compareValue) { + @Override public int compareTo(int rowId, byte[] compareValue) { return ByteUtil.UnsafeComparer.INSTANCE - .compareTo(data, index * columnValueSize, columnValueSize, compareValue, 0, + .compareTo(data, rowId * columnValueSize, columnValueSize, compareValue, 0, columnValueSize); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java index db83198..f498c6e 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java @@ -162,7 +162,7 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens } } - @Override public int compareTo(int index, byte[] compareValue) { + @Override public int compareTo(int rowId, byte[] compareValue) { // now to get the row from memory block we need to do following thing // 1. first get the current offset // 2. if it's not a last row- get the next row offset @@ -171,11 +171,11 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens // length // get the offset of set of data - int currentDataOffset = dataOffsets[index]; + int currentDataOffset = dataOffsets[rowId]; short length = 0; // calculating the length of data - if (index < numberOfRows - 1) { - length = (short) (dataOffsets[index + 1] - (currentDataOffset + if (rowId < numberOfRows - 1) { + length = (short) (dataOffsets[rowId + 1] - (currentDataOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); } else { // for last record
