http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java index 49c9373..94b685c 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java @@ -18,6 +18,7 @@ */ package org.apache.carbondata.core.carbon.datastore.chunk.reader.measure.v2; +import java.io.IOException; import java.util.ArrayList; import java.util.BitSet; import java.util.List; @@ -92,7 +93,8 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun * @param blockIndexes blocks range to be read * @return measure column chunks */ - public MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader, int[][] blockIndexes) { + public MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader, int[][] blockIndexes) + throws IOException { // read the column chunk based on block index and add MeasureColumnDataChunk[] dataChunks = new MeasureColumnDataChunk[measureColumnChunkOffsets.size()]; @@ -130,7 +132,8 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun * @param blockIndex block to be read * @return measure data chunk */ - @Override public MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex) { + @Override public MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex) + throws IOException { MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk(); DataChunk2 measureColumnChunk = null; byte[] measureDataChunk = null; @@ -141,6 +144,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun .readByteArray(filePath, measureColumnChunkOffsets.get(blockIndex), measureColumnChunkLength.get(blockIndex)); measureColumnChunk = CarbonUtil.readDataChunk(measureDataChunk); + assert measureColumnChunk != null; dataPage = fileReader.readByteArray(filePath, measureColumnChunkOffsets.get(blockIndex) + measureColumnChunkLength.get(blockIndex), measureColumnChunk.data_page_length); @@ -151,6 +155,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun measureDataChunk = new byte[measureColumnChunkLength.get(blockIndex)]; System.arraycopy(data, 0, measureDataChunk, 0, measureColumnChunkLength.get(blockIndex)); measureColumnChunk = CarbonUtil.readDataChunk(measureDataChunk); + assert measureColumnChunk != null; dataPage = new byte[measureColumnChunk.data_page_length]; System.arraycopy(data, measureColumnChunkLength.get(blockIndex), dataPage, 0, measureColumnChunk.data_page_length); @@ -188,13 +193,13 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun * @return measure column chunk array */ private MeasureColumnDataChunk[] readMeasureChunksInGroup(FileHolder fileReader, - int startBlockIndex, int endBlockIndex) { + int startBlockIndex, int endBlockIndex) throws IOException { long currentMeasureOffset = measureColumnChunkOffsets.get(startBlockIndex); byte[] data = fileReader.readByteArray(filePath, currentMeasureOffset, (int) (measureColumnChunkOffsets.get(endBlockIndex + 1) - currentMeasureOffset)); MeasureColumnDataChunk[] dataChunks = new MeasureColumnDataChunk[endBlockIndex - startBlockIndex + 1]; - MeasureColumnDataChunk dataChunk = new MeasureColumnDataChunk(); + MeasureColumnDataChunk dataChunk = null; int index = 0; int copyPoint = 0; byte[] measureDataChunk = null; @@ -205,6 +210,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun measureDataChunk = new byte[measureColumnChunkLength.get(i)]; System.arraycopy(data, copyPoint, measureDataChunk, 0, measureColumnChunkLength.get(i)); measureColumnChunk = CarbonUtil.readDataChunk(measureDataChunk); + assert measureColumnChunk != null; dataPage = new byte[measureColumnChunk.data_page_length]; copyPoint += measureColumnChunkLength.get(i); System.arraycopy(data, copyPoint, dataPage, 0, measureColumnChunk.data_page_length);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java index a9c7343..7049d65 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java @@ -19,6 +19,7 @@ package org.apache.carbondata.core.carbon.datastore.exception; +import java.io.IOException; import java.util.Locale; /** @@ -26,7 +27,7 @@ import java.util.Locale; * * @author Administrator */ -public class IndexBuilderException extends Exception { +public class IndexBuilderException extends IOException { /** * default serial version ID. */ @@ -40,7 +41,6 @@ public class IndexBuilderException extends Exception { /** * Constructor * - * @param errorCode The error code for this exception. * @param msg The error message for this exception. */ public IndexBuilderException(String msg) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java index d414d3e..99ae8fe 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java @@ -151,8 +151,7 @@ public abstract class AbstractBTreeBuilder implements BtreeBuilder { buffer.get(dictionaryKey); byte[] nonDictionaryKey = new byte[nonDictonaryKeySize]; buffer.get(nonDictionaryKey); - IndexKey entry = new IndexKey(dictionaryKey, nonDictionaryKey); - return entry; + return new IndexKey(dictionaryKey, nonDictionaryKey); } /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java index bff3286..d889bce 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java @@ -18,6 +18,8 @@ */ package org.apache.carbondata.core.carbon.datastore.impl.btree; +import java.io.IOException; + import org.apache.carbondata.core.carbon.datastore.DataRefNode; import org.apache.carbondata.core.carbon.datastore.IndexKey; import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk; @@ -109,7 +111,6 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode { * This method will be used to get the max value of all the columns this can * be used in case of filter query * - * @param max value of all the columns */ @Override public byte[][] getColumnsMaxValue() { return maxKeyOfColumns; @@ -119,7 +120,6 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode { * This method will be used to get the max value of all the columns this can * be used in case of filter query * - * @param max value of all the columns */ @Override public byte[][] getColumnsMinValue() { return minKeyOfColumns; @@ -172,7 +172,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode { * @return dimension data chunks */ @Override public DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader, - int[][] blockIndexes) { + int[][] blockIndexes) throws IOException { // No required here as leaf which will will be use this class will implement its own get // dimension chunks return null; @@ -186,7 +186,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode { * @return dimension data chunk */ @Override public DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader, - int blockIndex) { + int blockIndex) throws IOException { // No required here as leaf which will will be use this class will implement // its own get dimension chunks return null; @@ -200,7 +200,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode { * @return measure column data chunk */ @Override public MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader, - int[][] blockIndexes) { + int[][] blockIndexes) throws IOException { // No required here as leaf which will will be use this class will implement its own get // measure chunks return null; @@ -213,7 +213,8 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode { * @param blockIndex block index to be read from file * @return measure data chunk */ - @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex) { + @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex) + throws IOException { // No required here as leaf which will will be use this class will implement its own get // measure chunks return null; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java index 31273ff..911b643 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java @@ -69,8 +69,6 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder { * Below method will be used to get the first tentative data block based on * search key * - * @param dataBlocks complete data blocks present - * @param serachKey key to be search * @return data block */ @Override public DataRefNode findFirstDataBlock(DataRefNode dataRefBlock, IndexKey searchKey) { @@ -86,8 +84,6 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder { * Below method will be used to get the last data tentative block based on * search key * - * @param dataBlocks complete data blocks present - * @param serachKey key to be search * @return data block */ @Override public DataRefNode findLastDataBlock(DataRefNode dataRefBlock, IndexKey searchKey) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java index c558763..5be4ed1 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java @@ -62,7 +62,6 @@ public class BTreeNonLeafNode implements BTreeNode { * as it is a non leaf node it will have the reference of all the leaf node * under it, setting all the children * - * @param leaf nodes */ @Override public void setChildren(BTreeNode[] children) { this.children = children; @@ -87,7 +86,6 @@ public class BTreeNonLeafNode implements BTreeNode { /** * add a key of a leaf node * - * @param leaf node start keys */ @Override public void setKey(IndexKey key) { listOfKeys.add(key); @@ -135,7 +133,6 @@ public class BTreeNonLeafNode implements BTreeNode { * This method will be used to get the max value of all the columns this can * be used in case of filter query * - * @param max value of all the columns */ @Override public byte[][] getColumnsMaxValue() { // operation of getting the max value is not supported as its a non leaf @@ -150,7 +147,6 @@ public class BTreeNonLeafNode implements BTreeNode { * This method will be used to get the max value of all the columns this can * be used in case of filter query * - * @param min value of all the columns */ @Override public byte[][] getColumnsMinValue() { // operation of getting the min value is not supported as its a non leaf @@ -183,7 +179,6 @@ public class BTreeNonLeafNode implements BTreeNode { * Below method will be used to get the dimension chunk * * @param fileReader file reader to read the chunk from file - * @param blockIndex block index to be read * @return dimension data chunk */ @Override public DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader, http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java index 34468a6..d76ce4c 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java @@ -38,7 +38,6 @@ public class BlockBTreeLeafNode extends AbstractBTreeLeafNode { * * @param builderInfos builder infos which have required metadata to create a leaf * node - * @param leafIndex leaf node index * @param metadataIndex metadata index */ public BlockBTreeLeafNode(BTreeBuilderInfo builderInfos, int metadataIndex, long nodeNumber) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java index 79ee008..e98eae1 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java @@ -18,6 +18,8 @@ */ package org.apache.carbondata.core.carbon.datastore.impl.btree; +import java.io.IOException; + import org.apache.carbondata.core.carbon.datastore.BTreeBuilderInfo; import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk; import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk; @@ -85,7 +87,7 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode { * @return dimension data chunks */ @Override public DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader, - int[][] blockIndexes) { + int[][] blockIndexes) throws IOException { return dimensionChunksReader.readDimensionChunks(fileReader, blockIndexes); } @@ -97,7 +99,7 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode { * @return dimension data chunk */ @Override public DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader, - int blockIndex) { + int blockIndex) throws IOException { return dimensionChunksReader.readDimensionChunk(fileReader, blockIndex); } @@ -109,7 +111,7 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode { * @return measure column data chunk */ @Override public MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader, - int[][] blockIndexes) { + int[][] blockIndexes) throws IOException { return measureColumnChunkReader.readMeasureChunks(fileReader, blockIndexes); } @@ -120,7 +122,8 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode { * @param blockIndex block index to be read from file * @return measure data chunk */ - @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex) { + @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex) + throws IOException { return measureColumnChunkReader.readMeasureChunk(fileReader, blockIndex); } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java index 4f1ae37..e977aca 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java @@ -32,61 +32,5 @@ public class ChunkCompressorMeta implements Serializable { */ private static final long serialVersionUID = -6697087170420991140L; - /** - * data chunk compressor - */ - private CompressionCodec compressor; - - /** - * total byte size of all uncompressed pages in this column chunk (including the headers) - */ - private long uncompressedSize; - - /** - * total byte size of all compressed pages in this column chunk (including the headers) - */ - private long compressedSize; - - /** - * @return the compressor - */ - public CompressionCodec getCompressorCodec() { - return compressor; - } - - /** - * @param compressor the compressor to set - */ - public void setCompressor(CompressionCodec compressor) { - this.compressor = compressor; - } - - /** - * @return the uncompressedSize - */ - public long getUncompressedSize() { - return uncompressedSize; - } - - /** - * @param uncompressedSize the uncompressedSize to set - */ - public void setUncompressedSize(long uncompressedSize) { - this.uncompressedSize = uncompressedSize; - } - - /** - * @return the compressedSize - */ - public long getCompressedSize() { - return compressedSize; - } - - /** - * @param compressedSize the compressedSize to set - */ - public void setCompressedSize(long compressedSize) { - this.compressedSize = compressedSize; - } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/CompressionCodec.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/CompressionCodec.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/CompressionCodec.java deleted file mode 100644 index 6f302ec..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/CompressionCodec.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.carbon.metadata.blocklet.compressor; - -/** - * Compressions supported by Carbon Data. - */ -public enum CompressionCodec { - - /** - * snappy compression - */ - SNAPPY, -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java index 584e51f..f301fc6 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java @@ -21,8 +21,6 @@ package org.apache.carbondata.core.carbon.metadata.blocklet.datachunk; import java.io.Serializable; import java.util.List; -import org.apache.carbondata.core.carbon.metadata.blocklet.compressor.ChunkCompressorMeta; -import org.apache.carbondata.core.carbon.metadata.blocklet.sort.SortState; import org.apache.carbondata.core.carbon.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.ValueEncoderMeta; @@ -37,23 +35,11 @@ public class DataChunk implements Serializable { private static final long serialVersionUID = 1L; /** - * the compression meta data of a chunk - */ - private ChunkCompressorMeta chunkCompressionMeta; - - /** * whether this chunk is a row chunk or column chunk */ private boolean isRowMajor; /** - * the column IDs in this chunk, will have atleast - * one column ID for columnar format, many column ID for - * row major format - */ - private List<Integer> columnUniqueIdList; - - /** * Offset of data page */ private long dataPageOffset; @@ -89,21 +75,6 @@ public class DataChunk implements Serializable { private int rlePageLength; /** - * is rle is applied in the data chunk - */ - private boolean isRleApplied; - - /** - * is dictionary is applied in the column, only if it is a dimension column - */ - private boolean isNoDictonaryColumn; - - /** - * sorting type selected for chunk; - */ - private SortState sortState; - - /** * The List of encoders overriden at node level */ private List<Encoding> encodingList; @@ -115,20 +86,6 @@ public class DataChunk implements Serializable { private List<ValueEncoderMeta> valueEncoderMetaList; /** - * @return the chunkCompressionMeta - */ - public ChunkCompressorMeta getChunkCompressionMeta() { - return chunkCompressionMeta; - } - - /** - * @param chunkCompressionMeta the chunkCompressionMeta to set - */ - public void setChunkCompressionMeta(ChunkCompressorMeta chunkCompressionMeta) { - this.chunkCompressionMeta = chunkCompressionMeta; - } - - /** * @return the isRowMajor */ public boolean isRowMajor() { @@ -143,20 +100,6 @@ public class DataChunk implements Serializable { } /** - * @return the columnUniqueIdList - */ - public List<Integer> getColumnUniqueIdList() { - return columnUniqueIdList; - } - - /** - * @param columnUniqueIdList the columnUniqueIdList to set - */ - public void setColumnUniqueIdList(List<Integer> columnUniqueIdList) { - this.columnUniqueIdList = columnUniqueIdList; - } - - /** * @return the dataPageOffset */ public long getDataPageOffset() { @@ -255,48 +198,6 @@ public class DataChunk implements Serializable { } /** - * @return the isRleApplied - */ - public boolean isRleApplied() { - return isRleApplied; - } - - /** - * @param isRleApplied the isRleApplied to set - */ - public void setRleApplied(boolean isRleApplied) { - this.isRleApplied = isRleApplied; - } - - /** - * @return the isNoDictonaryColumn - */ - public boolean isNoDictonaryColumn() { - return isNoDictonaryColumn; - } - - /** - * @param isNoDictonaryColumn the isNoDictonaryColumn to set - */ - public void setNoDictonaryColumn(boolean isNoDictonaryColumn) { - this.isNoDictonaryColumn = isNoDictonaryColumn; - } - - /** - * @return the sortState - */ - public SortState getSortState() { - return sortState; - } - - /** - * @param sortState the sortState to set - */ - public void setSortState(SortState sortState) { - this.sortState = sortState; - } - - /** * @return the encoderList */ public List<Encoding> getEncodingList() { @@ -304,9 +205,9 @@ public class DataChunk implements Serializable { } /** - * @param encoderList the encoderList to set + * @param encodingList the encoderList to set */ - public void setEncoderList(List<Encoding> encodingList) { + public void setEncodingList(List<Encoding> encodingList) { this.encodingList = encodingList; } @@ -318,7 +219,7 @@ public class DataChunk implements Serializable { } /** - * @param valueEncoderMeta the valueEncoderMeta to set + * @param valueEncoderMetaList the valueEncoderMeta to set */ public void setValueEncoderMeta(List<ValueEncoderMeta> valueEncoderMetaList) { this.valueEncoderMetaList = valueEncoderMetaList; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/sort/SortState.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/sort/SortState.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/sort/SortState.java deleted file mode 100644 index 122198c..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/sort/SortState.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.carbon.metadata.blocklet.sort; - -/** - * Enum for sort type information - */ -public enum SortState { - - /** - * column is not sorted - */ - SORT_NONE, - - /** - * data from source was already in sorted order - */ - SORT_NATIVE, - - /** - * data from source was not sorted,so data is explicitly sorted - */ - SORT_EXPLICT; -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java index 4b2be5f..ce50c87 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java @@ -356,8 +356,6 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id()); wrapperColumnSchema.setScale(externalColumnSchema.getScale()); wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value()); - wrapperColumnSchema.setAggregateFunction(externalColumnSchema.getAggregate_function()); - wrapperColumnSchema.setColumnProperties(externalColumnSchema.getColumnProperties()); wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible()); wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId()); wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal()); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java index 416f8e7..5a46502 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java @@ -27,5 +27,5 @@ public enum Encoding { RLE, INVERTED_INDEX, BIT_PACKED, - DIRECT_DICTIONARY; + DIRECT_DICTIONARY } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java index f756ab1..eacacc1 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java @@ -531,10 +531,6 @@ public class CarbonTable implements Serializable { return blockSize; } - public void setBlockSizeInMB(int blockSize) { - this.blockSize = blockSize; - } - /** * to get the normal dimension or the primitive dimension of the complex type * http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java index 888b898..83013b4 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java @@ -116,34 +116,6 @@ public class TableInfo implements Serializable { this.databaseName = databaseName; } - public TableSchema getTableSchemaByName(String tableName) { - if (factTable.getTableName().equalsIgnoreCase(tableName)) { - return factTable; - } - for (TableSchema aggregatTableSchema : aggregateTableList) { - if (aggregatTableSchema.getTableName().equals(tableName)) { - return aggregatTableSchema; - } - } - return null; - } - - public TableSchema getTableSchemaByTableId(String tableId) { - if (factTable.getTableId().equals(tableId)) { - return factTable; - } - for (TableSchema aggregatTableSchema : aggregateTableList) { - if (aggregatTableSchema.getTableId().equals(tableId)) { - return aggregatTableSchema; - } - } - return null; - } - - public int getNumberOfAggregateTables() { - return aggregateTableList.size(); - } - /** * @return the tableUniqueName */ http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java index 9beeff2..8db78a8 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java @@ -128,36 +128,6 @@ public class TableSchema implements Serializable { this.tableName = tableName; } - /** - * to get the column schema present in the table by name - * - * @param columnName - * @return column schema if matches the name - */ - public ColumnSchema getColumnSchemaByName(String columnName) { - for (ColumnSchema tableColumn : listOfColumns) { - if (tableColumn.getColumnName().equals(columnName)) { - return tableColumn; - } - } - return null; - } - - /** - * to get the column schema present in the table by unique id - * - * @param columnUniqueId - * @return column schema if matches the id - */ - public ColumnSchema getColumnSchemaById(String columnUniqueId) { - for (ColumnSchema tableColumn : listOfColumns) { - if (tableColumn.getColumnUniqueId().equalsIgnoreCase(columnUniqueId)) { - return tableColumn; - } - } - return null; - } - @Override public int hashCode() { final int prime = 31; int result = 1; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java index ae0a25a..edf9527 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java @@ -30,11 +30,6 @@ public class CarbonMeasure extends CarbonColumn { private static final long serialVersionUID = 354341488059013977L; /** - * aggregator chosen for measure - */ - private String aggregateFunction; - - /** * Used when this column contains decimal data. */ private int scale; @@ -69,20 +64,6 @@ public class CarbonMeasure extends CarbonColumn { } /** - * @return the aggregator - */ - public String getAggregateFunction() { - return aggregateFunction; - } - - /** - * @param aggregateFunction the aggregateFunction to set - */ - public void setAggregateFunction(String aggregateFunction) { - this.aggregateFunction = aggregateFunction; - } - - /** * to check whether to dimension are equal or not */ @Override public boolean equals(Object obj) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java index c0b2569..ac1caca 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java @@ -102,11 +102,6 @@ public class ColumnSchema implements Serializable { private int numberOfChild; /** - * Used when this column is part of an aggregate function. - */ - private String aggregateFunction; - - /** * used in case of schema restructuring */ private byte[] defaultValue; @@ -199,7 +194,6 @@ public class ColumnSchema implements Serializable { } /** - * @param columnGroup the columnGroup to set */ public void setColumnGroup(int columnGroupId) { this.columnGroupId = columnGroupId; @@ -248,20 +242,6 @@ public class ColumnSchema implements Serializable { } /** - * @return the aggregator - */ - public String getAggregateFunction() { - return aggregateFunction; - } - - /** - * @param aggregateFunction the aggregator to set - */ - public void setAggregateFunction(String aggregateFunction) { - this.aggregateFunction = aggregateFunction; - } - - /** * @return the defaultValue */ public byte[] getDefaultValue() { @@ -332,7 +312,6 @@ public class ColumnSchema implements Serializable { } /** - * @param encoderList the encoderList to set */ public void setEncodingList(List<Encoding> encodingList) { this.encodingList = encodingList; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java index 8c9f297..8cf6edb 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java @@ -42,7 +42,6 @@ public class CarbonTablePath extends Path { protected static final String SCHEMA_FILE = "schema"; protected static final String TABLE_STATUS_FILE = "tablestatus"; protected static final String FACT_DIR = "Fact"; - protected static final String AGGREGATE_TABLE_PREFIX = "Agg"; protected static final String SEGMENT_PREFIX = "Segment_"; protected static final String PARTITION_PREFIX = "Part"; protected static final String CARBON_DATA_EXT = ".carbondata"; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java index 55b878b..4d64242 100644 --- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java +++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java @@ -30,7 +30,7 @@ public class QueryStatisticsRecorderDummy implements QueryStatisticsRecorder,Ser */ private static final long serialVersionUID = -5719752001674467864L; - public QueryStatisticsRecorderDummy(String queryId) { + public QueryStatisticsRecorderDummy() { } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java index 909369d..383a515 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java @@ -67,7 +67,7 @@ public class DoubleCompressor extends ValueCompressor { for (int j = 0; j < value.length; j++) { BigDecimal val = BigDecimal.valueOf(value[j]); double diff = max.subtract(val).doubleValue(); - longResult[i] = (long) (Math.round(diff * Math.pow(10, decimal))); + longResult[i] = Math.round(diff * Math.pow(10, decimal)); i++; } return longResult; @@ -122,7 +122,7 @@ public class DoubleCompressor extends ValueCompressor { case DATA_LONG: long[] longResult = new long[value.length]; for (int j = 0; j < value.length; j++) { - longResult[i] = (long) (Math.round(Math.pow(10, decimal) * value[j])); + longResult[i] = Math.round(Math.pow(10, decimal) * value[j]); i++; } return longResult; @@ -188,7 +188,7 @@ public class DoubleCompressor extends ValueCompressor { default: double[] defaultResult = new double[value.length]; for (int j = 0; j < value.length; j++) { - defaultResult[i] = (double) (maxValue - value[j]); + defaultResult[i] = maxValue - value[j]; i++; } return defaultResult; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 6c159e6..32339ff 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -778,16 +778,6 @@ public final class CarbonCommonConstants { public static final String SEGMENT_COMPACTED = "Compacted"; /** - * property for number of core to load the blocks in driver - */ - public static final String NUMBER_OF_CORE_TO_LOAD_DRIVER_SEGMENT = - "no.of.cores.to.load.blocks.in.driver"; - /** - * default number of cores - */ - public static final int NUMBER_OF_CORE_TO_LOAD_DRIVER_SEGMENT_DEFAULT_VALUE = 10; - - /** * ZOOKEEPERLOCK TYPE */ public static final String CARBON_LOCK_TYPE_ZOOKEEPER = @@ -833,11 +823,6 @@ public final class CarbonCommonConstants { public static final String HIVE_CONNECTION_URL = "javax.jdo.option.ConnectionURL"; /** - * Rocord size in case of compaction. - */ - public static final int COMPACTION_INMEMORY_RECORD_SIZE = 120000; - - /** * If the level 2 compaction is done in minor then new compacted segment will end with .2 */ public static String LEVEL2_COMPACTION_INDEX = ".2"; @@ -858,17 +843,6 @@ public final class CarbonCommonConstants { public static final String ZOOKEEPER_URL = "spark.deploy.zookeeper.url"; /** - * configure the minimum blocklet size eligible for blocklet distribution - */ - public static final String CARBON_BLOCKLETDISTRIBUTION_MIN_REQUIRED_SIZE = - "carbon.blockletdistribution.min.blocklet.size"; - - /** - * default blocklet size eligible for blocklet distribution - */ - public static final int DEFAULT_CARBON_BLOCKLETDISTRIBUTION_MIN_REQUIRED_SIZE = 2; - - /** * File created in case of minor compaction request */ public static String minorCompactionRequiredFile = "compactionRequired_minor"; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java index 9f8d8ed..1f99158 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java @@ -19,6 +19,7 @@ package org.apache.carbondata.core.datastorage.store; +import java.io.IOException; public interface FileHolder { /** @@ -30,7 +31,7 @@ public interface FileHolder { * @param length number of bytes to be read * @return read byte array */ - byte[] readByteArray(String filePath, long offset, int length); + byte[] readByteArray(String filePath, long offset, int length) throws IOException; /** * This method will be used to read the byte array from file based on length(number of bytes) @@ -39,7 +40,7 @@ public interface FileHolder { * @param length number of bytes to be read * @return read byte array */ - byte[] readByteArray(String filePath, int length); + byte[] readByteArray(String filePath, int length) throws IOException; /** * This method will be used to read int from file from postion(offset), here @@ -49,7 +50,7 @@ public interface FileHolder { * @param offset reading start position, * @return read int */ - int readInt(String filePath, long offset); + int readInt(String filePath, long offset) throws IOException; /** * This method will be used to read long from file from postion(offset), here @@ -59,7 +60,7 @@ public interface FileHolder { * @param offset reading start position, * @return read long */ - long readLong(String filePath, long offset); + long readLong(String filePath, long offset) throws IOException; /** * This method will be used to read int from file from postion(offset), here @@ -68,7 +69,7 @@ public interface FileHolder { * @param filePath fully qualified file path * @return read int */ - int readInt(String filePath); + int readInt(String filePath) throws IOException; /** * This method will be used to read long value from file from postion(offset), here @@ -78,10 +79,10 @@ public interface FileHolder { * @param offset reading start position, * @return read long */ - long readDouble(String filePath, long offset); + long readDouble(String filePath, long offset) throws IOException; /** * This method will be used to close all the streams currently present in the cache */ - void finish(); + void finish() throws IOException; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java deleted file mode 100644 index 42c5071..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store; - -public interface NodeKeyStore { - /** - * This method will be used to get the actual mdkeys array present in the - * store store - * - * @param fileHolder - * @return mdkey - */ - byte[] getBackArray(FileHolder fileHolder); - - /** - * This method will be used to insert mdkey to store - * - * @param index index of mdkey - * @param value mdkey - */ - void put(int index, byte[] value); - - /** - * This method will be used to get the writable key array. - * writable key array will hold below information: - * <size of key array><key array> - * total length will be 4 bytes for size + key array length - * - * @return writable array (compressed or normal) - */ - byte[] getWritableKeyArray(); - - /** - * This method will be used to get the mdkkey array based on index - * - * @param index index in store - * @param fileHolder file holder will be used to read the file - * @return mdkey - */ - byte[] get(int index, FileHolder fileHolder); - - /** - * This method will clear the store and create the new empty store - */ - void clear(); - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java index f2fe9d2..a174a18 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java @@ -32,10 +32,4 @@ public interface NodeMeasureDataStore { */ byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray); - MeasureDataWrapper getBackData(int[] cols, FileHolder fileHolder); - - MeasureDataWrapper getBackData(int cols, FileHolder fileHolder); - - short getLength(); - } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java deleted file mode 100644 index 9e12847..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -import org.apache.carbondata.core.datastorage.store.FileHolder; - -public interface ColumnarKeyStore { - /** - * This method will be used to get the actual mdkeys array present in the - * carbon store, it will read and uncomnpress the key - * - * @param fileHolder - * @return mdkey - * @noDictionaryValKeyIndexes, directkey indexes for determining the NO_DICTIONARY - * Col inorder to process the direct surrogates data. - */ - ColumnarKeyStoreDataHolder[] getUnCompressedKeyArray(FileHolder fileHolder, int[] blockIndex, - boolean[] needCompressedData, int[] noDictionaryValKeyIndexes); - - /** - * This method will be used to get the actual mdkeys array present in the - * carbon store, it will read and uncomnpress the key - * - * @param fileHolder - * @return mdkey - */ - ColumnarKeyStoreDataHolder getUnCompressedKeyArray(FileHolder fileHolder, int blockIndex, - boolean needCompressedData, int[] noDictionaryValKeyIndexes); - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java index b66d957..29887a3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java @@ -20,11 +20,9 @@ package org.apache.carbondata.core.datastorage.store.columnar; import java.nio.ByteBuffer; -import java.util.List; public class ColumnarKeyStoreDataHolder { private byte[] keyblockData; - private List<byte[]> noDictionaryValBasedKeyBlockData; private ColumnarKeyStoreMetadata columnarKeyStoreMetadata; public ColumnarKeyStoreDataHolder(final byte[] keyblockData, @@ -33,34 +31,10 @@ public class ColumnarKeyStoreDataHolder { this.columnarKeyStoreMetadata = columnarKeyStoreMetadata; } - //Added constructor for holding noDictionaryValBasedKeyBlockData - public ColumnarKeyStoreDataHolder(final List<byte[]> noDictionaryValBasedKeyBlockData, - final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) { - this.noDictionaryValBasedKeyBlockData = noDictionaryValBasedKeyBlockData; + public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) { this.columnarKeyStoreMetadata = columnarKeyStoreMetadata; } - public byte[] getKeyBlockData() { - return keyblockData; - } - - /** - * @return the columnarKeyStoreMetadata - */ - public ColumnarKeyStoreMetadata getColumnarKeyStoreMetadata() { - return columnarKeyStoreMetadata; - } - - public void unCompress() { - if (columnarKeyStoreMetadata.isUnCompressed()) { - return; - } - this.keyblockData = UnBlockIndexer - .uncompressData(keyblockData, columnarKeyStoreMetadata.getDataIndex(), - columnarKeyStoreMetadata.getEachRowSize()); - columnarKeyStoreMetadata.setUnCompressed(true); - } - public int getSurrogateKey(int columnIndex) { byte[] actual = new byte[4]; int startIndex; @@ -77,21 +51,4 @@ public class ColumnarKeyStoreDataHolder { return ByteBuffer.wrap(actual).getInt(); } - /** - * get the byte[] for high cardinality column block - * - * @return List<byte[]>. - */ - public List<byte[]> getNoDictionaryValBasedKeyBlockData() { - return noDictionaryValBasedKeyBlockData; - } - - /** - * set the byte[] for high cardinality column block - * - * @param noDictionaryValBasedKeyBlockData - */ - public void setNoDictionaryValBasedKeyBlockData(List<byte[]> noDictionaryValBasedKeyBlockData) { - this.noDictionaryValBasedKeyBlockData = noDictionaryValBasedKeyBlockData; - } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java deleted file mode 100644 index 8f0621e..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor; - -public class ColumnarKeyStoreInfo { - private int numberOfKeys; - - private int[] sizeOfEachBlock; - - private int[] keyBlockLengths; - - private long[] keyBlockOffsets; - - private int[] keyBlockIndexLength; - - private long[] keyBlockIndexOffsets; - - private String filePath; - - private boolean[] isSorted; - - private int[] cardinality; - - private NumberCompressor numberCompressor; - - private NumberCompressor[] keyBlockUnCompressor; - - private ColumnGroupModel hybridStoreModel; - - /** - * dataIndexMap - */ - private int[] dataIndexMapLength; - - /** - * dataIndexMap - */ - private long[] dataIndexMapOffsets; - - /** - * aggKeyBlock - */ - private boolean[] aggKeyBlock; - - /** - * @return the numberOfKeys - */ - public int getNumberOfKeys() { - return numberOfKeys; - } - - /** - * @param numberOfKeys the numberOfKeys to set - */ - public void setNumberOfKeys(int numberOfKeys) { - this.numberOfKeys = numberOfKeys; - } - - /** - * @return the sizeOfEachBlock - */ - public int[] getSizeOfEachBlock() { - return sizeOfEachBlock; - } - - /** - * @param sizeOfEachBlock the sizeOfEachBlock to set - */ - public void setSizeOfEachBlock(int[] sizeOfEachBlock) { - this.sizeOfEachBlock = sizeOfEachBlock; - } - - /** - * @return the keyBlockLengths - */ - public int[] getKeyBlockLengths() { - return keyBlockLengths; - } - - /** - * @param keyBlockLengths the keyBlockLengths to set - */ - public void setKeyBlockLengths(int[] keyBlockLengths) { - this.keyBlockLengths = keyBlockLengths; - } - - /** - * @return the keyBlockOffsets - */ - public long[] getKeyBlockOffsets() { - return keyBlockOffsets; - } - - /** - * @param keyBlockOffsets the keyBlockOffsets to set - */ - public void setKeyBlockOffsets(long[] keyBlockOffsets) { - this.keyBlockOffsets = keyBlockOffsets; - } - - /** - * @return the keyBlockIndexLength - */ - public int[] getKeyBlockIndexLength() { - return keyBlockIndexLength; - } - - /** - * @param keyBlockIndexLength the keyBlockIndexLength to set - */ - public void setKeyBlockIndexLength(int[] keyBlockIndexLength) { - this.keyBlockIndexLength = keyBlockIndexLength; - } - - /** - * @return the keyBlockIndexOffsets - */ - public long[] getKeyBlockIndexOffsets() { - return keyBlockIndexOffsets; - } - - /** - * @param keyBlockIndexOffsets the keyBlockIndexOffsets to set - */ - public void setKeyBlockIndexOffsets(long[] keyBlockIndexOffsets) { - this.keyBlockIndexOffsets = keyBlockIndexOffsets; - } - - /** - * @return the filePath - */ - public String getFilePath() { - return filePath; - } - - /** - * @param filePath the filePath to set - */ - public void setFilePath(String filePath) { - this.filePath = filePath; - } - - /** - * @return the isSorted - */ - public boolean[] getIsSorted() { - return isSorted; - } - - /** - * @param isSorted the isSorted to set - */ - public void setIsSorted(boolean[] isSorted) { - this.isSorted = isSorted; - } - - /** - * @return the numberCompressor - */ - public NumberCompressor getNumberCompressor() { - return numberCompressor; - } - - /** - * @param numberCompressor the numberCompressor to set - */ - public void setNumberCompressor(NumberCompressor numberCompressor) { - this.numberCompressor = numberCompressor; - } - - /** - * @return the dataIndexMapLength - */ - public int[] getDataIndexMapLength() { - return dataIndexMapLength; - } - - /** - * @param dataIndexMapLength the dataIndexMapLength to set - */ - public void setDataIndexMapLength(int[] dataIndexMapLength) { - this.dataIndexMapLength = dataIndexMapLength; - } - - /** - * @return the dataIndexMapOffsets - */ - public long[] getDataIndexMapOffsets() { - return dataIndexMapOffsets; - } - - /** - * @param dataIndexMapOffsets the dataIndexMapOffsets to set - */ - public void setDataIndexMapOffsets(long[] dataIndexMapOffsets) { - this.dataIndexMapOffsets = dataIndexMapOffsets; - } - - /** - * @return the aggKeyBlock - */ - public boolean[] getAggKeyBlock() { - return aggKeyBlock; - } - - /** - * @param aggKeyBlock the aggKeyBlock to set - */ - public void setAggKeyBlock(boolean[] aggKeyBlock) { - this.aggKeyBlock = aggKeyBlock; - } - - /** - * @return the keyBlockUnCompressor - */ - public NumberCompressor[] getKeyBlockUnCompressor() { - return keyBlockUnCompressor; - } - - /** - * @param keyBlockUnCompressor the keyBlockUnCompressor to set - */ - public void setKeyBlockUnCompressor(NumberCompressor[] keyBlockUnCompressor) { - this.keyBlockUnCompressor = keyBlockUnCompressor; - } - - public int[] getCardinality() { - return cardinality; - } - - public void setCardinality(int[] cardinality) { - this.cardinality = cardinality; - } - - public ColumnGroupModel getHybridStoreModel() { - return hybridStoreModel; - } - - public void setHybridStoreModel(ColumnGroupModel hybridStoreModel) { - this.hybridStoreModel = hybridStoreModel; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java index a9dcb35..7754ddb 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java @@ -19,132 +19,35 @@ package org.apache.carbondata.core.datastorage.store.columnar; -import org.apache.carbondata.core.keygenerator.KeyGenerator; -import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory; - -public class ColumnarKeyStoreMetadata { - private boolean isSorted; - - private int[] columnIndex; +class ColumnarKeyStoreMetadata { private int[] columnReverseIndex; private int eachRowSize; - private int[] dataIndex; - - private boolean isUnCompressed; - - private KeyGenerator keyGenerator; - - /** - * isNoDictionaryValColumn. - */ - private boolean isNoDictionaryValColumn; - private boolean isRowStore; - - public ColumnarKeyStoreMetadata(int eachRowSize) { + ColumnarKeyStoreMetadata(int eachRowSize) { this.eachRowSize = eachRowSize; - keyGenerator = KeyGeneratorFactory.getKeyGenerator(new int[] { eachRowSize }); - } - - /** - * @return the isSorted - */ - public boolean isSorted() { - return isSorted; - } - - /** - * @param isSorted the isSorted to set - */ - public void setSorted(boolean isSorted) { - this.isSorted = isSorted; - } - - /** - * @return the columnIndex - */ - public int[] getColumnIndex() { - return columnIndex; - } - - /** - * @param columnIndex the columnIndex to set - */ - public void setColumnIndex(int[] columnIndex) { - this.columnIndex = columnIndex; } /** * @return the eachRowSize */ - public int getEachRowSize() { + int getEachRowSize() { return eachRowSize; } /** - * @return the dataIndex - */ - public int[] getDataIndex() { - return dataIndex; - } - - /** - * @param dataIndex the dataIndex to set - */ - public void setDataIndex(int[] dataIndex) { - this.dataIndex = dataIndex; - } - - /** * @return the columnReverseIndex */ - public int[] getColumnReverseIndex() { + int[] getColumnReverseIndex() { return columnReverseIndex; } /** * @param columnReverseIndex the columnReverseIndex to set */ - public void setColumnReverseIndex(int[] columnReverseIndex) { + void setColumnReverseIndex(int[] columnReverseIndex) { this.columnReverseIndex = columnReverseIndex; } - public boolean isUnCompressed() { - return isUnCompressed; - } - - public void setUnCompressed(boolean isUnCompressed) { - this.isUnCompressed = isUnCompressed; - } - - public KeyGenerator getKeyGenerator() { - return keyGenerator; - } - - public boolean isRowStore() { - return isRowStore; - } - - public void setRowStore(boolean isRowStore) { - this.isRowStore = isRowStore; - } - - /** - * @return - */ - public boolean isNoDictionaryValColumn() { - return isNoDictionaryValColumn; - - } - - /** - * @param isNoDictionaryValColumn - */ - public void setNoDictionaryValColumn(boolean isNoDictionaryValColumn) { - this.isNoDictionaryValColumn = isNoDictionaryValColumn; - - } - } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java index e68a9fe..824076e 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java @@ -221,12 +221,11 @@ public class WriterCompressModel { this.uniqueValue = uniqueValue; } - public void setCompressionFinders(CompressionFinder[] compressionFinders) { this.compressionFinders = compressionFinders; } - public CompressionFinder[] getCompressionFinder() { + public CompressionFinder[] getCompressionFinders() { return this.compressionFinders; } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java index b8c8248..1b9b163 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java @@ -129,7 +129,7 @@ public class UnCompressByteArray implements ValueCompressonHolder.UnCompressValu return holder; } - public static enum ByteArrayType { + public enum ByteArrayType { BYTE_ARRAY, BIG_DECIMAL } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java index e5c3293..44bb60d 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java @@ -53,7 +53,7 @@ public class UnCompressMaxMinFloat implements UnCompressValue<float[]> { } @Override public void setValue(float[] value) { - this.value = (float[]) value; + this.value = value; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java index e7e2e4e..1c7f651 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java @@ -75,9 +75,6 @@ public class UnCompressNoneDefault implements UnCompressValue<double[]> { return null; } - /** - * @see ValueCompressonHolder.UnCompressValue#getCompressorObject() - */ @Override public UnCompressValue getCompressorObject() { return new UnCompressNoneByte(actualDataType); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java index e4e5a75..364d2fa 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java @@ -117,12 +117,8 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue private CarbonReadDataHolder unCompressLong() { CarbonReadDataHolder dataHldr = new CarbonReadDataHolder(); - long[] vals = new long[value.length]; - - for (int i = 0; i < vals.length; i++) { - vals[i] = value[i]; - } + System.arraycopy(value, 0, vals, 0, vals.length); dataHldr.setReadableLongValues(vals); return dataHldr; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java index 65c6556..6e2f5d7 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java @@ -23,8 +23,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastorage.store.FileHolder; @@ -34,9 +32,6 @@ import org.apache.hadoop.fs.Path; public class DFSFileHolderImpl implements FileHolder { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(DFSFileHolderImpl.class.getName()); /** * cache to hold filename and its stream */ @@ -47,10 +42,10 @@ public class DFSFileHolderImpl implements FileHolder { new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); } - @Override public byte[] readByteArray(String filePath, long offset, int length) { + @Override public byte[] readByteArray(String filePath, long offset, int length) + throws IOException { FSDataInputStream fileChannel = updateCache(filePath); - byte[] byteBffer = read(fileChannel, length, offset); - return byteBffer; + return read(fileChannel, length, offset); } /** @@ -61,17 +56,13 @@ public class DFSFileHolderImpl implements FileHolder { * @param filePath fully qualified file path * @return channel */ - private FSDataInputStream updateCache(String filePath) { + private FSDataInputStream updateCache(String filePath) throws IOException { FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath); - try { - if (null == fileChannel) { - Path pt = new Path(filePath); - FileSystem fs = FileSystem.get(FileFactory.getConfiguration()); - fileChannel = fs.open(pt); - fileNameAndStreamCache.put(filePath, fileChannel); - } - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); + if (null == fileChannel) { + Path pt = new Path(filePath); + FileSystem fs = FileSystem.get(FileFactory.getConfiguration()); + fileChannel = fs.open(pt); + fileNameAndStreamCache.put(filePath, fileChannel); } return fileChannel; } @@ -84,14 +75,10 @@ public class DFSFileHolderImpl implements FileHolder { * @param offset position * @return byte buffer */ - private byte[] read(FSDataInputStream channel, int size, long offset) { + private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException { byte[] byteBffer = new byte[size]; - try { - channel.seek(offset); - channel.readFully(byteBffer); - } catch (Exception e) { - LOGGER.error(e, e.getMessage()); - } + channel.seek(offset); + channel.readFully(byteBffer); return byteBffer; } @@ -102,82 +89,46 @@ public class DFSFileHolderImpl implements FileHolder { * @param size number of bytes * @return byte buffer */ - private byte[] read(FSDataInputStream channel, int size) { + private byte[] read(FSDataInputStream channel, int size) throws IOException { byte[] byteBffer = new byte[size]; - try { - channel.readFully(byteBffer); - } catch (Exception e) { - LOGGER.error(e, e.getMessage()); - } + channel.readFully(byteBffer); return byteBffer; } - @Override public int readInt(String filePath, long offset) { + @Override public int readInt(String filePath, long offset) throws IOException { FSDataInputStream fileChannel = updateCache(filePath); - int i = -1; - try { - fileChannel.seek(offset); - i = fileChannel.readInt(); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - - return i; + fileChannel.seek(offset); + return fileChannel.readInt(); } - @Override public long readDouble(String filePath, long offset) { + @Override public long readDouble(String filePath, long offset) throws IOException { FSDataInputStream fileChannel = updateCache(filePath); - long i = -1; - try { - fileChannel.seek(offset); - i = fileChannel.readLong(); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - - return i; + fileChannel.seek(offset); + return fileChannel.readLong(); } - @Override public void finish() { + @Override public void finish() throws IOException { for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) { - try { - FSDataInputStream channel = entry.getValue(); - if (null != channel) { - channel.close(); - } - } catch (IOException exception) { - LOGGER.error(exception, exception.getMessage()); + FSDataInputStream channel = entry.getValue(); + if (null != channel) { + channel.close(); } } - } - @Override public byte[] readByteArray(String filePath, int length) { + @Override public byte[] readByteArray(String filePath, int length) throws IOException { FSDataInputStream fileChannel = updateCache(filePath); - byte[] byteBffer = read(fileChannel, length); - return byteBffer; + return read(fileChannel, length); } - @Override public long readLong(String filePath, long offset) { + @Override public long readLong(String filePath, long offset) throws IOException { FSDataInputStream fileChannel = updateCache(filePath); - long i = -1; - try { - fileChannel.seek(offset); - i = fileChannel.readLong(); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return i; + fileChannel.seek(offset); + return fileChannel.readLong(); } - @Override public int readInt(String filePath) { + @Override public int readInt(String filePath) throws IOException { FSDataInputStream fileChannel = updateCache(filePath); - int i = -1; - try { - i = fileChannel.readInt(); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return i; + return fileChannel.readInt(); } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java index c540920..f0c424b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java @@ -51,20 +51,7 @@ import org.apache.hadoop.io.compress.GzipCodec; public final class FileFactory { private static Configuration configuration = null; - private static FileType storeDefaultFileType = FileType.LOCAL; - static { - String property = CarbonUtil.getCarbonStorePath(); - if (property != null) { - if (property.startsWith(CarbonUtil.HDFS_PREFIX)) { - storeDefaultFileType = FileType.HDFS; - } else if (property.startsWith(CarbonUtil.ALLUXIO_PREFIX)) { - storeDefaultFileType = FileType.ALLUXIO; - } else if (property.startsWith(CarbonUtil.VIEWFS_PREFIX)) { - storeDefaultFileType = FileType.VIEWFS; - } - } - configuration = new Configuration(); configuration.addResource(new Path("../core-default.xml")); } @@ -90,22 +77,6 @@ public final class FileFactory { } } - public static FileType getFileType() { - String property = CarbonUtil.getCarbonStorePath(); - if (property != null) { - if (property.startsWith(CarbonUtil.HDFS_PREFIX)) { - storeDefaultFileType = FileType.HDFS; - } - else if (property.startsWith(CarbonUtil.ALLUXIO_PREFIX)) { - storeDefaultFileType = FileType.ALLUXIO; - } - else if (property.startsWith(CarbonUtil.VIEWFS_PREFIX)) { - storeDefaultFileType = FileType.VIEWFS; - } - } - return storeDefaultFileType; - } - public static FileType getFileType(String path) { if (path.startsWith(CarbonUtil.HDFS_PREFIX)) { return FileType.HDFS; @@ -231,44 +202,6 @@ public final class FileFactory { } } - public static DataOutputStream getDataOutputStream(String path, FileType fileType, - short replicationFactor) throws IOException { - path = path.replace("\\", "/"); - switch (fileType) { - case LOCAL: - return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path))); - case HDFS: - case ALLUXIO: - case VIEWFS: - Path pt = new Path(path); - FileSystem fs = pt.getFileSystem(configuration); - FSDataOutputStream stream = fs.create(pt, replicationFactor); - return stream; - default: - return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path))); - } - } - - public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize) - throws IOException { - path = path.replace("\\", "/"); - switch (fileType) { - case LOCAL: - return new DataOutputStream( - new BufferedOutputStream(new FileOutputStream(path), bufferSize)); - case HDFS: - case ALLUXIO: - case VIEWFS: - Path pt = new Path(path); - FileSystem fs = pt.getFileSystem(configuration); - FSDataOutputStream stream = fs.create(pt, true, bufferSize); - return stream; - default: - return new DataOutputStream( - new BufferedOutputStream(new FileOutputStream(path), bufferSize)); - } - } - public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize, boolean append) throws IOException { path = path.replace("\\", "/"); @@ -428,7 +361,6 @@ public final class FileFactory { return path.delete(); } - public static boolean mkdirs(String filePath, FileType fileType) throws IOException { filePath = filePath.replace("\\", "/"); switch (fileType) {
