http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java deleted file mode 100644 index 013d873..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.ByteUtil; - -public class BlockIndexerStorageForInt implements IndexStorage<int[]> { - private boolean alreadySorted; - - private int[] dataAfterComp; - - private int[] indexMap; - - private byte[][] keyBlock; - - private int[] dataIndexMap; - - private int totalSize; - - public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary, - boolean isSortRequired) { - ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary); - if (isSortRequired) { - Arrays.sort(columnWithIndexs); - } - compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock)); - if (compressData) { - compressDataMyOwnWay(columnWithIndexs); - } - } - - /** - * Create an object with each column array and respective index - * - * @return - */ - private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock, - boolean isNoDictionary) { - ColumnWithIntIndex[] columnWithIndexs; - if (isNoDictionary) { - columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length]; - for (int i = 0; i < columnWithIndexs.length; i++) { - columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i); - } - - } else { - columnWithIndexs = new ColumnWithIntIndex[keyBlock.length]; - for (int i = 0; i < columnWithIndexs.length; i++) { - columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i); - } - } - - return columnWithIndexs; - } - - private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs, - byte[][] keyBlock) { - int[] indexes = new int[columnWithIndexs.length]; - for (int i = 0; i < indexes.length; i++) { - indexes[i] = columnWithIndexs[i].getIndex(); - keyBlock[i] = columnWithIndexs[i].getColumn(); - } - this.keyBlock = keyBlock; - return indexes; - } - - /** - * It compresses depends up on the sequence numbers. - * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In - * first array the start and end of sequential numbers and second array - * keeps the indexes of where sequential numbers starts. If there is no - * sequential numbers then the same array it returns with empty second - * array. - * - * @param indexes - */ - public void compressMyOwnWay(int[] indexes) { - List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - int k = 0; - int i = 1; - for (; i < indexes.length; i++) { - if (indexes[i] - indexes[i - 1] == 1) { - k++; - } else { - if (k > 0) { - map.add((list.size())); - list.add(indexes[i - k - 1]); - list.add(indexes[i - 1]); - } else { - list.add(indexes[i - 1]); - } - k = 0; - } - } - if (k > 0) { - map.add((list.size())); - list.add(indexes[i - k - 1]); - list.add(indexes[i - 1]); - } else { - list.add(indexes[i - 1]); - } - dataAfterComp = convertToArray(list); - if (indexes.length == dataAfterComp.length) { - indexMap = new int[0]; - } else { - indexMap = convertToArray(map); - } - if (dataAfterComp.length == 2 && indexMap.length == 1) { - alreadySorted = true; - } - } - - private int[] convertToArray(List<Integer> list) { - int[] shortArray = new int[list.size()]; - for (int i = 0; i < shortArray.length; i++) { - shortArray[i] = list.get(i); - } - return shortArray; - } - - /** - * @return the alreadySorted - */ - public boolean isAlreadySorted() { - return alreadySorted; - } - - /** - * @return the dataAfterComp - */ - public int[] getDataAfterComp() { - return dataAfterComp; - } - - /** - * @return the indexMap - */ - public int[] getIndexMap() { - return indexMap; - } - - /** - * @return the keyBlock - */ - public byte[][] getKeyBlock() { - return keyBlock; - } - - private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) { - byte[] prvKey = indexes[0].getColumn(); - List<ColumnWithIntIndex> list = - new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - list.add(indexes[0]); - int counter = 1; - int start = 0; - List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - for (int i = 1; i < indexes.length; i++) { - if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) { - prvKey = indexes[i].getColumn(); - list.add(indexes[i]); - map.add(start); - map.add(counter); - start += counter; - counter = 1; - continue; - } - counter++; - } - map.add(start); - map.add(counter); - this.keyBlock = convertToKeyArray(list); - if (indexes.length == keyBlock.length) { - dataIndexMap = new int[0]; - } else { - dataIndexMap = convertToArray(map); - } - } - - private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) { - byte[][] shortArray = new byte[list.size()][]; - for (int i = 0; i < shortArray.length; i++) { - shortArray[i] = list.get(i).getColumn(); - totalSize += shortArray[i].length; - } - return shortArray; - } - - @Override public int[] getDataIndexMap() { - return dataIndexMap; - } - - @Override public int getTotalSize() { - return totalSize; - } - - @Override public byte[] getMin() { - return keyBlock[0]; - } - - @Override public byte[] getMax() { - return keyBlock[keyBlock.length - 1]; - } -}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java deleted file mode 100644 index c7d43cf..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.ByteUtil; - -public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> { - private byte[][] keyBlock; - private byte[][] sortedBlock; - private int totalSize; - private int[] dataIndexMap; - - public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData, - boolean isNoDictionary) { - // without invertedindex but can be RLE - if (compressData) { - // with RLE - byte[] prvKey = keyBlockInput[0]; - List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - list.add(keyBlockInput[0]); - int counter = 1; - int start = 0; - List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - int length = keyBlockInput.length; - for(int i = 1; i < length; i++) { - if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) { - prvKey = keyBlockInput[i]; - list.add(keyBlockInput[i]); - map.add(start); - map.add(counter); - start += counter; - counter = 1; - continue; - } - counter++; - } - map.add(start); - map.add(counter); - this.keyBlock = convertToKeyArray(list); - if (keyBlockInput.length == this.keyBlock.length) { - dataIndexMap = new int[0]; - } else { - dataIndexMap = convertToArray(map); - } - } else { - this.keyBlock = keyBlockInput; - dataIndexMap = new int[0]; - } - - this.sortedBlock = new byte[keyBlock.length][]; - System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length); - if (isNoDictionary) { - Arrays.sort(sortedBlock, new Comparator<byte[]>() { - @Override - public int compare(byte[] col1, byte[] col2) { - return ByteUtil.UnsafeComparer.INSTANCE - .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2); - } - }); - } else { - Arrays.sort(sortedBlock, new Comparator<byte[]>() { - @Override - public int compare(byte[] col1, byte[] col2) { - return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2); - } - }); - } - - } - - private int[] convertToArray(List<Integer> list) { - int[] shortArray = new int[list.size()]; - for(int i = 0; i < shortArray.length; i++) { - shortArray[i] = list.get(i); - } - return shortArray; - } - - private byte[][] convertToKeyArray(List<byte[]> list) { - byte[][] shortArray = new byte[list.size()][]; - for (int i = 0; i < shortArray.length; i++) { - shortArray[i] = list.get(i); - totalSize += shortArray[i].length; - } - return shortArray; - } - - @Override - public int[] getDataIndexMap() { - return dataIndexMap; - } - - @Override - public int getTotalSize() { - return totalSize; - } - - @Override - public boolean isAlreadySorted() { - return true; - } - - /** - * no use - * @return - */ - @Override - public int[] getDataAfterComp() { - return new int[0]; - } - - /** - * no use - * @return - */ - @Override - public int[] getIndexMap() { - return new int[0]; - } - - /** - * @return the keyBlock - */ - public byte[][] getKeyBlock() { - return keyBlock; - } - - @Override public byte[] getMin() { - return sortedBlock[0]; - } - - @Override public byte[] getMax() { - return sortedBlock[sortedBlock.length - 1]; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java deleted file mode 100644 index cf9ba40..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.datastorage.store.columnar; - -public class ColumnGroupModel { - - /** - * number of columns in columnar block - */ - private int[] columnSplit; - - /** - * total number of columns - */ - private int noOfColumnsStore; - - /** - * whether given index is columnar or not - * true: columnar - * false: row block - */ - private boolean[] columnarStore; - - /** - * column groups - * e.g - * {{0,1,2},3,4,{5,6}} - */ - private int[][] columnGroups; - - /** - * return columnSplit - * - * @return - */ - public int[] getColumnSplit() { - return columnSplit; - } - - /** - * set columnSplit - * - * @param split - */ - public void setColumnSplit(int[] split) { - this.columnSplit = split; - } - - /** - * @return no of columnar block - */ - public int getNoOfColumnStore() { - return this.noOfColumnsStore; - } - - /** - * set no of columnar block - * - * @param noOfColumnsStore - */ - public void setNoOfColumnStore(int noOfColumnsStore) { - this.noOfColumnsStore = noOfColumnsStore; - } - - /** - * it's an identifier for row block or single column block - * - * @param columnarStore - */ - public void setColumnarStore(boolean[] columnarStore) { - this.columnarStore = columnarStore; - } - - /** - * set column groups - * - * @param columnGroups - */ - public void setColumnGroup(int[][] columnGroups) { - this.columnGroups = columnGroups; - } - - /** - * check if given column group is columnar - * - * @param colGroup - * @return true if given block is columnar - */ - public boolean isColumnar(int colGroup) { - return columnarStore[colGroup]; - } - - /** - * @return columngroups - */ - public int[][] getColumnGroup() { - return this.columnGroups; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java deleted file mode 100644 index 36606a5..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -import java.util.Arrays; - -import org.apache.carbondata.core.util.ByteUtil; - -public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> { - protected byte[] column; - - private int index; - - public ColumnWithIntIndex(byte[] column, int index) { - this.column = column; - this.index = index; - } - - public ColumnWithIntIndex() { - } - - /** - * @return the column - */ - public byte[] getColumn() { - return column; - } - - /** - * @param column the column to set - */ - public void setColumn(byte[] column) { - this.column = column; - } - - /** - * @return the index - */ - public int getIndex() { - return index; - } - - /** - * @param index the index to set - */ - public void setIndex(int index) { - this.index = index; - } - - @Override public int compareTo(ColumnWithIntIndex o) { - return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column); - } - - @Override public boolean equals(Object obj) { - if(obj == null || getClass() != obj.getClass()) { - return false; - } - ColumnWithIntIndex o = (ColumnWithIntIndex)obj; - return Arrays.equals(column, o.column) && index == o.index; - } - - @Override public int hashCode() { - return Arrays.hashCode(column) + index; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java deleted file mode 100644 index 61a1165..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -import java.util.Arrays; - -import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer; - -public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex - implements Comparable<ColumnWithIntIndex> { - - public ColumnWithIntIndexForHighCard(byte[] column, int index) { - super(column, index); - } - - @Override public int compareTo(ColumnWithIntIndex o) { - return UnsafeComparer.INSTANCE - .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2); - } - - @Override public boolean equals(Object obj) { - if(obj == null || getClass() != obj.getClass()) { - return false; - } - ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj; - return Arrays.equals(column, o.column) && getIndex() == o.getIndex(); - } - - @Override public int hashCode() { - return Arrays.hashCode(column) + getIndex(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java deleted file mode 100644 index 29887a3..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -import java.nio.ByteBuffer; - -public class ColumnarKeyStoreDataHolder { - private byte[] keyblockData; - private ColumnarKeyStoreMetadata columnarKeyStoreMetadata; - - public ColumnarKeyStoreDataHolder(final byte[] keyblockData, - final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) { - this.keyblockData = keyblockData; - this.columnarKeyStoreMetadata = columnarKeyStoreMetadata; - } - - public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) { - this.columnarKeyStoreMetadata = columnarKeyStoreMetadata; - } - - public int getSurrogateKey(int columnIndex) { - byte[] actual = new byte[4]; - int startIndex; - if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) { - startIndex = - columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata - .getEachRowSize(); - } else { - startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize(); - } - int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize(); - System.arraycopy(keyblockData, startIndex, actual, destPos, - columnarKeyStoreMetadata.getEachRowSize()); - return ByteBuffer.wrap(actual).getInt(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java deleted file mode 100644 index 7754ddb..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -class ColumnarKeyStoreMetadata { - - private int[] columnReverseIndex; - - private int eachRowSize; - - ColumnarKeyStoreMetadata(int eachRowSize) { - this.eachRowSize = eachRowSize; - } - - /** - * @return the eachRowSize - */ - int getEachRowSize() { - return eachRowSize; - } - - /** - * @return the columnReverseIndex - */ - int[] getColumnReverseIndex() { - return columnReverseIndex; - } - - /** - * @param columnReverseIndex the columnReverseIndex to set - */ - void setColumnReverseIndex(int[] columnReverseIndex) { - this.columnReverseIndex = columnReverseIndex; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/IndexStorage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/IndexStorage.java deleted file mode 100644 index e1f4548..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/IndexStorage.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -public interface IndexStorage<T> { - boolean isAlreadySorted(); - - T getDataAfterComp(); - - T getIndexMap(); - - byte[][] getKeyBlock(); - - T getDataIndexMap(); - - int getTotalSize(); - - /** - * @return min value of block - */ - byte[] getMin(); - - /** - * @return max value of block - */ - byte[] getMax(); -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java deleted file mode 100644 index 149facb..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.columnar; - -import java.util.Arrays; - -public final class UnBlockIndexer { - - private UnBlockIndexer() { - - } - - public static int[] uncompressIndex(int[] indexData, int[] indexMap) { - int actualSize = indexData.length; - for (int i = 0; i < indexMap.length; i++) { - actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1; - } - int[] indexes = new int[actualSize]; - int k = 0; - for (int i = 0; i < indexData.length; i++) { - int index = Arrays.binarySearch(indexMap, i); - if (index > -1) { - for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) { - indexes[k] = j; - k++; - } - i++; - } else { - indexes[k] = indexData[i]; - k++; - } - } - return indexes; - } - - public static byte[] uncompressData(byte[] data, int[] index, int keyLen) { - if (index.length < 1) { - return data; - } - int numberOfCopy = 0; - int actualSize = 0; - int srcPos = 0; - int destPos = 0; - for (int i = 1; i < index.length; i += 2) { - actualSize += index[i]; - } - byte[] uncompressedData = new byte[actualSize * keyLen]; - int picIndex = 0; - for (int i = 0; i < data.length; i += keyLen) { - numberOfCopy = index[picIndex * 2 + 1]; - picIndex++; - for (int j = 0; j < numberOfCopy; j++) { - System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen); - destPos += keyLen; - } - srcPos += keyLen; - } - return uncompressedData; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java deleted file mode 100644 index 1b90bc6..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression; - -public interface Compressor { - - byte[] compressByte(byte[] unCompInput); - - byte[] unCompressByte(byte[] compInput); - - byte[] unCompressByte(byte[] compInput, int offset, int length); - - byte[] compressShort(short[] unCompInput); - - short[] unCompressShort(byte[] compInput); - - short[] unCompressShort(byte[] compInput, int offset, int lenght); - - byte[] compressInt(int[] unCompInput); - - int[] unCompressInt(byte[] compInput); - - int[] unCompressInt(byte[] compInput, int offset, int length); - - byte[] compressLong(long[] unCompInput); - - long[] unCompressLong(byte[] compInput); - - long[] unCompressLong(byte[] compInput, int offset, int length); - - byte[] compressFloat(float[] unCompInput); - - float[] unCompressFloat(byte[] compInput); - - float[] unCompressFloat(byte[] compInput, int offset, int length); - - byte[] compressDouble(double[] unCompInput); - - double[] unCompressDouble(byte[] compInput); - - double[] unCompressDouble(byte[] compInput, int offset, int length); - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java deleted file mode 100644 index 5160879..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.CarbonProperties; - -public class CompressorFactory { - - private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory(); - - private final Compressor compressor; - - private CompressorFactory() { - String compressorType = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR); - switch (compressorType) { - case "snappy": - compressor = new SnappyCompressor(); - break; - default: - throw new RuntimeException( - "Invalid compressor type provided! Please provide valid compressor type"); - } - } - - public static CompressorFactory getInstance() { - return COMPRESSOR_FACTORY; - } - - public Compressor getCompressor() { - return compressor; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/MeasureMetaDataModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/MeasureMetaDataModel.java deleted file mode 100644 index f207478..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/MeasureMetaDataModel.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression; - -public class MeasureMetaDataModel { - /** - * maxValue - */ - private Object[] maxValue; - - /** - * minValue - */ - private Object[] minValue; - - /** - * mantissa - */ - private int[] mantissa; - - /** - * measureCount - */ - private int measureCount; - - /** - * uniqueValue - */ - private Object[] uniqueValue; - - /** - * type - */ - private char[] type; - - /** - * dataTypeSelected - */ - private byte[] dataTypeSelected; - - public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa, - int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) { - this.minValue = minValue; - this.maxValue = maxValue; - this.mantissa = mantissa; - this.measureCount = measureCount; - this.uniqueValue = uniqueValue; - this.type = type; - this.dataTypeSelected = dataTypeSelected; - } - - /** - * get Max value - * - * @return - */ - public Object[] getMaxValue() { - return maxValue; - } - - /** - * getMinValue - * - * @return - */ - public Object[] getMinValue() { - return minValue; - } - - /** - * getMantissa - * - * @return - */ - public int[] getMantissa() { - return mantissa; - } - - /** - * getMeasureCount - * - * @return - */ - public int getMeasureCount() { - return measureCount; - } - - /** - * getUniqueValue - * - * @return - */ - public Object[] getUniqueValue() { - return uniqueValue; - } - - /** - * @return the type - */ - public char[] getType() { - return type; - } - - /** - * @return the dataTypeSelected - */ - public byte[] getDataTypeSelected() { - return dataTypeSelected; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java deleted file mode 100644 index 51730a2..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression; - -import org.apache.carbondata.core.metadata.ValueEncoderMeta; -import org.apache.carbondata.core.util.ValueCompressionUtil; - -// Used in read path for decompression preparation -public class ReaderCompressModel { - private ValueEncoderMeta valueEncoderMeta; - - private ValueCompressionUtil.DataType convertedDataType; - - private ValueCompressionHolder valueHolder; - - public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) { - this.valueEncoderMeta = valueEncoderMeta; - } - - public ValueCompressionUtil.DataType getConvertedDataType() { - return convertedDataType; - } - - public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) { - this.convertedDataType = convertedDataType; - } - - public Object getMaxValue() { - return valueEncoderMeta.getMaxValue(); - } - - public int getMantissa() { - return valueEncoderMeta.getMantissa(); - } - - public ValueCompressionHolder getValueCompressionHolder() { - return valueHolder; - } - - public void setValueCompressionHolder(ValueCompressionHolder valueHolder) { - this.valueHolder = valueHolder; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java deleted file mode 100644 index 62d3b53..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression; - -import java.io.IOException; -import java.lang.reflect.Field; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; - -import org.xerial.snappy.Snappy; -import org.xerial.snappy.SnappyNative; - -public class SnappyCompressor implements Compressor { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(SnappyCompressor.class.getName()); - - private final SnappyNative snappyNative; - - public SnappyCompressor() { - Snappy snappy = new Snappy(); - Field privateField = null; - try { - privateField = snappy.getClass().getDeclaredField("impl"); - } catch (NoSuchFieldException | SecurityException e) { - throw new RuntimeException(e); - } - privateField.setAccessible(true); - try { - snappyNative = (SnappyNative) privateField.get(snappy); - } catch (IllegalArgumentException | IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - @Override public byte[] compressByte(byte[] unCompInput) { - try { - return Snappy.rawCompress(unCompInput, unCompInput.length); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public byte[] unCompressByte(byte[] compInput) { - try { - return Snappy.uncompress(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return compInput; - } - - @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) { - int uncompressedLength = 0; - byte[] data = null; - try { - uncompressedLength = Snappy.uncompressedLength(compInput, offset, length); - data = new byte[uncompressedLength]; - Snappy.uncompress(compInput, offset, length, data, 0); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return data; - } - - @Override public byte[] compressShort(short[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public short[] unCompressShort(byte[] compInput) { - try { - return Snappy.uncompressShortArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) { - try { - return Snappy.uncompressShortArray(compInput, offset, lenght); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public byte[] compressInt(int[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public int[] unCompressInt(byte[] compInput) { - try { - return Snappy.uncompressIntArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public int[] unCompressInt(byte[] compInput, int offset, int length) { - try { - return Snappy.uncompressIntArray(compInput, offset, length); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public byte[] compressLong(long[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public long[] unCompressLong(byte[] compInput) { - try { - return Snappy.uncompressLongArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public long[] unCompressLong(byte[] compInput, int offset, int length) { - try { - return Snappy.uncompressLongArray(compInput, offset, length); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public byte[] compressFloat(float[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public float[] unCompressFloat(byte[] compInput) { - try { - return Snappy.uncompressFloatArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) { - try { - return Snappy.uncompressFloatArray(compInput, offset, length); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public byte[] compressDouble(double[] unCompInput) { - try { - return Snappy.compress(unCompInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - return null; - } - } - - @Override public double[] unCompressDouble(byte[] compInput) { - try { - return Snappy.uncompressDoubleArray(compInput); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } - - @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) { - try { - int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length); - double[] result = new double[uncompressedLength / 8]; - snappyNative.rawUncompress(compInput, offset, length, result, 0); - return result; - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } - return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressionHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressionHolder.java deleted file mode 100644 index 6fee5ea..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressionHolder.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression; - -import java.math.BigDecimal; - -import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; - - - -/** - * ValueCompressionHolder is the base class for handling - * compression / decompression of the measure data chunk - */ -public abstract class ValueCompressionHolder<T> { - - /** - * compressedValue - */ - protected byte[] compressedValue; - - /** - * @param compressor the compressor used to decompress the data - * @param dataType data type of the data - * @param data compressed data - */ - public void unCompress(Compressor compressor, DataType dataType, byte[] data, - int offset, int length) { - switch (dataType) { - case DATA_BYTE: - setValue((T)compressor.unCompressByte(data, offset, length)); - break; - case DATA_SHORT: - setValue((T)compressor.unCompressShort(data, offset, length)); - break; - case DATA_INT: - setValue((T)compressor.unCompressInt(data, offset, length)); - break; - case DATA_LONG: - case DATA_BIGINT: - setValue((T)compressor.unCompressLong(data, offset, length)); - break; - case DATA_FLOAT: - setValue((T)compressor.unCompressFloat(data, offset, length)); - break; - default: - setValue((T)compressor.unCompressDouble(data, offset, length)); - break; - } - } - - /** - * @param compressor the compressor used to compress the data - * @param dataType data type of the data - * @param data original data - */ - public byte[] compress(Compressor compressor, DataType dataType, Object data) { - switch (dataType) { - case DATA_BYTE: - return compressor.compressByte((byte[])data); - case DATA_SHORT: - return compressor.compressShort((short[])data); - case DATA_INT: - return compressor.compressInt((int[])data); - case DATA_LONG: - case DATA_BIGINT: - return compressor.compressLong((long[])data); - case DATA_FLOAT: - return compressor.compressFloat((float[])data); - case DATA_DOUBLE: - default: - return compressor.compressDouble((double[])data); - } - } - - public abstract void setValue(T value); - - public abstract T getValue(); - - public abstract void setValueInBytes(byte[] value); - - public abstract void compress(); - - public abstract void uncompress(DataType dataType, byte[] compressData, int offset, - int length, int decimal, Object maxValueObject); - - public byte[] getCompressedData() { return compressedValue; } - - public abstract long getLongValue(int index); - - public abstract double getDoubleValue(int index); - - public abstract BigDecimal getBigDecimalValue(int index); - - public abstract void freeMemory(); - - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java deleted file mode 100644 index 2e93123..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression; - -import org.apache.carbondata.core.util.CompressionFinder; -import org.apache.carbondata.core.util.ValueCompressionUtil; - -public class WriterCompressModel { - - /** - * DataType[] variable. - */ - private ValueCompressionUtil.DataType[] convertedDataType; - /** - * DataType[] variable. - */ - private ValueCompressionUtil.DataType[] actualDataType; - - /** - * maxValue - */ - private Object[] maxValue; - /** - * minValue. - */ - private Object[] minValue; - - /** - * uniqueValue - */ - private Object[] uniqueValue; - /** - * mantissa. - */ - private int[] mantissa; - - /** - * aggType - */ - private char[] type; - - /** - * dataTypeSelected - */ - private byte[] dataTypeSelected; - /** - * unCompressValues. - */ - private ValueCompressionHolder[] valueHolder; - - private CompressionFinder[] compressionFinders; - - /** - * @return the convertedDataType - */ - public ValueCompressionUtil.DataType[] getConvertedDataType() { - return convertedDataType; - } - - /** - * @param convertedDataType the convertedDataType to set - */ - public void setConvertedDataType(ValueCompressionUtil.DataType[] convertedDataType) { - this.convertedDataType = convertedDataType; - } - - /** - * @return the actualDataType - */ - public ValueCompressionUtil.DataType[] getActualDataType() { - return actualDataType; - } - - /** - * @param actualDataType - */ - public void setActualDataType(ValueCompressionUtil.DataType[] actualDataType) { - this.actualDataType = actualDataType; - } - - /** - * @return the maxValue - */ - public Object[] getMaxValue() { - return maxValue; - } - - /** - * @param maxValue the maxValue to set - */ - public void setMaxValue(Object[] maxValue) { - this.maxValue = maxValue; - } - - /** - * @return the mantissa - */ - public int[] getMantissa() { - return mantissa; - } - - /** - * @param mantissa the mantissa to set - */ - public void setMantissa(int[] mantissa) { - this.mantissa = mantissa; - } - - /** - * getUnCompressValues(). - * - * @return the unCompressValues - */ - public ValueCompressionHolder[] getValueCompressionHolder() { - return valueHolder; - } - - /** - * @param valueHolder set the ValueCompressionHolder - */ - public void setValueCompressionHolder(ValueCompressionHolder[] valueHolder) { - this.valueHolder = valueHolder; - } - - /** - * getMinValue - * - * @return - */ - public Object[] getMinValue() { - return minValue; - } - - /** - * setMinValue. - * - * @param minValue - */ - public void setMinValue(Object[] minValue) { - this.minValue = minValue; - } - - /** - * @return the aggType - */ - public char[] getType() { - return type; - } - - /** - * @param type the type to set - */ - public void setType(char[] type) { - this.type = type; - } - - /** - * @return the dataTypeSelected - */ - public byte[] getDataTypeSelected() { - return dataTypeSelected; - } - - /** - * @param dataTypeSelected the dataTypeSelected to set - */ - public void setDataTypeSelected(byte[] dataTypeSelected) { - this.dataTypeSelected = dataTypeSelected; - } - - /** - * getUniqueValue - * - * @return - */ - public Object[] getUniqueValue() { - return uniqueValue; - } - - /** - * setUniqueValue - * - * @param uniqueValue - */ - public void setUniqueValue(Object[] uniqueValue) { - this.uniqueValue = uniqueValue; - } - - public void setCompressionFinders(CompressionFinder[] compressionFinders) { - this.compressionFinders = compressionFinders; - } - - public CompressionFinder[] getCompressionFinders() { - return this.compressionFinders; - } - - /** - * @return the compType - */ - public ValueCompressionUtil.COMPRESSION_TYPE getCompType(int index) { - return this.compressionFinders[index].getCompType(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinByte.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinByte.java deleted file mode 100644 index 75b9db0..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinByte.java +++ /dev/null @@ -1,115 +0,0 @@ -/* -3 * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression.decimal; - -import java.math.BigDecimal; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; -import org.apache.carbondata.core.datastorage.store.compression.Compressor; -import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder; -import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; - -public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> { - - /** - * Attribute for Carbon LOGGER - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(CompressionMaxMinByte.class.getName()); - - /** - * compressor. - */ - private static Compressor compressor = CompressorFactory.getInstance().getCompressor(); - - /** - * value. - */ - protected byte[] value; - - private MeasureDataChunkStore<byte[]> measureChunkStore; - - /** - * actual data type - */ - protected DataType actualDataType; - - private double maxValue; - - public CompressionMaxMinByte(DataType actualDataType) { - this.actualDataType = actualDataType; - } - - @Override public byte[] getValue() {return this.value; } - - @Override public void setValue(byte[] value) { - this.value = value; - } - - @Override public void compress() { - compressedValue = super.compress(compressor, DataType.DATA_BYTE, value); - } - - @Override - public void uncompress(DataType dataType, byte[] compressedData, int offset, int length, - int decimalPlaces, Object maxValueObject) { - super.unCompress(compressor, dataType, compressedData, offset, length); - setUncompressedValues(value, maxValueObject); - - } - - @Override public void setValueInBytes(byte[] value) { - this.value = value; - } - - @Override public long getLongValue(int index) { - byte byteValue = measureChunkStore.getByte(index); - return (long) (maxValue - byteValue); - } - - @Override public double getDoubleValue(int index) { - byte byteValue = measureChunkStore.getByte(index); - return (maxValue - byteValue); - } - - @Override public BigDecimal getBigDecimalValue(int index) { - throw new UnsupportedOperationException( - "Big decimal value is not defined for CompressionMaxMinByte"); - } - - private void setUncompressedValues(byte[] data, Object maxValueObject) { - this.measureChunkStore = - MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length); - this.measureChunkStore.putData(data); - if (maxValueObject instanceof Long) { - this.maxValue = (long)maxValueObject; - } else { - this.maxValue = (double) maxValueObject; - } - } - - @Override public void freeMemory() { - this.measureChunkStore.freeMemory(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinDefault.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinDefault.java deleted file mode 100644 index 00a993c..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinDefault.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression.decimal; - -import java.math.BigDecimal; -import java.nio.ByteBuffer; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; -import org.apache.carbondata.core.datastorage.store.compression.Compressor; -import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder; -import org.apache.carbondata.core.util.ValueCompressionUtil; -import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; - - -public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> { - - /** - * Attribute for Carbon LOGGER - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(CompressionMaxMinDefault.class.getName()); - - /** - * compressor. - */ - private static Compressor compressor = CompressorFactory.getInstance().getCompressor(); - - /** - * value. - */ - private double[] value; - - private MeasureDataChunkStore<double[]> measureChunkStore; - - /** - * actual data type - */ - private DataType actualDataType; - - private double maxValue; - - public CompressionMaxMinDefault(DataType actualDataType) { - this.actualDataType = actualDataType; - } - - @Override public void setValue(double[] value) { - this.value = value; - } - - @Override public double[] getValue() {return this.value; } - - @Override public void compress() { - compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value); - } - - @Override public void uncompress(DataType dataType, byte[] compressedData, - int offset, int length, int decimalPlaces, Object maxValueObject) { - super.unCompress(compressor, dataType, compressedData, offset, length); - setUncompressedValues(value, maxValueObject); - } - - @Override public void setValueInBytes(byte[] value) { - ByteBuffer buffer = ByteBuffer.wrap(value); - this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length); - } - - @Override public long getLongValue(int index) { - throw new UnsupportedOperationException( - "Long value is not defined for CompressionMaxMinDefault"); - } - - @Override public double getDoubleValue(int index) { - double doubleValue = measureChunkStore.getDouble(index); - return maxValue - doubleValue; - } - - @Override public BigDecimal getBigDecimalValue(int index) { - throw new UnsupportedOperationException( - "Big decimal value is not defined for CompressionMaxMinDefault"); - } - - private void setUncompressedValues(double[] data, Object maxValueObject) { - this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE - .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length); - this.measureChunkStore.putData(data); - if (maxValueObject instanceof Long) { - this.maxValue = (long) maxValueObject; - } else { - this.maxValue = (double) maxValueObject; - } - } - - @Override public void freeMemory() { - this.measureChunkStore.freeMemory(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinInt.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinInt.java deleted file mode 100644 index bf00cf8..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinInt.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression.decimal; - -import java.math.BigDecimal; -import java.nio.ByteBuffer; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; -import org.apache.carbondata.core.datastorage.store.compression.Compressor; -import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder; -import org.apache.carbondata.core.util.ValueCompressionUtil; -import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; - -public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> { - /** - * Attribute for Carbon LOGGER - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(CompressionMaxMinInt.class.getName()); - - /** - * compressor. - */ - private static Compressor compressor = CompressorFactory.getInstance().getCompressor(); - - private MeasureDataChunkStore<int[]> measureChunkStore; - - /** - * value. - */ - private int[] value; - - private DataType actualDataType; - - private double maxValue; - - public CompressionMaxMinInt(DataType actualType) { - this.actualDataType = actualType; - } - - @Override public void setValue(int[] value) { - this.value = value; - } - - @Override public int[] getValue() { return this.value; } - - @Override - public void uncompress(DataType dataType, byte[] compressedData, int offset, int length, - int decimalPlaces, Object maxValueObject) { - super.unCompress(compressor, dataType, compressedData, offset, length); - setUncompressedValues(value, maxValueObject); - } - - @Override public void compress() { - compressedValue = super.compress(compressor, DataType.DATA_INT, value); - } - - @Override public void setValueInBytes(byte[] value) { - ByteBuffer buffer = ByteBuffer.wrap(value); - this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length); - } - - @Override public long getLongValue(int index) { - int intValue = measureChunkStore.getInt(index); - return (long) (maxValue - intValue); - } - - @Override public double getDoubleValue(int index) { - int intValue = measureChunkStore.getInt(index); - return maxValue - intValue; - } - - @Override public BigDecimal getBigDecimalValue(int index) { - throw new UnsupportedOperationException( - "Big decimal value is not defined for CompressionMaxMinInt"); - } - - private void setUncompressedValues(int[] data, Object maxValueObject) { - this.measureChunkStore = - MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length); - this.measureChunkStore.putData(data); - if (maxValueObject instanceof Long) { - this.maxValue = (long) maxValueObject; - } else { - this.maxValue = (double) maxValueObject; - } - } - - @Override public void freeMemory() { - this.measureChunkStore.freeMemory(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinLong.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinLong.java deleted file mode 100644 index bebe4b7..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinLong.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression.decimal; - -import java.math.BigDecimal; -import java.nio.ByteBuffer; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; -import org.apache.carbondata.core.datastorage.store.compression.Compressor; -import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder; -import org.apache.carbondata.core.util.ValueCompressionUtil; -import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; - -public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> { - /** - * Attribute for Carbon LOGGER - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(CompressionMaxMinLong.class.getName()); - /** - * compressor. - */ - private static Compressor compressor = CompressorFactory.getInstance().getCompressor(); - - private MeasureDataChunkStore<long[]> measureChunkStore; - - /** - * value. - */ - protected long[] value; - - protected DataType actualDataType; - - private double maxValue; - - public CompressionMaxMinLong(DataType actualDataType) { - this.actualDataType = actualDataType; - } - - @Override public void compress() { - compressedValue = super.compress(compressor, DataType.DATA_LONG, value); - } - - @Override public void setValue(long[] value) { - this.value = value; - - } - - @Override - public void uncompress(DataType dataType, byte[] compressedData, int offset, int length, - int decimalPlaces, Object maxValueObject) { - super.unCompress(compressor, dataType, compressedData, offset, length); - setUncompressValues(value, maxValueObject); - } - - @Override public long[] getValue() {return this.value; } - - @Override public void setValueInBytes(byte[] value) { - ByteBuffer buffer = ByteBuffer.wrap(value); - this.value = ValueCompressionUtil.convertToLongArray(buffer, value.length); - } - - @Override public long getLongValue(int index) { - long longValue = measureChunkStore.getLong(index); - return (long) maxValue - longValue; - } - - @Override public double getDoubleValue(int index) { - long longValue = measureChunkStore.getLong(index); - return maxValue - longValue; - } - - @Override public BigDecimal getBigDecimalValue(int index) { - throw new UnsupportedOperationException( - "Big decimal value is not defined for CompressionMaxMinLong"); - } - - private void setUncompressValues(long[] data, Object maxValueObject) { - this.measureChunkStore = - MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length); - this.measureChunkStore.putData(data); - if (maxValueObject instanceof Long) { - this.maxValue = (long) maxValueObject; - } else { - this.maxValue = (double) maxValueObject; - } - } - - @Override public void freeMemory() { - this.measureChunkStore.freeMemory(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinShort.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinShort.java deleted file mode 100644 index f179bce..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinShort.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression.decimal; - -import java.math.BigDecimal; -import java.nio.ByteBuffer; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; -import org.apache.carbondata.core.datastorage.store.compression.Compressor; -import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder; -import org.apache.carbondata.core.util.ValueCompressionUtil; -import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; - -public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> { - - /** - * Attribute for Carbon LOGGER - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(CompressionMaxMinShort.class.getName()); - - /** - * shortCompressor. - */ - private static Compressor compressor = CompressorFactory.getInstance().getCompressor(); - - private MeasureDataChunkStore<short[]> measureChunkStore; - - /** - * value. - */ - private short[] value; - - private DataType actualDataType; - - private double maxValue; - - public CompressionMaxMinShort(DataType actualDataType) { - this.actualDataType = actualDataType; - } - - @Override public void setValue(short[] value) { - this.value = value; - - } - - @Override - public void uncompress(DataType dataType, byte[] compressedData, int offset, int length, - int decimalPlaces, Object maxValueObject) { - super.unCompress(compressor, dataType, compressedData, offset, length); - setUncompressedValues(value, maxValueObject); - } - - @Override public short[] getValue() {return this.value; } - - @Override public void compress() { - compressedValue = super.compress(compressor, DataType.DATA_SHORT, value); - } - - @Override public void setValueInBytes(byte[] value) { - ByteBuffer buffer = ByteBuffer.wrap(value); - this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length); - } - - @Override public long getLongValue(int index) { - short shortValue = measureChunkStore.getShort(index); - return (long) maxValue - shortValue; - } - - @Override public double getDoubleValue(int index) { - short shortValue = measureChunkStore.getShort(index); - return maxValue - shortValue; - } - - @Override public BigDecimal getBigDecimalValue(int index) { - throw new UnsupportedOperationException( - "Big decimal value is not defined for CompressionMaxMinShort"); - } - - private void setUncompressedValues(short[] data, Object maxValueObject) { - this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE - .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length); - this.measureChunkStore.putData(data); - if (maxValueObject instanceof Long) { - this.maxValue = (long) maxValueObject; - } else { - this.maxValue = (double) maxValueObject; - } - } - - @Override public void freeMemory() { - this.measureChunkStore.freeMemory(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalByte.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalByte.java deleted file mode 100644 index 1b7022b..0000000 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalByte.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.datastorage.store.compression.nondecimal; - -import java.math.BigDecimal; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory; -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; -import org.apache.carbondata.core.datastorage.store.compression.Compressor; -import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder; -import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; - -public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> { - /** - * Attribute for Carbon LOGGER - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(CompressionNonDecimalByte.class.getName()); - - /** - * compressor. - */ - private static Compressor compressor = CompressorFactory.getInstance().getCompressor(); - - /** - * value. - */ - private byte[] value; - - private MeasureDataChunkStore<byte[]> measureChunkStore; - - private double divisionFactory; - - @Override public void setValue(byte[] value) { - this.value = value; - } - - @Override public byte[] getValue() {return this.value; } - - @Override public void compress() { - compressedValue = super.compress(compressor, DataType.DATA_BYTE, value); - } - - @Override - public void uncompress(DataType dataType, byte[] compressedData, - int offset, int length, int decimalPlaces, Object maxValueObject) { - super.unCompress(compressor, dataType, compressedData, offset, length); - setUncompressedValues(value, decimalPlaces); - } - - @Override public void setValueInBytes(byte[] value) { - this.value = value; - } - - @Override public long getLongValue(int index) { - throw new UnsupportedOperationException( - "Long value is not defined for CompressionNonDecimalByte"); - } - - @Override public double getDoubleValue(int index) { - return (measureChunkStore.getByte(index) / this.divisionFactory); - } - - @Override public BigDecimal getBigDecimalValue(int index) { - throw new UnsupportedOperationException( - "Big decimal value is not defined for CompressionNonDecimalByte"); - } - - private void setUncompressedValues(byte[] data, int decimalPlaces) { - this.measureChunkStore = - MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length); - this.measureChunkStore.putData(data); - this.divisionFactory = Math.pow(10, decimalPlaces); - } - - @Override public void freeMemory() { - this.measureChunkStore.freeMemory(); - } -}
