Repository: incubator-carbondata Updated Branches: refs/heads/master 00c479463 -> 05497d0d1
rebased code. fixed review comment. BigDecimal compression Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/63d66264 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/63d66264 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/63d66264 Branch: refs/heads/master Commit: 63d66264cb60338845c87f7f627019bb843844aa Parents: 00c4794 Author: ashok.blend <ashok.bl...@gmail.com> Authored: Sat Dec 3 13:05:38 2016 -0800 Committer: jackylk <jacky.li...@huawei.com> Committed: Tue Dec 13 15:34:38 2016 +0800 ---------------------------------------------------------------------- .../core/compression/BigDecimalCompressor.java | 76 +++++ .../core/compression/BigIntCompressor.java | 26 ++ .../core/compression/ValueCompressor.java | 9 + .../store/compression/WriterCompressModel.java | 11 + .../compression/type/UnCompressBigDecimal.java | 112 +++++++ .../type/UnCompressBigDecimalByte.java | 134 +++++++++ .../store/dataholder/CarbonReadDataHolder.java | 3 + .../store/dataholder/CarbonWriteDataHolder.java | 50 ++++ ...ractHeavyCompressedDoubleArrayDataStore.java | 10 +- .../core/util/BigDecimalCompressionFinder.java | 96 ++++++ .../carbondata/core/util/CompressionFinder.java | 145 +++++++++ .../core/util/ValueCompressionUtil.java | 291 +++++++------------ ...mpressedMeasureChunkFileBasedReaderTest.java | 98 +++---- .../core/util/ValueCompressionUtilTest.java | 2 +- .../store/CarbonFactDataHandlerColumnar.java | 59 +++- 15 files changed, 863 insertions(+), 259 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java new file mode 100644 index 0000000..85c9927 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.compression; + +import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.util.BigDecimalCompressionFinder; +import org.apache.carbondata.core.util.CompressionFinder; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + +/** + * Bigdecimal data type compressor + * + */ +public class BigDecimalCompressor extends BigIntCompressor { + + private boolean readLeft = true; + + @Override + public Object getCompressedValues(CompressionFinder compressionFinder, + CarbonWriteDataHolder dataHolder, Object maxValue, int decimal) { + BigDecimalCompressionFinder bigdCompressionFinder = + (BigDecimalCompressionFinder) compressionFinder; + Long[] maxValues = (Long[]) maxValue; + Object leftCompressedValue = getCompressedValues( + bigdCompressionFinder.getLeftCompType(), dataHolder, + bigdCompressionFinder.getLeftChangedDataType(), maxValues[0], 0); + readLeft = false; + Object rightCompressedValue = getCompressedValues( + bigdCompressionFinder.getRightCompType(), dataHolder, + bigdCompressionFinder.getRightChangedDataType(), maxValues[1], 0); + return new Object[] { leftCompressedValue, rightCompressedValue }; + } + + @Override + protected Object compressMaxMin(DataType changedDataType, + CarbonWriteDataHolder dataHolder, Object max) { + long maxValue = (long) max; + long[][] writableBigDValues = dataHolder.getWritableBigDecimalValues(); + long[] value = null; + if (readLeft) { + value = writableBigDValues[0]; + } else { + value = writableBigDValues[1]; + } + return compressMaxMin(changedDataType, maxValue, value); + } + + @Override + protected Object compressNone(DataType changedDataType, + CarbonWriteDataHolder dataHolder) { + long[][] writableBigDValues = dataHolder.getWritableBigDecimalValues(); + long[] value = null; + if (readLeft) { + value = writableBigDValues[0]; + } else { + value = writableBigDValues[1]; + } + return compressNone(changedDataType, value); + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java index 7b9e52f..315b28f 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java @@ -44,6 +44,19 @@ public class BigIntCompressor extends ValueCompressor { Object max) { long maxValue = (long) max; long[] value = dataHolder.getWritableLongValues(); + return compressMaxMin(changedDataType, maxValue, value); + } + + /** + * 1. It gets delta value i.e difference of maximum value and actual value + * 2. Convert the delta value computed above to changedDatatype + * @param changedDataType + * @param maxValue + * @param value + * @return + */ + protected Object compressMaxMin(DataType changedDataType, long maxValue, + long[] value) { int i = 0; switch (changedDataType) { case DATA_BYTE: @@ -80,6 +93,19 @@ public class BigIntCompressor extends ValueCompressor { @Override protected Object compressNone(DataType changedDataType, CarbonWriteDataHolder dataHolder) { long[] value = dataHolder.getWritableLongValues(); + return compressNone(changedDataType, value); + } + + /** + * It convert the value to changed datatype. + * Changed datatype is computed based list of values it has. + * for instance if value is 2,10,12,45 + * these value can be easily fit in byte and hence below method convert to byte and store it. + * @param changedDataType + * @param value + * @return + */ + protected Object compressNone(DataType changedDataType, long[] value) { int i = 0; switch (changedDataType) { case DATA_BYTE: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java index 370c8d5..9e7862c 100644 --- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java +++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java @@ -19,6 +19,7 @@ package org.apache.carbondata.core.compression; import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.util.CompressionFinder; import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE; import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; @@ -27,6 +28,14 @@ import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; */ public abstract class ValueCompressor { + public Object getCompressedValues(CompressionFinder compressionFinder, + CarbonWriteDataHolder dataHolder, Object maxValue, int decimal) { + return getCompressedValues(compressionFinder.getCompType(), + dataHolder, + compressionFinder.getChangedDataType(), + maxValue, decimal); + } + /** * * @param compType http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java index ac21c8e..a9c2666 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java @@ -19,6 +19,7 @@ package org.apache.carbondata.core.datastorage.store.compression; +import org.apache.carbondata.core.util.CompressionFinder; import org.apache.carbondata.core.util.ValueCompressionUtil; public class WriterCompressModel { @@ -68,6 +69,8 @@ public class WriterCompressModel { */ private ValueCompressonHolder.UnCompressValue[] unCompressValues; + private CompressionFinder[] compressionFinders; + /** * @return the compType */ @@ -218,4 +221,12 @@ public class WriterCompressModel { this.uniqueValue = uniqueValue; } + + public void setCompressionFinders(CompressionFinder[] compressionFinders) { + this.compressionFinders = compressionFinders; + } + + public CompressionFinder[] getCompressionFinder() { + return this.compressionFinders; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimal.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimal.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimal.java new file mode 100644 index 0000000..bf2c11b --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimal.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.datastorage.store.compression.type; + +import java.nio.ByteBuffer; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue; +import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; +import org.apache.carbondata.core.util.BigDecimalCompressionFinder; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + +/** + * Big decimal compression/uncompression + */ +public class UnCompressBigDecimal<T> implements UnCompressValue<T> { + + private BigDecimalCompressionFinder compressionFinder; + + /** + * leftPart before decimal + */ + private UnCompressValue leftPart; + + /** + * rightPart after decimal + */ + private UnCompressValue rightPart; + + public UnCompressBigDecimal(BigDecimalCompressionFinder compressionFinder, + UnCompressValue leftPart, UnCompressValue rightPart) { + this.compressionFinder = compressionFinder; + this.leftPart = leftPart; + this.rightPart = rightPart; + } + + @Override + public void setValue(T value) { + Object[] values = (Object[]) value; + leftPart.setValue(values[0]); + rightPart.setValue(values[1]); + } + + @Override + public void setValueInBytes(byte[] value) { + // TODO Auto-generated method stub + + } + + @Override + public UnCompressValue<T> getNew() { + UnCompressValue leftUnCompressClone = leftPart.getNew(); + UnCompressValue rightUnCompressClone = rightPart.getNew(); + return new UnCompressBigDecimal(compressionFinder, leftUnCompressClone, + rightUnCompressClone); + } + + @Override + public UnCompressValue compress() { + UnCompressBigDecimal byt = new UnCompressBigDecimal<>(compressionFinder, + leftPart.compress(), rightPart.compress()); + return byt; + } + + @Override + public UnCompressValue uncompress(DataType dataType) { + // TODO Auto-generated method stub + return null; + } + + @Override + public byte[] getBackArrayData() { + byte[] leftdata = leftPart.getBackArrayData(); + byte[] rightdata = rightPart.getBackArrayData(); + ByteBuffer byteBuffer = ByteBuffer + .allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + leftdata.length + + rightdata.length); + byteBuffer.putInt(leftdata.length); + byteBuffer.put(leftdata); + byteBuffer.put(rightdata); + byteBuffer.flip(); + return byteBuffer.array(); + } + + @Override + public UnCompressValue getCompressorObject() { + return new UnCompressBigDecimalByte<>(compressionFinder, + leftPart.getCompressorObject(), rightPart.getCompressorObject()); + } + + @Override + public CarbonReadDataHolder getValues(int decimal, Object maxValue) { + // TODO Auto-generated method stub + return null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimalByte.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimalByte.java new file mode 100644 index 0000000..97e0d38 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimalByte.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.datastorage.store.compression.type; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; +import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue; +import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; +import org.apache.carbondata.core.util.BigDecimalCompressionFinder; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + +/** + * Big decimal compression/uncompression + */ +public class UnCompressBigDecimalByte<T> implements UnCompressValue<T> { + + private BigDecimalCompressionFinder compressionFinder; + + private UnCompressValue leftPart; + + private UnCompressValue rightPart; + + public UnCompressBigDecimalByte( + BigDecimalCompressionFinder compressionFinder, UnCompressValue leftPart, + UnCompressValue rightPart) { + this.compressionFinder = compressionFinder; + this.leftPart = leftPart; + this.rightPart = rightPart; + } + + @Override + public void setValue(T value) { + byte[] values = (byte[]) value; + ByteBuffer buffer = ByteBuffer.wrap(values); + buffer.rewind(); + int leftPartLen = buffer.getInt(); + int rightPartLen = values.length - leftPartLen + - CarbonCommonConstants.INT_SIZE_IN_BYTE; + byte[] leftValue = new byte[leftPartLen]; + byte[] rightValue = new byte[rightPartLen]; + buffer.get(leftValue); + buffer.get(rightValue); + leftPart.setValue(leftValue); + rightPart.setValue(rightValue); + } + + @Override + public void setValueInBytes(byte[] value) { + // TODO Auto-generated method stub + + } + + @Override + public UnCompressValue<T> getNew() { + UnCompressValue leftUnCompressClone = leftPart.getNew(); + UnCompressValue rightUnCompressClone = rightPart.getNew(); + return new UnCompressBigDecimal(compressionFinder, leftUnCompressClone, + rightUnCompressClone); + } + + @Override + public UnCompressValue compress() { + UnCompressBigDecimal byt = new UnCompressBigDecimal<>(compressionFinder, + leftPart.compress(), rightPart.compress()); + return byt; + } + + @Override + public UnCompressValue uncompress(DataType dataType) { + ValueCompressonHolder.UnCompressValue left = leftPart + .uncompress(compressionFinder.getLeftChangedDataType()); + ValueCompressonHolder.UnCompressValue right = rightPart + .uncompress(compressionFinder.getRightChangedDataType()); + return new UnCompressBigDecimalByte<>(compressionFinder, left, right); + } + + @Override + public byte[] getBackArrayData() { + return null; + } + + @Override + public UnCompressValue getCompressorObject() { + return new UnCompressBigDecimal<>(compressionFinder, + leftPart.getCompressorObject(), rightPart.getCompressorObject()); + } + + @Override + public CarbonReadDataHolder getValues(int decimal, Object maxValue) { + Long[] maxValues = (Long[]) maxValue; + CarbonReadDataHolder dataHolder = new CarbonReadDataHolder(); + CarbonReadDataHolder leftDataHolder = leftPart.getValues(decimal, + maxValues[0]); + long[] leftVals = leftDataHolder.getReadableLongValue(); + int size = leftVals.length; + long[] rightVals = new long[size]; + if (decimal > 0) { + CarbonReadDataHolder rightDataHolder = rightPart.getValues(decimal, + maxValues[1]); + rightVals = rightDataHolder.getReadableLongValue(); + } + BigDecimal[] values = new BigDecimal[size]; + for (int i = 0; i < size; i++) { + String decimalPart = Double.toString(rightVals[i]/Math.pow(10, decimal)); + String bigdStr = Long.toString(leftVals[i]) + + CarbonCommonConstants.POINT + + decimalPart.substring(decimalPart.indexOf(".")+1, decimalPart.length()); + BigDecimal bigdVal = new BigDecimal(bigdStr); + values[i] = bigdVal; + } + dataHolder.setReadableBigDecimalValues(values); + return dataHolder; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java index 1cfc7a2..7bf3dbe 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java @@ -72,6 +72,9 @@ public class CarbonReadDataHolder { return this.intValues[index]; } + public long[] getReadableLongValue() { + return this.longValues; + } public long getReadableLongValueByIndex(int index) { return this.longValues[index]; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonWriteDataHolder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonWriteDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonWriteDataHolder.java index 08f7786..c87b266 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonWriteDataHolder.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonWriteDataHolder.java @@ -31,6 +31,15 @@ public class CarbonWriteDataHolder { private long[] longValues; /** + * bigDecimal left part + */ + private long[] bigDecimalLeftValues; + + /** + * bigDecimal right part + */ + private long[] bigDecimalRightValues; + /** * byteValues */ private byte[][] byteValues; @@ -120,6 +129,14 @@ public class CarbonWriteDataHolder { longValues = new long[size]; } + public void initialiseBigDecimalValues(int size) { + if (size < 1) { + throw new IllegalArgumentException("Invalid array size"); + } + bigDecimalLeftValues = new long[size]; + bigDecimalRightValues = new long[size]; + } + /** * set double value by index * @@ -143,6 +160,17 @@ public class CarbonWriteDataHolder { } /** + * set bigdecimal value by index + * + * @param index + * @param value + */ + public void setWritableBigDecimalValueByIndex(int index, long[] value) { + bigDecimalLeftValues[index] = value[0]; + bigDecimalRightValues[index] = value[1]; + size++; + } + /** * set byte array value by index * * @param index @@ -227,4 +255,26 @@ public class CarbonWriteDataHolder { } return longValues; } + + /** + * Get Writable bigdecimal Values + * + * @return + */ + public long[][] getWritableBigDecimalValues() { + long[][] bigDecimalValues = new long[2][]; + if (size < bigDecimalLeftValues.length) { + long[] temp = new long[size]; + System.arraycopy(bigDecimalLeftValues, 0, temp, 0, size); + bigDecimalLeftValues = temp; + } + if (size < bigDecimalRightValues.length) { + long[] temp = new long[size]; + System.arraycopy(bigDecimalRightValues, 0, temp, 0, size); + bigDecimalRightValues = temp; + } + bigDecimalValues[0]= bigDecimalLeftValues; + bigDecimalValues[1] = bigDecimalRightValues; + return bigDecimalValues; + } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java index 9b6ce9f..3adc888 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java @@ -62,13 +62,11 @@ public abstract class AbstractHeavyCompressedDoubleArrayDataStore @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) { for (int i = 0; i < compressionModel.getUnCompressValues().length; i++) { values[i] = compressionModel.getUnCompressValues()[i].getNew(); - if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE - && type[i] != CarbonCommonConstants.BIG_DECIMAL_MEASURE) { - + if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) { values[i].setValue( - ValueCompressionUtil.getValueCompressor(compressionModel.getActualDataType()[i]) - .getCompressedValues(compressionModel.getCompType()[i], dataHolder[i], - compressionModel.getChangedDataType()[i], compressionModel.getMaxValue()[i], + ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinder()[i]) + .getCompressedValues(compressionModel.getCompressionFinder()[i], dataHolder[i], + compressionModel.getMaxValue()[i], compressionModel.getMantissa()[i])); } else { values[i].setValue(dataHolder[i].getWritableByteArrayValues()); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/util/BigDecimalCompressionFinder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/BigDecimalCompressionFinder.java b/core/src/main/java/org/apache/carbondata/core/util/BigDecimalCompressionFinder.java new file mode 100644 index 0000000..9a7ae13 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/util/BigDecimalCompressionFinder.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.util; + +import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + +public class BigDecimalCompressionFinder extends CompressionFinder { + + /** + * non decimal part compression type + */ + private COMPRESSION_TYPE leftCompType; + + /** + * decimal part compression type + */ + private COMPRESSION_TYPE rightCompType; + + /** + * non decimal actual data type + */ + private DataType leftActualDataType; + + /** + * decimal actual data type + */ + private DataType rightActualDataType; + + /** + * non decimal changed data type + */ + private DataType leftChangedDataType; + + /** + * decimal changed data type + */ + private DataType rightChangedDataType; + + public BigDecimalCompressionFinder(COMPRESSION_TYPE compType, + DataType actualDataType, DataType changedDataType, char measureStoreType) { + super(compType, actualDataType, changedDataType, measureStoreType); + } + + public BigDecimalCompressionFinder(COMPRESSION_TYPE[] compType, + DataType[] actualDataType, DataType[] changedDataType, char measureStoreType) { + super(null, null, null, measureStoreType); + this.leftCompType = compType[0]; + this.rightCompType = compType[1]; + this.leftActualDataType = actualDataType[0]; + this.rightActualDataType = actualDataType[1]; + this.leftChangedDataType = changedDataType[0]; + this.rightChangedDataType = changedDataType[1]; + } + + public COMPRESSION_TYPE getLeftCompType() { + return leftCompType; + } + + public COMPRESSION_TYPE getRightCompType() { + return rightCompType; + } + + public DataType getLeftActualDataType() { + return leftActualDataType; + } + + public DataType getRightActualDataType() { + return rightActualDataType; + } + + public DataType getLeftChangedDataType() { + return leftChangedDataType; + } + + public DataType getRightChangedDataType() { + return rightChangedDataType; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java new file mode 100644 index 0000000..b642a20 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java @@ -0,0 +1,145 @@ +package org.apache.carbondata.core.util; + +import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + + +/** + * through the size of data type,priority and compression type, select the + * best compression type + */ +public class CompressionFinder implements Comparable<CompressionFinder> { + private COMPRESSION_TYPE compType; + + private DataType actualDataType; + + private DataType changedDataType; + /** + * the size of changed data + */ + private int size; + + private PRIORITY priority; + + private char measureStoreType; + + /** + * CompressionFinder constructor. + * + * @param compType + * @param actualDataType + * @param changedDataType + */ + CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType, + DataType changedDataType, char measureStoreType) { + super(); + this.compType = compType; + this.actualDataType = actualDataType; + this.changedDataType = changedDataType; + this.measureStoreType = measureStoreType; + } + + /** + * CompressionFinder overloaded constructor. + * + * @param compType + * @param actualDataType + * @param changedDataType + * @param priority + */ + + CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType, DataType changedDataType, + PRIORITY priority, char measureStoreType) { + super(); + this.actualDataType = actualDataType; + this.changedDataType = changedDataType; + this.size = ValueCompressionUtil.getSize(changedDataType); + this.priority = priority; + this.compType = compType; + this.measureStoreType = measureStoreType; + } + + @Override public boolean equals(Object obj) { + boolean equals = false; + if (obj instanceof CompressionFinder) { + CompressionFinder cf = (CompressionFinder) obj; + + if (this.size == cf.size && this.priority == cf.priority) { + equals = true; + } + + } + return equals; + } + + @Override public int hashCode() { + final int code = 31; + int result = 1; + + result = code * result + this.size; + result = code * result + ((priority == null) ? 0 : priority.hashCode()); + return result; + } + + @Override public int compareTo(CompressionFinder o) { + int returnVal = 0; + // the big size have high priority + if (this.equals(o)) { + returnVal = 0; + } else if (this.size == o.size) { + // the compression type priority + if (priority.priority > o.priority.priority) { + returnVal = 1; + } else if (priority.priority < o.priority.priority) { + returnVal = -1; + } + + } else if (this.size > o.size) { + returnVal = 1; + } else { + returnVal = -1; + } + return returnVal; + } + + /** + * Compression type priority. + * ACTUAL is the highest priority and DIFFNONDECIMAL is the lowest + * priority + */ + enum PRIORITY { + ACTUAL(0), + DIFFSIZE(1), + MAXNONDECIMAL(2), + DIFFNONDECIMAL(3); + private int priority; + + private PRIORITY(int priority) { + this.priority = priority; + } + } + + public COMPRESSION_TYPE getCompType() { + return compType; + } + + public DataType getActualDataType() { + return actualDataType; + } + + public DataType getChangedDataType() { + return changedDataType; + } + + public int getSize() { + return size; + } + + public PRIORITY getPriority() { + return priority; + } + + public char getMeasureStoreType() { + return measureStoreType; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java index d01b30a..6210c30 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java @@ -23,12 +23,14 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.Arrays; +import org.apache.carbondata.core.compression.BigDecimalCompressor; import org.apache.carbondata.core.compression.BigIntCompressor; import org.apache.carbondata.core.compression.DoubleCompressor; import org.apache.carbondata.core.compression.ValueCompressor; import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel; import org.apache.carbondata.core.datastorage.store.compression.ReaderCompressModel; import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; +import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue; import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressByteArray; import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinByte; @@ -55,6 +57,7 @@ import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressN import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneInt; import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneLong; import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneShort; +import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressBigDecimal; import org.apache.carbondata.core.metadata.ValueEncoderMeta; public final class ValueCompressionUtil { @@ -138,19 +141,21 @@ public final class ValueCompressionUtil { // ''b' for decimal, 'l' for long, 'n' for double switch (measureStoreType) { case 'b': - return new CompressionFinder(COMPRESSION_TYPE.BIGDECIMAL, DataType.DATA_BYTE, - DataType.DATA_BYTE); + return getBigDecimalCompressorFinder(maxValue, minValue, 0, dataTypeSelected, + measureStoreType); case 'l': - return getLongCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected); + return getLongCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected, + measureStoreType); case 'n': - return getDoubleCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected); + return getDoubleCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected, + measureStoreType); default: throw new IllegalArgumentException("unsupported measure type"); } } private static CompressionFinder getDoubleCompressorFinder(Object maxValue, Object minValue, - int mantissa, byte dataTypeSelected) { + int mantissa, byte dataTypeSelected, char measureStoreType) { //Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1 is max, //but we can't use -1 to getDatatype, we should use -10000000. double absMaxValue = Math.abs((double) maxValue) >= Math.abs((double) minValue) ? @@ -165,13 +170,13 @@ public final class ValueCompressionUtil { int deltaSize = getSize(deltaDataType); if (adaptiveSize > deltaSize) { return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DATA_DOUBLE, - deltaDataType); + deltaDataType, measureStoreType); } else if (adaptiveSize < deltaSize) { return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_DOUBLE, - deltaDataType); + deltaDataType, measureStoreType); } else { return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_DOUBLE, - adaptiveDataType); + adaptiveDataType, measureStoreType); } } else { // double @@ -183,21 +188,47 @@ public final class ValueCompressionUtil { CompressionFinder[] finders = new CompressionFinder[] { new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, adaptiveDataType, adaptiveDataType, - CompressionFinder.PRIORITY.ACTUAL), + CompressionFinder.PRIORITY.ACTUAL, measureStoreType), new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, adaptiveDataType, deltaDataType, - CompressionFinder.PRIORITY.DIFFSIZE), + CompressionFinder.PRIORITY.DIFFSIZE, measureStoreType), new CompressionFinder(COMPRESSION_TYPE.BIGINT, adaptiveDataType, maxNonDecDataType, - CompressionFinder.PRIORITY.MAXNONDECIMAL), + CompressionFinder.PRIORITY.MAXNONDECIMAL, measureStoreType), new CompressionFinder(COMPRESSION_TYPE.DELTA_NON_DECIMAL, adaptiveDataType, - diffNonDecDataType, CompressionFinder.PRIORITY.DIFFNONDECIMAL) }; + diffNonDecDataType, CompressionFinder.PRIORITY.DIFFNONDECIMAL, measureStoreType) }; // sort the compressionFinder.The top have the highest priority Arrays.sort(finders); return finders[0]; } } + private static CompressionFinder getBigDecimalCompressorFinder( + Object maxValue, Object minValue, int mantissa, + byte dataTypeSelected, char measureStoreType) { + Long[] maxValues = (Long[])maxValue; + Long[] minValues = (Long[])minValue; + CompressionFinder leftCompressionFinder = getLongCompressorFinder(maxValues[0], minValues[0], + mantissa, dataTypeSelected, measureStoreType); + CompressionFinder rightCompressionFinder = getLongCompressorFinder(maxValues[1], minValues[1], + mantissa, dataTypeSelected, measureStoreType); + COMPRESSION_TYPE[] compressionTypes = new COMPRESSION_TYPE[2]; + DataType[] actualDataTypes = new DataType[2]; + DataType[] changedDataTypes = new DataType[2]; + compressionTypes[0] = leftCompressionFinder.getCompType(); + compressionTypes[1] = rightCompressionFinder.getCompType(); + + actualDataTypes[0] = leftCompressionFinder.getActualDataType(); + actualDataTypes[1] = rightCompressionFinder.getActualDataType(); + + changedDataTypes[0] = leftCompressionFinder.getChangedDataType(); + changedDataTypes[1] = rightCompressionFinder.getChangedDataType(); + + CompressionFinder bigdCompressionFinder = new BigDecimalCompressionFinder( + compressionTypes, actualDataTypes, changedDataTypes, measureStoreType); + return bigdCompressionFinder; + } + private static CompressionFinder getLongCompressorFinder(Object maxValue, Object minValue, - int mantissa, byte dataTypeSelected) { + int mantissa, byte dataTypeSelected, char measureStoreType) { DataType adaptiveDataType = getDataType((long) maxValue, mantissa, dataTypeSelected); int adaptiveSize = getSize(adaptiveDataType); DataType deltaDataType = getDataType((long) maxValue - (long) minValue, mantissa, @@ -205,13 +236,13 @@ public final class ValueCompressionUtil { int deltaSize = getSize(deltaDataType); if (adaptiveSize > deltaSize) { return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DATA_BIGINT, - deltaDataType); + deltaDataType, measureStoreType); } else if (adaptiveSize < deltaSize) { return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_BIGINT, - deltaDataType); + deltaDataType, measureStoreType); } else { return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_BIGINT, - adaptiveDataType); + adaptiveDataType, measureStoreType); } } @@ -240,49 +271,62 @@ public final class ValueCompressionUtil { /** * It returns Compressor for given datatype - * @param actualDataType + * @param msrType * @return compressor based on actualdatatype */ - public static ValueCompressor getValueCompressor(DataType actualDataType) { - switch (actualDataType) { - case DATA_BIGINT: - return new BigIntCompressor(); - default: - return new DoubleCompressor(); + public static ValueCompressor getValueCompressor(CompressionFinder compressorFinder) { + switch(compressorFinder.getMeasureStoreType()) { + case 'b': return new BigDecimalCompressor(); + case 'l': return new BigIntCompressor(); + default : return new DoubleCompressor(); } } - private static ValueCompressonHolder.UnCompressValue[] getUncompressedValues( - COMPRESSION_TYPE[] compType, DataType[] actualDataType, DataType[] changedDataType) { - - ValueCompressonHolder.UnCompressValue[] compressValue = - new ValueCompressonHolder.UnCompressValue[changedDataType.length]; - for (int i = 0; i < changedDataType.length; i++) { - switch (compType[i]) { - case ADAPTIVE: - compressValue[i] = getUnCompressNone(changedDataType[i], actualDataType[i]); - break; - - case DELTA_DOUBLE: - compressValue[i] = getUnCompressDecimalMaxMin(changedDataType[i], actualDataType[i]); - break; - - case BIGINT: - compressValue[i] = getUnCompressNonDecimal(changedDataType[i]); - break; - - case BIGDECIMAL: - compressValue[i] = new UnCompressByteArray(UnCompressByteArray.ByteArrayType.BIG_DECIMAL); - break; + /** + * get uncompressed object + * @param compressionFinders : Compression types for measures + * @return + */ + private static UnCompressValue[] getUncompressedValues( + CompressionFinder[] compressionFinders) { + UnCompressValue[] unCompressedValues = new UnCompressValue[compressionFinders.length]; + for (int i=0; i< compressionFinders.length; i++) { + unCompressedValues[i] = getUncompressedValue(compressionFinders[i]); + } + return unCompressedValues; + } + /** + * + * @param compressionFinder bigdecimal compression finder + * @return Uncompressed measure object + */ + private static UnCompressValue getUncompressedValue( + BigDecimalCompressionFinder compressionFinder) { + UnCompressValue leftPart = getUncompressedValue(compressionFinder.getLeftCompType(), + compressionFinder.getLeftActualDataType(), compressionFinder.getLeftChangedDataType()); + UnCompressValue rightPart = getUncompressedValue(compressionFinder.getRightCompType(), + compressionFinder.getRightActualDataType(), compressionFinder.getRightChangedDataType()); + return new UnCompressBigDecimal<>(compressionFinder, leftPart, rightPart); + } - default: - compressValue[i] = getUnCompressNonDecimalMaxMin(changedDataType[i]); - } + /** + * + * @param compressionFinder for measure other then bigdecimal + * @return + */ + private static UnCompressValue getUncompressedValue( + CompressionFinder compressionFinder) { + switch(compressionFinder.getMeasureStoreType()) { + case 'b': + return getUncompressedValue( + (BigDecimalCompressionFinder) compressionFinder); + default: + return getUncompressedValue(compressionFinder.getCompType(), + compressionFinder.getActualDataType(), compressionFinder.getChangedDataType()); } - return compressValue; } - private static ValueCompressonHolder.UnCompressValue getUncompressedValues( + private static ValueCompressonHolder.UnCompressValue getUncompressedValue( COMPRESSION_TYPE compType, DataType actualDataType, DataType changedDataType) { switch (compType) { case ADAPTIVE: @@ -685,14 +729,17 @@ public final class ValueCompressionUtil { DataType[] actualType = new DataType[measureCount]; DataType[] changedType = new DataType[measureCount]; COMPRESSION_TYPE[] compType = new COMPRESSION_TYPE[measureCount]; + CompressionFinder[] compressionFinders = new CompressionFinder[measureCount]; for (int i = 0; i < measureCount; i++) { CompressionFinder compresssionFinder = ValueCompressionUtil.getCompressionFinder(maxValue[i], minValue[i], mantissa[i], type[i], dataTypeSelected[i]); - actualType[i] = compresssionFinder.actualDataType; - changedType[i] = compresssionFinder.changedDataType; - compType[i] = compresssionFinder.compType; + compressionFinders[i] = compresssionFinder; + actualType[i] = compresssionFinder.getActualDataType(); + changedType[i] = compresssionFinder.getChangedDataType(); + compType[i] = compresssionFinder.getCompType(); } + compressionModel.setCompressionFinders(compressionFinders); compressionModel.setMaxValue(maxValue); compressionModel.setMantissa(mantissa); compressionModel.setChangedDataType(changedType); @@ -703,8 +750,7 @@ public final class ValueCompressionUtil { compressionModel.setType(type); compressionModel.setDataTypeSelected(dataTypeSelected); ValueCompressonHolder.UnCompressValue[] values = ValueCompressionUtil - .getUncompressedValues(compressionModel.getCompType(), compressionModel.getActualDataType(), - compressionModel.getChangedDataType()); + .getUncompressedValues(compressionFinders); compressionModel.setUnCompressValues(values); return compressionModel; } @@ -718,11 +764,8 @@ public final class ValueCompressionUtil { getCompressionFinder(meta.getMaxValue(), meta.getMinValue(), meta.getMantissa(), meta.getType(), meta.getDataTypeSelected()); compressModel.setUnCompressValues( - ValueCompressionUtil.getUncompressedValues( - compressFinder.compType, - compressFinder.actualDataType, - compressFinder.changedDataType)); - compressModel.setChangedDataType(compressFinder.changedDataType); + ValueCompressionUtil.getUncompressedValue(compressFinder)); + compressModel.setChangedDataType(compressFinder.getChangedDataType()); compressModel.setValueEncoderMeta(meta); return compressModel; } @@ -858,133 +901,5 @@ public final class ValueCompressionUtil { DATA_DOUBLE(); private DataType() { } - - } - - /** - * through the size of data type,priority and compression type, select the - * best compression type - */ - private static class CompressionFinder implements Comparable<CompressionFinder> { - - private COMPRESSION_TYPE compType; - - private DataType actualDataType; - - private DataType changedDataType; - /** - * the size of changed data - */ - private int size; - - private PRIORITY priority; - - /** - * CompressionFinder constructor. - * - * @param compType - * @param actualDataType - * @param changedDataType - */ - CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType, - DataType changedDataType) { - super(); - this.compType = compType; - this.actualDataType = actualDataType; - this.changedDataType = changedDataType; - } - - /** - * CompressionFinder overloaded constructor. - * - * @param compType - * @param actualDataType - * @param changedDataType - * @param priority - */ - - CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType, DataType changedDataType, - PRIORITY priority) { - super(); - this.actualDataType = actualDataType; - this.changedDataType = changedDataType; - this.size = getSize(changedDataType); - this.priority = priority; - this.compType = compType; - } - - @Override public boolean equals(Object obj) { - boolean equals = false; - if (obj instanceof CompressionFinder) { - CompressionFinder cf = (CompressionFinder) obj; - - if (this.size == cf.size && this.priority == cf.priority) { - equals = true; - } - - } - return equals; - } - - @Override public int hashCode() { - final int code = 31; - int result = 1; - - result = code * result + this.size; - result = code * result + ((priority == null) ? 0 : priority.hashCode()); - return result; - } - - @Override public int compareTo(CompressionFinder o) { - int returnVal = 0; - // the big size have high priority - if (this.equals(o)) { - returnVal = 0; - } else if (this.size == o.size) { - // the compression type priority - if (priority.priority > o.priority.priority) { - returnVal = 1; - } else if (priority.priority < o.priority.priority) { - returnVal = -1; - } - - } else if (this.size > o.size) { - returnVal = 1; - } else { - returnVal = -1; - } - return returnVal; - } - - /** - * Compression type priority. - * ACTUAL is the highest priority and DIFFNONDECIMAL is the lowest - * priority - */ - enum PRIORITY { - /** - * - */ - ACTUAL(0), /** - * - */ - DIFFSIZE(1), /** - * - */ - MAXNONDECIMAL(2), /** - * - */ - DIFFNONDECIMAL(3); - - /** - * priority. - */ - private int priority; - - private PRIORITY(int priority) { - this.priority = priority; - } - } } - } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java index 36c8ec3..c2a5d5d 100644 --- a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java +++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java @@ -2,6 +2,7 @@ package org.apache.carbondata.core.carbon.datastore.chunk.reader.measure; import static junit.framework.TestCase.assertEquals; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -13,10 +14,10 @@ import org.apache.carbondata.core.carbon.datastore.chunk.reader.measure.v1.Compr import org.apache.carbondata.core.carbon.metadata.blocklet.BlockletInfo; import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk; import org.apache.carbondata.core.datastorage.store.FileHolder; +import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel; import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; -import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressByteArray; -import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder; +import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; +import org.apache.carbondata.core.datastorage.util.StoreFactory; import org.apache.carbondata.core.metadata.ValueEncoderMeta; import org.apache.carbondata.core.util.ValueCompressionUtil; import org.junit.BeforeClass; @@ -25,28 +26,27 @@ import org.junit.Test; public class CompressedMeasureChunkFileBasedReaderTest { static CompressedMeasureChunkFileBasedReaderV1 compressedMeasureChunkFileBasedReader; + static CarbonWriteDataHolder[] dataHolder = new CarbonWriteDataHolder[1]; + static WriterCompressModel writerCompressModel; @BeforeClass public static void setup() { List<DataChunk> dataChunkList = new ArrayList<>(); dataChunkList.add(new DataChunk()); - WriterCompressModel writerCompressModel = new WriterCompressModel(); + writerCompressModel = new WriterCompressModel(); + Object maxValue[] = new Object[]{new Long[]{8L, 0L}}; + Object minValue[] = new Object[]{new Long[]{1L,0L}}; + byte[] dataTypeSelected = new byte[1]; + char[] aggType = new char[]{'b'}; + MeasureMetaDataModel measureMDMdl = + new MeasureMetaDataModel(minValue, maxValue, new int[]{1}, maxValue.length, null, + aggType, dataTypeSelected); + writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMDMdl); + - ValueCompressonHolder.UnCompressValue unCompressValue[] = - { new UnCompressByteArray(UnCompressByteArray.ByteArrayType.BYTE_ARRAY) }; - byte valueInByte[] = { 1, 5, 4, 8, 7 }; - unCompressValue[0].setValueInBytes(valueInByte); - ValueCompressionUtil.DataType dataType[] = { ValueCompressionUtil.DataType.DATA_BYTE }; - - writerCompressModel.setUnCompressValues(unCompressValue); - writerCompressModel.setChangedDataType(dataType); - int decimal[] = { 5, 8, 2 }; - writerCompressModel.setMantissa(decimal); - Object maxValue[] = { 8 }; - writerCompressModel.setMaxValue(maxValue); ValueEncoderMeta meta = new ValueEncoderMeta(); - meta.setMaxValue(8.0); - meta.setMinValue(1.0); + meta.setMaxValue(new Long[]{8L,0L}); + meta.setMinValue(new Long[]{1L,0L}); meta.setMantissa(1); meta.setType('b'); List<ValueEncoderMeta> valueEncoderMetaList = new ArrayList<>(); @@ -61,63 +61,45 @@ public class CompressedMeasureChunkFileBasedReaderTest { @Test public void readMeasureChunkTest() { FileHolder fileHolder = new MockUp<FileHolder>() { @Mock public byte[] readByteArray(String filePath, long offset, int length) { - byte mockedValue[] = { 1, 5, 4, 8, 7 }; - return mockedValue; + dataHolder[0] = new CarbonWriteDataHolder(); + dataHolder[0].initialiseBigDecimalValues(1); + dataHolder[0].setWritableBigDecimalValueByIndex(0, new long[]{2L,1L}); + byte[][] writableMeasureDataArray = + StoreFactory.createDataStore(writerCompressModel).getWritableMeasureDataArray(dataHolder) + .clone(); + return writableMeasureDataArray[0]; } }.getMockInstance(); - new MockUp<UnCompressByteArray>() { - @Mock public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { - List<byte[]> valsList = new ArrayList<byte[]>(); - byte mockedValue[] = { 3, 7, 9 }; - valsList.add(mockedValue); - CarbonReadDataHolder holder = new CarbonReadDataHolder(); - byte[][] value = new byte[valsList.size()][]; - valsList.toArray(value); - holder.setReadableByteValues(value); - return holder; - } - }; - MeasureColumnDataChunk measureColumnDataChunks = compressedMeasureChunkFileBasedReader.readMeasureChunk(fileHolder, 0); - byte expectedValue[] = { 3, 7, 9 }; - for (int i = 0; i < 3; i++) { - assertEquals(expectedValue[i], - measureColumnDataChunks.getMeasureDataHolder().getReadableByteArrayValueByIndex(0)[i]); - } + BigDecimal bigD = new BigDecimal("2.1"); + assertEquals(bigD, + measureColumnDataChunks.getMeasureDataHolder().getReadableBigDecimalValueByIndex(0)); + } @Test public void readMeasureChunksTest() { FileHolder fileHolder = new MockUp<FileHolder>() { @Mock public byte[] readByteArray(String filePath, long offset, int length) { - byte mockedValue[] = { 1, 5, 4, 8, 7 }; - return mockedValue; + dataHolder[0] = new CarbonWriteDataHolder(); + dataHolder[0].initialiseBigDecimalValues(1); + dataHolder[0].setWritableBigDecimalValueByIndex(0, new long[]{2L,1L}); + byte[][] writableMeasureDataArray = + StoreFactory.createDataStore(writerCompressModel).getWritableMeasureDataArray(dataHolder) + .clone(); + return writableMeasureDataArray[0]; } }.getMockInstance(); - new MockUp<UnCompressByteArray>() { - @Mock public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) { - List<byte[]> valsList = new ArrayList<byte[]>(); - byte mockedValue[] = { 3, 7, 9 }; - valsList.add(mockedValue); - CarbonReadDataHolder holder = new CarbonReadDataHolder(); - byte[][] value = new byte[valsList.size()][]; - valsList.toArray(value); - holder.setReadableByteValues(value); - return holder; - } - }; - int[][] blockIndexes = {{0,0}}; MeasureColumnDataChunk measureColumnDataChunks[] = compressedMeasureChunkFileBasedReader.readMeasureChunks(fileHolder, blockIndexes); - byte expectedValue[] = { 3, 7, 9 }; - for (int i = 0; i < 3; i++) { - assertEquals(expectedValue[i], - measureColumnDataChunks[0].getMeasureDataHolder().getReadableByteArrayValueByIndex(0)[i]); - } + BigDecimal bigD = new BigDecimal("2.1"); + assertEquals(bigD, + measureColumnDataChunks[0].getMeasureDataHolder().getReadableBigDecimalValueByIndex(0)); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java index a1bb6f0..739670c 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java @@ -590,7 +590,7 @@ public class ValueCompressionUtilTest { Object[] maxValues = { 10l, 20l, 30l }; Object[] minValues = { 1l, 2l, 3l }; int[] decimalLength = { 0, 0, 0 }; - Object[] uniqueValues = { 5, 3, 2l }; + Object[] uniqueValues = { 5, new Long[]{2l,4l}, 2l }; char[] types = { 'l', 'l', 'l' }; byte[] dataTypeSelected = { 1, 2, 4 }; MeasureMetaDataModel measureMetaDataModel = http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d66264/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java index 0398cd7..cde19bd 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java @@ -493,7 +493,11 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { } else if (type[i] == CarbonCommonConstants.SUM_COUNT_VALUE_MEASURE) { max[i] = -Double.MAX_VALUE; } else if (type[i] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) { - max[i] = new BigDecimal(0.0); + //max[i] = new BigDecimal(0.0); + Long[] bigdMinVal = new Long[2]; + bigdMinVal[0] = Long.MIN_VALUE; + bigdMinVal[1] = Long.MIN_VALUE; + max[i] = bigdMinVal; } else { max[i] = 0.0; } @@ -506,8 +510,16 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { min[i] = Double.MAX_VALUE; uniqueValue[i] = Double.MIN_VALUE; } else if (type[i] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) { - min[i] = new BigDecimal(Double.MAX_VALUE); + Long[] bigdMaxVal = new Long[2]; + bigdMaxVal[0] = Long.MAX_VALUE; + bigdMaxVal[1] = Long.MAX_VALUE; + //min[i] = new BigDecimal(Double.MAX_VALUE); + min[i] = bigdMaxVal; uniqueValue[i] = new BigDecimal(Double.MIN_VALUE); + Long[] bigdUniqueVal = new Long[2]; + bigdUniqueVal[0] = Long.MIN_VALUE; + bigdUniqueVal[1] = Long.MIN_VALUE; + uniqueValue[i] = bigdUniqueVal; } else { min[i] = 0.0; uniqueValue[i] = 0.0; @@ -588,11 +600,23 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { b = (byte[]) row[customMeasureIndex[i]]; } } + BigDecimal value = DataTypeUtil.byteToBigDecimal(b); + String[] bigdVals = value.toPlainString().split("\\."); + long[] bigDvalue = new long[2]; + if (bigdVals.length == 2) { + bigDvalue[0] = Long.parseLong(bigdVals[0]); + BigDecimal bd = new BigDecimal(CarbonCommonConstants.POINT+bigdVals[1]); + bigDvalue[1] = (long)(bd.doubleValue()*Math.pow(10, value.scale())); + //bigDvalue[1] = Long.parseLong(bigdVals[1]); + } else { + bigDvalue[0] = Long.parseLong(bigdVals[0]); + } byteBuffer = ByteBuffer.allocate(b.length + CarbonCommonConstants.INT_SIZE_IN_BYTE); byteBuffer.putInt(b.length); byteBuffer.put(b); byteBuffer.flip(); b = byteBuffer.array(); + dataHolder[customMeasureIndex[i]].setWritableBigDecimalValueByIndex(count, bigDvalue); dataHolder[customMeasureIndex[i]].setWritableByteArrayValueByIndex(count, b); } calculateMaxMin(max, min, decimal, customMeasureIndex, row); @@ -1207,8 +1231,28 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { buff = (byte[]) row[count]; } BigDecimal value = DataTypeUtil.byteToBigDecimal(buff); - BigDecimal minVal = (BigDecimal) min[count]; - min[count] = minVal.min(value); + decimal[count] = value.scale(); + String[] bigdVals = value.toPlainString().split("\\."); + Long[] maxVal = (Long[])max[count]; + Long[] minVal = (Long[])min[count]; + long maxLeftVal = (long)maxVal[0]; + long maxRightVal = (long)maxVal[1]; + long minLeftVal = (long) minVal[0]; + long minRightVal = (long)minVal[1]; + if (bigdVals.length == 2) { + long leftPart = Long.parseLong(bigdVals[0]); + BigDecimal bd = new BigDecimal(CarbonCommonConstants.POINT+bigdVals[1]); + long rightPart = (long)(bd.doubleValue()*Math.pow(10, value.scale())); + //Long.parseLong(bigdVals[1]); + maxVal[0] = (maxLeftVal > leftPart ? maxLeftVal : leftPart); + maxVal[1] = (maxRightVal > rightPart ? maxRightVal : rightPart); + minVal[0] = (minLeftVal < leftPart ? minLeftVal : leftPart); + minVal[1] = (minRightVal < rightPart ? minRightVal : rightPart); + } else { + long leftPart = Long.parseLong(bigdVals[0]); + maxVal[0] = (maxLeftVal > leftPart ? maxLeftVal : leftPart); + minVal[0] = (minLeftVal < leftPart ? minLeftVal : leftPart); + } } } } @@ -1226,8 +1270,10 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { if (type[i] == CarbonCommonConstants.BIG_INT_MEASURE) { uniqueValue[i] = (long) minValue[i] - 1; } else if (type[i] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) { - BigDecimal val = (BigDecimal) minValue[i]; - uniqueValue[i] = (val.subtract(new BigDecimal(1.0))); + Long[] bigdMinVal = (Long[])minValue[i]; + Long[] bigdUniqVal = (Long[])uniqueValue[i]; + bigdUniqVal[0] = bigdMinVal[0] -1; + bigdUniqVal[1] = bigdMinVal[1] -1; } else { uniqueValue[i] = (double) minValue[i] - 1; } @@ -1372,6 +1418,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { for (int i = 0; i < customMeasureIndex.length; i++) { dataHolder[customMeasureIndex[i]] = new CarbonWriteDataHolder(); dataHolder[customMeasureIndex[i]].initialiseByteArrayValues(size); + dataHolder[customMeasureIndex[i]].initialiseBigDecimalValues(size); } return dataHolder; }