Repository: incubator-carbondata Updated Branches: refs/heads/master b7f3be7e0 -> bc8265c9a
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java index be270e4..b3647a8 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java @@ -23,25 +23,17 @@ import mockit.Mock; import mockit.MockUp; import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties; -import org.apache.carbondata.core.carbon.metadata.blocklet.index.*; -import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex; import org.apache.carbondata.core.carbon.metadata.index.BlockIndexInfo; -import org.apache.carbondata.core.carbon.metadata.schema.table.column.*; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel; +import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; import org.apache.carbondata.core.metadata.BlockletInfoColumnar; import org.apache.carbondata.core.metadata.ValueEncoderMeta; import org.apache.carbondata.format.*; -import org.apache.carbondata.format.BlockletBTreeIndex; import org.apache.carbondata.format.BlockletMinMaxIndex; import org.apache.carbondata.format.ColumnSchema; import org.junit.BeforeClass; import org.junit.Test; -import java.io.ByteArrayInputStream; -import java.io.ObjectInputStream; import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; import java.util.ArrayList; import java.util.BitSet; import java.util.HashSet; @@ -49,7 +41,6 @@ import java.util.List; import java.util.Set; import static junit.framework.TestCase.*; -import static org.apache.carbondata.core.util.CarbonMetadataUtil.convertBlockletInfo; import static org.apache.carbondata.core.util.CarbonMetadataUtil.getIndexHeader; import static org.apache.carbondata.core.util.CarbonMetadataUtil.convertFileFooter; import static org.apache.carbondata.core.util.CarbonMetadataUtil.getBlockIndexInfo; @@ -119,7 +110,7 @@ public class CarbonMetadataUtilTest { blockletInfoList.add(blockletInfo); ValueEncoderMeta valueEncoderMeta = new ValueEncoderMeta(); - valueEncoderMeta.setDecimal(5); + valueEncoderMeta.setMantissa(5); valueEncoderMeta.setMinValue(objMinArr); valueEncoderMeta.setMaxValue(objMaxArr); valueEncoderMeta.setUniqueValue(lngObj); @@ -196,13 +187,13 @@ public class CarbonMetadataUtilTest { integerList.add(new Integer("1")); integerList.add(new Integer("2")); - ValueCompressionModel valueCompressionModel = new ValueCompressionModel(); - valueCompressionModel.setMaxValue(objMaxArr); - valueCompressionModel.setMinValue(objMinArr); - valueCompressionModel.setDataTypeSelected(byteArr); - valueCompressionModel.setDecimal(intArr); - valueCompressionModel.setType(charArr); - valueCompressionModel.setUniqueValue(objMinArr); + WriterCompressModel writerCompressModel = new WriterCompressModel(); + writerCompressModel.setMaxValue(objMaxArr); + writerCompressModel.setMinValue(objMinArr); + writerCompressModel.setDataTypeSelected(byteArr); + writerCompressModel.setMantissa(intArr); + writerCompressModel.setType(charArr); + writerCompressModel.setUniqueValue(objMinArr); BlockletInfoColumnar blockletInfoColumnar = new BlockletInfoColumnar(); @@ -226,7 +217,7 @@ public class CarbonMetadataUtilTest { blockletInfoColumnar.setMeasureLength(intArr); blockletInfoColumnar.setMeasureOffset(longArr); blockletInfoColumnar.setMeasureNullValueIndex(bitSetArr); - blockletInfoColumnar.setCompressionModel(valueCompressionModel); + blockletInfoColumnar.setCompressionModel(writerCompressModel); BlockletInfoColumnar blockletInfoColumnar1 = new BlockletInfoColumnar(); blockletInfoColumnar1.setColumnMaxData(maxByteArr); @@ -242,7 +233,7 @@ public class CarbonMetadataUtilTest { blockletInfoColumnar1.setMeasureLength(intArr); blockletInfoColumnar1.setMeasureOffset(longArr); blockletInfoColumnar1.setMeasureNullValueIndex(bitSetArr); - blockletInfoColumnar1.setCompressionModel(valueCompressionModel); + blockletInfoColumnar1.setCompressionModel(writerCompressModel); blockletInfoColumnar1.setColGrpBlocks(boolArr); List<BlockletInfoColumnar> blockletInfoColumnarList = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java index 869c8cc..14d9248 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java @@ -33,11 +33,10 @@ import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDime import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema; import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel; +import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; import org.apache.carbondata.core.datastorage.store.filesystem.LocalCarbonFile; import org.apache.carbondata.core.datastorage.store.impl.FileFactory; import org.apache.carbondata.core.metadata.ValueEncoderMeta; -import org.apache.carbondata.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.scan.model.QueryDimension; import org.apache.hadoop.security.UserGroupInformation; import org.junit.AfterClass; @@ -701,9 +700,9 @@ public class CarbonUtilTest { valueEncoderMetas.add(valueEncoderMeta); dataChunk.setValueEncoderMeta(valueEncoderMetas); dataChunkList.add(dataChunk); - ValueCompressionModel valueCompressionModel = + WriterCompressModel writerCompressModel = CarbonUtil.getValueCompressionModel(dataChunkList.get(0).getValueEncoderMeta()); - assertEquals(1, valueCompressionModel.getMaxValue().length); + assertEquals(1, writerCompressModel.getMaxValue().length); } @Test public void testToGetDictionaryChunkSize() { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java index 25d1426..a1bb6f0 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java @@ -19,9 +19,33 @@ package org.apache.carbondata.core.util; import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel; +import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder; -import org.apache.carbondata.core.datastorage.store.compression.type.*; +import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinByte; +import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinDefault; +import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinFloat; +import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinInt; +import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinLong; +import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinShort; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalByte; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal + .UnCompressNonDecimalDefault; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalFloat; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalInt; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalLong; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinByte; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinDefault; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinFloat; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinInt; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinLong; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinShort; +import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalShort; +import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneByte; +import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneDefault; +import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneFloat; +import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneInt; +import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneLong; +import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneShort; import org.junit.Test; @@ -45,7 +69,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataInt() { double[] values = { 25, 12, 22 }; int[] result = (int[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values, DataType.DATA_INT, 22, 0); int[] expectedResult = { -3, 10, 0 }; for (int i = 0; i < result.length; i++) { @@ -56,7 +80,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataByte() { double[] values = { 20, 21, 22 }; byte[] result = (byte[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values, DataType.DATA_BYTE, 22, 0); byte[] expectedResult = { 2, 1, 0 }; for (int i = 0; i < result.length; i++) { @@ -67,7 +91,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataShort() { double[] values = { 200, 21, 22 }; short[] result = (short[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values, DataType.DATA_SHORT, 22, 0); short[] expectedResult = { -178, 1, 0 }; for (int i = 0; i < result.length; i++) { @@ -78,7 +102,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataLong() { double[] values = { 2000, 2100, 2002 }; long[] result = (long[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values, DataType.DATA_LONG, 2125, 0); long[] expectedResult = { 125, 25, 123 }; for (int i = 0; i < result.length; i++) { @@ -89,7 +113,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataFloat() { double[] values = { 20.121, 21.223, 22.345 }; float[] result = (float[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values, DataType.DATA_FLOAT, 22.345, 3); float[] expectedResult = { 2.224f, 1.122f, 0f }; for (int i = 0; i < result.length; i++) { @@ -100,7 +124,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataDouble() { double[] values = { 20.121, 21.223, 22.345 }; double[] result = (double[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values, DataType.DATA_DOUBLE, 102.345, 3); double[] expectedResult = { 82.224, 81.122, 80.0 }; for (int i = 0; i < result.length; i++) { @@ -111,7 +135,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNoneForBigInt() { double[] values = { 20.121, 21.223, 22.345 }; long[] result = (long[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_BIGINT, 22, 0); long[] expectedResult = { 20, 21, 22 }; for (int i = 0; i < result.length; i++) { @@ -122,7 +146,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataByte() { double[] values = { 20, 21, 22 }; byte[] result = (byte[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, DataType.DATA_BYTE, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_BYTE, 22, 0); byte[] expectedResult = { 20, 21, 22 }; for (int i = 0; i < result.length; i++) { @@ -133,7 +157,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataShort() { double[] values = { 200000, 21, 22 }; short[] result = (short[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_SHORT, 22, 0); short[] expectedResult = { 3392, 21, 22 }; for (int i = 0; i < result.length; i++) { @@ -144,7 +168,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataInt() { double[] values = { 20, 21, 22 }; int[] result = (int[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, DataType.DATA_INT, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_INT, 22, 0); int[] expectedResult = { 20, 21, 22 }; for (int i = 0; i < result.length; i++) { @@ -155,7 +179,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataLong() { double[] values = { 20, 21, 22 }; long[] result = (long[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, DataType.DATA_LONG, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_LONG, 22, 0); long[] expectedResult = { 20, 21, 22 }; for (int i = 0; i < result.length; i++) { @@ -166,7 +190,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataFloat() { double[] values = { 20.121, 21.223, 22.345 }; float[] result = (float[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_FLOAT, 22, 3); float[] expectedResult = { 20.121f, 21.223f, 22.345f }; for (int i = 0; i < result.length; i++) { @@ -177,7 +201,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataDouble() { double[] values = { 20.121, 21.223, 22.345 }; double[] result = (double[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_DOUBLE, 22, 3); double[] expectedResult = { 20.121, 21.223, 22.345 }; for (int i = 0; i < result.length; i++) { @@ -188,7 +212,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForFloat() { double[] values = { 20.1, 21.2, 22.3 }; float[] result = (float[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values, DataType.DATA_FLOAT, 22, 1); float[] expectedResult = { 201f, 212f, 223f }; for (int i = 0; i < result.length; i++) { @@ -199,7 +223,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForByte() { double[] values = { 20.1, 21.2, 22.3 }; byte[] result = (byte[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values, DataType.DATA_BYTE, 22, 1); byte[] expectedResult = { -55, -44, -33 }; for (int i = 0; i < result.length; i++) { @@ -210,7 +234,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForShort() { double[] values = { 20.1, 21.2, 22.3 }; short[] result = (short[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values, DataType.DATA_SHORT, 22, 1); short[] expectedResult = { 201, 212, 223 }; for (int i = 0; i < result.length; i++) { @@ -221,7 +245,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForInt() { double[] values = { 20.1, 21.2, 22.3 }; int[] result = (int[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values, DataType.DATA_INT, 22, 1); int[] expectedResult = { 201, 212, 223 }; for (int i = 0; i < result.length; i++) { @@ -232,7 +256,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForLong() { double[] values = { 20.1, 21.2, 22.3 }; long[] result = (long[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values, DataType.DATA_LONG, 22, 1); long[] expectedResult = { 201, 212, 223 }; for (int i = 0; i < result.length; i++) { @@ -243,7 +267,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForDouble() { double[] values = { 20.1, 21.2, 22.3 }; double[] result = (double[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values, DataType.DATA_DOUBLE, 22, 1); double[] expectedResult = { 201, 212, 223 }; for (int i = 0; i < result.length; i++) { @@ -254,7 +278,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForByte() { double[] values = { 20, 21, 22 }; byte[] result = (byte[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values, DataType.DATA_BYTE, 22, 1); byte[] expectedResult = { 20, 10, 0 }; for (int i = 0; i < result.length; i++) { @@ -265,7 +289,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForInt() { double[] values = { 20, 21, 22 }; int[] result = (int[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values, DataType.DATA_INT, 22, 1); int[] expectedResult = { 20, 10, 0 }; for (int i = 0; i < result.length; i++) { @@ -276,7 +300,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForDouble() { double[] values = { 20, 21, 22 }; double[] result = (double[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values, DataType.DATA_DOUBLE, 22, 1); double[] expectedResult = { 20, 10, 0 }; for (int i = 0; i < result.length; i++) { @@ -287,7 +311,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForShort() { double[] values = { 20000, 21, 22 }; short[] result = (short[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values, DataType.DATA_SHORT, 22, 1); short[] expectedResult = { -3172, 10, 0 }; for (int i = 0; i < result.length; i++) { @@ -298,7 +322,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForLong() { double[] values = { 20, 21, 22 }; long[] result = (long[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values, DataType.DATA_LONG, 22, 1); long[] expectedResult = { 20, 10, 0 }; for (int i = 0; i < result.length; i++) { @@ -309,7 +333,7 @@ public class ValueCompressionUtilTest { @Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForFloat() { double[] values = { 20, 21, 22 }; float[] result = (float[]) ValueCompressionUtil - .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values, + .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values, DataType.DATA_FLOAT, 22, 1); float[] expectedResult = { 20f, 10f, 0f }; for (int i = 0; i < result.length; i++) { @@ -319,151 +343,151 @@ public class ValueCompressionUtilTest { @Test public void testToUnCompressNone() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNone(DataType.DATA_BIGINT, DataType.DATA_BIGINT); + ValueCompressionUtil.getUnCompressNone(DataType.DATA_BIGINT, DataType.DATA_BIGINT); assertEquals(result.getClass(), UnCompressNoneDefault.class); } @Test public void testToUnCompressNoneForByte() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNone(DataType.DATA_BYTE, DataType.DATA_FLOAT); + ValueCompressionUtil.getUnCompressNone(DataType.DATA_BYTE, DataType.DATA_FLOAT); assertEquals(result.getClass(), UnCompressNoneByte.class); } @Test public void testToUnCompressNoneForLong() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNone(DataType.DATA_LONG, DataType.DATA_FLOAT); + ValueCompressionUtil.getUnCompressNone(DataType.DATA_LONG, DataType.DATA_FLOAT); assertEquals(result.getClass(), UnCompressNoneLong.class); } @Test public void testToUnCompressNoneForShort() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNone(DataType.DATA_SHORT, DataType.DATA_FLOAT); + ValueCompressionUtil.getUnCompressNone(DataType.DATA_SHORT, DataType.DATA_FLOAT); assertEquals(result.getClass(), UnCompressNoneShort.class); } @Test public void testToUnCompressNoneForFloat() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNone(DataType.DATA_FLOAT, DataType.DATA_FLOAT); + ValueCompressionUtil.getUnCompressNone(DataType.DATA_FLOAT, DataType.DATA_FLOAT); assertEquals(result.getClass(), UnCompressNoneFloat.class); } @Test public void testToUnCompressNoneForInt() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNone(DataType.DATA_INT, DataType.DATA_FLOAT); + ValueCompressionUtil.getUnCompressNone(DataType.DATA_INT, DataType.DATA_FLOAT); assertEquals(result.getClass(), UnCompressNoneInt.class); } @Test public void testToUnCompressNoneForDouble() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNone(DataType.DATA_DOUBLE, DataType.DATA_FLOAT); + ValueCompressionUtil.getUnCompressNone(DataType.DATA_DOUBLE, DataType.DATA_FLOAT); assertEquals(result.getClass(), UnCompressNoneDefault.class); } @Test public void testToUnCompressMaxMinForDouble() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressMaxMin(DataType.DATA_DOUBLE, null); + ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_DOUBLE, null); assertEquals(result.getClass(), UnCompressMaxMinDefault.class); } @Test public void testToUnCompressMaxMinForInt() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressMaxMin(DataType.DATA_INT, null); + ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_INT, null); assertEquals(result.getClass(), UnCompressMaxMinInt.class); } @Test public void testToUnCompressMaxMinForFloat() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressMaxMin(DataType.DATA_FLOAT, null); + ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_FLOAT, null); assertEquals(result.getClass(), UnCompressMaxMinFloat.class); } @Test public void testToUnCompressMaxMinForLong() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressMaxMin(DataType.DATA_LONG, null); + ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_LONG, null); assertEquals(result.getClass(), UnCompressMaxMinLong.class); } @Test public void testToUnCompressMaxMinForByte() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressMaxMin(DataType.DATA_BYTE, null); + ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_BYTE, null); assertEquals(result.getClass(), UnCompressMaxMinByte.class); } @Test public void testToUnCompressMaxMinForShort() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressMaxMin(DataType.DATA_SHORT, null); + ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_SHORT, null); assertEquals(result.getClass(), UnCompressMaxMinShort.class); } @Test public void testToUnCompressNonDecimalForDouble() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_DOUBLE, null); + ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_DOUBLE); assertEquals(result.getClass(), UnCompressNonDecimalDefault.class); } @Test public void testToUnCompressNonDecimalForInt() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_INT, null); + ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_INT); assertEquals(result.getClass(), UnCompressNonDecimalInt.class); } @Test public void testToUnCompressNonDecimalForFloat() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_FLOAT, null); + ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_FLOAT); assertEquals(result.getClass(), UnCompressNonDecimalFloat.class); } @Test public void testToUnCompressNonDecimalForLong() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_LONG, null); + ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_LONG); assertEquals(result.getClass(), UnCompressNonDecimalLong.class); } @Test public void testToUnCompressNonDecimalForByte() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_BYTE, null); + ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_BYTE); assertEquals(result.getClass(), UnCompressNonDecimalByte.class); } @Test public void testToUnCompressNonDecimalForShort() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_SHORT, null); + ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_SHORT); assertEquals(result.getClass(), UnCompressNonDecimalShort.class); } @Test public void testToUnCompressNonDecimalMaxMinForDouble() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_DOUBLE, null); + ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_DOUBLE); assertEquals(result.getClass(), UnCompressNonDecimalMaxMinDefault.class); } @Test public void testToUnCompressNonDecimalMaxMinForInt() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_INT, null); + ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_INT); assertEquals(result.getClass(), UnCompressNonDecimalMaxMinInt.class); } @Test public void testToUnCompressNonDecimalMaxMinForFloat() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_FLOAT, null); + ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_FLOAT); assertEquals(result.getClass(), UnCompressNonDecimalMaxMinFloat.class); } @Test public void testToUnCompressNonDecimalMaxMinForLong() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_LONG, null); + ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_LONG); assertEquals(result.getClass(), UnCompressNonDecimalMaxMinLong.class); } @Test public void testToUnCompressNonDecimalMaxMinForByte() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_BYTE, null); + ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_BYTE); assertEquals(result.getClass(), UnCompressNonDecimalMaxMinByte.class); } @Test public void testToUnCompressNonDecimalMaxMinForShort() { ValueCompressonHolder.UnCompressValue result = - ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_SHORT, null); + ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_SHORT); assertEquals(result.getClass(), UnCompressNonDecimalMaxMinShort.class); } @@ -563,23 +587,23 @@ public class ValueCompressionUtilTest { } @Test public void testToGetValueCompressionModel() { - Object[] maxValues = { 10, 20, 30l }; - Object[] minValues = { 1, 2, 3l }; + Object[] maxValues = { 10l, 20l, 30l }; + Object[] minValues = { 1l, 2l, 3l }; int[] decimalLength = { 0, 0, 0 }; Object[] uniqueValues = { 5, 3, 2l }; - char[] types = { 'c', 'b', 'l' }; + char[] types = { 'l', 'l', 'l' }; byte[] dataTypeSelected = { 1, 2, 4 }; MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 3, uniqueValues, types, dataTypeSelected); - ValueCompressionModel valueCompressionModel = - ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.CUSTOM, - valueCompressionModel.getCompType()[0]); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.CUSTOM_BIGDECIMAL, - valueCompressionModel.getCompType()[1]); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NONE, - valueCompressionModel.getCompType()[2]); + WriterCompressModel writerCompressModel = + ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, + writerCompressModel.getCompType()[0]); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, + writerCompressModel.getCompType()[1]); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, + writerCompressModel.getCompType()[2]); } @Test public void testToGetValueCompressionModelForDefaultAggregatorType() { @@ -592,10 +616,10 @@ public class ValueCompressionUtilTest { MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected); - ValueCompressionModel valueCompressionModel = - ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NONE, - valueCompressionModel.getCompType()[0]); + WriterCompressModel writerCompressModel = + ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, + writerCompressModel.getCompType()[0]); } @Test public void testToGetValueCompressionModelForShortAndByte() { @@ -608,10 +632,10 @@ public class ValueCompressionUtilTest { MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected); - ValueCompressionModel valueCompressionModel = - ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, - valueCompressionModel.getCompType()[0]); + WriterCompressModel writerCompressModel = + ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, + writerCompressModel.getCompType()[0]); } @Test public void testToGetValueCompressionModelForIntAndShort() { @@ -624,10 +648,10 @@ public class ValueCompressionUtilTest { MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected); - ValueCompressionModel valueCompressionModel = - ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, - valueCompressionModel.getCompType()[0]); + WriterCompressModel writerCompressModel = + ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, + writerCompressModel.getCompType()[0]); } @Test public void testToGetValueCompressionModelForByteAndInt() { @@ -640,10 +664,10 @@ public class ValueCompressionUtilTest { MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected); - ValueCompressionModel valueCompressionModel = - ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NONE, - valueCompressionModel.getCompType()[0]); + WriterCompressModel writerCompressModel = + ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, + writerCompressModel.getCompType()[0]); } @Test public void testToGetValueCompressionModelForByteAndIntAndDecimal1() { @@ -656,10 +680,10 @@ public class ValueCompressionUtilTest { MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected); - ValueCompressionModel valueCompressionModel = - ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, - valueCompressionModel.getCompType()[0]); + WriterCompressModel writerCompressModel = + ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, + writerCompressModel.getCompType()[0]); } @Test public void testToGetValueCompressionModelForByteAndIntAndDataTypeSelected0() { @@ -672,10 +696,10 @@ public class ValueCompressionUtilTest { MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected); - ValueCompressionModel valueCompressionModel = - ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, - valueCompressionModel.getCompType()[0]); + WriterCompressModel writerCompressModel = + ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, + writerCompressModel.getCompType()[0]); } @Test public void testToGetValueCompressionModelForFloatAndDataTypeSelected1() { @@ -688,10 +712,10 @@ public class ValueCompressionUtilTest { MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected); - ValueCompressionModel valueCompressionModel = - ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel); - assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NONE, - valueCompressionModel.getCompType()[0]); + WriterCompressModel writerCompressModel = + ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel); + assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, + writerCompressModel.getCompType()[0]); } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java index b36233b..20a0790 100644 --- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java +++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java @@ -30,7 +30,7 @@ import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties; import org.apache.carbondata.core.carbon.metadata.datatype.DataType; import org.apache.carbondata.core.carbon.metadata.encoder.Encoding; import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel; +import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile; import org.apache.carbondata.core.datastorage.store.impl.FileFactory; import org.apache.carbondata.core.metadata.BlockletInfoColumnar; @@ -182,10 +182,10 @@ public class CarbonFooterWriterTest extends TestCase{ infoColumnar.setAggKeyBlock(new boolean[] { true, true, true, true }); infoColumnar.setColGrpBlocks(new boolean[] { false, false, false, false }); infoColumnar.setMeasureNullValueIndex(new BitSet[] {new BitSet(),new BitSet()}); - ValueCompressionModel compressionModel = new ValueCompressionModel(); + WriterCompressModel compressionModel = new WriterCompressModel(); compressionModel.setMaxValue(new Object[] { 44d, 55d }); compressionModel.setMinValue(new Object[] { 0d, 0d }); - compressionModel.setDecimal(new int[] { 0, 0 }); + compressionModel.setMantissa(new int[] { 0, 0 }); compressionModel.setType(new char[] { 'n', 'n' }); compressionModel.setUniqueValue(new Object[] { 0d, 0d }); compressionModel.setDataTypeSelected(new byte[2]); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/format/src/main/thrift/schema.thrift ---------------------------------------------------------------------- diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift index 885beba..1df59a5 100644 --- a/format/src/main/thrift/schema.thrift +++ b/format/src/main/thrift/schema.thrift @@ -68,7 +68,7 @@ struct ColumnSchema{ 6: required bool dimension; // Whether the column is a dimension or measure 7: optional i32 column_group_id; // The group ID for column used for row format columns, where in columns in each group are chunked together. /** - * Used when this column contains decimal data. + * Used when this column contains mantissa data. */ 8: optional i32 scale; 9: optional i32 precision; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala index d6cd075..39d0841 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala @@ -757,7 +757,7 @@ private[sql] case class DescribeCommandFormatted( if (colPropStr.length() > 0) { results ++= Seq((colPropStr, "", "")) } else { - results ++= Seq(("NONE", "", "")) + results ++= Seq(("ADAPTIVE", "", "")) } val dimension = carbonTable .getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName) http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml index 65fdd57..32b1452 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml +++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml @@ -202,7 +202,7 @@ <parentid>FIELD</parentid> </attribute> <attribute id="FIELD_DECIMAL"> - <xmlcode>decimal</xmlcode> + <xmlcode>mantissa</xmlcode> <repcode>field_decimal</repcode> <description>CsvInputDialog.DecimalColumn.Column</description> <tooltip/> http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java index c961700..0398cd7 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java @@ -51,7 +51,7 @@ import org.apache.carbondata.core.datastorage.store.columnar.BlockIndexerStorage import org.apache.carbondata.core.datastorage.store.columnar.BlockIndexerStorageForNoInvertedIndex; import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel; import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel; +import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder; import org.apache.carbondata.core.datastorage.util.StoreFactory; import org.apache.carbondata.core.keygenerator.KeyGenException; @@ -603,8 +603,8 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { if ((noDictionaryCount + complexColCount) > 0) { noDictionaryValueHolder = noDictionaryKeyDataHolder.getByteArrayValues(); } - ValueCompressionModel compressionModel = ValueCompressionUtil - .getValueCompressionModel(max, min, decimal, uniqueValue, type, new byte[max.length]); + WriterCompressModel compressionModel = ValueCompressionUtil + .getWriterCompressModel(max, min, decimal, uniqueValue, type, new byte[max.length]); byte[][] writableMeasureDataArray = StoreFactory.createDataStore(compressionModel).getWritableMeasureDataArray(dataHolder) .clone(); @@ -738,8 +738,8 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { if ((noDictionaryCount + complexColCount) > 0) { noDictionaryValueHolder = noDictionaryKeyDataHolder.getNonDictByteArrayValues(); } - ValueCompressionModel compressionModel = ValueCompressionUtil - .getValueCompressionModel(max, min, decimal, uniqueValue, type, new byte[max.length]); + WriterCompressModel compressionModel = ValueCompressionUtil + .getWriterCompressModel(max, min, decimal, uniqueValue, type, new byte[max.length]); byte[][] writableMeasureDataArray = StoreFactory.createDataStore(compressionModel).getWritableMeasureDataArray(dataHolder) .clone(); @@ -755,7 +755,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { // TODO remove after kettle flow is removed private NodeHolder getNodeHolderObject(byte[][] dataHolderLocal, byte[][] byteArrayValues, int entryCountLocal, byte[] startkeyLocal, byte[] endKeyLocal, - ValueCompressionModel compressionModel, byte[][] noDictionaryData, + WriterCompressModel compressionModel, byte[][] noDictionaryData, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) throws CarbonDataWriterException { byte[][][] noDictionaryColumnsData = null; @@ -882,7 +882,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { private NodeHolder getNodeHolderObjectWithOutKettle(byte[][] dataHolderLocal, byte[][] byteArrayValues, int entryCountLocal, byte[] startkeyLocal, byte[] endKeyLocal, - ValueCompressionModel compressionModel, byte[][][] noDictionaryData, + WriterCompressModel compressionModel, byte[][][] noDictionaryData, byte[][] noDictionaryStartKey, byte[][] noDictionaryEndKey) throws CarbonDataWriterException { byte[][][] noDictionaryColumnsData = null; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java index 3e31901..5745e9f 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java @@ -20,7 +20,7 @@ package org.apache.carbondata.processing.store.writer; import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel; +import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException; public interface CarbonFactDataWriter<T> { @@ -40,7 +40,7 @@ public interface CarbonFactDataWriter<T> { */ NodeHolder buildDataNodeHolder(IndexStorage<T>[] keyStorageArray, byte[][] dataArray, - int entryCount, byte[] startKey, byte[] endKey, ValueCompressionModel compressionModel, + int entryCount, byte[] startKey, byte[] endKey, WriterCompressModel compressionModel, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) throws CarbonDataWriterException; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java index a7d14f0..0864458 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java @@ -21,7 +21,7 @@ package org.apache.carbondata.processing.store.writer; import java.util.BitSet; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel; +import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; public class NodeHolder { /** @@ -110,7 +110,7 @@ public class NodeHolder { /** * compression model for numbers data block. */ - private ValueCompressionModel compressionModel; + private WriterCompressModel compressionModel; /** * array of aggBlocks flag to identify the aggBlocks @@ -382,11 +382,11 @@ public class NodeHolder { this.columnMinData = columnMinData; } - public ValueCompressionModel getCompressionModel() { + public WriterCompressModel getCompressionModel() { return compressionModel; } - public void setCompressionModel(ValueCompressionModel compressionModel) { + public void setCompressionModel(WriterCompressModel compressionModel) { this.compressionModel = compressionModel; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java index 19e781d..8d53fe4 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java @@ -28,7 +28,7 @@ import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage; -import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel; +import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel; import org.apache.carbondata.core.metadata.BlockletInfoColumnar; import org.apache.carbondata.core.util.CarbonMetadataUtil; import org.apache.carbondata.core.writer.CarbonFooterWriter; @@ -50,7 +50,7 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> { @Override public NodeHolder buildDataNodeHolder(IndexStorage<int[]>[] keyStorageArray, byte[][] dataArray, - int entryCount, byte[] startKey, byte[] endKey, ValueCompressionModel compressionModel, + int entryCount, byte[] startKey, byte[] endKey, WriterCompressModel compressionModel, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) throws CarbonDataWriterException { // if there are no NO-Dictionary column present in the table then // set the empty byte array
