Repository: incubator-carbondata Updated Branches: refs/heads/master 7ba906be1 -> bd09f9bc1
Fixed Compatibility issue Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f94bae5e Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f94bae5e Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f94bae5e Branch: refs/heads/master Commit: f94bae5eb68646987685ab740161b9d8820d8f69 Parents: 7ba906b Author: kumarvishal <kumarvishal.1...@gmail.com> Authored: Sat Jan 21 00:35:30 2017 +0800 Committer: kumarvishal <kumarvishal.1...@gmail.com> Committed: Sat Jan 21 01:00:46 2017 +0800 ---------------------------------------------------------------------- .../core/constants/CarbonCommonConstants.java | 2 +- .../compression/ReaderCompressModel.java | 2 +- .../core/metadata/ValueEncoderMeta.java | 12 +++--- .../core/util/CarbonMetadataUtil.java | 4 +- .../apache/carbondata/core/util/CarbonUtil.java | 2 +- .../core/util/ValueCompressionUtil.java | 10 +++-- .../core/util/path/CarbonTablePath.java | 6 ++- .../datastore/SegmentTaskIndexStoreTest.java | 42 -------------------- .../core/util/CarbonMetadataUtilTest.java | 2 +- .../carbondata/core/util/DataTypeUtilTest.java | 4 +- .../core/util/ValueCompressionUtilTest.java | 2 +- 11 files changed, 28 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index d1c5eb1..df3f9f1 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -631,7 +631,7 @@ public final class CarbonCommonConstants { /** * BIG_INT_MEASURE */ - public static final char BIG_INT_MEASURE = 'l'; + public static final char BIG_INT_MEASURE = 'd'; /** * This determines the size of array to be processed in data load steps. one http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java index 750888c..101a24b 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java @@ -45,7 +45,7 @@ public class ReaderCompressModel { } public int getMantissa() { - return valueEncoderMeta.getMantissa(); + return valueEncoderMeta.getDecimal(); } public ValueCompressionHolder getValueCompressionHolder() { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java index 9febc55..4a9007c 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java @@ -20,6 +20,8 @@ package org.apache.carbondata.core.metadata; import java.io.Serializable; /** + * DO NOT MODIFY THIS CLASS AND PACKAGE NAME, BECAUSE + * IT IS SERIALIZE TO STORE * It holds Value compression metadata for one data column */ public class ValueEncoderMeta implements Serializable { @@ -38,7 +40,7 @@ public class ValueEncoderMeta implements Serializable { */ private Object uniqueValue; - private int mantissa; + private int decimal; private char type; @@ -68,12 +70,12 @@ public class ValueEncoderMeta implements Serializable { this.uniqueValue = uniqueValue; } - public int getMantissa() { - return mantissa; + public int getDecimal() { + return decimal; } - public void setMantissa(int mantissa) { - this.mantissa = mantissa; + public void setDecimal(int decimal) { + this.decimal = decimal; } public char getType() { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java index 5a1d50a..25e7cfe 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java @@ -318,7 +318,7 @@ public class CarbonMetadataUtil { encoderMeta.setMaxValue(compressionModel.getMaxValue()[index]); encoderMeta.setMinValue(compressionModel.getMinValue()[index]); encoderMeta.setDataTypeSelected(compressionModel.getDataTypeSelected()[index]); - encoderMeta.setMantissa(compressionModel.getMantissa()[index]); + encoderMeta.setDecimal(compressionModel.getMantissa()[index]); encoderMeta.setType(compressionModel.getType()[index]); encoderMeta.setUniqueValue(compressionModel.getUniqueValue()[index]); return encoderMeta; @@ -428,7 +428,7 @@ public class CarbonMetadataUtil { for (int i = 0; i < encoderMetas.length; i++) { maxValue[i] = encoderMetas[i].getMaxValue(); minValue[i] = encoderMetas[i].getMinValue(); - decimalLength[i] = encoderMetas[i].getMantissa(); + decimalLength[i] = encoderMetas[i].getDecimal(); uniqueValue[i] = encoderMetas[i].getUniqueValue(); aggType[i] = encoderMetas[i].getType(); dataTypeSelected[i] = encoderMetas[i].getDataTypeSelected(); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 07ac127..11c8870 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -749,7 +749,7 @@ public final class CarbonUtil { maxValue[i] = valueEncoderMeta.getMaxValue(); minValue[i] = valueEncoderMeta.getMinValue(); uniqueValue[i] = valueEncoderMeta.getUniqueValue(); - decimal[i] = valueEncoderMeta.getMantissa(); + decimal[i] = valueEncoderMeta.getDecimal(); type[i] = valueEncoderMeta.getType(); dataTypeSelected[i] = valueEncoderMeta.getDataTypeSelected(); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java index 219f69e..b958445 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java @@ -116,9 +116,12 @@ public final class ValueCompressionUtil { case 'b': return new CompressionFinder(COMPRESSION_TYPE.BIGDECIMAL, DataType.DATA_BYTE, DataType.DATA_BYTE, measureStoreType); - case 'l': + case 'd': return getLongCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected, measureStoreType); + case 'l': + return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, + DataType.DATA_BIGINT, DataType.DATA_BIGINT, measureStoreType); case 'n': return getDoubleCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected, measureStoreType); @@ -223,7 +226,7 @@ public final class ValueCompressionUtil { */ public static ValueCompressor getValueCompressor(CompressionFinder compressorFinder) { switch(compressorFinder.getMeasureStoreType()) { - case 'l': return new BigIntCompressor(); + case 'd': return new BigIntCompressor(); default : return new DoubleCompressor(); } } @@ -565,6 +568,7 @@ public final class ValueCompressionUtil { case DATA_INT: return new CompressionNoneInt(actualDataType); case DATA_LONG: + case DATA_BIGINT: return new CompressionNoneLong(actualDataType); default: return new CompressionNoneDefault(actualDataType); @@ -683,7 +687,7 @@ public final class ValueCompressionUtil { public static ReaderCompressModel getReaderCompressModel(ValueEncoderMeta meta) { ReaderCompressModel compressModel = new ReaderCompressModel(); CompressionFinder compressFinder = - getCompressionFinder(meta.getMaxValue(), meta.getMinValue(), meta.getMantissa(), + getCompressionFinder(meta.getMaxValue(), meta.getMinValue(), meta.getDecimal(), meta.getType(), meta.getDataTypeSelected()); compressModel.setValueCompressionHolder( ValueCompressionUtil.getValueCompressionHolder(compressFinder)); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java index 1d1ebfa..39b3629 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java +++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java @@ -258,6 +258,10 @@ public class CarbonTablePath extends Path { CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() { @Override public boolean accept(CarbonFile file) { + if(bucketNumber.equals("-1")) { + return file.getName().startsWith(taskId) && file.getName() + .endsWith(INDEX_FILE_EXT); + } return file.getName().startsWith(taskId + "-" + bucketNumber) && file.getName() .endsWith(INDEX_FILE_EXT); } @@ -455,7 +459,7 @@ public class CarbonTablePath extends Path { int endIndex = fileName.indexOf("-", startIndex); // to support backward compatibility if (startIndex == -1 || endIndex == -1) { - return "0"; + return "-1"; } return fileName.substring(startIndex, endIndex); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java index 0b71f2e..c66398c 100644 --- a/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java +++ b/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java @@ -85,48 +85,6 @@ public class SegmentTaskIndexStoreTest { return footerList; } - @Test public void loadAndGetTaskIdToSegmentsMap() throws IOException { - new MockUp<CarbonTablePath.DataFileUtil>() { - @Mock String getTaskNo(String carbonDataFileName) { - return "100"; - } - }; - - new MockUp<CarbonUtil>() { - @Mock List<DataFileFooter> readCarbonIndexFile(String taskId, - String bucketNumber, - List<TableBlockInfo> tableBlockInfoList, - AbsoluteTableIdentifier absoluteTableIdentifier) { - return getDataFileFooters(); - } - }; - - new MockUp<CarbonTablePath>() { - @Mock public String getCarbonIndexFilePath(final String taskId, final String partitionId, - final String segmentId, final String bucketNumber) { - return "/src/test/resources"; - } - }; - - new MockUp<SegmentTaskIndex>() { - @Mock void buildIndex(List<DataFileFooter> footerList) { - } - }; - TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier = - new TableSegmentUniqueIdentifier(absoluteTableIdentifier, "SG100"); - - HashMap<String, List<TableBlockInfo>> segmentToTableBlocksInfos = - new HashMap<String, List<TableBlockInfo>>() {{ - put("SG100", Arrays.asList(tableBlockInfo)); - }}; - tableSegmentUniqueIdentifier.setSegmentToTableBlocksInfos(segmentToTableBlocksInfos); - Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> result = - taskIndexStore.get(tableSegmentUniqueIdentifier).getTaskIdToTableSegmentMap(); - - assertEquals(result.size(), 1); - assertTrue(result.containsKey(new SegmentTaskIndexStore.TaskBucketHolder("100", "0"))); - } - @Test public void checkExistenceOfSegmentBTree() { TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier = new TableSegmentUniqueIdentifier(absoluteTableIdentifier, "SG100"); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java index 3146ea8..3935bdc 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java @@ -108,7 +108,7 @@ public class CarbonMetadataUtilTest { blockletInfoList.add(blockletInfo); ValueEncoderMeta valueEncoderMeta = new ValueEncoderMeta(); - valueEncoderMeta.setMantissa(5); + valueEncoderMeta.setDecimal(5); valueEncoderMeta.setMinValue(objMinArr); valueEncoderMeta.setMaxValue(objMaxArr); valueEncoderMeta.setUniqueValue(lngObj); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java index c7ead8d..f2d110f 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java @@ -52,8 +52,8 @@ public class DataTypeUtilTest { @Test public void testGetAggType() { assertTrue(getAggType(DataType.DECIMAL) == 'b'); - assertTrue(getAggType(DataType.INT) == 'l'); - assertTrue(getAggType(DataType.LONG) == 'l'); + assertTrue(getAggType(DataType.INT) == 'd'); + assertTrue(getAggType(DataType.LONG) == 'd'); assertTrue(getAggType(DataType.NULL) == 'n'); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f94bae5e/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java index 04605e7..a72197f 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java @@ -322,7 +322,7 @@ public class ValueCompressionUtilTest { @Test public void testToUnCompressNone() { ValueCompressionHolder result = ValueCompressionUtil.getCompressionNone(DataType.DATA_BIGINT, DataType.DATA_BIGINT); - assertEquals(result.getClass(), CompressionNoneDefault.class); + assertEquals(result.getClass(), CompressionNoneLong.class); } @Test public void testToUnCompressNoneForByte() {
