Repository: incubator-carbondata Updated Branches: refs/heads/master 83ea402ab -> 1706af46c
code cleanup Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e5d26701 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e5d26701 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e5d26701 Branch: refs/heads/master Commit: e5d26701caced3d793d1f43823744e5c2e303ae4 Parents: 83ea402 Author: Jihong Ma <jihon...@apache.org> Authored: Wed Jan 18 22:41:16 2017 -0800 Committer: Jihong Ma <jihon...@apache.org> Committed: Mon Jan 23 11:33:33 2017 -0800 ---------------------------------------------------------------------- .../MultiDimKeyVarLengthEquiSplitGenerator.java | 10 ++-- .../reader/CarbonDeleteDeltaFileReaderImpl.java | 26 --------- .../core/scan/expression/ExpressionResult.java | 7 ++- .../core/scan/result/AbstractScannedResult.java | 46 +--------------- .../result/impl/FilterQueryScannedResult.java | 56 -------------------- .../impl/NonFilterQueryScannedResult.java | 55 ------------------- .../core/util/ValueCompressionUtil.java | 50 ----------------- .../core/util/ValueCompressionUtilTest.java | 55 ------------------- 8 files changed, 9 insertions(+), 296 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5d26701/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java index 9cc56c2..f5a0f69 100644 --- a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java @@ -18,7 +18,6 @@ package org.apache.carbondata.core.keygenerator.columnar.impl; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.Set; @@ -143,8 +142,9 @@ public class MultiDimKeyVarLengthEquiSplitGenerator extends MultiDimKeyVarLength private int[] convertToArray(List<Integer> list) { int[] ints = new int[list.size()]; - for (int i = 0; i < ints.length; i++) { - ints[i] = list.get(i); + int i = 0; + for (Integer ii : list) { + ints[i++] = ii; } return ints; } @@ -152,8 +152,8 @@ public class MultiDimKeyVarLengthEquiSplitGenerator extends MultiDimKeyVarLength private int[] convertToArray(Set<Integer> set) { int[] ints = new int[set.size()]; int i = 0; - for (Iterator iterator = set.iterator(); iterator.hasNext(); ) { - ints[i++] = (Integer) iterator.next(); + for (Integer ii: set) { + ints[i++] = ii; } return ints; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5d26701/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java index dfc4537..b75d796 100644 --- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java @@ -51,8 +51,6 @@ public class CarbonDeleteDeltaFileReaderImpl implements CarbonDeleteDeltaFileRea private DataInputStream dataInputStream = null; - private BufferedReader buffReader = null; - private InputStreamReader inputStream = null; private static final int DEFAULT_BUFFER_SIZE = 258; @@ -83,7 +81,6 @@ public class CarbonDeleteDeltaFileReaderImpl implements CarbonDeleteDeltaFileRea dataInputStream = FileFactory.getDataInputStream(filePath, fileType); inputStream = new InputStreamReader(dataInputStream, CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT); - buffReader = new BufferedReader(inputStream); int n = 0; while (-1 != (n = inputStream.read(buffer))) { sw.write(buffer, 0, n); @@ -123,27 +120,4 @@ public class CarbonDeleteDeltaFileReaderImpl implements CarbonDeleteDeltaFileRea return deleteDeltaBlockDetails; } - - /** - * Returns all deleted records from specified delete delta file - * - * @return - * @throws IOException - */ - public int[] getDeleteDeltaRows() throws IOException { - String[] stringRows = read().split(CarbonCommonConstants.COMMA); - int[] rows = new int[stringRows.length]; - int rowsLength = stringRows.length; - for (int i = 0; i < rowsLength; i++) { - try { - rows[i] = Integer.parseInt(stringRows[i]); - } catch (NumberFormatException nfe) { - LOGGER.error("Invalid row : " + stringRows[i] + nfe.getLocalizedMessage()); - throw new IOException("Invalid row : " + nfe.getLocalizedMessage()); - } - } - - return rows; - } - } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5d26701/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java index 1e57bf2..28d0f7e 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java @@ -522,12 +522,11 @@ public class ExpressionResult implements Comparable<ExpressionResult> { return val1.compareTo(val2); case DATE: case TIMESTAMP: + String format= CarbonUtil.getFormatFromProperty(o.dataType); SimpleDateFormat parser = new SimpleDateFormat(format); - Date date1 = null; - Date date2 = null; - date1 = parser.parse(this.getString()); - date2 = parser.parse(o.getString()); + Date date1 = parser.parse(this.getString()); + Date date2 = parser.parse(o.getString()); return date1.compareTo(date2); case STRING: default: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5d26701/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java index 7a8e557..60e6f67 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java @@ -331,7 +331,7 @@ public abstract class AbstractScannedResult { } /** - * @param blockletId + * @param rowId */ public void setRowId(long rowId) { this.rowId = rowId; @@ -483,14 +483,6 @@ public abstract class AbstractScannedResult { public abstract int[] getDictionaryKeyIntegerArray(); /** - * Return the dimension data based on dimension ordinal - * - * @param dimensionOrdinal dimension ordinal - * @return dimension data - */ - public abstract byte[] getDimensionKey(int dimensionOrdinal); - - /** * Below method will be used to get the complex type key array * * @return complex type key array @@ -514,42 +506,6 @@ public abstract class AbstractScannedResult { public abstract String[] getNoDictionaryKeyStringArray(); /** - * Below method will be used to to check whether measure value - * is null or for a measure - * - * @param ordinal measure ordinal - * @return is null or not - */ - public abstract boolean isNullMeasureValue(int ordinal); - - /** - * Below method will be used to get the measure value for measure - * of long data type - * - * @param ordinal measure ordinal - * @return long value of measure - */ - public abstract long getLongMeasureValue(int ordinal); - - /** - * Below method will be used to get the value of measure of double - * type - * - * @param ordinal measure ordinal - * @return measure value - */ - public abstract double getDoubleMeasureValue(int ordinal); - - /** - * Below method will be used to get the data of big decimal type - * of a measure - * - * @param ordinal measure ordinal - * @return measure value - */ - public abstract BigDecimal getBigDecimalMeasureValue(int ordinal); - - /** * * @return BlockletLevelDeleteDeltaDataCache. */ http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5d26701/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java index 04ea85b..6ca0570 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java @@ -16,8 +16,6 @@ */ package org.apache.carbondata.core.scan.result.impl; -import java.math.BigDecimal; - import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; import org.apache.carbondata.core.scan.result.AbstractScannedResult; import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo; @@ -90,60 +88,6 @@ public class FilterQueryScannedResult extends AbstractScannedResult { } /** - * Return the dimension data based on dimension ordinal - * - * @param dimensionOrdinal dimension ordinal - * @return dimension data - */ - @Override public byte[] getDimensionKey(int dimensionOrdinal) { - return getDimensionData(dimensionOrdinal, rowMapping[currentRow]); - } - - /** - * Below method will be used to to check whether measure value - * is null or for a measure - * - * @param ordinal measure ordinal - * @return is null or not - */ - @Override public boolean isNullMeasureValue(int ordinal) { - return isNullMeasureValue(ordinal, rowMapping[currentRow]); - } - - /** - * Below method will be used to get the measure value for measure - * of long data type - * - * @param ordinal measure ordinal - * @return long value of measure - */ - @Override public long getLongMeasureValue(int ordinal) { - return getLongMeasureValue(ordinal, rowMapping[currentRow]); - } - - /** - * Below method will be used to get the value of measure of double - * type - * - * @param ordinal measure ordinal - * @return measure value - */ - @Override public double getDoubleMeasureValue(int ordinal) { - return getDoubleMeasureValue(ordinal, rowMapping[currentRow]); - } - - /** - * Below method will be used to get the data of big decimal type - * of a measure - * - * @param ordinal measure ordinal - * @return measure value - */ - @Override public BigDecimal getBigDecimalMeasureValue(int ordinal) { - return getBigDecimalMeasureValue(ordinal, rowMapping[currentRow]); - } - - /** * Fill the column data to vector */ public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5d26701/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java index e1ffb62..6527930 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java @@ -16,8 +16,6 @@ */ package org.apache.carbondata.core.scan.result.impl; -import java.math.BigDecimal; - import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; import org.apache.carbondata.core.scan.result.AbstractScannedResult; @@ -88,57 +86,4 @@ public class NonFilterQueryScannedResult extends AbstractScannedResult { return currentRow; } - /** - * Return the dimension data based on dimension ordinal - * - * @param dimensionOrdinal dimension ordinal - * @return dimension data - */ - @Override public byte[] getDimensionKey(int dimensionOrdinal) { - return getDimensionData(dimensionOrdinal, currentRow); - } - - /** - * Below method will be used to to check whether measure value is null or - * for a measure - * - * @param ordinal measure ordinal - * @return is null or not - */ - @Override public boolean isNullMeasureValue(int ordinal) { - return isNullMeasureValue(ordinal, currentRow); - } - - /** - * Below method will be used to get the measure value for measure of long - * data type - * - * @param ordinal measure ordinal - * @return long value of measure - */ - @Override public long getLongMeasureValue(int ordinal) { - return getLongMeasureValue(ordinal, currentRow); - } - - /** - * Below method will be used to get the value of measure of double type - * - * @param ordinal measure ordinal - * @return measure value - */ - @Override public double getDoubleMeasureValue(int ordinal) { - return getDoubleMeasureValue(ordinal, currentRow); - } - - /** - * Below method will be used to get the data of big decimal type of a - * measure - * - * @param ordinal measure ordinal - * @return measure value - */ - @Override public BigDecimal getBigDecimalMeasureValue(int ordinal) { - return getBigDecimalMeasureValue(ordinal, currentRow); - } - } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5d26701/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java index b958445..d4e084d 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java @@ -696,46 +696,6 @@ public final class ValueCompressionUtil { return compressModel; } - public static byte[] convertToBytes(short[] values) { - ByteBuffer buffer = ByteBuffer.allocate(values.length * 2); - for (short val : values) { - buffer.putShort(val); - } - return buffer.array(); - } - - public static byte[] convertToBytes(int[] values) { - ByteBuffer buffer = ByteBuffer.allocate(values.length * 4); - for (int val : values) { - buffer.putInt(val); - } - return buffer.array(); - } - - public static byte[] convertToBytes(float[] values) { - ByteBuffer buffer = ByteBuffer.allocate(values.length * 4); - for (float val : values) { - buffer.putFloat(val); - } - return buffer.array(); - } - - public static byte[] convertToBytes(long[] values) { - ByteBuffer buffer = ByteBuffer.allocate(values.length * 8); - for (long val : values) { - buffer.putLong(val); - } - return buffer.array(); - } - - public static byte[] convertToBytes(double[] values) { - ByteBuffer buffer = ByteBuffer.allocate(values.length * 8); - for (double val : values) { - buffer.putDouble(val); - } - return buffer.array(); - } - public static short[] convertToShortArray(ByteBuffer buffer, int length) { buffer.rewind(); short[] values = new short[length / 2]; @@ -756,16 +716,6 @@ public final class ValueCompressionUtil { return values; } - public static float[] convertToFloatArray(ByteBuffer buffer, int length) { - buffer.rewind(); - float[] values = new float[length / 4]; - - for (int i = 0; i < values.length; i++) { - values[i] = buffer.getFloat(); - } - return values; - } - public static long[] convertToLongArray(ByteBuffer buffer, int length) { buffer.rewind(); long[] values = new long[length / 8]; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e5d26701/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java index a72197f..6252ca1 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java @@ -445,15 +445,6 @@ public class ValueCompressionUtilTest { assertEquals(result.getClass(), CompressionNonDecimalMaxMinShort.class); } - @Test public void testToConvertToBytesForInt() { - int[] input = { 120000, 200000, 300000 }; - byte[] result = ValueCompressionUtil.convertToBytes(input); - byte[] expectedResult = { 0, 1, -44 }; - for (int i = 0; i < input.length; i++) { - assertEquals(expectedResult[i], result[i]); - } - } - @Test public void testToConvertToInt() { ByteBuffer byteBuffer = ByteBuffer.allocate(4); byteBuffer.putInt(123); @@ -464,34 +455,6 @@ public class ValueCompressionUtilTest { } } - @Test public void testToConvertToBytesForFloat() { - float[] input = { 120.001f, 200.012f, 300.123f }; - byte[] result = ValueCompressionUtil.convertToBytes(input); - byte[] expectedResult = { 66, -16, 0, -125, 67, 72, 3, 18, 67, -106, 15, -66 }; - for (int i = 0; i < result.length; i++) { - assertEquals(expectedResult[i], result[i]); - } - } - - @Test public void testToConvertToFloat() { - ByteBuffer byteBuffer = ByteBuffer.allocate(4); - byteBuffer.putFloat(123.23f); - float[] result = ValueCompressionUtil.convertToFloatArray(byteBuffer, 4); - float[] expectedResult = { 123.23f }; - for (int i = 0; i < result.length; i++) { - assertEquals(expectedResult[i], result[i],2); - } - } - - @Test public void testToConvertToBytesForShort() { - short[] input = { 20000, -30000, 23 }; - byte[] result = ValueCompressionUtil.convertToBytes(input); - byte[] expectedResult = { 78, 32, -118, -48, 0, 23 }; - for (int i = 0; i < result.length; i++) { - assertEquals(expectedResult[i], result[i]); - } - } - @Test public void testToConvertToShort() { ByteBuffer byteBuffer = ByteBuffer.allocate(2); byteBuffer.putShort((short) 3); @@ -502,15 +465,6 @@ public class ValueCompressionUtilTest { } } - @Test public void testToConvertToBytesForLong() { - long[] input = { 2000000, 300654 }; - byte[] result = ValueCompressionUtil.convertToBytes(input); - byte[] expectedResult = { 0, 0, 0, 0, 0, 30, -124, -128, 0, 0, 0, 0, 0, 4, -106, 110 }; - for (int i = 0; i < result.length; i++) { - assertEquals(expectedResult[i], result[i]); - } - } - @Test public void testToConvertToLong() { ByteBuffer byteBuffer = ByteBuffer.allocate(8); byteBuffer.putLong(321654987); @@ -521,15 +475,6 @@ public class ValueCompressionUtilTest { } } - @Test public void testToConvertToBytesForDouble() { - double[] input = { 20.236, 30.001 }; - byte[] result = ValueCompressionUtil.convertToBytes(input); - byte[] expectedResult = { 64, 52, 60, 106, 126, -7, -37, 35, 64, 62, 0, 65, -119, 55, 75, -57 }; - for (int i = 0; i < result.length; i++) { - assertEquals(expectedResult[i], result[i]); - } - } - @Test public void testToConvertToDouble() { ByteBuffer byteBuffer = ByteBuffer.allocate(8); byteBuffer.putDouble(3216.54987);
