DRILL-1007: Unify handling of negative values in Decimal28 and Decimal38 data type
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0dec032f Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0dec032f Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0dec032f Branch: refs/heads/master Commit: 0dec032fb922b8eaae975d33adeb06e2407797ed Parents: 199f467 Author: Mehant Baid <[email protected]> Authored: Tue Jun 17 19:06:48 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Wed Jun 18 20:28:59 2014 -0700 ---------------------------------------------------------------------- .../templates/ConvertToNullableHolder.java | 2 +- .../Decimal/CastDecimalDenseDecimalSparse.java | 2 +- .../templates/Decimal/CastDecimalInt.java | 2 +- .../templates/Decimal/CastDecimalSimilar.java | 3 +- .../Decimal/CastDecimalSparseDecimalDense.java | 3 +- .../templates/Decimal/CastDecimalVarchar.java | 2 +- .../templates/Decimal/CastFloatDecimal.java | 8 +- .../templates/Decimal/CastIntDecimal.java | 2 +- .../templates/Decimal/CastSrcDecimalSimple.java | 4 +- .../templates/Decimal/CastVarCharDecimal.java | 4 +- .../templates/Decimal/DecimalFunctions.java | 69 ++++++++------- .../templates/DecimalAggrTypeFunctions1.java | 30 +++---- .../codegen/templates/FixedValueVectors.java | 42 --------- .../templates/ParquetOutputRecordWriter.java | 2 +- .../main/codegen/templates/ValueHolders.java | 13 ++- .../drill/exec/vector/ValueHolderHelper.java | 4 +- .../drill/exec/physical/impl/TestDecimal.java | 91 -------------------- 17 files changed, 87 insertions(+), 196 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java b/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java index 548d645..0d2be65 100644 --- a/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java +++ b/exec/java-exec/src/main/codegen/templates/ConvertToNullableHolder.java @@ -57,7 +57,7 @@ public class ${className} implements DrillSimpleFunc { output.scale = input.scale; output.precision = input.precision; <#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")> - output.sign = input.sign; + output.setSign(input.getSign()); output.start = input.start; output.buffer = input.buffer; <#else> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java index a486cf2..e9c03c8 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalDenseDecimalSparse.java @@ -63,7 +63,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{ } out.scale = (int) scale.value; out.precision = (int) precision.value; - out.sign = in.sign; + out.setSign(in.getSign()); /* We store base 1 Billion integers in our representation, which requires * 30 bits, but a typical integer requires 32 bits. In our dense representation http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java index a89965f..cf7a634 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalInt.java @@ -96,7 +96,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc { out.value = (${type.javatype}) ((out.value * org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) + in.getInteger(i)); } - if (in.sign == true) { + if (in.getSign() == true) { out.value *= -1; } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java index a59cfdb..9a2d3e5 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSimilar.java @@ -61,7 +61,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{ out.start = 0; out.scale = (int) scale.value; out.precision = (int) precision.value; - out.sign = in.sign; + boolean sign = (in.getSign()); // Re initialize the buffer everytime for (int i = 0; i < ${type.arraySize}; i++) { @@ -79,6 +79,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{ if (in.scale != out.scale) { org.apache.drill.common.util.DecimalUtility.roundDecimal(out.buffer, out.start, out.nDecimalDigits, out.scale, in.scale); } + out.setSign(sign); } } </#if> <#-- type.major --> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java index 4798b34..e9e9711 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalSparseDecimalDense.java @@ -66,8 +66,6 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{ out.scale = (int) scale.value; out.precision = (int) precision.value; - out.sign = in.sign; - /* Before converting from a sparse representation to a dense representation * we need to convert it to an intermediate representation. In the sparse * representation we separate out the scale and the integer part of the decimal @@ -169,6 +167,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{ // Set the bytes in the buffer out.buffer.setBytes(dstIndex, intermediateBytes, 1, (size - 1)); + out.setSign(in.getSign()); } } </#if> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java index 783165f..dc8eca5 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalVarchar.java @@ -146,7 +146,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc { // If we have valid digits print '-' sign - if ((in.sign == true) && index < ${type.arraySize}) { + if ((in.getSign() == true) && index < ${type.arraySize}) { str.append("-"); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java index 9ebb86f..903634b 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastFloatDecimal.java @@ -70,14 +70,14 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc { out.start = 0; out.buffer = buffer; - if (in.value < 0) { - out.sign = true; - } - // Initialize the buffer for (int i = 0; i < ${type.arraySize}; i++) { out.setInteger(i, 0); } + + if (in.value < 0) { + out.setSign(true); + } // Assign the integer part of the decimal to the output holder org.apache.drill.common.util.DecimalUtility.getSparseFromBigDecimal(new java.math.BigDecimal(String.valueOf(in.value)), out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java index 979e7e2..78bde2e 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastIntDecimal.java @@ -79,7 +79,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc { // check if input is a negative number and store the sign if (in.value < 0) { - out.sign = true; + out.setSign(true); } // Figure out how many array positions to be left for the scale part http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java index b298c66..7ef806f 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastSrcDecimalSimple.java @@ -72,7 +72,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc { out.buffer = buffer; out.start = 0; - out.sign = (in.value < 0) ? true : false; + out.setSign((in.value < 0)); /* Since we will be dividing the decimal value with base 1 billion * we don't want negative results if the decimal is negative. @@ -169,7 +169,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{ out.buffer = buffer; out.start = 0; - out.sign = (in.value < 0) ? true : false; + out.setSign((in.value < 0)); /* Since we will be dividing the decimal value with base 1 billion * we don't want negative results if the decimal is negative. http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java index 9ca0533..ceebc0a 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java @@ -186,6 +186,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc { out.scale = (int) scale.value; out.precision = (int) precision.value; + boolean sign = false; // Initialize the output buffer for (int i = 0; i < ${type.arraySize}; i++) { @@ -207,7 +208,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc { if (next == '-') { readIndex++; - out.sign = true; + sign = true; } if (next == '.') { @@ -340,6 +341,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc { int padding = (int) org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) (org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - ndigits)); out.setInteger(decimalBufferIndex, out.getInteger(decimalBufferIndex) * padding); } + out.setSign(sign); } } </#if> <#-- type.major --> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java index a41fb20..3f5b5cd 100644 --- a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java +++ b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalFunctions.java @@ -23,9 +23,9 @@ import org.apache.drill.exec.expr.annotations.Workspace; <#macro compareBlock holderType left right absCompare output> outside:{ - ${output} = org.apache.drill.common.util.DecimalUtility.compareSparseBytes(left.buffer, left.start, left.sign, + ${output} = org.apache.drill.common.util.DecimalUtility.compareSparseBytes(left.buffer, left.start, left.getSign(), left.scale, left.precision, right.buffer, - right.start, right.sign, right.precision, + right.start, right.getSign(), right.precision, right.scale, left.WIDTH, left.nDecimalDigits, ${absCompare}); } @@ -295,9 +295,9 @@ public class ${type.name}Functions { * causes the sign of one of the inputs to change and hence it effectively * becomes addition */ - if (left.sign != right.sign) { + if (left.getSign() != right.getSign()) { <@addBlock holderType=type.name in1="left" in2="right" result="result"/> - result.sign = left.sign; + result.setSign(left.getSign()); } else { /* Sign of the inputs are the same, meaning we have to perform subtraction * For subtraction we need left input to be greater than right input @@ -313,10 +313,10 @@ public class ${type.name}Functions { } //Determine the sign of the result - if ((left.sign == false && cmp == -1) || (left.sign == true && cmp == 1)) { - result.sign = true; + if ((left.getSign() == false && cmp == -1) || (left.getSign() == true && cmp == 1)) { + result.setSign(true); } else { - result.sign = false; + result.setSign(false); } } @@ -347,7 +347,7 @@ public class ${type.name}Functions { } /* If sign is different use the subtraction logic */ - if (left.sign != right.sign) { + if (left.getSign() != right.getSign()) { /* Subtract logic assumes, left input is greater than right input * swap if necessary @@ -357,17 +357,17 @@ public class ${type.name}Functions { if (cmp == -1) { <@subtractBlock holderType=type.name in1="right" in2="left" result="result"/> - result.sign = right.sign; + result.setSign(right.getSign()); } else { <@subtractBlock holderType=type.name in1="left" in2="right" result="result"/> - result.sign = left.sign; + result.setSign(left.getSign()); } } else { /* Sign of the two input decimals is the same, use the add logic */ <@addBlock holderType=type.name in1="left" in2="right" result="result"/> - result.sign = left.sign; + result.setSign(left.getSign()); } } } @@ -486,7 +486,7 @@ public class ${type.name}Functions { result.setInteger(outputIndex--, 0); } - result.sign = (left.sign == right.sign) ? false : true; + result.setSign(left.getSign() != right.getSign()); } } @@ -609,7 +609,7 @@ public class ${type.name}Functions { boolean zeroValue = true; - if (in.sign == true) { + if (in.getSign() == true) { out.value = -1; } else { for (int i = 0; i < ${type.storage}; i++) { @@ -637,7 +637,7 @@ public class ${type.name}Functions { out.precision = in.precision; out.buffer = in.buffer; out.start = in.start; - out.sign = in.sign; + boolean sign = in.getSign(); // Indicates whether we need to add 1 to the integer part, while performing ceil int carry = 0; @@ -645,7 +645,7 @@ public class ${type.name}Functions { int scaleStartIndex = ${type.storage} - org.apache.drill.common.util.DecimalUtility.roundUp(in.scale); int srcIntIndex = scaleStartIndex - 1; - if (out.sign == false) { + if (sign == false) { // For negative values ceil we don't need to increment the integer part while (scaleStartIndex < ${type.storage}) { if (out.getInteger(scaleStartIndex) != 0) { @@ -684,6 +684,8 @@ public class ${type.name}Functions { } } } + // set the sign + out.setSign(sign); } } @@ -701,7 +703,7 @@ public class ${type.name}Functions { out.precision = in.precision; out.buffer = in.buffer; out.start = in.start; - out.sign = in.sign; + boolean sign = in.getSign(); // Indicates whether we need to decrement 1 from the integer part, while performing floor, done for -ve values int carry = 0; @@ -709,7 +711,7 @@ public class ${type.name}Functions { int scaleStartIndex = ${type.storage} - org.apache.drill.common.util.DecimalUtility.roundUp(in.scale); int srcIntIndex = scaleStartIndex - 1; - if (out.sign == true) { + if (sign == true) { // For negative values ceil we don't need to increment the integer part while (scaleStartIndex < ${type.storage}) { if (out.getInteger(scaleStartIndex) != 0) { @@ -730,7 +732,6 @@ public class ${type.name}Functions { while (destIndex >= 0) { out.setInteger(destIndex--, 0); } - // Add the carry if (carry != 0) { destIndex = ${type.storage} - 1; @@ -748,6 +749,8 @@ public class ${type.name}Functions { } } } + // set the sign + out.setSign(sign); } } @@ -765,7 +768,7 @@ public class ${type.name}Functions { out.precision = in.precision; out.buffer = in.buffer; out.start = in.start; - out.sign = in.sign; + boolean sign = in.getSign(); // Integer part's src index int srcIntIndex = ${type.storage} - org.apache.drill.common.util.DecimalUtility.roundUp(in.scale) - 1; @@ -780,6 +783,8 @@ public class ${type.name}Functions { while (destIndex >= 0) { out.setInteger(destIndex--, 0); } + // set the sign + out.setSign(sign); } } @@ -798,7 +803,7 @@ public class ${type.name}Functions { result.precision = left.precision; result.buffer = left.buffer; result.start = left.start; - result.sign = left.sign; + boolean sign = left.getSign(); int newScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(right.value); int origScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(left.scale); @@ -855,6 +860,8 @@ public class ${type.name}Functions { } } } + // set the sign + result.setSign(sign); } } @@ -872,7 +879,7 @@ public class ${type.name}Functions { out.precision = in.precision; out.buffer = in.buffer; out.start = in.start; - out.sign = in.sign; + boolean sign = in.getSign(); boolean roundUp = false; @@ -915,6 +922,8 @@ public class ${type.name}Functions { } } } + // set the sign + out.setSign(sign); } } @@ -933,9 +942,11 @@ public class ${type.name}Functions { result.precision = left.precision; result.buffer = left.buffer; result.start = left.start; - result.sign = left.sign; + boolean sign = left.getSign(); org.apache.drill.common.util.DecimalUtility.roundDecimal(result.buffer, result.start, result.nDecimalDigits, result.scale, left.scale); + // set the sign + result.setSign(sign); } } @@ -1074,7 +1085,7 @@ public class ${type.name}Functions { public void setup(RecordBatch incoming) {} public void eval() { - out.value = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.sign, right.buffer, right.start, right.sign, left.WIDTH); + out.value = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH); } } @@ -1087,7 +1098,7 @@ public class ${type.name}Functions { public void setup(RecordBatch incoming) {} public void eval() { - int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.sign, right.buffer, right.start, right.sign, left.WIDTH); + int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH); out.value = cmp == -1 ? 1 : 0; } } @@ -1101,7 +1112,7 @@ public class ${type.name}Functions { public void setup(RecordBatch incoming) {} public void eval() { - int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.sign, right.buffer, right.start, right.sign, left.WIDTH); + int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH); out.value = cmp < 1 ? 1 : 0; } } @@ -1115,7 +1126,7 @@ public class ${type.name}Functions { public void setup(RecordBatch incoming) {} public void eval() { - int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.sign, right.buffer, right.start, right.sign, left.WIDTH); + int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH); out.value = cmp == 1 ? 1 : 0; } } @@ -1129,7 +1140,7 @@ public class ${type.name}Functions { public void setup(RecordBatch incoming) {} public void eval() { - int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.sign, right.buffer, right.start, right.sign, left.WIDTH); + int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH); out.value = cmp > -1 ? 1 : 0; } } @@ -1143,7 +1154,7 @@ public class ${type.name}Functions { public void setup(RecordBatch incoming) {} public void eval() { - int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.sign, right.buffer, right.start, right.sign, left.WIDTH); + int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH); out.value = cmp == 0 ? 1 : 0; } } @@ -1159,7 +1170,7 @@ public class ${type.name}Functions { public void eval() { - int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.sign, right.buffer, right.start, right.sign, left.WIDTH); + int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(left.buffer, left.start, left.getSign(), right.buffer, right.start, right.getSign(), left.WIDTH); out.value = cmp != 0 ? 1 : 0; } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/DecimalAggrTypeFunctions1.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/DecimalAggrTypeFunctions1.java b/exec/java-exec/src/main/codegen/templates/DecimalAggrTypeFunctions1.java index 5e02ff0..c5a927c 100644 --- a/exec/java-exec/src/main/codegen/templates/DecimalAggrTypeFunctions1.java +++ b/exec/java-exec/src/main/codegen/templates/DecimalAggrTypeFunctions1.java @@ -72,13 +72,13 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu for (int i = 0; i < value.nDecimalDigits; i++) { value.setInteger(i, 0xFFFFFFFF); } - value.sign = true; + value.setSign(true); <#elseif aggrtype.funcName == "min"> for (int i = 0; i < value.nDecimalDigits; i++) { value.setInteger(i, 0x7FFFFFFF); } // Set sign to be positive so initial value is maximum - value.sign = false; + value.setSign(false); value.precision = ${type.runningType}Holder.maxPrecision; </#if> <#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18"> @@ -101,21 +101,21 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu value.value++; <#elseif aggrtype.funcName == "max"> <#if type.outputType.endsWith("Dense")> - int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(in.buffer, in.start, in.sign, value.buffer, value.start, value.sign, in.WIDTH); + int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(in.buffer, in.start, in.getSign(), value.buffer, value.start, value.getSign(), in.WIDTH); if (cmp == 1) { in.buffer.getBytes(in.start, value.buffer, 0, value.WIDTH); - value.sign = in.sign; + value.setSign(in.getSign()); value.scale = in.scale; value.precision = in.precision; } <#elseif type.outputType.endsWith("Sparse")> - int cmp = org.apache.drill.common.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.sign, + int cmp = org.apache.drill.common.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.getSign(), in.scale, in.precision, value.buffer, - value.start, value.sign, value.precision, + value.start, value.getSign(), value.precision, value.scale, in.WIDTH, in.nDecimalDigits, false); if (cmp == 1) { in.buffer.getBytes(in.start, value.buffer, 0, value.WIDTH); - value.sign = in.sign; + value.setSign(in.getSign()); value.scale = in.scale; value.precision = in.precision; } @@ -124,21 +124,21 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu </#if> <#elseif aggrtype.funcName == "min"> <#if type.outputType.endsWith("Dense")> - int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(in.buffer, in.start, in.sign, value.buffer, value.start, value.sign, in.WIDTH); + int cmp = org.apache.drill.common.util.DecimalUtility.compareDenseBytes(in.buffer, in.start, in.getSign(), value.buffer, value.start, value.getSign(), in.WIDTH); if (cmp == -1) { in.buffer.getBytes(in.start, value.buffer, 0, value.WIDTH); - value.sign = in.sign; + value.setSign(in.getSign()); value.scale = in.scale; value.precision = in.precision; } <#elseif type.outputType.endsWith("Sparse")> - int cmp = org.apache.drill.common.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.sign, + int cmp = org.apache.drill.common.util.DecimalUtility.compareSparseBytes(in.buffer, in.start, in.getSign(), in.scale, in.precision, value.buffer, - value.start, value.sign, value.precision, + value.start, value.getSign(), value.precision, value.scale, in.WIDTH, in.nDecimalDigits, false); if (cmp == -1) { in.buffer.getBytes(in.start, value.buffer, 0, value.WIDTH); - value.sign = in.sign; + value.setSign(in.getSign()); value.scale = in.scale; value.precision = in.precision; } @@ -159,7 +159,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu <#if type.outputType.endsWith("Dense") || type.outputType.endsWith("Sparse")> out.buffer = value.buffer; out.start = value.start; - out.sign = value.sign; + out.setSign(value.getSign()); <#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18"> out.value = value.value; </#if> @@ -184,9 +184,9 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu } <#if aggrtype.funcName == "min"> // Set sign to be positive so initial value is maximum - value.sign = false; + value.setSign(false); <#elseif aggrtype.funcName == "max"> - value.sign = true; + value.setSign(true); </#if> <#elseif type.outputType == "Decimal9" || type.outputType == "Decimal18"> value.value = ${type.initValue}; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java index 7ff7327..d9715a7 100644 --- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java @@ -402,20 +402,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F * big endian ordering makes sense for this purpose. So we have to deal with * the sign bit for the two representation in a slightly different fashion */ - - // Get the sign of the decimal - <#if minor.class.endsWith("Sparse")> - if ((holder.buffer.getInt(holder.start) & 0x80000000) != 0) { - <#elseif minor.class.endsWith("Dense")> - if ((holder.buffer.getInt(holder.start) & 0x00000080) != 0) { - </#if> - holder.sign = true; - } - holder.scale = getField().getScale(); holder.precision = getField().getPrecision(); - - } public void get(int index, Nullable${minor.class}Holder holder) { @@ -423,15 +411,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F holder.start = index * ${type.width}; holder.buffer = data; - - // Get the sign the of the decimal - <#if minor.class.endsWith("Sparse")> - if ((holder.buffer.getInt(holder.start) & 0x80000000) != 0) { - <#elseif minor.class.endsWith("Dense")> - if ((holder.buffer.getInt(holder.start) & 0x00000080) != 0) { - </#if> - holder.sign = true; - } } @Override @@ -690,31 +669,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public void set(int index, ${minor.class}Holder holder){ data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width}); - - // Set the sign of the decimal - if (holder.sign == true) { - int value = data.getInt(index * ${type.width}); - <#if minor.class.endsWith("Sparse")> - data.setInt(index * ${type.width}, (value | 0x80000000)); - <#elseif minor.class.endsWith("Dense")> - data.setInt(index * ${type.width}, (value | 0x00000080)); - </#if> - - } } void set(int index, Nullable${minor.class}Holder holder){ data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width}); - - // Set the sign of the decimal - if (holder.sign == true) { - int value = data.getInt(index * ${type.width}); - <#if minor.class.endsWith("Sparse")> - data.setInt(index * ${type.width}, (value | 0x80000000)); - <#elseif minor.class.endsWith("Dense")> - data.setInt(index * ${type.width}, (value | 0x00000080)); - </#if> - } } public boolean setSafe(int index, Nullable${minor.class}Holder holder){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java index 07bd449..92267e7 100644 --- a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java @@ -163,7 +163,7 @@ public abstract class ParquetOutputRecordWriter implements RecordWriter { byte[] bytes = DecimalUtility.getBigDecimalFromSparse( valueHolder.buffer, valueHolder.start, ${minor.class}Holder.nDecimalDigits, valueHolder.scale).unscaledValue().toByteArray(); byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.${minor.class?upper_case})]; - if (valueHolder.sign) { + if (valueHolder.getSign()) { Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF); } else { Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/codegen/templates/ValueHolders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/ValueHolders.java b/exec/java-exec/src/main/codegen/templates/ValueHolders.java index cdc4d74..46cf9e0 100644 --- a/exec/java-exec/src/main/codegen/templates/ValueHolders.java +++ b/exec/java-exec/src/main/codegen/templates/ValueHolders.java @@ -72,7 +72,6 @@ public final class ${className} implements ValueHolder{ public int precision; public static final int maxPrecision = ${minor.maxPrecisionDigits}; <#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")> - public boolean sign; public int start; public ByteBuf buffer; public static final int nDecimalDigits = ${minor.nDecimalDigits}; @@ -96,6 +95,18 @@ public final class ${className} implements ValueHolder{ buffer.setInt(start + (index * 4), value); } + public void setSign(boolean sign) { + // Set MSB to 1 if sign is negative + if (sign == true) { + int value = getInteger(0); + setInteger(0, (value | 0x80000000)); + } + } + + public boolean getSign() { + return ((buffer.getInt(start) & 0x80000000) != 0); + } + <#else> public ${minor.javaType!type.javaType} value; </#if> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java index fb9dfd0..6968c21 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java @@ -85,7 +85,7 @@ public class ValueHolderHelper { dch.scale = bigDecimal.scale(); dch.precision = bigDecimal.precision(); - dch.sign = (bigDecimal.signum() == -1); + dch.setSign(bigDecimal.signum() == -1); dch.start = 0; dch.buffer = Unpooled.wrappedBuffer(new byte[5 * DecimalUtility.integerSize]); @@ -104,7 +104,7 @@ public class ValueHolderHelper { dch.scale = bigDecimal.scale(); dch.precision = bigDecimal.precision(); - dch.sign = (bigDecimal.signum() == -1); + dch.setSign(bigDecimal.signum() == -1); dch.start = 0; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0dec032f/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java index f485378..093366f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java @@ -278,95 +278,4 @@ public class TestDecimal extends PopUnitTestBase{ } } } - - @Test - public void testDenseSparseConversion() throws Exception { - - /* Function checks the following workflow - * VarChar -> Sparse -> Dense -> Sort(Dense) -> Sparse -> VarChar - */ - try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); - Drillbit bit = new Drillbit(CONFIG, serviceSet); - DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) { - - // run query. - bit.run(); - client.connect(); - List<QueryResultBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/decimal/test_decimal_dense_sparse.json"), Charsets.UTF_8) - .replace("#{TEST_FILE}", "/input_complex_decimal.json") - ); - - RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator()); - - QueryResultBatch batch = results.get(0); - assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData())); - - String sortOutput[] = {"-100000000001.000000000000", "-100000000001.000000000000", "-0.120000000000", "0.100000000001", "11.123456789012", "100000000001.123456789001", "123456789123456789.000000000000"}; - - Iterator<VectorWrapper<?>> itr = batchLoader.iterator(); - - // Check the output of sort - VectorWrapper<?> v = itr.next(); - ValueVector.Accessor accessor = v.getValueVector().getAccessor(); - - for (int i = 0; i < accessor.getValueCount(); i++) { - assertEquals(accessor.getObject(i).toString(), sortOutput[i]); - } - assertEquals(7, accessor.getValueCount()); - - batchLoader.clear(); - for (QueryResultBatch result : results) { - result.release(); - } - } - } - - @Test - public void testDenseSparseConversion1() throws Exception { - - /* Function checks the following cast sequence. - * VarChar -> Decimal28Sparse - * Decimal28Sparse -> Decimal28Dense - * Decimal28Dense -> Decimal38Dense - * - * Goal is to test the similar casting functionality 28Dense -> 38Dense - * - */ - try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); - Drillbit bit = new Drillbit(CONFIG, serviceSet); - DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) { - - // run query. - bit.run(); - client.connect(); - List<QueryResultBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, - Files.toString(FileUtils.getResourceAsFile("/decimal/test_decimal_sparse_dense_dense.json"), Charsets.UTF_8) - .replace("#{TEST_FILE}", "/input_simple_decimal.json") - ); - - RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator()); - - QueryResultBatch batch = results.get(0); - assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData())); - - String output[] = {"99.0000", "11.1234", "0.1000", "-0.1200", "-123.1234", "-1.0001"}; - - Iterator<VectorWrapper<?>> itr = batchLoader.iterator(); - - // Check the output of sort - VectorWrapper<?> v = itr.next(); - ValueVector.Accessor accessor = v.getValueVector().getAccessor(); - - for (int i = 0; i < accessor.getValueCount(); i++) { - assertEquals(accessor.getObject(i).toString(), output[i]); - } - assertEquals(6, accessor.getValueCount()); - - batchLoader.clear(); - for (QueryResultBatch result : results) { - result.release(); - } - } - } }
