Switch to DrillBuf Add @Inject DrillBuf Move comparison functions to memory sensitive ones Add scalar replacement functionality for value holders Simplify date parsing function Add local compiled code caching
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5c5cef06 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5c5cef06 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5c5cef06 Branch: refs/heads/master Commit: 5c5cef06881c0ee8d1c2b203e166dffac69bfd14 Parents: fb93576 Author: Jacques Nadeau <[email protected]> Authored: Wed Jul 9 14:01:35 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Sun Aug 24 21:00:43 2014 -0700 ---------------------------------------------------------------------- .../expression/ExpressionStringBuilder.java | 19 +- .../common/expression/ValueExpressions.java | 152 ++-- .../drill/common/util/CoreDecimalUtility.java | 76 ++ .../DecimalScalePrecisionDivideFunction.java | 3 +- .../util/DecimalScalePrecisionModFunction.java | 3 +- .../drill/common/util/DecimalUtility.java | 735 ---------------- .../codegen/templates/ObjectInspectors.java | 2 +- exec/java-exec/pom.xml | 18 +- exec/java-exec/src/main/codegen/data/Casts.tdd | 2 +- .../src/main/codegen/data/ValueVectorTypes.tdd | 66 +- .../src/main/codegen/includes/vv_imports.ftl | 4 +- .../main/codegen/templates/CastDateVarChar.java | 7 +- .../main/codegen/templates/CastFunctions.java | 2 + .../templates/CastFunctionsSrcVarLen.java | 2 + .../CastFunctionsSrcVarLenTargetVarLen.java | 2 + .../templates/CastFunctionsTargetVarLen.java | 8 +- .../codegen/templates/CastIntervalInterval.java | 6 +- .../codegen/templates/CastIntervalVarChar.java | 26 +- .../main/codegen/templates/CastVarCharDate.java | 11 +- .../codegen/templates/CastVarCharInterval.java | 6 +- .../codegen/templates/ComparisonFunctions.java | 46 +- .../templates/ConvertToNullableHolder.java | 6 +- .../templates/DateIntervalAggrFunctions1.java | 28 +- .../DateDateArithmeticFunctions.java | 6 +- .../DateIntervalArithmeticFunctions.java | 6 +- .../DateToCharFunctions.java | 7 +- .../DateIntervalFunctionTemplates/Extract.java | 12 +- .../IntervalIntervalArithmetic.java | 12 +- .../IntervalNumericArithmetic.java | 10 +- .../ToTimeStampFunction.java | 2 +- .../templates/DateIntervalFunctions.java | 28 +- .../Decimal/CastDecimalDenseDecimalSparse.java | 34 +- .../templates/Decimal/CastDecimalFloat.java | 12 +- .../templates/Decimal/CastDecimalInt.java | 18 +- .../templates/Decimal/CastDecimalSimilar.java | 22 +- .../Decimal/CastDecimalSparseDecimalDense.java | 38 +- .../templates/Decimal/CastDecimalVarchar.java | 51 +- .../templates/Decimal/CastDownwardDecimal.java | 44 +- .../templates/Decimal/CastFloatDecimal.java | 20 +- .../templates/Decimal/CastIntDecimal.java | 26 +- .../templates/Decimal/CastSrcDecimalSimple.java | 57 +- .../templates/Decimal/CastVarCharDecimal.java | 58 +- .../Decimal/DecimalAggrTypeFunctions1.java | 51 +- .../Decimal/DecimalAggrTypeFunctions2.java | 14 +- .../templates/Decimal/DecimalFunctions.java | 348 ++++---- .../codegen/templates/FixedValueVectors.java | 221 +++-- .../codegen/templates/HolderReaderImpl.java | 31 +- .../templates/IntervalAggrFunctions2.java | 8 +- .../templates/JsonOutputRecordWriter.java | 2 +- .../codegen/templates/NullableValueVectors.java | 106 ++- .../templates/NumericToCharFunctions.java | 12 +- .../templates/ParquetOutputRecordWriter.java | 4 +- .../codegen/templates/RepeatedValueVectors.java | 18 +- .../main/codegen/templates/SqlAccessors.java | 2 +- .../main/codegen/templates/ValueHolders.java | 164 ++-- .../templates/VarCharAggrFunctions1.java | 56 +- .../templates/VariableLengthVectors.java | 72 +- .../java/io/netty/buffer/AccountingByteBuf.java | 866 ------------------- .../src/main/java/io/netty/buffer/DrillBuf.java | 674 +++++++++++++++ .../java/io/netty/buffer/FakeAllocator.java | 137 +++ .../netty/buffer/PooledByteBufAllocatorL.java | 2 +- .../netty/buffer/UnsafeDirectLittleEndian.java | 24 +- .../cache/VectorAccessibleSerializable.java | 5 +- .../drill/exec/compile/ByteCodeLoader.java | 4 +- .../drill/exec/compile/ClassTransformer.java | 103 +-- .../apache/drill/exec/compile/CodeCompiler.java | 89 ++ .../exec/compile/DrillInitMethodVisitor.java | 44 + .../apache/drill/exec/compile/MergeAdapter.java | 110 ++- .../exec/compile/TemplateClassDefinition.java | 17 +- .../exec/compile/bytecode/DirectSorter.java | 33 + .../compile/bytecode/InstructionModifier.java | 259 ++++++ .../compile/bytecode/ReplacingBasicValue.java | 57 ++ .../compile/bytecode/ReplacingInterpreter.java | 91 ++ .../compile/bytecode/ScalarReplacementNode.java | 66 ++ .../bytecode/ScalarReplacementTypes.java | 117 +++ .../bytecode/TrackingInstructionList.java | 69 ++ .../exec/compile/bytecode/ValueHolderIden.java | 223 +++++ .../bytecode/ValueHolderReplacementVisitor.java | 75 ++ .../exec/compile/sig/CodeGeneratorMethod.java | 18 +- .../exec/compile/sig/GeneratorMapping.java | 12 +- .../drill/exec/compile/sig/MappingSet.java | 4 + .../drill/exec/compile/sig/SignatureHolder.java | 39 +- .../apache/drill/exec/expr/ClassGenerator.java | 6 +- .../apache/drill/exec/expr/CodeGenerator.java | 61 +- .../drill/exec/expr/EvaluationVisitor.java | 138 ++- .../drill/exec/expr/GetSetVectorHelper.java | 175 ++++ .../drill/exec/expr/fn/DrillAggFuncHolder.java | 70 +- .../exec/expr/fn/DrillDecimalAddFuncHolder.java | 3 +- .../expr/fn/DrillDecimalDivScaleFuncHolder.java | 3 +- .../expr/fn/DrillDecimalModScaleFuncHolder.java | 3 +- .../expr/fn/DrillDecimalSumScaleFuncHolder.java | 3 +- .../drill/exec/expr/fn/DrillFuncHolder.java | 35 +- .../drill/exec/expr/fn/FunctionConverter.java | 36 +- .../exec/expr/fn/impl/ByteFunctionHelpers.java | 96 ++ .../exec/expr/fn/impl/CastVarBinaryToDate.java | 43 + .../exec/expr/fn/impl/CastVarCharVar16Char.java | 14 +- .../exec/expr/fn/impl/DateTypeFunctions.java | 21 +- .../drill/exec/expr/fn/impl/HashFunctions.java | 8 +- .../exec/expr/fn/impl/SimpleCastFunctions.java | 19 +- .../expr/fn/impl/StringFunctionHelpers.java | 125 +++ .../exec/expr/fn/impl/StringFunctions.java | 193 ++--- .../expr/fn/impl/conv/BigIntBEConvertTo.java | 9 +- .../exec/expr/fn/impl/conv/BigIntConvertTo.java | 8 +- .../expr/fn/impl/conv/BigIntVLongConvertTo.java | 8 +- .../expr/fn/impl/conv/BooleanByteConvertTo.java | 8 +- .../expr/fn/impl/conv/DateEpochBEConvertTo.java | 8 +- .../expr/fn/impl/conv/DateEpochConvertTo.java | 8 +- .../exec/expr/fn/impl/conv/DoubleConvertTo.java | 8 +- .../exec/expr/fn/impl/conv/FloatConvertTo.java | 8 +- .../exec/expr/fn/impl/conv/IntBEConvertTo.java | 8 +- .../exec/expr/fn/impl/conv/IntConvertTo.java | 8 +- .../expr/fn/impl/conv/IntVIntConvertTo.java | 8 +- .../exec/expr/fn/impl/conv/JsonConvertFrom.java | 20 +- .../exec/expr/fn/impl/conv/JsonConvertTo.java | 25 +- .../expr/fn/impl/conv/SmallIntBEConvertTo.java | 8 +- .../expr/fn/impl/conv/SmallIntConvertTo.java | 8 +- .../expr/fn/impl/conv/TimeEpochBEConvertTo.java | 8 +- .../expr/fn/impl/conv/TimeEpochConvertTo.java | 8 +- .../expr/fn/impl/conv/TinyIntConvertTo.java | 8 +- .../exec/expr/fn/impl/conv/UInt8ConvertTo.java | 8 +- .../org/apache/drill/exec/memory/Accountor.java | 10 +- .../drill/exec/memory/BufferAllocator.java | 11 +- .../drill/exec/memory/TopLevelAllocator.java | 59 +- .../apache/drill/exec/ops/FragmentContext.java | 39 +- .../apache/drill/exec/ops/OperatorContext.java | 29 + .../drill/exec/physical/impl/OutputMutator.java | 8 +- .../drill/exec/physical/impl/ScanBatch.java | 30 +- .../impl/aggregate/HashAggTemplate.java | 47 +- .../exec/physical/impl/common/HashTable.java | 10 +- .../physical/impl/common/HashTableTemplate.java | 68 +- .../exec/physical/impl/common/IndexPointer.java | 24 + .../exec/physical/impl/join/HashJoinBatch.java | 15 +- .../planner/sql/handlers/DefaultSqlHandler.java | 1 + .../org/apache/drill/exec/record/DeadBuf.java | 19 +- .../drill/exec/record/RawFragmentBatch.java | 7 +- .../drill/exec/record/RecordBatchLoader.java | 7 +- .../apache/drill/exec/record/TypedFieldId.java | 9 + .../apache/drill/exec/record/WritableBatch.java | 42 +- .../exec/record/selection/SelectionVector2.java | 23 +- .../drill/exec/resolver/TypeCastRules.java | 2 +- .../drill/exec/rpc/ProtobufLengthDecoder.java | 11 +- .../exec/rpc/data/DataResponseHandler.java | 5 +- .../exec/rpc/data/DataResponseHandlerImpl.java | 4 +- .../apache/drill/exec/rpc/data/DataServer.java | 6 +- .../drill/exec/rpc/user/QueryResultBatch.java | 25 +- .../drill/exec/rpc/user/QueryResultHandler.java | 3 +- .../drill/exec/server/DrillbitContext.java | 11 +- .../server/options/SystemOptionManager.java | 3 +- .../exec/store/easy/json/JSONRecordReader2.java | 6 +- .../columnreaders/FixedByteAlignedReader.java | 13 +- .../NullableFixedByteAlignedReaders.java | 20 +- .../NullableVarLengthValuesColumn.java | 8 +- .../store/parquet/columnreaders/PageReader.java | 12 +- .../columnreaders/VarLengthColumnReaders.java | 77 +- .../columnreaders/VarLengthValuesColumn.java | 8 +- .../parquet2/DrillParquetGroupConverter.java | 59 +- .../exec/store/parquet2/DrillParquetReader.java | 2 +- .../DrillParquetRecordMaterializer.java | 11 +- .../drill/exec/store/pojo/PojoRecordReader.java | 47 +- .../apache/drill/exec/store/pojo/Writers.java | 24 +- .../org/apache/drill/exec/util/ByteBufUtil.java | 45 +- .../apache/drill/exec/util/DecimalUtility.java | 692 +++++++++++++++ .../drill/exec/vector/BaseDataValueVector.java | 27 +- .../drill/exec/vector/BaseValueVector.java | 2 +- .../org/apache/drill/exec/vector/BitVector.java | 11 +- .../drill/exec/vector/FixedWidthVector.java | 12 +- .../apache/drill/exec/vector/ObjectVector.java | 6 +- .../exec/vector/RepeatedFixedWidthVector.java | 6 +- .../vector/RepeatedVariableWidthVector.java | 4 +- .../drill/exec/vector/ValueHolderHelper.java | 43 +- .../apache/drill/exec/vector/ValueVector.java | 10 +- .../drill/exec/vector/VariableWidthVector.java | 12 +- .../apache/drill/exec/vector/VectorTrimmer.java | 4 +- .../drill/exec/vector/complex/MapVector.java | 12 +- .../exec/vector/complex/RepeatedListVector.java | 8 +- .../exec/vector/complex/RepeatedMapVector.java | 14 +- .../exec/vector/complex/fn/JsonReader.java | 31 +- .../vector/complex/fn/JsonReaderWithState.java | 8 +- .../exec/work/batch/SpoolingRawBatchBuffer.java | 7 +- .../java/org/apache/drill/exec/RunRootExec.java | 74 ++ .../apache/drill/exec/client/DumpCatTest.java | 4 + .../exec/compile/TestClassTransformation.java | 9 +- .../exec/compile/TestEvaluationVisitor.java | 2 +- .../compile/bytecode/ReplaceMethodInvoke.java | 105 +++ .../apache/drill/exec/expr/ExpressionTest.java | 2 +- .../fn/impl/TestByteComparisonFunctions.java | 129 +++ .../drill/exec/fn/impl/TestMathFunctions.java | 2 + .../exec/fn/impl/TestNewMathFunctions.java | 2 + .../exec/fn/impl/TestRepeatedFunction.java | 2 + .../exec/physical/impl/TestCastFunctions.java | 14 +- .../physical/impl/TestComparisonFunctions.java | 2 + .../physical/impl/TestConvertFunctions.java | 6 +- .../impl/TestImplicitCastFunctions.java | 2 + .../exec/physical/impl/TestOptiqPlans.java | 1 + .../exec/physical/impl/TestSimpleFunctions.java | 11 +- .../exec/physical/impl/TestStringFunctions.java | 8 +- .../drill/exec/physical/impl/agg/TestAgg.java | 2 + .../physical/impl/common/TestHashTable.java | 2 + .../physical/impl/filter/TestSimpleFilter.java | 3 + .../exec/physical/impl/join/TestHashJoin.java | 6 +- .../exec/physical/impl/join/TestMergeJoin.java | 8 +- .../physical/impl/limit/TestSimpleLimit.java | 4 + .../impl/project/TestSimpleProjection.java | 2 + .../exec/physical/impl/sort/TestSimpleSort.java | 3 + .../physical/impl/svremover/TestSVRemover.java | 2 + .../impl/trace/TestTraceMultiRecordBatch.java | 6 + .../impl/trace/TestTraceOutputDump.java | 2 + .../physical/impl/union/TestSimpleUnion.java | 2 + .../drill/exec/record/vector/TestLoad.java | 3 +- .../apache/drill/exec/server/TestBitRpc.java | 3 +- .../drill/exec/store/TestOutputMutator.java | 7 + .../store/parquet/ParquetRecordReaderTest.java | 9 + .../vector/complex/writer/TestJsonReader.java | 7 +- exec/java-exec/src/test/sh/runexec | 33 + 214 files changed, 5900 insertions(+), 3522 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java index 0e778c5..a15536e 100644 --- a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java +++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java @@ -17,36 +17,29 @@ */ package org.apache.drill.common.expression; -import java.io.IOException; +import java.math.BigDecimal; import org.apache.drill.common.expression.IfExpression.IfCondition; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; import org.apache.drill.common.expression.ValueExpressions.DateExpression; +import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression; import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; import org.apache.drill.common.expression.ValueExpressions.FloatExpression; import org.apache.drill.common.expression.ValueExpressions.IntExpression; import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression; import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression; import org.apache.drill.common.expression.ValueExpressions.LongExpression; -import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression; -import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression; -import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression; -import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression; import org.apache.drill.common.expression.ValueExpressions.QuotedString; import org.apache.drill.common.expression.ValueExpressions.TimeExpression; import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; import org.apache.drill.common.types.TypeProtos.MajorType; import org.joda.time.Period; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.DateTimeFormatterBuilder; -import org.joda.time.format.DateTimeParser; import com.google.common.collect.ImmutableList; -import org.apache.drill.common.util.DecimalUtility; - -import java.math.BigDecimal; public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBuilder, RuntimeException>{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionStringBuilder.class); @@ -76,7 +69,7 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui sb.append(") "); return null; } - + @Override public Void visitBooleanOperator(BooleanOperator op, StringBuilder sb) throws RuntimeException { return visitFunctionCall(op, sb); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java index a1f69c2..c4237d5 100644 --- a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java +++ b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java @@ -17,8 +17,8 @@ */ package org.apache.drill.common.expression; -import java.util.GregorianCalendar; import java.math.BigDecimal; +import java.util.GregorianCalendar; import java.util.Iterator; import org.apache.drill.common.expression.visitors.ExprVisitor; @@ -26,7 +26,7 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; -import org.apache.drill.common.util.DecimalUtility; +import org.apache.drill.common.util.CoreDecimalUtility; import com.google.common.collect.Iterators; @@ -96,7 +96,7 @@ public class ValueExpressions { } public static LogicalExpression getNumericExpression(String sign, String s, ExpressionPosition ep) { - String numStr = (sign == null) ? s : sign+s; + String numStr = (sign == null) ? s : sign+s; try { int a = Integer.parseInt(numStr); return new IntExpression(a, ep); @@ -138,12 +138,12 @@ public class ValueExpressions { } @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } @@ -208,12 +208,12 @@ public class ValueExpressions { } @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } @@ -248,17 +248,17 @@ public class ValueExpressions { public Iterator<LogicalExpression> iterator() { return Iterators.emptyIterator(); } - + @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } - + } @@ -272,7 +272,7 @@ public class ValueExpressions { super(pos); this.scale = input.scale(); this.precision = input.precision(); - this.decimal = DecimalUtility.getDecimal9FromBigDecimal(input, scale, precision); + this.decimal = CoreDecimalUtility.getDecimal9FromBigDecimal(input, scale, precision); } @@ -302,17 +302,17 @@ public class ValueExpressions { public Iterator<LogicalExpression> iterator() { return Iterators.emptyIterator(); } - + @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } - + } @@ -326,7 +326,7 @@ public class ValueExpressions { super(pos); this.scale = input.scale(); this.precision = input.precision(); - this.decimal = DecimalUtility.getDecimal18FromBigDecimal(input, scale, precision); + this.decimal = CoreDecimalUtility.getDecimal18FromBigDecimal(input, scale, precision); } @@ -356,14 +356,14 @@ public class ValueExpressions { public Iterator<LogicalExpression> iterator() { return Iterators.emptyIterator(); } - + @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } @@ -397,14 +397,14 @@ public class ValueExpressions { public Iterator<LogicalExpression> iterator() { return Iterators.emptyIterator(); } - + @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } } @@ -438,15 +438,15 @@ public class ValueExpressions { } @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } - + } @@ -478,17 +478,17 @@ public class ValueExpressions { public Iterator<LogicalExpression> iterator() { return Iterators.emptyIterator(); } - + @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } - + } public static class LongExpression extends LogicalExpressionBase { @@ -526,15 +526,15 @@ public class ValueExpressions { } @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } - + } @@ -572,14 +572,14 @@ public class ValueExpressions { public Iterator<LogicalExpression> iterator() { return Iterators.emptyIterator(); } - + @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } @@ -621,15 +621,15 @@ public class ValueExpressions { } @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } - + } public static class TimeStampExpression extends LogicalExpressionBase { @@ -665,14 +665,14 @@ public class ValueExpressions { public Iterator<LogicalExpression> iterator() { return Iterators.emptyIterator(); } - + @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } @@ -711,14 +711,14 @@ public class ValueExpressions { public Iterator<LogicalExpression> iterator() { return Iterators.emptyIterator(); } - + @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } @@ -766,15 +766,15 @@ public class ValueExpressions { } @Override - public int getSelfCost() { - return 0; // TODO + public int getSelfCost() { + return 0; // TODO } - + @Override - public int getCumulativeCost() { + public int getCumulativeCost() { return 0; // TODO } - + } public static class QuotedString extends ValueExpression<String> { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java new file mode 100644 index 0000000..c5cd8c0 --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.util; + +import java.math.BigDecimal; + +import org.apache.drill.common.types.TypeProtos; + +public class CoreDecimalUtility { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CoreDecimalUtility.class); + + public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) { + // Truncate or pad to set the input to the correct scale + input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); + + return (input.unscaledValue().longValue()); + } + + public static int getMaxPrecision(TypeProtos.MinorType decimalType) { + if (decimalType == TypeProtos.MinorType.DECIMAL9) { + return 9; + } else if (decimalType == TypeProtos.MinorType.DECIMAL18) { + return 18; + } else if (decimalType == TypeProtos.MinorType.DECIMAL28SPARSE) { + return 28; + } else if (decimalType == TypeProtos.MinorType.DECIMAL38SPARSE) { + return 38; + } + return 0; + } + + /* + * Function returns the Minor decimal type given the precision + */ + public static TypeProtos.MinorType getDecimalDataType(int precision) { + if (precision <= 9) { + return TypeProtos.MinorType.DECIMAL9; + } else if (precision <= 18) { + return TypeProtos.MinorType.DECIMAL18; + } else if (precision <= 28) { + return TypeProtos.MinorType.DECIMAL28SPARSE; + } else { + return TypeProtos.MinorType.DECIMAL38SPARSE; + } + } + + /* + * Given a precision it provides the max precision of that decimal data type; + * For eg: given the precision 12, we would use DECIMAL18 to store the data + * which has a max precision range of 18 digits + */ + public static int getPrecisionRange(int precision) { + return getMaxPrecision(getDecimalDataType(precision)); + } + public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) { + // Truncate/ or pad to set the input to the correct scale + input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); + + return (input.unscaledValue().intValue()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java index 4d3b939..6214e3e 100644 --- a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java +++ b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionDivideFunction.java @@ -18,6 +18,7 @@ package org.apache.drill.common.util; + /* * Here we compute the scale and precision of the output decimal data type * based on the input scale and precision. Since division operation can be @@ -51,7 +52,7 @@ public class DecimalScalePrecisionDivideFunction extends DrillBaseComputeScalePr int maxResultIntegerDigits = leftIntegerDigits + rightScale; - outputPrecision = DecimalUtility.getPrecisionRange(outputScale + maxResultIntegerDigits); + outputPrecision = CoreDecimalUtility.getPrecisionRange(outputScale + maxResultIntegerDigits); // Output precision should be greater or equal to the input precision outputPrecision = Math.max(outputPrecision, Math.max(leftPrecision, rightPrecision)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java index e1e587b..1c41197 100644 --- a/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java +++ b/common/src/main/java/org/apache/drill/common/util/DecimalScalePrecisionModFunction.java @@ -18,6 +18,7 @@ package org.apache.drill.common.util; + public class DecimalScalePrecisionModFunction extends DrillBaseComputeScalePrecision { public DecimalScalePrecisionModFunction(int leftPrecision, int leftScale, int rightPrecision, int rightScale) { @@ -31,7 +32,7 @@ public class DecimalScalePrecisionModFunction extends DrillBaseComputeScalePreci outputScale = Math.max(leftScale, rightScale); int leftIntegerDigits = leftPrecision - leftScale; - outputPrecision = DecimalUtility.getPrecisionRange(outputScale + leftIntegerDigits); + outputPrecision = CoreDecimalUtility.getPrecisionRange(outputScale + leftIntegerDigits); if (outputScale + leftIntegerDigits > outputPrecision) { outputScale = outputPrecision - leftIntegerDigits; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java deleted file mode 100644 index 85ba918..0000000 --- a/common/src/main/java/org/apache/drill/common/util/DecimalUtility.java +++ /dev/null @@ -1,735 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.common.util; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import org.apache.drill.common.types.TypeProtos; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.util.Arrays; - -public class DecimalUtility { - - public final static int MAX_DIGITS = 9; - public final static int DIGITS_BASE = 1000000000; - public final static int DIGITS_MAX = 999999999; - public final static int integerSize = (Integer.SIZE/8); - - public final static String[] decimalToString = {"", - "0", - "00", - "000", - "0000", - "00000", - "000000", - "0000000", - "00000000", - "000000000"}; - - public final static long[] scale_long_constants = { - 1, - 10, - 100, - 1000, - 10000, - 100000, - 1000000, - 10000000, - 100000000, - 1000000000, - 10000000000l, - 100000000000l, - 1000000000000l, - 10000000000000l, - 100000000000000l, - 1000000000000000l, - 10000000000000000l, - 100000000000000000l, - 1000000000000000000l}; - - /* - * Simple function that returns the static precomputed - * power of ten, instead of using Math.pow - */ - public static long getPowerOfTen(int power) { - assert power >= 0 && power < scale_long_constants.length; - return scale_long_constants[(power)]; - } - - /* - * Math.pow returns a double and while multiplying with large digits - * in the decimal data type we encounter noise. So instead of multiplying - * with Math.pow we use the static constants to perform the multiplication - */ - public static long adjustScaleMultiply(long input, int factor) { - int index = Math.abs(factor); - assert index >= 0 && index < scale_long_constants.length; - if (factor >= 0) { - return input * scale_long_constants[index]; - } else { - return input / scale_long_constants[index]; - } - } - - public static long adjustScaleDivide(long input, int factor) { - int index = Math.abs(factor); - assert index >= 0 && index < scale_long_constants.length; - if (factor >= 0) { - return input / scale_long_constants[index]; - } else { - return input * scale_long_constants[index]; - } - } - - /* Given the number of actual digits this function returns the - * number of indexes it will occupy in the array of integers - * which are stored in base 1 billion - */ - public static int roundUp(int ndigits) { - return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS; - } - - /* Returns a string representation of the given integer - * If the length of the given integer is less than the - * passed length, this function will prepend zeroes to the string - */ - public static StringBuilder toStringWithZeroes(int number, int desiredLength) { - String value = ((Integer) number).toString(); - int length = value.length(); - - StringBuilder str = new StringBuilder(); - str.append(decimalToString[desiredLength - length]); - str.append(value); - - return str; - } - - public static StringBuilder toStringWithZeroes(long number, int desiredLength) { - String value = ((Long) number).toString(); - int length = value.length(); - - StringBuilder str = new StringBuilder(); - - // Desired length can be > MAX_DIGITS - int zeroesLength = desiredLength - length; - while (zeroesLength > MAX_DIGITS) { - str.append(decimalToString[MAX_DIGITS]); - zeroesLength -= MAX_DIGITS; - } - str.append(decimalToString[zeroesLength]); - str.append(value); - - return str; - } - - public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) { - - // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false - return getBigDecimalFromByteBuf(data, startIndex, nDecimalDigits, scale, false); - } - - public static BigDecimal getBigDecimalFromSparse(ByteBuf data, int startIndex, int nDecimalDigits, int scale) { - - // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate - return getBigDecimalFromByteBuf(data, startIndex, nDecimalDigits, scale, true); - } - - - /* Create a BigDecimal object using the data in the ByteBuf. - * This function assumes that data is provided in a non-dense format - * It works on both sparse and intermediate representations. - */ - public static BigDecimal getBigDecimalFromByteBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale, boolean truncateScale) { - - // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal. - int actualDigits; - - // Initialize the BigDecimal, first digit in the ByteBuf has the sign so mask it out - BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF); - - BigInteger base = BigInteger.valueOf(DIGITS_BASE); - - for (int i = 1; i < nDecimalDigits; i++) { - - BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * integerSize))); - decimalDigits = decimalDigits.multiply(base); - decimalDigits = decimalDigits.add(temp); - } - - // Truncate any additional padding we might have added - if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) { - BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits))); - decimalDigits = decimalDigits.divide(truncate); - } - - // set the sign - if ((data.getInt(startIndex) & 0x80000000) != 0) { - decimalDigits = decimalDigits.negate(); - } - - BigDecimal decimal = new BigDecimal(decimalDigits, scale); - - return decimal; - } - - /* This function returns a BigDecimal object from the dense decimal representation. - * First step is to convert the dense representation into an intermediate representation - * and then invoke getBigDecimalFromByteBuf() to get the BigDecimal object - */ - public static BigDecimal getBigDecimalFromDense(ByteBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) { - - /* This method converts the dense representation to - * an intermediate representation. The intermediate - * representation has one more integer than the dense - * representation. - */ - byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * integerSize)]; - - // Start storing from the least significant byte of the first integer - int intermediateIndex = 3; - - int[] mask = {0x03, 0x0F, 0x3F, 0xFF}; - int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00}; - - int maskIndex; - int shiftOrder; - byte shiftBits; - - // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code - if (maxPrecision == 38) { - maskIndex = 0; - shiftOrder = 6; - shiftBits = 0x00; - intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F); - } else if (maxPrecision == 28) { - maskIndex = 1; - shiftOrder = 4; - shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder); - intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2); - } else { - throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported"); - } - - int inputIndex = 1; - boolean sign = false; - - if ((data.getByte(startIndex) & 0x80) != 0) { - sign = true; - } - - while (inputIndex < width) { - - intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder))); - - shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder); - - inputIndex++; - intermediateIndex++; - - if (((inputIndex - 1) % integerSize) == 0) { - shiftBits = (byte) ((shiftBits & 0xFF) >>> 2); - maskIndex++; - shiftOrder -= 2; - } - - } - /* copy the last byte */ - intermediateBytes[intermediateIndex] = shiftBits; - - if (sign == true) { - intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80); - } - - ByteBuf intermediateData = Unpooled.wrappedBuffer(intermediateBytes); - - return getBigDecimalFromIntermediate(intermediateData, 0, nDecimalDigits + 1, scale); - } - - /* - * Function converts the BigDecimal and stores it in out internal sparse representation - */ - public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision, int nDecimalDigits) { - - // Initialize the buffer - for (int i = 0; i < nDecimalDigits; i++) { - data.setInt(startIndex + (i * integerSize), 0); - } - - boolean sign = false; - - if (input.signum() == -1) { - // negative input - sign = true; - input = input.abs(); - } - - // Truncate the input as per the scale provided - input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); - - // Separate out the integer part - BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN); - - int destIndex = nDecimalDigits - roundUp(scale) - 1; - - // we use base 1 billion integer digits for out integernal representation - BigDecimal base = new BigDecimal(DIGITS_BASE); - - while (integerPart.compareTo(BigDecimal.ZERO) == 1) { - // store the modulo as the integer value - data.setInt(startIndex + (destIndex * integerSize), (integerPart.remainder(base)).intValue()); - destIndex--; - // Divide by base 1 billion - integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN); - } - - /* Sparse representation contains padding of additional zeroes - * so each digit contains MAX_DIGITS for ease of arithmetic - */ - int actualDigits; - if ((actualDigits = (scale % MAX_DIGITS)) != 0) { - // Pad additional zeroes - scale = scale + (MAX_DIGITS - actualDigits); - input = input.setScale(scale, BigDecimal.ROUND_DOWN); - } - - //separate out the fractional part - BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale); - - destIndex = nDecimalDigits - 1; - - while (scale > 0) { - // Get next set of MAX_DIGITS (9) store it in the ByteBuf - fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS); - BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE); - - data.setInt(startIndex + (destIndex * integerSize), (temp.unscaledValue().intValue())); - destIndex--; - - fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN); - scale -= MAX_DIGITS; - } - - // Set the negative sign - if (sign == true) { - data.setInt(startIndex, data.getInt(startIndex) | 0x80000000); - } - - } - public static int getDecimal9FromBigDecimal(BigDecimal input, int scale, int precision) { - // Truncate/ or pad to set the input to the correct scale - input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); - - return (input.unscaledValue().intValue()); - } - - public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) { - // Truncate or pad to set the input to the correct scale - input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); - - return (input.unscaledValue().longValue()); - } - - public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) { - return BigDecimal.valueOf(input, scale); - } - - public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) { - return BigDecimal.valueOf(input, scale); - } - - - public static int compareDenseBytes(ByteBuf left, int leftStart, boolean leftSign, ByteBuf right, int rightStart, boolean rightSign, int width) { - - int invert = 1; - - /* If signs are different then simply look at the - * sign of the two inputs and determine which is greater - */ - if (leftSign != rightSign) { - - return((leftSign == true) ? -1 : 1); - } else if(leftSign == true) { - /* Both inputs are negative, at the end we will - * have to invert the comparison - */ - invert = -1; - } - - int cmp = 0; - - for (int i = 0; i < width; i++) { - byte leftByte = left.getByte(leftStart + i); - byte rightByte = right.getByte(rightStart + i); - // Unsigned byte comparison - if ((leftByte & 0xFF) > (rightByte & 0xFF)) { - cmp = 1; - break; - } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) { - cmp = -1; - break; - } - } - cmp *= invert; // invert the comparison if both were negative values - - return cmp; - } - - public static int getIntegerFromSparseBuffer(ByteBuf buffer, int start, int index) { - int value = buffer.getInt(start + (index * 4)); - - if (index == 0) { - /* the first byte contains sign bit, return value without it */ - value = (value & 0x7FFFFFFF); - } - return value; - } - - public static void setInteger(ByteBuf buffer, int start, int index, int value) { - buffer.setInt(start + (index * 4), value); - } - - public static int compareSparseBytes(ByteBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ByteBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) { - - int invert = 1; - - if (absCompare == false) { - if (leftSign != rightSign) { - return (leftSign == true) ? -1 : 1; - } - - // Both values are negative invert the outcome of the comparison - if (leftSign == true) { - invert = -1; - } - } - - int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits); - return cmp * invert; - } - public static int compareSparseBytesInner(ByteBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ByteBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) { - /* compute the number of integer digits in each decimal */ - int leftInt = leftPrecision - leftScale; - int rightInt = rightPrecision - rightScale; - - /* compute the number of indexes required for storing integer digits */ - int leftIntRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(leftInt); - int rightIntRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(rightInt); - - /* compute number of indexes required for storing scale */ - int leftScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(leftScale); - int rightScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(rightScale); - - /* compute index of the most significant integer digits */ - int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp; - int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp; - - int leftStopIndex = nDecimalDigits - leftScaleRoundedUp; - int rightStopIndex = nDecimalDigits - rightScaleRoundedUp; - - /* Discard the zeroes in the integer part */ - while (leftIndex1 < leftStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) { - break; - } - - /* Digit in this location is zero, decrement the actual number - * of integer digits - */ - leftIntRoundedUp--; - leftIndex1++; - } - - /* If we reached the stop index then the number of integers is zero */ - if (leftIndex1 == leftStopIndex) { - leftIntRoundedUp = 0; - } - - while (rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) { - break; - } - - /* Digit in this location is zero, decrement the actual number - * of integer digits - */ - rightIntRoundedUp--; - rightIndex1++; - } - - if (rightIndex1 == rightStopIndex) { - rightIntRoundedUp = 0; - } - - /* We have the accurate number of non-zero integer digits, - * if the number of integer digits are different then we can determine - * which decimal is larger and needn't go down to comparing individual values - */ - if (leftIntRoundedUp > rightIntRoundedUp) { - return 1; - } - else if (rightIntRoundedUp > leftIntRoundedUp) { - return -1; - } - - /* The number of integer digits are the same, set the each index - * to the first non-zero integer and compare each digit - */ - leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp; - rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp; - - while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) { - return 1; - } - else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) { - return -1; - } - - leftIndex1++; - rightIndex1++; - } - - /* The integer part of both the decimal's are equal, now compare - * each individual fractional part. Set the index to be at the - * beginning of the fractional part - */ - leftIndex1 = leftStopIndex; - rightIndex1 = rightStopIndex; - - /* Stop indexes will be the end of the array */ - leftStopIndex = nDecimalDigits; - rightStopIndex = nDecimalDigits; - - /* compare the two fractional parts of the decimal */ - while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) { - return 1; - } - else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) { - return -1; - } - - leftIndex1++; - rightIndex1++; - } - - /* Till now the fractional part of the decimals are equal, check - * if one of the decimal has fractional part that is remaining - * and is non-zero - */ - while (leftIndex1 < leftStopIndex) { - if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) { - return 1; - } - leftIndex1++; - } - - while(rightIndex1 < rightStopIndex) { - if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) { - return -1; - } - rightIndex1++; - } - - /* Both decimal values are equal */ - return 0; - } - - public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) { - byte[] value = Arrays.copyOfRange(bytes, start, start + length); - BigInteger unscaledValue = new BigInteger(value); - return new BigDecimal(unscaledValue, scale); - } - - - public static BigDecimal getBigDecimalFromByteBuf(ByteBuf bytebuf, int start, int length, int scale) { - byte[] value = new byte[length]; - bytebuf.getBytes(start, value, 0, length); - BigInteger unscaledValue = new BigInteger(value); - return new BigDecimal(unscaledValue, scale); - } - - public static void roundDecimal(ByteBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) { - int newScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(desiredScale); - int origScaleRoundedUp = org.apache.drill.common.util.DecimalUtility.roundUp(currentScale); - - if (desiredScale < currentScale) { - - boolean roundUp = false; - - //Extract the first digit to be truncated to check if we need to round up - int truncatedScaleIndex = desiredScale + 1; - if (truncatedScaleIndex <= currentScale) { - int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1; - extractDigitIndex += org.apache.drill.common.util.DecimalUtility.roundUp(truncatedScaleIndex); - int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex); - int temp = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS); - if (temp != 0) { - extractDigit = extractDigit / (int) (Math.pow(10, temp)); - } - if ((extractDigit % 10) > 4) { - roundUp = true; - } - } - - // Get the source index beyond which we will truncate - int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1; - int srcIndex = srcIntIndex + newScaleRoundedUp; - - // Truncate the remaining fractional part, move the integer part - int destIndex = nDecimalDigits - 1; - if (srcIndex != destIndex) { - while (srcIndex >= 0) { - setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--)); - } - - // Set the remaining portion of the decimal to be zeroes - while (destIndex >= 0) { - setInteger(result, start, destIndex--, 0); - } - srcIndex = nDecimalDigits - 1; - } - - // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit - int truncateFactor = org.apache.drill.common.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.common.util.DecimalUtility.MAX_DIGITS); - if (truncateFactor != org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) { - truncateFactor = (int) Math.pow(10, truncateFactor); - int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1); - fractionalDigits /= truncateFactor; - setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor); - } - - // Finally round up the digit if needed - if (roundUp == true) { - srcIndex = nDecimalDigits - 1; - int carry; - if (truncateFactor != org.apache.drill.common.util.DecimalUtility.MAX_DIGITS) { - carry = truncateFactor; - } else { - carry = 1; - } - - while (srcIndex >= 0) { - int value = getIntegerFromSparseBuffer(result, start, srcIndex); - value += carry; - - if (value >= org.apache.drill.common.util.DecimalUtility.DIGITS_BASE) { - setInteger(result, start, srcIndex--, value % org.apache.drill.common.util.DecimalUtility.DIGITS_BASE); - carry = value / org.apache.drill.common.util.DecimalUtility.DIGITS_BASE; - } else { - setInteger(result, start, srcIndex--, value); - carry = 0; - break; - } - } - } - } else if (desiredScale > currentScale) { - // Add fractional digits to the decimal - - // Check if we need to shift the decimal digits to the left - if (newScaleRoundedUp > origScaleRoundedUp) { - int srcIndex = 0; - int destIndex = newScaleRoundedUp - origScaleRoundedUp; - - // Check while extending scale, we are not overwriting integer part - while (srcIndex < destIndex) { - if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) { - throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified"); - } - } - - srcIndex = 0; - while (destIndex < nDecimalDigits) { - setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++)); - } - - // Clear the remaining part - while (srcIndex < nDecimalDigits) { - setInteger(result, start, srcIndex++, 0); - } - } - } - } - - /* - * Function returns the Minor decimal type given the precision - */ - public static TypeProtos.MinorType getDecimalDataType(int precision) { - if (precision <= 9) { - return TypeProtos.MinorType.DECIMAL9; - } else if (precision <= 18) { - return TypeProtos.MinorType.DECIMAL18; - } else if (precision <= 28) { - return TypeProtos.MinorType.DECIMAL28SPARSE; - } else { - return TypeProtos.MinorType.DECIMAL38SPARSE; - } - } - - public static int getMaxPrecision(TypeProtos.MinorType decimalType) { - if (decimalType == TypeProtos.MinorType.DECIMAL9) { - return 9; - } else if (decimalType == TypeProtos.MinorType.DECIMAL18) { - return 18; - } else if (decimalType == TypeProtos.MinorType.DECIMAL28SPARSE) { - return 28; - } else if (decimalType == TypeProtos.MinorType.DECIMAL38SPARSE) { - return 38; - } - return 0; - } - - - /* - * Given a precision it provides the max precision of that decimal data type; - * For eg: given the precision 12, we would use DECIMAL18 to store the data - * which has a max precision range of 18 digits - */ - public static int getPrecisionRange(int precision) { - return getMaxPrecision(getDecimalDataType(precision)); - } - - public static int getFirstFractionalDigit(int decimal, int scale) { - if (scale == 0) { - return 0; - } - int temp = (int) adjustScaleDivide(decimal, scale - 1); - return Math.abs(temp % 10); - } - - public static int getFirstFractionalDigit(long decimal, int scale) { - if (scale == 0) { - return 0; - } - long temp = adjustScaleDivide(decimal, scale - 1); - return (int) (Math.abs(temp % 10)); - } - - public static int getFirstFractionalDigit(ByteBuf data, int scale, int start, int nDecimalDigits) { - if (scale == 0) { - return 0; - } - - int index = nDecimalDigits - roundUp(scale); - return (int) (adjustScaleDivide(data.getInt(start + (index * integerSize)), MAX_DIGITS - 1)); - } -} - http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java index 2267ad5..576fa47 100644 --- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java +++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java @@ -24,7 +24,7 @@ package org.apache.drill.exec.expr.fn.impl.hive; -import org.apache.drill.common.util.DecimalUtility; +import org.apache.drill.exec.util.DecimalUtility; import org.apache.drill.exec.expr.holders.*; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/pom.xml ---------------------------------------------------------------------- diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index 903f953..d1f9754 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -21,10 +21,16 @@ <name>exec/Java Execution Engine</name> <dependencies> +<!-- <dependency> --> +<!-- <groupId>org.ow2.asm</groupId> --> +<!-- <artifactId>asm-util</artifactId> --> +<!-- <version>4.1</version> --> +<!-- </dependency> --> <dependency> <groupId>org.ow2.asm</groupId> - <artifactId>asm-util</artifactId> + <artifactId>asm-debug-all</artifactId> <version>4.1</version> +<!-- <scope>test</scope> --> </dependency> <dependency> <groupId>pentaho</groupId> @@ -38,11 +44,11 @@ <version>${project.version}</version> <scope>test</scope> </dependency> - <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm-commons</artifactId> - <version>4.1</version> - </dependency> +<!-- <dependency> --> +<!-- <groupId>org.ow2.asm</groupId> --> +<!-- <artifactId>asm-commons</artifactId> --> +<!-- <version>4.1</version> --> +<!-- </dependency> --> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-pool2</artifactId> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/data/Casts.tdd ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd index 2b76514..a4f5fb1 100644 --- a/exec/java-exec/src/main/codegen/data/Casts.tdd +++ b/exec/java-exec/src/main/codegen/data/Casts.tdd @@ -78,7 +78,7 @@ {from: "IntervalDay", to: "VarChar", major: "IntervalDayVarChar", bufferLength: "43"}, {from: "IntervalYear", to: "Interval", major: "IntervalSimpleToComplex", months: "in.value", days: "0", millis: "0"}, - {from: "IntervalDay", to: "Interval", major: "IntervalSimpleToComplex", months: "0", days: "in.days", millis: "in.milliSeconds"}, + {from: "IntervalDay", to: "Interval", major: "IntervalSimpleToComplex", months: "0", days: "in.days", millis: "in.milliseconds"}, {from: "Interval", to: "IntervalYear", major: "IntervalComplexToSimple"}, {from: "Interval", to: "IntervalDay", major: "IntervalComplexToSimple"} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd b/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd index d3b3ac4..813f43a 100644 --- a/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd +++ b/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd @@ -26,9 +26,10 @@ width: 1, javaType: "byte", boxedType: "Byte", + fields: [{name: "value", type: "byte"}], minor: [ - { class: "TinyInt", valueHolder: "IntHolder"}, - { class: "UInt1", valueHolder: "UInt1Holder"} + { class: "TinyInt", valueHolder: "IntHolder" }, + { class: "UInt1", valueHolder: "UInt1Holder" } ] }, { @@ -36,6 +37,7 @@ width: 2, javaType: "char", boxedType: "Character", + fields: [{name: "value", type: "char"}], minor: [ { class: "UInt2", valueHolder: "UInt2Holder"} ] @@ -44,6 +46,7 @@ width: 2, javaType: "short", boxedType: "Short", + fields: [{name: "value", type: "short"}], minor: [ { class: "SmallInt", valueHolder: "Int2Holder"}, ] @@ -53,13 +56,14 @@ width: 4, javaType: "int", boxedType: "Integer", + fields: [{name: "value", type: "int"}], minor: [ - { class: "Int", valueHolder: "IntHolder" }, + { class: "Int", valueHolder: "IntHolder"}, { class: "UInt4", valueHolder: "UInt4Holder" }, - { class: "Float4", javaType: "float" , boxedType: "Float" }, + { class: "Float4", javaType: "float" , boxedType: "Float", fields: [{name: "value", type: "float"}]}, { class: "Time", javaType: "int", friendlyType: "DateTime" }, { class: "IntervalYear", javaType: "int", friendlyType: "Period" } - { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal" }, + { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal", fields: [{name:"value", type:"int"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }, ] }, { @@ -67,13 +71,14 @@ width: 8, javaType: "long", boxedType: "Long", + fields: [{name: "value", type: "long"}], minor: [ - { class: "BigInt" }, + { class: "BigInt"}, { class: "UInt8" }, - { class: "Float8", javaType: "double" , boxedType: "Double" }, + { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], }, { class: "Date", javaType: "long", friendlyType: "DateTime" }, { class: "TimeStamp", javaType: "long", friendlyType: "DateTime" } - { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal" }, + { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal", fields: [{name:"value", type:"long"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }, <#-- { class: "Money", maxPrecisionDigits: 2, scale: 1, }, --> @@ -82,68 +87,69 @@ { major: "Fixed", width: 12, - javaType: "ByteBuf", - boxedType: "ByteBuf", + javaType: "DrillBuf", + boxedType: "DrillBuf", minor: [ - { class: "TimeStampTZ", milliSecondsSize: 8, friendlyType: "DateTime" }, - { class: "IntervalDay", milliSecondsOffset: 4, friendlyType: "Period" } + { class: "TimeStampTZ", millisecondsSize: 8, friendlyType: "DateTime", fields: [ {name: "value", type:"long"}, {name: "index", type: "int"}] }, + { class: "IntervalDay", millisecondsOffset: 4, friendlyType: "Period", fields: [ {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] } ] }, { major: "Fixed", width: 16, - javaType: "ByteBuf" - boxedType: "ByteBuf", + javaType: "DrillBuf" + boxedType: "DrillBuf", minor: [ - { class: "Interval", daysOffset: 4, milliSecondsOffset: 8, friendlyType: "Period" } + { class: "Interval", daysOffset: 4, millisecondsOffset: 8, friendlyType: "Period", fields: [ {name: "months", type: "int"}, {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] } ] }, { major: "Fixed", width: 12, - javaType: "ByteBuf", - boxedType: "ByteBuf", + javaType: "DrillBuf", + boxedType: "DrillBuf", minor: [ <#-- { class: "TimeTZ" }, { class: "Interval" } --> - { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal" } + { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] } ] }, { major: "Fixed", width: 16, - javaType: "ByteBuf", - boxedType: "ByteBuf", + javaType: "DrillBuf", + boxedType: "DrillBuf", minor: [ - { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal" } + { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] } ] }, { major: "Fixed", width: 24, - javaType: "ByteBuf", - boxedType: "ByteBuf", + javaType: "DrillBuf", + boxedType: "DrillBuf", minor: [ - { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal" } + { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] } ] }, { major: "Fixed", width: 20, - javaType: "ByteBuf", - boxedType: "ByteBuf", + javaType: "DrillBuf", + boxedType: "DrillBuf", minor: [ - { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal" } + { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "DrillBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] } ] }, { major: "VarLen", width: 4, javaType: "int", - boxedType: "ByteBuf", + boxedType: "DrillBuf", + fields: [{name: "start", type: "int"}, {name: "end", type: "int"}, {name: "buffer", type: "DrillBuf"}], minor: [ { class: "VarBinary" , friendlyType: "byte[]" }, { class: "VarChar" , friendlyType: "Text" } @@ -154,9 +160,9 @@ major: "Bit", width: 1, javaType: "int", - boxedType: "ByteBuf", + boxedType: "Integer", minor: [ - { class: "Bit" , friendlyType: "Boolean" } + { class: "Bit" , friendlyType: "Boolean", fields: [{name: "value", type: "int"}] } ] } ] http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/includes/vv_imports.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl index 67c6bff..371d8d0 100644 --- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl +++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl @@ -45,6 +45,8 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; import com.sun.codemodel.JType; import com.sun.codemodel.JCodeModel; +import javax.inject.Inject; + import java.util.Arrays; import java.util.Random; import java.util.List; @@ -65,7 +67,7 @@ import org.joda.time.Period; import org.apache.hadoop.io.Text; import org.apache.drill.exec.vector.accessor.sql.TimePrintMillis; - +import javax.inject.Inject; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java b/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java index e2fd9d5..c261e29 100644 --- a/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java +++ b/exec/java-exec/src/main/codegen/templates/CastDateVarChar.java @@ -28,7 +28,10 @@ package org.apache.drill.exec.expr.fn.impl.gcast; +<#include "/@includes/vv_imports.ftl" /> + import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; import org.apache.drill.exec.expr.DrillSimpleFunc; import org.apache.drill.exec.expr.annotations.FunctionTemplate; @@ -51,11 +54,11 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { @Param ${type.from}Holder in; @Param BigIntHolder len; - @Workspace ByteBuf buffer; + @Inject DrillBuf buffer; @Output ${type.to}Holder out; public void setup(RecordBatch incoming) { - buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]); + buffer = buffer.reallocIfNeeded((int) len.value); } public void eval() { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctions.java b/exec/java-exec/src/main/codegen/templates/CastFunctions.java index 4c750dd..41c1557 100644 --- a/exec/java-exec/src/main/codegen/templates/CastFunctions.java +++ b/exec/java-exec/src/main/codegen/templates/CastFunctions.java @@ -34,6 +34,8 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import org.apache.drill.exec.expr.annotations.Output; import org.apache.drill.exec.expr.annotations.Param; import org.apache.drill.exec.expr.holders.*; +import javax.inject.Inject; +import io.netty.buffer.DrillBuf; import org.apache.drill.exec.record.RecordBatch; @SuppressWarnings("unused") http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java index aa216cd..d8ce44a 100644 --- a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java +++ b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLen.java @@ -41,6 +41,8 @@ import org.apache.drill.exec.expr.annotations.Output; import org.apache.drill.exec.expr.annotations.Param; import org.apache.drill.exec.expr.holders.*; import org.apache.drill.exec.record.RecordBatch; +import javax.inject.Inject; +import io.netty.buffer.DrillBuf; @SuppressWarnings("unused") @FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java index 8594d5a..0096be8 100644 --- a/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java +++ b/exec/java-exec/src/main/codegen/templates/CastFunctionsSrcVarLenTargetVarLen.java @@ -38,6 +38,8 @@ import org.apache.drill.exec.expr.annotations.Param; import org.apache.drill.exec.expr.annotations.Workspace; import org.apache.drill.exec.expr.holders.*; import org.apache.drill.exec.record.RecordBatch; +import javax.inject.Inject; +import io.netty.buffer.DrillBuf; @SuppressWarnings("unused") @FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java b/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java index 319ab6b..455ab13 100644 --- a/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java +++ b/exec/java-exec/src/main/codegen/templates/CastFunctionsTargetVarLen.java @@ -29,6 +29,7 @@ package org.apache.drill.exec.expr.fn.impl.gcast; import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; import org.apache.drill.exec.expr.DrillSimpleFunc; import org.apache.drill.exec.expr.annotations.FunctionTemplate; @@ -38,6 +39,8 @@ import org.apache.drill.exec.expr.annotations.Param; import org.apache.drill.exec.expr.annotations.Workspace; import org.apache.drill.exec.expr.holders.*; import org.apache.drill.exec.record.RecordBatch; +import javax.inject.Inject; +import io.netty.buffer.DrillBuf; @SuppressWarnings("unused") @FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) @@ -45,15 +48,14 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc{ @Param ${type.from}Holder in; @Param BigIntHolder len; - @Workspace ByteBuf buffer; + @Inject DrillBuf buffer; @Output ${type.to}Holder out; public void setup(RecordBatch incoming) { - //TODO: max bufferLength should = parameter.len - buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]); } public void eval() { + buffer = buffer.reallocIfNeeded((int) len.value); String istr = (new ${type.javaType}(in.value)).toString(); out.buffer = buffer; out.start = 0; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java b/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java index 19c218f..e85b724 100644 --- a/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java +++ b/exec/java-exec/src/main/codegen/templates/CastIntervalInterval.java @@ -26,6 +26,8 @@ package org.apache.drill.exec.expr.fn.impl.gcast; +<#include "/@includes/vv_imports.ftl" /> + import io.netty.buffer.ByteBuf; import org.apache.drill.exec.expr.DrillSimpleFunc; @@ -54,7 +56,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { public void eval() { out.months = ${type.months}; out.days = ${type.days}; - out.milliSeconds = ${type.millis}; + out.milliseconds = ${type.millis}; } } @@ -96,7 +98,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { out.value = in.months; <#elseif type.to == "IntervalDay"> out.days = in.days; - out.milliSeconds = in.milliSeconds; + out.milliseconds = in.milliseconds; </#if> } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java b/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java index 4c88fcc..829fe47 100644 --- a/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java +++ b/exec/java-exec/src/main/codegen/templates/CastIntervalVarChar.java @@ -27,7 +27,10 @@ package org.apache.drill.exec.expr.fn.impl.gcast; +<#include "/@includes/vv_imports.ftl" /> + import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; import org.apache.drill.exec.expr.DrillSimpleFunc; import org.apache.drill.exec.expr.annotations.FunctionTemplate; @@ -49,11 +52,11 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { @Param ${type.from}Holder in; @Param BigIntHolder len; - @Workspace ByteBuf buffer; + @Inject DrillBuf buffer; @Output ${type.to}Holder out; public void setup(RecordBatch incoming) { - buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]); + buffer.reallocIfNeeded(${type.bufferLength}); } public void eval() { @@ -61,7 +64,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { int years = (in.months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); int months = (in.months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - long millis = in.milliSeconds; + long millis = in.milliseconds; long hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); @@ -99,6 +102,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { package org.apache.drill.exec.expr.fn.impl.gcast; +<#include "/@includes/vv_imports.ftl" /> import io.netty.buffer.ByteBuf; @@ -122,11 +126,11 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { @Param ${type.from}Holder in; @Param BigIntHolder len; - @Workspace ByteBuf buffer; + @Inject DrillBuf buffer; @Output ${type.to}Holder out; public void setup(RecordBatch incoming) { - buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]); + buffer = buffer.reallocIfNeeded((int) len.value); } public void eval() { @@ -154,6 +158,9 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { package org.apache.drill.exec.expr.fn.impl.gcast; + + + import io.netty.buffer.ByteBuf; import org.apache.drill.exec.expr.DrillSimpleFunc; @@ -169,6 +176,8 @@ import org.joda.time.MutableDateTime; import org.joda.time.DateTimeZone; import org.joda.time.DateMidnight; import org.apache.drill.exec.expr.fn.impl.DateUtility; +import javax.inject.Inject; +import io.netty.buffer.DrillBuf; @SuppressWarnings("unused") @FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL, @@ -177,16 +186,15 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { @Param ${type.from}Holder in; @Param BigIntHolder len; - @Workspace ByteBuf buffer; + @Inject DrillBuf buffer; @Output ${type.to}Holder out; public void setup(RecordBatch incoming) { - buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[${type.bufferLength}]); + buffer = buffer.reallocIfNeeded((int) len.value); } public void eval() { - - long millis = in.milliSeconds; + long millis = in.milliseconds; long hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java b/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java index e2e1143..cf3ae90 100644 --- a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java +++ b/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java @@ -40,6 +40,8 @@ import org.joda.time.MutableDateTime; import org.joda.time.DateTimeZone; import org.joda.time.DateMidnight; import org.apache.drill.exec.expr.fn.impl.DateUtility; +import javax.inject.Inject; +import io.netty.buffer.DrillBuf; @SuppressWarnings("unused") @FunctionTemplate(names = {"cast${type.to?upper_case}", "${type.alias}"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL, @@ -48,19 +50,20 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { @Param ${type.from}Holder in; @Output ${type.to}Holder out; - + public void setup(RecordBatch incoming) { } public void eval() { + <#if type.to != "Date"> byte[] buf = new byte[in.end - in.start]; in.buffer.getBytes(in.start, buf, 0, in.end - in.start); String input = new String(buf, com.google.common.base.Charsets.UTF_8); - + </#if> + <#if type.to == "Date"> - org.joda.time.format.DateTimeFormatter f = org.apache.drill.exec.expr.fn.impl.DateUtility.getDateTimeFormatter(); - out.value = (org.joda.time.DateMidnight.parse(input, f).withZoneRetainFields(org.joda.time.DateTimeZone.UTC)).getMillis(); + out.value = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getDate(in.buffer.memoryAddress(), in.start, in.end); <#elseif type.to == "TimeStamp"> org.joda.time.format.DateTimeFormatter f = org.apache.drill.exec.expr.fn.impl.DateUtility.getDateTimeFormatter(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java b/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java index 135abdd..eaebf02 100644 --- a/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java +++ b/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java @@ -40,6 +40,8 @@ import org.joda.time.MutableDateTime; import org.joda.time.DateTimeZone; import org.joda.time.DateMidnight; import org.apache.drill.exec.expr.fn.impl.DateUtility; +import javax.inject.Inject; +import io.netty.buffer.DrillBuf; @SuppressWarnings("unused") @FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL) @@ -65,7 +67,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { out.days = period.getDays(); - out.milliSeconds = (period.getHours() * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) + + out.milliseconds = (period.getHours() * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) + (period.getMinutes() * org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis) + (period.getSeconds() * org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis) + (period.getMillis()); @@ -73,7 +75,7 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc { <#elseif type.to == "IntervalDay"> out.days = period.getDays(); - out.milliSeconds = (period.getHours() * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) + + out.milliseconds = (period.getHours() * org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis) + (period.getMinutes() * org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis) + (period.getSeconds() * org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis) + (period.getMillis());
