Allow implicit cast of VARCHAR type arg to VAR16CHAR type when resolving Hive UDFs
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/95c5b569 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/95c5b569 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/95c5b569 Branch: refs/heads/master Commit: 95c5b5698abf0facac127a35c0d960fac1e286b2 Parents: 6e65f04 Author: vkorukanti <[email protected]> Authored: Tue Jul 29 19:14:16 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Tue Jul 29 19:56:44 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/expr/fn/HiveFuncHolder.java | 10 ++++ .../exec/expr/fn/HiveFunctionRegistry.java | 25 +++++++- .../drill/exec/fn/hive/TestSampleHiveUDFs.java | 3 +- .../exec/expr/ExpressionTreeMaterializer.java | 17 +++++- .../drill/exec/expr/fn/AbstractFuncHolder.java | 5 ++ .../drill/exec/expr/fn/DrillFuncHolder.java | 2 + .../exec/expr/fn/impl/CastVarCharVar16Char.java | 61 ++++++++++++++++++++ 7 files changed, 119 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/95c5b569/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java index 68a9b97..45dbafe 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java @@ -112,6 +112,16 @@ public class HiveFuncHolder extends AbstractFuncHolder { return isRandom; } + @Override + public MajorType getParmMajorType(int i) { + return argTypes[i]; + } + + @Override + public int getParamCount() { + return argTypes.length; + } + /** * Start generating code * @return workspace variables http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/95c5b569/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java index 119c93b..c4813a7 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java @@ -24,6 +24,7 @@ import com.google.common.collect.Sets; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; import org.apache.drill.common.util.PathScanner; import org.apache.drill.exec.expr.fn.impl.hive.ObjectInspectorHelper; @@ -94,13 +95,35 @@ public class HiveFunctionRegistry implements PluggableFunctionRegistry{ * @return */ @Override - public HiveFuncHolder getFunction(FunctionCall call){ + public HiveFuncHolder getFunction(FunctionCall call) { + HiveFuncHolder h; + + h = resolveFunction(call, false); + if (h != null) { + return h; + } + + return resolveFunction(call, true); + } + + /** + * Helper method which resolves the given function call to a Hive UDF. It takes an argument + * <i>convertVarCharToVar16Char</i> which tells to implicitly cast input arguments of type VARCHAR to VAR16CHAR + * and search Hive UDF registry using implicitly casted argument types. + * + * TODO: This is a rudimentary function resolver. Need to include more implicit casting such as DECIMAL28 to + * DECIMAL38 as Hive UDFs can accept only DECIMAL38 type. + */ + private HiveFuncHolder resolveFunction(FunctionCall call, boolean convertVarCharToVar16Char) { HiveFuncHolder holder; MajorType[] argTypes = new MajorType[call.args.size()]; ObjectInspector[] argOIs = new ObjectInspector[call.args.size()]; for(int i=0; i<call.args.size(); i++) { try { argTypes[i] = call.args.get(i).getMajorType(); + if (convertVarCharToVar16Char && argTypes[i].getMinorType() == MinorType.VARCHAR) { + argTypes[i] = Types.withMode(MinorType.VAR16CHAR, argTypes[i].getMode()); + } argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMode(), argTypes[i].getMinorType()); } catch(Exception e) { // Hive throws errors if there are unsupported types. Consider there is no hive UDF supporting the http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/95c5b569/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java index 31be13f..8517d20 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java @@ -95,10 +95,9 @@ public class TestSampleHiveUDFs extends BaseTestQuery { } @Test - @Ignore // TODO: need var16char input. String type field read from Hive is converted to VarChar type public void stringInOut() throws Exception{ String query = "SELECT testHiveUDFString(string_field) as col1 FROM hive.readtest"; - String expected = "col1\n" + "8.345\n" + "null\n"; + String expected = "col1\n" + "stringfield\n" + "\n"; helper(query, expected); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/95c5b569/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index 60bc78c..d3c2f8e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -232,7 +232,22 @@ public class ExpressionTreeMaterializer { // as no drill func is found, search for a non-Drill function. AbstractFuncHolder matchedNonDrillFuncHolder = registry.findNonDrillFunction(call); if (matchedNonDrillFuncHolder != null) { - return matchedNonDrillFuncHolder.getExpr(call.getName(), call.args, call.getPosition()); + // Insert implicit cast function holder expressions if required + List<LogicalExpression> extArgsWithCast = Lists.newArrayList(); + + for (int i = 0; i < call.args.size(); ++i) { + LogicalExpression currentArg = call.args.get(i); + TypeProtos.MajorType parmType = matchedNonDrillFuncHolder.getParmMajorType(i); + + if (Types.softEquals(parmType, currentArg.getMajorType(), true)) { + extArgsWithCast.add(currentArg); + } else { + // Insert cast if param type is different from arg type. + extArgsWithCast.add(addCastExpression(call.args.get(i), parmType, registry)); + } + } + + return matchedNonDrillFuncHolder.getExpr(call.getName(), extArgsWithCast, call.getPosition()); } logFunctionResolutionError(errorCollector, call); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/95c5b569/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/AbstractFuncHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/AbstractFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/AbstractFuncHolder.java index bf6ec56..f79450c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/AbstractFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/AbstractFuncHolder.java @@ -22,6 +22,7 @@ import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.fn.FuncHolder; +import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.expr.ClassGenerator; import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer; @@ -42,4 +43,8 @@ public abstract class AbstractFuncHolder implements FuncHolder { } public abstract FunctionHolderExpression getExpr(String name, List<LogicalExpression> args, ExpressionPosition pos); + + public abstract MajorType getParmMajorType(int i); + + public abstract int getParamCount(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/95c5b569/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java index d066a00..5ca48ce 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java @@ -196,10 +196,12 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder { return true; } + @Override public MajorType getParmMajorType(int i) { return this.parameters[i].type; } + @Override public int getParamCount() { return this.parameters.length; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/95c5b569/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarCharVar16Char.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarCharVar16Char.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarCharVar16Char.java new file mode 100644 index 0000000..12656d4 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CastVarCharVar16Char.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl; + +import io.netty.buffer.ByteBuf; +import org.apache.drill.exec.expr.DrillSimpleFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.annotations.Output; +import org.apache.drill.exec.expr.annotations.Param; +import org.apache.drill.exec.expr.annotations.Workspace; +import org.apache.drill.exec.expr.holders.BigIntHolder; +import org.apache.drill.exec.expr.holders.Var16CharHolder; +import org.apache.drill.exec.expr.holders.VarCharHolder; +import org.apache.drill.exec.record.RecordBatch; + +import java.nio.charset.Charset; + +@SuppressWarnings("unused") +@FunctionTemplate(names = {"castVAR16CHAR", "to_var16char", "to_string"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls= NullHandling.NULL_IF_NULL) +public class CastVarCharVar16Char implements DrillSimpleFunc { + @Param VarCharHolder in; + @Param BigIntHolder length; + @Output Var16CharHolder out; + @Workspace Charset charset; + @Workspace ByteBuf buffer; + + @Override + public void setup(RecordBatch incoming) { + charset = java.nio.charset.Charset.forName("UTF-16"); + buffer = io.netty.buffer.Unpooled.wrappedBuffer(new byte[65536]); + } + + @Override + public void eval() { + //if input length <= target_type length, do nothing + //else truncate based on target_type length + byte[] buf = in.toString().getBytes(charset); + buffer.setBytes(0, buf); + + out.start = 0; + out.end = buf.length; + out.buffer = buffer; + } +} +
