Repository: incubator-drill Updated Branches: refs/heads/master cdc5daed5 -> 1b20b6e9e
DRILL-692: Add Hive UDFs to Drill SQL operator table. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e602b2a9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e602b2a9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e602b2a9 Branch: refs/heads/master Commit: e602b2a9cd2e7550be4c2fd38e2b5942dc623d41 Parents: cdc5dae Author: vkorukanti <[email protected]> Authored: Sat May 10 12:24:55 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Mon May 12 11:38:56 2014 -0700 ---------------------------------------------------------------------- .../templates/ObjectInspectorHelper.java | 10 ++- .../exec/expr/ExpressionTreeMaterializer.java | 2 +- .../fn/HiveFunctionImplementationRegistry.java | 17 ++++- .../exec/planner/sql/DrillOperatorTable.java | 13 ++-- .../drill/exec/planner/sql/HiveUDFOperator.java | 70 ++++++++++++++++++++ .../drill/exec/physical/impl/TestHiveUDFs.java | 10 +-- .../record/ExpressionTreeMaterializerTest.java | 3 +- .../resources/functions/hive/GenericUDF.json | 6 +- .../drill/jdbc/test/TestHiveScalarUDFs.java | 57 ++++++++++++++++ 9 files changed, 169 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e602b2a9/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java b/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java index 23b969d..22a9eb2 100644 --- a/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java +++ b/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java @@ -70,7 +70,7 @@ public class ObjectInspectorHelper { JType holderClass = TypeHelper.getHolderType(m, returnType, TypeProtos.DataMode.OPTIONAL); block.assign(returnValueHolder, JExpr._new(holderClass)); - <#if entry.hiveType == "VARCHAR" || entry.hiveType == "STRING"> + <#if entry.hiveType == "VARCHAR" || entry.hiveType == "STRING" || entry.hiveType == "BINARY"> block.assign(returnValueHolder.ref("buffer"), m.directClass(io.netty.buffer.Unpooled.class.getCanonicalName()) .staticInvoke("wrappedBuffer") @@ -160,6 +160,14 @@ public class ObjectInspectorHelper { .invoke("setBytes").arg(JExpr.lit(0)).arg(data)); jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0)); jc._else().assign(returnValueHolder.ref("end"), data.ref("length")); + <#elseif entry.hiveType == "BINARY"> + + JVar data = jc._else().decl(m.directClass(byte[].class.getCanonicalName()), "data", + castedOI.invoke("getPrimitiveJavaObject").arg(returnValue)); + jc._else().add(returnValueHolder.ref("buffer") + .invoke("setBytes").arg(JExpr.lit(0)).arg(data)); + jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0)); + jc._else().assign(returnValueHolder.ref("end"), data.ref("length")); <#else> jc._else().assign(returnValueHolder.ref("value"), http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e602b2a9/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index fc7fb6a..0267be3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -192,7 +192,7 @@ public class ExpressionTreeMaterializer { return new HiveFuncHolderExpr(call.getName(), matchedHiveHolder, call.args, call.getPosition()); logFunctionResolutionError(errorCollector, call); - return null; + return NullExpression.INSTANCE; } private void logFunctionResolutionError(ErrorCollector errorCollector, FunctionCall call) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e602b2a9/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java index 634b24f..e5c890e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java @@ -76,9 +76,22 @@ public class HiveFunctionImplementationRegistry { for(int i=0; i<names.length;i++){ methods.put(names[i], clazz); + if (!names[i].toLowerCase().equals(names[i])) { + // After the Optiq-Drill conversion of function calls, function names are in lowercase + // and we fail to find them in the map. Add a lowercase name entry. + methods.put(names[i].toLowerCase(), clazz); + } } } + public ArrayListMultimap<String, Class<? extends GenericUDF>> getGenericUDFs() { + return methodsGenericUDF; + } + + public ArrayListMultimap<String, Class<? extends UDF>> getUDFs() { + return methodsUDF; + } + /** * Find the UDF class for given function name and check if it accepts the given input argument * types. If a match is found, create a holder and return @@ -127,7 +140,7 @@ public class HiveFunctionImplementationRegistry { nonDeterministicUDFs.contains(udfClazz)); } catch(IllegalAccessException | InstantiationException e) { logger.debug("Failed to instantiate class", e); - } catch(UDFArgumentException e) { /*ignore this*/ } + } catch(Exception e) { /*ignore this*/ } return null; } @@ -147,7 +160,7 @@ public class HiveFunctionImplementationRegistry { returnOI, Types.optional(ObjectInspectorHelper.getDrillType(returnOI)), nonDeterministicUDFs.contains(udfClazz)); - } catch(UDFArgumentException e) { /*ignore this*/ } + } catch(Exception e) { /*ignore this*/ } return null; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e602b2a9/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java index 7c8bce2..772b3b9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.google.common.collect.Sets; import org.apache.drill.exec.expr.fn.DrillFuncHolder; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.eigenbase.sql.SqlFunctionCategory; @@ -33,7 +34,6 @@ import org.eigenbase.sql.fun.SqlStdOperatorTable; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Lists; -import com.google.hive12.common.collect.Sets; public class DrillOperatorTable extends SqlStdOperatorTable { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillOperatorTable.class); @@ -63,10 +63,15 @@ public class DrillOperatorTable extends SqlStdOperatorTable { } } - // TODO: add hive functions. - } - + for (String name : Sets.union( + registry.getHiveRegistry().getGenericUDFs().asMap().keySet(), + registry.getHiveRegistry().getUDFs().asMap().keySet())) { + SqlOperator op = new HiveUDFOperator(name.toUpperCase()); + operators.add(op); + opMap.put(name, op); + } + } @Override public void lookupOperatorOverloads(SqlIdentifier opName, SqlFunctionCategory category, SqlSyntax syntax, List<SqlOperator> operatorList) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e602b2a9/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java new file mode 100644 index 0000000..71860c3 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.planner.sql; + +import org.eigenbase.reltype.RelDataType; +import org.eigenbase.sql.SqlCall; +import org.eigenbase.sql.SqlCallBinding; +import org.eigenbase.sql.SqlFunction; +import org.eigenbase.sql.SqlFunctionCategory; +import org.eigenbase.sql.SqlIdentifier; +import org.eigenbase.sql.SqlOperandCountRange; +import org.eigenbase.sql.SqlOperator; +import org.eigenbase.sql.parser.SqlParserPos; +import org.eigenbase.sql.type.SqlOperandCountRanges; +import org.eigenbase.sql.type.SqlOperandTypeChecker; +import org.eigenbase.sql.type.SqlTypeName; +import org.eigenbase.sql.validate.SqlValidator; +import org.eigenbase.sql.validate.SqlValidatorScope; + +public class HiveUDFOperator extends SqlFunction { + + public HiveUDFOperator(String name) { + super(new SqlIdentifier(name, SqlParserPos.ZERO), DynamicReturnType.INSTANCE, null, new ArgChecker(), null, + SqlFunctionCategory.USER_DEFINED_FUNCTION); + } + + @Override + public RelDataType deriveType(SqlValidator validator, SqlValidatorScope scope, SqlCall call) { + return validator.getTypeFactory().createSqlType(SqlTypeName.ANY); + } + + /** Argument Checker for variable number of arguments */ + public static class ArgChecker implements SqlOperandTypeChecker { + + public static ArgChecker INSTANCE = new ArgChecker(); + + private SqlOperandCountRange range = SqlOperandCountRanges.any(); + + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + return true; + } + + @Override + public SqlOperandCountRange getOperandCountRange() { + return range; + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) { + return opName + "(HiveUDF - Opaque)"; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e602b2a9/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java index eabeda2..b2fa898 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java @@ -101,10 +101,8 @@ public class TestHiveUDFs extends ExecTest { NullableVar16CharVector concatV = (NullableVar16CharVector) vv.next(); Float4Vector flt1V = (Float4Vector) vv.next(); NullableVar16CharVector format_numberV = (NullableVar16CharVector) vv.next(); - - /* DRILL-425 NullableVar16CharVector nullableStr1V = ((NullableVar16CharVector) vv.next()); - NullableVar16CharVector upperNullableStr1V = ((NullableVar16CharVector) vv.next()); */ + NullableVar16CharVector upperNullableStr1V = ((NullableVar16CharVector) vv.next()); for(int i=0; i<exec.getRecordCount(); i++) { @@ -123,14 +121,12 @@ public class TestHiveUDFs extends ExecTest { String nullableStr1 = null; - /* DRILL-425 if (!nullableStr1V.getAccessor().isNull(i)) - nullableStr1 = new String(nullableStr1V.getAccessor().get(i), Charsets.UTF_16);*/ + nullableStr1 = new String(nullableStr1V.getAccessor().get(i), Charsets.UTF_16); String upperNullableStr1 = null; - /* DRILL-425 if (!upperNullableStr1V.getAccessor().isNull(i)) - upperNullableStr1 = new String(upperNullableStr1V.getAccessor().get(i), Charsets.UTF_16); */ + upperNullableStr1 = new String(upperNullableStr1V.getAccessor().get(i), Charsets.UTF_16); assertEquals(nullableStr1 != null, upperNullableStr1 != null); if (nullableStr1 != null) http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e602b2a9/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java index 8a31a27..d07ce85 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java @@ -32,6 +32,7 @@ import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; @@ -204,7 +205,7 @@ public class ExpressionTreeMaterializerTest extends ExecTest { ImmutableList.of((LogicalExpression) new FieldReference("test", ExpressionPosition.UNKNOWN) ), ExpressionPosition.UNKNOWN); LogicalExpression newExpr = ExpressionTreeMaterializer.materialize(functionCallExpr, batch, ec, registry); - assertEquals(newExpr, null); + assertTrue(newExpr instanceof TypedNullConstant); assertEquals(1, ec.getErrorCount()); System.out.println(ec.toErrorString()); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e602b2a9/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json b/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json index 7b82570..e849e00 100644 --- a/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json +++ b/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json @@ -16,7 +16,7 @@ {name: "str1", type: "VAR16CHAR", mode: "REQUIRED"}, {name: "str2", type: "VAR16CHAR", mode: "REQUIRED"}, {name: "str3", type: "VAR16CHAR", mode: "REQUIRED"}, - /* DRILL-425 {name: "nullableStr1", type: "VAR16CHAR", mode: "OPTIONAL"}, */ + {name: "nullableStr1", type: "VAR16CHAR", mode: "OPTIONAL"}, {name: "flt1", type: "FLOAT4", mode: "REQUIRED"} ]} ] @@ -31,9 +31,9 @@ { ref: "unix_timestamp", expr: "unix_timestamp()" }, { ref: "concat", expr: "concat_ws('-', str2, str3)" }, { ref: "flt1", expr: "flt1" }, - { ref: "format_number", expr: "format_number(cast(flt1 as float8), cast(2 as int))" } /* DRILL-425, + { ref: "format_number", expr: "format_number(cast(flt1 as float8), cast(2 as int))" }, { ref: "nullableStr1", expr: "nullableStr1" }, - { ref: "upperNulableStr1", expr: "upper(nullableStr1)" } */ + { ref: "upperNulableStr1", expr: "upper(nullableStr1)" } ] }, { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e602b2a9/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveScalarUDFs.java ---------------------------------------------------------------------- diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveScalarUDFs.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveScalarUDFs.java new file mode 100644 index 0000000..76af1b0 --- /dev/null +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestHiveScalarUDFs.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.jdbc.test; + + +import org.apache.drill.exec.store.hive.HiveTestDataGenerator; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.junit.BeforeClass; +import org.junit.Test; + + +public class TestHiveScalarUDFs { + + @BeforeClass + public static void generateHive() throws Exception{ + new HiveTestDataGenerator().generateTestData(); + } + + /** Test a hive function that implements the interface {@link org.apache.hadoop.hive.ql.exec.UDF}. */ + @Test + public void simpleUDF() throws Exception { + JdbcAssert.withNoDefaultSchema() + .sql("SELECT " + + "from_unixtime(1237573801) as unix_timestamp, " + + "UDFDegrees(cast(26.89 as DOUBLE)) as degrees " + + "FROM cp.`employee.json` LIMIT 1") + .returns("unix_timestamp=2009-03-20 11:30:01; degrees=1540.6835111067835"); + } + + /** Test a hive function that implements the interface {@link org.apache.hadoop.hive.ql.udf.generic.GenericUDF}. */ + @Test + public void simpleGenericUDF() throws Exception{ + JdbcAssert.withNoDefaultSchema() + .sql("SELECT CAST(" + + "encode('text', 'UTF-8') " + + "AS VARCHAR(5)) " + + "FROM cp.`employee.json` LIMIT 1") + .returns("EXPR$0=text"); + } +}
