DRILL-1192: Hive Scalar UDFs: Add Date, TimeStamp and Decimal type support Also following refactoring: + Minimize the number of variables in HiveTypes.tdd + Make use of Hive TypeEntries and Hive AbstractPrimitiveObjectInspector to simplify Drill ObjectInspectors implementations.
Test: + Add Hive UDF test implementations and testcases to cover all supported types (passing data into Hive UDF and reading data returned from Hive UDF). Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/1e9930fb Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/1e9930fb Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/1e9930fb Branch: refs/heads/master Commit: 1e9930fbae8279afe3eed9e7f18392fb1a08688a Parents: 9b2eae2 Author: vkorukanti <[email protected]> Authored: Wed Jul 23 09:33:15 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Fri Jul 25 14:34:36 2014 -0700 ---------------------------------------------------------------------- contrib/storage-hive/core/pom.xml | 8 + .../core/src/main/codegen/data/HiveTypes.tdd | 58 +++-- .../templates/ObjectInspectorHelper.java | 45 +++- .../codegen/templates/ObjectInspectors.java | 223 +++++++++++++---- .../drill/exec/expr/fn/HiveFuncHolder.java | 6 +- .../exec/expr/fn/HiveFunctionRegistry.java | 11 +- .../AbstractDrillPrimitiveObjectInspector.java | 39 +++ .../hive/AbstractPrimitiveObjectInspector.java | 70 ------ .../drill/exec/TestHiveProjectPushDown.java | 8 +- .../drill/exec/fn/hive/HiveTestUDFImpls.java | 237 +++++++++++++++++++ .../drill/exec/fn/hive/TestSampleHiveUDFs.java | 154 ++++++++++++ .../exec/store/hive/HiveTestDataGenerator.java | 1 + .../core/src/test/resources/drill-module.conf | 5 + .../org/apache/drill/exec/util/VectorUtil.java | 35 +++ .../java/org/apache/drill/BaseTestQuery.java | 19 ++ 15 files changed, 751 insertions(+), 168 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index 38e4a41..9d54444 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -123,6 +123,14 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <forkCount>1</forkCount> + <reuseForks>false</reuseForks> + </configuration> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd b/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd index c23f981..2b4338b 100644 --- a/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd +++ b/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd @@ -19,82 +19,80 @@ { hiveType: "BOOLEAN", hiveOI: "BooleanObjectInspector", - serdeConstant: "BOOLEAN_TYPE_NAME", javaType: "boolean", - minorType: "BIT", - holder: "Bit" + drillType: "Bit" }, { hiveType: "BYTE", hiveOI: "ByteObjectInspector", - serdeConstant: "TINYINT_TYPE_NAME", javaType: "byte", - minorType: "TINYINT", - holder: "TinyInt" + drillType: "TinyInt" }, { hiveType: "SHORT", hiveOI: "ShortObjectInspector", - serdeConstant: "SMALLINT_TYPE_NAME", javaType: "short", - minorType: "SMALLINT", - holder: "SmallInt" + drillType: "SmallInt" }, { hiveType: "INT", hiveOI: "IntObjectInspector", - serdeConstant: "INT_TYPE_NAME", javaType: "int", - minorType: "INT", - holder: "Int" + drillType: "Int" }, { hiveType: "LONG", hiveOI: "LongObjectInspector", - serdeConstant: "BIGINT_TYPE_NAME", javaType: "long", - minorType: "BIGINT", - holder: "BigInt" + drillType: "BigInt" }, { hiveType: "FLOAT", hiveOI: "FloatObjectInspector", - serdeConstant: "FLOAT_TYPE_NAME", javaType: "float", - minorType: "FLOAT4", - holder: "Float4" + drillType: "Float4" }, { hiveType: "DOUBLE", hiveOI: "DoubleObjectInspector", - serdeConstant: "DOUBLE_TYPE_NAME", javaType: "double", - minorType: "FLOAT8", - holder: "Float8" + drillType: "Float8" }, { hiveType: "VARCHAR", hiveOI: "HiveVarcharObjectInspector", - serdeConstant: "VARCHAR_TYPE_NAME", javaType: "", - minorType: "VARCHAR", - holder: "VarChar" + drillType: "VarChar" }, { hiveType: "STRING", hiveOI: "StringObjectInspector", - serdeConstant: "STRING_TYPE_NAME", javaType: "", - minorType: "VAR16CHAR", - holder: "Var16Char" + drillType: "Var16Char" }, { hiveType: "BINARY", hiveOI: "BinaryObjectInspector", - serdeConstant: "BINARY_TYPE_NAME", javaType: "", - minorType: "VARBINARY", - holder: "VarBinary" + drillType: "VarBinary" + }, + { + hiveType: "TIMESTAMP", + hiveOI: "TimestampObjectInspector", + javaType: "java.sql.Timestamp", + drillType: "TimeStamp" + }, + { + hiveType: "DECIMAL", + hiveOI: "HiveDecimalObjectInspector", + javaType: "org.apache.hadoop.hive.common.type.HiveDecimal", + drillType: "Decimal38Sparse" + }, + { + hiveType: "DATE", + hiveOI: "DateObjectInspector", + javaType: "java.sql.Date", + drillType: "Date" } ] } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java index 22a9eb2..091f027 100644 --- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java +++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + <@pp.dropOutputFile /> <@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/ObjectInspectorHelper.java" /> @@ -25,38 +26,51 @@ package org.apache.drill.exec.expr.fn.impl.hive; import com.sun.codemodel.*; import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.expr.DirectExpression; import org.apache.drill.exec.expr.TypeHelper; -import org.apache.drill.exec.expr.holders.*; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.*; +import java.lang.UnsupportedOperationException; +import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; public class ObjectInspectorHelper { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ObjectInspectorHelper.class); - private static Map<MinorType, Class> OIMAP = new HashMap<>(); + private static Map<MinorType, Class> OIMAP_REQUIRED = new HashMap<>(); + private static Map<MinorType, Class> OIMAP_OPTIONAL = new HashMap<>(); static { <#list drillOI.map as entry> - OIMAP.put(MinorType.${entry.minorType}, Drill${entry.holder}ObjectInspector.class); + OIMAP_REQUIRED.put(MinorType.${entry.drillType?upper_case}, Drill${entry.drillType}ObjectInspector.Required.class); + OIMAP_OPTIONAL.put(MinorType.${entry.drillType?upper_case}, Drill${entry.drillType}ObjectInspector.Optional.class); </#list> } - public static ObjectInspector getDrillObjectInspector(MinorType drillType) { - if (OIMAP.containsKey(drillType)) { - try { - return (ObjectInspector)OIMAP.get(drillType).newInstance(); - } catch(InstantiationException | IllegalAccessException e) { - throw new RuntimeException("Failed to instantiate ObjectInspector", e); + public static ObjectInspector getDrillObjectInspector(DataMode mode, MinorType minorType) { + try { + if (mode == DataMode.REQUIRED) { + if (OIMAP_REQUIRED.containsKey(minorType)) { + return (ObjectInspector) OIMAP_REQUIRED.get(minorType).newInstance(); + } + } else if (mode == DataMode.OPTIONAL) { + if (OIMAP_OPTIONAL.containsKey(minorType)) { + return (ObjectInspector) OIMAP_OPTIONAL.get(minorType).newInstance(); + } + } else { + throw new UnsupportedOperationException("Repeated types are not supported as arguement to Hive UDFs"); } + } catch(InstantiationException | IllegalAccessException e) { + throw new RuntimeException("Failed to instantiate ObjectInspector", e); } - throw new UnsupportedOperationException(drillType.toString()); + throw new UnsupportedOperationException( + String.format("Type %s[%s] not supported as arguement to Hive UDFs", minorType.toString(), mode.toString())); } public static JBlock initReturnValueHolder(JCodeModel m, JVar returnValueHolder, ObjectInspector oi, MinorType returnType) { @@ -95,7 +109,7 @@ public class ObjectInspectorHelper { private static Map<PrimitiveCategory, MinorType> TYPE_HIVE2DRILL = new HashMap<>(); static { <#list drillOI.map as entry> - TYPE_HIVE2DRILL.put(PrimitiveCategory.${entry.hiveType}, MinorType.${entry.minorType}); + TYPE_HIVE2DRILL.put(PrimitiveCategory.${entry.hiveType}, MinorType.${entry.drillType?upper_case}); </#list> } @@ -168,7 +182,14 @@ public class ObjectInspectorHelper { .invoke("setBytes").arg(JExpr.lit(0)).arg(data)); jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0)); jc._else().assign(returnValueHolder.ref("end"), data.ref("length")); - + <#elseif entry.hiveType == "TIMESTAMP"> + JVar tsVar = jc._else().decl(m.directClass(java.sql.Timestamp.class.getCanonicalName()), "ts", + castedOI.invoke("getPrimitiveJavaObject").arg(returnValue)); + jc._else().assign(returnValueHolder.ref("value"), tsVar.invoke("getTime")); + <#elseif entry.hiveType == "DATE"> + JVar dVar = jc._else().decl(m.directClass(java.sql.Date.class.getCanonicalName()), "d", + castedOI.invoke("getPrimitiveJavaObject").arg(returnValue)); + jc._else().assign(returnValueHolder.ref("value"), dVar.invoke("getTime")); <#else> jc._else().assign(returnValueHolder.ref("value"), castedOI.invoke("get").arg(returnValue)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java index 9a8c837..2267ad5 100644 --- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java +++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java @@ -18,28 +18,40 @@ <@pp.dropOutputFile /> <#list drillOI.map as entry> -<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/Drill${entry.holder}ObjectInspector.java" /> +<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/Drill${entry.drillType}ObjectInspector.java" /> <#include "/@includes/license.ftl" /> package org.apache.drill.exec.expr.fn.impl.hive; +import org.apache.drill.common.util.DecimalUtility; import org.apache.drill.exec.expr.holders.*; +import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.io.ByteWritable; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.primitive.*; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; -public class Drill${entry.holder}ObjectInspector extends AbstractPrimitiveObjectInspector +public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDrillPrimitiveObjectInspector implements ${entry.hiveOI} { - @Override - public String getTypeName() { - return serdeConstants.${entry.serdeConstant}; + public Drill${entry.drillType}ObjectInspector() { + super(PrimitiveObjectInspectorUtils.${entry.hiveType?lower_case}TypeEntry); } -<#if entry.minorType == "VARCHAR"> +<#if entry.drillType == "VarChar"> @Override public HiveVarcharWritable getPrimitiveWritableObject(Object o) { HiveVarcharWritable valW = new HiveVarcharWritable(); @@ -47,67 +59,182 @@ public class Drill${entry.holder}ObjectInspector extends AbstractPrimitiveObject return valW; } - @Override - public HiveVarchar getPrimitiveJavaObject(Object o) { - String val = ((VarCharHolder)o).toString(); - return new HiveVarchar(val, HiveVarchar.MAX_VARCHAR_LENGTH); + public static class Required extends Drill${entry.drillType}ObjectInspector { + @Override + public HiveVarchar getPrimitiveJavaObject(Object o) { + return new HiveVarchar(((VarCharHolder)o).toString(), HiveVarchar.MAX_VARCHAR_LENGTH); + } + } + + public static class Optional extends Drill${entry.drillType}ObjectInspector { + @Override + public HiveVarchar getPrimitiveJavaObject(Object o) { + return new HiveVarchar(((NullableVarCharHolder)o).toString(), HiveVarchar.MAX_VARCHAR_LENGTH); + } } -<#elseif entry.minorType == "VAR16CHAR"> + +<#elseif entry.drillType == "Var16Char"> @Override public Text getPrimitiveWritableObject(Object o) { throw new UnsupportedOperationException(); } - @Override - public String getPrimitiveJavaObject(Object o) { - if (o instanceof Var16CharHolder) - return ((Var16CharHolder)o).toString(); - else - return ((NullableVar16CharHolder)o).toString(); + public static class Required extends Drill${entry.drillType}ObjectInspector { + @Override + public String getPrimitiveJavaObject(Object o){ + return((Var16CharHolder)o).toString(); + } } -<#elseif entry.minorType == "VARBINARY"> -@Override -public org.apache.hadoop.io.BytesWritable getPrimitiveWritableObject(Object o) { - throw new UnsupportedOperationException(); -} -@Override -public byte[] getPrimitiveJavaObject(Object o) { - if (o instanceof VarBinaryHolder){ - VarBinaryHolder h = (VarBinaryHolder)o; - byte[] buf = new byte[h.end-h.start]; - h.buffer.getBytes(h.start, buf, 0, h.end-h.start); - return buf; - }else{ - NullableVarBinaryHolder h = (NullableVarBinaryHolder)o; - byte[] buf = new byte[h.end-h.start]; - h.buffer.getBytes(h.start, buf, 0, h.end-h.start); - return buf; - + public static class Optional extends Drill${entry.drillType}ObjectInspector { + @Override + public String getPrimitiveJavaObject(Object o){ + return((NullableVar16CharHolder)o).toString(); + } } -} -<#elseif entry.minorType == "BIT"> +<#elseif entry.drillType == "VarBinary"> @Override - public boolean get(Object o) { - if (o instanceof BitHolder) - return ((BitHolder)o).value == 0 ? false : true; - else + public BytesWritable getPrimitiveWritableObject(Object o) { + return new BytesWritable(getPrimitiveJavaObject(o)); + } + + public static class Required extends Drill${entry.drillType}ObjectInspector { + @Override + public byte[] getPrimitiveJavaObject(Object o) { + VarBinaryHolder h = (VarBinaryHolder)o; + byte[] buf = new byte[h.end-h.start]; + h.buffer.getBytes(h.start, buf, 0, h.end-h.start); + return buf; + } + } + + public static class Optional extends Drill${entry.drillType}ObjectInspector { + @Override + public byte[] getPrimitiveJavaObject(Object o) { + NullableVarBinaryHolder h = (NullableVarBinaryHolder)o; + byte[] buf = new byte[h.end-h.start]; + h.buffer.getBytes(h.start, buf, 0, h.end-h.start); + return buf; + } + } + +<#elseif entry.drillType == "Bit"> + public static class Required extends Drill${entry.drillType}ObjectInspector { + @Override + public boolean get(Object o) { + return ((BitHolder)o).value == 0 ? false : true; + } + } + + public static class Optional extends Drill${entry.drillType}ObjectInspector { + @Override + public boolean get(Object o) { return ((NullableBitHolder)o).value == 0 ? false : true; + } + } + + @Override + public BooleanWritable getPrimitiveWritableObject(Object o) { + return new BooleanWritable(get(o)); + } + + @Override + public Boolean getPrimitiveJavaObject(Object o) { + return new Boolean(get(o)); + } + +<#elseif entry.drillType == "Decimal38Sparse"> + public HiveDecimalWritable getPrimitiveWritableObject(Object o) { + return new HiveDecimalWritable(getPrimitiveJavaObject(o)); + } + + public static class Required extends Drill${entry.drillType}ObjectInspector{ + @Override + public HiveDecimal getPrimitiveJavaObject(Object o){ + Decimal38SparseHolder h = (Decimal38SparseHolder) o; + return new HiveDecimal(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale)); + } + } + + public static class Optional extends Drill${entry.drillType}ObjectInspector{ + @Override + public HiveDecimal getPrimitiveJavaObject(Object o){ + NullableDecimal38SparseHolder h = (NullableDecimal38SparseHolder) o; + return new HiveDecimal(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale)); + } + } + +<#elseif entry.drillType == "TimeStamp"> + @Override + public TimestampWritable getPrimitiveWritableObject(Object o) { + return new TimestampWritable(getPrimitiveJavaObject(o)); + } + + public static class Required extends Drill${entry.drillType}ObjectInspector{ + @Override + public java.sql.Timestamp getPrimitiveJavaObject(Object o){ + return new java.sql.Timestamp(((TimeStampHolder)o).value); + } + } + + public static class Optional extends Drill${entry.drillType}ObjectInspector{ + @Override + public java.sql.Timestamp getPrimitiveJavaObject(Object o){ + return new java.sql.Timestamp(((NullableTimeStampHolder)o).value); + } + } + +<#elseif entry.drillType == "Date"> + @Override + public DateWritable getPrimitiveWritableObject(Object o) { + return new DateWritable(getPrimitiveJavaObject(o)); + } + + public static class Required extends Drill${entry.drillType}ObjectInspector{ + @Override + public java.sql.Date getPrimitiveJavaObject(Object o){ + return new java.sql.Date(((DateHolder)o).value); + } + } + + public static class Optional extends Drill${entry.drillType}ObjectInspector{ + @Override + public java.sql.Date getPrimitiveJavaObject(Object o){ + return new java.sql.Date(((NullableDateHolder)o).value); + } } <#else> +<#if entry.drillType == "Int"> @Override - public ${entry.javaType} get(Object o) { - if (o instanceof ${entry.holder}Holder) - return ((${entry.holder}Holder)o).value; - else - return ((Nullable${entry.holder}Holder)o).value; + public Integer getPrimitiveJavaObject(Object o) { + return new Integer(get(o)); + } +<#else> + @Override + public ${entry.javaType?cap_first} getPrimitiveJavaObject(Object o) { + return new ${entry.javaType?cap_first}(get(o)); } </#if> @Override - public PrimitiveCategory getPrimitiveCategory() { - return PrimitiveCategory.${entry.hiveType}; + public ${entry.javaType?cap_first}Writable getPrimitiveWritableObject(Object o) { + return new ${entry.javaType?cap_first}Writable(get(o)); + } + + public static class Required extends Drill${entry.drillType}ObjectInspector{ + @Override + public ${entry.javaType} get(Object o){ + return((${entry.drillType}Holder)o).value; + } } + + public static class Optional extends Drill${entry.drillType}ObjectInspector{ + @Override + public ${entry.javaType} get(Object o){ + return((Nullable${entry.drillType}Holder)o).value; + } + } +</#if> } </#list> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java index 813d4c5..68a9b97 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java @@ -30,6 +30,7 @@ import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.expr.ClassGenerator; import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer; @@ -170,12 +171,13 @@ public class HiveFuncHolder extends AbstractFuncHolder { JClass oih = m.directClass(ObjectInspectorHelper.class.getCanonicalName()); JClass mt = m.directClass(TypeProtos.MinorType.class.getCanonicalName()); + JClass mode = m.directClass(DataMode.class.getCanonicalName()); for(int i=0; i<argTypes.length; i++) { sub.assign( oiArray.component(JExpr.lit(i)), oih.staticInvoke("getDrillObjectInspector") - .arg(mt.staticInvoke("valueOf") - .arg(JExpr.lit(argTypes[i].getMinorType().getNumber())))); + .arg(mode.staticInvoke("valueOf").arg(JExpr.lit(argTypes[i].getMode().getNumber()))) + .arg(mt.staticInvoke("valueOf").arg(JExpr.lit(argTypes[i].getMinorType().getNumber())))); } // declare and instantiate DeferredObject array http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java index b9369ed..119c93b 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java @@ -99,8 +99,15 @@ public class HiveFunctionRegistry implements PluggableFunctionRegistry{ MajorType[] argTypes = new MajorType[call.args.size()]; ObjectInspector[] argOIs = new ObjectInspector[call.args.size()]; for(int i=0; i<call.args.size(); i++) { - argTypes[i] = call.args.get(i).getMajorType(); - argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMinorType()); + try { + argTypes[i] = call.args.get(i).getMajorType(); + argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMode(), argTypes[i].getMinorType()); + } catch(Exception e) { + // Hive throws errors if there are unsupported types. Consider there is no hive UDF supporting the + // given argument types + logger.trace("Failed to find a hive function for given FunctionCall: '{}'", call.toString(), e); + return null; + } } String funcName = call.getName().toLowerCase(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java new file mode 100644 index 0000000..2a98398 --- /dev/null +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn.impl.hive; + +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; + + +public abstract class AbstractDrillPrimitiveObjectInspector extends AbstractPrimitiveObjectInspector { + + public AbstractDrillPrimitiveObjectInspector(PrimitiveTypeEntry typeEntry) { + super(typeEntry); + } + + @Override + public Object copyObject(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean preferWritable() { + return false; + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java deleted file mode 100644 index 04b552e..0000000 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.expr.fn.impl.hive; - -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; - - -public abstract class AbstractPrimitiveObjectInspector implements PrimitiveObjectInspector { - - @Override - public Class<?> getPrimitiveWritableClass() { - throw new UnsupportedOperationException(); - } - - @Override - public Object getPrimitiveWritableObject(Object o) { - throw new UnsupportedOperationException(); - } - - @Override - public Class<?> getJavaPrimitiveClass() { - throw new UnsupportedOperationException(); - } - - @Override - public Object getPrimitiveJavaObject(Object o) { - throw new UnsupportedOperationException(); - } - - @Override - public Object copyObject(Object o) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean preferWritable() { - return false; - } - - @Override - public BaseTypeParams getTypeParams() { - throw new UnsupportedOperationException(); - } - - @Override - public void setTypeParams(BaseTypeParams baseTypeParams) { - throw new UnsupportedOperationException(); - } - - @Override - public Category getCategory() { - return Category.PRIMITIVE; - } -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java index 6ced636..a985cae 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java @@ -52,7 +52,7 @@ public class TestHiveProjectPushDown extends PlanTestBase { String query = "SELECT boolean_field as b_f, tinyint_field as ti_f FROM hive.`default`.readtest"; String expectedColNames = " \"columns\" : [ \"`boolean_field`\", \"`tinyint_field`\" ]"; - testHelper(query, expectedColNames, 1); + testHelper(query, expectedColNames, 2); } @Test @@ -60,7 +60,7 @@ public class TestHiveProjectPushDown extends PlanTestBase { String query = "SELECT double_part as dbl_p FROM hive.`default`.readtest"; String expectedColNames = " \"columns\" : [ \"`double_part`\" ]"; - testHelper(query, expectedColNames, 1); + testHelper(query, expectedColNames, 2); } @Test @@ -68,7 +68,7 @@ public class TestHiveProjectPushDown extends PlanTestBase { String query = "SELECT double_part as dbl_p, decimal_part as dec_p FROM hive.`default`.readtest"; String expectedColNames = " \"columns\" : [ \"`double_part`\", \"`decimal_part`\" ]"; - testHelper(query, expectedColNames, 1); + testHelper(query, expectedColNames, 2); } @Test @@ -78,7 +78,7 @@ public class TestHiveProjectPushDown extends PlanTestBase { String expectedColNames = " \"columns\" : [ \"`boolean_field`\", \"`tinyint_field`\", " + "\"`double_part`\", \"`decimal_part`\" ]"; - testHelper(query, expectedColNames, 1); + testHelper(query, expectedColNames, 2); } @Test http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java new file mode 100644 index 0000000..8fc059b --- /dev/null +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java @@ -0,0 +1,237 @@ + +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.fn.hive; + +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; + +/** + * Contains test Hive UDF that take a particular data type and return the same data type and value. + * These test UDFs exercise the code paths in Drill ObjectInspectors and parsing the return value from Hive UDF + * for all supported data types in Hive UDF. + */ +public class HiveTestUDFImpls { + public static abstract class GenericUDFTestBase extends GenericUDF { + protected final String udfName; + protected final PrimitiveCategory inputType; + protected final PrimitiveCategory outputType; + protected ObjectInspector argumentOI; + + public GenericUDFTestBase(String udfName, PrimitiveCategory type) { + this(udfName, type, type); + } + + public GenericUDFTestBase(String udfName, PrimitiveCategory inputType, PrimitiveCategory outputType) { + this.udfName = udfName; + this.inputType = inputType; + this.outputType = outputType; + } + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentLengthException(String.format("%s needs 1 argument, got %d", udfName, arguments.length)); + } + + if (arguments[0].getCategory() != Category.PRIMITIVE || + ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() != inputType) { + String actual = arguments[0].getCategory() + (arguments[0].getCategory() == Category.PRIMITIVE ? + "[" + ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() + "]" : ""); + throw new UDFArgumentException( + String.format("%s only takes primitive type %s, got %s", udfName, inputType, actual)); + } + argumentOI = arguments[0]; + return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(outputType); + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append(udfName); + sb.append("("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(","); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + if (arguments[0] == null) { + return null; + } + + Object input = arguments[0].get(); + switch(inputType) { + case BOOLEAN: + return ((BooleanObjectInspector)argumentOI).get(input) ? Boolean.TRUE : Boolean.FALSE; + case BYTE: + return new Byte(((ByteObjectInspector)argumentOI).get(input)); + case SHORT: + return new Short(((ShortObjectInspector)argumentOI).get(input)); + case INT: + return new Integer(((IntObjectInspector)argumentOI).get(input)); + case LONG: + return new Long(((LongObjectInspector)argumentOI).get(input)); + case FLOAT: + return new Float(((FloatObjectInspector)argumentOI).get(input)); + case DOUBLE: + return new Double(((DoubleObjectInspector)argumentOI).get(input)); + case STRING: + return PrimitiveObjectInspectorUtils.getString(input, (StringObjectInspector)argumentOI); + case BINARY: + return PrimitiveObjectInspectorUtils.getBinary(input, (BinaryObjectInspector) argumentOI).getBytes(); + case VARCHAR: + return PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector)argumentOI); + case DATE: + return PrimitiveObjectInspectorUtils.getDate(input, (DateObjectInspector) argumentOI); + case TIMESTAMP: + return PrimitiveObjectInspectorUtils.getTimestamp(input, (TimestampObjectInspector) argumentOI); + case DECIMAL: + // return type is a HiveVarchar + HiveDecimal decimalValue = + PrimitiveObjectInspectorUtils.getHiveDecimal(input, (HiveDecimalObjectInspector) argumentOI); + return new HiveVarchar(decimalValue.toString(), HiveVarchar.MAX_VARCHAR_LENGTH); + } + + throw new UnsupportedOperationException(String.format("Unexpected input type '%s' in Test UDF", inputType)); + } + } + + @Description(name = "testHiveUDFBOOLEAN", value = "_FUNC_(BOOLEAN) - Tests boolean data as input and output") + public static class GenericUDFTestBOOLEAN extends GenericUDFTestBase { + public GenericUDFTestBOOLEAN() { + super("testHiveUDFBOOLEAN", PrimitiveCategory.BOOLEAN); + } + } + + @Description(name = "testHiveUDFBYTE", value = "_FUNC_(BYTE) - Tests byte data as input and output") + public static class GenericUDFTestBYTE extends GenericUDFTestBase { + public GenericUDFTestBYTE() { + super("testHiveUDFBYTE", PrimitiveCategory.BYTE); + } + } + + @Description(name = "testHiveUDFSHORT", value = "_FUNC_(SHORT) - Tests short data as input and output") + public static class GenericUDFTestSHORT extends GenericUDFTestBase { + public GenericUDFTestSHORT() { + super("testHiveUDFSHORT", PrimitiveCategory.SHORT); + } + } + + @Description(name = "testHiveUDFINT", value = "_FUNC_(INT) - Tests int data as input and output") + public static class GenericUDFTestINT extends GenericUDFTestBase { + public GenericUDFTestINT() { + super("testHiveUDFINT", PrimitiveCategory.INT); + } + } + + @Description(name = "testHiveUDFLONG", value = "_FUNC_(LONG) - Tests long data as input and output") + public static class GenericUDFTestLONG extends GenericUDFTestBase { + public GenericUDFTestLONG() { + super("testHiveUDFLONG", PrimitiveCategory.LONG); + } + } + + @Description(name = "testHiveUDFFLOAT", value = "_FUNC_(FLOAT) - Tests float data as input and output") + public static class GenericUDFTestFLOAT extends GenericUDFTestBase { + public GenericUDFTestFLOAT() { + super("testHiveUDFFLOAT", PrimitiveCategory.FLOAT); + } + } + + @Description(name = "testHiveUDFDOUBLE", value = "_FUNC_(DOUBLE) - Tests double data as input and output") + public static class GenericUDFTestDOUBLE extends GenericUDFTestBase { + public GenericUDFTestDOUBLE() { + super("testHiveUDFDOUBLE", PrimitiveCategory.DOUBLE); + } + } + + @Description(name = "testHiveUDFVARCHAR", value = "_FUNC_(VARCHAR) - Tests varchar data as input and output") + public static class GenericUDFTestVARCHAR extends GenericUDFTestBase { + public GenericUDFTestVARCHAR() { + super("testHiveUDFVARCHAR", PrimitiveCategory.VARCHAR); + } + } + + @Description(name = "testHiveUDFSTRING", value = "_FUNC_(STRING) - Tests string data as input and output") + public static class GenericUDFTestSTRING extends GenericUDFTestBase { + public GenericUDFTestSTRING() { + super("testHiveUDFSTRING", PrimitiveCategory.STRING); + } + } + + @Description(name = "testHiveUDFBINARY", value = "_FUNC_(BINARY) - Tests binary data as input and output") + public static class GenericUDFTestBINARY extends GenericUDFTestBase { + public GenericUDFTestBINARY() { + super("testHiveUDFBINARY", PrimitiveCategory.BINARY); + } + } + + @Description(name = "testHiveUDFTIMESTAMP", value = "_FUNC_(TIMESTAMP) - Tests timestamp data as input and output") + public static class GenericUDFTestTIMESTAMP extends GenericUDFTestBase { + public GenericUDFTestTIMESTAMP() { + super("testHiveUDFTIMESTAMP", PrimitiveCategory.TIMESTAMP); + } + } + + @Description(name = "testHiveUDFDATE", value = "_FUNC_(DATE) - Tests date data as input and output") + public static class GenericUDFTestDATE extends GenericUDFTestBase { + public GenericUDFTestDATE() { + super("testHiveUDFDATE", PrimitiveCategory.DATE); + } + } + + @Description(name = "testHiveUDFDECIMAL", value = "_FUNC_(DECIMAL) - Tests decimal data as input and output") + public static class GenericUDFTestDECIMAL extends GenericUDFTestBase { + public GenericUDFTestDECIMAL() { + super("testHiveUDFDECIMAL", PrimitiveCategory.DECIMAL, PrimitiveCategory.VARCHAR); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java new file mode 100644 index 0000000..2881a18 --- /dev/null +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.fn.hive; + +import org.apache.drill.BaseTestQuery; +import org.apache.drill.exec.rpc.user.QueryResultBatch; +import org.apache.drill.exec.store.hive.HiveTestDataGenerator; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertTrue; + +public class TestSampleHiveUDFs extends BaseTestQuery { + + @BeforeClass + public static void addStoragePlugins() throws Exception{ + new HiveTestDataGenerator().createAndAddHiveTestPlugin(bit.getContext().getStorage()); + } + + private void helper(String query, String expected) throws Exception { + List<QueryResultBatch> results = testSqlWithResults(query); + String actual = getResultString(results, ","); + assertTrue(String.format("Result:\n%s\ndoes not match:\n%s", actual, expected), expected.equals(actual)); + } + + @Test + public void booleanInOut() throws Exception{ + String query = "SELECT " + + "testHiveUDFBoolean(true) as col1," + + "testHiveUDFBoolean(false) as col2," + + "testHiveUDFBoolean(cast(null as boolean)) as col3 " + + "FROM hive.kv LIMIT 1"; + + String expected = "col1,col2,col3\n" + "true,false,null\n"; + helper(query, expected); + } + + @Test + public void byteInOut() throws Exception{ + String query = "SELECT testHiveUDFByte(tinyint_field) as col1 FROM hive.readtest"; + String expected = "col1\n" + "34\n" + "null\n"; + helper(query, expected); + } + + @Test + public void shortInOut() throws Exception{ + String query = "SELECT testHiveUDFShort(smallint_field) as col1 FROM hive.readtest"; + String expected = "col1\n" + "3455\n" + "null\n"; + helper(query, expected); + } + + @Test + public void intInOut() throws Exception{ + String query = "SELECT testHiveUDFInt(int_field) as col1 FROM hive.readtest"; + String expected = "col1\n" + "123456\n" + "null\n"; + helper(query, expected); + } + + @Test + public void longInOut() throws Exception{ + String query = "SELECT testHiveUDFLong(bigint_field) as col1 FROM hive.readtest"; + String expected = "col1\n" + "234235\n" + "null\n"; + helper(query, expected); + } + + @Test + public void floatInOut() throws Exception{ + String query = "SELECT testHiveUDFFloat(float_field) as col1 FROM hive.readtest"; + String expected = "col1\n" + "4.67\n" + "null\n"; helper(query, expected); + } + + @Test + public void doubleInOut() throws Exception{ + String query = "SELECT testHiveUDFDouble(double_field) as col1 FROM hive.readtest"; + String expected = "col1\n" + "8.345\n" + "null\n"; + helper(query, expected); + } + + @Test + @Ignore // TODO: need var16char input. String type field read from Hive is converted to VarChar type + public void stringInOut() throws Exception{ + String query = "SELECT testHiveUDFString(string_field) as col1 FROM hive.readtest"; + String expected = "col1\n" + "8.345\n" + "null\n"; + helper(query, expected); + } + + @Test + public void binaryInOut() throws Exception{ + String query = "SELECT testHiveUDFBinary(binary_field) as col1 FROM hive.readtest"; + String expected = "col1\n" + "binaryfield\n" + "\n"; + helper(query, expected); helper(query, expected); + } + + @Test + public void varcharInOut() throws Exception{ + String query = "SELECT " + + "testHiveUDFVarChar('This is a varchar') as col1," + + "testHiveUDFVarChar(cast(null as varchar)) as col2 " + + "FROM hive.kv LIMIT 1"; + + String expected = "col1,col2\n" + "This is a varchar,null\n"; + helper(query, expected); + } + + @Test + public void dateInOut() throws Exception{ + String query = "SELECT " + + "testHiveUDFDate(cast('1970-01-02 10:20:33' as date)) as col1," + + "testHiveUDFDate(cast(null as date)) as col2 " + + "FROM hive.kv LIMIT 1"; + + String expected = "col1,col2\n" + "1970-01-01T08:00:00.000-08:00,null\n"; + helper(query, expected); + } + + @Test + public void timestampInOut() throws Exception{ + String query = "SELECT " + + "testHiveUDFTimeStamp(cast('1970-01-02 10:20:33' as timestamp)) as col1," + + "testHiveUDFTimeStamp(cast(null as timestamp)) as col2 " + + "FROM hive.kv LIMIT 1"; + + String expected = "col1,col2\n" + "1970-01-02T10:20:33.000-08:00,null\n"; + helper(query, expected); + } + + @Test + public void decimalInOut() throws Exception{ + String query = "SELECT " + + "testHiveUDFDecimal(cast('1234567891234567891234567891234567891.4' as decimal(38, 1))) as col1 " + + "FROM hive.kv LIMIT 1"; + + String expected = "col1\n" + "1234567891234567891234567891234567891.4\n"; + helper(query, expected); + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index adb007d..2f500b8 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -261,6 +261,7 @@ public class HiveTestDataGenerator { PrintWriter printWriter = new PrintWriter(file); printWriter.println("YmluYXJ5ZmllbGQ=,false,34,3489423929323435243,8.345,4.67,123456,234235,3455,stringfield,varcharfield,2013-07-05 17:01:00,2013-07-05"); + printWriter.println(",,,,,,,,,,,,"); printWriter.close(); return file.getPath(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/resources/drill-module.conf ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/resources/drill-module.conf b/contrib/storage-hive/core/src/test/resources/drill-module.conf new file mode 100644 index 0000000..271395e --- /dev/null +++ b/contrib/storage-hive/core/src/test/resources/drill-module.conf @@ -0,0 +1,5 @@ +// This file tells Drill to consider this module when class path scanning. +// This file can also include any supplementary configuration information. +// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. + +drill.logical.function.packages += "org.apache.hadoop.hive" http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java index 98585c4..81fce5d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java @@ -19,6 +19,7 @@ package org.apache.drill.exec.util; import java.util.List; +import com.google.common.base.Joiner; import org.apache.commons.lang.StringUtils; import org.apache.drill.common.util.DrillStringUtils; import org.apache.drill.exec.record.MaterializedField; @@ -69,6 +70,40 @@ public class VectorUtil { } } + public static void appendVectorAccessibleContent(VectorAccessible va, StringBuilder formattedResults, + final String delimiter, boolean includeHeader) { + if (includeHeader) { + List<String> columns = Lists.newArrayList(); + for (VectorWrapper<?> vw : va) { + columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath()); + } + + formattedResults.append(Joiner.on(delimiter).join(columns)); + formattedResults.append("\n"); + } + + int rows = va.getRecordCount(); + for (int row = 0; row < rows; row++) { + List<String> rowValues = Lists.newArrayList(); + for (VectorWrapper<?> vw : va) { + Object o = vw.getValueVector().getAccessor().getObject(row); + if (o == null) { + rowValues.add("null"); + } else if (o instanceof byte[]) { + rowValues.add(new String((byte[]) o)); + } else { + rowValues.add(o.toString()); + } + } + formattedResults.append(Joiner.on(delimiter).join(rowValues)); + formattedResults.append("\n"); + } + + for (VectorWrapper<?> vw : va) { + vw.clear(); + } + } + public static void showVectorAccessibleContent(VectorAccessible va) { showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java index 9aa76ec..5e52e82 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java @@ -275,4 +275,23 @@ public class BaseTestQuery extends ExecTest{ return rowCount; } + protected String getResultString(List<QueryResultBatch> results, String delimiter) throws SchemaChangeException { + StringBuilder formattedResults = new StringBuilder(); + boolean includeHeader = true; + RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); + for(QueryResultBatch result : results){ + loader.load(result.getHeader().getDef(), result.getData()); + if (loader.getRecordCount() <= 0) { + break; + } + VectorUtil.appendVectorAccessibleContent(loader, formattedResults, delimiter, includeHeader); + if (!includeHeader) { + includeHeader = false; + } + loader.clear(); + result.release(); + } + + return formattedResults.toString(); + } }
