Author: namit Date: Fri Feb 22 05:32:57 2013 New Revision: 1448900 URL: http://svn.apache.org/r1448900 Log: HIVE-4025 Add reflect UDF for member method invocation of column (Navis via namit)
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java hive/trunk/ql/src/test/queries/clientpositive/udf_reflect2.q hive/trunk/ql/src/test/results/clientpositive/udf_reflect2.q.out Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1448900&r1=1448899&r2=1448900&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Fri Feb 22 05:32:57 2013 @@ -192,6 +192,7 @@ import org.apache.hadoop.hive.ql.udf.gen import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFPrintf; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFReflect; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFReflect2; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSentences; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSize; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSortArray; @@ -441,6 +442,7 @@ public final class FunctionRegistry { // Generic UDFs registerGenericUDF("reflect", GenericUDFReflect.class); + registerGenericUDF("reflect2", GenericUDFReflect2.class); registerGenericUDF("java_method", GenericUDFReflect.class); registerGenericUDF("array", GenericUDFArray.class); Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java?rev=1448900&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java Fri Feb 22 05:32:57 2013 @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.*; + +/** + * common class for reflective UDFs + */ +public abstract class AbstractGenericUDFReflect extends GenericUDF { + + private PrimitiveObjectInspector[] parameterOIs; + private PrimitiveTypeEntry[] parameterTypes; + private Class[] parameterClasses; + + private Object[] parameterJavaValues; + + void setupParameterOIs(ObjectInspector[] arguments, int start) throws UDFArgumentTypeException { + int length = arguments.length - start; + parameterOIs = new PrimitiveObjectInspector[length]; + parameterTypes = new PrimitiveTypeEntry[length]; + parameterClasses = new Class[length]; + parameterJavaValues = new Object[length]; + for (int i = 0; i < length; i++) { + if (arguments[i + start].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(i, + "The parameters of GenericUDFReflect(class,method[,arg1[,arg2]...])" + + " must be primitive (int, double, string, etc)."); + } + parameterOIs[i] = (PrimitiveObjectInspector)arguments[i + start]; + parameterTypes[i] = PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveCategory( + parameterOIs[i].getPrimitiveCategory()); + parameterClasses[i] = parameterTypes[i].primitiveJavaType == null ? + parameterTypes[i].primitiveJavaClass : parameterTypes[i].primitiveJavaType; + } + } + + Object[] setupParameters(DeferredObject[] arguments, int start) throws HiveException { + // Get the parameter values + for (int i = 0; i < parameterOIs.length; i++) { + Object argument = arguments[i + start].get(); + parameterJavaValues[i] = parameterOIs[i].getPrimitiveJavaObject(argument); + } + return parameterJavaValues; + } + + // a(string,int,int) can be matched with methods like + // a(string,int,int), a(string,int,Integer), a(string,Integer,int) and a(string,Integer,Integer) + // and accepts the first one clazz.getMethods() returns + Method findMethod(Class clazz, String name, Class<?> retType, boolean memberOnly) + throws Exception { + for (Method method : clazz.getMethods()) { + if (!method.getName().equals(name) || + (retType != null && !retType.isAssignableFrom(method.getReturnType())) || + (memberOnly && Modifier.isStatic(method.getReturnType().getModifiers())) || + method.getParameterTypes().length != parameterTypes.length) { + continue; + } + // returns first one matches all of the params + boolean match = true; + Class<?>[] types = method.getParameterTypes(); + for (int i = 0; i < parameterTypes.length; i++) { + if (types[i] != parameterTypes[i].primitiveJavaType && + types[i] != parameterTypes[i].primitiveJavaClass && + !types[i].isAssignableFrom(parameterTypes[i].primitiveJavaClass)) { + match = false; + break; + } + } + if (match) { + return method; + } + } + // tried all, back to original code (for error message) + return clazz.getMethod(name, parameterClasses); + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append(functionName()).append('('); + for (int i = 0; i < children.length; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(children[i]); + } + sb.append(')'); + return sb.toString(); + } + + protected abstract String functionName(); +} Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java?rev=1448900&r1=1448899&r2=1448900&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java Fri Feb 22 05:32:57 2013 @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.ql.udf.generic; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; @@ -29,12 +28,9 @@ import org.apache.hadoop.hive.ql.udf.UDF import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.apache.hadoop.util.ReflectionUtils; /** @@ -44,16 +40,14 @@ import org.apache.hadoop.util.Reflection value = "_FUNC_(class,method[,arg1[,arg2..]]) calls method with reflection", extended = "Use this UDF to call Java methods by matching the argument signature\n") @UDFType(deterministic = false) -public class GenericUDFReflect extends GenericUDF { +public class GenericUDFReflect extends AbstractGenericUDFReflect { + + StringObjectInspector inputClassNameOI; + StringObjectInspector inputMethodNameOI; - PrimitiveObjectInspector[] argumentOIs; StringObjectInspector classNameOI; StringObjectInspector methodNameOI; - - PrimitiveTypeEntry[] parameterTypes; - Class[] parameterClasses; - Object[] parameterJavaValues; - + @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { @@ -71,35 +65,16 @@ public class GenericUDFReflect extends G + " should be string."); } } - + inputClassNameOI = (StringObjectInspector) arguments[0]; + inputMethodNameOI = (StringObjectInspector) arguments[1]; + classNameOI = (StringObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(arguments[0]); methodNameOI = (StringObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(arguments[1]); - - parameterTypes = new PrimitiveTypeEntry[arguments.length - 2]; - parameterClasses = new Class[arguments.length - 2]; - for (int i = 2; i < arguments.length; i++) { - if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { - throw new UDFArgumentTypeException(i, - "The parameters of GenericUDFReflect(class,method[,arg1[,arg2]...])" - + " must be primitive (int, double, string, etc)."); - } - PrimitiveCategory category = - ((PrimitiveObjectInspector)arguments[i]).getPrimitiveCategory(); - parameterTypes[i - 2] = - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveCategory(category); - parameterClasses[i - 2] = parameterTypes[i - 2].primitiveJavaType == null ? - parameterTypes[i - 2].primitiveJavaClass : parameterTypes[i - 2].primitiveJavaType; - } - - parameterJavaValues = new Object[arguments.length - 2]; - argumentOIs = new PrimitiveObjectInspector[arguments.length]; - for (int i = 0; i < arguments.length; i++) { - argumentOIs[i] = (PrimitiveObjectInspector)arguments[i]; - } - + setupParameterOIs(arguments, 2); + return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( PrimitiveCategory.STRING); } @@ -109,8 +84,7 @@ public class GenericUDFReflect extends G Method m; Object className; Object methodName; - String result; - + @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { @@ -118,15 +92,14 @@ public class GenericUDFReflect extends G // Skip class loading if the class name didn't change boolean classNameChanged = false; - ObjectInspector newClassNameOI = argumentOIs[0]; Object newClassName = arguments[0].get(); // We compare class name/method name using ObjectInspectorUtils.compare(...), to avoid // any object conversion (which may cause object creation) in most cases, when the class // name/method name is constant Java String, or constant Text (StringWritable). if (className == null || ObjectInspectorUtils.compare(className, classNameOI, newClassName, - newClassNameOI) != 0) { - className = ObjectInspectorUtils.copyToStandardObject(newClassName, newClassNameOI); + inputClassNameOI) != 0) { + className = ObjectInspectorUtils.copyToStandardObject(newClassName, inputClassNameOI); String classNameString = classNameOI.getPrimitiveJavaObject(className); try { c = Class.forName(classNameString); @@ -144,80 +117,32 @@ public class GenericUDFReflect extends G // Try to find the method // Skip method finding if the method name didn't change, and class name didn't change. - ObjectInspector newMethodNameOI = argumentOIs[1]; Object newMethodName = arguments[1].get(); - + if (methodName == null || ObjectInspectorUtils.compare(methodName, methodNameOI, newMethodName, - newMethodNameOI) != 0 || classNameChanged) { - methodName = ObjectInspectorUtils.copyToStandardObject(newMethodName, newMethodNameOI); + inputMethodNameOI) != 0 || classNameChanged) { + methodName = ObjectInspectorUtils.copyToStandardObject(newMethodName, inputMethodNameOI); String methodNameString = methodNameOI.getPrimitiveJavaObject(methodName); try { - m = findMethod(c, methodNameString, parameterTypes, parameterClasses); + m = findMethod(c, methodNameString, String.class, false); } catch (Exception e) { throw new HiveException("UDFReflect getMethod ", e); } } - // Get the parameter values - for (int i = 2; i < arguments.length; i++) { - parameterJavaValues[i - 2] = argumentOIs[i].getPrimitiveJavaObject(arguments[i].get()); - } + Object[] parameterJavaValues = setupParameters(arguments, 2); try { - result = String.valueOf(m.invoke(o, parameterJavaValues)); - return result; - } catch (IllegalArgumentException e1) { - System.err.println("UDFReflect evaluate "+ e1 + " method = " + m + " args = " + - Arrays.asList(parameterJavaValues)); - } catch (IllegalAccessException e1) { - System.err.println("UDFReflect evaluate "+ e1 + " method = " + m + " args = " + - Arrays.asList(parameterJavaValues)); - } catch (InvocationTargetException e1) { - System.err.println("UDFReflect evaluate "+ e1 + " method = " + m + " args = " + + return String.valueOf(m.invoke(o, parameterJavaValues)); + } catch (Exception e1) { + System.err.println("UDFReflect evaluate " + e1 + " method = " + m + " args = " + Arrays.asList(parameterJavaValues)); } return null; } @Override - public String getDisplayString(String[] children) { - StringBuilder sb = new StringBuilder(); - sb.append("reflect("); - for (int i = 0; i < children.length; i++) { - if (i > 0) { - sb.append(','); - } - sb.append(children[i]); - } - sb.append(')'); - return sb.toString(); - } - - // a(string,int,int) can be matched with methods like - // a(string,int,int), a(string,int,Integer), a(string,Integer,int) and a(string,Integer,Integer) - // and accepts the first one clazz.getMethods() returns - private Method findMethod(Class clazz, String name, PrimitiveTypeEntry[] parameterTypes, - Class[] parameterClasses) throws Exception { - for (Method method : clazz.getMethods()) { - if (!method.getName().equals(name) || method.getReturnType() != String.class || - method.getParameterTypes().length != parameterTypes.length) { - continue; - } - // returns first one matches all of the params - boolean match = true; - Class<?>[] types = method.getParameterTypes(); - for (int i = 0; i < parameterTypes.length; i++) { - if (types[i] != parameterTypes[i].primitiveJavaType && - types[i] != parameterTypes[i].primitiveJavaClass) { - match = false; - break; - } - } - if (match) { - return method; - } - } - // tried all, back to original code (for error message) - return clazz.getMethod(name, parameterClasses); + protected String functionName() { + return "reflect"; } } Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java?rev=1448900&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java Fri Feb 22 05:32:57 2013 @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.math.BigDecimal; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.io.ByteWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; + +/** + * A simple generic udf to call java functions via reflection. + */ +@Description(name = "reflect2", + value = "_FUNC_(arg0,method[,arg1[,arg2..]]) calls method of arg0 with reflection", + extended = "Use this UDF to call Java methods by matching the argument signature\n") +@UDFType(deterministic = true) +public class GenericUDFReflect2 extends AbstractGenericUDFReflect { + + PrimitiveObjectInspector targetOI; + PrimitiveObjectInspector returnOI; + Method method; + + transient Writable returnObj; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length < 2) { + throw new UDFArgumentLengthException( + "The function GenericUDFReflect2(arg0,method[,arg1[,arg2]...])" + + " accepts 2 or more arguments."); + } + if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(1, "The target instance should be a primitive type."); + } + targetOI = (PrimitiveObjectInspector) arguments[0]; + + if (!(arguments[1] instanceof StringObjectInspector)) { + throw new UDFArgumentTypeException(1, "The method name should be string type."); + } + if (!(arguments[1] instanceof ConstantObjectInspector)) { + throw new UDFArgumentTypeException(1, "The method name should be a constant."); + } + Text methodName = (Text) ((ConstantObjectInspector)arguments[1]).getWritableConstantValue(); + if (methodName.toString().equals("hashCode") && arguments.length == 2) { + // it's non-deterministic + throw new UDFArgumentTypeException(1, "Use hash() UDF instead of this."); + } + setupParameterOIs(arguments, 2); + + Class<?> targetClass = PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveCategory( + targetOI.getPrimitiveCategory()).primitiveJavaClass; + + try { + method = findMethod(targetClass, methodName.toString(), null, true); + returnOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + getTypeFor(method.getReturnType()).primitiveCategory); + returnObj = (Writable) returnOI.getPrimitiveWritableClass().newInstance(); + } catch (Exception e) { + throw new UDFArgumentException(e); + } + return returnOI; + } + + private PrimitiveObjectInspectorUtils.PrimitiveTypeEntry getTypeFor(Class<?> retType) + throws UDFArgumentException { + PrimitiveObjectInspectorUtils.PrimitiveTypeEntry entry = + PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaType(retType); + if (entry == null) { + entry = PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(retType); + } + if (entry == null) { + throw new UDFArgumentException("Invalid return type " + retType); + } + return entry; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object targetObject = targetOI.getPrimitiveJavaObject(arguments[0].get()); + if (targetObject == null) { + return null; + } + Object result = null; + try { + result = method.invoke(targetObject, setupParameters(arguments, 2)); + } catch (InvocationTargetException e) { + throw new HiveException(e.getCause()); + } catch (Exception e) { + throw new HiveException(e); + } + if (result == null) { + return null; + } + switch (returnOI.getPrimitiveCategory()) { + case VOID: + return null; + case BOOLEAN: + ((BooleanWritable)returnObj).set((Boolean)result); + return returnObj; + case BYTE: + ((ByteWritable)returnObj).set((Byte)result); + return returnObj; + case SHORT: + ((ShortWritable)returnObj).set((Short)result); + return returnObj; + case INT: + ((IntWritable)returnObj).set((Integer)result); + return returnObj; + case LONG: + ((LongWritable)returnObj).set((Long)result); + return returnObj; + case FLOAT: + ((FloatWritable)returnObj).set((Float)result); + return returnObj; + case DOUBLE: + ((DoubleWritable)returnObj).set((Double)result); + return returnObj; + case STRING: + ((Text)returnObj).set((String)result); + return returnObj; + case TIMESTAMP: + ((TimestampWritable)returnObj).set((Timestamp)result); + return returnObj; + case BINARY: + ((BytesWritable)returnObj).set((byte[])result, 0, ((byte[]) result).length); + return returnObj; + case DECIMAL: + ((BigDecimalWritable)returnObj).set((BigDecimal)result); + return returnObj; + } + throw new HiveException("Invalid type " + returnOI.getPrimitiveCategory()); + } + + @Override + protected String functionName() { + return "reflect2"; + } +} Added: hive/trunk/ql/src/test/queries/clientpositive/udf_reflect2.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/udf_reflect2.q?rev=1448900&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/udf_reflect2.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/udf_reflect2.q Fri Feb 22 05:32:57 2013 @@ -0,0 +1,70 @@ +DESCRIBE FUNCTION reflect2; +DESCRIBE FUNCTION EXTENDED reflect2; + +EXPLAIN EXTENDED +SELECT key, + reflect2(key, "byteValue"), + reflect2(key, "shortValue"), + reflect2(key, "intValue"), + reflect2(key, "longValue"), + reflect2(key, "floatValue"), + reflect2(key, "doubleValue"), + reflect2(key, "toString"), + value, + reflect2(value, "concat", "_concat"), + reflect2(value, "contains", "86"), + reflect2(value, "startsWith", "v"), + reflect2(value, "endsWith", "6"), + reflect2(value, "equals", "val_86"), + reflect2(value, "equalsIgnoreCase", "VAL_86"), + reflect2(value, "getBytes"), + reflect2(value, "indexOf", "1"), + reflect2(value, "lastIndexOf", "1"), + reflect2(value, "replace", "val", "VALUE"), + reflect2(value, "substring", 1), + reflect2(value, "substring", 1, 5), + reflect2(value, "toUpperCase"), + reflect2(value, "trim"), + ts, + reflect2(ts, "getYear"), + reflect2(ts, "getMonth"), + reflect2(ts, "getDay"), + reflect2(ts, "getHours"), + reflect2(ts, "getMinutes"), + reflect2(ts, "getSeconds"), + reflect2(ts, "getTime") +FROM (select cast(key as int) key, value, cast('2013-02-15 19:41:20' as timestamp) ts from src) a LIMIT 5; + + +SELECT key, + reflect2(key, "byteValue"), + reflect2(key, "shortValue"), + reflect2(key, "intValue"), + reflect2(key, "longValue"), + reflect2(key, "floatValue"), + reflect2(key, "doubleValue"), + reflect2(key, "toString"), + value, + reflect2(value, "concat", "_concat"), + reflect2(value, "contains", "86"), + reflect2(value, "startsWith", "v"), + reflect2(value, "endsWith", "6"), + reflect2(value, "equals", "val_86"), + reflect2(value, "equalsIgnoreCase", "VAL_86"), + reflect2(value, "getBytes"), + reflect2(value, "indexOf", "1"), + reflect2(value, "lastIndexOf", "1"), + reflect2(value, "replace", "val", "VALUE"), + reflect2(value, "substring", 1), + reflect2(value, "substring", 1, 5), + reflect2(value, "toUpperCase"), + reflect2(value, "trim"), + ts, + reflect2(ts, "getYear"), + reflect2(ts, "getMonth"), + reflect2(ts, "getDay"), + reflect2(ts, "getHours"), + reflect2(ts, "getMinutes"), + reflect2(ts, "getSeconds"), + reflect2(ts, "getTime") +FROM (select cast(key as int) key, value, cast('2013-02-15 19:41:20' as timestamp) ts from src) a LIMIT 5; Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1448900&r1=1448899&r2=1448900&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Fri Feb 22 05:32:57 2013 @@ -120,6 +120,7 @@ printf radians rand reflect +reflect2 regexp regexp_extract regexp_replace Added: hive/trunk/ql/src/test/results/clientpositive/udf_reflect2.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udf_reflect2.q.out?rev=1448900&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/udf_reflect2.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/udf_reflect2.q.out Fri Feb 22 05:32:57 2013 @@ -0,0 +1,316 @@ +PREHOOK: query: DESCRIBE FUNCTION reflect2 +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION reflect2 +POSTHOOK: type: DESCFUNCTION +reflect2(arg0,method[,arg1[,arg2..]]) calls method of arg0 with reflection +PREHOOK: query: DESCRIBE FUNCTION EXTENDED reflect2 +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED reflect2 +POSTHOOK: type: DESCFUNCTION +reflect2(arg0,method[,arg1[,arg2..]]) calls method of arg0 with reflection +Use this UDF to call Java methods by matching the argument signature + +PREHOOK: query: EXPLAIN EXTENDED +SELECT key, + reflect2(key, "byteValue"), + reflect2(key, "shortValue"), + reflect2(key, "intValue"), + reflect2(key, "longValue"), + reflect2(key, "floatValue"), + reflect2(key, "doubleValue"), + reflect2(key, "toString"), + value, + reflect2(value, "concat", "_concat"), + reflect2(value, "contains", "86"), + reflect2(value, "startsWith", "v"), + reflect2(value, "endsWith", "6"), + reflect2(value, "equals", "val_86"), + reflect2(value, "equalsIgnoreCase", "VAL_86"), + reflect2(value, "getBytes"), + reflect2(value, "indexOf", "1"), + reflect2(value, "lastIndexOf", "1"), + reflect2(value, "replace", "val", "VALUE"), + reflect2(value, "substring", 1), + reflect2(value, "substring", 1, 5), + reflect2(value, "toUpperCase"), + reflect2(value, "trim"), + ts, + reflect2(ts, "getYear"), + reflect2(ts, "getMonth"), + reflect2(ts, "getDay"), + reflect2(ts, "getHours"), + reflect2(ts, "getMinutes"), + reflect2(ts, "getSeconds"), + reflect2(ts, "getTime") +FROM (select cast(key as int) key, value, cast('2013-02-15 19:41:20' as timestamp) ts from src) a LIMIT 5 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +SELECT key, + reflect2(key, "byteValue"), + reflect2(key, "shortValue"), + reflect2(key, "intValue"), + reflect2(key, "longValue"), + reflect2(key, "floatValue"), + reflect2(key, "doubleValue"), + reflect2(key, "toString"), + value, + reflect2(value, "concat", "_concat"), + reflect2(value, "contains", "86"), + reflect2(value, "startsWith", "v"), + reflect2(value, "endsWith", "6"), + reflect2(value, "equals", "val_86"), + reflect2(value, "equalsIgnoreCase", "VAL_86"), + reflect2(value, "getBytes"), + reflect2(value, "indexOf", "1"), + reflect2(value, "lastIndexOf", "1"), + reflect2(value, "replace", "val", "VALUE"), + reflect2(value, "substring", 1), + reflect2(value, "substring", 1, 5), + reflect2(value, "toUpperCase"), + reflect2(value, "trim"), + ts, + reflect2(ts, "getYear"), + reflect2(ts, "getMonth"), + reflect2(ts, "getDay"), + reflect2(ts, "getHours"), + reflect2(ts, "getMinutes"), + reflect2(ts, "getSeconds"), + reflect2(ts, "getTime") +FROM (select cast(key as int) key, value, cast('2013-02-15 19:41:20' as timestamp) ts from src) a LIMIT 5 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION TOK_INT (TOK_TABLE_OR_COL key)) key) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION TOK_TIMESTAMP '2013-02-15 19:41:20') ts)))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL key) "byteValue")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL key) "shortValue")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL key) "intValue")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL key) "longValue")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL key) "floatValue")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL key) "doubleValue")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL key) "toString")) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (T OK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "concat" "_concat")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "contains" "86")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "startsWith" "v")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "endsWith" "6")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "equals" "val_86")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "equalsIgnoreCase" "VAL_86")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "getBytes")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "indexOf" "1")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "lastIndexOf" "1")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "replace" "val" "VALUE")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "substring" 1)) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "substring" 1 5)) (TOK_SELEXPR (TOK_FUNCTIO N reflect2 (TOK_TABLE_OR_COL value) "toUpperCase")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL value) "trim")) (TOK_SELEXPR (TOK_TABLE_OR_COL ts)) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL ts) "getYear")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL ts) "getMonth")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL ts) "getDay")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL ts) "getHours")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL ts) "getMinutes")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL ts) "getSeconds")) (TOK_SELEXPR (TOK_FUNCTION reflect2 (TOK_TABLE_OR_COL ts) "getTime"))) (TOK_LIMIT 5))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + a:src + TableScan + alias: src + GatherStats: false + Select Operator + expressions: + expr: UDFToInteger(key) + type: int + expr: value + type: string + expr: CAST( '2013-02-15 19:41:20' AS TIMESTAMP) + type: timestamp + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: int + expr: reflect2(_col0,'byteValue') + type: tinyint + expr: reflect2(_col0,'shortValue') + type: smallint + expr: reflect2(_col0,'intValue') + type: int + expr: reflect2(_col0,'longValue') + type: bigint + expr: reflect2(_col0,'floatValue') + type: float + expr: reflect2(_col0,'doubleValue') + type: double + expr: reflect2(_col0,'toString') + type: string + expr: _col1 + type: string + expr: reflect2(_col1,'concat','_concat') + type: string + expr: reflect2(_col1,'contains','86') + type: boolean + expr: reflect2(_col1,'startsWith','v') + type: boolean + expr: reflect2(_col1,'endsWith','6') + type: boolean + expr: reflect2(_col1,'equals','val_86') + type: boolean + expr: reflect2(_col1,'equalsIgnoreCase','VAL_86') + type: boolean + expr: reflect2(_col1,'getBytes') + type: binary + expr: reflect2(_col1,'indexOf','1') + type: int + expr: reflect2(_col1,'lastIndexOf','1') + type: int + expr: reflect2(_col1,'replace','val','VALUE') + type: string + expr: reflect2(_col1,'substring',1) + type: string + expr: reflect2(_col1,'substring',1,5) + type: string + expr: reflect2(_col1,'toUpperCase') + type: string + expr: reflect2(_col1,'trim') + type: string + expr: _col2 + type: timestamp + expr: reflect2(_col2,'getYear') + type: int + expr: reflect2(_col2,'getMonth') + type: int + expr: reflect2(_col2,'getDay') + type: int + expr: reflect2(_col2,'getHours') + type: int + expr: reflect2(_col2,'getMinutes') + type: int + expr: reflect2(_col2,'getSeconds') + type: int + expr: reflect2(_col2,'getTime') + type: bigint + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16,_col17,_col18,_col19,_col20,_col21,_col22,_col23,_col24,_col25,_col26,_col27,_col28,_col29,_col30 + columns.types int:tinyint:smallint:int:bigint:float:double:string:string:string:boolean:boolean:boolean:boolean:boolean:binary:int:int:string:string:string:string:string:timestamp:int:int:int:int:int:int:bigint + escape.delim \ + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numPartitions 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numPartitions 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [a:src] + + Stage: Stage-0 + Fetch Operator + limit: 5 + + +PREHOOK: query: SELECT key, + reflect2(key, "byteValue"), + reflect2(key, "shortValue"), + reflect2(key, "intValue"), + reflect2(key, "longValue"), + reflect2(key, "floatValue"), + reflect2(key, "doubleValue"), + reflect2(key, "toString"), + value, + reflect2(value, "concat", "_concat"), + reflect2(value, "contains", "86"), + reflect2(value, "startsWith", "v"), + reflect2(value, "endsWith", "6"), + reflect2(value, "equals", "val_86"), + reflect2(value, "equalsIgnoreCase", "VAL_86"), + reflect2(value, "getBytes"), + reflect2(value, "indexOf", "1"), + reflect2(value, "lastIndexOf", "1"), + reflect2(value, "replace", "val", "VALUE"), + reflect2(value, "substring", 1), + reflect2(value, "substring", 1, 5), + reflect2(value, "toUpperCase"), + reflect2(value, "trim"), + ts, + reflect2(ts, "getYear"), + reflect2(ts, "getMonth"), + reflect2(ts, "getDay"), + reflect2(ts, "getHours"), + reflect2(ts, "getMinutes"), + reflect2(ts, "getSeconds"), + reflect2(ts, "getTime") +FROM (select cast(key as int) key, value, cast('2013-02-15 19:41:20' as timestamp) ts from src) a LIMIT 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, + reflect2(key, "byteValue"), + reflect2(key, "shortValue"), + reflect2(key, "intValue"), + reflect2(key, "longValue"), + reflect2(key, "floatValue"), + reflect2(key, "doubleValue"), + reflect2(key, "toString"), + value, + reflect2(value, "concat", "_concat"), + reflect2(value, "contains", "86"), + reflect2(value, "startsWith", "v"), + reflect2(value, "endsWith", "6"), + reflect2(value, "equals", "val_86"), + reflect2(value, "equalsIgnoreCase", "VAL_86"), + reflect2(value, "getBytes"), + reflect2(value, "indexOf", "1"), + reflect2(value, "lastIndexOf", "1"), + reflect2(value, "replace", "val", "VALUE"), + reflect2(value, "substring", 1), + reflect2(value, "substring", 1, 5), + reflect2(value, "toUpperCase"), + reflect2(value, "trim"), + ts, + reflect2(ts, "getYear"), + reflect2(ts, "getMonth"), + reflect2(ts, "getDay"), + reflect2(ts, "getHours"), + reflect2(ts, "getMinutes"), + reflect2(ts, "getSeconds"), + reflect2(ts, "getTime") +FROM (select cast(key as int) key, value, cast('2013-02-15 19:41:20' as timestamp) ts from src) a LIMIT 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +238 -18 238 238 238 238.0 238.0 238 val_238 val_238_concat false true false false false val_238 -1 -1 VALUE_238 al_238 al_2 VAL_238 val_238 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000 +86 86 86 86 86 86.0 86.0 86 val_86 val_86_concat true true true true true val_86 -1 -1 VALUE_86 al_86 al_8 VAL_86 val_86 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000 +311 55 311 311 311 311.0 311.0 311 val_311 val_311_concat false true false false false val_311 5 6 VALUE_311 al_311 al_3 VAL_311 val_311 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000 +27 27 27 27 27 27.0 27.0 27 val_27 val_27_concat false true false false false val_27 -1 -1 VALUE_27 al_27 al_2 VAL_27 val_27 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000 +165 -91 165 165 165 165.0 165.0 165 val_165 val_165_concat false true false false false val_165 4 4 VALUE_165 al_165 al_1 VAL_165 val_165 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000