This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 1cafaef HIVE-25738: NullIf doesn't support complex types (#2816) (Zoltan Haindrich reviewed by Zhihua Deng and Stamatis Zampetakis) 1cafaef is described below commit 1cafaef0456fd2714c20f57b7c81a4968f425d44 Author: Zoltan Haindrich <k...@rxd.hu> AuthorDate: Wed Dec 1 17:25:57 2021 +0100 HIVE-25738: NullIf doesn't support complex types (#2816) (Zoltan Haindrich reviewed by Zhihua Deng and Stamatis Zampetakis) --- .../hive/ql/udf/generic/GenericUDFNullif.java | 24 ++++++---- .../hive/ql/udf/generic/TestGenericUDFNullif.java | 47 +++++++++++++++++++ ql/src/test/queries/clientnegative/nullif_union.q | 3 ++ ql/src/test/queries/clientpositive/udf_nullif.q | 9 ++++ .../test/results/clientnegative/nullif_union.q.out | 1 + .../results/clientpositive/llap/udf_nullif.q.out | 54 ++++++++++++++++++++++ 6 files changed, 130 insertions(+), 8 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNullif.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNullif.java index a47882e..b99efa1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNullif.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNullif.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; @@ -49,6 +50,17 @@ public class GenericUDFNullif extends GenericUDF { returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true); returnOIResolver.update(arguments[0]); + switch (arguments[0].getCategory()) { + case LIST: + case MAP: + case STRUCT: + case PRIMITIVE: + break; + case UNION: + default: + throw new UDFArgumentTypeException(0, "Unsupported Argument type category: " + arguments[0].getCategory()); + } + boolean isPrimitive = (arguments[0] instanceof PrimitiveObjectInspector); if (isPrimitive) { @@ -86,17 +98,13 @@ public class GenericUDFNullif extends GenericUDF { public Object evaluate(DeferredObject[] arguments) throws HiveException { Object arg0 = arguments[0].get(); Object arg1 = arguments[1].get(); - Object value0 = null; - if (arg0 != null) { - value0 = returnOIResolver.convertIfNecessary(arg0, argumentOIs[0], false); - } + Object value0 = returnOIResolver.convertIfNecessary(arg0, argumentOIs[0], false); if (arg0 == null || arg1 == null) { return value0; } - PrimitiveObjectInspector compareOI = (PrimitiveObjectInspector) returnOIResolver.get(); - if (PrimitiveObjectInspectorUtils.comparePrimitiveObjects( - value0, compareOI, - returnOIResolver.convertIfNecessary(arg1, argumentOIs[1], false), compareOI)) { + Object value1 = returnOIResolver.convertIfNecessary(arg1, argumentOIs[1], false); + ObjectInspector compareOI = returnOIResolver.get(); + if (ObjectInspectorUtils.compare(value0, compareOI, value1, compareOI) == 0) { return null; } return value0; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNullif.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNullif.java index 281b0d5..2ff4408 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNullif.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNullif.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.udf.generic; +import static java.util.Arrays.asList; + import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; @@ -27,7 +29,9 @@ import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.lazy.LazyInteger; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.DoubleWritable; @@ -149,4 +153,47 @@ public class TestGenericUDFNullif { Assert.assertEquals(TypeInfoFactory.intTypeInfo, oi.getTypeInfo()); Assert.assertEquals(null, udf.evaluate(args)); } + + @Test + public void testArrayNull() throws Exception { + GenericUDFNullif udf = new GenericUDFNullif(); + + ObjectInspector[] inputOIs = { + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector), + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector) }; + + Object i1 = asList(new Text("aa"), new Text("dd"), new Text("cc"), new Text("bb")); + Object i2 = asList(new Text("aa"), new Text("dd"), new Text("cc"), new Text("bb")); + + DeferredObject[] args = { new DeferredJavaObject(i1), new DeferredJavaObject(i2) }; + + StandardListObjectInspector oi = (StandardListObjectInspector) udf.initialize(inputOIs); + Assert.assertEquals("array<string>", oi.getTypeName()); + Assert.assertEquals(null, udf.evaluate(args)); + + } + + @Test + public void testArray1() throws Exception { + GenericUDFNullif udf = new GenericUDFNullif(); + + ObjectInspector[] inputOIs = { + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector), + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector) }; + + Object i1 = asList(new Text("aa"), new Text("dd"), new Text("cc"), new Text("bb")); + Object i2 = asList(new Text("aa"), new Text("dd"), new Text("cc"), new Text("xx")); + + DeferredObject[] args = { new DeferredJavaObject(i1), new DeferredJavaObject(i2) }; + + StandardListObjectInspector oi = (StandardListObjectInspector) udf.initialize(inputOIs); + Assert.assertEquals("array<string>", oi.getTypeName()); + Assert.assertEquals(i1, udf.evaluate(args)); + + } + } diff --git a/ql/src/test/queries/clientnegative/nullif_union.q b/ql/src/test/queries/clientnegative/nullif_union.q new file mode 100644 index 0000000..ca59fcb --- /dev/null +++ b/ql/src/test/queries/clientnegative/nullif_union.q @@ -0,0 +1,3 @@ +-- union is not supported +SELECT NULLIF(create_union(0,1,2),create_union(0,1,3)) + diff --git a/ql/src/test/queries/clientpositive/udf_nullif.q b/ql/src/test/queries/clientpositive/udf_nullif.q index 00dfa23..3479426 100644 --- a/ql/src/test/queries/clientpositive/udf_nullif.q +++ b/ql/src/test/queries/clientpositive/udf_nullif.q @@ -27,3 +27,12 @@ select nullif(a,b), nullif(b,c), nullif(c,d), nullif(d,a) from t0; + +SELECT assert_true(NULLIF(array(1,2,3),array(1,2,3)) is null); +SELECT assert_true(NULLIF(array(1,2,3),array(3,2,1)) is not null); + +SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 1)) is null); +SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 2)) is not null); + +SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',2)) is null); +SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',3)) is not null); diff --git a/ql/src/test/results/clientnegative/nullif_union.q.out b/ql/src/test/results/clientnegative/nullif_union.q.out new file mode 100644 index 0000000..8d7bc60 --- /dev/null +++ b/ql/src/test/results/clientnegative/nullif_union.q.out @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10016]: Line 2:14 Argument type mismatch '2': Unsupported Argument type category: UNION diff --git a/ql/src/test/results/clientpositive/llap/udf_nullif.q.out b/ql/src/test/results/clientpositive/llap/udf_nullif.q.out index 492b412..916007a 100644 --- a/ql/src/test/results/clientpositive/llap/udf_nullif.q.out +++ b/ql/src/test/results/clientpositive/llap/udf_nullif.q.out @@ -213,3 +213,57 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@t0 #### A masked pattern was here #### 1 2 3.1 4.1 +PREHOOK: query: SELECT assert_true(NULLIF(array(1,2,3),array(1,2,3)) is null) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT assert_true(NULLIF(array(1,2,3),array(1,2,3)) is null) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT assert_true(NULLIF(array(1,2,3),array(3,2,1)) is not null) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT assert_true(NULLIF(array(1,2,3),array(3,2,1)) is not null) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 1)) is null) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 1)) is null) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 2)) is not null) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 2)) is not null) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',2)) is null) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',2)) is null) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',3)) is not null) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',3)) is not null) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +NULL