Repository: hive Updated Branches: refs/heads/master a51ae9c8f -> 527d13b08
HIVE-17530: ClassCastException when converting uniontype (Anthony Hsu via Carl Steinbach) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/527d13b0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/527d13b0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/527d13b0 Branch: refs/heads/master Commit: 527d13b080ed970b527c7cb2bfd98fd6fd17f5d5 Parents: a51ae9c Author: Carl Steinbach <c...@apache.org> Authored: Mon Sep 18 11:47:29 2017 -0700 Committer: Carl Steinbach <c...@apache.org> Committed: Mon Sep 18 11:47:29 2017 -0700 ---------------------------------------------------------------------- .../ql/exec/vector/VectorDeserializeRow.java | 4 +- .../orc_avro_partition_uniontype.q | 9 +++++ .../orc_avro_partition_uniontype.q.out | 40 ++++++++++++++++++++ .../ObjectInspectorConverters.java | 4 +- .../SettableUnionObjectInspector.java | 6 +-- .../StandardUnionObjectInspector.java | 16 +++++--- .../TestObjectInspectorConverters.java | 30 ++++++++------- 7 files changed, 83 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/527d13b0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java index 2ad06fc..3826182 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java @@ -1251,8 +1251,8 @@ public final class VectorDeserializeRow<T extends DeserializeRead> { final Object union = unionOI.create(); final int tag = deserializeRead.currentInt; - unionOI.addField(union, new StandardUnion((byte) tag, - convertComplexFieldRowColumn(unionColumnVector.fields[tag], batchIndex, fields[tag]))); + unionOI.setFieldAndTag(union, new StandardUnion((byte) tag, + convertComplexFieldRowColumn(unionColumnVector.fields[tag], batchIndex, fields[tag])), (byte) tag); deserializeRead.finishComplexVariableFieldsType(); return union; } http://git-wip-us.apache.org/repos/asf/hive/blob/527d13b0/ql/src/test/queries/clientpositive/orc_avro_partition_uniontype.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/orc_avro_partition_uniontype.q b/ql/src/test/queries/clientpositive/orc_avro_partition_uniontype.q new file mode 100644 index 0000000..47ac526 --- /dev/null +++ b/ql/src/test/queries/clientpositive/orc_avro_partition_uniontype.q @@ -0,0 +1,9 @@ +SET hive.exec.schema.evolution = false; + +CREATE TABLE avro_orc_partitioned_uniontype (a uniontype<boolean, string>) PARTITIONED BY (b int) STORED AS ORC; + +INSERT INTO avro_orc_partitioned_uniontype PARTITION (b=1) SELECT create_union(1, true, value) FROM src LIMIT 5; + +ALTER TABLE avro_orc_partitioned_uniontype SET FILEFORMAT AVRO; + +SELECT * FROM avro_orc_partitioned_uniontype; http://git-wip-us.apache.org/repos/asf/hive/blob/527d13b0/ql/src/test/results/clientpositive/orc_avro_partition_uniontype.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/orc_avro_partition_uniontype.q.out b/ql/src/test/results/clientpositive/orc_avro_partition_uniontype.q.out new file mode 100644 index 0000000..da8d9cc --- /dev/null +++ b/ql/src/test/results/clientpositive/orc_avro_partition_uniontype.q.out @@ -0,0 +1,40 @@ +PREHOOK: query: CREATE TABLE avro_orc_partitioned_uniontype (a uniontype<boolean, string>) PARTITIONED BY (b int) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_orc_partitioned_uniontype +POSTHOOK: query: CREATE TABLE avro_orc_partitioned_uniontype (a uniontype<boolean, string>) PARTITIONED BY (b int) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_orc_partitioned_uniontype +PREHOOK: query: INSERT INTO avro_orc_partitioned_uniontype PARTITION (b=1) SELECT create_union(1, true, value) FROM src LIMIT 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@avro_orc_partitioned_uniontype@b=1 +POSTHOOK: query: INSERT INTO avro_orc_partitioned_uniontype PARTITION (b=1) SELECT create_union(1, true, value) FROM src LIMIT 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@avro_orc_partitioned_uniontype@b=1 +POSTHOOK: Lineage: avro_orc_partitioned_uniontype PARTITION(b=1).a EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE avro_orc_partitioned_uniontype SET FILEFORMAT AVRO +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@avro_orc_partitioned_uniontype +PREHOOK: Output: default@avro_orc_partitioned_uniontype +POSTHOOK: query: ALTER TABLE avro_orc_partitioned_uniontype SET FILEFORMAT AVRO +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@avro_orc_partitioned_uniontype +POSTHOOK: Output: default@avro_orc_partitioned_uniontype +PREHOOK: query: SELECT * FROM avro_orc_partitioned_uniontype +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_orc_partitioned_uniontype +PREHOOK: Input: default@avro_orc_partitioned_uniontype@b=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM avro_orc_partitioned_uniontype +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_orc_partitioned_uniontype +POSTHOOK: Input: default@avro_orc_partitioned_uniontype@b=1 +#### A masked pattern was here #### +{1:"val_165"} 1 +{1:"val_27"} 1 +{1:"val_311"} 1 +{1:"val_86"} 1 +{1:"val_238"} 1 http://git-wip-us.apache.org/repos/asf/hive/blob/527d13b0/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java index cdf5ed4..faa0a1f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java @@ -471,7 +471,7 @@ public final class ObjectInspectorConverters { } Object inputFieldValue = inputOI.getField(input); - Object inputFieldTag = inputOI.getTag(input); + byte inputFieldTag = inputOI.getTag(input); Object outputFieldValue = null; int inputFieldTagIndex = ((Byte)inputFieldTag).intValue(); @@ -480,7 +480,7 @@ public final class ObjectInspectorConverters { outputFieldValue = fieldConverters.get(inputFieldTagIndex).convert(inputFieldValue); } - outputOI.addField(output, outputFieldValue); + outputOI.setFieldAndTag(output, outputFieldValue, inputFieldTag); return output; } http://git-wip-us.apache.org/repos/asf/hive/blob/527d13b0/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java index 564d8d6..e48ba03 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java @@ -26,9 +26,9 @@ package org.apache.hadoop.hive.serde2.objectinspector; public abstract class SettableUnionObjectInspector implements UnionObjectInspector { - /* Create an empty object */ + /* Creates an empty union object. */ public abstract Object create(); - /* Add field to the object */ - public abstract Object addField(Object union, Object field); + /* Sets the field and tag in the union. Returns the union. */ + public abstract Object setFieldAndTag(Object union, Object field, byte tag); } http://git-wip-us.apache.org/repos/asf/hive/blob/527d13b0/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java index 7b28682..d948965 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java @@ -144,15 +144,19 @@ public class StandardUnionObjectInspector extends SettableUnionObjectInspector { @Override public Object create() { - ArrayList<Object> a = new ArrayList<Object>(); - return a; + return new StandardUnion(); } @Override - public Object addField(Object union, Object field) { - ArrayList<Object> a = (ArrayList<Object>) union; - a.add(field); - return a; + public Object setFieldAndTag(Object union, Object field, byte tag) { + StandardUnion unionObject = (StandardUnion) union; + unionObject.setObject(field); + if (field == null) { + unionObject.setTag((byte) -1); + } else { + unionObject.setTag(tag); + } + return unionObject; } } http://git-wip-us.apache.org/repos/asf/hive/blob/527d13b0/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java ---------------------------------------------------------------------- diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java index 2e1bb22..219a6cd 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java @@ -30,8 +30,8 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -295,27 +295,30 @@ public class TestObjectInspectorConverters extends TestCase { Converter unionConverter0 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2)); - Object convertedObject0 = unionConverter0.convert(new StandardUnionObjectInspector.StandardUnion((byte)0, 1)); - List<String> expectedObject0 = new ArrayList<String>(); - expectedObject0.add("1"); + Object convertedObject0 = unionConverter0.convert(new StandardUnion((byte)0, 1)); + StandardUnion expectedObject0 = new StandardUnion(); + expectedObject0.setTag((byte) 0); + expectedObject0.setObject("1"); assertEquals(expectedObject0, convertedObject0); Converter unionConverter1 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2)); - Object convertedObject1 = unionConverter1.convert(new StandardUnionObjectInspector.StandardUnion((byte)1, "1")); - List<Integer> expectedObject1 = new ArrayList<Integer>(); - expectedObject1.add(1); + Object convertedObject1 = unionConverter1.convert(new StandardUnion((byte)1, "1")); + StandardUnion expectedObject1 = new StandardUnion(); + expectedObject1.setTag((byte) 1); + expectedObject1.setObject(1); assertEquals(expectedObject1, convertedObject1); Converter unionConverter2 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2)); - Object convertedObject2 = unionConverter2.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true)); - List<Boolean> expectedObject2 = new ArrayList<Boolean>(); - expectedObject2.add(true); + Object convertedObject2 = unionConverter2.convert(new StandardUnion((byte)2, true)); + StandardUnion expectedObject2 = new StandardUnion(); + expectedObject2.setTag((byte) 2); + expectedObject2.setObject(true); assertEquals(expectedObject2, convertedObject2); @@ -344,9 +347,10 @@ public class TestObjectInspectorConverters extends TestCase { Converter unionConverterExtra = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra2)); - Object convertedObjectExtra = unionConverterExtra.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true)); - List<Object> expectedObjectExtra = new ArrayList<Object>(); - expectedObjectExtra.add(null); + Object convertedObjectExtra = unionConverterExtra.convert(new StandardUnion((byte)2, true)); + StandardUnion expectedObjectExtra = new StandardUnion(); + expectedObjectExtra.setTag((byte) -1); + expectedObjectExtra.setObject(null); assertEquals(expectedObjectExtra, convertedObjectExtra); // we should get back null
