Repository: nifi Updated Branches: refs/heads/master aa196bc01 -> 090e74872
NIFI-4647: Fix support for strings in unions for ConvertAvroToORC Signed-off-by: Pierre Villard <[email protected]> This closes #2644. Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/090e7487 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/090e7487 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/090e7487 Branch: refs/heads/master Commit: 090e748726b319a96ec941f51299d1436f5f5136 Parents: aa196bc Author: Matthew Burgess <[email protected]> Authored: Wed Apr 18 14:49:16 2018 -0400 Committer: Pierre Villard <[email protected]> Committed: Thu Apr 19 09:49:32 2018 +0200 ---------------------------------------------------------------------- .../hadoop/hive/ql/io/orc/NiFiOrcUtils.java | 8 +++++++- .../apache/nifi/util/orc/TestNiFiOrcUtils.java | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/090e7487/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java index c9624b6..687073e 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java @@ -67,9 +67,15 @@ public class NiFiOrcUtils { if (o != null) { if (typeInfo instanceof UnionTypeInfo) { OrcUnion union = new OrcUnion(); + // Avro uses Utf8 and GenericData.EnumSymbol objects instead of Strings. This is handled in other places in the method, but here + // we need to determine the union types from the objects, so choose String.class if the object is one of those Avro classes + Class clazzToCompareTo = o.getClass(); + if (o instanceof org.apache.avro.util.Utf8 || o instanceof GenericData.EnumSymbol) { + clazzToCompareTo = String.class; + } // Need to find which of the union types correspond to the primitive object TypeInfo objectTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector( - ObjectInspectorFactory.getReflectionObjectInspector(o.getClass(), ObjectInspectorFactory.ObjectInspectorOptions.JAVA)); + ObjectInspectorFactory.getReflectionObjectInspector(clazzToCompareTo, ObjectInspectorFactory.ObjectInspectorOptions.JAVA)); List<TypeInfo> unionTypeInfos = ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos(); int index = 0; http://git-wip-us.apache.org/repos/asf/nifi/blob/090e7487/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java index 342aed9..47ee3a5 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/util/orc/TestNiFiOrcUtils.java @@ -20,7 +20,9 @@ package org.apache.nifi.util.orc; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericData; +import org.apache.avro.util.Utf8; import org.apache.hadoop.hive.ql.io.orc.NiFiOrcUtils; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -267,6 +269,23 @@ public class TestNiFiOrcUtils { + " STORED AS ORC", ddl); } + @Test + public void test_convertToORCObject() { + Schema schema = SchemaBuilder.enumeration("myEnum").symbols("x","y","z"); + List<Object> objects = Arrays.asList(new Utf8("Hello"), new GenericData.EnumSymbol(schema, "x")); + objects.forEach((avroObject) -> { + Object o = NiFiOrcUtils.convertToORCObject(TypeInfoUtils.getTypeInfoFromTypeString("uniontype<bigint,string>"), avroObject); + assertTrue(o instanceof UnionObject); + UnionObject uo = (UnionObject) o; + assertTrue(uo.getObject() instanceof Text); + }); + } + + @Test(expected = IllegalArgumentException.class) + public void test_convertToORCObjectBadUnion() { + NiFiOrcUtils.convertToORCObject(TypeInfoUtils.getTypeInfoFromTypeString("uniontype<bigint,long>"), "Hello"); + } + ////////////////// // Helper methods
