Author: cws Date: Fri Oct 19 00:23:11 2012 New Revision: 1399935 URL: http://svn.apache.org/viewvc?rev=1399935&view=rev Log: HIVE-3525. Avro Maps with Nullable Values fail with NPE (Sean Busbey via cws)
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java?rev=1399935&r1=1399934&r2=1399935&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java Fri Oct 19 00:23:11 2012 @@ -43,7 +43,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Hashtable; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -246,7 +246,7 @@ class AvroDeserializer { throws AvroSerdeException { // Avro only allows maps with Strings for keys, so we only have to worry // about deserializing the values - Map<String, Object> map = new Hashtable<String, Object>(); + Map<String, Object> map = new HashMap<String, Object>(); Map<Utf8, Object> mapDatum = (Map)datum; Schema valueSchema = mapSchema.getValueType(); TypeInfo valueTypeInfo = columnType.getMapValueTypeInfo(); Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java?rev=1399935&r1=1399934&r2=1399935&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java Fri Oct 19 00:23:11 2012 @@ -39,7 +39,7 @@ import org.apache.hadoop.io.Writable; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Hashtable; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -229,7 +229,7 @@ class AvroSerializer { Map<?,?> map = fieldOI.getMap(structFieldData); Schema valueType = schema.getValueType(); - Map<Object, Object> deserialized = new Hashtable<Object, Object>(fieldOI.getMapSize(structFieldData)); + Map<Object, Object> deserialized = new HashMap<Object, Object>(fieldOI.getMapSize(structFieldData)); for (Map.Entry<?, ?> entry : map.entrySet()) { deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector, entry.getKey(), null), // This works, but is a bit fragile. Construct a single String schema? Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java?rev=1399935&r1=1399934&r2=1399935&view=diff ============================================================================== --- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java (original) +++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java Fri Oct 19 00:23:11 2012 @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.Hashtable; import java.util.List; import java.util.Map; +import java.util.HashMap; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -404,6 +405,57 @@ public class TestAvroDeserializer { verifyNullableType(record, s, null); } + @Test + public void canDeserializeMapWithNullablePrimitiveValues() throws SerDeException, IOException { + Schema s = Schema.parse(TestAvroObjectInspectorGenerator.MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA); + GenericData.Record record = new GenericData.Record(s); + + Map<String, Long> m = new HashMap<String, Long>(); + m.put("one", 1l); + m.put("two", 2l); + m.put("three", 3l); + m.put("mu", null); + + record.put("aMap", m); + assertTrue(GENERIC_DATA.validate(s, record)); + System.out.println("record = " + record); + + AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record); + + AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s); + + AvroDeserializer de = new AvroDeserializer(); + + ArrayList<Object> row = (ArrayList<Object>)de.deserialize(aoig.getColumnNames(), + aoig.getColumnTypes(), garw, s); + assertEquals(1, row.size()); + Object theMapObject = row.get(0); + assertTrue(theMapObject instanceof Map); + Map theMap = (Map)theMapObject; + + // Verify the raw object that's been created + assertEquals(1l, theMap.get("one")); + assertEquals(2l, theMap.get("two")); + assertEquals(3l, theMap.get("three")); + assertTrue(theMap.containsKey("mu")); + assertEquals(null, theMap.get("mu")); + + // Verify that the provided object inspector can pull out these same values + StandardStructObjectInspector oi = + (StandardStructObjectInspector)aoig.getObjectInspector(); + + List<Object> z = oi.getStructFieldsDataAsList(row); + assertEquals(1, z.size()); + StructField fieldRef = oi.getStructFieldRef("amap"); + + Map theMap2 = (Map)oi.getStructFieldData(row, fieldRef); + assertEquals(1l, theMap2.get("one")); + assertEquals(2l, theMap2.get("two")); + assertEquals(3l, theMap2.get("three")); + assertTrue(theMap2.containsKey("mu")); + assertEquals(null, theMap2.get("mu")); + } + private void verifyNullableType(GenericData.Record record, Schema s, String expected) throws SerDeException, IOException { assertTrue(GENERIC_DATA.validate(s, record)); Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java?rev=1399935&r1=1399934&r2=1399935&view=diff ============================================================================== --- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java (original) +++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java Fri Oct 19 00:23:11 2012 @@ -142,6 +142,18 @@ public class TestAvroObjectInspectorGene " {\"name\":\"nullableString\", \"type\":[\"null\", \"string\"]}\n" + " ]\n" + "}"; + public static final String MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA = "{\n" + + " \"namespace\": \"testing\",\n" + + " \"name\": \"mapWithNullableUnionTest\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aMap\",\n" + + " \"type\":{\"type\":\"map\",\n" + + " \"values\":[\"null\",\"long\"]}\n" + + "\t}\n" + + " ]\n" + + "}"; public static final String BYTES_SCHEMA = "{\n" + " \"type\": \"record\", \n" + " \"name\": \"bytesTest\",\n" + @@ -325,10 +337,19 @@ public class TestAvroObjectInspectorGene public void canHandleMapsWithPrimitiveValueTypes() throws SerDeException { Schema s = Schema.parse(MAP_WITH_PRIMITIVE_VALUE_TYPE); AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s); - + verifyMap(aoig, "aMap"); + } + + /** + * Check a given AvroObjectInspectorGenerator to verify that it matches our test + * schema's expected map. + * @param aoig should already have been intitialized, may not be null + * @param fieldName name of the contianed column, will always fail if null. + */ + private void verifyMap(final AvroObjectInspectorGenerator aoig, final String fieldName) { // Column names assertEquals(1, aoig.getColumnNames().size()); - assertEquals("aMap", aoig.getColumnNames().get(0)); + assertEquals(fieldName, aoig.getColumnNames().get(0)); // Column types assertEquals(1, aoig.getColumnTypes().size()); @@ -483,6 +504,13 @@ public class TestAvroObjectInspectorGene assertEquals(PrimitiveObjectInspector.PrimitiveCategory.STRING, pti.getPrimitiveCategory()); } + @Test // That Union[T, NULL] is converted to just T, within a Map + public void convertsMapsWithNullablePrimitiveTypes() throws SerDeException { + Schema s = Schema.parse(MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA); + AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s); + verifyMap(aoig, "aMap"); + } + @Test public void objectInspectorsAreCached() throws SerDeException { // Verify that Hive is caching the object inspectors for us. Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java?rev=1399935&r1=1399934&r2=1399935&view=diff ============================================================================== --- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java (original) +++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java Fri Oct 19 00:23:11 2012 @@ -31,6 +31,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.Hashtable; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -206,6 +207,21 @@ public class TestAvroSerializer { assertNull(r.get("nullableint")); } + @Test + public void canSerializeMapsWithNullablePrimitiveValues() throws SerDeException, IOException { + String field = "{ \"name\":\"mapWithNulls\", \"type\": " + + "{\"type\":\"map\", \"values\": [\"null\", \"boolean\"]} }"; + + Map<String, Boolean> m = new HashMap<String, Boolean>(); + m.put("yes", true); + m.put("no", false); + m.put("maybe", null); + GenericRecord r = serializeAndDeserialize(field, "mapWithNulls", m); + + Object result = r.get("mapWithNulls"); + assertEquals(m, result); + } + @Test public void canSerializeBytes() throws SerDeException, IOException { String field = "{ \"name\":\"bytes1\", \"type\":\"bytes\" }";