Author: cws
Date: Fri Oct 19 00:23:11 2012
New Revision: 1399935
URL: http://svn.apache.org/viewvc?rev=1399935&view=rev
Log:
HIVE-3525. Avro Maps with Nullable Values fail with NPE (Sean Busbey via cws)
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java?rev=1399935&r1=1399934&r2=1399935&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
Fri Oct 19 00:23:11 2012
@@ -43,7 +43,7 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Hashtable;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -246,7 +246,7 @@ class AvroDeserializer {
throws AvroSerdeException {
// Avro only allows maps with Strings for keys, so we only have to worry
// about deserializing the values
- Map<String, Object> map = new Hashtable<String, Object>();
+ Map<String, Object> map = new HashMap<String, Object>();
Map<Utf8, Object> mapDatum = (Map)datum;
Schema valueSchema = mapSchema.getValueType();
TypeInfo valueTypeInfo = columnType.getMapValueTypeInfo();
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java?rev=1399935&r1=1399934&r2=1399935&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java
Fri Oct 19 00:23:11 2012
@@ -39,7 +39,7 @@ import org.apache.hadoop.io.Writable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Hashtable;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -229,7 +229,7 @@ class AvroSerializer {
Map<?,?> map = fieldOI.getMap(structFieldData);
Schema valueType = schema.getValueType();
- Map<Object, Object> deserialized = new Hashtable<Object,
Object>(fieldOI.getMapSize(structFieldData));
+ Map<Object, Object> deserialized = new HashMap<Object,
Object>(fieldOI.getMapSize(structFieldData));
for (Map.Entry<?, ?> entry : map.entrySet()) {
deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector,
entry.getKey(), null), // This works, but is a bit fragile. Construct a single
String schema?
Modified:
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java?rev=1399935&r1=1399934&r2=1399935&view=diff
==============================================================================
---
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
(original)
+++
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
Fri Oct 19 00:23:11 2012
@@ -36,6 +36,7 @@ import java.util.ArrayList;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
+import java.util.HashMap;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@@ -404,6 +405,57 @@ public class TestAvroDeserializer {
verifyNullableType(record, s, null);
}
+ @Test
+ public void canDeserializeMapWithNullablePrimitiveValues() throws
SerDeException, IOException {
+ Schema s =
Schema.parse(TestAvroObjectInspectorGenerator.MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA);
+ GenericData.Record record = new GenericData.Record(s);
+
+ Map<String, Long> m = new HashMap<String, Long>();
+ m.put("one", 1l);
+ m.put("two", 2l);
+ m.put("three", 3l);
+ m.put("mu", null);
+
+ record.put("aMap", m);
+ assertTrue(GENERIC_DATA.validate(s, record));
+ System.out.println("record = " + record);
+
+ AvroGenericRecordWritable garw =
Utils.serializeAndDeserializeRecord(record);
+
+ AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
+
+ AvroDeserializer de = new AvroDeserializer();
+
+ ArrayList<Object> row =
(ArrayList<Object>)de.deserialize(aoig.getColumnNames(),
+ aoig.getColumnTypes(), garw, s);
+ assertEquals(1, row.size());
+ Object theMapObject = row.get(0);
+ assertTrue(theMapObject instanceof Map);
+ Map theMap = (Map)theMapObject;
+
+ // Verify the raw object that's been created
+ assertEquals(1l, theMap.get("one"));
+ assertEquals(2l, theMap.get("two"));
+ assertEquals(3l, theMap.get("three"));
+ assertTrue(theMap.containsKey("mu"));
+ assertEquals(null, theMap.get("mu"));
+
+ // Verify that the provided object inspector can pull out these same values
+ StandardStructObjectInspector oi =
+ (StandardStructObjectInspector)aoig.getObjectInspector();
+
+ List<Object> z = oi.getStructFieldsDataAsList(row);
+ assertEquals(1, z.size());
+ StructField fieldRef = oi.getStructFieldRef("amap");
+
+ Map theMap2 = (Map)oi.getStructFieldData(row, fieldRef);
+ assertEquals(1l, theMap2.get("one"));
+ assertEquals(2l, theMap2.get("two"));
+ assertEquals(3l, theMap2.get("three"));
+ assertTrue(theMap2.containsKey("mu"));
+ assertEquals(null, theMap2.get("mu"));
+ }
+
private void verifyNullableType(GenericData.Record record, Schema s,
String expected) throws SerDeException,
IOException {
assertTrue(GENERIC_DATA.validate(s, record));
Modified:
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java?rev=1399935&r1=1399934&r2=1399935&view=diff
==============================================================================
---
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
(original)
+++
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java
Fri Oct 19 00:23:11 2012
@@ -142,6 +142,18 @@ public class TestAvroObjectInspectorGene
" {\"name\":\"nullableString\", \"type\":[\"null\", \"string\"]}\n" +
" ]\n" +
"}";
+ public static final String MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA =
"{\n" +
+ " \"namespace\": \"testing\",\n" +
+ " \"name\": \"mapWithNullableUnionTest\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"aMap\",\n" +
+ " \"type\":{\"type\":\"map\",\n" +
+ " \"values\":[\"null\",\"long\"]}\n" +
+ "\t}\n" +
+ " ]\n" +
+ "}";
public static final String BYTES_SCHEMA = "{\n" +
" \"type\": \"record\", \n" +
" \"name\": \"bytesTest\",\n" +
@@ -325,10 +337,19 @@ public class TestAvroObjectInspectorGene
public void canHandleMapsWithPrimitiveValueTypes() throws SerDeException {
Schema s = Schema.parse(MAP_WITH_PRIMITIVE_VALUE_TYPE);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
-
+ verifyMap(aoig, "aMap");
+ }
+
+ /**
+ * Check a given AvroObjectInspectorGenerator to verify that it matches our
test
+ * schema's expected map.
+ * @param aoig should already have been intitialized, may not be null
+ * @param fieldName name of the contianed column, will always fail if null.
+ */
+ private void verifyMap(final AvroObjectInspectorGenerator aoig, final String
fieldName) {
// Column names
assertEquals(1, aoig.getColumnNames().size());
- assertEquals("aMap", aoig.getColumnNames().get(0));
+ assertEquals(fieldName, aoig.getColumnNames().get(0));
// Column types
assertEquals(1, aoig.getColumnTypes().size());
@@ -483,6 +504,13 @@ public class TestAvroObjectInspectorGene
assertEquals(PrimitiveObjectInspector.PrimitiveCategory.STRING,
pti.getPrimitiveCategory());
}
+ @Test // That Union[T, NULL] is converted to just T, within a Map
+ public void convertsMapsWithNullablePrimitiveTypes() throws SerDeException {
+ Schema s = Schema.parse(MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA);
+ AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
+ verifyMap(aoig, "aMap");
+ }
+
@Test
public void objectInspectorsAreCached() throws SerDeException {
// Verify that Hive is caching the object inspectors for us.
Modified:
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java?rev=1399935&r1=1399934&r2=1399935&view=diff
==============================================================================
---
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
(original)
+++
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
Fri Oct 19 00:23:11 2012
@@ -31,6 +31,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Hashtable;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -206,6 +207,21 @@ public class TestAvroSerializer {
assertNull(r.get("nullableint"));
}
+ @Test
+ public void canSerializeMapsWithNullablePrimitiveValues() throws
SerDeException, IOException {
+ String field = "{ \"name\":\"mapWithNulls\", \"type\": " +
+ "{\"type\":\"map\", \"values\": [\"null\", \"boolean\"]} }";
+
+ Map<String, Boolean> m = new HashMap<String, Boolean>();
+ m.put("yes", true);
+ m.put("no", false);
+ m.put("maybe", null);
+ GenericRecord r = serializeAndDeserialize(field, "mapWithNulls", m);
+
+ Object result = r.get("mapWithNulls");
+ assertEquals(m, result);
+ }
+
@Test
public void canSerializeBytes() throws SerDeException, IOException {
String field = "{ \"name\":\"bytes1\", \"type\":\"bytes\" }";
