Author: daijy Date: Thu Mar 5 17:45:56 2015 New Revision: 1664432 URL: http://svn.apache.org/r1664432 Log: PIG-4447: Pig Cannot handle nullable values (arrays and records) in avro records
Added: pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc Modified: pig/trunk/CHANGES.txt pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java Modified: pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1664432&r1=1664431&r2=1664432&view=diff ============================================================================== --- pig/trunk/CHANGES.txt (original) +++ pig/trunk/CHANGES.txt Thu Mar 5 17:45:56 2015 @@ -50,6 +50,8 @@ PIG-4333: Split BigData tests into multi BUG FIXES +PIG-4447: Pig Cannot handle nullable values (arrays and records) in avro records (rdsr via daijy) + PIG-4444: Fix unit test failure TestTezAutoParallelism (daijy) PIG-4445: VALUELIST and VALUESET outputSchema does not match actual schema of data returned when map value schema Modified: pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java?rev=1664432&r1=1664431&r2=1664432&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java (original) +++ pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java Thu Mar 5 17:45:56 2015 @@ -222,7 +222,7 @@ public class AvroStorageSchemaConversion switch(mapAvroSchema.getType()) { case RECORD: ResourceSchema innerResourceSchemaRecord = - avroSchemaToResourceSchema(fieldSchema.getValueType(), schemasInStack, + avroSchemaToResourceSchema(mapAvroSchema, schemasInStack, alreadyDefinedSchemas, allowRecursiveSchema); mapSchemaFields[0] = new ResourceSchema.ResourceFieldSchema(); mapSchemaFields[0].setType(DataType.TUPLE); @@ -235,7 +235,7 @@ public class AvroStorageSchemaConversion case MAP: case ARRAY: ResourceSchema innerResourceSchema = - avroSchemaToResourceSchema(fieldSchema.getValueType(), schemasInStack, + avroSchemaToResourceSchema(mapAvroSchema, schemasInStack, alreadyDefinedSchemas, allowRecursiveSchema); rf.setSchema(innerResourceSchema); break; Added: pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc?rev=1664432&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc (added) +++ pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc Thu Mar 5 17:45:56 2015 @@ -0,0 +1,22 @@ +{ + "name" : "nullableRecordInMap", + "namespace" : "org.apache.pig.test.builtin", + "type" : "record", + "fields" : [ + {"name" : "key", "type" : "string"}, + {"name" : "value", "type" : "int"}, + { + "name" : "parameters", + "type": { + "type": "map", + "values": [ + "null", + { + "type": "array", + "items": "string" + } + ] + } + } + ] +} Added: pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc?rev=1664432&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc (added) +++ pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc Thu Mar 5 17:45:56 2015 @@ -0,0 +1,34 @@ +{ + "name" : "nullableRecordInMap", + "namespace" : "org.apache.pig.test.builtin", + "type" : "record", + "fields" : [ + {"name" : "key", "type" : "string"}, + {"name" : "value", "type" : "int"}, + { + "name" : "parameters", + "type": [ + "null", + { + "type": "map", + "values": [ + "null", + { + "type": "record", + "name": "nullable_record", + "fields": [ + { + "name": "id", + "type": [ + "null", + "string" + ] + } + ] + } + ] + } + ] + } + ] +}