[
https://issues.apache.org/jira/browse/AVRO-2034?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16717681#comment-16717681
]
ASF GitHub Bot commented on AVRO-2034:
--------------------------------------
dkulp closed pull request #224: AVRO-2034 Nested schema types with unexpected
fields causes json parse failure
URL: https://github.com/apache/avro/pull/224
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java
b/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java
index 78fafaa83..cd2742453 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java
@@ -494,14 +494,23 @@ public Symbol doAction(Symbol input, Symbol top) throws
IOException {
throw error("record-start");
}
} else if (top == Symbol.RECORD_END || top == Symbol.UNION_END) {
- if (in.getCurrentToken() == JsonToken.END_OBJECT) {
+ //AVRO-2034 advance to the end of our object
+ while(in.getCurrentToken() != JsonToken.END_OBJECT){
in.nextToken();
+ }
+
+ if (in.getCurrentToken() == JsonToken.END_OBJECT) {
+
if (top == Symbol.RECORD_END) {
if (currentReorderBuffer != null &&
!currentReorderBuffer.savedFields.isEmpty()) {
throw error("Unknown fields: " +
currentReorderBuffer.savedFields.keySet());
}
currentReorderBuffer = reorderBuffers.pop();
}
+
+ //AVRO-2034 advance beyond the end object for the next record.
+ in.nextToken();
+
} else {
throw error(top == Symbol.RECORD_END ? "record-end" : "union-end");
}
diff --git
a/lang/java/avro/src/test/java/org/apache/avro/TestNestedRecords.java
b/lang/java/avro/src/test/java/org/apache/avro/TestNestedRecords.java
new file mode 100644
index 000000000..8900b1ee9
--- /dev/null
+++ b/lang/java/avro/src/test/java/org/apache/avro/TestNestedRecords.java
@@ -0,0 +1,110 @@
+package org.apache.avro;
+
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.JsonDecoder;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * This test demonstrates the fix for a complex nested schema type.
+ */
+public class TestNestedRecords {
+
+
+ @Test
+ public void testSingleSubRecord() throws IOException {
+
+ final Schema child = SchemaBuilder.record("Child")
+ .namespace("org.apache.avro.nested")
+ .fields()
+ .requiredString("childField").endRecord();
+
+
+ final Schema parent = SchemaBuilder.record("Parent")
+ .namespace("org.apache.avro.nested")
+ .fields()
+ .requiredString("parentField1")
+ .name("child1").type(child).noDefault()
+ .requiredString("parentField2").endRecord();
+
+
+
+ final String inputAsExpected = "{\n" +
+ " \"parentField1\": \"parentValue1\",\n" +
+ " \"child1\":{\n" +
+ " \"childField\":\"childValue1\"\n" +
+ " },\n" +
+ " \"parentField2\":\"parentValue2\"\n" +
+ "}";
+
+
+ final ByteArrayInputStream inputStream = new
ByteArrayInputStream(inputAsExpected.getBytes());
+
+ final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(parent,
inputStream);
+ final DatumReader<Object> reader = new GenericDatumReader<Object>(parent);
+
+ final GenericData.Record decoded = (GenericData.Record) reader.read(null,
decoder);
+
+
+ assertThat(decoded.get("parentField1").toString(),
equalTo("parentValue1"));
+ assertThat(decoded.get("parentField2").toString(),
equalTo("parentValue2"));
+
+
assertThat(((GenericData.Record)decoded.get("child1")).get("childField").toString(),
equalTo("childValue1"));
+
+ }
+
+
+
+ @Test
+ public void testSingleSubRecordExtraField() throws IOException {
+
+ final Schema child = SchemaBuilder.record("Child")
+ .namespace("org.apache.avro.nested")
+ .fields()
+ .requiredString("childField").endRecord();
+
+
+ final Schema parent = SchemaBuilder.record("Parent")
+ .namespace("org.apache.avro.nested")
+ .fields()
+ .requiredString("parentField1")
+ .name("child1").type(child).noDefault()
+ .requiredString("parentField2").endRecord();
+
+
+ final String inputAsExpected = "{\n" +
+ " \"parentField1\": \"parentValue1\",\n" +
+ " \"child1\":{\n" +
+ " \"childField\":\"childValue1\",\n" +
+
+ //this field should be safely ignored
+ " \"extraField\":\"extraValue\"\n" +
+ " },\n" +
+ " \"parentField2\":\"parentValue2\"\n" +
+ "}";
+
+
+ final ByteArrayInputStream inputStream = new
ByteArrayInputStream(inputAsExpected.getBytes());
+
+ final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(parent,
inputStream);
+ final DatumReader<Object> reader = new GenericDatumReader<Object>(parent);
+
+ final GenericData.Record decoded = (GenericData.Record) reader.read(null,
decoder);
+
+ assertThat(decoded.get("parentField1").toString(),
equalTo("parentValue1"));
+ assertThat(decoded.get("parentField2").toString(),
equalTo("parentValue2"));
+
+
assertThat(((GenericData.Record)decoded.get("child1")).get("childField").toString(),
equalTo("childValue1"));
+
+
+ }
+
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Nested schema types with unexpected fields causes json parse failure
> --------------------------------------------------------------------
>
> Key: AVRO-2034
> URL: https://issues.apache.org/jira/browse/AVRO-2034
> Project: Apache Avro
> Issue Type: Bug
> Components: java
> Affects Versions: 1.8.1
> Reporter: Todd Nine
> Priority: Major
>
> When parsing a nested type with an unexpected field using the JSON parser,
> this results in an error. To reproduce, see the class {{TestNestedRecords}}
> in the referenced PR.
> https://github.com/apache/avro/pull/224
> Note that this only occurs when the following pattern exists in the schema.
> # regular field
> # nested record with additional field
> # Any subsequent field following the nested record with an unexpected field
> appears to reproduce the problem.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
