[
https://issues.apache.org/jira/browse/BEAM-4454?focusedWorklogId=171423&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-171423
]
ASF GitHub Bot logged work on BEAM-4454:
----------------------------------------
Author: ASF GitHub Bot
Created on: 03/Dec/18 00:24
Start Date: 03/Dec/18 00:24
Worklog Time Spent: 10m
Work Description: reuvenlax commented on a change in pull request #7181:
[BEAM-4454] Add more AVRO utilities to convert between Beam and Avro.
URL: https://github.com/apache/beam/pull/7181#discussion_r238122666
##########
File path:
sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroUtilsTest.java
##########
@@ -89,10 +100,202 @@ public void testUnwrapNullableSchemaToUnion() {
org.apache.avro.Schema.create(Type.LONG),
org.apache.avro.Schema.create(Type.NULL));
+ TypeWithNullability typeWithNullability = new
TypeWithNullability(avroSchema);
+ assertTrue(typeWithNullability.nullable);
assertEquals(
org.apache.avro.Schema.createUnion(
org.apache.avro.Schema.create(Type.STRING),
org.apache.avro.Schema.create(Type.LONG)),
- AvroUtils.unwrapNullableSchema(avroSchema));
+ typeWithNullability.type);
+ }
+
+ private org.apache.avro.Schema getAvroSubSchema() {
+ List<org.apache.avro.Schema.Field> fields = Lists.newArrayList();
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "bool", org.apache.avro.Schema.create(Type.BOOLEAN), "", null));
+ fields.add(
+ new org.apache.avro.Schema.Field("int",
org.apache.avro.Schema.create(Type.INT), "", null));
+ return org.apache.avro.Schema.createRecord(fields);
+ }
+
+ private org.apache.avro.Schema getAvroSchema() {
+ List<org.apache.avro.Schema.Field> fields = Lists.newArrayList();
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "bool", org.apache.avro.Schema.create(Type.BOOLEAN), "", null));
+ fields.add(
+ new org.apache.avro.Schema.Field("int",
org.apache.avro.Schema.create(Type.INT), "", null));
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "long", org.apache.avro.Schema.create(Type.LONG), "", null));
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "float", org.apache.avro.Schema.create(Type.FLOAT), "", null));
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "double", org.apache.avro.Schema.create(Type.DOUBLE), "", null));
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "string", org.apache.avro.Schema.create(Type.STRING), "", null));
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "bytes", org.apache.avro.Schema.create(Type.BYTES), "", null));
+ fields.add(new org.apache.avro.Schema.Field("row", getAvroSubSchema(), "",
null));
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "array", org.apache.avro.Schema.createArray(getAvroSubSchema()),
"", null));
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "map", org.apache.avro.Schema.createMap(getAvroSubSchema()), "",
null));
+ return org.apache.avro.Schema.createRecord(fields);
+ }
+
+ private Schema getBeamSubSchema() {
+ return new Schema.Builder()
+ .addField(Field.of("bool", FieldType.BOOLEAN))
+ .addField(Field.of("int", FieldType.INT32))
+ .build();
+ }
+
+ private Schema getBeamSchema() {
+ Schema subSchema = getBeamSubSchema();
+ return new Schema.Builder()
+ .addField(Field.of("bool", FieldType.BOOLEAN))
+ .addField(Field.of("int", FieldType.INT32))
+ .addField(Field.of("long", FieldType.INT64))
+ .addField(Field.of("float", FieldType.FLOAT))
+ .addField(Field.of("double", FieldType.DOUBLE))
+ .addField(Field.of("string", FieldType.STRING))
+ .addField(Field.of("bytes", FieldType.BYTES))
+ .addField(Field.of("row", FieldType.row(subSchema)))
+ .addField(Field.of("array", FieldType.array(FieldType.row(subSchema))))
+ .addField(Field.of("map", FieldType.map(FieldType.STRING,
FieldType.row(subSchema))))
+ .build();
+ }
+
+ private Row getBeamRow() {
+ Row subRow = Row.withSchema(getBeamSubSchema()).addValues(true,
42).build();
+
+ byte[] a0 = new byte[] {1, 2, 3, 4};
+ return Row.withSchema(getBeamSchema())
+ .addValue(true)
+ .addValue(43)
+ .addValue(44L)
+ .addValue((float) 44.1)
+ .addValue((double) 44.2)
+ .addValue("string")
+ .addValue(a0)
+ .addValue(subRow)
+ .addValue(ImmutableList.of(subRow, subRow))
+ .addValue(ImmutableMap.of("k1", subRow, "k2", subRow))
+ .build();
+ }
+
+ private GenericRecord getGenericRecord() {
+ byte[] a0 = new byte[] {1, 2, 3, 4};
+
+ GenericRecord subRecord =
+ new GenericRecordBuilder(getAvroSubSchema()).set("bool",
true).set("int", 42).build();
+
+ return new GenericRecordBuilder(getAvroSchema())
+ .set("bool", true)
+ .set("int", 43)
+ .set("long", 44L)
+ .set("float", (float) 44.1)
+ .set("double", (double) 44.2)
+ .set("string", "string")
+ .set("bytes", ByteBuffer.wrap(a0))
+ .set("row", subRecord)
+ .set("array", ImmutableList.of(subRecord, subRecord))
+ .set("map", ImmutableMap.of("k1", subRecord, "k2", subRecord))
+ .build();
+ }
+
+ @Test
+ public void testFromAvroSchema() {
+ assertEquals(getBeamSchema(), AvroUtils.toBeamSchema(getAvroSchema()));
+ }
+
+ @Test
+ public void testFromBeamSchema() {
+ Schema beamSchema = getBeamSchema();
+ org.apache.avro.Schema avroSchema = AvroUtils.toAvroSchema(beamSchema);
+ assertEquals(getAvroSchema(), avroSchema);
+ }
+
+ @Test
+ public void testNullableFieldInAvroSchema() {
+ List<org.apache.avro.Schema.Field> fields = Lists.newArrayList();
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "int",
ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT)), "", null));
+ fields.add(
+ new org.apache.avro.Schema.Field(
+ "array",
+ org.apache.avro.Schema.createArray(
+
ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT))),
+ "",
+ null));
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 171423)
Time Spent: 1.5h (was: 1h 20m)
> Provide automatic schema registration for AVROs
> -----------------------------------------------
>
> Key: BEAM-4454
> URL: https://issues.apache.org/jira/browse/BEAM-4454
> Project: Beam
> Issue Type: Sub-task
> Components: sdk-java-core
> Reporter: Reuven Lax
> Assignee: Reuven Lax
> Priority: Major
> Time Spent: 1.5h
> Remaining Estimate: 0h
>
> Need to make sure this is a compatible change
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
