ravjot28 commented on code in PR #19872:
URL: https://github.com/apache/hudi/pull/19872#discussion_r4540329642
##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
}
protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
- if (record != null) {
- if (record.getSchema() == schema) {
- return Option.of(record);
- }
- // if the schema does not match, we need to deserialize with the proper
schema to match legacy behavior
- recordBytes = getRecordBytes();
+ if (record != null && record.getSchema() == schema) {
+ return Option.of(record);
}
- if (recordBytes == null || recordBytes.length == 0) {
+ byte[] bytes = getRecordBytes();
+ if (bytes.length == 0) {
return Option.empty();
}
- record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+ if (writerSchema == null) {
+ // Legacy serialized payloads did not include a writer schema.
+ writerSchema = schema;
+ }
+ Map<String, String> renames = new HashMap<>();
+ collectRenames(writerSchema, schema, "", renames, new HashSet<>());
+ if (renames.isEmpty()) {
+ record = HoodieAvroUtils.bytesToAvro(bytes, writerSchema, schema);
+ } else {
+ GenericRecord original = HoodieAvroUtils.bytesToAvro(bytes,
writerSchema);
+ record = HoodieAvroUtils.rewriteRecordWithNewSchema(original, schema,
renames);
+ }
return Option.of(record);
}
+ private static void collectRenames(Schema writer, Schema reader, String
prefix,
+ Map<String, String> renames,
Set<Pair<Schema, Schema>> visiting) {
+ if (writer.equals(reader)) {
+ return;
+ }
+ Pair<Schema, Schema> pair = Pair.of(writer, reader);
+ if (!visiting.add(pair)) {
+ return;
+ }
+ try {
+ if (writer.getType() == Schema.Type.UNION || reader.getType() ==
Schema.Type.UNION) {
+ for (Schema w : writer.getType() == Schema.Type.UNION ?
writer.getTypes() : java.util.Collections.singletonList(writer)) {
+ for (Schema r : reader.getType() == Schema.Type.UNION ?
reader.getTypes() : java.util.Collections.singletonList(reader)) {
+ boolean sameRecordBranch = w.getType() != Schema.Type.RECORD ||
r.getType() != Schema.Type.RECORD
+ || w.getFullName().equals(r.getFullName()) ||
r.getAliases().contains(w.getFullName())
+ || (writer.getType() != Schema.Type.UNION ||
writer.getTypes().stream().filter(s -> s.getType() ==
Schema.Type.RECORD).count() == 1)
+ && (reader.getType() != Schema.Type.UNION ||
reader.getTypes().stream().filter(s -> s.getType() ==
Schema.Type.RECORD).count() == 1);
Review Comment:
Done in 6cd646b8519: the condition is in `isSameRecordBranch`.
##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
}
protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
- if (record != null) {
- if (record.getSchema() == schema) {
- return Option.of(record);
- }
- // if the schema does not match, we need to deserialize with the proper
schema to match legacy behavior
- recordBytes = getRecordBytes();
+ if (record != null && record.getSchema() == schema) {
+ return Option.of(record);
}
- if (recordBytes == null || recordBytes.length == 0) {
+ byte[] bytes = getRecordBytes();
+ if (bytes.length == 0) {
return Option.empty();
}
- record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+ if (writerSchema == null) {
+ // Legacy serialized payloads did not include a writer schema.
+ writerSchema = schema;
+ }
+ Map<String, String> renames = new HashMap<>();
+ collectRenames(writerSchema, schema, "", renames, new HashSet<>());
+ if (renames.isEmpty()) {
+ record = HoodieAvroUtils.bytesToAvro(bytes, writerSchema, schema);
+ } else {
+ GenericRecord original = HoodieAvroUtils.bytesToAvro(bytes,
writerSchema);
+ record = HoodieAvroUtils.rewriteRecordWithNewSchema(original, schema,
renames);
+ }
return Option.of(record);
}
+ private static void collectRenames(Schema writer, Schema reader, String
prefix,
+ Map<String, String> renames,
Set<Pair<Schema, Schema>> visiting) {
+ if (writer.equals(reader)) {
+ return;
+ }
+ Pair<Schema, Schema> pair = Pair.of(writer, reader);
+ if (!visiting.add(pair)) {
+ return;
+ }
+ try {
+ if (writer.getType() == Schema.Type.UNION || reader.getType() ==
Schema.Type.UNION) {
+ for (Schema w : writer.getType() == Schema.Type.UNION ?
writer.getTypes() : java.util.Collections.singletonList(writer)) {
+ for (Schema r : reader.getType() == Schema.Type.UNION ?
reader.getTypes() : java.util.Collections.singletonList(reader)) {
+ boolean sameRecordBranch = w.getType() != Schema.Type.RECORD ||
r.getType() != Schema.Type.RECORD
+ || w.getFullName().equals(r.getFullName()) ||
r.getAliases().contains(w.getFullName())
Review Comment:
Done in 6cd646b8519: `Collections` is imported.
##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
}
protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
- if (record != null) {
- if (record.getSchema() == schema) {
- return Option.of(record);
- }
- // if the schema does not match, we need to deserialize with the proper
schema to match legacy behavior
- recordBytes = getRecordBytes();
+ if (record != null && record.getSchema() == schema) {
+ return Option.of(record);
}
- if (recordBytes == null || recordBytes.length == 0) {
+ byte[] bytes = getRecordBytes();
+ if (bytes.length == 0) {
return Option.empty();
}
- record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+ if (writerSchema == null) {
+ // Legacy serialized payloads did not include a writer schema.
+ writerSchema = schema;
+ }
+ Map<String, String> renames = new HashMap<>();
Review Comment:
6cd646b8519 replaces the walk: a renamed copy of the writer schema is built
by `copy` (unions, arrays, maps) and `copyRecord`, and the field lookup is in
`resolveFields`, `findByNameOrAlias` and `findPositionalFallback`.
##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +169,189 @@ protected boolean isEmptyRecord() {
}
protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
- if (record != null) {
- if (record.getSchema() == schema) {
- return Option.of(record);
- }
- // if the schema does not match, we need to deserialize with the proper
schema to match legacy behavior
- recordBytes = getRecordBytes();
+ if (record != null && record.getSchema() == schema) {
+ return Option.of(record);
}
- if (recordBytes == null || recordBytes.length == 0) {
+ byte[] bytes = getRecordBytes();
+ if (bytes.length == 0) {
return Option.empty();
}
- record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+ if (writerSchema == null) {
+ // Legacy bytes have no schema. Preserve positional decoding until a
supplied schema
+ // consumes the complete record; a prefix projection must not become the
writer schema.
+ BinaryDecoder decoder = HoodieAvroUtils.getBinaryDecoder(bytes, 0,
bytes.length);
+ record = new GenericDatumReader<GenericRecord>(schema).read(null,
decoder);
+ if (decoder.isEnd()) {
+ writerSchema = schema;
+ }
+ return Option.of(record);
+ }
+ Map<String, String> renames = new HashMap<>();
+ collectRenames(writerSchema, schema, "", renames, new HashSet<>());
Review Comment:
Done in 6cd646b8519. The resolution (equal layout, or the renamed copy of
the writer schema) is cached per pair of schema instances in a bounded Caffeine
cache with weak keys, which compare by identity: up to 1024 writer schemas,
each with up to 64 reader schemas. A hit is two lookups; an evicted pair is
computed again. When the two schemas are equal, the bytes decode positionally,
as on master.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]