ravjot28 commented on code in PR #19872:
URL: https://github.com/apache/hudi/pull/19872#discussion_r4540329642


##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
   }
 
   protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
-    if (record != null) {
-      if (record.getSchema() == schema) {
-        return Option.of(record);
-      }
-      // if the schema does not match, we need to deserialize with the proper 
schema to match legacy behavior
-      recordBytes = getRecordBytes();
+    if (record != null && record.getSchema() == schema) {
+      return Option.of(record);
     }
-    if (recordBytes == null || recordBytes.length == 0) {
+    byte[] bytes = getRecordBytes();
+    if (bytes.length == 0) {
       return Option.empty();
     }
-    record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+    if (writerSchema == null) {
+      // Legacy serialized payloads did not include a writer schema.
+      writerSchema = schema;
+    }
+    Map<String, String> renames = new HashMap<>();
+    collectRenames(writerSchema, schema, "", renames, new HashSet<>());
+    if (renames.isEmpty()) {
+      record = HoodieAvroUtils.bytesToAvro(bytes, writerSchema, schema);
+    } else {
+      GenericRecord original = HoodieAvroUtils.bytesToAvro(bytes, 
writerSchema);
+      record = HoodieAvroUtils.rewriteRecordWithNewSchema(original, schema, 
renames);
+    }
     return Option.of(record);
   }
 
+  private static void collectRenames(Schema writer, Schema reader, String 
prefix,
+                                     Map<String, String> renames, 
Set<Pair<Schema, Schema>> visiting) {
+    if (writer.equals(reader)) {
+      return;
+    }
+    Pair<Schema, Schema> pair = Pair.of(writer, reader);
+    if (!visiting.add(pair)) {
+      return;
+    }
+    try {
+      if (writer.getType() == Schema.Type.UNION || reader.getType() == 
Schema.Type.UNION) {
+        for (Schema w : writer.getType() == Schema.Type.UNION ? 
writer.getTypes() : java.util.Collections.singletonList(writer)) {
+          for (Schema r : reader.getType() == Schema.Type.UNION ? 
reader.getTypes() : java.util.Collections.singletonList(reader)) {
+            boolean sameRecordBranch = w.getType() != Schema.Type.RECORD || 
r.getType() != Schema.Type.RECORD
+                || w.getFullName().equals(r.getFullName()) || 
r.getAliases().contains(w.getFullName())
+                || (writer.getType() != Schema.Type.UNION || 
writer.getTypes().stream().filter(s -> s.getType() == 
Schema.Type.RECORD).count() == 1)
+                && (reader.getType() != Schema.Type.UNION || 
reader.getTypes().stream().filter(s -> s.getType() == 
Schema.Type.RECORD).count() == 1);

Review Comment:
   Done in 6cd646b8519: the condition is in `isSameRecordBranch`.
   



##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
   }
 
   protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
-    if (record != null) {
-      if (record.getSchema() == schema) {
-        return Option.of(record);
-      }
-      // if the schema does not match, we need to deserialize with the proper 
schema to match legacy behavior
-      recordBytes = getRecordBytes();
+    if (record != null && record.getSchema() == schema) {
+      return Option.of(record);
     }
-    if (recordBytes == null || recordBytes.length == 0) {
+    byte[] bytes = getRecordBytes();
+    if (bytes.length == 0) {
       return Option.empty();
     }
-    record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+    if (writerSchema == null) {
+      // Legacy serialized payloads did not include a writer schema.
+      writerSchema = schema;
+    }
+    Map<String, String> renames = new HashMap<>();
+    collectRenames(writerSchema, schema, "", renames, new HashSet<>());
+    if (renames.isEmpty()) {
+      record = HoodieAvroUtils.bytesToAvro(bytes, writerSchema, schema);
+    } else {
+      GenericRecord original = HoodieAvroUtils.bytesToAvro(bytes, 
writerSchema);
+      record = HoodieAvroUtils.rewriteRecordWithNewSchema(original, schema, 
renames);
+    }
     return Option.of(record);
   }
 
+  private static void collectRenames(Schema writer, Schema reader, String 
prefix,
+                                     Map<String, String> renames, 
Set<Pair<Schema, Schema>> visiting) {
+    if (writer.equals(reader)) {
+      return;
+    }
+    Pair<Schema, Schema> pair = Pair.of(writer, reader);
+    if (!visiting.add(pair)) {
+      return;
+    }
+    try {
+      if (writer.getType() == Schema.Type.UNION || reader.getType() == 
Schema.Type.UNION) {
+        for (Schema w : writer.getType() == Schema.Type.UNION ? 
writer.getTypes() : java.util.Collections.singletonList(writer)) {
+          for (Schema r : reader.getType() == Schema.Type.UNION ? 
reader.getTypes() : java.util.Collections.singletonList(reader)) {
+            boolean sameRecordBranch = w.getType() != Schema.Type.RECORD || 
r.getType() != Schema.Type.RECORD
+                || w.getFullName().equals(r.getFullName()) || 
r.getAliases().contains(w.getFullName())

Review Comment:
   Done in 6cd646b8519: `Collections` is imported.
   



##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
   }
 
   protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
-    if (record != null) {
-      if (record.getSchema() == schema) {
-        return Option.of(record);
-      }
-      // if the schema does not match, we need to deserialize with the proper 
schema to match legacy behavior
-      recordBytes = getRecordBytes();
+    if (record != null && record.getSchema() == schema) {
+      return Option.of(record);
     }
-    if (recordBytes == null || recordBytes.length == 0) {
+    byte[] bytes = getRecordBytes();
+    if (bytes.length == 0) {
       return Option.empty();
     }
-    record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+    if (writerSchema == null) {
+      // Legacy serialized payloads did not include a writer schema.
+      writerSchema = schema;
+    }
+    Map<String, String> renames = new HashMap<>();

Review Comment:
   6cd646b8519 replaces the walk: a renamed copy of the writer schema is built 
by `copy` (unions, arrays, maps) and `copyRecord`, and the field lookup is in 
`resolveFields`, `findByNameOrAlias` and `findPositionalFallback`.
   



##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +169,189 @@ protected boolean isEmptyRecord() {
   }
 
   protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
-    if (record != null) {
-      if (record.getSchema() == schema) {
-        return Option.of(record);
-      }
-      // if the schema does not match, we need to deserialize with the proper 
schema to match legacy behavior
-      recordBytes = getRecordBytes();
+    if (record != null && record.getSchema() == schema) {
+      return Option.of(record);
     }
-    if (recordBytes == null || recordBytes.length == 0) {
+    byte[] bytes = getRecordBytes();
+    if (bytes.length == 0) {
       return Option.empty();
     }
-    record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+    if (writerSchema == null) {
+      // Legacy bytes have no schema. Preserve positional decoding until a 
supplied schema
+      // consumes the complete record; a prefix projection must not become the 
writer schema.
+      BinaryDecoder decoder = HoodieAvroUtils.getBinaryDecoder(bytes, 0, 
bytes.length);
+      record = new GenericDatumReader<GenericRecord>(schema).read(null, 
decoder);
+      if (decoder.isEnd()) {
+        writerSchema = schema;
+      }
+      return Option.of(record);
+    }
+    Map<String, String> renames = new HashMap<>();
+    collectRenames(writerSchema, schema, "", renames, new HashSet<>());

Review Comment:
   Done in 6cd646b8519. The resolution (equal layout, or the renamed copy of 
the writer schema) is cached per pair of schema instances in a bounded Caffeine 
cache with weak keys, which compare by identity: up to 1024 writer schemas, 
each with up to 64 reader schemas. A hit is two lookups; an evicted pair is 
computed again. When the two schemas are equal, the bytes decode positionally, 
as on master.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to