ravjot28 commented on code in PR #19872:
URL: https://github.com/apache/hudi/pull/19872#discussion_r4540327282


##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
   }
 
   protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
-    if (record != null) {
-      if (record.getSchema() == schema) {
-        return Option.of(record);
-      }
-      // if the schema does not match, we need to deserialize with the proper 
schema to match legacy behavior
-      recordBytes = getRecordBytes();
+    if (record != null && record.getSchema() == schema) {
+      return Option.of(record);
     }
-    if (recordBytes == null || recordBytes.length == 0) {
+    byte[] bytes = getRecordBytes();
+    if (bytes.length == 0) {
       return Option.empty();
     }
-    record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+    if (writerSchema == null) {
+      // Legacy serialized payloads did not include a writer schema.
+      writerSchema = schema;
+    }
+    Map<String, String> renames = new HashMap<>();
+    collectRenames(writerSchema, schema, "", renames, new HashSet<>());
+    if (renames.isEmpty()) {
+      record = HoodieAvroUtils.bytesToAvro(bytes, writerSchema, schema);
+    } else {
+      GenericRecord original = HoodieAvroUtils.bytesToAvro(bytes, 
writerSchema);
+      record = HoodieAvroUtils.rewriteRecordWithNewSchema(original, schema, 
renames);
+    }
     return Option.of(record);
   }
 
+  private static void collectRenames(Schema writer, Schema reader, String 
prefix,
+                                     Map<String, String> renames, 
Set<Pair<Schema, Schema>> visiting) {
+    if (writer.equals(reader)) {
+      return;
+    }
+    Pair<Schema, Schema> pair = Pair.of(writer, reader);
+    if (!visiting.add(pair)) {
+      return;
+    }
+    try {
+      if (writer.getType() == Schema.Type.UNION || reader.getType() == 
Schema.Type.UNION) {
+        for (Schema w : writer.getType() == Schema.Type.UNION ? 
writer.getTypes() : java.util.Collections.singletonList(writer)) {
+          for (Schema r : reader.getType() == Schema.Type.UNION ? 
reader.getTypes() : java.util.Collections.singletonList(reader)) {
+            boolean sameRecordBranch = w.getType() != Schema.Type.RECORD || 
r.getType() != Schema.Type.RECORD
+                || w.getFullName().equals(r.getFullName()) || 
r.getAliases().contains(w.getFullName())
+                || (writer.getType() != Schema.Type.UNION || 
writer.getTypes().stream().filter(s -> s.getType() == 
Schema.Type.RECORD).count() == 1)
+                && (reader.getType() != Schema.Type.UNION || 
reader.getTypes().stream().filter(s -> s.getType() == 
Schema.Type.RECORD).count() == 1);
+            if (w.getType() == r.getType() && sameRecordBranch) {
+              collectRenames(w, r, prefix, renames, visiting);
+            }
+          }
+        }
+      } else if (writer.getType() == Schema.Type.RECORD && reader.getType() == 
Schema.Type.RECORD) {
+        for (Schema.Field field : reader.getFields()) {
+          Schema.Field source = writer.getField(field.name());
+          if (source == null) {
+            for (String alias : field.aliases()) {
+              if (writer.getField(alias) != null) {
+                source = writer.getField(alias);
+                break;
+              }
+            }
+          }
+          // Preserve the historical rename contract only for a defaulted 
field replacing a removed
+          // field at the same position. Added fields and named projections 
must never shift values.
+          if (source == null && field.defaultVal() != null
+              && writer.getFields().size() == reader.getFields().size()) {
+            Schema.Field candidate = writer.getFields().get(field.pos());
+            if (reader.getField(candidate.name()) == null) {
+              source = candidate;
+            }
+          }
+          if (source == null) {
+            if (field.defaultVal() == null) {
+              throw new AvroTypeException("Field '" + prefix + field.name() + 
"' has no writer field or default");
+            }
+            continue;
+          }
+          if (!source.name().equals(field.name())) {
+            renames.put(prefix + field.name(), source.name());
+          }
+          collectRenames(source.schema(), field.schema(), prefix + 
field.name() + ".", renames, visiting);
+        }
+      } else if (writer.getType() == Schema.Type.ARRAY && reader.getType() == 
Schema.Type.ARRAY) {
+        collectRenames(writer.getElementType(), reader.getElementType(), 
prefix + "element.", renames, visiting);
+      } else if (writer.getType() == Schema.Type.MAP && reader.getType() == 
Schema.Type.MAP) {
+        collectRenames(writer.getValueType(), reader.getValueType(), prefix + 
"value.", renames, visiting);
+      }
+    } finally {
+      visiting.remove(pair);
+    }
+  }
+
+  private void writeObject(ObjectOutputStream output) throws IOException {
+    getRecordBytes();
+    output.defaultWriteObject();
+    output.writeObject(writerSchema == null ? null : 
SCHEMA_STRINGS.get(writerSchema));
+  }
+
+  private void readObject(ObjectInputStream input) throws IOException, 
ClassNotFoundException {
+    input.defaultReadObject();
+    try {
+      String schemaJson = (String) input.readObject();
+      writerSchema = schemaJson == null ? null : 
PARSED_SCHEMAS.get(schemaJson);
+    } catch (OptionalDataException e) {
+      if (!e.eof) {
+        throw e;
+      }
+      // Java-serialized payloads from before schema retention contain only 
the default fields.
+    }
+  }
+
+  /**
+   * Selects the legacy payload format for a transport that supplies the 
writer schema separately.
+   * Other Kryo instances retain a self-contained writer schema by default. 
The setting survives
+   * graph resets and does not affect Java serialization or other Kryo 
instances.
+   */
+  public static void useLegacyKryoFormat(Kryo kryo) {
+    kryo.getContext().put(KRYO_WRITE_LEGACY_FORMAT, Boolean.TRUE);
+  }
+
   @Override
   public void write(Kryo kryo, Output output) {
     byte[] bytes = getRecordBytes();
-    output.writeInt(bytes.length);
+    if (Boolean.TRUE.equals(kryo.getContext().get(KRYO_WRITE_LEGACY_FORMAT))) {
+      output.writeInt(bytes.length);
+    } else {
+      // Negative lengths distinguish schema-bearing payloads from the legacy 
non-negative format.
+      output.writeInt(-bytes.length - 1);
+      output.writeString(writerSchema == null ? null : 
SCHEMA_STRINGS.get(writerSchema));

Review Comment:
   Agreed, thanks. 6cd646b8519 removes the writer schema from the serialized 
forms instead of adding an opt-out: `write`/`read` produce master's Kryo layout 
(length, record bytes, ordering value, delete flag) and Java serialization uses 
the default form again, so `SerializationUtils`, the spill maps and Spark 
shuffles need no special setup, and `useLegacyKryoFormat` and the registrar 
change are gone. A per-stream schema table was not an option for the spill 
maps, whose entries are read back one at a time in any order.
   
   The writer schema is now kept in memory only, as a reference to the record's 
own schema. `serializedBytesOfEncodedPayloadMatchEarlierBytes` checks that the 
Kryo bytes of an encoded payload equal master's layout and its Java bytes equal 
golden bytes written by the class before this change, both before and after a 
projection request, and that the Java bytes of an encoded tombstone equal 
golden bytes; the same test passes on master. One state does change on purpose: 
a payload whose bytes were never computed used to be Java-serialized with null 
bytes and lost its record; it now writes them. The cost of this choice: a 
payload read back from a spill or a shuffle has no writer schema and decodes 
positionally with the requested schema, as on master, so the by-name fix covers 
payloads built from a record and decoded before they are serialized.
   



##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
   }
 
   protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
-    if (record != null) {
-      if (record.getSchema() == schema) {
-        return Option.of(record);
-      }
-      // if the schema does not match, we need to deserialize with the proper 
schema to match legacy behavior
-      recordBytes = getRecordBytes();
+    if (record != null && record.getSchema() == schema) {
+      return Option.of(record);
     }
-    if (recordBytes == null || recordBytes.length == 0) {
+    byte[] bytes = getRecordBytes();
+    if (bytes.length == 0) {
       return Option.empty();
     }
-    record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+    if (writerSchema == null) {
+      // Legacy serialized payloads did not include a writer schema.
+      writerSchema = schema;
+    }
+    Map<String, String> renames = new HashMap<>();
+    collectRenames(writerSchema, schema, "", renames, new HashSet<>());
+    if (renames.isEmpty()) {
+      record = HoodieAvroUtils.bytesToAvro(bytes, writerSchema, schema);
+    } else {
+      GenericRecord original = HoodieAvroUtils.bytesToAvro(bytes, 
writerSchema);
+      record = HoodieAvroUtils.rewriteRecordWithNewSchema(original, schema, 
renames);
+    }
     return Option.of(record);
   }
 
+  private static void collectRenames(Schema writer, Schema reader, String 
prefix,
+                                     Map<String, String> renames, 
Set<Pair<Schema, Schema>> visiting) {
+    if (writer.equals(reader)) {
+      return;
+    }
+    Pair<Schema, Schema> pair = Pair.of(writer, reader);
+    if (!visiting.add(pair)) {
+      return;
+    }
+    try {
+      if (writer.getType() == Schema.Type.UNION || reader.getType() == 
Schema.Type.UNION) {
+        for (Schema w : writer.getType() == Schema.Type.UNION ? 
writer.getTypes() : java.util.Collections.singletonList(writer)) {
+          for (Schema r : reader.getType() == Schema.Type.UNION ? 
reader.getTypes() : java.util.Collections.singletonList(reader)) {
+            boolean sameRecordBranch = w.getType() != Schema.Type.RECORD || 
r.getType() != Schema.Type.RECORD
+                || w.getFullName().equals(r.getFullName()) || 
r.getAliases().contains(w.getFullName())
+                || (writer.getType() != Schema.Type.UNION || 
writer.getTypes().stream().filter(s -> s.getType() == 
Schema.Type.RECORD).count() == 1)
+                && (reader.getType() != Schema.Type.UNION || 
reader.getTypes().stream().filter(s -> s.getType() == 
Schema.Type.RECORD).count() == 1);
+            if (w.getType() == r.getType() && sameRecordBranch) {
+              collectRenames(w, r, prefix, renames, visiting);
+            }
+          }
+        }
+      } else if (writer.getType() == Schema.Type.RECORD && reader.getType() == 
Schema.Type.RECORD) {
+        for (Schema.Field field : reader.getFields()) {
+          Schema.Field source = writer.getField(field.name());
+          if (source == null) {
+            for (String alias : field.aliases()) {
+              if (writer.getField(alias) != null) {
+                source = writer.getField(alias);
+                break;
+              }
+            }
+          }
+          // Preserve the historical rename contract only for a defaulted 
field replacing a removed
+          // field at the same position. Added fields and named projections 
must never shift values.
+          if (source == null && field.defaultVal() != null
+              && writer.getFields().size() == reader.getFields().size()) {

Review Comment:
   The same-position fallback now applies to a reader field only when no writer 
field matches it by name or alias, both records have the same number of fields, 
no reader field claims the writer field at that position by name or alias, and 
that writer field's schema is compatible with the reader field's schema. 
Otherwise an unmatched reader field with a default gets its default. 
`incompatibleReplacementUsesDefault` covers your example: `{id, removed: 
string}` read as `{id, added: [null, long] default null}` gives `added = null`.
   
   One more change in 6cd646b8519: an unmatched reader field without a default 
may also take the value at its position under the same conditions, because that 
is what master's positional decode gives for a renamed required field 
(`renamedRequiredFieldKeepsValueAtSamePosition`); before, it threw. With no 
candidate it still fails with an `AvroTypeException` naming the field. When the 
two records have a different number of fields the fallback does not apply, so 
an unmatched field gets its default or fails.
   
   The renames are now resolved per pair of record schemas: the writer schema 
is copied with each field named after the reader field it resolves to, and 
Avro's resolver decodes the bytes into the requested schema. Union branches 
with the same field path and recursive records each keep their own renames 
(`unionBranchesResolveRenamesIndependently`, 
`recursiveRenameKeepsNestedValues`).
   



##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
   }
 
   protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
-    if (record != null) {
-      if (record.getSchema() == schema) {
-        return Option.of(record);
-      }
-      // if the schema does not match, we need to deserialize with the proper 
schema to match legacy behavior
-      recordBytes = getRecordBytes();
+    if (record != null && record.getSchema() == schema) {
+      return Option.of(record);
     }
-    if (recordBytes == null || recordBytes.length == 0) {
+    byte[] bytes = getRecordBytes();
+    if (bytes.length == 0) {
       return Option.empty();
     }
-    record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+    if (writerSchema == null) {
+      // Legacy serialized payloads did not include a writer schema.
+      writerSchema = schema;

Review Comment:
   Good catch. 6cd646b8519 removes the inference instead of narrowing it: 
legacy bytes alone cannot tell which schema wrote them. A payload without a 
writer schema in memory, which includes every payload read back from Java or 
Kryo serialization, decodes positionally with the requested schema on every 
request, as on master. Neither the bytes nor that rule depend on earlier 
requests, so the fix no longer relies on the first request after a shuffle 
being the full schema. `deserializedPayloadDecodesEachRequestPositionally` 
requests a prefix first and then the full schema, with and without a second 
round trip, and reads the trailing value.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to