ravjot28 commented on code in PR #19872:
URL: https://github.com/apache/hudi/pull/19872#discussion_r4540327282
##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
}
protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
- if (record != null) {
- if (record.getSchema() == schema) {
- return Option.of(record);
- }
- // if the schema does not match, we need to deserialize with the proper
schema to match legacy behavior
- recordBytes = getRecordBytes();
+ if (record != null && record.getSchema() == schema) {
+ return Option.of(record);
}
- if (recordBytes == null || recordBytes.length == 0) {
+ byte[] bytes = getRecordBytes();
+ if (bytes.length == 0) {
return Option.empty();
}
- record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+ if (writerSchema == null) {
+ // Legacy serialized payloads did not include a writer schema.
+ writerSchema = schema;
+ }
+ Map<String, String> renames = new HashMap<>();
+ collectRenames(writerSchema, schema, "", renames, new HashSet<>());
+ if (renames.isEmpty()) {
+ record = HoodieAvroUtils.bytesToAvro(bytes, writerSchema, schema);
+ } else {
+ GenericRecord original = HoodieAvroUtils.bytesToAvro(bytes,
writerSchema);
+ record = HoodieAvroUtils.rewriteRecordWithNewSchema(original, schema,
renames);
+ }
return Option.of(record);
}
+ private static void collectRenames(Schema writer, Schema reader, String
prefix,
+ Map<String, String> renames,
Set<Pair<Schema, Schema>> visiting) {
+ if (writer.equals(reader)) {
+ return;
+ }
+ Pair<Schema, Schema> pair = Pair.of(writer, reader);
+ if (!visiting.add(pair)) {
+ return;
+ }
+ try {
+ if (writer.getType() == Schema.Type.UNION || reader.getType() ==
Schema.Type.UNION) {
+ for (Schema w : writer.getType() == Schema.Type.UNION ?
writer.getTypes() : java.util.Collections.singletonList(writer)) {
+ for (Schema r : reader.getType() == Schema.Type.UNION ?
reader.getTypes() : java.util.Collections.singletonList(reader)) {
+ boolean sameRecordBranch = w.getType() != Schema.Type.RECORD ||
r.getType() != Schema.Type.RECORD
+ || w.getFullName().equals(r.getFullName()) ||
r.getAliases().contains(w.getFullName())
+ || (writer.getType() != Schema.Type.UNION ||
writer.getTypes().stream().filter(s -> s.getType() ==
Schema.Type.RECORD).count() == 1)
+ && (reader.getType() != Schema.Type.UNION ||
reader.getTypes().stream().filter(s -> s.getType() ==
Schema.Type.RECORD).count() == 1);
+ if (w.getType() == r.getType() && sameRecordBranch) {
+ collectRenames(w, r, prefix, renames, visiting);
+ }
+ }
+ }
+ } else if (writer.getType() == Schema.Type.RECORD && reader.getType() ==
Schema.Type.RECORD) {
+ for (Schema.Field field : reader.getFields()) {
+ Schema.Field source = writer.getField(field.name());
+ if (source == null) {
+ for (String alias : field.aliases()) {
+ if (writer.getField(alias) != null) {
+ source = writer.getField(alias);
+ break;
+ }
+ }
+ }
+ // Preserve the historical rename contract only for a defaulted
field replacing a removed
+ // field at the same position. Added fields and named projections
must never shift values.
+ if (source == null && field.defaultVal() != null
+ && writer.getFields().size() == reader.getFields().size()) {
+ Schema.Field candidate = writer.getFields().get(field.pos());
+ if (reader.getField(candidate.name()) == null) {
+ source = candidate;
+ }
+ }
+ if (source == null) {
+ if (field.defaultVal() == null) {
+ throw new AvroTypeException("Field '" + prefix + field.name() +
"' has no writer field or default");
+ }
+ continue;
+ }
+ if (!source.name().equals(field.name())) {
+ renames.put(prefix + field.name(), source.name());
+ }
+ collectRenames(source.schema(), field.schema(), prefix +
field.name() + ".", renames, visiting);
+ }
+ } else if (writer.getType() == Schema.Type.ARRAY && reader.getType() ==
Schema.Type.ARRAY) {
+ collectRenames(writer.getElementType(), reader.getElementType(),
prefix + "element.", renames, visiting);
+ } else if (writer.getType() == Schema.Type.MAP && reader.getType() ==
Schema.Type.MAP) {
+ collectRenames(writer.getValueType(), reader.getValueType(), prefix +
"value.", renames, visiting);
+ }
+ } finally {
+ visiting.remove(pair);
+ }
+ }
+
+ private void writeObject(ObjectOutputStream output) throws IOException {
+ getRecordBytes();
+ output.defaultWriteObject();
+ output.writeObject(writerSchema == null ? null :
SCHEMA_STRINGS.get(writerSchema));
+ }
+
+ private void readObject(ObjectInputStream input) throws IOException,
ClassNotFoundException {
+ input.defaultReadObject();
+ try {
+ String schemaJson = (String) input.readObject();
+ writerSchema = schemaJson == null ? null :
PARSED_SCHEMAS.get(schemaJson);
+ } catch (OptionalDataException e) {
+ if (!e.eof) {
+ throw e;
+ }
+ // Java-serialized payloads from before schema retention contain only
the default fields.
+ }
+ }
+
+ /**
+ * Selects the legacy payload format for a transport that supplies the
writer schema separately.
+ * Other Kryo instances retain a self-contained writer schema by default.
The setting survives
+ * graph resets and does not affect Java serialization or other Kryo
instances.
+ */
+ public static void useLegacyKryoFormat(Kryo kryo) {
+ kryo.getContext().put(KRYO_WRITE_LEGACY_FORMAT, Boolean.TRUE);
+ }
+
@Override
public void write(Kryo kryo, Output output) {
byte[] bytes = getRecordBytes();
- output.writeInt(bytes.length);
+ if (Boolean.TRUE.equals(kryo.getContext().get(KRYO_WRITE_LEGACY_FORMAT))) {
+ output.writeInt(bytes.length);
+ } else {
+ // Negative lengths distinguish schema-bearing payloads from the legacy
non-negative format.
+ output.writeInt(-bytes.length - 1);
+ output.writeString(writerSchema == null ? null :
SCHEMA_STRINGS.get(writerSchema));
Review Comment:
Agreed, thanks. 6cd646b8519 removes the writer schema from the serialized
forms instead of adding an opt-out: `write`/`read` produce master's Kryo layout
(length, record bytes, ordering value, delete flag) and Java serialization uses
the default form again, so `SerializationUtils`, the spill maps and Spark
shuffles need no special setup, and `useLegacyKryoFormat` and the registrar
change are gone. A per-stream schema table was not an option for the spill
maps, whose entries are read back one at a time in any order.
The writer schema is now kept in memory only, as a reference to the record's
own schema. `serializedBytesOfEncodedPayloadMatchEarlierBytes` checks that the
Kryo bytes of an encoded payload equal master's layout and its Java bytes equal
golden bytes written by the class before this change, both before and after a
projection request, and that the Java bytes of an encoded tombstone equal
golden bytes; the same test passes on master. One state does change on purpose:
a payload whose bytes were never computed used to be Java-serialized with null
bytes and lost its record; it now writes them. The cost of this choice: a
payload read back from a spill or a shuffle has no writer schema and decodes
positionally with the requested schema, as on master, so the by-name fix covers
payloads built from a record and decoded before they are serialized.
##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
}
protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
- if (record != null) {
- if (record.getSchema() == schema) {
- return Option.of(record);
- }
- // if the schema does not match, we need to deserialize with the proper
schema to match legacy behavior
- recordBytes = getRecordBytes();
+ if (record != null && record.getSchema() == schema) {
+ return Option.of(record);
}
- if (recordBytes == null || recordBytes.length == 0) {
+ byte[] bytes = getRecordBytes();
+ if (bytes.length == 0) {
return Option.empty();
}
- record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+ if (writerSchema == null) {
+ // Legacy serialized payloads did not include a writer schema.
+ writerSchema = schema;
+ }
+ Map<String, String> renames = new HashMap<>();
+ collectRenames(writerSchema, schema, "", renames, new HashSet<>());
+ if (renames.isEmpty()) {
+ record = HoodieAvroUtils.bytesToAvro(bytes, writerSchema, schema);
+ } else {
+ GenericRecord original = HoodieAvroUtils.bytesToAvro(bytes,
writerSchema);
+ record = HoodieAvroUtils.rewriteRecordWithNewSchema(original, schema,
renames);
+ }
return Option.of(record);
}
+ private static void collectRenames(Schema writer, Schema reader, String
prefix,
+ Map<String, String> renames,
Set<Pair<Schema, Schema>> visiting) {
+ if (writer.equals(reader)) {
+ return;
+ }
+ Pair<Schema, Schema> pair = Pair.of(writer, reader);
+ if (!visiting.add(pair)) {
+ return;
+ }
+ try {
+ if (writer.getType() == Schema.Type.UNION || reader.getType() ==
Schema.Type.UNION) {
+ for (Schema w : writer.getType() == Schema.Type.UNION ?
writer.getTypes() : java.util.Collections.singletonList(writer)) {
+ for (Schema r : reader.getType() == Schema.Type.UNION ?
reader.getTypes() : java.util.Collections.singletonList(reader)) {
+ boolean sameRecordBranch = w.getType() != Schema.Type.RECORD ||
r.getType() != Schema.Type.RECORD
+ || w.getFullName().equals(r.getFullName()) ||
r.getAliases().contains(w.getFullName())
+ || (writer.getType() != Schema.Type.UNION ||
writer.getTypes().stream().filter(s -> s.getType() ==
Schema.Type.RECORD).count() == 1)
+ && (reader.getType() != Schema.Type.UNION ||
reader.getTypes().stream().filter(s -> s.getType() ==
Schema.Type.RECORD).count() == 1);
+ if (w.getType() == r.getType() && sameRecordBranch) {
+ collectRenames(w, r, prefix, renames, visiting);
+ }
+ }
+ }
+ } else if (writer.getType() == Schema.Type.RECORD && reader.getType() ==
Schema.Type.RECORD) {
+ for (Schema.Field field : reader.getFields()) {
+ Schema.Field source = writer.getField(field.name());
+ if (source == null) {
+ for (String alias : field.aliases()) {
+ if (writer.getField(alias) != null) {
+ source = writer.getField(alias);
+ break;
+ }
+ }
+ }
+ // Preserve the historical rename contract only for a defaulted
field replacing a removed
+ // field at the same position. Added fields and named projections
must never shift values.
+ if (source == null && field.defaultVal() != null
+ && writer.getFields().size() == reader.getFields().size()) {
Review Comment:
The same-position fallback now applies to a reader field only when no writer
field matches it by name or alias, both records have the same number of fields,
no reader field claims the writer field at that position by name or alias, and
that writer field's schema is compatible with the reader field's schema.
Otherwise an unmatched reader field with a default gets its default.
`incompatibleReplacementUsesDefault` covers your example: `{id, removed:
string}` read as `{id, added: [null, long] default null}` gives `added = null`.
One more change in 6cd646b8519: an unmatched reader field without a default
may also take the value at its position under the same conditions, because that
is what master's positional decode gives for a renamed required field
(`renamedRequiredFieldKeepsValueAtSamePosition`); before, it threw. With no
candidate it still fails with an `AvroTypeException` naming the field. When the
two records have a different number of fields the fallback does not apply, so
an unmatched field gets its default or fails.
The renames are now resolved per pair of record schemas: the writer schema
is copied with each field named after the reader field it resolves to, and
Avro's resolver decodes the bytes into the requested schema. Union branches
with the same field path and recursive records each keep their own renames
(`unionBranchesResolveRenamesIndependently`,
`recursiveRenameKeepsNestedValues`).
##########
hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java:
##########
@@ -126,24 +154,129 @@ protected boolean isEmptyRecord() {
}
protected Option<IndexedRecord> getRecord(Schema schema) throws IOException {
- if (record != null) {
- if (record.getSchema() == schema) {
- return Option.of(record);
- }
- // if the schema does not match, we need to deserialize with the proper
schema to match legacy behavior
- recordBytes = getRecordBytes();
+ if (record != null && record.getSchema() == schema) {
+ return Option.of(record);
}
- if (recordBytes == null || recordBytes.length == 0) {
+ byte[] bytes = getRecordBytes();
+ if (bytes.length == 0) {
return Option.empty();
}
- record = SerializableIndexedRecord.fromAvroBytes(schema, recordBytes);
+ if (writerSchema == null) {
+ // Legacy serialized payloads did not include a writer schema.
+ writerSchema = schema;
Review Comment:
Good catch. 6cd646b8519 removes the inference instead of narrowing it:
legacy bytes alone cannot tell which schema wrote them. A payload without a
writer schema in memory, which includes every payload read back from Java or
Kryo serialization, decodes positionally with the requested schema on every
request, as on master. Neither the bytes nor that rule depend on earlier
requests, so the fix no longer relies on the first request after a shuffle
being the full schema. `deserializedPayloadDecodesEachRequestPositionally`
requests a prefix first and then the full schema, with and without a second
round trip, and reads the trailing value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]