Fokko commented on code in PR #40:
URL: https://github.com/apache/iceberg-python/pull/40#discussion_r1351918311
##########
pyiceberg/avro/resolver.py:
##########
@@ -233,7 +255,107 @@ def skip(self, decoder: BinaryDecoder) -> None:
pass
-class SchemaResolver(PrimitiveWithPartnerVisitor[IcebergType, Reader]):
+class WriteSchemaResolver(PrimitiveWithPartnerVisitor[IcebergType, Writer]):
+ def schema(self, schema: Schema, expected_schema: Optional[IcebergType],
result: Writer) -> Writer:
+ return result
+
+ def struct(self, struct: StructType, provided_struct:
Optional[IcebergType], field_writers: List[Writer]) -> Writer:
+ if not isinstance(provided_struct, StructType):
+ raise ResolveError(f"File/write schema are not aligned for struct,
got {provided_struct}")
+
+ provided_struct_positions: Dict[int, int] = {field.field_id: pos for
pos, field in enumerate(provided_struct.fields)}
+
+ results: List[Tuple[Optional[int], Writer]] = []
+ iter(field_writers)
+
+ for pos, write_field in enumerate(struct.fields):
+ if write_field.field_id in provided_struct_positions:
+
results.append((provided_struct_positions[write_field.field_id],
field_writers[pos]))
+ else:
+ # There is a default value
+ if isinstance(write_field, NestedField) and
write_field.write_default is not None:
+ # The field is not in the record, but there is a write
default value
+ default_writer = DefaultWriter(
+ writer=visit(write_field.field_type,
CONSTRUCT_WRITER_VISITOR), value=write_field.write_default
Review Comment:
@rdblue Just to clarify, the type annotation here is not just a hint, it
will be enforced by Pydantic. If you pass in something other than what the type
allows, it will raise a Pydantic `ValidationError`. An assertion would be
similar (but then it would be done in Python land instead of Rust 🦀 ).
##########
pyiceberg/avro/resolver.py:
##########
@@ -233,7 +255,107 @@ def skip(self, decoder: BinaryDecoder) -> None:
pass
-class SchemaResolver(PrimitiveWithPartnerVisitor[IcebergType, Reader]):
+class WriteSchemaResolver(PrimitiveWithPartnerVisitor[IcebergType, Writer]):
+ def schema(self, schema: Schema, expected_schema: Optional[IcebergType],
result: Writer) -> Writer:
+ return result
+
+ def struct(self, struct: StructType, provided_struct:
Optional[IcebergType], field_writers: List[Writer]) -> Writer:
+ if not isinstance(provided_struct, StructType):
+ raise ResolveError(f"File/write schema are not aligned for struct,
got {provided_struct}")
+
+ provided_struct_positions: Dict[int, int] = {field.field_id: pos for
pos, field in enumerate(provided_struct.fields)}
+
+ results: List[Tuple[Optional[int], Writer]] = []
+ iter(field_writers)
+
+ for pos, write_field in enumerate(struct.fields):
+ if write_field.field_id in provided_struct_positions:
+
results.append((provided_struct_positions[write_field.field_id],
field_writers[pos]))
+ else:
+ # There is a default value
+ if isinstance(write_field, NestedField) and
write_field.write_default is not None:
+ # The field is not in the record, but there is a write
default value
+ default_writer = DefaultWriter(
+ writer=visit(write_field.field_type,
CONSTRUCT_WRITER_VISITOR), value=write_field.write_default
Review Comment:
@rdblue Just to clarify, the type annotation here is not just a hint, it
will be enforced by Pydantic. If you pass in something other than what the type
allows, it will raise a Pydantic `ValidationError`. An assertion would be
similar (but then it would be done in Python land instead of Rust 🦀 ).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
