Stephen0421 commented on code in PR #10570:
URL: https://github.com/apache/paimon/pull/10570#discussion_r4307307089
##########
paimon-python/pypaimon/casting/schema_evolution.py:
##########
@@ -187,4 +187,47 @@ def cast_array_for_schema_evolution(array, target_type,
source_paimon_type=None,
if (pa.types.is_signed_integer(target_type)
and (pa.types.is_floating(array.type) or
pa.types.is_decimal(array.type))):
return _numeric_to_integer(array, target_type)
- return array.cast(target_type, safe=False)
+ try:
+ return array.cast(target_type, safe=False)
+ except pa.ArrowNotImplementedError:
+ # PyArrow 6 cannot cast a map or list as a whole when only the child
+ # binary type changes. Child arrays themselves still cast.
+ rebuilt = _cast_nested_container(array, target_type)
+ if rebuilt is None:
+ raise
+ return rebuilt
+
+
+def _cast_child(array, target_type):
+ if array.type == target_type:
+ return array
+ try:
+ return array.cast(target_type, safe=False)
+ except pa.ArrowNotImplementedError:
+ return _cast_nested_container(array, target_type)
+
+
+def _cast_nested_container(array, target_type):
+ """Rebuild a list or map after casting its children."""
+ if isinstance(array, pa.ChunkedArray):
+ chunks = [_cast_nested_container(chunk, target_type) for chunk in
array.chunks]
+ if any(chunk is None for chunk in chunks):
+ return None
+ return pa.chunked_array(chunks, type=target_type)
+ if pa.types.is_list(array.type) and pa.types.is_list(target_type):
+ values = _cast_child(array.values, target_type.value_type)
+ if values is None:
+ return None
+ return pa.ListArray.from_arrays(array.offsets, values,
mask=array.is_null())
+ if pa.types.is_large_list(array.type) and
pa.types.is_large_list(target_type):
+ values = _cast_child(array.values, target_type.value_type)
+ if values is None:
+ return None
+ return pa.LargeListArray.from_arrays(array.offsets, values,
mask=array.is_null())
+ if pa.types.is_map(array.type) and pa.types.is_map(target_type):
+ keys = _cast_child(array.keys, target_type.key_type)
+ items = _cast_child(array.items, target_type.item_type)
+ if keys is None or items is None:
+ return None
+ return pa.MapArray.from_arrays(array.offsets, keys, items,
mask=array.is_null())
Review Comment:
Fixed in 556033789.
The nested fallback no longer uses `MapArray.offsets` or
`MapArray.from_arrays(..., mask=)`. Offsets are copied from `buffers()[1]` at
`array.offset`, including slices, and a null map is a null entry in that offset
array. The numeric offsets stay in place so neighboring rows still index the
child arrays.
`test_map_binary_values_rebuild_when_container_cast_is_missing` covers a
null map, a list rebuild, and a sliced map whose offsets are `[1, 1, 3]`.
Python 3.6.15 CI now passes both mixed tests: `test_py_write_managed_blob` and
`test_read_python_managed_blob_after_java_gc`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
