leaves12138 commented on code in PR #10508:
URL: https://github.com/apache/paimon/pull/10508#discussion_r4244336368
##########
paimon-python/pypaimon/casting/schema_evolution.py:
##########
@@ -77,19 +77,60 @@ def _truncate_timestamp(array, target_type,
source_paimon_type, target_paimon_ty
return pa.array(values, type=pa.int64()).cast(target_type)
+def _timestamp_to_bigint(array):
+ ticks = {'s': 1, 'ms': 1000, 'us': 1000000, 'ns':
1000000000}[array.type.unit]
+ values = []
+ for value in array.cast(pa.int64()).to_pylist():
+ if value is None:
+ values.append(None)
+ elif array.type.tz is None:
+ values.append(value // ticks)
+ else:
+ # Java's LTZ rule round-trips Timestamp through LocalDateTime.
+ # Timestamp stores its epoch day as an int in that conversion.
+ millis = value * 1000 if array.type.unit == 's' else value //
(ticks // 1000)
+ if not -(1 << 63) <= millis < (1 << 63):
+ values.append(None)
+ continue
+ day = _narrow_integer(millis // 86400000, 32)
+ values.append((day * 86400000 + millis % 86400000) // 1000)
+ return pa.array(values, type=pa.int64())
+
+
+def _bigint_to_timestamp(array, target_type):
+ ticks = {'s': 1, 'ms': 1000, 'us': 1000000, 'ns':
1000000000}[target_type.unit]
+ values = []
+ for value in array.to_pylist():
+ if value is None:
+ values.append(None)
+ continue
+ # NumericPrimitiveToTimestamp performs value.longValue() * 1000
+ # before conversion. Java's signed long multiplication wraps.
+ millis = _narrow_integer(value * 1000, 64)
+ value = millis // 1000 if target_type.unit == 's' else millis * (ticks
// 1000)
+ values.append(value if -(1 << 63) <= value < (1 << 63) else None)
+ return pa.array(values, type=pa.int64()).cast(target_type)
+
+
def cast_array_for_schema_evolution(array, target_type,
source_paimon_type=None, target_paimon_type=None):
"""Materialize existing type changes using Java numeric and timestamp
casts.
Declared Paimon types are needed for precision changes within one Arrow
- timestamp unit. Numeric narrowing preserves low bits, while floating-point
- values truncate and saturate to int/long before byte/short narrowing.
+ timestamp unit. BIGINT/timestamp conversions use epoch seconds, including
+ pre-epoch flooring and Java's millis multiplication. Numeric narrowing
+ preserves low bits, while floating-point values truncate and saturate to
+ int/long before byte/short narrowing.
DECIMAL scale reduction uses HALF_UP and returns NULL on precision
overflow.
"""
timestamp = _truncate_timestamp(array, target_type, source_paimon_type,
target_paimon_type)
if timestamp is not None:
return timestamp
if array.type == target_type:
return array
+ if pa.types.is_timestamp(array.type) and target_type == pa.int64():
+ return _timestamp_to_bigint(array)
+ if array.type == pa.int64() and pa.types.is_timestamp(target_type) and
target_type.tz is None:
Review Comment:
This condition only handles `pa.int64()`, but Java
`NumericPrimitiveToTimestamp.create()` supports both `INTEGER` and `BIGINT`.
Paimon `INT` is represented as `pa.int32()`, so an existing `INT ->
TIMESTAMP(...)` evolution falls through to `array.cast(target_type)` and
PyArrow raises `ArrowNotImplementedError` (`Unsupported cast from int32 to
timestamp`). Please include the `pa.int32()` source path (and add a regression
case for INT -> TIMESTAMP) so this matches the Rust/Java behavior.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
