This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 6aa60c5c Align temporal string schema evolution with Java (#1115)
6aa60c5c is described below
commit 6aa60c5c43fc124eeac0ce4452b2dcea6f0b16a8
Author: Jingsong Lee <[email protected]>
AuthorDate: Tue Oct 13 07:20:50 2026 +0800
Align temporal string schema evolution with Java (#1115)
---
crates/paimon/src/arrow/evolution_string.rs | 429 ++++++++++++++++++++++++++++
crates/paimon/src/arrow/mod.rs | 1 +
crates/paimon/src/arrow/nested_evolution.rs | 48 +++-
crates/paimon/src/common/date_time.rs | 134 +++++++++
crates/paimon/src/common/mod.rs | 1 +
crates/paimon/src/spec/partition_utils.rs | 94 ++----
6 files changed, 621 insertions(+), 86 deletions(-)
diff --git a/crates/paimon/src/arrow/evolution_string.rs
b/crates/paimon/src/arrow/evolution_string.rs
new file mode 100644
index 00000000..ee5207d9
--- /dev/null
+++ b/crates/paimon/src/arrow/evolution_string.rs
@@ -0,0 +1,429 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Temporal character casts: Java formatting, local zone and CHAR/VARCHAR
length.
+
+use std::sync::Arc;
+
+use arrow_array::{cast::AsArray, types::Int64Type, ArrayRef, StringArray,
Time32MillisecondArray};
+use arrow_schema::{DataType as ArrowType, TimeUnit};
+use jiff::tz::TimeZone;
+
+use crate::common::date_time::{format_epoch_timestamp, format_time};
+use crate::spec::DataType;
+
+/// Return None when this is not a temporal-to-character conversion.
+pub(super) fn temporal_to_string(
+ source: &ArrayRef,
+ source_type: &DataType,
+ target_type: &DataType,
+) -> crate::Result<Option<ArrayRef>> {
+ let (length, pad) = match target_type {
+ DataType::Char(t) => (t.length(), true),
+ DataType::VarChar(t) => (t.length() as usize, false),
+ _ => return Ok(None),
+ };
+ let values = match source_type {
+ DataType::Time(t) => {
+ let millis = arrow_cast::cast(source,
&ArrowType::Time32(TimeUnit::Millisecond))
+ .map_err(|error| invalid(error.to_string()))?;
+ let millis = millis
+ .as_any()
+ .downcast_ref::<Time32MillisecondArray>()
+ .ok_or_else(|| invalid("Expected TIME array"))?;
+ millis
+ .iter()
+ .map(|value| value.map(|value| format_time(value,
t.precision())))
+ .collect::<Vec<_>>()
+ }
+ DataType::Timestamp(_) | DataType::LocalZonedTimestamp(_) => {
+ let (precision, local) = match source_type {
+ DataType::Timestamp(t) => (t.precision(), false),
+ DataType::LocalZonedTimestamp(t) => (t.precision(), true),
+ _ => unreachable!(),
+ };
+ let ArrowType::Timestamp(unit, _) = source.data_type() else {
+ return Err(invalid("Expected TIMESTAMP array"));
+ };
+ let zone = local.then(TimeZone::system);
+ let raw = arrow_cast::cast(source, &ArrowType::Int64)
+ .map_err(|error| invalid(error.to_string()))?;
+ raw.as_primitive::<Int64Type>()
+ .iter()
+ .map(|value| {
+ value.map(|value| render_timestamp(value, unit, precision,
zone.as_ref()))
+ })
+ .collect::<Vec<_>>()
+ }
+ _ => return Ok(None),
+ };
+ // The temporal representation is ASCII. Java toCharacterString counts
+ // characters, truncates both string kinds, and pads only CHAR.
+ let strings = values.into_iter().map(|value| {
+ value.map(|mut value| {
+ value.truncate(length);
+ if pad && value.len() < length {
+ value.extend(std::iter::repeat_n(' ', length - value.len()));
+ }
+ value
+ })
+ });
+ Ok(Some(Arc::new(StringArray::from_iter(strings))))
+}
+
+fn render_timestamp(
+ value: i64,
+ unit: &TimeUnit,
+ precision: u32,
+ zone: Option<&TimeZone>,
+) -> String {
+ let millis = match unit {
+ TimeUnit::Second => value.wrapping_mul(1000),
+ TimeUnit::Millisecond => value,
+ TimeUnit::Microsecond => value.div_euclid(1000),
+ TimeUnit::Nanosecond => value.div_euclid(1_000_000),
+ };
+ let sub_milli = match unit {
+ TimeUnit::Microsecond => value.rem_euclid(1000) * 1000,
+ TimeUnit::Nanosecond => value.rem_euclid(1_000_000),
+ _ => 0,
+ };
+ let millis = match zone {
+ Some(zone) => {
+ let offset = local_offset(millis.div_euclid(1000), zone);
+ // Timestamp.fromLocalDateTime uses signed long millisecond
storage.
+ // Applying an offset can wrap and change the millisecond fraction.
+ millis.wrapping_add(i64::from(offset) * 1000)
+ }
+ None => millis,
+ };
+ let nanos = (millis.rem_euclid(1000) * 1_000_000 + sub_milli) as u32;
+ format_epoch_timestamp(millis.div_euclid(1000), nanos, precision)
+}
+
+fn local_offset(seconds: i64, zone: &TimeZone) -> i32 {
+ let timestamp = jiff::Timestamp::from_second(seconds).unwrap_or_else(|_| {
+ if seconds < 0 {
+ // Java ZoneRules uses its first wall offset before recorded
history.
+ jiff::Timestamp::MIN
+ } else {
+ // Java's final transition rules repeat with the Gregorian
calendar:
+ // 400 years contain 146097 days, a whole number of weeks. Look up
+ // the same month/day/weekday/time in 2370..2770, after historical
+ // transitions, while retaining the original millis for formatting.
+ const CYCLE_SECONDS: i64 = 146_097 * 86_400;
+ jiff::Timestamp::from_second(seconds.rem_euclid(CYCLE_SECONDS) +
CYCLE_SECONDS)
+ .expect("a projected Gregorian cycle is within jiff's
timestamp range")
+ }
+ });
+ zone.to_offset(timestamp).seconds()
+}
+
+fn invalid(message: impl Into<String>) -> crate::Error {
+ crate::Error::DataInvalid {
+ message: message.into(),
+ source: None,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::arrow::nested_evolution::evolve_column;
+ use crate::spec::{CharType, TimeType, TimestampType, VarCharType};
+ use arrow_array::{Array, Int64Array, TimestampNanosecondArray};
+
+ fn strings(source: ArrayRef, source_type: DataType, target: DataType) ->
StringArray {
+ evolve_column(&source, &source_type, &target)
+ .unwrap()
+ .as_any()
+ .downcast_ref::<StringArray>()
+ .unwrap()
+ .clone()
+ }
+
+ #[test]
+ fn time_string_matches_java_fraction_loop_and_negative_normalization() {
+ let source: ArrayRef = Arc::new(Time32MillisecondArray::from(vec![
+ None,
+ Some(0),
+ Some(100),
+ Some(101),
+ Some(-1),
+ None,
+ Some(0),
+ ]));
+ for (precision, expected) in [
+ (0, ["00:00:00", "00:00:00", "00:00:00", "23:59:59"]),
+ (1, ["00:00:00.0", "00:00:00.1", "00:00:00.1", "23:59:59.9"]),
+ (
+ 2,
+ ["00:00:00.0", "00:00:00.1", "00:00:00.10", "23:59:59.99"],
+ ),
+ (
+ 3,
+ ["00:00:00.0", "00:00:00.1", "00:00:00.101", "23:59:59.999"],
+ ),
+ (
+ 9,
+ ["00:00:00.0", "00:00:00.1", "00:00:00.101", "23:59:59.999"],
+ ),
+ ] {
+ let actual = strings(
+ source.slice(1, 5),
+ DataType::Time(TimeType::new(precision).unwrap()),
+ DataType::VarChar(VarCharType::string_type()),
+ );
+ assert_eq!(
+ actual,
+
StringArray::from_iter(expected.into_iter().map(Some).chain([None]))
+ );
+ }
+ }
+
+ #[test]
+ fn timestamp_string_keeps_nanos_and_minimum_fraction_width() {
+ let source: ArrayRef = Arc::new(TimestampNanosecondArray::from(vec![
+ None,
+ Some(0),
+ Some(123_456_789),
+ Some(-1),
+ None,
+ Some(0),
+ ]));
+ for (precision, whole) in [
+ (0, "1970-01-01 00:00:00"),
+ (1, "1970-01-01 00:00:00.0"),
+ (3, "1970-01-01 00:00:00.000"),
+ (6, "1970-01-01 00:00:00.000000"),
+ (9, "1970-01-01 00:00:00.000000000"),
+ ] {
+ let actual = strings(
+ source.slice(1, 4),
+ DataType::Timestamp(TimestampType::new(precision).unwrap()),
+ DataType::VarChar(VarCharType::string_type()),
+ );
+ assert_eq!(
+ actual,
+ StringArray::from(vec![
+ Some(whole),
+ Some("1970-01-01 00:00:00.123456789"),
+ Some("1969-12-31 23:59:59.999999999"),
+ None
+ ])
+ );
+ }
+ }
+
+ #[test]
+ fn timestamp_string_handles_all_physical_units_and_character_lengths() {
+ for (unit, ticks) in [
+ (TimeUnit::Second, 1),
+ (TimeUnit::Millisecond, 1_000),
+ (TimeUnit::Microsecond, 1_000_000),
+ (TimeUnit::Nanosecond, 1_000_000_000),
+ ] {
+ let raw: ArrayRef = Arc::new(Int64Array::from(vec![Some(ticks),
Some(-ticks), None]));
+ let source = arrow_cast::cast(&raw, &ArrowType::Timestamp(unit,
None)).unwrap();
+ for (target, first, second) in [
+ (
+ DataType::VarChar(VarCharType::string_type()),
+ "1970-01-01 00:00:01.000",
+ "1969-12-31 23:59:59.000",
+ ),
+ (
+ DataType::VarChar(VarCharType::new(12).unwrap()),
+ "1970-01-01 0",
+ "1969-12-31 2",
+ ),
+ (
+ DataType::Char(CharType::new(25).unwrap()),
+ "1970-01-01 00:00:01.000 ",
+ "1969-12-31 23:59:59.000 ",
+ ),
+ (DataType::Char(CharType::new(4).unwrap()), "1970", "1969"),
+ ] {
+ let result = strings(
+ source.clone(),
+ DataType::Timestamp(TimestampType::new(3).unwrap()),
+ target,
+ );
+ assert_eq!(
+ result,
+ StringArray::from(vec![Some(first), Some(second), None])
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn timestamp_string_uses_java_four_digits_outside_datetime_year_range() {
+ use arrow_array::TimestampMillisecondArray;
+ let source: ArrayRef = Arc::new(TimestampMillisecondArray::from(vec![
+ Some(253_402_300_800_000),
+ Some(-62_167_219_200_000),
+ Some(-62_198_755_200_000),
+ Some(i64::MAX),
+ Some(i64::MIN),
+ None,
+ ]));
+ let actual = strings(
+ source,
+ DataType::Timestamp(TimestampType::new(3).unwrap()),
+ DataType::VarChar(VarCharType::string_type()),
+ );
+ assert_eq!(
+ actual,
+ StringArray::from(vec![
+ Some("0000-01-01 00:00:00.000"),
+ Some("0000-01-01 00:00:00.000"),
+ Some("000/-01-01 00:00:00.000"),
+ Some("/+-.-04-06 07:12:55.807"),
+ Some("5471-09-26 16:47:04.192"),
+ None
+ ])
+ );
+ }
+
+ #[test]
+ fn timestamp_seconds_wrapping_keeps_wrapped_millisecond_fraction() {
+ use arrow_array::TimestampSecondArray;
+ let source: ArrayRef = Arc::new(TimestampSecondArray::from(vec![
+ Some(9_223_372_036_854_776),
+ None,
+ ]));
+ let actual = strings(
+ source,
+ DataType::Timestamp(TimestampType::new(3).unwrap()),
+ DataType::VarChar(VarCharType::string_type()),
+ );
+ assert_eq!(
+ actual,
+ StringArray::from(vec![Some("5471-09-26 16:47:04.384"), None])
+ );
+ }
+
+ #[test]
+ fn local_timestamp_string_extreme_millis_does_not_fail_the_batch() {
+ use crate::spec::LocalZonedTimestampType;
+ use arrow_array::TimestampMillisecondArray;
+ let source: ArrayRef = Arc::new(
+ TimestampMillisecondArray::from(vec![Some(0), Some(i64::MAX),
None, Some(i64::MIN)])
+ .with_timezone("UTC"),
+ );
+ let result = evolve_column(
+ &source,
+
&DataType::LocalZonedTimestamp(LocalZonedTimestampType::new(3).unwrap()),
+ &DataType::VarChar(VarCharType::string_type()),
+ );
+ let result = result.expect("LTZ conversion must not fail on a non-NULL
extreme value");
+ assert_eq!(result.len(), 4);
+ assert_eq!(result.null_count(), 1);
+ assert!(result.is_null(2));
+ assert!(result.is_valid(0) && result.is_valid(1) &&
result.is_valid(3));
+ }
+
+ #[test]
+ fn local_timestamp_strings_match_java_extremes_and_zone_rules() {
+ // Golden strings from Java CastExecutors in these explicitly selected
+ // zones, including far-future DST and historical offsets with seconds.
+ let values = [
+ i64::MAX,
+ i64::MIN,
+ i64::MAX - 181 * 86_400_000,
+ 1_704_067_200_000,
+ 1_719_792_000_000,
+ ];
+ for (name, expected) in [
+ (
+ "UTC",
+ [
+ "/+-.-04-06 07:12:55.807",
+ "5471-09-26 16:47:04.192",
+ "/+---10-08 07:12:55.807",
+ "2024-01-01 00:00:00.000",
+ "2024-07-01 00:00:00.000",
+ ],
+ ),
+ (
+ "Asia/Shanghai",
+ [
+ "5471-09-27 00:47:04.191",
+ "5471-09-27 00:52:47.192",
+ "/+---10-08 15:12:55.807",
+ "2024-01-01 08:00:00.000",
+ "2024-07-01 08:00:00.000",
+ ],
+ ),
+ (
+ "America/New_York",
+ [
+ "/+-.-04-06 03:12:55.807",
+ "/+-.-04-06 02:16:53.808",
+ "/+---10-08 02:12:55.807",
+ "2023-12-31 19:00:00.000",
+ "2024-06-30 20:00:00.000",
+ ],
+ ),
+ (
+ "Australia/Sydney",
+ [
+ "5471-09-27 02:47:04.191",
+ "5471-09-27 02:51:56.192",
+ "/+---10-08 18:12:55.807",
+ "2024-01-01 11:00:00.000",
+ "2024-07-01 10:00:00.000",
+ ],
+ ),
+ ] {
+ let zone = TimeZone::get(name).unwrap();
+ for (value, expected) in values.into_iter().zip(expected) {
+ assert_eq!(
+ render_timestamp(value, &TimeUnit::Millisecond, 3,
Some(&zone)),
+ expected,
+ "{value} in {name}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn local_timestamp_preserves_sub_millisecond_digits_in_all_ranges() {
+ let zone = TimeZone::get("America/New_York").unwrap();
+ for (value, unit, precision, expected) in [
+ (
+ i64::MAX,
+ TimeUnit::Microsecond,
+ 6,
+ "4247-01-09 23:00:54.775807",
+ ),
+ (-1, TimeUnit::Microsecond, 6, "1969-12-31 18:59:59.999999"),
+ (
+ i64::MAX,
+ TimeUnit::Nanosecond,
+ 9,
+ "2262-04-11 19:47:16.854775807",
+ ),
+ (-1, TimeUnit::Nanosecond, 9, "1969-12-31 18:59:59.999999999"),
+ ] {
+ assert_eq!(
+ render_timestamp(value, &unit, precision, Some(&zone)),
+ expected
+ );
+ }
+ }
+}
diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs
index 2ef79ced..06f9d4f7 100644
--- a/crates/paimon/src/arrow/mod.rs
+++ b/crates/paimon/src/arrow/mod.rs
@@ -16,6 +16,7 @@
// under the License.
mod evolution_cast;
+mod evolution_string;
pub(crate) mod filtering;
pub(crate) mod format;
pub(crate) mod nested_evolution;
diff --git a/crates/paimon/src/arrow/nested_evolution.rs
b/crates/paimon/src/arrow/nested_evolution.rs
index 883b89c3..2a79f6e3 100644
--- a/crates/paimon/src/arrow/nested_evolution.rs
+++ b/crates/paimon/src/arrow/nested_evolution.rs
@@ -204,6 +204,12 @@ pub(crate) fn evolve_column(
_ => {}
}
+ if let Some(result) =
+ super::evolution_string::temporal_to_string(source, source_type,
target_type)?
+ {
+ return Ok(result);
+ }
+
// Apply the epoch-seconds rule at every leaf, including nested containers.
// Numeric -> LTZ depends on Java's system time zone and retains its
existing
// path; ordinary TIMESTAMP has no such runtime dependency.
@@ -344,14 +350,17 @@ fn render_string_values(
.collect())
}
_ => {
- let casted = cast(source, &ArrowDataType::Utf8).map_err(|error| {
- crate::Error::UnexpectedError {
- message: format!(
- "failed to render {:?} as string during schema
evolution",
- source.data_type()
- ),
- source: Some(Box::new(error)),
- }
+ let casted = evolve_column(
+ source,
+ source_type,
+ &DataType::VarChar(crate::spec::VarCharType::string_type()),
+ )
+ .map_err(|error| crate::Error::UnexpectedError {
+ message: format!(
+ "failed to render {:?} as string during schema evolution",
+ source.data_type()
+ ),
+ source: Some(Box::new(error)),
})?;
let strings = casted
.as_any()
@@ -623,6 +632,29 @@ mod tests {
use arrow_buffer::NullBuffer;
use arrow_schema::{DataType as ArrowDataType, Fields};
+ #[test]
+ fn timestamp_string_evolution_uses_java_separator_and_precision() {
+ let source: ArrayRef = Arc::new(TimestampMillisecondArray::from(vec![
+ Some(0),
+ Some(-1),
+ None,
+ ]));
+ let result = evolve_column(
+ &source,
+ &DataType::Timestamp(TimestampType::new(3).unwrap()),
+ &DataType::VarChar(VarCharType::string_type()),
+ )
+ .unwrap();
+ assert_eq!(
+ result.as_any().downcast_ref::<StringArray>().unwrap(),
+ &StringArray::from(vec![
+ Some("1970-01-01 00:00:00.000"),
+ Some("1969-12-31 23:59:59.999"),
+ None
+ ])
+ );
+ }
+
#[test]
fn selected_map_read_type_uses_key_metadata_not_child_names() {
use arrow_array::builder::{Int32Builder, MapBuilder, StringBuilder};
diff --git a/crates/paimon/src/common/date_time.rs
b/crates/paimon/src/common/date_time.rs
new file mode 100644
index 00000000..a30f5cc5
--- /dev/null
+++ b/crates/paimon/src/common/date_time.rs
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Java temporal string formatting shared by casts and partition paths.
+
+use chrono::{Datelike, NaiveDateTime, Timelike};
+
+/// Format millis-since-midnight to `HH:mm:ss[.f...]` using Java
`DateTimeUtils` semantics.
+///
+/// Matches Java `DateTimeUtils.formatTimestampMillis(time, precision)`:
+/// outputs up to `precision` fractional digits, stopping after the first
digit whose remaining fraction is zero.
+pub(crate) fn format_time(millis_of_day: i32, precision: u32) -> String {
+ let mut ms = millis_of_day;
+ // Handle negative millis (same guard as Java)
+ while ms < 0 {
+ ms += 86_400_000;
+ }
+ let ms = ms as u32;
+ let h = ms / 3_600_000 % 100;
+ let m = (ms % 3_600_000) / 60_000;
+ let s = (ms % 60_000) / 1_000;
+ let mut frac_ms = ms % 1_000;
+
+ let hms = format!("{h:02}:{m:02}:{s:02}");
+ if precision == 0 {
+ return hms;
+ }
+
+ // Emit up to `precision` digits, each extracted from the ms value.
+ // Matches Java's digit-by-digit loop with trailing-zero break.
+ let mut frac = String::with_capacity(precision as usize);
+ let mut remaining = precision;
+ while remaining > 0 {
+ frac.push((b'0' + (frac_ms / 100) as u8) as char);
+ frac_ms = (frac_ms % 100) * 10;
+ if frac_ms == 0 {
+ break;
+ }
+ remaining -= 1;
+ }
+
+ format!("{hms}.{frac}")
+}
+
+/// Format a timestamp using non-legacy `DateTimeUtils.formatTimestamp()`
semantics.
+///
+/// Always uses space separator: `yyyy-MM-dd HH:mm:ss[.fraction]`.
+/// Fraction: pad nano to 9 digits, strip trailing zeros while keeping at
least `precision` digits.
+pub(crate) fn format_timestamp(dt: NaiveDateTime, precision: u32) -> String {
+ format_timestamp_parts(
+ (dt.year(), dt.month(), dt.day()),
+ (dt.hour(), dt.minute(), dt.second()),
+ dt.nanosecond(),
+ precision,
+ )
+}
+
+/// Java Timestamp.toLocalDateTime stores its epoch day in a signed int.
+/// Use Gregorian arithmetic so that this also covers dates beyond chrono's
range.
+pub(crate) fn format_epoch_timestamp(seconds: i64, nanos: u32, precision: u32)
-> String {
+ let (year, month, day) = epoch_day_to_date(seconds.div_euclid(86_400) as
i32);
+ let time = seconds.rem_euclid(86_400) as u32;
+ format_timestamp_parts(
+ (year, month, day),
+ (time / 3600, time / 60 % 60, time % 60),
+ nanos,
+ precision,
+ )
+}
+
+fn epoch_day_to_date(day: i32) -> (i32, u32, u32) {
+ // Gregorian eras start on March 1, so the leap day is at the end of a
year.
+ let days = i64::from(day) + 719_468;
+ let era = days.div_euclid(146_097);
+ let day_of_era = days - era * 146_097;
+ let year_of_era =
+ (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era /
146_096) / 365;
+ let mut year = year_of_era + era * 400;
+ let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 -
year_of_era / 100);
+ let march_month = (5 * day_of_year + 2) / 153;
+ let day = day_of_year - (153 * march_month + 2) / 5 + 1;
+ let month = march_month + if march_month < 10 { 3 } else { -9 };
+ year += i64::from(month <= 2);
+ (year as i32, month as u32, day as u32)
+}
+
+fn format_timestamp_parts(
+ date: (i32, u32, u32),
+ time: (u32, u32, u32),
+ nano: u32,
+ precision: u32,
+) -> String {
+ let (year, month, day) = date;
+ let (hour, minute, second) = time;
+ // DateTimeUtils.int4 writes exactly four digits using signed division and
+ // remainder, including years outside 0000..9999.
+ let year = [1000, 100, 10, 1]
+ .into_iter()
+ .map(|divisor| (i32::from(b'0') + year / divisor % 10) as u8 as char)
+ .collect::<String>();
+ let ymdhms = format!(
+ "{year}-{:02}-{:02} {:02}:{:02}:{:02}",
+ month, day, hour, minute, second
+ );
+
+ // Whole seconds keep the declared precision; nonzero fractions are never
truncated.
+ let nano_str = format!("{nano:09}");
+ let mut fraction = &nano_str[..];
+
+ // Strip trailing zeros, but don't go below the precision boundary.
+ while fraction.len() > precision as usize && fraction.ends_with('0') {
+ fraction = &fraction[..fraction.len() - 1];
+ }
+
+ if fraction.is_empty() {
+ ymdhms
+ } else {
+ format!("{ymdhms}.{fraction}")
+ }
+}
diff --git a/crates/paimon/src/common/mod.rs b/crates/paimon/src/common/mod.rs
index 792cbfe5..5376e76d 100644
--- a/crates/paimon/src/common/mod.rs
+++ b/crates/paimon/src/common/mod.rs
@@ -17,6 +17,7 @@
//! Common utilities for Paimon.
+pub(crate) mod date_time;
pub mod options;
pub use options::{CatalogOptions, Options};
diff --git a/crates/paimon/src/spec/partition_utils.rs
b/crates/paimon/src/spec/partition_utils.rs
index b0466eaa..3785e619 100644
--- a/crates/paimon/src/spec/partition_utils.rs
+++ b/crates/paimon/src/spec/partition_utils.rs
@@ -23,6 +23,7 @@
//! - Java `InternalRowPartitionComputer` for value formatting
//! - Java `PartitionPathUtils` for path escaping and assembly
+use crate::common::date_time::{format_time, format_timestamp};
use crate::error::*;
use crate::spec::types::DataType;
use crate::spec::BinaryRow;
@@ -436,7 +437,7 @@ fn format_partition_value(
if legacy {
format_timestamp_legacy(dt)
} else {
- format_timestamp_non_legacy(dt, t.precision())
+ format_timestamp(dt, t.precision())
}
}
@@ -452,7 +453,7 @@ fn format_partition_value(
if legacy {
format_timestamp_legacy(dt)
} else {
- format_timestamp_non_legacy(dt, t.precision())
+ format_timestamp(dt, t.precision())
}
}
@@ -642,43 +643,6 @@ fn format_date(epoch_days: i32) -> String {
date.format("%Y-%m-%d").to_string()
}
-/// Format millis-since-midnight to `HH:mm:ss[.f...]` for non-legacy TIME
partitions.
-///
-/// Matches Java `DateTimeUtils.formatTimestampMillis(time, precision)`:
-/// outputs up to `precision` fractional digits, stripping trailing zeros.
-fn format_time(millis_of_day: i32, precision: u32) -> String {
- let mut ms = millis_of_day;
- // Handle negative millis (same guard as Java)
- while ms < 0 {
- ms += 86_400_000;
- }
- let ms = ms as u32;
- let h = ms / 3_600_000;
- let m = (ms % 3_600_000) / 60_000;
- let s = (ms % 60_000) / 1_000;
- let mut frac_ms = ms % 1_000;
-
- let hms = format!("{h:02}:{m:02}:{s:02}");
- if precision == 0 || frac_ms == 0 {
- return hms;
- }
-
- // Emit up to `precision` digits, each extracted from the ms value.
- // Matches Java's digit-by-digit loop with trailing-zero break.
- let mut frac = String::with_capacity(precision as usize);
- let mut remaining = precision;
- while remaining > 0 {
- frac.push((b'0' + (frac_ms / 100) as u8) as char);
- frac_ms = (frac_ms % 100) * 10;
- if frac_ms == 0 {
- break;
- }
- remaining -= 1;
- }
-
- format!("{hms}.{frac}")
-}
-
/// Format an unscaled i128 value with the given scale to a plain decimal
string.
///
/// Matches Java `BigDecimal.toPlainString()` semantics: no scientific
notation,
@@ -792,30 +756,6 @@ fn format_timestamp_legacy(dt: NaiveDateTime) -> String {
result
}
-/// Format a timestamp using non-legacy `DateTimeUtils.formatTimestamp()`
semantics.
-///
-/// Always uses space separator: `yyyy-MM-dd HH:mm:ss[.fraction]`.
-/// Fraction: pad nano to 9 digits, strip trailing zeros while keeping at
least `precision` digits.
-fn format_timestamp_non_legacy(dt: NaiveDateTime, precision: u32) -> String {
- let nano = dt.nanosecond();
- let ymdhms = dt.format("%Y-%m-%d %H:%M:%S").to_string();
-
- // Whole seconds keep the declared precision; nonzero fractions are never
truncated.
- let nano_str = format!("{nano:09}");
- let mut fraction = &nano_str[..];
-
- // Strip trailing zeros, but don't go below the precision boundary.
- while fraction.len() > precision as usize && fraction.ends_with('0') {
- fraction = &fraction[..fraction.len() - 1];
- }
-
- if fraction.is_empty() {
- ymdhms
- } else {
- format!("{ymdhms}.{fraction}")
- }
-}
-
/// Escape a path component following Java `PartitionPathUtils.escapePathName`.
///
/// Characters that need escaping are encoded as `%XX` (uppercase hex).
@@ -1418,20 +1358,11 @@ mod tests {
.and_hms_nano_opt(12, 0, 0, 123_000_000)
.unwrap();
// Precision is a minimum fraction width, not a truncation boundary.
- assert_eq!(
- format_timestamp_non_legacy(dt, 0),
- "2024-01-01 12:00:00.123"
- );
+ assert_eq!(format_timestamp(dt, 0), "2024-01-01 12:00:00.123");
// precision=3 → strip trailing zeros down to 3 digits
- assert_eq!(
- format_timestamp_non_legacy(dt, 3),
- "2024-01-01 12:00:00.123"
- );
+ assert_eq!(format_timestamp(dt, 3), "2024-01-01 12:00:00.123");
// precision=9 → preserve all 9 digits (no stripping since len ==
precision)
- assert_eq!(
- format_timestamp_non_legacy(dt, 9),
- "2024-01-01 12:00:00.123000000"
- );
+ assert_eq!(format_timestamp(dt, 9), "2024-01-01 12:00:00.123000000");
}
#[test]
@@ -1447,7 +1378,7 @@ mod tests {
(6, "2024-01-01 12:00:00.000000"),
(9, "2024-01-01 12:00:00.000000000"),
] {
- assert_eq!(format_timestamp_non_legacy(dt, precision), expected);
+ assert_eq!(format_timestamp(dt, precision), expected);
}
}
@@ -1464,7 +1395,7 @@ mod tests {
.unwrap()
.and_hms_nano_opt(12, 0, 0, nanos)
.unwrap();
- assert_eq!(format_timestamp_non_legacy(dt, precision), expected);
+ assert_eq!(format_timestamp(dt, precision), expected);
}
}
@@ -1734,7 +1665,7 @@ mod tests {
#[test]
fn test_format_time() {
// precision=3, 12:34:56 = 45_296_000 ms
- assert_eq!(format_time(45_296_000, 3), "12:34:56");
+ assert_eq!(format_time(45_296_000, 3), "12:34:56.0");
// precision=3, 12:34:56.123 = 45_296_123 ms
assert_eq!(format_time(45_296_123, 3), "12:34:56.123");
// precision=3, 00:00:00.100 = 100 ms → trailing zero stripped to .1
@@ -1769,6 +1700,13 @@ mod tests {
"t=12%3A34%3A56.123/",
false,
);
+ assert_single_partition(
+ "t",
+ DataType::Time(TimeType::new(3).unwrap()),
+ |b| b.write_int(0, 0),
+ "t=00%3A00%3A00.0/",
+ false,
+ );
}
// ======================== Corrupted row tests ========================
