This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 535a5c944f [lance] Support TIMESTAMP_LTZ type (#10558)
535a5c944f is described below
commit 535a5c944f854cb8b68196d44677e576cf22120b
Author: Eunbin Son <[email protected]>
AuthorDate: Fri Oct 16 09:51:57 2026 +0900
[lance] Support TIMESTAMP_LTZ type (#10558)
---
docs/docs/concepts/spec/fileformat.md | 3 ++-
.../org/apache/paimon/format/lance/LanceFileFormat.java | 11 ++++-------
.../paimon/format/lance/LanceFileFormatReadWriteTest.java | 4 ++++
.../apache/paimon/format/lance/LanceFileFormatTest.java | 14 ++++++++++++++
4 files changed, 24 insertions(+), 8 deletions(-)
diff --git a/docs/docs/concepts/spec/fileformat.md
b/docs/docs/concepts/spec/fileformat.md
index 8af08a4a1b..18c1bb59d9 100644
--- a/docs/docs/concepts/spec/fileformat.md
+++ b/docs/docs/concepts/spec/fileformat.md
@@ -248,11 +248,12 @@ The following table lists the type mapping from Paimon
type to Lance (Arrow) typ
| DATE | DATE32 |
| TIME | TIME32 / TIME64 |
| TIMESTAMP(P) | TIMESTAMP (unit based on precision) |
+| TIMESTAMP_LOCAL_ZONE(P) | TIMESTAMP (unit based on precision, UTC time zone)
|
| ARRAY | LIST |
| ROW | STRUCT |
Limitations:
-1. Lance file format does not support `MAP`, `MULTISET`,
`TIMESTAMP_LOCAL_ZONE` or `VARIANT` types.
+1. Lance file format does not support `MAP`, `MULTISET` or `VARIANT` types.
2. Lance file format does not support `BLOB` fields stored inline in data
files (`blob-descriptor-field` or `blob-view-field`). Regular `BLOB` fields are
written to dedicated blob files and are not affected.
## Vortex
diff --git
a/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceFileFormat.java
b/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceFileFormat.java
index 2eee2639b7..8a0619fe06 100644
---
a/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceFileFormat.java
+++
b/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceFileFormat.java
@@ -169,13 +169,10 @@ public class LanceFileFormat extends FileFormat {
@Override
public Void visit(LocalZonedTimestampType localZonedTimestampType) {
- // lance has a bug here, if the local in GMT-10:00, it failed by
reson: called
- // `Result::unwrap()` on an `Err` value: Schema { message:
"Unsupported timestamp type:
- // timestamp:us:GMT-10:00", location: Location { file:
- // "rust/lance-core/src/datatypes.rs", line: 326, column: 39 } }
- // Disable it for now.
- throw new UnsupportedOperationException(
- "Lance file format does not support type
LOCAL_ZONED_TIMESTAMP");
+ // Lance cannot parse timestamp time zone ids containing ':' (e.g.
GMT-10:00), which
+ // aborts the JVM. This is safe because ArrowFieldTypeConversion
always writes
+ // LOCAL_ZONED_TIMESTAMP with the "UTC" time zone.
+ return null;
}
@Override
diff --git
a/paimon-lance/src/test/java/org/apache/paimon/format/lance/LanceFileFormatReadWriteTest.java
b/paimon-lance/src/test/java/org/apache/paimon/format/lance/LanceFileFormatReadWriteTest.java
index efd1636932..30df66db08 100644
---
a/paimon-lance/src/test/java/org/apache/paimon/format/lance/LanceFileFormatReadWriteTest.java
+++
b/paimon-lance/src/test/java/org/apache/paimon/format/lance/LanceFileFormatReadWriteTest.java
@@ -64,6 +64,8 @@ public class LanceFileFormatReadWriteTest extends
FormatReadWriteTest {
.field("bytes", DataTypes.BYTES())
.field("timestamp", DataTypes.TIMESTAMP())
.field("timestamp_3", DataTypes.TIMESTAMP(3))
+ .field("timestamp_ltz",
DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE())
+ .field("timestamp_ltz_3",
DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3))
.field("date", DataTypes.DATE())
.field("decimal", DataTypes.DECIMAL(2, 2))
.field("decimal2", DataTypes.DECIMAL(38, 2))
@@ -106,6 +108,8 @@ public class LanceFileFormatReadWriteTest extends
FormatReadWriteTest {
new byte[] {1, 5, 2},
Timestamp.fromMicros(123123123),
Timestamp.fromEpochMillis(123123123),
+ Timestamp.fromMicros(123123123),
+ Timestamp.fromEpochMillis(123123123),
2456,
Decimal.fromBigDecimal(new BigDecimal("0.22"), 2, 2),
Decimal.fromBigDecimal(new BigDecimal("12312455.22"),
38, 2),
diff --git
a/paimon-lance/src/test/java/org/apache/paimon/format/lance/LanceFileFormatTest.java
b/paimon-lance/src/test/java/org/apache/paimon/format/lance/LanceFileFormatTest.java
index 9a7152e259..b2aaf47654 100644
---
a/paimon-lance/src/test/java/org/apache/paimon/format/lance/LanceFileFormatTest.java
+++
b/paimon-lance/src/test/java/org/apache/paimon/format/lance/LanceFileFormatTest.java
@@ -102,6 +102,20 @@ public class LanceFileFormatTest {
assertDoesNotThrow(() -> format.validateDataFields(rowType));
}
+ @Test
+ public void testValidateDataFields_SupportedLocalZonedTimestampType() {
+ LanceFileFormat format =
+ new LanceFileFormat(new FileFormatFactory.FormatContext(new
Options(), 1024, 1024));
+ RowType rowType =
+ RowType.of(
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0),
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3),
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6),
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(9),
+
DataTypes.ARRAY(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()));
+ assertDoesNotThrow(() -> format.validateDataFields(rowType));
+ }
+
@Test
public void testValidateDataFields_UnsupportedVariantType() {
LanceFileFormat format =
