This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d3772cb05c9 [fix](iceberg) Preserve timestamp precision and semantics
in metadata scans (#68985)
d3772cb05c9 is described below
commit d3772cb05c967e29667d575f2e1c92209dc7c363
Author: Oliveira <[email protected]>
AuthorDate: Thu Oct 15 09:51:50 2026 +0800
[fix](iceberg) Preserve timestamp precision and semantics in metadata scans
(#68985)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #68276 (supersedes its metadata-scanner fix; this PR targets
master)
Problem Summary:
Iceberg metadata table timestamps are encoded as epoch microseconds. The
JNI scanner converts them through milliseconds, losing fractional
precision. It also applies the session time zone to
timestamp-without-zone partition values, changing their wall-clock
value.
For example, a `$files.partition` timestamp of `1234567` microseconds is
truncated to `1970-01-01 00:00:01.234000` rather than `1970-01-01
00:00:01.234567`. Switching from UTC to Asia/Shanghai incorrectly shifts
timestamp-without-zone values by eight hours.
This PR:
- Decodes signed epoch microseconds with floor division/modulus,
preserving pre-epoch fractions.
- Propagates Iceberg types through struct/list/map values to distinguish
wall-clock timestamps from instants.
- Resolves requested field types by name while continuing to read
projected rows by position.
- Adds unit tests and a `$files.partition` regression covering
`-1000001`, `-1`, `1`, `999`, `1000`, `1234567`, and NULL,
UTC/Asia/Shanghai, both timestamp mapping settings, and a projected
partition child.
Scope is the Java Iceberg metadata scanner. This PR does not modify
regular data-table timestamp reads or the FE/native `$position_deletes`
serialization path. There are no storage-format or configuration
changes.
### Release note
Fix microsecond precision loss and incorrect session-time-zone shifts
for timestamp-without-zone values in Iceberg metadata tables.
### Check List (For Author)
- Test
- [x] Regression test
- `test_iceberg_files_timestamp_partition`: generated the golden output,
checked its timestamp values, and reran normal comparison successfully
(1 suite, 0 failures/skips).
- `test_iceberg_system_table_projection`: passed.
- `test_iceberg_position_deletes_sys_table`: passed.
- [x] Unit Test
- `IcebergSysTableColumnValueTest`: 2 tests passed; the
FE/Java-extension reactor completed successfully.
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
Local External tests used an isolated Iceberg fixture stack and the
newly packaged metadata-scanner plugin deployed to the personal Doris
BE. The full FE/BE deployment build was interrupted during C++ submodule
downloads; FE/BE executables were not rebuilt for these runs. These are
targeted local results, not full community External/P0/coverage results.
- Behavior changed:
- [ ] No.
- [x] Yes. Metadata timestamps retain microseconds;
timestamp-without-zone values no longer shift with the session time
zone. Timestamp-with-zone values preserve their instant semantics.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../doris/iceberg/IcebergSysTableColumnValue.java | 36 +++++++---
.../doris/iceberg/IcebergSysTableJniScanner.java | 11 ++-
.../iceberg/IcebergSysTableColumnValueTest.java | 80 ++++++++++++++++++++++
.../test_iceberg_files_timestamp_partition.out | 73 ++++++++++++++++++++
.../iceberg/write/test_iceberg_write_stats2.out | 4 +-
.../test_iceberg_files_timestamp_partition.groovy | 72 +++++++++++++++++++
6 files changed, 265 insertions(+), 11 deletions(-)
diff --git
a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java
b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java
index 5857872de4e..a010c5dc213 100644
---
a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java
+++
b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java
@@ -20,6 +20,8 @@ package org.apache.doris.iceberg;
import org.apache.doris.jni.spi.vec.ColumnValue;
import org.apache.iceberg.StructLike;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -38,14 +40,20 @@ public class IcebergSysTableColumnValue implements
ColumnValue {
private final Object fieldData;
private final String timezone;
+ private final Type icebergType;
public IcebergSysTableColumnValue(Object fieldData) {
this(fieldData, DEFAULT_TIME_ZONE);
}
public IcebergSysTableColumnValue(Object fieldData, String timezone) {
+ this(fieldData, timezone, null);
+ }
+
+ public IcebergSysTableColumnValue(Object fieldData, String timezone, Type
icebergType) {
this.fieldData = fieldData;
this.timezone = timezone;
+ this.icebergType = icebergType;
}
@Override
@@ -136,14 +144,22 @@ public class IcebergSysTableColumnValue implements
ColumnValue {
@Override
public LocalDateTime getDateTime() {
- Instant instant = Instant.ofEpochMilli((((long) fieldData) / 1000));
- return LocalDateTime.ofInstant(instant, ZoneId.of(timezone));
+ // NTZ is a wall-clock value; only timestamptz mapped to DATETIME uses
the session zone.
+ ZoneId zone = icebergType instanceof Types.TimestampType
+ && !((Types.TimestampType) icebergType).shouldAdjustToUTC()
+ ? ZoneOffset.UTC : ZoneId.of(timezone);
+ return LocalDateTime.ofInstant(timestampInstant(), zone);
}
@Override
public LocalDateTime getTimeStampTz() {
- Instant instant = Instant.ofEpochMilli((((long) fieldData) / 1000));
- return LocalDateTime.ofInstant(instant, ZoneId.of("UTC"));
+ return LocalDateTime.ofInstant(timestampInstant(), ZoneOffset.UTC);
+ }
+
+ private Instant timestampInstant() {
+ long micros = (long) fieldData;
+ return Instant.ofEpochSecond(Math.floorDiv(micros, 1_000_000L),
+ Math.floorMod(micros, 1_000_000L) * 1_000L);
}
@Override
@@ -162,7 +178,8 @@ public class IcebergSysTableColumnValue implements
ColumnValue {
public void unpackArray(List<ColumnValue> values) {
List<?> items = (List<?>) fieldData;
for (Object item : items) {
- values.add(new IcebergSysTableColumnValue(item, timezone));
+ values.add(new IcebergSysTableColumnValue(item, timezone,
+ icebergType == null ? null :
icebergType.asListType().elementType()));
}
}
@@ -170,8 +187,10 @@ public class IcebergSysTableColumnValue implements
ColumnValue {
public void unpackMap(List<ColumnValue> keys, List<ColumnValue> values) {
Map<?, ?> data = (Map<?, ?>) fieldData;
data.forEach((key, value) -> {
- keys.add(new IcebergSysTableColumnValue(key, timezone));
- values.add(new IcebergSysTableColumnValue(value, timezone));
+ keys.add(new IcebergSysTableColumnValue(key, timezone,
+ icebergType == null ? null :
icebergType.asMapType().keyType()));
+ values.add(new IcebergSysTableColumnValue(value, timezone,
+ icebergType == null ? null :
icebergType.asMapType().valueType()));
});
}
@@ -180,7 +199,8 @@ public class IcebergSysTableColumnValue implements
ColumnValue {
StructLike record = (StructLike) fieldData;
for (Integer fieldIndex : structFieldIndex) {
Object rawValue = record.get(fieldIndex, Object.class);
- values.add(new IcebergSysTableColumnValue(rawValue, timezone));
+ values.add(new IcebergSysTableColumnValue(rawValue, timezone,
+ icebergType == null ? null :
icebergType.asStructType().fields().get(fieldIndex).type()));
}
}
}
diff --git
a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java
b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java
index cd99848ccb9..df6f6ae5124 100644
---
a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java
+++
b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java
@@ -27,6 +27,7 @@ import com.google.common.base.Preconditions;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.StructLike;
import org.apache.iceberg.io.CloseableIterator;
+import org.apache.iceberg.types.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,6 +45,7 @@ public class IcebergSysTableJniScanner extends JniScanner {
private final PreExecutionAuthenticator preExecutionAuthenticator;
private final FileScanTask scanTask;
private final int requiredFieldCount;
+ private final Type[] icebergTypes;
private final String timezone;
private CloseableIterator<StructLike> reader;
@@ -57,6 +59,13 @@ public class IcebergSysTableJniScanner extends JniScanner {
"required_fields should not be empty");
String[] requiredFields = requiredFieldsParam.split(",");
this.requiredFieldCount = requiredFields.length;
+ this.icebergTypes = new Type[requiredFieldCount];
+ for (int i = 0; i < requiredFieldCount; i++) {
+ // Resolve types by name, but read projected ROWS by position
below. StaticDataTask.schema()
+ // can be the full schema even when its rows contain only the
requested fields.
+ this.icebergTypes[i] =
scanTask.schema().findType(requiredFields[i]);
+ Preconditions.checkNotNull(icebergTypes[i], "Missing Iceberg
metadata field %s", requiredFields[i]);
+ }
this.timezone = params.getOrDefault("time_zone",
TimeZone.getDefault().getID());
Map<String, String> hadoopOptionParams = params.entrySet().stream()
.filter(kv -> kv.getKey().startsWith(HADOOP_OPTION_PREFIX))
@@ -104,7 +113,7 @@ public class IcebergSysTableJniScanner extends JniScanner {
// narrowed StructProjection, so a full-schema ordinal
overruns the projected row (upstream
// #65262 -- reverting this to a by-name/schema() lookup
reintroduces ArrayIndexOutOfBounds).
Object value = row.get(i, Object.class);
- ColumnValue columnValue = new
IcebergSysTableColumnValue(value, timezone);
+ ColumnValue columnValue = new
IcebergSysTableColumnValue(value, timezone, icebergTypes[i]);
appendData(i, columnValue);
}
rows++;
diff --git
a/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java
b/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java
new file mode 100644
index 00000000000..1c4186449f5
--- /dev/null
+++
b/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergSysTableColumnValueTest.java
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.iceberg;
+
+import org.apache.doris.jni.spi.vec.ColumnValue;
+
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class IcebergSysTableColumnValueTest {
+ @Test
+ public void preservesMicrosAndTimestampSemantics() {
+ long[] micros = {-1000001L, -1L, 1L, 999L, 1000L, 1234567L};
+ String[] expected = {"1969-12-31T23:59:58.999999",
"1969-12-31T23:59:59.999999",
+ "1970-01-01T00:00:00.000001", "1970-01-01T00:00:00.000999",
+ "1970-01-01T00:00:00.001", "1970-01-01T00:00:01.234567"};
+ for (int i = 0; i < micros.length; i++) {
+ LocalDateTime utc = LocalDateTime.parse(expected[i]);
+ for (String zone : Arrays.asList("UTC", "Asia/Shanghai")) {
+ ColumnValue ntz = new IcebergSysTableColumnValue(micros[i],
zone, Types.TimestampType.withoutZone());
+ ColumnValue tz = new IcebergSysTableColumnValue(micros[i],
zone, Types.TimestampType.withZone());
+ Assertions.assertEquals(utc, ntz.getDateTime());
+ Assertions.assertEquals(zone.equals("UTC") ? utc :
utc.plusHours(8), tz.getDateTime());
+ Assertions.assertEquals(utc, tz.getTimeStampTz());
+ }
+ }
+ }
+
+ @Test
+ public void propagatesTypesThroughNestedContainers() {
+ Types.StructType struct = Types.StructType.of(
+ Types.NestedField.optional(1, "ts",
Types.TimestampType.withoutZone()),
+ Types.NestedField.optional(2, "tz",
Types.TimestampType.withZone()));
+ PartitionData record = new PartitionData(struct);
+ record.set(0, -1L);
+ record.set(1, -1L);
+ List<ColumnValue> fields = new ArrayList<>();
+ new IcebergSysTableColumnValue(record, "Asia/Shanghai", struct)
+ .unpackStruct(Arrays.asList(1, 0), fields);
+ LocalDateTime utc = LocalDateTime.parse("1969-12-31T23:59:59.999999");
+ Assertions.assertEquals(utc.plusHours(8), fields.get(0).getDateTime());
+ Assertions.assertEquals(utc, fields.get(1).getDateTime());
+
+ List<ColumnValue> elements = new ArrayList<>();
+ new IcebergSysTableColumnValue(Collections.singletonList(-1L),
"Asia/Shanghai",
+ Types.ListType.ofOptional(3,
Types.TimestampType.withoutZone())).unpackArray(elements);
+ Assertions.assertEquals(utc, elements.get(0).getDateTime());
+
+ List<ColumnValue> keys = new ArrayList<>();
+ List<ColumnValue> values = new ArrayList<>();
+ new IcebergSysTableColumnValue(Collections.singletonMap(-1L, -1L),
"Asia/Shanghai",
+ Types.MapType.ofOptional(4, 5,
Types.TimestampType.withoutZone(), Types.TimestampType.withZone()))
+ .unpackMap(keys, values);
+ Assertions.assertEquals(utc, keys.get(0).getDateTime());
+ Assertions.assertEquals(utc.plusHours(8), values.get(0).getDateTime());
+ }
+}
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out
new file mode 100644
index 00000000000..2a605bebeea
--- /dev/null
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.out
@@ -0,0 +1,73 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !false_UTC --
+{"ts":"1969-12-31 23:59:58.999999", "tz":"1969-12-31 23:59:58.999999+00:00"}
+{"ts":"1969-12-31 23:59:59.999999", "tz":"1969-12-31 23:59:59.999999+00:00"}
+{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 00:00:00.000001+00:00"}
+{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 00:00:00.000999+00:00"}
+{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 00:00:00.001000+00:00"}
+{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 00:00:01.234567+00:00"}
+{"ts":null, "tz":null}
+
+-- !false_UTC_ntz --
+\N
+1969-12-31T23:59:58.999999
+1969-12-31T23:59:59.999999
+1970-01-01T00:00:00.000001
+1970-01-01T00:00:00.000999
+1970-01-01T00:00:00.001
+1970-01-01T00:00:01.234567
+
+-- !false_Asia_Shanghai --
+{"ts":"1969-12-31 23:59:58.999999", "tz":"1970-01-01 07:59:58.999999+08:00"}
+{"ts":"1969-12-31 23:59:59.999999", "tz":"1970-01-01 07:59:59.999999+08:00"}
+{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 08:00:00.000001+08:00"}
+{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 08:00:00.000999+08:00"}
+{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 08:00:00.001000+08:00"}
+{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 08:00:01.234567+08:00"}
+{"ts":null, "tz":null}
+
+-- !false_Asia_Shanghai_ntz --
+\N
+1969-12-31T23:59:58.999999
+1969-12-31T23:59:59.999999
+1970-01-01T00:00:00.000001
+1970-01-01T00:00:00.000999
+1970-01-01T00:00:00.001
+1970-01-01T00:00:01.234567
+
+-- !true_UTC --
+{"ts":"1969-12-31 23:59:58.999999", "tz":"1969-12-31 23:59:58.999999+00:00"}
+{"ts":"1969-12-31 23:59:59.999999", "tz":"1969-12-31 23:59:59.999999+00:00"}
+{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 00:00:00.000001+00:00"}
+{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 00:00:00.000999+00:00"}
+{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 00:00:00.001000+00:00"}
+{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 00:00:01.234567+00:00"}
+{"ts":null, "tz":null}
+
+-- !true_UTC_ntz --
+\N
+1969-12-31T23:59:58.999999
+1969-12-31T23:59:59.999999
+1970-01-01T00:00:00.000001
+1970-01-01T00:00:00.000999
+1970-01-01T00:00:00.001
+1970-01-01T00:00:01.234567
+
+-- !true_Asia_Shanghai --
+{"ts":"1969-12-31 23:59:58.999999", "tz":"1970-01-01 07:59:58.999999+08:00"}
+{"ts":"1969-12-31 23:59:59.999999", "tz":"1970-01-01 07:59:59.999999+08:00"}
+{"ts":"1970-01-01 00:00:00.000001", "tz":"1970-01-01 08:00:00.000001+08:00"}
+{"ts":"1970-01-01 00:00:00.000999", "tz":"1970-01-01 08:00:00.000999+08:00"}
+{"ts":"1970-01-01 00:00:00.001000", "tz":"1970-01-01 08:00:00.001000+08:00"}
+{"ts":"1970-01-01 00:00:01.234567", "tz":"1970-01-01 08:00:01.234567+08:00"}
+{"ts":null, "tz":null}
+
+-- !true_Asia_Shanghai_ntz --
+\N
+1969-12-31T23:59:58.999999
+1969-12-31T23:59:59.999999
+1970-01-01T00:00:00.000001
+1970-01-01T00:00:00.000999
+1970-01-01T00:00:00.001
+1970-01-01T00:00:01.234567
+
diff --git
a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out
b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out
index de452a61eaa..df313184df9 100644
---
a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out
+++
b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out
@@ -7,7 +7,7 @@ true 11 111 1.1 1.1 1111 1234.5678
1234.567890 123456789012345678.123456789012 a
0 PARQUET 2 {1:2, 2:2, 3:2, 4:2, 5:2, 6:2, 7:2, 8:2, 9:2, 10:2,
11:2, 12:2} {1:0, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0, 10:0, 11:0, 12:0}
{1:0x00, 2:0x0B000000, 3:0x6F00000000000000, 4:0xCDCC8C3F,
5:0x9A9999999999F13F, 6:0x0457, 7:0x00BC614E, 8:0x499602D2,
9:0x018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000,
12:0x005CE70F33F10500} {1:0x01, 2:0x16000000, 3:0xDE00000000000000,
4:0xCDCC0C40, 5:0x9A99999999990140, 6:0x08AE, 7:0x05397FB1, 8:0x020A75E124,
9:0x0C7748819DFFB62505316873C [...]
-- !sql_2 --
-{"bigint_col":{"column_size":74, "value_count":2, "null_value_count":0,
"nan_value_count":null, "lower_bound":111, "upper_bound":222},
"boolean_col":{"column_size":33, "value_count":2, "null_value_count":0,
"nan_value_count":null, "lower_bound":0, "upper_bound":1},
"date_col":{"column_size":66, "value_count":2, "null_value_count":0,
"nan_value_count":null, "lower_bound":"2023-01-01",
"upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":74,
"value_count":2, "null_value_count":0, "n [...]
+{"bigint_col":{"column_size":74, "value_count":2, "null_value_count":0,
"nan_value_count":null, "lower_bound":111, "upper_bound":222},
"boolean_col":{"column_size":33, "value_count":2, "null_value_count":0,
"nan_value_count":null, "lower_bound":0, "upper_bound":1},
"date_col":{"column_size":66, "value_count":2, "null_value_count":0,
"nan_value_count":null, "lower_bound":"2023-01-01",
"upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":74,
"value_count":2, "null_value_count":0, "n [...]
-- !sql_3 --
false 22 222 2.2 2.2 2222 8765.4321 8765.432100
987654321098765432.987654321099 bbb 2023-06-15 2023-06-15T23:45:01
@@ -17,7 +17,7 @@ true 11 111 1.1 1.1 1111 1234.5678
1234.567890 123456789012345678.123456789012 a
0 ORC 2 {1:2, 2:2, 3:2, 4:2, 5:2, 6:2, 7:2, 8:2, 9:2, 10:2,
11:2, 12:2} {} {1:0x00, 2:0x0B00000000000000, 3:0x6F00000000000000,
4:0xCDCC8C3F, 5:0x9A9999999999F13F, 6:0x0457, 7:0x00BC614E, 8:0x075BCD15,
9:0x018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000,
12:0x005CE70F33F10500} {1:0x01, 2:0x1600000000000000,
3:0xDE00000000000000, 4:0xCDCC0C40, 5:0x9A99999999990140, 6:0x08AE,
7:0x05397FB1, 8:0x05397FB1, 9:0x0C7748819DFFB62505316873CB, 10:0x626262,
11:0x434C0000, 12:0x40D91FA833FE0500}
-- !sql_5 --
-{"bigint_col":{"column_size":null, "value_count":2, "null_value_count":null,
"nan_value_count":null, "lower_bound":111, "upper_bound":222},
"boolean_col":{"column_size":null, "value_count":2, "null_value_count":null,
"nan_value_count":null, "lower_bound":0, "upper_bound":1},
"date_col":{"column_size":null, "value_count":2, "null_value_count":null,
"nan_value_count":null, "lower_bound":"2023-01-01",
"upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":null,
"value_count":2, "null_v [...]
+{"bigint_col":{"column_size":null, "value_count":2, "null_value_count":null,
"nan_value_count":null, "lower_bound":111, "upper_bound":222},
"boolean_col":{"column_size":null, "value_count":2, "null_value_count":null,
"nan_value_count":null, "lower_bound":0, "upper_bound":1},
"date_col":{"column_size":null, "value_count":2, "null_value_count":null,
"nan_value_count":null, "lower_bound":"2023-01-01",
"upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":null,
"value_count":2, "null_v [...]
-- !sql_6 --
{1:0x00, 2:0x0B000000, 3:0x6F00000000000000, 4:0xCDCC8C3F,
5:0x9A9999999999F13F, 6:0x0457, 7:0x00BC614E, 8:0x499602D2,
9:0x018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000,
12:0x005CE70F33F10500}
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.groovy
new file mode 100644
index 00000000000..4ec5d37c41c
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_files_timestamp_partition.groovy
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_files_timestamp_partition", "p0,external,iceberg") {
+ if
(!context.config.otherConfigs.get("enableIcebergTest")?.toString()?.equalsIgnoreCase("true"))
{
+ return
+ }
+ String host = context.config.otherConfigs.get("externalEnvIp")
+ String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ def previousZone = sql("SELECT @@time_zone")[0][0]
+ // Parquet supports both pre-epoch fractions and NTZ. Use identity
partitions so the
+ // metadata rows retain the original microseconds, not a day/hour
transform.
+ spark_iceberg_multi """
+ SET spark.sql.session.timeZone=UTC;
+ CREATE DATABASE IF NOT EXISTS demo.files_timestamp_partition_db;
+ DROP TABLE IF EXISTS
demo.files_timestamp_partition_db.metadata_timestamp_precision;
+ CREATE TABLE
demo.files_timestamp_partition_db.metadata_timestamp_precision (
+ id INT, ts TIMESTAMP_NTZ, tz TIMESTAMP
+ ) USING iceberg PARTITIONED BY (ts, tz)
+ TBLPROPERTIES ('write.format.default'='parquet');
+ INSERT INTO
demo.files_timestamp_partition_db.metadata_timestamp_precision
+ SELECT id, CAST(value AS TIMESTAMP_NTZ), CAST(value AS TIMESTAMP)
+ FROM VALUES
+ (0, '1969-12-31 23:59:58.999999'),
+ (1, '1969-12-31 23:59:59.999999'),
+ (2, '1970-01-01 00:00:00.000001'),
+ (3, '1970-01-01 00:00:00.000999'),
+ (4, '1970-01-01 00:00:00.001000'),
+ (5, '1970-01-01 00:00:01.234567'),
+ (6, CAST(NULL AS STRING)) AS input(id, value);
+ """
+ try {
+ for (boolean mapping : [false, true]) {
+ sql "DROP CATALOG IF EXISTS test_iceberg_files_timestamp_partition"
+ sql """CREATE CATALOG test_iceberg_files_timestamp_partition
PROPERTIES (
+ 'type'='iceberg', 'iceberg.catalog.type'='rest',
+ 'uri'='http://${host}:${restPort}',
+ 's3.endpoint'='http://${host}:${minioPort}',
+ 's3.access_key'='admin', 's3.secret_key'='password',
's3.region'='us-east-1',
+ 'enable.mapping.timestamp_tz'='${mapping}')"""
+ sql "SWITCH test_iceberg_files_timestamp_partition"
+ sql "USE files_timestamp_partition_db"
+ for (String zone : ["UTC", "Asia/Shanghai"]) {
+ sql "SET time_zone = '${zone}'"
+ String tag = "${mapping}_${zone.replace('/', '_')}"
+ "order_qt_${tag}" """SELECT `partition`
+ FROM metadata_timestamp_precision\$files ORDER BY
file_path"""
+ // Exercise a pruned struct child as well as the entire
partition above.
+ "order_qt_${tag}_ntz" """SELECT `partition`.ts
+ FROM metadata_timestamp_precision\$files ORDER BY
file_path"""
+ }
+ }
+ } finally {
+ sql "SET time_zone = '${previousZone}'"
+ sql "SWITCH internal"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
