This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d9e9926b12c [bugfix](iceberg)fix read NULL with date partition (#30478)
d9e9926b12c is described below
commit d9e9926b12cc8330d8395474816f1881fb862d03
Author: wuwenchi <[email protected]>
AuthorDate: Tue Jan 30 10:30:57 2024 +0800
[bugfix](iceberg)fix read NULL with date partition (#30478)
* fix date
* fix date
* add case
---
.../planner/external/iceberg/IcebergScanNode.java | 19 +++++++++++--
.../test_external_catalog_iceberg_partition.out | 33 ++++++++++++++++++++++
.../test_external_catalog_iceberg_partition.groovy | 21 ++++++++++++++
3 files changed, 71 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
index ae3efccadc5..a335ccfa021 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
@@ -69,6 +69,9 @@ import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.types.Conversions;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DateTimeUtil;
import org.apache.iceberg.util.TableScanUtil;
import java.io.IOException;
@@ -220,11 +223,23 @@ public class IcebergScanNode extends FileQueryScanNode {
List<String> partitionValues = new ArrayList<>();
if (isPartitionedTable) {
StructLike structLike = splitTask.file().partition();
+ List<PartitionField> fields = splitTask.spec().fields();
+ Types.StructType structType =
icebergTable.schema().asStruct();
// set partitionValue for this IcebergSplit
for (int i = 0; i < structLike.size(); i++) {
- String partition = String.valueOf(structLike.get(i,
Object.class));
- partitionValues.add(partition);
+ Object obj = structLike.get(i, Object.class);
+ String value = String.valueOf(obj);
+ PartitionField partitionField = fields.get(i);
+ if (partitionField.transform().isIdentity()) {
+ Type type =
structType.fieldType(partitionField.name());
+ if (type != null &&
type.typeId().equals(Type.TypeID.DATE)) {
+ // iceberg use integer to store date,
+ // we need transform it to string
+ value = DateTimeUtil.daysToIsoDate((Integer)
obj);
+ }
+ }
+ partitionValues.add(value);
}
// Counts the number of partitions read
diff --git
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_partition.out
b/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_partition.out
index c2582691ccb..a0fd39be5f2 100644
---
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_partition.out
+++
b/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_partition.out
@@ -71,3 +71,36 @@
-- !q09 --
100 0.3 test3 2023-01-03T00:00
+-- !q01 --
+1 true 2020-01-02 2020-01-02T01:02:03.123456
+
+-- !q02 --
+1 true 2020-01-02 2020-01-02T01:02:03.123456
+
+-- !q03 --
+1 true 2020-01-02 2020-01-02T01:02:03.123456
+
+-- !q04 --
+1 true 2020-01-02 2020-01-02T01:02:03.123456
+
+-- !q05 --
+1 true 2020-01-02 2020-01-02T01:02:03.123456
+
+-- !q06 --
+1 2020-01-02
+
+-- !q07 --
+1 2020-01-02
+
+-- !q08 --
+1 2020-01-02
+
+-- !q09 --
+1 2020-01-02
+
+-- !q10 --
+1 2020-01-02
+
+-- !q11 --
+1 2020-01-02
+
diff --git
a/regression-test/suites/external_table_p2/iceberg/test_external_catalog_iceberg_partition.groovy
b/regression-test/suites/external_table_p2/iceberg/test_external_catalog_iceberg_partition.groovy
index 9429887e8c0..dfdd923bcc4 100644
---
a/regression-test/suites/external_table_p2/iceberg/test_external_catalog_iceberg_partition.groovy
+++
b/regression-test/suites/external_table_p2/iceberg/test_external_catalog_iceberg_partition.groovy
@@ -55,9 +55,30 @@ suite("test_external_catalog_iceberg_partition",
"p2,external,iceberg,external_r
qt_q08 """ select * from
iceberg_catalog.orc_partitioned_truncate_and_fields where t_int is null order
by t_float """
qt_q09 """ select * from
iceberg_catalog.orc_partitioned_truncate_and_fields where t_int is not null
order by t_float """
}
+
+ // test date for partition and predict
+ def q01_date = {
+
+ qt_q01 """ select * from user_case_date_without_partition where d
= '2020-01-02' """
+ qt_q02 """ select * from user_case_date_without_partition where d
> '2020-01-01' """
+ qt_q03 """ select * from user_case_date_without_partition where d
< '2020-01-03' """
+ qt_q04 """ select * from user_case_date_without_partition where ts
< '2020-01-03' """
+ qt_q05 """ select * from user_case_date_without_partition where ts
> '2020-01-01' """
+
+ qt_q06 """ select * from user_case_date_with_date_partition where
d = '2020-01-02' """
+ qt_q07 """ select * from user_case_date_with_date_partition where
d < '2020-01-03' """
+ qt_q08 """ select * from user_case_date_with_date_partition where
d > '2020-01-01' """
+
+ qt_q09 """ select * from user_case_date_with_days_date_partition
where d = '2020-01-02' """
+ qt_q10 """ select * from user_case_date_with_days_date_partition
where d < '2020-01-03' """
+ qt_q11 """ select * from user_case_date_with_days_date_partition
where d > '2020-01-01' """
+
+ }
+
sql """ use `iceberg_catalog`; """
q01_parquet()
q01_orc()
+ q01_date()
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
