This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 26daf52776b [bugfix](paimon)Fix field case issues (#36239)
26daf52776b is described below
commit 26daf52776bb05a243ca126acad4e6e3611a251f
Author: wuwenchi <[email protected]>
AuthorDate: Mon Jun 17 18:37:43 2024 +0800
[bugfix](paimon)Fix field case issues (#36239)
Issue Number: close #36021
Doris ignores the case of the fields, so lowercase fields are used to
identify field names.
---
.../apache/doris/paimon/PaimonScannerUtils.java | 1 +
.../datasource/paimon/PaimonExternalTable.java | 2 +-
.../paimon/source/PaimonPredicateConverter.java | 2 +-
.../paimon/test_paimon_catalog.out | 56 +++++++++++++++-------
.../paimon/test_paimon_catalog.groovy | 26 ++++++----
5 files changed, 58 insertions(+), 29 deletions(-)
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonScannerUtils.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonScannerUtils.java
index 102b6f01d50..9b1a493874e 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonScannerUtils.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonScannerUtils.java
@@ -40,6 +40,7 @@ public class PaimonScannerUtils {
public static List<String> fieldNames(RowType rowType) {
return rowType.getFields().stream()
.map(DataField::name)
+ .map(String::toLowerCase)
.collect(Collectors.toList());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
index d9e43bdd6cf..ab4bb8eac9b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
@@ -80,7 +80,7 @@ public class PaimonExternalTable extends ExternalTable {
List<DataField> columns = schema.fields();
List<Column> tmpSchema =
Lists.newArrayListWithCapacity(columns.size());
for (DataField field : columns) {
- tmpSchema.add(new Column(field.name(),
+ tmpSchema.add(new Column(field.name().toLowerCase(),
paimonTypeToDorisType(field.type()), true, null, true,
field.description(), true,
field.id()));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java
index 848e419b2fb..605bc1b321a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java
@@ -45,7 +45,7 @@ public class PaimonPredicateConverter {
public PaimonPredicateConverter(RowType rowType) {
this.builder = new PredicateBuilder(rowType);
- this.fieldNames =
rowType.getFields().stream().map(DataField::name).collect(Collectors.toList());
+ this.fieldNames = rowType.getFields().stream().map(f ->
f.name().toLowerCase()).collect(Collectors.toList());
this.paimonFieldTypes =
rowType.getFields().stream().map(DataField::type).collect(Collectors.toList());
}
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
index b2faa237100..c29201fb8a1 100644
--- a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
@@ -448,47 +448,47 @@ bbb
bbb
bbb
--- !c80 --
+-- !c81 --
\N
2023-10-24T16:10:40.533
2023-10-24T18:19:51.673
--- !c80 --
+-- !c82 --
1
1
1
--- !c80 --
+-- !c83 --
3
3
3
--- !c80 --
+-- !c84 --
4
4
4
--- !c80 --
+-- !c85 --
6
6
6
--- !c80 --
+-- !c85 --
7
7
7
--- !c80 --
+-- !c86 --
19
19
19
--- !c80 --
+-- !c86 --
\N
\N
21
--- !c80 --
+-- !c89 --
22
22
22
@@ -568,6 +568,16 @@ bbb
3 3_1
4 4_1
+-- !c106 --
+1 a
+
+-- !c107 --
+
+-- !c108 --
+1
+
+-- !c109 --
+
-- !all --
1 2 3 4 5 6 7 8 9.1 10.1
11.10 2020-02-02 13str 14varchar a true aaaa
2023-08-13T09:32:38.530
10 20 30 40 50 60 70 80 90.1 100.1
110.10 2020-03-02 130str 140varchar b false bbbb
2023-08-14T08:32:52.821
@@ -1017,47 +1027,47 @@ bbb
bbb
bbb
--- !c80 --
+-- !c81 --
\N
2023-10-24T16:10:40.533
2023-10-24T18:19:51.673
--- !c80 --
+-- !c82 --
1
1
1
--- !c80 --
+-- !c83 --
3
3
3
--- !c80 --
+-- !c84 --
4
4
4
--- !c80 --
+-- !c85 --
6
6
6
--- !c80 --
+-- !c85 --
7
7
7
--- !c80 --
+-- !c86 --
19
19
19
--- !c80 --
+-- !c86 --
\N
\N
21
--- !c80 --
+-- !c89 --
22
22
22
@@ -1137,3 +1147,13 @@ bbb
3 3_1
4 4_1
+-- !c106 --
+1 a
+
+-- !c107 --
+
+-- !c108 --
+1
+
+-- !c109 --
+
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index 80ec63fb6f9..3b278c7e450 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -176,6 +176,10 @@ suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_
def c104= """select * from deletion_vector_orc;"""
def c105= """select * from deletion_vector_parquet;"""
+ def c106= """ select * from tb_with_upper_case """
+ def c107= """ select id from tb_with_upper_case where id > 1 """
+ def c108= """ select id from tb_with_upper_case where id = 1 """
+ def c109= """ select id from tb_with_upper_case where id < 1 """
String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
String catalog_name = "ctl_test_paimon_catalog"
@@ -256,15 +260,15 @@ suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_
qt_c78 c78
qt_c79 c79
qt_c80 c80
- qt_c80 c81
- qt_c80 c82
- qt_c80 c83
- qt_c80 c84
- qt_c80 c85
- qt_c80 c86
- qt_c80 c87
- qt_c80 c88
- qt_c80 c89
+ qt_c81 c81
+ qt_c82 c82
+ qt_c83 c83
+ qt_c84 c84
+ qt_c85 c85
+ qt_c85 c86
+ qt_c86 c87
+ qt_c86 c88
+ qt_c89 c89
qt_c90 c90
qt_c91 c91
qt_c92 c92
@@ -280,6 +284,10 @@ suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_
qt_c103 c103
qt_c104 c104
qt_c105 c105
+ qt_c106 c106
+ qt_c107 c107
+ qt_c108 c108
+ qt_c109 c109
}
test_cases("false")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
