This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7a84ab67c23 [fix](fe) Preserve inferred JSON column name casing
(#67411)
7a84ab67c23 is described below
commit 7a84ab67c23d714bc6280f42011e0e0d8b1c226c
Author: Refrain <[email protected]>
AuthorDate: Wed Oct 21 17:06:02 2026 +0800
[fix](fe) Preserve inferred JSON column name casing (#67411)
Problem Summary: JSON schema inference lowercased field names before
creating the table-valued function schema, while JSON readers match
object keys case-sensitively. Uppercase keys therefore appeared as
queryable lowercase columns but were silently materialized as NULL.
Preserve the inferred key casing for JSON only, while retaining
case-insensitive duplicate detection and the existing lowercase behavior
for other file formats.
### Release note
JSON table-valued functions now read inferred mixed-case and uppercase
keys without silently producing NULL values.
---
.../ExternalFileTableValuedFunction.java | 11 +++--
.../tvf/test_json_uppercase_keys.out | 9 ++++
.../tvf/test_json_uppercase_keys/case_keys.jsonl | 2 +
.../tvf/test_json_uppercase_keys.groovy | 49 ++++++++++++++++++++++
4 files changed, 67 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index 2f4495ab643..3f15a190ccd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -477,15 +477,18 @@ public abstract class ExternalFileTableValuedFunction
extends TableValuedFunctio
}
// add fetched file columns
Set<String> columnLowerNames = new HashSet<>();
+ boolean preserveColumnCase = fileFormatProperties.getFileFormatType()
== TFileFormatType.FORMAT_JSON;
for (int idx = 0; idx < result.getColumnNums(); ++idx) {
PTypeDesc type = result.getColumnTypes(idx);
- String colName = result.getColumnNames(idx).toLowerCase();
+ String originalColName = result.getColumnNames(idx);
+ String colLowerName = originalColName.toLowerCase();
// Since doris does not distinguish between upper and lower case
columns when querying, in order to avoid
// query ambiguity, two columns with the same name but different
capitalization are not allowed.
- if (columnLowerNames.contains(colName)) {
- throw new NotSupportedException("Repeated lowercase column
names: " + colName);
+ if (columnLowerNames.contains(colLowerName)) {
+ throw new NotSupportedException("Repeated lowercase column
names: " + colLowerName);
} else {
- columnLowerNames.add(colName);
+ columnLowerNames.add(colLowerName);
+ String colName = preserveColumnCase ? originalColName :
colLowerName;
columns.add(new Column(colName,
getColumnType(type.getTypesList(), 0).key(), true));
}
}
diff --git
a/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out
b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out
new file mode 100644
index 00000000000..1dba9bb2590
--- /dev/null
+++ b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out
@@ -0,0 +1,9 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !json_uppercase_keys --
+1 Beijing 12.5
+2 Shanghai 7.75
+
+-- !json_uppercase_keys_with_jsonpaths --
+1 Beijing 12.5
+2 Shanghai 7.75
+
diff --git
a/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl
b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl
new file mode 100644
index 00000000000..8b1e11f4b33
--- /dev/null
+++
b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl
@@ -0,0 +1,2 @@
+{"ID":1,"City":"Beijing","Metric":12.5}
+{"ID":2,"City":"Shanghai","Metric":7.75}
diff --git
a/regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy
b/regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy
new file mode 100644
index 00000000000..9953855e883
--- /dev/null
+++
b/regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_json_uppercase_keys", "p0,external") {
+ String ak = getS3AK()
+ String sk = getS3SK()
+ String s3Endpoint = getS3Endpoint()
+ String bucket = context.config.otherConfigs.get("s3BucketName")
+ String uri =
"https://${bucket}.${s3Endpoint}/regression/tvf/test_json_uppercase_keys/case_keys.jsonl";
+
+ order_qt_json_uppercase_keys """
+ select id, city, metric from
+ s3(
+ "URI" = "${uri}",
+ "s3.access_key" = "${ak}",
+ "s3.secret_key" = "${sk}",
+ "FORMAT" = "json",
+ "read_json_by_line" = "true",
+ "fuzzy_parse" = "true",
+ "use_path_style" = "false") order by id;
+ """
+
+ order_qt_json_uppercase_keys_with_jsonpaths """
+ select id, city, metric from
+ s3(
+ "URI" = "${uri}",
+ "s3.access_key" = "${ak}",
+ "s3.secret_key" = "${sk}",
+ "FORMAT" = "json",
+ "read_json_by_line" = "true",
+ "fuzzy_parse" = "true",
+ "jsonpaths" = '["\$.ID", "\$.City", "\$.Metric"]',
+ "use_path_style" = "false") order by id;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
