This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 56043de25cf branch-4.2: [fix](fe) Preserve inferred JSON column name
casing (#69451)
56043de25cf is described below
commit 56043de25cf9bfe58cd5aae670358a549a19605d
Author: hui lai <[email protected]>
AuthorDate: Fri Oct 23 14:06:29 2026 +0800
branch-4.2: [fix](fe) Preserve inferred JSON column name casing (#69451)
### What problem does this PR solve?
Issue Number: None
Related PR: #67411
Problem Summary: Backport #67411 to branch-4.2. JSON schema inference
lowercased field names while JSON readers match object keys
case-sensitively, causing mixed-case and uppercase keys to return NULL.
Preserve inferred JSON key casing, retain case-insensitive duplicate
detection, and keep existing lowercase behavior for other file formats.
The conflict resolution also preserves branch-4.2's existing Lance
column name casing behavior.
### Release note
JSON table-valued functions now read inferred mixed-case and uppercase
keys without silently producing NULL values.
### Check List (For Author)
- Test: Compilation and tests skipped as requested. The original PR's
regression test and fixture are included in this backport. CI requested
with `run buildall`.
- Behavior changed: Yes. Inferred JSON field names preserve source
casing.
- Does this need documentation: No.
---
.../ExternalFileTableValuedFunction.java | 14 ++++---
.../tvf/test_json_uppercase_keys.out | 9 ++++
.../tvf/test_json_uppercase_keys/case_keys.jsonl | 2 +
.../tvf/test_json_uppercase_keys.groovy | 49 ++++++++++++++++++++++
4 files changed, 68 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index 398cbb4e02c..c26bc2f472f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -544,18 +544,20 @@ public abstract class ExternalFileTableValuedFunction
extends TableValuedFunctio
}
// add fetched file columns
Set<String> columnLowerNames = new HashSet<>();
+ boolean preserveColumnCase = isLanceFormat()
+ || fileFormatProperties.getFileFormatType() ==
TFileFormatType.FORMAT_JSON;
for (int idx = 0; idx < result.getColumnNums(); ++idx) {
PTypeDesc type = result.getColumnTypes(idx);
String originalColName = result.getColumnNames(idx);
- String colName = originalColName.toLowerCase();
+ String colLowerName = originalColName.toLowerCase();
// Since doris does not distinguish between upper and lower case
columns when querying, in order to avoid
// query ambiguity, two columns with the same name but different
capitalization are not allowed.
- if (columnLowerNames.contains(colName)) {
- throw new NotSupportedException("Repeated lowercase column
names: " + colName);
+ if (columnLowerNames.contains(colLowerName)) {
+ throw new NotSupportedException("Repeated lowercase column
names: " + colLowerName);
} else {
- columnLowerNames.add(colName);
- columns.add(new Column(isLanceFormat() ? originalColName :
colName,
- getColumnType(type.getTypesList(), 0).key(), true));
+ columnLowerNames.add(colLowerName);
+ String colName = preserveColumnCase ? originalColName :
colLowerName;
+ columns.add(new Column(colName,
getColumnType(type.getTypesList(), 0).key(), true));
}
}
// add path columns
diff --git
a/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out
b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out
new file mode 100644
index 00000000000..1dba9bb2590
--- /dev/null
+++ b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys.out
@@ -0,0 +1,9 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !json_uppercase_keys --
+1 Beijing 12.5
+2 Shanghai 7.75
+
+-- !json_uppercase_keys_with_jsonpaths --
+1 Beijing 12.5
+2 Shanghai 7.75
+
diff --git
a/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl
b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl
new file mode 100644
index 00000000000..8b1e11f4b33
--- /dev/null
+++
b/regression-test/data/external_table_p0/tvf/test_json_uppercase_keys/case_keys.jsonl
@@ -0,0 +1,2 @@
+{"ID":1,"City":"Beijing","Metric":12.5}
+{"ID":2,"City":"Shanghai","Metric":7.75}
diff --git
a/regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy
b/regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy
new file mode 100644
index 00000000000..9953855e883
--- /dev/null
+++
b/regression-test/suites/external_table_p0/tvf/test_json_uppercase_keys.groovy
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_json_uppercase_keys", "p0,external") {
+ String ak = getS3AK()
+ String sk = getS3SK()
+ String s3Endpoint = getS3Endpoint()
+ String bucket = context.config.otherConfigs.get("s3BucketName")
+ String uri =
"https://${bucket}.${s3Endpoint}/regression/tvf/test_json_uppercase_keys/case_keys.jsonl";
+
+ order_qt_json_uppercase_keys """
+ select id, city, metric from
+ s3(
+ "URI" = "${uri}",
+ "s3.access_key" = "${ak}",
+ "s3.secret_key" = "${sk}",
+ "FORMAT" = "json",
+ "read_json_by_line" = "true",
+ "fuzzy_parse" = "true",
+ "use_path_style" = "false") order by id;
+ """
+
+ order_qt_json_uppercase_keys_with_jsonpaths """
+ select id, city, metric from
+ s3(
+ "URI" = "${uri}",
+ "s3.access_key" = "${ak}",
+ "s3.secret_key" = "${sk}",
+ "FORMAT" = "json",
+ "read_json_by_line" = "true",
+ "fuzzy_parse" = "true",
+ "jsonpaths" = '["\$.ID", "\$.City", "\$.Metric"]',
+ "use_path_style" = "false") order by id;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
