This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 8c51eb23e12 branch-4.0: [fix](fe) Normalize default HDFS paths in
LocationPath #63476 (#63769)
8c51eb23e12 is described below
commit 8c51eb23e12acd0320a1e4e707107d24c6eb35a2
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Jun 26 11:14:45 2026 +0800
branch-4.0: [fix](fe) Normalize default HDFS paths in LocationPath #63476
(#63769)
Cherry-picked from #63476
Co-authored-by: Socrates <[email protected]>
---
.../java/org/apache/doris/common/util/LocationPath.java | 14 +++++++++++++-
.../org/apache/doris/common/util/LocationPathTest.java | 15 +++++++++++++++
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index 1ce6576635d..21d7c736846 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -149,6 +149,9 @@ public class LocationPath {
String encodedLocation = encodedLocation(normalizedLocation);
URI uri = URI.create(encodedLocation);
String fsIdentifier = Strings.nullToEmpty(uri.getScheme()) + "://" +
Strings.nullToEmpty(uri.getAuthority());
+ if (StringUtils.isBlank(schema)) {
+ schema = Strings.nullToEmpty(uri.getScheme());
+ }
return new LocationPath(schema, normalizedLocation, fsIdentifier,
storageProperties);
}
@@ -193,6 +196,9 @@ public class LocationPath {
URI uri = URI.create(encodedLocation);
String fsIdentifier = Strings.nullToEmpty(uri.getScheme()) + "://"
+ Strings.nullToEmpty(uri.getAuthority());
+ if (StringUtils.isBlank(schema)) {
+ schema = Strings.nullToEmpty(uri.getScheme());
+ }
return new LocationPath(schema, normalizedLocation, fsIdentifier,
storageProperties);
} catch (UserException e) {
throw new StoragePropertiesException("Failed to create
LocationPath for location: " + location, e);
@@ -234,6 +240,7 @@ public class LocationPath {
String normalizedLocation =
storageProperties.validateAndNormalizeUri(location);
String fsIdentifier;
+ String schema = cachedSchema;
if (cachedFsIdPrefix != null &&
normalizedLocation.startsWith(cachedFsIdPrefix)) {
// Fast path: extract authority from normalized location
without full URI parsing
int authorityStart = cachedFsIdPrefix.length();
@@ -246,6 +253,9 @@ public class LocationPath {
throw new StoragePropertiesException("Invalid location,
missing authority: " + normalizedLocation);
}
fsIdentifier = cachedFsIdPrefix + authority;
+ if (StringUtils.isBlank(schema)) {
+ schema = cachedFsIdPrefix.substring(0,
cachedFsIdPrefix.length() - SCHEME_DELIM.length());
+ }
} else {
// Fallback to full URI parsing
String encodedLocation = encodedLocation(normalizedLocation);
@@ -256,9 +266,11 @@ public class LocationPath {
}
fsIdentifier = Strings.nullToEmpty(uri.getScheme()) + "://"
+ authority;
+ if (StringUtils.isBlank(schema)) {
+ schema = Strings.nullToEmpty(uri.getScheme());
+ }
}
- String schema = cachedSchema != null ? cachedSchema :
extractScheme(location);
return new LocationPath(schema, normalizedLocation, fsIdentifier,
storageProperties);
} catch (UserException e) {
throw new StoragePropertiesException("Failed to create
LocationPath for location: " + location, e);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
index c2efae6532e..6288f33ea51 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
@@ -333,6 +333,21 @@ public class LocationPathTest {
Assertions.assertEquals(full.getSchema(), cached.getSchema());
}
+ @Test
+ public void testLocationPathWithCacheUsesDefaultFsForHdfsPath() {
+ StorageProperties storageProperties =
STORAGE_PROPERTIES_MAP.get(StorageProperties.Type.HDFS);
+ String location =
"/hadoop_catalog/fdm/f_csm_t_consume_info/data/data_dt=20220407/file.parquet";
+ LocationPath cached = LocationPath.ofWithCache(location,
storageProperties, null, null);
+ LocationPath full = LocationPath.of(location, STORAGE_PROPERTIES_MAP);
+ Assertions.assertEquals(full.getNormalizedLocation(),
cached.getNormalizedLocation());
+
Assertions.assertEquals("hdfs://namenode:8020/hadoop_catalog/fdm/f_csm_t_consume_info/data/"
+ + "data_dt=20220407/file.parquet",
cached.getNormalizedLocation());
+ Assertions.assertEquals("hdfs://namenode:8020",
cached.getFsIdentifier());
+ Assertions.assertEquals("hdfs", cached.getSchema());
+ Assertions.assertEquals(TFileType.FILE_HDFS,
cached.getTFileTypeForBE());
+ Assertions.assertEquals(FileSystemType.HDFS,
cached.getFileSystemType());
+ }
+
@Test
public void testLocationPathWithCacheMissingAuthority() {
StorageProperties storageProperties =
STORAGE_PROPERTIES_MAP.get(StorageProperties.Type.S3);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
