This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new bfe74695603 branch-2.1: [fix](oss) the write to hive table on oss-hdfs
may fail (#49754) (#49911)
bfe74695603 is described below
commit bfe7469560302b55a42284ba766c1d8454b38a61
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Thu Apr 10 07:01:42 2025 -0700
branch-2.1: [fix](oss) the write to hive table on oss-hdfs may fail
(#49754) (#49911)
bp #49754
---
.../org/apache/doris/common/util/LocationPath.java | 5 ++--
.../doris/datasource/hive/HiveMetaStoreCache.java | 13 +++++----
.../doris/fs/remote/SwitchingFileSystem.java | 2 +-
.../apache/doris/common/util/LocationPathTest.java | 33 +++++++++++++++-------
4 files changed, 34 insertions(+), 19 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index 4ca8f9605a0..7e8b357395c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -203,8 +203,9 @@ public class LocationPath {
// Return the file system type and the file system identity.
// The file system identity is the scheme and authority of the URI, eg.
"hdfs://host:port" or "s3://bucket".
- public static Pair<FileSystemType, String> getFSIdentity(String location,
String bindBrokerName) {
- LocationPath locationPath = new LocationPath(location,
Collections.emptyMap(), true);
+ public static Pair<FileSystemType, String> getFSIdentity(String location,
+ Map<String, String> properties, String bindBrokerName) {
+ LocationPath locationPath = new LocationPath(location, properties,
true);
FileSystemType fsType = (bindBrokerName != null) ?
FileSystemType.BROKER : locationPath.getFileSystemType();
URI uri = locationPath.getPath().toUri();
String fsIdent = Strings.nullToEmpty(uri.getScheme()) + "://" +
Strings.nullToEmpty(uri.getAuthority());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 751919e85f4..ecb0fa60693 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -349,10 +349,11 @@ public class HiveMetaStoreCache {
List<String> partitionValues,
String bindBrokerName) throws UserException {
FileCacheValue result = new FileCacheValue();
+ Map<String, String> properties =
catalog.getCatalogProperty().getProperties();
RemoteFileSystem fs =
Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
new
FileSystemCache.FileSystemCacheKey(LocationPath.getFSIdentity(
- location, bindBrokerName),
- catalog.getCatalogProperty().getProperties(),
+ location, properties, bindBrokerName),
+ properties,
bindBrokerName, jobConf));
result.setSplittable(HiveUtil.isSplittable(fs, inputFormat, location));
// For Tez engine, it may generate subdirectoies for "union" query.
@@ -737,8 +738,8 @@ public class HiveMetaStoreCache {
boolean isFullAcid, boolean skipCheckingAcidVersionFile, long
tableId, String bindBrokerName) {
List<FileCacheValue> fileCacheValues = Lists.newArrayList();
try {
+ Map<String, String> properties =
catalog.getCatalogProperty().getProperties();
for (HivePartition partition : partitions) {
-
AuthenticationConfig authenticationConfig =
AuthenticationConfig.getKerberosConfig(jobConf);
HadoopAuthenticator hadoopAuthenticator =
HadoopAuthenticator.getHadoopAuthenticator(authenticationConfig);
@@ -771,7 +772,7 @@ public class HiveMetaStoreCache {
RemoteFileSystem fs =
Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
new FileSystemCache.FileSystemCacheKey(
LocationPath.getFSIdentity(baseOrDeltaPath.toUri().toString(),
- bindBrokerName),
+ properties, bindBrokerName),
catalog.getCatalogProperty().getProperties(),
bindBrokerName, jobConf));
Status status = fs.exists(acidVersionPath);
@@ -798,7 +799,7 @@ public class HiveMetaStoreCache {
String location = delta.getPath().toString();
RemoteFileSystem fs =
Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
new FileSystemCache.FileSystemCacheKey(
- LocationPath.getFSIdentity(location,
bindBrokerName),
+ LocationPath.getFSIdentity(location,
properties, bindBrokerName),
catalog.getCatalogProperty().getProperties(), bindBrokerName, jobConf));
List<RemoteFile> remoteFiles = new ArrayList<>();
Status status = fs.listFiles(location, false, remoteFiles);
@@ -826,7 +827,7 @@ public class HiveMetaStoreCache {
String location = directory.getBaseDirectory().toString();
RemoteFileSystem fs =
Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
new FileSystemCache.FileSystemCacheKey(
- LocationPath.getFSIdentity(location,
bindBrokerName),
+ LocationPath.getFSIdentity(location,
properties, bindBrokerName),
catalog.getCatalogProperty().getProperties(), bindBrokerName, jobConf));
List<RemoteFile> remoteFiles = new ArrayList<>();
Status status = fs.listFiles(location, false, remoteFiles);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java
index 00802922ef3..ab7c91d693a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java
@@ -125,7 +125,7 @@ public class SwitchingFileSystem implements FileSystem {
public FileSystem fileSystem(String location) {
return extMetaCacheMgr.getFsCache().getRemoteFileSystem(
new FileSystemCache.FileSystemCacheKey(
- LocationPath.getFSIdentity(location,
+ LocationPath.getFSIdentity(location, properties,
bindBrokerName), properties, bindBrokerName));
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
index 4457b7dd1ef..1e9e3e1ab88 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
@@ -25,6 +25,7 @@ import org.apache.doris.fs.FileSystemType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -39,7 +40,8 @@ public class LocationPathTest {
String beLocation = locationPath.toStorageLocation().toString();
Assertions.assertTrue(beLocation.startsWith("hdfs://"));
- Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
null).first, FileSystemType.DFS);
+ Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
Collections.emptyMap(), null).first,
+ FileSystemType.DFS);
// HA props
Map<String, String> props = new HashMap<>();
@@ -92,7 +94,8 @@ public class LocationPathTest {
// BE
loc = locationPath.toStorageLocation().toString();
Assertions.assertTrue(loc.startsWith("jfs://"));
- Assertions.assertEquals(LocationPath.getFSIdentity(loc, null).first,
FileSystemType.JFS);
+ Assertions.assertEquals(LocationPath.getFSIdentity(loc,
Collections.emptyMap(), null).first,
+ FileSystemType.JFS);
}
@Test
@@ -106,7 +109,8 @@ public class LocationPathTest {
// BE
String beLoc = locationPath.toStorageLocation().toString();
Assertions.assertTrue(beLoc.startsWith("s3://"));
- Assertions.assertEquals(LocationPath.getFSIdentity(beLoc, null).first,
FileSystemType.S3);
+ Assertions.assertEquals(LocationPath.getFSIdentity(beLoc,
Collections.emptyMap(), null).first,
+ FileSystemType.S3);
}
@Test
@@ -118,17 +122,21 @@ public class LocationPathTest {
// BE
String beLocation = locationPath.toStorageLocation().toString();
Assertions.assertTrue(beLocation.startsWith("s3://"));
- Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
null).first, FileSystemType.S3);
+ Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
Collections.emptyMap(), null).first,
+ FileSystemType.S3);
+ // test oss-hdfs
rangeProps.put(OssProperties.ENDPOINT, "oss-dls.aliyuncs.com");
locationPath = new
LocationPath("oss://test.oss-dls.aliyuncs.com/path", rangeProps);
+ Assertions.assertEquals("oss://test.oss-dls.aliyuncs.com/path",
locationPath.get());
+ Assertions.assertEquals(LocationPath.getFSIdentity(locationPath.get(),
rangeProps, null).first,
+ FileSystemType.DFS);
// FE
Assertions.assertTrue(locationPath.get().startsWith("oss://test.oss-dls.aliyuncs"));
// BE
beLocation = locationPath.toStorageLocation().toString();
Assertions.assertTrue(beLocation.startsWith("oss://test.oss-dls.aliyuncs"));
Assertions.assertEquals(locationPath.getFileSystemType(),
FileSystemType.DFS);
-
}
@Test
@@ -140,7 +148,8 @@ public class LocationPathTest {
String beLocation = locationPath.toStorageLocation().toString();
// BE
Assertions.assertTrue(beLocation.startsWith("s3://"));
- Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
null).first, FileSystemType.S3);
+ Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
Collections.emptyMap(), null).first,
+ FileSystemType.S3);
locationPath = new LocationPath("cosn://test.com", rangeProps);
// FE
@@ -148,7 +157,8 @@ public class LocationPathTest {
// BE
beLocation = locationPath.toStorageLocation().toString();
Assertions.assertTrue(beLocation.startsWith("s3://"));
- Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
null).first, FileSystemType.S3);
+ Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
Collections.emptyMap(), null).first,
+ FileSystemType.S3);
locationPath = new LocationPath("ofs://test.com", rangeProps);
// FE
@@ -156,7 +166,8 @@ public class LocationPathTest {
// BE
beLocation = locationPath.toStorageLocation().toString();
Assertions.assertTrue(beLocation.startsWith("ofs://"));
- Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
null).first, FileSystemType.OFS);
+ Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
Collections.emptyMap(), null).first,
+ FileSystemType.OFS);
// GFS is now equals to DFS
locationPath = new LocationPath("gfs://test.com", rangeProps);
@@ -165,7 +176,8 @@ public class LocationPathTest {
// BE
beLocation = locationPath.toStorageLocation().toString();
Assertions.assertTrue(beLocation.startsWith("gfs://"));
- Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
null).first, FileSystemType.DFS);
+ Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
Collections.emptyMap(), null).first,
+ FileSystemType.DFS);
}
@Test
@@ -177,7 +189,8 @@ public class LocationPathTest {
// BE
String beLocation = locationPath.toStorageLocation().toString();
Assertions.assertTrue(beLocation.startsWith("s3://"));
- Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
null).first, FileSystemType.S3);
+ Assertions.assertEquals(LocationPath.getFSIdentity(beLocation,
Collections.emptyMap(), null).first,
+ FileSystemType.S3);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
