This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 94dee833cd [fix](multi-catalog)fix compatible with hdfs HA empty
prefix (#22424)
94dee833cd is described below
commit 94dee833cd76320a8bdfa5b22219767183945725
Author: slothever <[email protected]>
AuthorDate: Tue Aug 1 21:48:16 2023 +0800
[fix](multi-catalog)fix compatible with hdfs HA empty prefix (#22424)
---
docs/en/docs/lakehouse/multi-catalog/iceberg.md | 15 ++++++++
docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md | 15 ++++++++
.../org/apache/doris/catalog/HdfsResource.java | 2 +-
.../java/org/apache/doris/common/util/S3Util.java | 45 +++++++++++++++-------
4 files changed, 62 insertions(+), 15 deletions(-)
diff --git a/docs/en/docs/lakehouse/multi-catalog/iceberg.md
b/docs/en/docs/lakehouse/multi-catalog/iceberg.md
index 4509fbc4fa..54af57bee8 100644
--- a/docs/en/docs/lakehouse/multi-catalog/iceberg.md
+++ b/docs/en/docs/lakehouse/multi-catalog/iceberg.md
@@ -93,11 +93,26 @@ see [Alibaba Cloud DLF Catalog](dlf.md)
This method needs to provide REST services in advance, and users need to
implement the REST interface for obtaining Iceberg metadata.
+```sql
+CREATE CATALOG iceberg PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://172.21.0.1:8181'
+);
+```
+
+If the data is on HDFS and High Availability (HA) is set up, need to add HA
configuration to the Catalog.
+
```sql
CREATE CATALOG iceberg PROPERTIES (
'type'='iceberg',
'iceberg.catalog.type'='rest',
'uri' = 'http://172.21.0.1:8181',
+ 'dfs.nameservices'='your-nameservice',
+ 'dfs.ha.namenodes.your-nameservice'='nn1,nn2',
+ 'dfs.namenode.rpc-address.your-nameservice.nn1'='172.21.0.1:8020',
+ 'dfs.namenode.rpc-address.your-nameservice.nn2'='172.21.0.2:8020',
+
'dfs.client.failover.proxy.provider.your-nameservice'='org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider'
);
```
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
b/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
index c93fca28e5..ab5b447005 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
@@ -93,11 +93,26 @@ Iceberg 属性详情参见 [Iceberg Glue
Catalog](https://iceberg.apache.org/doc
该方式需要预先提供REST服务，用户需实现获取Iceberg元数据的REST接口。
+```sql
+CREATE CATALOG iceberg PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://172.21.0.1:8181'
+);
+```
+
+如果使用HDFS存储数据，并开启了高可用模式，还需在Catalog中增加HDFS高可用配置：
+
```sql
CREATE CATALOG iceberg PROPERTIES (
'type'='iceberg',
'iceberg.catalog.type'='rest',
'uri' = 'http://172.21.0.1:8181',
+ 'dfs.nameservices'='your-nameservice',
+ 'dfs.ha.namenodes.your-nameservice'='nn1,nn2',
+ 'dfs.namenode.rpc-address.your-nameservice.nn1'='172.21.0.1:8020',
+ 'dfs.namenode.rpc-address.your-nameservice.nn2'='172.21.0.2:8020',
+
'dfs.client.failover.proxy.provider.your-nameservice'='org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider'
);
```
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
index 2b50ec63b6..9735f2f059 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
@@ -53,7 +53,7 @@ public class HdfsResource extends Resource {
public static String HADOOP_SHORT_CIRCUIT = "dfs.client.read.shortcircuit";
public static String HADOOP_SOCKET_PATH = "dfs.domain.socket.path";
public static String DSF_NAMESERVICES = "dfs.nameservices";
- public static final String HDFS_PREFIX = "hdfs://";
+ public static final String HDFS_PREFIX = "hdfs:";
@SerializedName(value = "properties")
private Map<String, String> properties;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
index a47d838537..623e699fb6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
@@ -81,20 +81,8 @@ public class S3Util {
private static String normalizedLocation(String location, Map<String,
String> props) {
try {
- URI normalizedUri = new URI(location);
- if (StringUtils.isEmpty(normalizedUri.getHost()) &&
location.startsWith(HdfsResource.HDFS_PREFIX)) {
- // Need add hdfs host to location
- String host = props.get(HdfsResource.DSF_NAMESERVICES);
- if (StringUtils.isNotEmpty(host)) {
- // Replace 'hdfs://' to 'hdfs://name_service', for
example: hdfs:///abc to hdfs://name_service/abc
- return location.replace(HdfsResource.HDFS_PREFIX,
HdfsResource.HDFS_PREFIX + host);
- } else {
- // If no hadoop HA config
- if (location.startsWith(HdfsResource.HDFS_PREFIX + '/')) {
- // Do not support hdfs:///location
- throw new RuntimeException("Invalid location with
empty host: " + location);
- }
- }
+ if (location.startsWith(HdfsResource.HDFS_PREFIX)) {
+ return normalizedHdfsPath(location, props);
}
return location;
} catch (URISyntaxException e) {
@@ -102,6 +90,35 @@ public class S3Util {
}
}
+ private static String normalizedHdfsPath(String location, Map<String,
String> props) throws URISyntaxException {
+ URI normalizedUri = new URI(location);
+ // compatible with 'hdfs:///' or 'hdfs:/'
+ if (StringUtils.isEmpty(normalizedUri.getHost())) {
+ String normalizedPrefix = HdfsResource.HDFS_PREFIX + "//";
+ String brokenPrefix = HdfsResource.HDFS_PREFIX + "/";
+ if (location.startsWith(brokenPrefix) &&
!location.startsWith(normalizedPrefix)) {
+ location = location.replace(brokenPrefix, normalizedPrefix);
+ }
+ // Need add hdfs host to location
+ String host = props.get(HdfsResource.DSF_NAMESERVICES);
+ if (StringUtils.isNotEmpty(host)) {
+ // Replace 'hdfs://key/' to 'hdfs://name_service/key/'
+ // Or hdfs:///abc to hdfs://name_service/abc
+ return location.replace(normalizedPrefix, normalizedPrefix +
host + "/");
+ } else {
+ // 'hdfs://null/' equals the 'hdfs:///'
+ if (location.startsWith(HdfsResource.HDFS_PREFIX + "///")) {
+ // Do not support hdfs:///location
+ throw new RuntimeException("Invalid location with empty
host: " + location);
+ } else {
+ // Replace 'hdfs://key/' to '/key/', try access local
NameNode on BE.
+ return location.replace(normalizedPrefix, "/");
+ }
+ }
+ }
+ return location;
+ }
+
/**
* The converted path is used for BE
* @param location origin split path
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
