This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ab9b29f11f [fix](multi-catalog)fix broker and tvf params (#23252)
ab9b29f11f is described below
commit ab9b29f11f9d4dab4ad0aa2f1d654a104c715161
Author: slothever <[email protected]>
AuthorDate: Wed Aug 23 10:33:24 2023 +0800
[fix](multi-catalog)fix broker and tvf params (#23252)
fix fs specifics, broker and tvf params, and NPE
---
fe/fe-core/src/main/java/org/apache/doris/analysis/S3TvfLoadStmt.java | 4 +++-
fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java | 3 ++-
.../src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java | 2 ++
.../org/apache/doris/planner/external/iceberg/IcebergScanNode.java | 2 +-
4 files changed, 8 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/S3TvfLoadStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/S3TvfLoadStmt.java
index 60ed5de036..acbdb92070 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/S3TvfLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/S3TvfLoadStmt.java
@@ -415,7 +415,9 @@ public class S3TvfLoadStmt extends NativeInsertStmt {
// rewrite where predicate and order by elements
final SelectStmt selectStmt = (SelectStmt) getQueryStmt();
rewriteSlotRefInExpr(selectStmt.getWhereClause());
- selectStmt.getOrderByElements().forEach(orderByElement ->
rewriteSlotRefInExpr(orderByElement.getExpr()));
+ if (selectStmt.getOrderByElements() != null) {
+ selectStmt.getOrderByElements().forEach(orderByElement ->
rewriteSlotRefInExpr(orderByElement.getExpr()));
+ }
}
private void rewriteSlotRefInExpr(Expr expr) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
index eca8384cfa..c2a070cdfa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
@@ -62,8 +62,9 @@ public class FileSystemFactory {
if (S3Util.isHdfsOnOssEndpoint(location)) {
// if hdfs service is enabled on oss, use hdfs lib to access
oss.
fsType = FileSystemType.DFS;
+ } else {
+ fsType = FileSystemType.S3;
}
- fsType = FileSystemType.S3;
} else if (location.startsWith(FeConstants.FS_PREFIX_HDFS) ||
location.startsWith(FeConstants.FS_PREFIX_GFS)) {
fsType = FileSystemType.DFS;
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS) ||
location.startsWith(FeConstants.FS_PREFIX_COSN)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java
index 1373ac1d6d..cb87150928 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java
@@ -25,6 +25,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.BrokerUtil;
+import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.fs.operations.BrokerFileOperations;
import org.apache.doris.fs.operations.OpParams;
import org.apache.doris.service.FrontendOptions;
@@ -74,6 +75,7 @@ public class BrokerFileSystem extends RemoteFileSystem {
public BrokerFileSystem(String name, Map<String, String> properties) {
super(name, StorageBackend.StorageType.BROKER);
+
properties.putAll(PropertyConverter.convertToHadoopFSProperties(properties));
this.properties = properties;
this.operations = new BrokerFileOperations(name, properties);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
index c89c606b8b..9dcd1716be 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
@@ -321,7 +321,7 @@ public class IcebergScanNode extends FileQueryScanNode {
private String normalizeLocation(String location) {
Map<String, String> props = source.getCatalog().getProperties();
String icebergCatalogType =
props.get(IcebergExternalCatalog.ICEBERG_CATALOG_TYPE);
- if (icebergCatalogType.equalsIgnoreCase("hadoop")) {
+ if ("hadoop".equalsIgnoreCase(icebergCatalogType)) {
if (!location.startsWith(HdfsResource.HDFS_PREFIX)) {
String fsName = props.get(HdfsResource.HADOOP_FS_NAME);
location = fsName + location;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
