This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 19ccb9517f [fix](iceberg) should call UserGroupInformation when enable
security authentication (#24614)
19ccb9517f is described below
commit 19ccb9517f98f0b9aab372c647573d0138ff7c86
Author: Ashin Gau <[email protected]>
AuthorDate: Tue Sep 19 22:39:58 2023 +0800
[fix](iceberg) should call UserGroupInformation when enable security
authentication (#24614)
Fix two bugs:
1. Call `UserGroupInformation.doAs` when enable security authentication
2. `catalogId` is 0 when `IcebergExternalCatalog` is loaded from fe image
---
.../doris/catalog/HiveMetaStoreClientHelper.java | 19 ++++++++++++++++
.../catalog/external/IcebergExternalTable.java | 26 +++++++++++++---------
.../apache/doris/datasource/ExternalCatalog.java | 11 +++++++++
.../datasource/iceberg/IcebergExternalCatalog.java | 20 +++--------------
.../datasource/paimon/PaimonExternalCatalog.java | 11 ---------
.../doris/planner/external/hudi/HudiScanNode.java | 19 +++-------------
.../external/iceberg/IcebergMetadataCache.java | 6 +++--
.../planner/external/iceberg/IcebergScanNode.java | 5 +++++
8 files changed, 60 insertions(+), 57 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
index 4b5c2b1d7c..8aa7d39090 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
@@ -35,6 +35,7 @@ import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.property.constants.HMSProperties;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.fs.RemoteFiles;
@@ -927,6 +928,24 @@ public class HiveMetaStoreClientHelper {
return ugi;
}
+ public static <T> T ugiDoAs(long catalogId, PrivilegedExceptionAction<T>
action) {
+ return ugiDoAs(((ExternalCatalog)
Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogId)).getConfiguration(),
+ action);
+ }
+
+ public static <T> T ugiDoAs(Configuration conf,
PrivilegedExceptionAction<T> action) {
+ UserGroupInformation ugi = getUserGroupInformation(conf);
+ try {
+ if (ugi != null) {
+ return ugi.doAs(action);
+ } else {
+ return action.run();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e.getMessage(), e.getCause());
+ }
+ }
+
public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
String hudiBasePath = table.getRemoteTable().getSd().getLocation();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
index c2be8b90a0..bede9b99e4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java
@@ -19,6 +19,7 @@ package org.apache.doris.catalog.external;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.HiveMetaStoreClientHelper;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
@@ -60,15 +61,17 @@ public class IcebergExternalTable extends ExternalTable {
@Override
public List<Column> initSchema() {
- Schema schema = ((IcebergExternalCatalog)
catalog).getIcebergTable(dbName, name).schema();
- List<Types.NestedField> columns = schema.columns();
- List<Column> tmpSchema =
Lists.newArrayListWithCapacity(columns.size());
- for (Types.NestedField field : columns) {
- tmpSchema.add(new Column(field.name(),
- icebergTypeToDorisType(field.type()), true, null, true,
field.doc(), true,
- schema.caseInsensitiveFindField(field.name()).fieldId()));
- }
- return tmpSchema;
+ return HiveMetaStoreClientHelper.ugiDoAs(catalog.getConfiguration(),
() -> {
+ Schema schema = ((IcebergExternalCatalog)
catalog).getIcebergTable(dbName, name).schema();
+ List<Types.NestedField> columns = schema.columns();
+ List<Column> tmpSchema =
Lists.newArrayListWithCapacity(columns.size());
+ for (Types.NestedField field : columns) {
+ tmpSchema.add(new Column(field.name(),
+ icebergTypeToDorisType(field.type()), true, null,
true, field.doc(), true,
+
schema.caseInsensitiveFindField(field.name()).fieldId()));
+ }
+ return tmpSchema;
+ });
}
private Type
icebergPrimitiveTypeToDorisType(org.apache.iceberg.types.Type.PrimitiveType
primitive) {
@@ -141,7 +144,8 @@ public class IcebergExternalTable extends ExternalTable {
@Override
public Optional<ColumnStatistic> getColumnStatistic(String colName) {
makeSureInitialized();
- return StatisticsUtil.getIcebergColumnStats(colName,
- ((IcebergExternalCatalog) catalog).getIcebergTable(dbName, name));
+ return HiveMetaStoreClientHelper.ugiDoAs(catalog.getConfiguration(),
+ () -> StatisticsUtil.getIcebergColumnStats(colName,
+ ((IcebergExternalCatalog)
catalog).getIcebergTable(dbName, name)));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 21617b2f05..986c63aa96 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -49,6 +49,8 @@ import com.google.gson.annotations.SerializedName;
import lombok.Data;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
@@ -108,6 +110,15 @@ public abstract class ExternalCatalog
this.comment = com.google.common.base.Strings.nullToEmpty(comment);
}
+ public Configuration getConfiguration() {
+ Configuration conf = new HdfsConfiguration();
+ Map<String, String> catalogProperties =
catalogProperty.getHadoopProperties();
+ for (Map.Entry<String, String> entry : catalogProperties.entrySet()) {
+ conf.set(entry.getKey(), entry.getValue());
+ }
+ return conf;
+ }
+
protected List<String> listDatabaseNames() {
throw new UnsupportedOperationException("Unsupported operation: "
+ "listDatabaseNames from remote client when init catalog with
" + logType.name());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
index 8df4acfc8f..20e2a7ebfe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
@@ -25,8 +25,6 @@ import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.SessionContext;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.SupportsNamespaces;
@@ -35,7 +33,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
public abstract class IcebergExternalCatalog extends ExternalCatalog {
@@ -50,11 +47,9 @@ public abstract class IcebergExternalCatalog extends
ExternalCatalog {
protected String icebergCatalogType;
protected Catalog catalog;
protected SupportsNamespaces nsCatalog;
- private final long catalogId;
public IcebergExternalCatalog(long catalogId, String name, String comment)
{
super(catalogId, name, InitCatalogLog.Type.ICEBERG, comment);
- this.catalogId = catalogId;
}
@Override
@@ -63,15 +58,6 @@ public abstract class IcebergExternalCatalog extends
ExternalCatalog {
super.init();
}
- protected Configuration getConfiguration() {
- Configuration conf = new HdfsConfiguration();
- Map<String, String> catalogProperties =
catalogProperty.getHadoopProperties();
- for (Map.Entry<String, String> entry : catalogProperties.entrySet()) {
- conf.set(entry.getKey(), entry.getValue());
- }
- return conf;
- }
-
public Catalog getCatalog() {
makeSureInitialized();
return catalog;
@@ -118,8 +104,8 @@ public abstract class IcebergExternalCatalog extends
ExternalCatalog {
public org.apache.iceberg.Table getIcebergTable(String dbName, String
tblName) {
makeSureInitialized();
return Env.getCurrentEnv()
- .getExtMetaCacheMgr()
- .getIcebergMetadataCache()
- .getIcebergTable(catalog, catalogId, dbName, tblName);
+ .getExtMetaCacheMgr()
+ .getIcebergMetadataCache()
+ .getIcebergTable(catalog, id, dbName, tblName);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
index 9e9422a95e..d2a3571f3b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
@@ -23,8 +23,6 @@ import org.apache.doris.datasource.SessionContext;
import org.apache.doris.datasource.property.constants.PaimonProperties;
import com.google.common.collect.Maps;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.paimon.catalog.Catalog;
@@ -54,15 +52,6 @@ public abstract class PaimonExternalCatalog extends
ExternalCatalog {
super.init();
}
- protected Configuration getConfiguration() {
- Configuration conf = new HdfsConfiguration();
- Map<String, String> catalogProperties =
catalogProperty.getHadoopProperties();
- for (Map.Entry<String, String> entry : catalogProperties.entrySet()) {
- conf.set(entry.getKey(), entry.getValue());
- }
- return conf;
- }
-
public Catalog getCatalog() {
makeSureInitialized();
return catalog;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
index 328c7b0f19..b0306ab19c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
@@ -46,7 +46,6 @@ import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.BaseFile;
@@ -62,7 +61,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -254,20 +252,9 @@ public class HudiScanNode extends HiveScanNode {
snapshotTimestamp = Option.empty();
}
// Non partition table will get one dummy partition
- UserGroupInformation ugi =
HiveMetaStoreClientHelper.getUserGroupInformation(
- HiveMetaStoreClientHelper.getConfiguration(hmsTable));
- List<HivePartition> partitions;
- if (ugi != null) {
- try {
- partitions = ugi.doAs(
- (PrivilegedExceptionAction<List<HivePartition>>) () ->
getPrunedPartitions(hudiClient,
- snapshotTimestamp));
- } catch (Exception e) {
- throw new UserException(e);
- }
- } else {
- partitions = getPrunedPartitions(hudiClient, snapshotTimestamp);
- }
+ List<HivePartition> partitions = HiveMetaStoreClientHelper.ugiDoAs(
+ HiveMetaStoreClientHelper.getConfiguration(hmsTable),
+ () -> getPrunedPartitions(hudiClient, snapshotTimestamp));
Executor executor = ((HudiCachedPartitionProcessor) Env.getCurrentEnv()
.getExtMetaCacheMgr().getHudiPartitionProcess(hmsTable.getCatalog())).getExecutor();
List<Split> splits = Collections.synchronizedList(new ArrayList<>());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java
index 564c1b4955..1f8b226e45 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java
@@ -18,6 +18,7 @@
package org.apache.doris.planner.external.iceberg;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.HiveMetaStoreClientHelper;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.Config;
import org.apache.doris.common.MetaNotFoundException;
@@ -132,7 +133,8 @@ public class IcebergMetadataCache {
if (cacheTable != null) {
return cacheTable;
}
- Table table = catalog.loadTable(TableIdentifier.of(dbName, tbName));
+ Table table = HiveMetaStoreClientHelper.ugiDoAs(catalogId,
+ () -> catalog.loadTable(TableIdentifier.of(dbName, tbName)));
initIcebergTableFileIO(table);
tableCache.put(key, table);
@@ -200,7 +202,7 @@ public class IcebergMetadataCache {
catalogProperties.put("uri", uri);
hiveCatalog.initialize("hive", catalogProperties);
- Table table = hiveCatalog.loadTable(TableIdentifier.of(db, tbl));
+ Table table = HiveMetaStoreClientHelper.ugiDoAs(conf, () ->
hiveCatalog.loadTable(TableIdentifier.of(db, tbl)));
initIcebergTableFileIO(table);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
index 4837ba5545..85a68aa785 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
@@ -24,6 +24,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.HdfsResource;
+import org.apache.doris.catalog.HiveMetaStoreClientHelper;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalTable;
@@ -169,6 +170,10 @@ public class IcebergScanNode extends FileQueryScanNode {
@Override
public List<Split> getSplits() throws UserException {
+ return
HiveMetaStoreClientHelper.ugiDoAs(source.getCatalog().getConfiguration(),
this::doGetSplits);
+ }
+
+ private List<Split> doGetSplits() throws UserException {
TableScan scan = icebergTable.newScan();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
