morningman commented on code in PR #10088:
URL: https://github.com/apache/incubator-doris/pull/10088#discussion_r896543681
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java:
##########
@@ -19,108 +19,165 @@
import org.apache.doris.alter.AlterCancelException;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* External table represent tables that are not self-managed by Doris.
* Such as tables from hive, iceberg, es, etc.
*/
public class ExternalTable implements TableIf {
+ private static final Logger LOG =
LogManager.getLogger(ExternalTable.class);
+
+ protected long id;
+ protected String name;
+ protected ReentrantReadWriteLock rwLock;
+ protected TableType type;
+ protected List<Column> fullSchema = null;
+
+ public ExternalTable() {
+ }
+
+ /**
+ * Create external table.
+ *
+ * @param id Table id.
+ * @param name Table name.
+ */
+ public ExternalTable(long id, String name) {
+ this.id = id;
+ this.name = name;
+ this.rwLock = new ReentrantReadWriteLock(true);
Review Comment:
Move this to the field definition place above.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java:
##########
@@ -599,4 +624,63 @@ public ExprBuilder val(TypeInfo ti, Object val) {
return this;
}
}
+
+ /**
+ * Convert hive type to doris type.
+ */
+ public static Type hiveTypeToDorisType(String hiveType) {
+ String lowerCaseType = hiveType.toLowerCase();
+ if (lowerCaseType.equals("boolean")) {
Review Comment:
Can use `switch case` to do this.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java:
##########
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog.external;
+
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.datasource.ExternalDataSource;
+import org.apache.doris.datasource.HMSExternalDataSource;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Hive metastore external database.
+ */
+public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
+
+ private static final Logger LOG =
LogManager.getLogger(HMSExternalDatabase.class);
+
+ private ConcurrentHashMap<String, Long> tableNameToId = new
ConcurrentHashMap<>();
+ private AtomicLong nextId = new AtomicLong(0);
+ private final String hiveMetastoreUri;
+ private final TableIf.TableType tableType;
+
+ /**
+ * Create HMS external database.
+ *
+ * @param extDataSource External data source this database belongs to.
+ * @param id database id.
+ * @param name database name.
+ * @param uri Hive metastore uri.
+ */
+ public HMSExternalDatabase(ExternalDataSource extDataSource, long id,
String name, String uri) {
+ super(extDataSource, id, name);
+ this.hiveMetastoreUri = uri;
+ HMSExternalDataSource.HMSType hmsType = ((HMSExternalDataSource)
extDataSource).getHmsType();
+ if (hmsType.equals(HMSExternalDataSource.HMSType.HIVE)) {
+ tableType = TableIf.TableType.HIVE;
+ } else if (hmsType.equals(HMSExternalDataSource.HMSType.ICEBERG)) {
+ tableType = TableIf.TableType.ICEBERG;
+ } else {
+ tableType = TableIf.TableType.HUDI;
+ }
+ }
+
+ @Override
+ public List<HMSExternalTable> getTables() {
+ List<HMSExternalTable> tables = new ArrayList<>();
+ List<String> tableNames = extDataSource.listTableNames(null, name);
+ for (String tableName : tableNames) {
+ tables.add(new HMSExternalTable(nextId.incrementAndGet(),
tableName, name, hiveMetastoreUri, tableType));
+ }
+ return tables;
+ }
+
+ @Override
+ public List<HMSExternalTable> getTablesOnIdOrder() {
+ // Sort the name instead, because the id may change.
+ return
getTables().stream().sorted(Comparator.comparing(TableIf::getName)).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<HMSExternalTable> getTablesOnIdOrderIfExist(List<Long>
tableIdList) {
+ return null;
Review Comment:
Throw NotImplementedException if we don't need to implement it.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java:
##########
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog.external;
+
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.datasource.ExternalDataSource;
+import org.apache.doris.datasource.HMSExternalDataSource;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Hive metastore external database.
+ */
+public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
+
+ private static final Logger LOG =
LogManager.getLogger(HMSExternalDatabase.class);
+
+ private ConcurrentHashMap<String, Long> tableNameToId = new
ConcurrentHashMap<>();
+ private AtomicLong nextId = new AtomicLong(0);
+ private final String hiveMetastoreUri;
+ private final TableIf.TableType tableType;
+
+ /**
+ * Create HMS external database.
+ *
+ * @param extDataSource External data source this database belongs to.
+ * @param id database id.
+ * @param name database name.
+ * @param uri Hive metastore uri.
+ */
+ public HMSExternalDatabase(ExternalDataSource extDataSource, long id,
String name, String uri) {
+ super(extDataSource, id, name);
+ this.hiveMetastoreUri = uri;
+ HMSExternalDataSource.HMSType hmsType = ((HMSExternalDataSource)
extDataSource).getHmsType();
Review Comment:
A `HMSExternalDataSource` may contains different `HmsType`, so I think we
can not put `tableType` in `HMSExternalDatabase`
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java:
##########
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog.external;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.HiveMetaStoreClientHelper;
+import org.apache.doris.common.DdlException;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Hive metastore external table.
+ */
+public class HMSExternalTable extends ExternalTable {
+
+ private static final Logger LOG =
LogManager.getLogger(HMSExternalTable.class);
+
+ private final String metastoreUri;
+ private final String dbName;
+ private org.apache.hadoop.hive.metastore.api.Table remoteTable = null;
+
+ /**
+ * Create hive metastore external table.
+ *
+ * @param id Table id.
+ * @param name Table name.
+ * @param dbName Database name.
+ * @param uri Hive metastore uri.
+ * @param type Table type.
+ */
+ public HMSExternalTable(long id, String name, String dbName, String uri,
TableType type) {
+ super(id, name, type);
+ this.dbName = dbName;
+ this.metastoreUri = uri;
+ }
+
+ /**
+ * Get the related remote hive table.
+ */
+ public org.apache.hadoop.hive.metastore.api.Table getRemoteTable() {
+ if (remoteTable == null) {
+ synchronized (this) {
+ if (remoteTable == null) {
+ try {
+ remoteTable =
HiveMetaStoreClientHelper.getTable(dbName, name, metastoreUri);
+ } catch (DdlException e) {
+ LOG.warn("Fail to get remote hive table. db {}, table
{}, uri {}", dbName, name, metastoreUri);
+ remoteTable = null;
+ }
+ }
+ }
+ }
+ return remoteTable;
+ }
+
+ @Override
+ public List<Column> getFullSchema() {
+ List<Column> schema = new ArrayList<>();
+ try {
+ for (FieldSchema field :
HiveMetaStoreClientHelper.getSchema(dbName, name, metastoreUri)) {
+ schema.add(new Column(field.getName(),
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()),
+ true, null, true, null, field.getComment()));
+ }
+ } catch (DdlException e) {
+ LOG.warn("Fail to get schema of hms table {}", name, e);
+ }
+ synchronized (this) {
Review Comment:
Should be act same as `getRemoteTable`.
Currently, I think we can just get schema once, and ignore the meta refresh,
it can be implement later.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java:
##########
@@ -32,31 +48,170 @@ public class HMSExternalDataSource extends
ExternalDataSource {
private static final Logger LOG =
LogManager.getLogger(HMSExternalDataSource.class);
public static final String HIVE_METASTORE_URIS = "hive.metastore.uris";
- public HMSExternalDataSource() {
+ protected final String hiveMetastoreUris;
+ protected final HiveMetaStoreClient client;
+ private final HMSType hmsType;
+
+ private ConcurrentHashMap<String, DatabaseIf> nameToDb = new
ConcurrentHashMap();
+
+ /**
+ * Hive metastore data source implementation.
+ *
+ * @param hiveMetastoreUris e.g. thrift://127.0.0.1:9083
+ * @param hmsType Hive metastore type. Hive, Iceberg or Hudi.
+ */
+ public HMSExternalDataSource(String hiveMetastoreUris, HMSType hmsType)
throws DdlException {
+ this.hiveMetastoreUris = hiveMetastoreUris;
+ this.hmsType = hmsType;
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, hiveMetastoreUris);
+ try {
+ client = new HiveMetaStoreClient(hiveConf);
+ } catch (MetaException e) {
+ LOG.warn("Create HiveMetaStoreClient failed: {}", e.getMessage());
+ throw new DdlException("Create HiveMetaStoreClient failed: " +
e.getMessage());
+ }
+ }
+
+ public HMSType getHmsType() {
+ return hmsType;
}
@Override
public List<String> listDatabaseNames(SessionContext ctx) {
+ try {
Review Comment:
I think we should throw exception instead of return null here?
Better change the interface.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java:
##########
@@ -19,108 +19,165 @@
import org.apache.doris.alter.AlterCancelException;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* External table represent tables that are not self-managed by Doris.
* Such as tables from hive, iceberg, es, etc.
*/
public class ExternalTable implements TableIf {
+ private static final Logger LOG =
LogManager.getLogger(ExternalTable.class);
+
+ protected long id;
+ protected String name;
+ protected ReentrantReadWriteLock rwLock;
+ protected TableType type;
+ protected List<Column> fullSchema = null;
+
+ public ExternalTable() {
+ }
+
+ /**
+ * Create external table.
+ *
+ * @param id Table id.
+ * @param name Table name.
+ */
+ public ExternalTable(long id, String name) {
+ this.id = id;
+ this.name = name;
+ this.rwLock = new ReentrantReadWriteLock(true);
+ }
+
+ /**
+ * Create external table.
+ *
+ * @param id Table id.
+ * @param name Table name.
+ * @param type Table type.
+ */
+ public ExternalTable(long id, String name, TableType type) {
+ this.id = id;
+ this.name = name;
+ this.type = type;
+ this.rwLock = new ReentrantReadWriteLock(true);
Review Comment:
Move this to the field definition place above.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java:
##########
@@ -46,74 +46,91 @@
private static final Logger LOG =
LogManager.getLogger(ExternalDatabase.class);
- private long id;
- private String name;
private ReentrantReadWriteLock rwLock;
- private ExternalDataSource extDataSource;
-
- public ExternalDatabase() {
+ protected long id;
+ protected String name;
+ protected ExternalDataSource extDataSource;
+ protected DatabaseProperty dbProperties;
+ /**
+ * Create external database.
+ *
+ * @param extDataSource The data source this database belongs to.
+ * @param id Database id.
+ * @param name Database name.
+ */
+ public ExternalDatabase(ExternalDataSource extDataSource, long id, String
name) {
+ this.extDataSource = extDataSource;
+ this.rwLock = new ReentrantReadWriteLock(true);
Review Comment:
Move this to the field definition place above.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java:
##########
@@ -19,108 +19,165 @@
import org.apache.doris.alter.AlterCancelException;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* External table represent tables that are not self-managed by Doris.
* Such as tables from hive, iceberg, es, etc.
*/
public class ExternalTable implements TableIf {
+ private static final Logger LOG =
LogManager.getLogger(ExternalTable.class);
+
+ protected long id;
+ protected String name;
+ protected ReentrantReadWriteLock rwLock;
+ protected TableType type;
+ protected List<Column> fullSchema = null;
+
+ public ExternalTable() {
+ }
+
+ /**
+ * Create external table.
+ *
+ * @param id Table id.
+ * @param name Table name.
+ */
+ public ExternalTable(long id, String name) {
+ this.id = id;
+ this.name = name;
+ this.rwLock = new ReentrantReadWriteLock(true);
+ }
+
+ /**
+ * Create external table.
+ *
+ * @param id Table id.
+ * @param name Table name.
+ * @param type Table type.
+ */
+ public ExternalTable(long id, String name, TableType type) {
+ this.id = id;
+ this.name = name;
+ this.type = type;
+ this.rwLock = new ReentrantReadWriteLock(true);
+ }
+
@Override
public void readLock() {
-
+ this.rwLock.readLock().lock();
}
@Override
public boolean tryReadLock(long timeout, TimeUnit unit) {
- return false;
+ try {
+ return this.rwLock.readLock().tryLock(timeout, unit);
+ } catch (InterruptedException e) {
+ LOG.warn("failed to try read lock at table[" + name + "]", e);
+ return false;
+ }
}
@Override
public void readUnlock() {
-
+ this.rwLock.readLock().unlock();
}
@Override
public void writeLock() {
-
+ this.rwLock.writeLock().lock();
}
@Override
public boolean writeLockIfExist() {
- return false;
+ this.rwLock.writeLock().lock();
Review Comment:
```suggestion
writeLock();
```
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java:
##########
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog.external;
+
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.datasource.ExternalDataSource;
+import org.apache.doris.datasource.HMSExternalDataSource;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Hive metastore external database.
+ */
+public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
+
+ private static final Logger LOG =
LogManager.getLogger(HMSExternalDatabase.class);
+
+ private ConcurrentHashMap<String, Long> tableNameToId = new
ConcurrentHashMap<>();
+ private AtomicLong nextId = new AtomicLong(0);
+ private final String hiveMetastoreUri;
+ private final TableIf.TableType tableType;
+
+ /**
+ * Create HMS external database.
+ *
+ * @param extDataSource External data source this database belongs to.
+ * @param id database id.
+ * @param name database name.
+ * @param uri Hive metastore uri.
+ */
+ public HMSExternalDatabase(ExternalDataSource extDataSource, long id,
String name, String uri) {
+ super(extDataSource, id, name);
+ this.hiveMetastoreUri = uri;
+ HMSExternalDataSource.HMSType hmsType = ((HMSExternalDataSource)
extDataSource).getHmsType();
+ if (hmsType.equals(HMSExternalDataSource.HMSType.HIVE)) {
+ tableType = TableIf.TableType.HIVE;
+ } else if (hmsType.equals(HMSExternalDataSource.HMSType.ICEBERG)) {
+ tableType = TableIf.TableType.ICEBERG;
+ } else {
+ tableType = TableIf.TableType.HUDI;
+ }
+ }
+
+ @Override
+ public List<HMSExternalTable> getTables() {
+ List<HMSExternalTable> tables = new ArrayList<>();
+ List<String> tableNames = extDataSource.listTableNames(null, name);
+ for (String tableName : tableNames) {
+ tables.add(new HMSExternalTable(nextId.incrementAndGet(),
tableName, name, hiveMetastoreUri, tableType));
+ }
+ return tables;
+ }
+
+ @Override
+ public List<HMSExternalTable> getTablesOnIdOrder() {
+ // Sort the name instead, because the id may change.
+ return
getTables().stream().sorted(Comparator.comparing(TableIf::getName)).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<HMSExternalTable> getTablesOnIdOrderIfExist(List<Long>
tableIdList) {
+ return null;
+ }
+
+ @Override
+ public List<HMSExternalTable>
getTablesOnIdOrderOrThrowException(List<Long> tableIdList)
+ throws MetaNotFoundException {
+ return null;
+ }
+
+ @Override
+ public Set<String> getTableNamesWithLock() {
+ readLock();
Review Comment:
Looks like we don't need to `read lock` here.
Because we visit the hms for each call directly, and the rpc may hold the
lock for a long time.
I think you can just remove the readlock, and add a comment to explain it.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java:
##########
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog.external;
+
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.datasource.ExternalDataSource;
+import org.apache.doris.datasource.HMSExternalDataSource;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Hive metastore external database.
+ */
+public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
+
+ private static final Logger LOG =
LogManager.getLogger(HMSExternalDatabase.class);
+
+ private ConcurrentHashMap<String, Long> tableNameToId = new
ConcurrentHashMap<>();
+ private AtomicLong nextId = new AtomicLong(0);
+ private final String hiveMetastoreUri;
+ private final TableIf.TableType tableType;
+
+ /**
+ * Create HMS external database.
+ *
+ * @param extDataSource External data source this database belongs to.
+ * @param id database id.
+ * @param name database name.
+ * @param uri Hive metastore uri.
+ */
+ public HMSExternalDatabase(ExternalDataSource extDataSource, long id,
String name, String uri) {
+ super(extDataSource, id, name);
+ this.hiveMetastoreUri = uri;
+ HMSExternalDataSource.HMSType hmsType = ((HMSExternalDataSource)
extDataSource).getHmsType();
+ if (hmsType.equals(HMSExternalDataSource.HMSType.HIVE)) {
+ tableType = TableIf.TableType.HIVE;
+ } else if (hmsType.equals(HMSExternalDataSource.HMSType.ICEBERG)) {
+ tableType = TableIf.TableType.ICEBERG;
+ } else {
+ tableType = TableIf.TableType.HUDI;
+ }
+ }
+
+ @Override
+ public List<HMSExternalTable> getTables() {
+ List<HMSExternalTable> tables = new ArrayList<>();
+ List<String> tableNames = extDataSource.listTableNames(null, name);
+ for (String tableName : tableNames) {
+ tables.add(new HMSExternalTable(nextId.incrementAndGet(),
tableName, name, hiveMetastoreUri, tableType));
+ }
+ return tables;
+ }
+
+ @Override
+ public List<HMSExternalTable> getTablesOnIdOrder() {
+ // Sort the name instead, because the id may change.
+ return
getTables().stream().sorted(Comparator.comparing(TableIf::getName)).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<HMSExternalTable> getTablesOnIdOrderIfExist(List<Long>
tableIdList) {
+ return null;
+ }
+
+ @Override
+ public List<HMSExternalTable>
getTablesOnIdOrderOrThrowException(List<Long> tableIdList)
+ throws MetaNotFoundException {
+ return null;
Review Comment:
Throw NotImplementedException if we don't need to implement it.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java:
##########
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog.external;
+
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.datasource.ExternalDataSource;
+import org.apache.doris.datasource.HMSExternalDataSource;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Hive metastore external database.
+ */
+public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
+
+ private static final Logger LOG =
LogManager.getLogger(HMSExternalDatabase.class);
+
+ private ConcurrentHashMap<String, Long> tableNameToId = new
ConcurrentHashMap<>();
+ private AtomicLong nextId = new AtomicLong(0);
+ private final String hiveMetastoreUri;
+ private final TableIf.TableType tableType;
+
+ /**
+ * Create HMS external database.
+ *
+ * @param extDataSource External data source this database belongs to.
+ * @param id database id.
+ * @param name database name.
+ * @param uri Hive metastore uri.
+ */
+ public HMSExternalDatabase(ExternalDataSource extDataSource, long id,
String name, String uri) {
+ super(extDataSource, id, name);
+ this.hiveMetastoreUri = uri;
+ HMSExternalDataSource.HMSType hmsType = ((HMSExternalDataSource)
extDataSource).getHmsType();
+ if (hmsType.equals(HMSExternalDataSource.HMSType.HIVE)) {
+ tableType = TableIf.TableType.HIVE;
+ } else if (hmsType.equals(HMSExternalDataSource.HMSType.ICEBERG)) {
+ tableType = TableIf.TableType.ICEBERG;
+ } else {
+ tableType = TableIf.TableType.HUDI;
+ }
+ }
+
+ @Override
+ public List<HMSExternalTable> getTables() {
+ List<HMSExternalTable> tables = new ArrayList<>();
+ List<String> tableNames = extDataSource.listTableNames(null, name);
+ for (String tableName : tableNames) {
+ tables.add(new HMSExternalTable(nextId.incrementAndGet(),
tableName, name, hiveMetastoreUri, tableType));
+ }
+ return tables;
+ }
+
+ @Override
+ public List<HMSExternalTable> getTablesOnIdOrder() {
+ // Sort the name instead, because the id may change.
+ return
getTables().stream().sorted(Comparator.comparing(TableIf::getName)).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<HMSExternalTable> getTablesOnIdOrderIfExist(List<Long>
tableIdList) {
+ return null;
+ }
+
+ @Override
+ public List<HMSExternalTable>
getTablesOnIdOrderOrThrowException(List<Long> tableIdList)
+ throws MetaNotFoundException {
+ return null;
+ }
+
+ @Override
+ public Set<String> getTableNamesWithLock() {
+ readLock();
+ try {
+ return new HashSet<>(extDataSource.listTableNames(null, name));
+ } finally {
+ readUnlock();
+ }
+ }
+
+ @Override
+ public HMSExternalTable getTableNullable(String tableName) {
+ if (extDataSource.tableExist(null, name, tableName)) {
+ return new HMSExternalTable(nextId.incrementAndGet(), tableName,
name, hiveMetastoreUri, tableType);
+ }
+ return null;
+ }
+
+ @Override
+ public Optional<HMSExternalTable> getTable(String tableName) {
+ return Optional.ofNullable(getTableNullable(tableName));
+ }
+
+ @Override
+ public Optional<HMSExternalTable> getTable(long tableId) {
+ return Optional.empty();
+ }
+
+ @Override
+ public <E extends Exception> HMSExternalTable getTableOrException(String
tableName, Function<String, E> e)
+ throws E {
+ HMSExternalTable table = getTableNullable(tableName);
+ if (table == null) {
+ throw e.apply(tableName);
+ }
+ return table;
+ }
+
+ @Override
+ public <E extends Exception> HMSExternalTable getTableOrException(long
tableId, Function<Long, E> e) throws E {
+ return null;
+ }
+
+ @Override
+ public HMSExternalTable getTableOrMetaException(String tableName) throws
MetaNotFoundException {
+ return getTableOrException(tableName, t -> new
MetaNotFoundException("unknown table, tableName=" + t));
+ }
+
+ @Override
+ public HMSExternalTable getTableOrMetaException(long tableId) throws
MetaNotFoundException {
+ return null;
Review Comment:
NotImplementException, and all other not implemented method in this class.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java:
##########
@@ -32,31 +48,170 @@ public class HMSExternalDataSource extends
ExternalDataSource {
private static final Logger LOG =
LogManager.getLogger(HMSExternalDataSource.class);
public static final String HIVE_METASTORE_URIS = "hive.metastore.uris";
- public HMSExternalDataSource() {
+ protected final String hiveMetastoreUris;
+ protected final HiveMetaStoreClient client;
+ private final HMSType hmsType;
+
+ private ConcurrentHashMap<String, DatabaseIf> nameToDb = new
ConcurrentHashMap();
+
+ /**
+ * Hive metastore data source implementation.
+ *
+ * @param hiveMetastoreUris e.g. thrift://127.0.0.1:9083
+ * @param hmsType Hive metastore type. Hive, Iceberg or Hudi.
+ */
+ public HMSExternalDataSource(String hiveMetastoreUris, HMSType hmsType)
throws DdlException {
+ this.hiveMetastoreUris = hiveMetastoreUris;
+ this.hmsType = hmsType;
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, hiveMetastoreUris);
+ try {
+ client = new HiveMetaStoreClient(hiveConf);
+ } catch (MetaException e) {
+ LOG.warn("Create HiveMetaStoreClient failed: {}", e.getMessage());
+ throw new DdlException("Create HiveMetaStoreClient failed: " +
e.getMessage());
+ }
+ }
+
+ public HMSType getHmsType() {
+ return hmsType;
}
@Override
public List<String> listDatabaseNames(SessionContext ctx) {
+ try {
+ return client.getAllDatabases();
+ } catch (MetaException e) {
+ LOG.warn("List Database Names failed. " + e.getMessage());
+ }
return null;
}
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
+ try {
+ return client.getAllTables(dbName);
+ } catch (MetaException e) {
+ LOG.warn("List Table Names failed. " + e.getMessage());
+ }
return null;
}
@Override
public boolean tableExist(SessionContext ctx, String dbName, String
tblName) {
+ try {
+ return client.tableExists(dbName, tblName);
+ } catch (TException e) {
+ LOG.warn("Check table exist failed. " + e.getMessage());
+ }
return false;
}
@Override
public List<Column> getSchema(SessionContext ctx, String dbName, String
tblName) {
- return null;
+ List<Column> hiveMetastoreSchema = Lists.newArrayList();
+ try {
+ List<FieldSchema> hmsSchema = client.getSchema(dbName, tblName);
+ for (FieldSchema field : hmsSchema) {
+ hiveMetastoreSchema.add(new Column(field.getName(),
+
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()),
+ true, null, true, null, field.getComment()));
+ }
+ } catch (TException e) {
+ LOG.warn("Get table schema failed. " + e.getMessage());
+ }
+ return hiveMetastoreSchema;
}
+ @Nullable
@Override
- public List<ExternalScanRange> getExternalScanRanges(SessionContext ctx) {
+ public DatabaseIf getDbNullable(String dbName) {
+ try {
+ client.getDatabase(dbName);
+ } catch (TException e) {
+ LOG.warn("External database {} not exist.", dbName);
+ return null;
+ }
+ // TODO: get a valid id.
+ return new HMSExternalDatabase(this, 0, dbName, hiveMetastoreUris);
+ }
+
+ @Nullable
+ @Override
+ public DatabaseIf getDbNullable(long dbId) {
return null;
Review Comment:
NotImplementException, and all other methods in this class
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
